From 945e5c70f6cb20a77fbb654a0ab6bef7d2b25aac Mon Sep 17 00:00:00 2001 From: Jakub Doka Date: Tue, 17 Dec 2024 00:18:44 +0100 Subject: [PATCH] extractng Nodes from son.rs Signed-off-by: Jakub Doka --- lang/src/backend/hbvm.rs | 2 +- lang/src/backend/hbvm/regalloc.rs | 2 +- lang/src/lib.rs | 3 +- lang/src/nodes.rs | 2194 ++++++++++++++++ lang/src/son.rs | 2277 +---------------- ...son_tests_storing_into_nullable_struct.txt | 48 +- 6 files changed, 2283 insertions(+), 2243 deletions(-) create mode 100644 lang/src/nodes.rs diff --git a/lang/src/backend/hbvm.rs b/lang/src/backend/hbvm.rs index 27734b49..f1a241a4 100644 --- a/lang/src/backend/hbvm.rs +++ b/lang/src/backend/hbvm.rs @@ -2,8 +2,8 @@ use { super::{AssemblySpec, Backend}, crate::{ lexer::TokenKind, + nodes::{Kind, Nid, Nodes, MEM}, parser, - son::{Kind, Nid, Nodes, MEM}, ty::{self, Loc, Module, Offset, Size, Types}, utils::{EntSlice, EntVec}, }, diff --git a/lang/src/backend/hbvm/regalloc.rs b/lang/src/backend/hbvm/regalloc.rs index 0d080da1..1d60af78 100644 --- a/lang/src/backend/hbvm/regalloc.rs +++ b/lang/src/backend/hbvm/regalloc.rs @@ -5,8 +5,8 @@ use { HbvmBackend, Nid, Nodes, PLoc, Reloc, TypedReloc, }, lexer::TokenKind, + nodes::{Kind, ARG_START, MEM, VOID}, parser, quad_sort, - son::{Kind, ARG_START, MEM, VOID}, ty::{self, Arg, Loc, Module, Offset, Sig, Types}, utils::{BitSet, EntSlice}, }, diff --git a/lang/src/lib.rs b/lang/src/lib.rs index 21a9d8e6..7e178829 100644 --- a/lang/src/lib.rs +++ b/lang/src/lib.rs @@ -60,6 +60,7 @@ pub mod fmt; pub mod fs; pub mod fuzz; pub mod lexer; +pub mod nodes; pub mod parser; pub mod son; pub mod ty; @@ -67,8 +68,8 @@ pub mod ty; pub mod backend { use { crate::{ + nodes::Nodes, parser, - son::Nodes, ty::{self, Module, Types}, utils::EntSlice, }, diff --git a/lang/src/nodes.rs b/lang/src/nodes.rs new file mode 100644 index 00000000..22999209 --- /dev/null +++ b/lang/src/nodes.rs @@ -0,0 +1,2194 @@ +use { + crate::{ + ctx_map::CtxEntry, + debug, + lexer::{self, TokenKind}, + parser::Pos, + ty::{self, Loc, Types}, + utils::{BitSet, Vc}, + }, + alloc::{string::String, vec::Vec}, + core::{ + assert_matches::debug_assert_matches, + cell::Cell, + fmt::{self, Debug, Write}, + mem, + ops::{self, Range}, + }, + hashbrown::hash_map, +}; + +pub const VOID: Nid = 0; +pub const NEVER: Nid = 1; +pub const ENTRY: Nid = 2; +pub const MEM: Nid = 3; +pub const LOOPS: Nid = 4; +pub const ARG_START: usize = 3; + +pub type AClassId = i16; +pub type LoopDepth = u16; +pub type LockRc = u16; +pub type IDomDepth = u16; +pub type Nid = u16; + +type Lookup = crate::ctx_map::CtxMap; + +#[derive(Clone)] +pub struct Nodes { + values: Vec>, + queued_peeps: Vec, + free: Nid, + lookup: Lookup, +} + +impl Default for Nodes { + fn default() -> Self { + Self { + values: Default::default(), + queued_peeps: Default::default(), + free: Nid::MAX, + lookup: Default::default(), + } + } +} + +impl Nodes { + #[inline] + pub fn len(&self) -> usize { + self.values.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.values.is_empty() + } + + pub fn as_ty(&self, cint: Nid) -> ty::Id { + if self[cint].ty == ty::Id::NEVER { + return ty::Id::NEVER; + } + debug_assert_eq!(self[cint].ty, ty::Id::TYPE); + ty::Id::from(match self[cint].kind { + Kind::CInt { value } => value as u64, + _ => unreachable!("triing to cast non constant to a type: {:?}", self[cint]), + }) + } + + pub fn loop_depth(&self, target: Nid, scheds: Option<&[Nid]>) -> LoopDepth { + self[target].loop_depth.set(match self[target].kind { + Kind::Region | Kind::Entry | Kind::Then | Kind::Else | Kind::Call { .. } | Kind::If => { + if self[target].loop_depth.get() != 0 { + return self[target].loop_depth.get(); + } + self.loop_depth(self[target].inputs[0], scheds) + } + Kind::Loop => { + if self[target].loop_depth.get() + == self.loop_depth(self[target].inputs[0], scheds) + 1 + { + return self[target].loop_depth.get(); + } + let depth = self.loop_depth(self[target].inputs[0], scheds) + 1; + self[target].loop_depth.set(depth); + let mut cursor = self[target].inputs[1]; + while cursor != target { + self[cursor].loop_depth.set(depth); + let next = self.idom(cursor, scheds); + debug_assert_ne!(next, 0); + if matches!(self[cursor].kind, Kind::Then | Kind::Else) { + debug_assert_eq!(self[next].kind, Kind::If); + let other = self[next].outputs[(self[next].outputs[0] == cursor) as usize]; + self[other].loop_depth.set(depth - 1); + } + cursor = next; + } + depth + } + Kind::Start | Kind::End | Kind::Die | Kind::Return { .. } => 1, + u => unreachable!("{u:?}"), + }); + + self[target].loop_depth.get() + } + + pub fn idepth(&self, target: Nid, scheds: Option<&[Nid]>) -> IDomDepth { + if target == VOID { + return 0; + } + if self[target].depth.get() == 0 { + let depth = match self[target].kind { + Kind::End | Kind::Start => unreachable!("{:?}", self[target].kind), + Kind::Region => self + .idepth(self[target].inputs[0], scheds) + .max(self.idepth(self[target].inputs[1], scheds)), + _ if self[target].kind.is_pinned() => self.idepth(self[target].inputs[0], scheds), + _ if let Some(scheds) = scheds => { + self.idepth(scheds[target as usize], Some(scheds)) + } + _ => self.idepth(self[target].inputs[0], scheds), + } + 1; + self[target].depth.set(depth); + } + self[target].depth.get() + } + + fn fix_loops(&mut self, stack: &mut Vec, seen: &mut BitSet) { + debug_assert!(stack.is_empty()); + + stack.push(NEVER); + + while let Some(node) = stack.pop() { + if seen.set(node) && self.is_cfg(node) { + stack.extend(self[node].inputs.iter()); + } + } + + for l in self[LOOPS].outputs.clone() { + if !seen.get(l) { + self.bind(l, NEVER); + } + } + } + + fn push_up_impl(&self, node: Nid, visited: &mut BitSet, scheds: &mut [Nid]) { + if !visited.set(node) { + return; + } + + for &inp in &self[node].inputs[1..] { + if !self[inp].kind.is_pinned() { + self.push_up_impl(inp, visited, scheds); + } + } + + if self[node].kind.is_pinned() { + return; + } + + let mut deepest = self[node].inputs[0]; + for &inp in self[node].inputs[1..].iter() { + if self.idepth(inp, Some(scheds)) > self.idepth(deepest, Some(scheds)) { + if self[inp].kind.is_call() { + deepest = inp; + } else { + debug_assert!(!self.is_cfg(inp)); + deepest = self.idom(inp, Some(scheds)); + } + } + } + + scheds[node as usize] = deepest; + } + + fn collect_rpo(&self, node: Nid, rpo: &mut Vec, visited: &mut BitSet) { + if !self.is_cfg(node) || !visited.set(node) { + return; + } + + for &n in self[node].outputs.iter() { + self.collect_rpo(n, rpo, visited); + } + + rpo.push(node); + } + + fn push_up(&self, rpo: &mut Vec, visited: &mut BitSet, scheds: &mut [Nid]) { + debug_assert!(rpo.is_empty()); + self.collect_rpo(VOID, rpo, visited); + + for &node in rpo.iter().rev() { + self.loop_depth(node, Some(scheds)); + for i in 0..self[node].inputs.len() { + self.push_up_impl(self[node].inputs[i], visited, scheds); + } + + if matches!(self[node].kind, Kind::Loop | Kind::Region) { + for i in 0..self[node].outputs.len() { + let usage = self[node].outputs[i]; + if self[usage].kind == Kind::Phi { + self.push_up_impl(usage, visited, scheds); + } + } + } + } + + debug_assert_eq!( + self.iter() + .map(|(n, _)| n) + .filter(|&n| !visited.get(n) + && !matches!(self[n].kind, Kind::Arg | Kind::Mem | Kind::Loops)) + .collect::>(), + vec![], + "{:?}", + self.iter() + .filter(|&(n, nod)| !visited.get(n) + && !matches!(nod.kind, Kind::Arg | Kind::Mem | Kind::Loops)) + .collect::>() + ); + + rpo.clear(); + } + + fn better(&self, is: Nid, then: Nid, scheds: Option<&[Nid]>) -> bool { + debug_assert_ne!(self.idepth(is, scheds), self.idepth(then, scheds), "{is} {then}"); + self.loop_depth(is, scheds) < self.loop_depth(then, scheds) + || self.idepth(is, scheds) > self.idepth(then, scheds) + || self[then].kind == Kind::If + } + + fn is_forward_edge(&self, usage: Nid, def: Nid) -> bool { + match self[usage].kind { + Kind::Phi => { + self[usage].inputs[2] != def || self[self[usage].inputs[0]].kind != Kind::Loop + } + Kind::Loop => self[usage].inputs[1] != def, + _ => true, + } + } + + fn schedule_inside_blocks( + &mut self, + cfg_nodes: &mut Vec, + buf: &mut Vec, + seen: &mut BitSet, + ) { + debug_assert!(cfg_nodes.is_empty()); + debug_assert!(buf.is_empty()); + cfg_nodes.extend( + self.iter() + // skip VOID and NEVER + .skip(2) + .filter(|(_, n)| n.kind.is_cfg() && !n.kind.ends_basic_block()) + .map(|(n, _)| n), + ); + + for &block in &*cfg_nodes { + debug_assert!(block != VOID); + seen.clear(self.values.len()); + let mut outputs = mem::take(&mut self[block].outputs); + self.reschedule_block(block, &mut outputs, buf, seen); + self[block].outputs = outputs; + } + + cfg_nodes.clear(); + } + + fn reschedule_block( + &self, + from: Nid, + outputs: &mut [Nid], + buf: &mut Vec, + seen: &mut BitSet, + ) { + debug_assert!(buf.is_empty()); + + // NOTE: this code is horible + let fromc = Some(&from); + + let cfg_idx = outputs.iter().position(|&n| self.is_cfg(n)).unwrap(); + outputs.swap(cfg_idx, 0); + for &o in outputs.iter() { + if (!self.is_cfg(o) + && self[o].outputs.iter().any(|&oi| { + self[oi].kind != Kind::Phi && self[oi].inputs.first() == fromc && !seen.get(oi) + })) + || !seen.set(o) + { + continue; + } + let mut cursor = buf.len(); + for &o in outputs.iter().filter(|&&n| n == o) { + buf.push(o); + } + while let Some(&n) = buf.get(cursor) { + for &i in &self[n].inputs[1..] { + if fromc == self[i].inputs.first() + && self[i].outputs.iter().all(|&o| { + self[o].kind == Kind::Phi + || self[o].inputs.first() != fromc + || seen.get(o) + }) + && seen.set(i) + { + for &o in outputs.iter().filter(|&&n| n == i) { + buf.push(o); + } + } + } + cursor += 1; + } + } + + debug_assert_eq!( + outputs.iter().filter(|&&n| !seen.get(n)).copied().collect::>(), + vec![], + "{:?} {from:?} {:?}", + outputs + .iter() + .filter(|&&n| !seen.get(n)) + .copied() + .map(|n| (n, &self[n])) + .collect::>(), + self[from] + ); + + let bf = &buf; + debug_assert_eq!( + bf.iter() + .enumerate() + .filter(|(_, &b)| !self[b].kind.is_pinned()) + .flat_map(|(i, &b)| self[b] + .inputs + .iter() + .filter(|&&b| !self[b].kind.is_pinned()) + .filter_map(move |&inp| bf + .iter() + .position(|&n| inp == n) + .filter(|&j| i > j) + .map(|j| (bf[i], bf[j])))) + .collect::>(), + vec![], + "{:?}", + bf + ); + + debug_assert!(self.is_cfg(bf[0]) || self[bf[0]].kind == Kind::Phi, "{:?}", self[bf[0]]); + + if outputs.len() != buf.len() { + panic!("{:?} {:?}", outputs, buf); + } + outputs.copy_from_slice(buf); + buf.clear(); + } + + fn push_down( + &self, + node: Nid, + visited: &mut BitSet, + antideps: &mut [Nid], + scheds: &mut [Nid], + antidep_bounds: &mut Vec, + ) { + if !visited.set(node) { + return; + } + + for &usage in self[node].outputs.iter() { + if self.is_forward_edge(usage, node) && self[node].kind == Kind::Stre { + self.push_down(usage, visited, antideps, scheds, antidep_bounds); + } + } + + for &usage in self[node].outputs.iter() { + if self.is_forward_edge(usage, node) { + self.push_down(usage, visited, antideps, scheds, antidep_bounds); + } + } + + if self[node].kind.is_pinned() { + return; + } + + let mut min = None::; + for i in 0..self[node].outputs.len() { + let usage = self[node].outputs[i]; + let ub = self.use_block(node, usage, Some(scheds)); + min = min.map(|m| self.common_dom(ub, m, Some(scheds))).or(Some(ub)); + } + let mut min = min.unwrap(); + + debug_assert!(self.dominates(scheds[node as usize], min, Some(scheds))); + + let mut cursor = min; + let mut fuel = self.values.len(); + while cursor != scheds[node as usize] { + debug_assert!(fuel != 0); + fuel -= 1; + cursor = self.idom(cursor, Some(scheds)); + if self.better(cursor, min, Some(scheds)) { + min = cursor; + } + } + + if self[node].kind == Kind::Load { + min = self.find_antideps(node, min, antideps, scheds, antidep_bounds); + } + + if self[min].kind.ends_basic_block() { + min = self.idom(min, Some(scheds)); + } + + self.assert_dominance(node, min, true, Some(scheds)); + + debug_assert!( + self.idepth(min, Some(scheds)) >= self.idepth(scheds[node as usize], Some(scheds)) + ); + scheds[node as usize] = min; + } + + fn find_antideps( + &self, + load: Nid, + mut min: Nid, + antideps: &mut [Nid], + scheds: &[Nid], + antidep_bounds: &mut Vec, + ) -> Nid { + debug_assert!(self[load].kind == Kind::Load); + debug_assert!(self.dominates(scheds[load as usize], min, Some(scheds)), "{load}"); + + let (aclass, _) = self.aclass_index(self[load].inputs[1]); + + let mut cursor = min; + while cursor != scheds[load as usize] { + antideps[cursor as usize] = load; + if self[cursor].clobbers.get(aclass as _) { + min = self[cursor].inputs[0]; + } + cursor = self.idom(cursor, Some(scheds)); + } + + if self[load].inputs[2] == MEM { + return min; + } + + for &out in self[self[load].inputs[2]].outputs.iter() { + match self[out].kind { + Kind::Stre => { + let mut cursor = scheds[out as usize]; + if cursor == scheds[load as usize] { + antidep_bounds.extend([load, out]); + } + while cursor != scheds[load as usize] + && self.idepth(cursor, Some(scheds)) + > self.idepth(scheds[load as usize], Some(scheds)) + { + if antideps[cursor as usize] == load { + min = self.common_dom(min, cursor, Some(scheds)); + if min == cursor { + antidep_bounds.extend([load, out]); + } + break; + } + cursor = self.idom(cursor, Some(scheds)); + } + break; + } + Kind::Phi => { + let side = self[out].inputs[1..] + .iter() + .position(|&n| n == self[load].inputs[2]) + .unwrap(); + let ctrl = self[out].inputs[0]; + let mut cursor = self[ctrl].inputs[side]; + while cursor != scheds[load as usize] + && self.idepth(cursor, Some(scheds)) + > self.idepth(scheds[load as usize], Some(scheds)) + { + if antideps[cursor as usize] == load { + min = self.common_dom(min, cursor, Some(scheds)); + break; + } + cursor = self.idom(cursor, Some(scheds)); + } + } + _ => {} + } + } + + min + } + + pub fn bind(&mut self, from: Nid, to: Nid) { + debug_assert_ne!(to, 0); + debug_assert_ne!(self[to].kind, Kind::Phi); + self[from].outputs.push(to); + self[to].inputs.push(from); + } + + pub fn use_block(&self, target: Nid, from: Nid, scheds: Option<&[Nid]>) -> Nid { + if self[from].kind != Kind::Phi { + return self.idom(from, scheds); + } + + let index = self[from].inputs.iter().position(|&n| n == target).unwrap_or_else(|| { + panic!("from {from} {:?} target {target} {:?}", self[from], self[target]) + }); + self[self[from].inputs[0]].inputs[index - 1] + } + + pub fn idom(&self, target: Nid, scheds: Option<&[Nid]>) -> Nid { + match self[target].kind { + Kind::Start => unreachable!(), + Kind::End => unreachable!(), + Kind::Region => { + let &[lcfg, rcfg] = self[target].inputs.as_slice() else { unreachable!() }; + self.common_dom(lcfg, rcfg, scheds) + } + _ if self[target].kind.is_pinned() => self[target].inputs[0], + _ if let Some(scheds) = scheds => scheds[target as usize], + _ => self[target].inputs[0], + } + } + + fn common_dom(&self, mut a: Nid, mut b: Nid, scheds: Option<&[Nid]>) -> Nid { + while a != b { + let [ldepth, rdepth] = [self.idepth(a, scheds), self.idepth(b, scheds)]; + if ldepth >= rdepth { + a = self.idom(a, scheds); + } + if ldepth <= rdepth { + b = self.idom(b, scheds); + } + } + a + } + + fn graphviz_low(&self, disp: ty::Display, out: &mut String) -> core::fmt::Result { + use core::fmt::Write; + + writeln!(out)?; + writeln!(out, "digraph G {{")?; + writeln!(out, "rankdir=BT;")?; + writeln!(out, "concentrate=true;")?; + writeln!(out, "compound=true;")?; + + for (i, node) in self.iter() { + let color = match () { + _ if node.lock_rc.get() == Nid::MAX => "orange", + _ if node.lock_rc.get() == Nid::MAX - 1 => "blue", + _ if node.lock_rc.get() != 0 => "red", + _ if node.outputs.is_empty() => "purple", + _ if node.is_mem() => "green", + _ if self.is_cfg(i) => "yellow", + _ => "white", + }; + + if node.ty != ty::Id::VOID { + writeln!( + out, + " node{i}[label=\"{i} {} {} {}\" color={color}]", + node.kind, + disp.rety(node.ty), + node.aclass, + )?; + } else { + writeln!( + out, + " node{i}[label=\"{i} {} {}\" color={color}]", + node.kind, node.aclass, + )?; + } + + for (j, &o) in node.outputs.iter().enumerate() { + let color = if self.is_cfg(i) && self.is_cfg(o) { "red" } else { "lightgray" }; + let index = self[o].inputs.iter().position(|&inp| i == inp).unwrap(); + let style = if index == 0 && !self.is_cfg(o) { "style=dotted" } else { "" }; + writeln!( + out, + " node{o} -> node{i}[color={color} taillabel={index} headlabel={j} {style}]", + )?; + } + } + + writeln!(out, "}}")?; + + Ok(()) + } + + pub fn graphviz(&self, disp: ty::Display) { + let out = &mut String::new(); + _ = self.graphviz_low(disp, out); + log::info!("{out}"); + } + + pub fn graphviz_in_browser(&self, _disp: ty::Display) { + #[cfg(all(test, feature = "std"))] + { + let out = &mut String::new(); + _ = self.graphviz_low(_disp, out); + if !std::process::Command::new("brave") + .arg(format!("https://dreampuf.github.io/GraphvizOnline/#{out}")) + .status() + .unwrap() + .success() + { + log::error!("{out}"); + } + } + } + + pub fn gcm(&mut self, scratch: &mut Vec, bind_buf: &mut Vec, visited: &mut BitSet) { + visited.clear(self.values.len()); + self.fix_loops(bind_buf, visited); + debug_assert!(bind_buf.is_empty()); + debug_assert!(scratch.is_empty()); + scratch.resize(self.values.len() * 2, Nid::MAX); + let (antideps, scheds) = scratch.split_at_mut(self.values.len()); + visited.clear(self.values.len()); + self.push_up(bind_buf, visited, scheds); + visited.clear(self.values.len()); + self.push_down(VOID, visited, antideps, scheds, bind_buf); + + for &[from, to] in bind_buf.array_chunks() { + self.bind(from, to); + } + + self[VOID].outputs = + self[VOID].outputs.iter().filter(|&&n| self[n].kind.is_at_start()).copied().collect(); + + for (&shed, n) in scheds.iter().zip(0u16..) { + if shed == Nid::MAX { + continue; + } + + let prev = mem::replace(&mut self[n].inputs[0], shed); + if prev != VOID { + let index = self[prev].outputs.iter().position(|&o| o == n).unwrap(); + self[prev].outputs.swap_remove(index); + } + self[shed].outputs.push(n); + } + + bind_buf.clear(); + scratch.clear(); + visited.clear(self.values.len()); + self.schedule_inside_blocks(bind_buf, scratch, visited); + } + + pub fn clear(&mut self) { + self.values.clear(); + self.lookup.clear(); + self.free = Nid::MAX; + } + + pub fn new_node_nop(&mut self, ty: ty::Id, kind: Kind, inps: impl Into) -> Nid { + let node = Node { inputs: inps.into(), kind, ty, ..Default::default() }; + + if node.kind == Kind::Phi && node.ty != ty::Id::VOID { + debug_assert_ne!( + self[node.inputs[1]].ty, + ty::Id::VOID, + "{:?} {:?}", + self[node.inputs[1]], + node.ty.expand(), + ); + + if self[node.inputs[0]].kind != Kind::Loop { + debug_assert_ne!( + self[node.inputs[2]].ty, + ty::Id::VOID, + "{:?} {:?}", + self[node.inputs[2]], + node.ty.expand(), + ); + } + + debug_assert!(!matches!(node.ty.expand(), ty::Kind::Struct(_))); + } + + let mut lookup_meta = None; + if !node.is_not_gvnd() { + let (raw_entry, hash) = self.lookup.entry(node.key(), &self.values); + + let entry = match raw_entry { + hash_map::RawEntryMut::Occupied(o) => return o.get_key_value().0.value, + hash_map::RawEntryMut::Vacant(v) => v, + }; + + lookup_meta = Some((entry, hash)); + } + + if self.free == Nid::MAX { + self.free = self.values.len() as _; + self.values.push(Err((Nid::MAX, debug::trace()))); + } + + let free = self.free; + for &d in node.inputs.as_slice() { + debug_assert_ne!(d, free); + self.values[d as usize].as_mut().unwrap_or_else(|_| panic!("{d}")).outputs.push(free); + } + self.free = mem::replace(&mut self.values[free as usize], Ok(node)).unwrap_err().0; + + if let Some((entry, hash)) = lookup_meta { + entry.insert(crate::ctx_map::Key { value: free, hash }, ()); + } + free + } + + fn remove_node_lookup(&mut self, target: Nid) { + if !self[target].is_not_gvnd() { + self.lookup + .remove(&target, &self.values) + .unwrap_or_else(|| panic!("{:?}", self[target])); + } + } + + pub fn new_node(&mut self, ty: ty::Id, kind: Kind, inps: impl Into, tys: &Types) -> Nid { + let id = self.new_node_nop(ty, kind, inps); + if let Some(opt) = self.peephole(id, tys) { + debug_assert_ne!(opt, id); + for peep in mem::take(&mut self.queued_peeps) { + self.unlock(peep); + } + self.lock(opt); + self.remove(id); + self.unlock(opt); + opt + } else { + id + } + } + + pub fn new_const(&mut self, ty: ty::Id, value: impl Into) -> Nid { + self.new_node_nop(ty, Kind::CInt { value: value.into() }, [VOID]) + } + + // TODO: make this internal to son and force backends to track locks thelself + + pub fn is_locked(&self, target: Nid) -> bool { + self[target].lock_rc.get() != 0 + } + + pub fn is_unlocked(&self, target: Nid) -> bool { + self[target].lock_rc.get() == 0 + } + + pub fn lock(&self, target: Nid) { + self[target].lock_rc.set(self[target].lock_rc.get() + 1); + } + + #[track_caller] + pub fn unlock(&self, target: Nid) { + self[target].lock_rc.set(self[target].lock_rc.get() - 1); + } + + pub fn remove(&mut self, target: Nid) -> bool { + if !self[target].is_dangling() { + return false; + } + + for i in 0..self[target].inputs.len() { + let inp = self[target].inputs[i]; + let index = self[inp].outputs.iter().position(|&p| p == target).unwrap(); + self[inp].outputs.swap_remove(index); + self.remove(inp); + } + + self.remove_node_lookup(target); + + if cfg!(debug_assertions) { + mem::replace(&mut self.values[target as usize], Err((Nid::MAX, debug::trace()))) + .unwrap(); + } else { + mem::replace(&mut self.values[target as usize], Err((self.free, debug::trace()))) + .unwrap(); + self.free = target; + } + + true + } + + pub fn late_peephole(&mut self, target: Nid, tys: &Types) -> Option { + if let Some(id) = self.peephole(target, tys) { + for peep in mem::take(&mut self.queued_peeps) { + self.unlock(peep); + } + self.replace(target, id); + return None; + } + None + } + + pub fn iter_peeps(&mut self, mut fuel: usize, stack: &mut Vec, tys: &Types) { + debug_assert!(stack.is_empty()); + debug_assert!(self.queued_peeps.is_empty()); + + self.iter() + .filter_map(|(id, node)| node.kind.is_peeped().then_some(id)) + .collect_into(stack); + stack.iter().for_each(|&s| self.lock(s)); + + while fuel != 0 + && let Some(node) = stack.pop() + { + fuel -= 1; + + if self.unlock_remove(node) { + continue; + } + + if let Some(new) = self.peephole(node, tys) { + self.replace(node, new); + self.push_adjacent_nodes(new, stack); + } + stack.append(&mut self.queued_peeps); + + //debug_assert_matches!( + // self.iter().find(|(i, n)| n.lock_rc.get() != 0 + // && n.kind.is_peeped() + // && !stack.contains(i)), + // None + //); + } + + debug_assert!(self.queued_peeps.is_empty()); + + stack.drain(..).for_each(|s| _ = self.unlock_remove(s)); + } + + fn push_adjacent_nodes(&mut self, of: Nid, stack: &mut Vec) { + let prev_len = stack.len(); + for &i in self[of] + .outputs + .iter() + .chain(self[of].inputs.iter()) + .chain(self[of].peep_triggers.iter()) + { + if self.values[i as usize].is_ok() + && self[i].kind.is_peeped() + && self[i].lock_rc.get() == 0 + { + stack.push(i); + } + } + + self[of].peep_triggers = Vc::default(); + stack.iter().skip(prev_len).for_each(|&n| self.lock(n)); + } + + pub fn aclass_index(&self, region: Nid) -> (usize, Nid) { + if self[region].aclass >= 0 { + (self[region].aclass as _, region) + } else { + ( + self[self[region].aclass.unsigned_abs() - 1].aclass as _, + self[region].aclass.unsigned_abs() - 1, + ) + } + } + + pub fn pass_aclass(&mut self, from: Nid, to: Nid) { + debug_assert!(self[from].aclass >= 0); + if from != to { + self[to].aclass = -(from as AClassId + 1); + } + } + + fn peephole(&mut self, target: Nid, tys: &Types) -> Option { + use {Kind as K, TokenKind as T}; + match self[target].kind { + K::BinOp { op } => { + let &[ctrl, mut lhs, mut rhs] = self[target].inputs.as_slice() else { + unreachable!() + }; + let ty = self[target].ty; + + let is_float = self[target].ty.is_float(); + + if let (&K::CInt { value: a }, &K::CInt { value: b }) = + (&self[lhs].kind, &self[rhs].kind) + { + return Some(self.new_const(ty, op.apply_binop(a, b, is_float))); + } + + if lhs == rhs { + match op { + T::Ne | T::Gt | T::Lt | T::Sub => return Some(self.new_const(ty, 0)), + T::Eq | T::Ge | T::Le => return Some(self.new_const(ty, 1)), + T::Add => { + let rhs = self.new_const(ty, 2); + return Some(self.new_node( + ty, + K::BinOp { op: T::Mul }, + [ctrl, lhs, rhs], + tys, + )); + } + _ => {} + } + } + + // this is more general the pushing constants to left to help deduplicate expressions more + let mut changed = false; + if op.is_comutative() && self[lhs].key() < self[rhs].key() { + mem::swap(&mut lhs, &mut rhs); + changed = true; + } + + if let K::CInt { value } = self[rhs].kind { + match (op, value) { + (T::Eq, 0) if self[lhs].ty.is_pointer() || self[lhs].kind == Kind::Stck => { + return Some(self.new_const(ty::Id::BOOL, 0)); + } + (T::Ne, 0) if self[lhs].ty.is_pointer() || self[lhs].kind == Kind::Stck => { + return Some(self.new_const(ty::Id::BOOL, 1)); + } + (T::Add | T::Sub | T::Shl, 0) | (T::Mul | T::Div, 1) => return Some(lhs), + (T::Mul, 0) => return Some(rhs), + _ => {} + } + } + + if op.is_comutative() && self[lhs].kind == (K::BinOp { op }) { + let &[_, a, b] = self[lhs].inputs.as_slice() else { unreachable!() }; + if let K::CInt { value: av } = self[b].kind + && let K::CInt { value: bv } = self[rhs].kind + { + // (a op #b) op #c => a op (#b op #c) + let new_rhs = self.new_const(ty, op.apply_binop(av, bv, is_float)); + return Some(self.new_node(ty, K::BinOp { op }, [ctrl, a, new_rhs], tys)); + } + + if self.is_const(b) { + // (a op #b) op c => (a op c) op #b + let new_lhs = self.new_node(ty, K::BinOp { op }, [ctrl, a, rhs], tys); + return Some(self.new_node(ty, K::BinOp { op }, [ctrl, new_lhs, b], tys)); + } + + self.add_trigger(b, target); + } + + if op == T::Add + && self[lhs].kind == (K::BinOp { op: T::Mul }) + && self[lhs].inputs[1] == rhs + && let K::CInt { value } = self[self[lhs].inputs[2]].kind + { + // a * #n + a => a * (#n + 1) + let new_rhs = self.new_const(ty, value + 1); + return Some(self.new_node( + ty, + K::BinOp { op: T::Mul }, + [ctrl, rhs, new_rhs], + tys, + )); + } + + if op == T::Sub + && self[lhs].kind == (K::BinOp { op: T::Add }) + && let K::CInt { value: a } = self[rhs].kind + && let K::CInt { value: b } = self[self[lhs].inputs[2]].kind + { + let new_rhs = self.new_const(ty, b - a); + return Some(self.new_node( + ty, + K::BinOp { op: T::Add }, + [ctrl, self[lhs].inputs[1], new_rhs], + tys, + )); + } + + if op == T::Sub && self[lhs].kind == (K::BinOp { op }) { + // (a - b) - c => a - (b + c) + let &[_, a, b] = self[lhs].inputs.as_slice() else { unreachable!() }; + let c = rhs; + let new_rhs = self.new_node(ty, K::BinOp { op: T::Add }, [ctrl, b, c], tys); + return Some(self.new_node(ty, K::BinOp { op }, [ctrl, a, new_rhs], tys)); + } + + if op == T::Add + && self[rhs].kind == (K::BinOp { op: T::Mul }) + && let &[_, index, step] = self[rhs].inputs.as_slice() + && self[index].kind == K::Phi + && let &[iter_loop, index_init, new_index] = self[index].inputs.as_slice() + && new_index != VOID + && self[iter_loop].kind == K::Loop + && self[new_index].kind == (K::BinOp { op: T::Add }) + && self[new_index].inputs[1] == index + && let Some(&iter_cond) = self[index].outputs.iter().find( + |&&n| matches!(self[n].kind, Kind::BinOp { op } if op.is_compatison()), + ) + && self[index].outputs.iter().all(|n| [iter_cond, rhs, new_index].contains(n)) + { + // arr := @as([u32; 10], idk) + // + // i := 0 + // loop if i == 10 break else { + // arr[i] = 0 + // i += 1 + // } + // + // ||||| + // VVVVV + // + // cursor := &arr[0] + 0 + // end := &arr[0] + 10 + // loop if cursor == end else { + // *cursor = 0 + // i += 1 + // } + + debug_assert!(self[iter_cond].inputs.contains(&index)); + let iter_bound_index = + self[iter_cond].inputs.iter().rposition(|&n| n != index).unwrap(); + debug_assert_ne!(iter_bound_index, 0); + let end_shift = self.new_node( + self[rhs].ty, + K::BinOp { op: T::Mul }, + [ctrl, self[iter_cond].inputs[iter_bound_index], step], + tys, + ); + let end = + self.new_node(ty, K::BinOp { op: T::Add }, [ctrl, lhs, end_shift], tys); + + let init_shift = self.new_node( + self[rhs].ty, + K::BinOp { op: T::Mul }, + [ctrl, index_init, step], + tys, + ); + let init = + self.new_node(ty, K::BinOp { op: T::Add }, [ctrl, lhs, init_shift], tys); + + let new_value = self.new_node_nop(ty, K::Phi, [iter_loop, init, 0]); + let next = + self.new_node(ty, Kind::BinOp { op: T::Add }, [ctrl, new_value, step], tys); + + let mut new_cond_inputs = self[iter_cond].inputs.clone(); + new_cond_inputs[iter_bound_index] = end; + new_cond_inputs[3 - iter_bound_index] = new_value; + let new_cond = self.new_node(ty, self[iter_cond].kind, new_cond_inputs, tys); + self.replace(iter_cond, new_cond); + + return Some(self.modify_input(new_value, 2, next)); + } + + if changed { + return Some(self.new_node(ty, self[target].kind, [ctrl, lhs, rhs], tys)); + } + } + K::UnOp { op } => { + let &[_, oper] = self[target].inputs.as_slice() else { unreachable!() }; + let ty = self[target].ty; + + if matches!(op, TokenKind::Number | TokenKind::Float) + && tys.size_of(self[oper].ty) == tys.size_of(ty) + && self[oper].ty.is_integer() + && ty.is_integer() + { + return Some(oper); + } + + if let K::CInt { value } = self[oper].kind { + let is_float = self[oper].ty.is_float(); + return Some(self.new_const(ty, op.apply_unop(value, is_float))); + } + } + K::If => { + if self[target].inputs[0] == NEVER { + return Some(NEVER); + } + + if self[target].ty == ty::Id::VOID { + match self.try_match_cond(target) { + CondOptRes::Unknown => {} + CondOptRes::Known { value, .. } => { + let ty = if value { + ty::Id::RIGHT_UNREACHABLE + } else { + ty::Id::LEFT_UNREACHABLE + }; + return Some(self.new_node_nop(ty, K::If, self[target].inputs.clone())); + } + } + } + } + K::Then => { + if self[target].inputs[0] == NEVER { + return Some(NEVER); + } + + if self[self[target].inputs[0]].ty == ty::Id::LEFT_UNREACHABLE { + return Some(NEVER); + } else if self[self[target].inputs[0]].ty == ty::Id::RIGHT_UNREACHABLE { + return Some(self[self[target].inputs[0]].inputs[0]); + } + } + K::Else => { + if self[target].inputs[0] == NEVER { + return Some(NEVER); + } + + if self[self[target].inputs[0]].ty == ty::Id::RIGHT_UNREACHABLE { + return Some(NEVER); + } else if self[self[target].inputs[0]].ty == ty::Id::LEFT_UNREACHABLE { + return Some(self[self[target].inputs[0]].inputs[0]); + } + } + K::Region => { + let (ctrl, side) = match self[target].inputs.as_slice() { + [NEVER, NEVER] => return Some(NEVER), + &[NEVER, ctrl] => (ctrl, 2), + &[ctrl, NEVER] => (ctrl, 1), + _ => return None, + }; + + self.lock(target); + for i in self[target].outputs.clone() { + if self[i].kind == Kind::Phi { + for o in self[i].outputs.clone() { + if self.is_unlocked(o) { + self.lock(o); + self.queued_peeps.push(o); + } + } + self.replace(i, self[i].inputs[side]); + } + } + self.unlock(target); + + return Some(ctrl); + } + K::Call { .. } => { + if self[target].inputs[0] == NEVER { + return Some(NEVER); + } + } + K::Return { file } => { + if self[target].inputs[0] == NEVER { + return Some(NEVER); + } + + let mut new_inps = Vc::from(&self[target].inputs[..2]); + 'a: for &n in self[target].inputs.clone().iter().skip(2) { + if self[n].kind != Kind::Stre { + new_inps.push(n); + continue; + } + + if let Some(&load) = + self[n].outputs.iter().find(|&&n| self[n].kind == Kind::Load) + { + self.add_trigger(load, target); + continue; + } + + let mut cursor = n; + let class = self.aclass_index(self[cursor].inputs[2]); + + if self[class.1].kind != Kind::Stck { + new_inps.push(n); + continue; + } + + if self[class.1].outputs.iter().any(|&n| { + self[n].kind != Kind::Stre + && self[n].outputs.iter().any(|&n| self[n].kind != Kind::Stre) + }) { + new_inps.push(n); + continue; + } + + cursor = self[cursor].inputs[3]; + while cursor != MEM { + debug_assert_eq!(self[cursor].kind, Kind::Stre); + if self.aclass_index(self[cursor].inputs[2]) != class { + new_inps.push(n); + continue 'a; + } + + if let Some(&load) = + self[cursor].outputs.iter().find(|&&n| self[n].kind == Kind::Load) + { + self.add_trigger(load, target); + continue 'a; + } + + cursor = self[cursor].inputs[3]; + } + } + + if new_inps.as_slice() != self[target].inputs.as_slice() { + let ret = self.new_node_nop(ty::Id::VOID, Kind::Return { file }, new_inps); + self[ret].pos = self[target].pos; + return Some(ret); + } + } + K::Phi => { + let &[ctrl, lhs, rhs] = self[target].inputs.as_slice() else { unreachable!() }; + + if rhs == target || lhs == rhs { + return Some(lhs); + } + + // TODO: travese the graph downward and chech if this phi is only consumed by it + // self + if self[target].outputs.as_slice() == [rhs] + && self[rhs].outputs.as_slice() == [target] + { + return Some(lhs); + } + + if self[lhs].kind == Kind::Stre + && self[rhs].kind == Kind::Stre + && self[lhs].ty == self[rhs].ty + && self[lhs].ty.loc(tys) == Loc::Reg + && self[lhs].inputs[2] == self[rhs].inputs[2] + && self[lhs].inputs[3] == self[rhs].inputs[3] + { + let pick_value = self.new_node( + self[lhs].ty, + Kind::Phi, + [ctrl, self[lhs].inputs[1], self[rhs].inputs[1]], + tys, + ); + let mut vc = self[lhs].inputs.clone(); + vc[1] = pick_value; + return Some(self.new_node(self[lhs].ty, Kind::Stre, vc, tys)); + } + + // broken + //let ty = self[target].ty; + //if let Kind::BinOp { op } = self[lhs].kind + // && self[rhs].kind == (Kind::BinOp { op }) + //{ + // debug_assert!(ty != ty::Id::VOID); + // debug_assert_eq!( + // self[lhs].ty.simple_size(), + // ty.simple_size(), + // "{:?} {:?}", + // self[lhs].ty.expand(), + // ty.expand() + // ); + // debug_assert_eq!( + // self[rhs].ty.simple_size(), + // ty.simple_size(), + // "{:?} {:?}", + // self[rhs].ty.expand(), + // ty.expand() + // ); + // let inps = [ctrl, self[lhs].inputs[1], self[rhs].inputs[1]]; + // let nlhs = self.new_node(ty, Kind::Phi, inps, tys); + // let inps = [ctrl, self[lhs].inputs[2], self[rhs].inputs[2]]; + // let nrhs = self.new_node(ty, Kind::Phi, inps, tys); + // return Some(self.new_node(ty, Kind::BinOp { op }, [VOID, nlhs, nrhs], tys)); + //} + } + K::Stck => { + if let &[mut a, mut b] = self[target].outputs.as_slice() { + if self[a].kind == Kind::Load { + mem::swap(&mut a, &mut b); + } + + if self[a].kind.is_call() + && self[a].inputs.last() == Some(&target) + && self[b].kind == Kind::Load + && let &[store] = self[b].outputs.as_slice() + && self[store].kind == Kind::Stre + { + let len = self[a].inputs.len(); + let stre = self[store].inputs[3]; + if stre != MEM { + self[a].inputs.push(stre); + self[a].inputs.swap(len - 1, len); + self[stre].outputs.push(a); + } + return Some(self[store].inputs[2]); + } + } + } + K::Stre => { + let &[_, value, region, store, ..] = self[target].inputs.as_slice() else { + unreachable!() + }; + + if self[value].kind == Kind::Load && self[value].inputs[1] == region { + return Some(store); + } + + let mut cursor = target; + while self[cursor].kind == Kind::Stre + && self[cursor].inputs[1] != VOID + && let &[next_store] = self[cursor].outputs.as_slice() + { + if self[next_store].inputs[2] == region + && self[next_store].ty == self[target].ty + { + return Some(store); + } + cursor = next_store; + } + + 'eliminate: { + if self[target].outputs.is_empty() { + break 'eliminate; + } + + if self[value].kind != Kind::Load + || self[value].outputs.iter().any(|&n| self[n].kind != Kind::Stre) + { + for &ele in self[value].outputs.clone().iter().filter(|&&n| n != target) { + self.add_trigger(ele, target); + } + break 'eliminate; + } + + let &[_, stack, last_store] = self[value].inputs.as_slice() else { + unreachable!() + }; + + if self[stack].ty != self[value].ty || self[stack].kind != Kind::Stck { + break 'eliminate; + } + + let mut unidentifed = self[stack].outputs.clone(); + let load_idx = unidentifed.iter().position(|&n| n == value).unwrap(); + unidentifed.swap_remove(load_idx); + + let mut saved = Vc::default(); + let mut cursor = last_store; + let mut first_store = last_store; + while cursor != MEM && self[cursor].kind == Kind::Stre { + let mut contact_point = cursor; + let mut region = self[cursor].inputs[2]; + if let Kind::BinOp { op } = self[region].kind { + debug_assert_matches!(op, TokenKind::Add | TokenKind::Sub); + contact_point = region; + region = self[region].inputs[1] + } + + if region != stack { + break; + } + let Some(index) = unidentifed.iter().position(|&n| n == contact_point) + else { + break 'eliminate; + }; + if self[self[cursor].inputs[1]].kind == Kind::Load + && self[value].outputs.iter().any(|&n| { + self.aclass_index(self[self[cursor].inputs[1]].inputs[1]).0 + == self.aclass_index(self[n].inputs[2]).0 + }) + { + break 'eliminate; + } + unidentifed.remove(index); + saved.push(contact_point); + first_store = cursor; + cursor = *self[cursor].inputs.get(3).unwrap_or(&MEM); + + if unidentifed.is_empty() { + break; + } + } + + if !unidentifed.is_empty() { + break 'eliminate; + } + + debug_assert_matches!( + self[last_store].kind, + Kind::Stre | Kind::Mem, + "{:?}", + self[last_store] + ); + debug_assert_matches!( + self[first_store].kind, + Kind::Stre | Kind::Mem, + "{:?}", + self[first_store] + ); + + // FIXME: when the loads and stores become parallel we will need to get saved + // differently + let mut prev_store = store; + for mut oper in saved.into_iter().rev() { + let mut region = region; + if let Kind::BinOp { op } = self[oper].kind { + debug_assert_eq!(self[oper].outputs.len(), 1); + debug_assert_eq!(self[self[oper].outputs[0]].kind, Kind::Stre); + let new_region = self.new_node( + self[oper].ty, + Kind::BinOp { op }, + [VOID, region, self[oper].inputs[2]], + tys, + ); + self.pass_aclass(self.aclass_index(region).1, new_region); + region = new_region; + oper = self[oper].outputs[0]; + } + + let mut inps = self[oper].inputs.clone(); + debug_assert_eq!(inps.len(), 4); + inps[2] = region; + inps[3] = prev_store; + prev_store = self.new_node_nop(self[oper].ty, Kind::Stre, inps); + if self.is_unlocked(prev_store) { + self.lock(prev_store); + self.queued_peeps.push(prev_store); + } + } + + return Some(prev_store); + } + + if let Some(&load) = + self[target].outputs.iter().find(|&&n| self[n].kind == Kind::Load) + { + self.add_trigger(load, target); + } else if value != VOID + && self[value].kind != Kind::Load + && self[store].kind == Kind::Stre + && self[store].inputs[2] == region + { + if self[store].inputs[1] == value { + return Some(store); + } + + let mut inps = self[target].inputs.clone(); + inps[3] = self[store].inputs[3]; + return Some(self.new_node_nop(self[target].ty, Kind::Stre, inps)); + } + } + K::Load => { + fn range_of(s: &Nodes, mut region: Nid, ty: ty::Id, tys: &Types) -> Range { + let loc = s.aclass_index(region).1; + let full_size = tys.size_of( + if matches!(s[loc].kind, Kind::Stck | Kind::Arg | Kind::Global { .. }) { + s[loc].ty + } else if let Some(ptr) = tys.base_of(s[loc].ty) { + ptr + } else { + return 0..usize::MAX; + }, + ); + let size = tys.size_of(ty); + let mut offset = 0; + loop { + match s[region].kind { + _ if region == loc => { + break offset as usize..offset as usize + size as usize + } + Kind::Assert { kind: AssertKind::NullCheck, .. } => { + region = s[region].inputs[2] + } + Kind::BinOp { op: TokenKind::Add | TokenKind::Sub } + if let Kind::CInt { value } = s[s[region].inputs[2]].kind => + { + offset += value; + region = s[region].inputs[1]; + } + _ => break 0..full_size as usize, + }; + } + } + + let &[ctrl, region, store] = self[target].inputs.as_slice() else { unreachable!() }; + let load_range = range_of(self, region, self[target].ty, tys); + + let mut cursor = store; + while cursor != MEM && self[cursor].kind != Kind::Phi { + if self[cursor].inputs[0] == ctrl + && self[cursor].inputs[2] == region + && self[cursor].ty == self[target].ty + && (self[self[cursor].inputs[1]].kind != Kind::Load + || (!self[target].outputs.is_empty() + && self[target].outputs.iter().all(|&n| { + self[n].kind != Kind::Stre + || self + .aclass_index(self[self[cursor].inputs[1]].inputs[1]) + .0 + != self.aclass_index(self[n].inputs[2]).0 + }))) + { + return Some(self[cursor].inputs[1]); + } + let range = range_of(self, self[cursor].inputs[2], self[cursor].ty, tys); + if range.start >= load_range.end || range.end <= load_range.start { + cursor = self[cursor].inputs[3]; + } else { + let reg = self.aclass_index(self[cursor].inputs[2]).1; + self.add_trigger(reg, target); + break; + } + } + + if store != cursor { + return Some(self.new_node( + self[target].ty, + Kind::Load, + [ctrl, region, cursor], + tys, + )); + } + } + K::Loop => { + if self[target].inputs[1] == NEVER || self[target].inputs[0] == NEVER { + self.lock(target); + for o in self[target].outputs.clone() { + if self[o].kind == Kind::Phi { + self.remove_node_lookup(target); + + let prev = self[o].inputs[2]; + self[o].inputs[2] = VOID; + self[VOID].outputs.push(o); + let index = self[prev].outputs.iter().position(|&n| n == o).unwrap(); + self[prev].outputs.swap_remove(index); + self.lock(o); + self.remove(prev); + self.unlock(o); + + for o in self[o].outputs.clone() { + if self.is_unlocked(o) { + self.lock(o); + self.queued_peeps.push(o); + } + } + self.replace(o, self[o].inputs[1]); + } + } + self.unlock(target); + return Some(self[target].inputs[0]); + } + } + K::Die => { + if self[target].inputs[0] == NEVER { + return Some(NEVER); + } + } + K::Assert { kind, .. } => 'b: { + let pin = match (kind, self.try_match_cond(target)) { + (AssertKind::NullCheck, CondOptRes::Known { value: false, pin }) => pin, + (AssertKind::UnwrapCheck, CondOptRes::Unknown) => None, + _ => break 'b, + } + .unwrap_or(self[target].inputs[0]); + + for out in self[target].outputs.clone() { + if !self[out].kind.is_pinned() && self[out].inputs[0] != pin { + self.modify_input(out, 0, pin); + } + } + return Some(self[target].inputs[2]); + } + K::Start => {} + _ if self.is_cfg(target) && self.idom(target, None) == NEVER => panic!(), + K::Entry + | K::Mem + | K::Loops + | K::End + | K::CInt { .. } + | K::Arg + | K::Global { .. } + | K::Join => {} + } + + None + } + + pub fn try_match_cond(&self, target: Nid) -> CondOptRes { + let &[ctrl, cond, ..] = self[target].inputs.as_slice() else { unreachable!() }; + if let Kind::CInt { value } = self[cond].kind { + return CondOptRes::Known { value: value != 0, pin: None }; + } + + let mut cursor = ctrl; + while cursor != ENTRY { + let ctrl = &self[cursor]; + // TODO: do more inteligent checks on the condition + if matches!(ctrl.kind, Kind::Then | Kind::Else) { + if self[ctrl.inputs[0]].kind == Kind::End { + return CondOptRes::Unknown; + } + debug_assert_eq!(self[ctrl.inputs[0]].kind, Kind::If); + let other_cond = self[ctrl.inputs[0]].inputs[1]; + if let Some(value) = self.matches_cond(cond, other_cond) { + return CondOptRes::Known { + value: (ctrl.kind == Kind::Then) ^ !value, + pin: Some(cursor), + }; + } + } + + cursor = self.idom(cursor, None); + } + + CondOptRes::Unknown + } + + fn matches_cond(&self, to_match: Nid, matches: Nid) -> Option { + use TokenKind as K; + let [tn, mn] = [&self[to_match], &self[matches]]; + match (tn.kind, mn.kind) { + _ if to_match == matches => Some(true), + (Kind::BinOp { op: K::Ne }, Kind::BinOp { op: K::Eq }) + | (Kind::BinOp { op: K::Eq }, Kind::BinOp { op: K::Ne }) + if tn.inputs[1..] == mn.inputs[1..] => + { + Some(false) + } + (_, Kind::BinOp { op: K::Band }) => self + .matches_cond(to_match, mn.inputs[1]) + .or(self.matches_cond(to_match, mn.inputs[2])), + (_, Kind::BinOp { op: K::Bor }) => match ( + self.matches_cond(to_match, mn.inputs[1]), + self.matches_cond(to_match, mn.inputs[2]), + ) { + (None, Some(a)) | (Some(a), None) => Some(a), + (Some(b), Some(a)) if a == b => Some(a), + _ => None, + }, + _ => None, + } + } + + pub fn is_const(&self, id: Nid) -> bool { + matches!(self[id].kind, Kind::CInt { .. }) + } + + pub fn replace(&mut self, target: Nid, with: Nid) { + debug_assert_ne!(target, with, "{:?}", self[target]); + for out in self[target].outputs.clone() { + let index = self[out].inputs.iter().position(|&p| p == target).unwrap(); + self.modify_input(out, index, with); + } + } + + pub fn modify_input(&mut self, target: Nid, inp_index: usize, with: Nid) -> Nid { + self.remove_node_lookup(target); + debug_assert_ne!( + self[target].inputs[inp_index], with, + "{:?} {:?}", + self[target], self[with] + ); + + if self[target].is_not_gvnd() && (self[target].kind != Kind::Phi || with == 0) { + let prev = self[target].inputs[inp_index]; + self[target].inputs[inp_index] = with; + self[with].outputs.push(target); + let index = self[prev].outputs.iter().position(|&o| o == target).unwrap(); + self[prev].outputs.swap_remove(index); + self.remove(prev); + target + } else { + let prev = self[target].inputs[inp_index]; + self[target].inputs[inp_index] = with; + let (entry, hash) = self.lookup.entry(target.key(&self.values), &self.values); + match entry { + hash_map::RawEntryMut::Occupied(other) => { + let rpl = other.get_key_value().0.value; + self[target].inputs[inp_index] = prev; + self.lookup.insert(target.key(&self.values), target, &self.values); + self.replace(target, rpl); + rpl + } + hash_map::RawEntryMut::Vacant(slot) => { + slot.insert(crate::ctx_map::Key { value: target, hash }, ()); + let index = self[prev].outputs.iter().position(|&o| o == target).unwrap(); + self[prev].outputs.swap_remove(index); + self[with].outputs.push(target); + self.remove(prev); + target + } + } + } + } + + #[track_caller] + pub fn unlock_remove(&mut self, id: Nid) -> bool { + self.unlock(id); + self.remove(id) + } + + pub fn iter(&self) -> impl DoubleEndedIterator { + self.values.iter().enumerate().filter_map(|(i, s)| Some((i as _, s.as_ref().ok()?))) + } + + #[expect(clippy::format_in_format_args)] + fn basic_blocks_instr(&self, out: &mut String, node: Nid) -> core::fmt::Result { + match self[node].kind { + Kind::Assert { .. } | Kind::Start => unreachable!("{} {out}", self[node].kind), + Kind::End => return Ok(()), + Kind::If => write!(out, " if: "), + Kind::Region | Kind::Loop => writeln!(out, " goto: {node}"), + Kind::Return { .. } => write!(out, " ret: "), + Kind::Die => write!(out, " die: "), + Kind::CInt { value } => write!(out, "cint: #{value:<4}"), + Kind::Phi => write!(out, " phi: "), + Kind::Arg => write!( + out, + " arg: {:<5}", + self[VOID].outputs.iter().position(|&n| n == node).unwrap() - 2 + ), + Kind::BinOp { op } | Kind::UnOp { op } => { + write!(out, "{:>4}: ", op.name()) + } + Kind::Call { func, args: _ } => { + write!(out, "call: {func} {} ", self[node].depth.get()) + } + Kind::Global { global } => write!(out, "glob: {global:<5}"), + Kind::Entry => write!(out, "ctrl: {:<5}", "entry"), + Kind::Then => write!(out, "ctrl: {:<5}", "then"), + Kind::Else => write!(out, "ctrl: {:<5}", "else"), + Kind::Stck => write!(out, "stck: "), + Kind::Load => write!(out, "load: "), + Kind::Stre => write!(out, "stre: "), + Kind::Mem => write!(out, " mem: "), + Kind::Loops => write!(out, "loops: "), + Kind::Join => write!(out, "join: "), + }?; + + if self[node].kind != Kind::Loop && self[node].kind != Kind::Region { + writeln!( + out, + " {:<3} {:<14} {}", + node, + format!("{:?}", self[node].inputs), + format!("{:?}", self[node].outputs) + )?; + } + + Ok(()) + } + + fn basic_blocks_low( + &self, + out: &mut String, + mut node: Nid, + visited: &mut BitSet, + ) -> core::fmt::Result { + let iter = |nodes: &Nodes, node| nodes[node].outputs.clone().into_iter().rev(); + while visited.set(node) { + match self[node].kind { + Kind::Start => { + writeln!(out, "start: {}", self[node].depth.get())?; + let mut cfg_index = Nid::MAX; + for o in iter(self, node) { + self.basic_blocks_instr(out, o)?; + if self[o].kind.is_cfg() { + cfg_index = o; + } + } + node = cfg_index; + } + Kind::End => break, + Kind::If => { + self.basic_blocks_low(out, self[node].outputs[0], visited)?; + node = self[node].outputs[1]; + } + Kind::Region => { + writeln!( + out, + "region{node}: {} {} {:?}", + self[node].depth.get(), + self[node].loop_depth.get(), + self[node].inputs + )?; + let mut cfg_index = Nid::MAX; + for o in iter(self, node) { + self.basic_blocks_instr(out, o)?; + if self.is_cfg(o) { + cfg_index = o; + } + } + node = cfg_index; + } + Kind::Loop => { + writeln!( + out, + "loop{node}: {} {} {:?}", + self[node].depth.get(), + self[node].loop_depth.get(), + self[node].outputs + )?; + let mut cfg_index = Nid::MAX; + for o in iter(self, node) { + self.basic_blocks_instr(out, o)?; + if self.is_cfg(o) { + cfg_index = o; + } + } + node = cfg_index; + } + Kind::Return { .. } | Kind::Die => { + node = self[node].outputs[0]; + } + Kind::Then | Kind::Else | Kind::Entry => { + writeln!( + out, + "b{node}: {} {} {:?}", + self[node].depth.get(), + self[node].loop_depth.get(), + self[node].outputs + )?; + let mut cfg_index = Nid::MAX; + for o in iter(self, node) { + self.basic_blocks_instr(out, o)?; + if self.is_cfg(o) { + cfg_index = o; + } + } + node = cfg_index; + } + Kind::Call { .. } => { + let mut cfg_index = Nid::MAX; + let mut print_ret = true; + for o in iter(self, node) { + if self[o].inputs[0] == node + && (self[node].outputs[0] != o || mem::take(&mut print_ret)) + { + self.basic_blocks_instr(out, o)?; + } + if self.is_cfg(o) { + cfg_index = o; + } + } + node = cfg_index; + } + _ => unreachable!(), + } + } + + Ok(()) + } + + pub fn basic_blocks(&self) { + let mut out = String::new(); + let mut visited = BitSet::default(); + self.basic_blocks_low(&mut out, VOID, &mut visited).unwrap(); + log::info!("{out}"); + } + + pub fn is_cfg(&self, o: Nid) -> bool { + self[o].kind.is_cfg() + } + + pub fn check_final_integrity(&self, disp: ty::Display) { + if !cfg!(debug_assertions) { + return; + } + + let mut failed = false; + for (id, node) in self.iter() { + if self.is_locked(id) { + log::error!("{} {} {:?}", node.lock_rc.get(), 0, node.kind); + failed = true; + } + if !matches!(node.kind, Kind::End | Kind::Mem | Kind::Arg | Kind::Loops) + && node.outputs.is_empty() + { + log::error!("outputs are empry {id} {:?}", node); + failed = true; + } + if node.inputs.first() == Some(&NEVER) && id != NEVER { + log::error!("is unreachable but still present {id} {:?}", node.kind); + failed = true; + } + if node.outputs.contains(&id) && !matches!(node.kind, Kind::Loop | Kind::End) { + log::error!("node depends on it self and its not a loop {id} {:?}", node); + failed = true; + } + } + + if failed { + self.graphviz_in_browser(disp); + panic!() + } + } + + pub fn check_loop_depth_integrity(&self, disp: ty::Display) { + if !cfg!(debug_assertions) { + return; + } + + let mut failed = false; + for &loob in self[LOOPS].outputs.iter() { + let mut stack = vec![self[loob].inputs[1]]; + let mut seen = BitSet::default(); + seen.set(loob); + let depth = self.loop_depth(loob, None); + while let Some(nid) = stack.pop() { + if seen.set(nid) { + if depth > self.loop_depth(nid, None) { + failed = true; + log::error!("{depth} {} {nid} {:?}", self.loop_depth(nid, None), self[nid]); + } + + match self[nid].kind { + Kind::Loop | Kind::Region => { + stack.extend(&self[nid].inputs[..2]); + } + _ => stack.push(self[nid].inputs[0]), + } + } + } + } + + if failed { + self.graphviz_in_browser(disp); + panic!() + } + } + + fn assert_dominance(&self, nd: Nid, min: Nid, check_outputs: bool, scheds: Option<&[Nid]>) { + if !cfg!(debug_assertions) { + return; + } + + let node = &self[nd]; + for &i in &node.inputs[1..] { + let dom = self.idom(i, scheds); + debug_assert!( + self.dominates(dom, min, scheds), + "{dom} {min} {node:?} {:?}", + self.basic_blocks() + ); + } + if check_outputs { + for &o in node.outputs.iter() { + let dom = self.use_block(nd, o, scheds); + debug_assert!( + self.dominates(min, dom, scheds), + "{min} {dom} {node:?} {:?}", + self.basic_blocks() + ); + } + } + } + + pub fn dominates(&self, dominator: Nid, mut dominated: Nid, scheds: Option<&[Nid]>) -> bool { + loop { + if dominator == dominated { + break true; + } + + debug_assert!(dominated != VOID); + + if self.idepth(dominator, scheds) > self.idepth(dominated, scheds) { + break false; + } + + dominated = self.idom(dominated, scheds); + } + } + + pub fn this_or_delegates<'a>(&'a self, source: Nid, target: &'a Nid) -> (Nid, &'a [Nid]) { + if self.is_unlocked(*target) { + (source, core::slice::from_ref(target)) + } else { + (*target, self[*target].outputs.as_slice()) + } + } + + pub fn is_hard_zero(&self, nid: Nid) -> bool { + self[nid].kind == Kind::CInt { value: 0 } + && self[nid].outputs.iter().all(|&n| self[n].kind != Kind::Phi) + } + + fn add_trigger(&mut self, blocker: Nid, target: Nid) { + if !self[blocker].peep_triggers.contains(&target) { + self[blocker].peep_triggers.push(target); + } + } +} + +impl ops::Index for Nodes { + type Output = Node; + + fn index(&self, index: Nid) -> &Self::Output { + self.values[index as usize].as_ref().unwrap_or_else(|(_, bt)| panic!("{index} {bt:#?}")) + } +} + +impl ops::IndexMut for Nodes { + fn index_mut(&mut self, index: Nid) -> &mut Self::Output { + self.values[index as usize].as_mut().unwrap_or_else(|(_, bt)| panic!("{index} {bt:#?}")) + } +} + +impl crate::ctx_map::CtxEntry for Nid { + type Ctx = [Result]; + type Key<'a> = (Kind, &'a [Nid], ty::Id); + + fn key<'a>(&self, ctx: &'a Self::Ctx) -> Self::Key<'a> { + ctx[*self as usize].as_ref().unwrap_or_else(|(_, t)| panic!("{t:#?}")).key() + } +} + +#[derive(Debug, Default, Clone)] +pub struct Node { + pub kind: Kind, + pub inputs: Vc, + pub outputs: Vc, + pub peep_triggers: Vc, + pub clobbers: BitSet, + pub ty: ty::Id, + pub pos: Pos, + + pub depth: Cell, + pub lock_rc: Cell, + pub loop_depth: Cell, + pub aclass: AClassId, +} + +impl Node { + fn is_dangling(&self) -> bool { + self.outputs.is_empty() && self.lock_rc.get() == 0 && self.kind != Kind::Arg + } + + fn key(&self) -> (Kind, &[Nid], ty::Id) { + (self.kind, &self.inputs, self.ty) + } + + pub fn is_lazy_phi(&self, loob: Nid) -> bool { + self.kind == Kind::Phi && self.inputs[2] == 0 && self.inputs[0] == loob + } + + fn is_not_gvnd(&self) -> bool { + (self.kind == Kind::Phi && self.inputs[2] == 0) + || matches!(self.kind, Kind::Arg | Kind::Stck | Kind::Stre) + || self.kind.is_cfg() + } + + fn is_mem(&self) -> bool { + matches!(self.kind, Kind::Stre | Kind::Load | Kind::Stck) + } + + pub fn is_data_phi(&self) -> bool { + self.kind == Kind::Phi && self.ty != ty::Id::VOID + } + + pub fn has_no_value(&self) -> bool { + (self.kind.is_cfg() && (!self.kind.is_call() || self.ty == ty::Id::VOID)) + || matches!(self.kind, Kind::Stre) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum AssertKind { + NullCheck, + UnwrapCheck, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] +#[repr(u8)] +pub enum Kind { + #[default] + Start, + // [ctrl] + Entry, + // [VOID] + Mem, + // [VOID] + Loops, + // [terms...] + End, + // [ctrl, cond] + If, + Then, + Else, + // [lhs, rhs] + Region, + // [entry, back] + Loop, + // [ctrl, ?value] + Return { + file: ty::Module, + }, + // [ctrl] + Die, + // [ctrl] + CInt { + value: i64, + }, + // [ctrl, lhs, rhs] + Phi, + Arg, + // [ctrl, oper] + UnOp { + op: lexer::TokenKind, + }, + // [ctrl, lhs, rhs] + BinOp { + op: lexer::TokenKind, + }, + // [ctrl] + Global { + global: ty::Global, + }, + // [ctrl, ...args] + Call { + func: ty::Func, + args: ty::Tuple, + }, + // [ctrl, cond, value] + Assert { + kind: AssertKind, + pos: Pos, + }, + // [ctrl] + Stck, + // [ctrl, memory] + Load, + // [ctrl, value, memory] + Stre, + // [ctrl, a, b] + Join, +} + +impl Kind { + pub fn is_call(&self) -> bool { + matches!(self, Kind::Call { .. }) + } + + pub fn is_eca(&self) -> bool { + matches!(self, Kind::Call { func: ty::Func::ECA, .. }) + } + + fn is_pinned(&self) -> bool { + self.is_cfg() || self.is_at_start() || matches!(self, Self::Phi | Kind::Assert { .. }) + } + + fn is_at_start(&self) -> bool { + matches!(self, Self::Arg | Self::Mem | Self::Loops | Self::Entry) + } + + pub fn is_cfg(&self) -> bool { + matches!( + self, + Self::Start + | Self::End + | Self::Return { .. } + | Self::Die + | Self::Entry + | Self::Then + | Self::Else + | Self::Call { .. } + | Self::If + | Self::Region + | Self::Loop + ) + } + + fn ends_basic_block(&self) -> bool { + matches!(self, Self::Return { .. } | Self::If | Self::End | Self::Die) + } + + pub fn starts_basic_block(&self) -> bool { + matches!(self, Self::Region | Self::Loop | Self::Start | Kind::Then | Kind::Else) + } + + fn is_peeped(&self) -> bool { + !matches!(self, Self::End | Self::Arg | Self::Mem | Self::Loops) + } +} + +impl fmt::Display for Kind { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Kind::CInt { value } => write!(f, "#{value}"), + Kind::Entry => write!(f, "ctrl[entry]"), + Kind::Then => write!(f, "ctrl[then]"), + Kind::Else => write!(f, "ctrl[else]"), + Kind::BinOp { op } => write!(f, "{op}"), + Kind::Call { func, .. } => write!(f, "call {func}"), + slf => write!(f, "{slf:?}"), + } + } +} + +pub enum CondOptRes { + Unknown, + Known { value: bool, pin: Option }, +} diff --git a/lang/src/son.rs b/lang/src/son.rs index 6cac0d14..56eb0a66 100644 --- a/lang/src/son.rs +++ b/lang/src/son.rs @@ -5,9 +5,9 @@ use { hbvm::{Comptime, HbvmBackend}, Backend, }, - ctx_map::CtxEntry, debug, - lexer::{self, TokenKind}, + lexer::TokenKind, + nodes::*, parser::{ self, idfl::{self}, @@ -24,38 +24,16 @@ use { alloc::{string::String, vec::Vec}, core::{ assert_matches::debug_assert_matches, - cell::{Cell, RefCell}, + cell::RefCell, fmt::{self, Debug, Display, Write}, format_args as fa, mem, - ops::{self, Range}, }, - hashbrown::hash_map, hbbytecode::DisasmError, }; -pub const VOID: Nid = 0; -pub const NEVER: Nid = 1; -pub const ENTRY: Nid = 2; -pub const MEM: Nid = 3; -pub const LOOPS: Nid = 4; -pub const ARG_START: usize = 3; const DEFAULT_ACLASS: usize = 0; const GLOBAL_ACLASS: usize = 1; -pub type Nid = u16; -type AClassId = i16; - -type Lookup = crate::ctx_map::CtxMap; - -impl crate::ctx_map::CtxEntry for Nid { - type Ctx = [Result]; - type Key<'a> = (Kind, &'a [Nid], ty::Id); - - fn key<'a>(&self, ctx: &'a Self::Ctx) -> Self::Key<'a> { - ctx[*self as usize].as_ref().unwrap_or_else(|(_, t)| panic!("{t:#?}")).key() - } -} - macro_rules! inference { ($ty:ident, $ctx:expr, $self:expr, $pos:expr, $subject:literal, $example:literal) => { let Some($ty) = $ctx.ty else { @@ -74,1942 +52,7 @@ macro_rules! inference { }; } -#[derive(Clone)] -pub struct Nodes { - values: Vec>, - queued_peeps: Vec, - free: Nid, - lookup: Lookup, -} - -impl Default for Nodes { - fn default() -> Self { - Self { - values: Default::default(), - queued_peeps: Default::default(), - free: Nid::MAX, - lookup: Default::default(), - } - } -} - impl Nodes { - #[inline] - pub fn len(&self) -> usize { - self.values.len() - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.values.is_empty() - } - - fn as_ty(&self, cint: Nid) -> ty::Id { - if self[cint].ty == ty::Id::NEVER { - return ty::Id::NEVER; - } - debug_assert_eq!(self[cint].ty, ty::Id::TYPE); - ty::Id::from(match self[cint].kind { - Kind::CInt { value } => value as u64, - _ => unreachable!("triing to cast non constant to a type: {:?}", self[cint]), - }) - } - - pub fn loop_depth(&self, target: Nid, scheds: Option<&[Nid]>) -> LoopDepth { - self[target].loop_depth.set(match self[target].kind { - Kind::Region | Kind::Entry | Kind::Then | Kind::Else | Kind::Call { .. } | Kind::If => { - if self[target].loop_depth.get() != 0 { - return self[target].loop_depth.get(); - } - self.loop_depth(self[target].inputs[0], scheds) - } - Kind::Loop => { - if self[target].loop_depth.get() - == self.loop_depth(self[target].inputs[0], scheds) + 1 - { - return self[target].loop_depth.get(); - } - let depth = self.loop_depth(self[target].inputs[0], scheds) + 1; - self[target].loop_depth.set(depth); - let mut cursor = self[target].inputs[1]; - while cursor != target { - self[cursor].loop_depth.set(depth); - let next = self.idom(cursor, scheds); - debug_assert_ne!(next, 0); - if matches!(self[cursor].kind, Kind::Then | Kind::Else) { - debug_assert_eq!(self[next].kind, Kind::If); - let other = self[next].outputs[(self[next].outputs[0] == cursor) as usize]; - self[other].loop_depth.set(depth - 1); - } - cursor = next; - } - depth - } - Kind::Start | Kind::End | Kind::Die | Kind::Return { .. } => 1, - u => unreachable!("{u:?}"), - }); - - self[target].loop_depth.get() - } - - fn idepth(&self, target: Nid, scheds: Option<&[Nid]>) -> IDomDepth { - if target == VOID { - return 0; - } - if self[target].depth.get() == 0 { - let depth = match self[target].kind { - Kind::End | Kind::Start => unreachable!("{:?}", self[target].kind), - Kind::Region => self - .idepth(self[target].inputs[0], scheds) - .max(self.idepth(self[target].inputs[1], scheds)), - _ if self[target].kind.is_pinned() => self.idepth(self[target].inputs[0], scheds), - _ if let Some(scheds) = scheds => { - self.idepth(scheds[target as usize], Some(scheds)) - } - _ => self.idepth(self[target].inputs[0], scheds), - } + 1; - self[target].depth.set(depth); - } - self[target].depth.get() - } - - fn fix_loops(&mut self, stack: &mut Vec, seen: &mut BitSet) { - debug_assert!(stack.is_empty()); - - stack.push(NEVER); - - while let Some(node) = stack.pop() { - if seen.set(node) && self.is_cfg(node) { - stack.extend(self[node].inputs.iter()); - } - } - - for l in self[LOOPS].outputs.clone() { - if !seen.get(l) { - self.bind(l, NEVER); - } - } - } - - fn push_up_impl(&self, node: Nid, visited: &mut BitSet, scheds: &mut [Nid]) { - if !visited.set(node) { - return; - } - - for &inp in &self[node].inputs[1..] { - if !self[inp].kind.is_pinned() { - self.push_up_impl(inp, visited, scheds); - } - } - - if self[node].kind.is_pinned() { - return; - } - - let mut deepest = self[node].inputs[0]; - for &inp in self[node].inputs[1..].iter() { - if self.idepth(inp, Some(scheds)) > self.idepth(deepest, Some(scheds)) { - if self[inp].kind.is_call() { - deepest = inp; - } else { - debug_assert!(!self.is_cfg(inp)); - deepest = self.idom(inp, Some(scheds)); - } - } - } - - scheds[node as usize] = deepest; - } - - fn collect_rpo(&self, node: Nid, rpo: &mut Vec, visited: &mut BitSet) { - if !self.is_cfg(node) || !visited.set(node) { - return; - } - - for &n in self[node].outputs.iter() { - self.collect_rpo(n, rpo, visited); - } - - rpo.push(node); - } - - fn push_up(&self, rpo: &mut Vec, visited: &mut BitSet, scheds: &mut [Nid]) { - debug_assert!(rpo.is_empty()); - self.collect_rpo(VOID, rpo, visited); - - for &node in rpo.iter().rev() { - self.loop_depth(node, Some(scheds)); - for i in 0..self[node].inputs.len() { - self.push_up_impl(self[node].inputs[i], visited, scheds); - } - - if matches!(self[node].kind, Kind::Loop | Kind::Region) { - for i in 0..self[node].outputs.len() { - let usage = self[node].outputs[i]; - if self[usage].kind == Kind::Phi { - self.push_up_impl(usage, visited, scheds); - } - } - } - } - - debug_assert_eq!( - self.iter() - .map(|(n, _)| n) - .filter(|&n| !visited.get(n) - && !matches!(self[n].kind, Kind::Arg | Kind::Mem | Kind::Loops)) - .collect::>(), - vec![], - "{:?}", - self.iter() - .filter(|&(n, nod)| !visited.get(n) - && !matches!(nod.kind, Kind::Arg | Kind::Mem | Kind::Loops)) - .collect::>() - ); - - rpo.clear(); - } - - fn better(&self, is: Nid, then: Nid, scheds: Option<&[Nid]>) -> bool { - debug_assert_ne!(self.idepth(is, scheds), self.idepth(then, scheds), "{is} {then}"); - self.loop_depth(is, scheds) < self.loop_depth(then, scheds) - || self.idepth(is, scheds) > self.idepth(then, scheds) - || self[then].kind == Kind::If - } - - fn is_forward_edge(&self, usage: Nid, def: Nid) -> bool { - match self[usage].kind { - Kind::Phi => { - self[usage].inputs[2] != def || self[self[usage].inputs[0]].kind != Kind::Loop - } - Kind::Loop => self[usage].inputs[1] != def, - _ => true, - } - } - - fn schedule_inside_blocks( - &mut self, - cfg_nodes: &mut Vec, - buf: &mut Vec, - seen: &mut BitSet, - ) { - debug_assert!(cfg_nodes.is_empty()); - debug_assert!(buf.is_empty()); - cfg_nodes.extend( - self.iter() - // skip VOID and NEVER - .skip(2) - .filter(|(_, n)| n.kind.is_cfg() && !n.kind.ends_basic_block()) - .map(|(n, _)| n), - ); - - for &block in &*cfg_nodes { - debug_assert!(block != VOID); - seen.clear(self.values.len()); - let mut outputs = mem::take(&mut self[block].outputs); - self.reschedule_block(block, &mut outputs, buf, seen); - self[block].outputs = outputs; - } - - cfg_nodes.clear(); - } - - fn reschedule_block( - &self, - from: Nid, - outputs: &mut [Nid], - buf: &mut Vec, - seen: &mut BitSet, - ) { - debug_assert!(buf.is_empty()); - - // NOTE: this code is horible - let fromc = Some(&from); - - let cfg_idx = outputs.iter().position(|&n| self.is_cfg(n)).unwrap(); - outputs.swap(cfg_idx, 0); - for &o in outputs.iter() { - if (!self.is_cfg(o) - && self[o].outputs.iter().any(|&oi| { - self[oi].kind != Kind::Phi && self[oi].inputs.first() == fromc && !seen.get(oi) - })) - || !seen.set(o) - { - continue; - } - let mut cursor = buf.len(); - for &o in outputs.iter().filter(|&&n| n == o) { - buf.push(o); - } - while let Some(&n) = buf.get(cursor) { - for &i in &self[n].inputs[1..] { - if fromc == self[i].inputs.first() - && self[i].outputs.iter().all(|&o| { - self[o].kind == Kind::Phi - || self[o].inputs.first() != fromc - || seen.get(o) - }) - && seen.set(i) - { - for &o in outputs.iter().filter(|&&n| n == i) { - buf.push(o); - } - } - } - cursor += 1; - } - } - - debug_assert_eq!( - outputs.iter().filter(|&&n| !seen.get(n)).copied().collect::>(), - vec![], - "{:?} {from:?} {:?}", - outputs - .iter() - .filter(|&&n| !seen.get(n)) - .copied() - .map(|n| (n, &self[n])) - .collect::>(), - self[from] - ); - - let bf = &buf; - debug_assert_eq!( - bf.iter() - .enumerate() - .filter(|(_, &b)| !self[b].kind.is_pinned()) - .flat_map(|(i, &b)| self[b] - .inputs - .iter() - .filter(|&&b| !self[b].kind.is_pinned()) - .filter_map(move |&inp| bf - .iter() - .position(|&n| inp == n) - .filter(|&j| i > j) - .map(|j| (bf[i], bf[j])))) - .collect::>(), - vec![], - "{:?}", - bf - ); - - debug_assert!(self.is_cfg(bf[0]) || self[bf[0]].kind == Kind::Phi, "{:?}", self[bf[0]]); - - if outputs.len() != buf.len() { - panic!("{:?} {:?}", outputs, buf); - } - outputs.copy_from_slice(buf); - buf.clear(); - } - - fn push_down( - &self, - node: Nid, - visited: &mut BitSet, - antideps: &mut [Nid], - scheds: &mut [Nid], - antidep_bounds: &mut Vec, - ) { - if !visited.set(node) { - return; - } - - for &usage in self[node].outputs.iter() { - if self.is_forward_edge(usage, node) && self[node].kind == Kind::Stre { - self.push_down(usage, visited, antideps, scheds, antidep_bounds); - } - } - - for &usage in self[node].outputs.iter() { - if self.is_forward_edge(usage, node) { - self.push_down(usage, visited, antideps, scheds, antidep_bounds); - } - } - - if self[node].kind.is_pinned() { - return; - } - - let mut min = None::; - for i in 0..self[node].outputs.len() { - let usage = self[node].outputs[i]; - let ub = self.use_block(node, usage, Some(scheds)); - min = min.map(|m| self.common_dom(ub, m, Some(scheds))).or(Some(ub)); - } - let mut min = min.unwrap(); - - debug_assert!(self.dominates(scheds[node as usize], min, Some(scheds))); - - let mut cursor = min; - let mut fuel = self.values.len(); - while cursor != scheds[node as usize] { - debug_assert!(fuel != 0); - fuel -= 1; - cursor = self.idom(cursor, Some(scheds)); - if self.better(cursor, min, Some(scheds)) { - min = cursor; - } - } - - if self[node].kind == Kind::Load { - min = self.find_antideps(node, min, antideps, scheds, antidep_bounds); - } - - if self[min].kind.ends_basic_block() { - min = self.idom(min, Some(scheds)); - } - - self.assert_dominance(node, min, true, Some(scheds)); - - debug_assert!( - self.idepth(min, Some(scheds)) >= self.idepth(scheds[node as usize], Some(scheds)) - ); - scheds[node as usize] = min; - } - - fn find_antideps( - &self, - load: Nid, - mut min: Nid, - antideps: &mut [Nid], - scheds: &[Nid], - antidep_bounds: &mut Vec, - ) -> Nid { - debug_assert!(self[load].kind == Kind::Load); - debug_assert!(self.dominates(scheds[load as usize], min, Some(scheds)), "{load}"); - - let (aclass, _) = self.aclass_index(self[load].inputs[1]); - - let mut cursor = min; - while cursor != scheds[load as usize] { - antideps[cursor as usize] = load; - if self[cursor].clobbers.get(aclass as _) { - min = self[cursor].inputs[0]; - } - cursor = self.idom(cursor, Some(scheds)); - } - - if self[load].inputs[2] == MEM { - return min; - } - - for &out in self[self[load].inputs[2]].outputs.iter() { - match self[out].kind { - Kind::Stre => { - let mut cursor = scheds[out as usize]; - if cursor == scheds[load as usize] { - antidep_bounds.extend([load, out]); - } - while cursor != scheds[load as usize] - && self.idepth(cursor, Some(scheds)) - > self.idepth(scheds[load as usize], Some(scheds)) - { - if antideps[cursor as usize] == load { - min = self.common_dom(min, cursor, Some(scheds)); - if min == cursor { - antidep_bounds.extend([load, out]); - } - break; - } - cursor = self.idom(cursor, Some(scheds)); - } - break; - } - Kind::Phi => { - let side = self[out].inputs[1..] - .iter() - .position(|&n| n == self[load].inputs[2]) - .unwrap(); - let ctrl = self[out].inputs[0]; - let mut cursor = self[ctrl].inputs[side]; - while cursor != scheds[load as usize] - && self.idepth(cursor, Some(scheds)) - > self.idepth(scheds[load as usize], Some(scheds)) - { - if antideps[cursor as usize] == load { - min = self.common_dom(min, cursor, Some(scheds)); - break; - } - cursor = self.idom(cursor, Some(scheds)); - } - } - _ => {} - } - } - - min - } - - fn bind(&mut self, from: Nid, to: Nid) { - debug_assert_ne!(to, 0); - debug_assert_ne!(self[to].kind, Kind::Phi); - self[from].outputs.push(to); - self[to].inputs.push(from); - } - - pub fn use_block(&self, target: Nid, from: Nid, scheds: Option<&[Nid]>) -> Nid { - if self[from].kind != Kind::Phi { - return self.idom(from, scheds); - } - - let index = self[from].inputs.iter().position(|&n| n == target).unwrap_or_else(|| { - panic!("from {from} {:?} target {target} {:?}", self[from], self[target]) - }); - self[self[from].inputs[0]].inputs[index - 1] - } - - pub fn idom(&self, target: Nid, scheds: Option<&[Nid]>) -> Nid { - match self[target].kind { - Kind::Start => unreachable!(), - Kind::End => unreachable!(), - Kind::Region => { - let &[lcfg, rcfg] = self[target].inputs.as_slice() else { unreachable!() }; - self.common_dom(lcfg, rcfg, scheds) - } - _ if self[target].kind.is_pinned() => self[target].inputs[0], - _ if let Some(scheds) = scheds => scheds[target as usize], - _ => self[target].inputs[0], - } - } - - fn common_dom(&self, mut a: Nid, mut b: Nid, scheds: Option<&[Nid]>) -> Nid { - while a != b { - let [ldepth, rdepth] = [self.idepth(a, scheds), self.idepth(b, scheds)]; - if ldepth >= rdepth { - a = self.idom(a, scheds); - } - if ldepth <= rdepth { - b = self.idom(b, scheds); - } - } - a - } - - fn merge_scopes( - &mut self, - loops: &mut [Loop], - ctrl: &StrongRef, - to: &mut Scope, - from: &mut Scope, - tys: &Types, - ) { - for (i, (to_value, from_value)) in to.vars.iter_mut().zip(from.vars.iter_mut()).enumerate() - { - debug_assert_eq!(to_value.ty, from_value.ty); - if to_value.value() != from_value.value() { - self.load_loop_var(i, from_value, loops); - self.load_loop_var(i, to_value, loops); - if to_value.value() != from_value.value() { - debug_assert!(!to_value.ptr); - debug_assert!(!from_value.ptr); - let inps = [ctrl.get(), from_value.value(), to_value.value()]; - to_value - .set_value_remove(self.new_node(from_value.ty, Kind::Phi, inps, tys), self); - } - } - } - - for (i, (to_class, from_class)) in - to.aclasses.iter_mut().zip(from.aclasses.iter_mut()).enumerate() - { - if to_class.last_store.get() != from_class.last_store.get() { - self.load_loop_aclass(i, from_class, loops); - self.load_loop_aclass(i, to_class, loops); - if to_class.last_store.get() != from_class.last_store.get() { - let inps = [ctrl.get(), from_class.last_store.get(), to_class.last_store.get()]; - to_class - .last_store - .set_remove(self.new_node(ty::Id::VOID, Kind::Phi, inps, tys), self); - } - } - } - } - - fn graphviz_low(&self, disp: ty::Display, out: &mut String) -> core::fmt::Result { - use core::fmt::Write; - - writeln!(out)?; - writeln!(out, "digraph G {{")?; - writeln!(out, "rankdir=BT;")?; - writeln!(out, "concentrate=true;")?; - writeln!(out, "compound=true;")?; - - for (i, node) in self.iter() { - let color = match () { - _ if node.lock_rc.get() == Nid::MAX => "orange", - _ if node.lock_rc.get() == Nid::MAX - 1 => "blue", - _ if node.lock_rc.get() != 0 => "red", - _ if node.outputs.is_empty() => "purple", - _ if node.is_mem() => "green", - _ if self.is_cfg(i) => "yellow", - _ => "white", - }; - - if node.ty != ty::Id::VOID { - writeln!( - out, - " node{i}[label=\"{i} {} {} {}\" color={color}]", - node.kind, - disp.rety(node.ty), - node.aclass, - )?; - } else { - writeln!( - out, - " node{i}[label=\"{i} {} {}\" color={color}]", - node.kind, node.aclass, - )?; - } - - for (j, &o) in node.outputs.iter().enumerate() { - let color = if self.is_cfg(i) && self.is_cfg(o) { "red" } else { "lightgray" }; - let index = self[o].inputs.iter().position(|&inp| i == inp).unwrap(); - let style = if index == 0 && !self.is_cfg(o) { "style=dotted" } else { "" }; - writeln!( - out, - " node{o} -> node{i}[color={color} taillabel={index} headlabel={j} {style}]", - )?; - } - } - - writeln!(out, "}}")?; - - Ok(()) - } - - fn graphviz(&self, disp: ty::Display) { - let out = &mut String::new(); - _ = self.graphviz_low(disp, out); - log::info!("{out}"); - } - - fn graphviz_in_browser(&self, _disp: ty::Display) { - #[cfg(all(test, feature = "std"))] - { - let out = &mut String::new(); - _ = self.graphviz_low(_disp, out); - if !std::process::Command::new("brave") - .arg(format!("https://dreampuf.github.io/GraphvizOnline/#{out}")) - .status() - .unwrap() - .success() - { - log::error!("{out}"); - } - } - } - - fn gcm(&mut self, scratch: &mut Vec, bind_buf: &mut Vec, visited: &mut BitSet) { - visited.clear(self.values.len()); - self.fix_loops(bind_buf, visited); - debug_assert!(bind_buf.is_empty()); - debug_assert!(scratch.is_empty()); - scratch.resize(self.values.len() * 2, Nid::MAX); - let (antideps, scheds) = scratch.split_at_mut(self.values.len()); - visited.clear(self.values.len()); - self.push_up(bind_buf, visited, scheds); - visited.clear(self.values.len()); - self.push_down(VOID, visited, antideps, scheds, bind_buf); - - for &[from, to] in bind_buf.array_chunks() { - self.bind(from, to); - } - - self[VOID].outputs = - self[VOID].outputs.iter().filter(|&&n| self[n].kind.is_at_start()).copied().collect(); - - for (&shed, n) in scheds.iter().zip(0u16..) { - if shed == Nid::MAX { - continue; - } - - let prev = mem::replace(&mut self[n].inputs[0], shed); - if prev != VOID { - let index = self[prev].outputs.iter().position(|&o| o == n).unwrap(); - self[prev].outputs.swap_remove(index); - } - self[shed].outputs.push(n); - } - - bind_buf.clear(); - scratch.clear(); - visited.clear(self.values.len()); - self.schedule_inside_blocks(bind_buf, scratch, visited); - } - - fn clear(&mut self) { - self.values.clear(); - self.lookup.clear(); - self.free = Nid::MAX; - } - - fn new_node_nop(&mut self, ty: ty::Id, kind: Kind, inps: impl Into) -> Nid { - let node = Node { inputs: inps.into(), kind, ty, ..Default::default() }; - - if node.kind == Kind::Phi && node.ty != ty::Id::VOID { - debug_assert_ne!( - self[node.inputs[1]].ty, - ty::Id::VOID, - "{:?} {:?}", - self[node.inputs[1]], - node.ty.expand(), - ); - - if self[node.inputs[0]].kind != Kind::Loop { - debug_assert_ne!( - self[node.inputs[2]].ty, - ty::Id::VOID, - "{:?} {:?}", - self[node.inputs[2]], - node.ty.expand(), - ); - } - - debug_assert!(!matches!(node.ty.expand(), ty::Kind::Struct(_))); - } - - let mut lookup_meta = None; - if !node.is_not_gvnd() { - let (raw_entry, hash) = self.lookup.entry(node.key(), &self.values); - - let entry = match raw_entry { - hash_map::RawEntryMut::Occupied(o) => return o.get_key_value().0.value, - hash_map::RawEntryMut::Vacant(v) => v, - }; - - lookup_meta = Some((entry, hash)); - } - - if self.free == Nid::MAX { - self.free = self.values.len() as _; - self.values.push(Err((Nid::MAX, debug::trace()))); - } - - let free = self.free; - for &d in node.inputs.as_slice() { - debug_assert_ne!(d, free); - self.values[d as usize].as_mut().unwrap_or_else(|_| panic!("{d}")).outputs.push(free); - } - self.free = mem::replace(&mut self.values[free as usize], Ok(node)).unwrap_err().0; - - if let Some((entry, hash)) = lookup_meta { - entry.insert(crate::ctx_map::Key { value: free, hash }, ()); - } - free - } - - fn remove_node_lookup(&mut self, target: Nid) { - if !self[target].is_not_gvnd() { - self.lookup - .remove(&target, &self.values) - .unwrap_or_else(|| panic!("{:?}", self[target])); - } - } - - fn new_node(&mut self, ty: ty::Id, kind: Kind, inps: impl Into, tys: &Types) -> Nid { - let id = self.new_node_nop(ty, kind, inps); - if let Some(opt) = self.peephole(id, tys) { - debug_assert_ne!(opt, id); - for peep in mem::take(&mut self.queued_peeps) { - self.unlock(peep); - } - self.lock(opt); - self.remove(id); - self.unlock(opt); - opt - } else { - id - } - } - - fn new_const(&mut self, ty: ty::Id, value: impl Into) -> Nid { - self.new_node_nop(ty, Kind::CInt { value: value.into() }, [VOID]) - } - - fn new_const_lit(&mut self, ty: ty::Id, value: impl Into) -> Value { - Value::new(self.new_const(ty, value)).ty(ty) - } - - fn new_node_lit(&mut self, ty: ty::Id, kind: Kind, inps: impl Into, tys: &Types) -> Value { - Value::new(self.new_node(ty, kind, inps, tys)).ty(ty) - } - - // TODO: make this internal to son and force backends to track locks thelself - - pub fn is_locked(&self, target: Nid) -> bool { - self[target].lock_rc.get() != 0 - } - - pub fn is_unlocked(&self, target: Nid) -> bool { - self[target].lock_rc.get() == 0 - } - - pub fn lock(&self, target: Nid) { - self[target].lock_rc.set(self[target].lock_rc.get() + 1); - } - - #[track_caller] - pub fn unlock(&self, target: Nid) { - self[target].lock_rc.set(self[target].lock_rc.get() - 1); - } - - fn remove(&mut self, target: Nid) -> bool { - if !self[target].is_dangling() { - return false; - } - - for i in 0..self[target].inputs.len() { - let inp = self[target].inputs[i]; - let index = self[inp].outputs.iter().position(|&p| p == target).unwrap(); - self[inp].outputs.swap_remove(index); - self.remove(inp); - } - - self.remove_node_lookup(target); - - if cfg!(debug_assertions) { - mem::replace(&mut self.values[target as usize], Err((Nid::MAX, debug::trace()))) - .unwrap(); - } else { - mem::replace(&mut self.values[target as usize], Err((self.free, debug::trace()))) - .unwrap(); - self.free = target; - } - - true - } - - fn late_peephole(&mut self, target: Nid, tys: &Types) -> Option { - if let Some(id) = self.peephole(target, tys) { - for peep in mem::take(&mut self.queued_peeps) { - self.unlock(peep); - } - self.replace(target, id); - return None; - } - None - } - - fn iter_peeps(&mut self, mut fuel: usize, stack: &mut Vec, tys: &Types) { - debug_assert!(stack.is_empty()); - debug_assert!(self.queued_peeps.is_empty()); - - self.iter() - .filter_map(|(id, node)| node.kind.is_peeped().then_some(id)) - .collect_into(stack); - stack.iter().for_each(|&s| self.lock(s)); - - while fuel != 0 - && let Some(node) = stack.pop() - { - fuel -= 1; - - if self.unlock_remove(node) { - continue; - } - - if let Some(new) = self.peephole(node, tys) { - self.replace(node, new); - self.push_adjacent_nodes(new, stack); - } - stack.append(&mut self.queued_peeps); - - //debug_assert_matches!( - // self.iter().find(|(i, n)| n.lock_rc.get() != 0 - // && n.kind.is_peeped() - // && !stack.contains(i)), - // None - //); - } - - debug_assert!(self.queued_peeps.is_empty()); - - stack.drain(..).for_each(|s| _ = self.unlock_remove(s)); - } - - fn push_adjacent_nodes(&mut self, of: Nid, stack: &mut Vec) { - let prev_len = stack.len(); - for &i in self[of] - .outputs - .iter() - .chain(self[of].inputs.iter()) - .chain(self[of].peep_triggers.iter()) - { - if self.values[i as usize].is_ok() - && self[i].kind.is_peeped() - && self[i].lock_rc.get() == 0 - { - stack.push(i); - } - } - - self[of].peep_triggers = Vc::default(); - stack.iter().skip(prev_len).for_each(|&n| self.lock(n)); - } - - fn aclass_index(&self, region: Nid) -> (usize, Nid) { - if self[region].aclass >= 0 { - (self[region].aclass as _, region) - } else { - ( - self[self[region].aclass.unsigned_abs() - 1].aclass as _, - self[region].aclass.unsigned_abs() - 1, - ) - } - } - - fn pass_aclass(&mut self, from: Nid, to: Nid) { - debug_assert!(self[from].aclass >= 0); - if from != to { - self[to].aclass = -(from as AClassId + 1); - } - } - - fn peephole(&mut self, target: Nid, tys: &Types) -> Option { - use {Kind as K, TokenKind as T}; - match self[target].kind { - K::BinOp { op } => { - let &[ctrl, mut lhs, mut rhs] = self[target].inputs.as_slice() else { - unreachable!() - }; - let ty = self[target].ty; - - let is_float = self[target].ty.is_float(); - - if let (&K::CInt { value: a }, &K::CInt { value: b }) = - (&self[lhs].kind, &self[rhs].kind) - { - return Some(self.new_const(ty, op.apply_binop(a, b, is_float))); - } - - if lhs == rhs { - match op { - T::Ne | T::Gt | T::Lt | T::Sub => return Some(self.new_const(ty, 0)), - T::Eq | T::Ge | T::Le => return Some(self.new_const(ty, 1)), - T::Add => { - let rhs = self.new_const(ty, 2); - return Some(self.new_node( - ty, - K::BinOp { op: T::Mul }, - [ctrl, lhs, rhs], - tys, - )); - } - _ => {} - } - } - - // this is more general the pushing constants to left to help deduplicate expressions more - let mut changed = false; - if op.is_comutative() && self[lhs].key() < self[rhs].key() { - mem::swap(&mut lhs, &mut rhs); - changed = true; - } - - if let K::CInt { value } = self[rhs].kind { - match (op, value) { - (T::Eq, 0) if self[lhs].ty.is_pointer() || self[lhs].kind == Kind::Stck => { - return Some(self.new_const(ty::Id::BOOL, 0)); - } - (T::Ne, 0) if self[lhs].ty.is_pointer() || self[lhs].kind == Kind::Stck => { - return Some(self.new_const(ty::Id::BOOL, 1)); - } - (T::Add | T::Sub | T::Shl, 0) | (T::Mul | T::Div, 1) => return Some(lhs), - (T::Mul, 0) => return Some(rhs), - _ => {} - } - } - - if op.is_comutative() && self[lhs].kind == (K::BinOp { op }) { - let &[_, a, b] = self[lhs].inputs.as_slice() else { unreachable!() }; - if let K::CInt { value: av } = self[b].kind - && let K::CInt { value: bv } = self[rhs].kind - { - // (a op #b) op #c => a op (#b op #c) - let new_rhs = self.new_const(ty, op.apply_binop(av, bv, is_float)); - return Some(self.new_node(ty, K::BinOp { op }, [ctrl, a, new_rhs], tys)); - } - - if self.is_const(b) { - // (a op #b) op c => (a op c) op #b - let new_lhs = self.new_node(ty, K::BinOp { op }, [ctrl, a, rhs], tys); - return Some(self.new_node(ty, K::BinOp { op }, [ctrl, new_lhs, b], tys)); - } - - self.add_trigger(b, target); - } - - if op == T::Add - && self[lhs].kind == (K::BinOp { op: T::Mul }) - && self[lhs].inputs[1] == rhs - && let K::CInt { value } = self[self[lhs].inputs[2]].kind - { - // a * #n + a => a * (#n + 1) - let new_rhs = self.new_const(ty, value + 1); - return Some(self.new_node( - ty, - K::BinOp { op: T::Mul }, - [ctrl, rhs, new_rhs], - tys, - )); - } - - if op == T::Sub - && self[lhs].kind == (K::BinOp { op: T::Add }) - && let K::CInt { value: a } = self[rhs].kind - && let K::CInt { value: b } = self[self[lhs].inputs[2]].kind - { - let new_rhs = self.new_const(ty, b - a); - return Some(self.new_node( - ty, - K::BinOp { op: T::Add }, - [ctrl, self[lhs].inputs[1], new_rhs], - tys, - )); - } - - if op == T::Sub && self[lhs].kind == (K::BinOp { op }) { - // (a - b) - c => a - (b + c) - let &[_, a, b] = self[lhs].inputs.as_slice() else { unreachable!() }; - let c = rhs; - let new_rhs = self.new_node(ty, K::BinOp { op: T::Add }, [ctrl, b, c], tys); - return Some(self.new_node(ty, K::BinOp { op }, [ctrl, a, new_rhs], tys)); - } - - if op == T::Add - && self[rhs].kind == (K::BinOp { op: T::Mul }) - && let &[_, index, step] = self[rhs].inputs.as_slice() - && self[index].kind == K::Phi - && let &[iter_loop, index_init, new_index] = self[index].inputs.as_slice() - && new_index != VOID - && self[iter_loop].kind == K::Loop - && self[new_index].kind == (K::BinOp { op: T::Add }) - && self[new_index].inputs[1] == index - && let Some(&iter_cond) = self[index].outputs.iter().find( - |&&n| matches!(self[n].kind, Kind::BinOp { op } if op.is_compatison()), - ) - && self[index].outputs.iter().all(|n| [iter_cond, rhs, new_index].contains(n)) - { - // arr := @as([u32; 10], idk) - // - // i := 0 - // loop if i == 10 break else { - // arr[i] = 0 - // i += 1 - // } - // - // ||||| - // VVVVV - // - // cursor := &arr[0] + 0 - // end := &arr[0] + 10 - // loop if cursor == end else { - // *cursor = 0 - // i += 1 - // } - - debug_assert!(self[iter_cond].inputs.contains(&index)); - let iter_bound_index = - self[iter_cond].inputs.iter().rposition(|&n| n != index).unwrap(); - debug_assert_ne!(iter_bound_index, 0); - let end_shift = self.new_node( - self[rhs].ty, - K::BinOp { op: T::Mul }, - [ctrl, self[iter_cond].inputs[iter_bound_index], step], - tys, - ); - let end = - self.new_node(ty, K::BinOp { op: T::Add }, [ctrl, lhs, end_shift], tys); - - let init_shift = self.new_node( - self[rhs].ty, - K::BinOp { op: T::Mul }, - [ctrl, index_init, step], - tys, - ); - let init = - self.new_node(ty, K::BinOp { op: T::Add }, [ctrl, lhs, init_shift], tys); - - let new_value = self.new_node_nop(ty, K::Phi, [iter_loop, init, 0]); - let next = - self.new_node(ty, Kind::BinOp { op: T::Add }, [ctrl, new_value, step], tys); - - let mut new_cond_inputs = self[iter_cond].inputs.clone(); - new_cond_inputs[iter_bound_index] = end; - new_cond_inputs[3 - iter_bound_index] = new_value; - let new_cond = self.new_node(ty, self[iter_cond].kind, new_cond_inputs, tys); - self.replace(iter_cond, new_cond); - - return Some(self.modify_input(new_value, 2, next)); - } - - if changed { - return Some(self.new_node(ty, self[target].kind, [ctrl, lhs, rhs], tys)); - } - } - K::UnOp { op } => { - let &[_, oper] = self[target].inputs.as_slice() else { unreachable!() }; - let ty = self[target].ty; - - if matches!(op, TokenKind::Number | TokenKind::Float) - && tys.size_of(self[oper].ty) == tys.size_of(ty) - && self[oper].ty.is_integer() - && ty.is_integer() - { - return Some(oper); - } - - if let K::CInt { value } = self[oper].kind { - let is_float = self[oper].ty.is_float(); - return Some(self.new_const(ty, op.apply_unop(value, is_float))); - } - } - K::If => { - if self[target].inputs[0] == NEVER { - return Some(NEVER); - } - - if self[target].ty == ty::Id::VOID { - match self.try_match_cond(target) { - CondOptRes::Unknown => {} - CondOptRes::Known { value, .. } => { - let ty = if value { - ty::Id::RIGHT_UNREACHABLE - } else { - ty::Id::LEFT_UNREACHABLE - }; - return Some(self.new_node_nop(ty, K::If, self[target].inputs.clone())); - } - } - } - } - K::Then => { - if self[target].inputs[0] == NEVER { - return Some(NEVER); - } - - if self[self[target].inputs[0]].ty == ty::Id::LEFT_UNREACHABLE { - return Some(NEVER); - } else if self[self[target].inputs[0]].ty == ty::Id::RIGHT_UNREACHABLE { - return Some(self[self[target].inputs[0]].inputs[0]); - } - } - K::Else => { - if self[target].inputs[0] == NEVER { - return Some(NEVER); - } - - if self[self[target].inputs[0]].ty == ty::Id::RIGHT_UNREACHABLE { - return Some(NEVER); - } else if self[self[target].inputs[0]].ty == ty::Id::LEFT_UNREACHABLE { - return Some(self[self[target].inputs[0]].inputs[0]); - } - } - K::Region => { - let (ctrl, side) = match self[target].inputs.as_slice() { - [NEVER, NEVER] => return Some(NEVER), - &[NEVER, ctrl] => (ctrl, 2), - &[ctrl, NEVER] => (ctrl, 1), - _ => return None, - }; - - self.lock(target); - for i in self[target].outputs.clone() { - if self[i].kind == Kind::Phi { - for o in self[i].outputs.clone() { - if self.is_unlocked(o) { - self.lock(o); - self.queued_peeps.push(o); - } - } - self.replace(i, self[i].inputs[side]); - } - } - self.unlock(target); - - return Some(ctrl); - } - K::Call { .. } => { - if self[target].inputs[0] == NEVER { - return Some(NEVER); - } - } - K::Return { file } => { - if self[target].inputs[0] == NEVER { - return Some(NEVER); - } - - let mut new_inps = Vc::from(&self[target].inputs[..2]); - 'a: for &n in self[target].inputs.clone().iter().skip(2) { - if self[n].kind != Kind::Stre { - new_inps.push(n); - continue; - } - - if let Some(&load) = - self[n].outputs.iter().find(|&&n| self[n].kind == Kind::Load) - { - self.add_trigger(load, target); - continue; - } - - let mut cursor = n; - let class = self.aclass_index(self[cursor].inputs[2]); - - if self[class.1].kind != Kind::Stck { - new_inps.push(n); - continue; - } - - if self[class.1].outputs.iter().any(|&n| { - self[n].kind != Kind::Stre - && self[n].outputs.iter().any(|&n| self[n].kind != Kind::Stre) - }) { - new_inps.push(n); - continue; - } - - cursor = self[cursor].inputs[3]; - while cursor != MEM { - debug_assert_eq!(self[cursor].kind, Kind::Stre); - if self.aclass_index(self[cursor].inputs[2]) != class { - new_inps.push(n); - continue 'a; - } - - if let Some(&load) = - self[cursor].outputs.iter().find(|&&n| self[n].kind == Kind::Load) - { - self.add_trigger(load, target); - continue 'a; - } - - cursor = self[cursor].inputs[3]; - } - } - - if new_inps.as_slice() != self[target].inputs.as_slice() { - let ret = self.new_node_nop(ty::Id::VOID, Kind::Return { file }, new_inps); - self[ret].pos = self[target].pos; - return Some(ret); - } - } - K::Phi => { - let &[ctrl, lhs, rhs] = self[target].inputs.as_slice() else { unreachable!() }; - - if rhs == target || lhs == rhs { - return Some(lhs); - } - - // TODO: travese the graph downward and chech if this phi is only consumed by it - // self - if self[target].outputs.as_slice() == [rhs] - && self[rhs].outputs.as_slice() == [target] - { - return Some(lhs); - } - - if self[lhs].kind == Kind::Stre - && self[rhs].kind == Kind::Stre - && self[lhs].ty == self[rhs].ty - && self[lhs].ty.loc(tys) == Loc::Reg - && self[lhs].inputs[2] == self[rhs].inputs[2] - && self[lhs].inputs[3] == self[rhs].inputs[3] - { - let pick_value = self.new_node( - self[lhs].ty, - Kind::Phi, - [ctrl, self[lhs].inputs[1], self[rhs].inputs[1]], - tys, - ); - let mut vc = self[lhs].inputs.clone(); - vc[1] = pick_value; - return Some(self.new_node(self[lhs].ty, Kind::Stre, vc, tys)); - } - - // broken - //let ty = self[target].ty; - //if let Kind::BinOp { op } = self[lhs].kind - // && self[rhs].kind == (Kind::BinOp { op }) - //{ - // debug_assert!(ty != ty::Id::VOID); - // debug_assert_eq!( - // self[lhs].ty.simple_size(), - // ty.simple_size(), - // "{:?} {:?}", - // self[lhs].ty.expand(), - // ty.expand() - // ); - // debug_assert_eq!( - // self[rhs].ty.simple_size(), - // ty.simple_size(), - // "{:?} {:?}", - // self[rhs].ty.expand(), - // ty.expand() - // ); - // let inps = [ctrl, self[lhs].inputs[1], self[rhs].inputs[1]]; - // let nlhs = self.new_node(ty, Kind::Phi, inps, tys); - // let inps = [ctrl, self[lhs].inputs[2], self[rhs].inputs[2]]; - // let nrhs = self.new_node(ty, Kind::Phi, inps, tys); - // return Some(self.new_node(ty, Kind::BinOp { op }, [VOID, nlhs, nrhs], tys)); - //} - } - K::Stck => { - if let &[mut a, mut b] = self[target].outputs.as_slice() { - if self[a].kind == Kind::Load { - mem::swap(&mut a, &mut b); - } - - if self[a].kind.is_call() - && self[a].inputs.last() == Some(&target) - && self[b].kind == Kind::Load - && let &[store] = self[b].outputs.as_slice() - && self[store].kind == Kind::Stre - { - let len = self[a].inputs.len(); - let stre = self[store].inputs[3]; - if stre != MEM { - self[a].inputs.push(stre); - self[a].inputs.swap(len - 1, len); - self[stre].outputs.push(a); - } - return Some(self[store].inputs[2]); - } - } - } - K::Stre => { - let &[_, value, region, store, ..] = self[target].inputs.as_slice() else { - unreachable!() - }; - - if self[value].kind == Kind::Load && self[value].inputs[1] == region { - return Some(store); - } - - let mut cursor = target; - while self[cursor].kind == Kind::Stre - && self[cursor].inputs[1] != VOID - && let &[next_store] = self[cursor].outputs.as_slice() - { - if self[next_store].inputs[2] == region - && self[next_store].ty == self[target].ty - { - return Some(store); - } - cursor = next_store; - } - - 'eliminate: { - if self[target].outputs.is_empty() { - break 'eliminate; - } - - if self[value].kind != Kind::Load - || self[value].outputs.iter().any(|&n| self[n].kind != Kind::Stre) - { - for &ele in self[value].outputs.clone().iter().filter(|&&n| n != target) { - self.add_trigger(ele, target); - } - break 'eliminate; - } - - let &[_, stack, last_store] = self[value].inputs.as_slice() else { - unreachable!() - }; - - if self[stack].ty != self[value].ty || self[stack].kind != Kind::Stck { - break 'eliminate; - } - - let mut unidentifed = self[stack].outputs.clone(); - let load_idx = unidentifed.iter().position(|&n| n == value).unwrap(); - unidentifed.swap_remove(load_idx); - - let mut saved = Vc::default(); - let mut cursor = last_store; - let mut first_store = last_store; - while cursor != MEM && self[cursor].kind == Kind::Stre { - let mut contact_point = cursor; - let mut region = self[cursor].inputs[2]; - if let Kind::BinOp { op } = self[region].kind { - debug_assert_matches!(op, TokenKind::Add | TokenKind::Sub); - contact_point = region; - region = self[region].inputs[1] - } - - if region != stack { - break; - } - let Some(index) = unidentifed.iter().position(|&n| n == contact_point) - else { - break 'eliminate; - }; - if self[self[cursor].inputs[1]].kind == Kind::Load - && self[value].outputs.iter().any(|&n| { - self.aclass_index(self[self[cursor].inputs[1]].inputs[1]).0 - == self.aclass_index(self[n].inputs[2]).0 - }) - { - break 'eliminate; - } - unidentifed.remove(index); - saved.push(contact_point); - first_store = cursor; - cursor = *self[cursor].inputs.get(3).unwrap_or(&MEM); - - if unidentifed.is_empty() { - break; - } - } - - if !unidentifed.is_empty() { - break 'eliminate; - } - - debug_assert_matches!( - self[last_store].kind, - Kind::Stre | Kind::Mem, - "{:?}", - self[last_store] - ); - debug_assert_matches!( - self[first_store].kind, - Kind::Stre | Kind::Mem, - "{:?}", - self[first_store] - ); - - // FIXME: when the loads and stores become parallel we will need to get saved - // differently - let mut prev_store = store; - for mut oper in saved.into_iter().rev() { - let mut region = region; - if let Kind::BinOp { op } = self[oper].kind { - debug_assert_eq!(self[oper].outputs.len(), 1); - debug_assert_eq!(self[self[oper].outputs[0]].kind, Kind::Stre); - let new_region = self.new_node( - self[oper].ty, - Kind::BinOp { op }, - [VOID, region, self[oper].inputs[2]], - tys, - ); - self.pass_aclass(self.aclass_index(region).1, new_region); - region = new_region; - oper = self[oper].outputs[0]; - } - - let mut inps = self[oper].inputs.clone(); - debug_assert_eq!(inps.len(), 4); - inps[2] = region; - inps[3] = prev_store; - prev_store = self.new_node_nop(self[oper].ty, Kind::Stre, inps); - if self.is_unlocked(prev_store) { - self.lock(prev_store); - self.queued_peeps.push(prev_store); - } - } - - return Some(prev_store); - } - - if let Some(&load) = - self[target].outputs.iter().find(|&&n| self[n].kind == Kind::Load) - { - self.add_trigger(load, target); - } else if value != VOID - && self[value].kind != Kind::Load - && self[store].kind == Kind::Stre - && self[store].inputs[2] == region - { - if self[store].inputs[1] == value { - return Some(store); - } - - let mut inps = self[target].inputs.clone(); - inps[3] = self[store].inputs[3]; - return Some(self.new_node_nop(self[target].ty, Kind::Stre, inps)); - } - } - K::Load => { - fn range_of(s: &Nodes, mut region: Nid, ty: ty::Id, tys: &Types) -> Range { - let loc = s.aclass_index(region).1; - let full_size = tys.size_of( - if matches!(s[loc].kind, Kind::Stck | Kind::Arg | Kind::Global { .. }) { - s[loc].ty - } else if let Some(ptr) = tys.base_of(s[loc].ty) { - ptr - } else { - return 0..usize::MAX; - }, - ); - let size = tys.size_of(ty); - let mut offset = 0; - loop { - match s[region].kind { - _ if region == loc => { - break offset as usize..offset as usize + size as usize - } - Kind::Assert { kind: AssertKind::NullCheck, .. } => { - region = s[region].inputs[2] - } - Kind::BinOp { op: TokenKind::Add | TokenKind::Sub } - if let Kind::CInt { value } = s[s[region].inputs[2]].kind => - { - offset += value; - region = s[region].inputs[1]; - } - _ => break 0..full_size as usize, - }; - } - } - - let &[ctrl, region, store] = self[target].inputs.as_slice() else { unreachable!() }; - let load_range = range_of(self, region, self[target].ty, tys); - - let mut cursor = store; - while cursor != MEM && self[cursor].kind != Kind::Phi { - if self[cursor].inputs[0] == ctrl - && self[cursor].inputs[2] == region - && self[cursor].ty == self[target].ty - && (self[self[cursor].inputs[1]].kind != Kind::Load - || (!self[target].outputs.is_empty() - && self[target].outputs.iter().all(|&n| { - self[n].kind != Kind::Stre - || self - .aclass_index(self[self[cursor].inputs[1]].inputs[1]) - .0 - != self.aclass_index(self[n].inputs[2]).0 - }))) - { - return Some(self[cursor].inputs[1]); - } - let range = range_of(self, self[cursor].inputs[2], self[cursor].ty, tys); - if range.start >= load_range.end || range.end <= load_range.start { - cursor = self[cursor].inputs[3]; - } else { - let reg = self.aclass_index(self[cursor].inputs[2]).1; - self.add_trigger(reg, target); - break; - } - } - - if store != cursor { - return Some(self.new_node( - self[target].ty, - Kind::Load, - [ctrl, region, cursor], - tys, - )); - } - } - K::Loop => { - if self[target].inputs[1] == NEVER || self[target].inputs[0] == NEVER { - self.lock(target); - for o in self[target].outputs.clone() { - if self[o].kind == Kind::Phi { - self.remove_node_lookup(target); - - let prev = self[o].inputs[2]; - self[o].inputs[2] = VOID; - self[VOID].outputs.push(o); - let index = self[prev].outputs.iter().position(|&n| n == o).unwrap(); - self[prev].outputs.swap_remove(index); - self.lock(o); - self.remove(prev); - self.unlock(o); - - for o in self[o].outputs.clone() { - if self.is_unlocked(o) { - self.lock(o); - self.queued_peeps.push(o); - } - } - self.replace(o, self[o].inputs[1]); - } - } - self.unlock(target); - return Some(self[target].inputs[0]); - } - } - K::Die => { - if self[target].inputs[0] == NEVER { - return Some(NEVER); - } - } - K::Assert { kind, .. } => 'b: { - let pin = match (kind, self.try_match_cond(target)) { - (AssertKind::NullCheck, CondOptRes::Known { value: false, pin }) => pin, - (AssertKind::UnwrapCheck, CondOptRes::Unknown) => None, - _ => break 'b, - } - .unwrap_or(self[target].inputs[0]); - - for out in self[target].outputs.clone() { - if !self[out].kind.is_pinned() && self[out].inputs[0] != pin { - self.modify_input(out, 0, pin); - } - } - return Some(self[target].inputs[2]); - } - K::Start => {} - _ if self.is_cfg(target) && self.idom(target, None) == NEVER => panic!(), - K::Entry - | K::Mem - | K::Loops - | K::End - | K::CInt { .. } - | K::Arg - | K::Global { .. } - | K::Join => {} - } - - None - } - - fn try_match_cond(&self, target: Nid) -> CondOptRes { - let &[ctrl, cond, ..] = self[target].inputs.as_slice() else { unreachable!() }; - if let Kind::CInt { value } = self[cond].kind { - return CondOptRes::Known { value: value != 0, pin: None }; - } - - let mut cursor = ctrl; - while cursor != ENTRY { - let ctrl = &self[cursor]; - // TODO: do more inteligent checks on the condition - if matches!(ctrl.kind, Kind::Then | Kind::Else) { - if self[ctrl.inputs[0]].kind == Kind::End { - return CondOptRes::Unknown; - } - debug_assert_eq!(self[ctrl.inputs[0]].kind, Kind::If); - let other_cond = self[ctrl.inputs[0]].inputs[1]; - if let Some(value) = self.matches_cond(cond, other_cond) { - return CondOptRes::Known { - value: (ctrl.kind == Kind::Then) ^ !value, - pin: Some(cursor), - }; - } - } - - cursor = self.idom(cursor, None); - } - - CondOptRes::Unknown - } - - fn matches_cond(&self, to_match: Nid, matches: Nid) -> Option { - use TokenKind as K; - let [tn, mn] = [&self[to_match], &self[matches]]; - match (tn.kind, mn.kind) { - _ if to_match == matches => Some(true), - (Kind::BinOp { op: K::Ne }, Kind::BinOp { op: K::Eq }) - | (Kind::BinOp { op: K::Eq }, Kind::BinOp { op: K::Ne }) - if tn.inputs[1..] == mn.inputs[1..] => - { - Some(false) - } - (_, Kind::BinOp { op: K::Band }) => self - .matches_cond(to_match, mn.inputs[1]) - .or(self.matches_cond(to_match, mn.inputs[2])), - (_, Kind::BinOp { op: K::Bor }) => match ( - self.matches_cond(to_match, mn.inputs[1]), - self.matches_cond(to_match, mn.inputs[2]), - ) { - (None, Some(a)) | (Some(a), None) => Some(a), - (Some(b), Some(a)) if a == b => Some(a), - _ => None, - }, - _ => None, - } - } - - pub fn is_const(&self, id: Nid) -> bool { - matches!(self[id].kind, Kind::CInt { .. }) - } - - fn replace(&mut self, target: Nid, with: Nid) { - debug_assert_ne!(target, with, "{:?}", self[target]); - for out in self[target].outputs.clone() { - let index = self[out].inputs.iter().position(|&p| p == target).unwrap(); - self.modify_input(out, index, with); - } - } - - fn modify_input(&mut self, target: Nid, inp_index: usize, with: Nid) -> Nid { - self.remove_node_lookup(target); - debug_assert_ne!( - self[target].inputs[inp_index], with, - "{:?} {:?}", - self[target], self[with] - ); - - if self[target].is_not_gvnd() && (self[target].kind != Kind::Phi || with == 0) { - let prev = self[target].inputs[inp_index]; - self[target].inputs[inp_index] = with; - self[with].outputs.push(target); - let index = self[prev].outputs.iter().position(|&o| o == target).unwrap(); - self[prev].outputs.swap_remove(index); - self.remove(prev); - target - } else { - let prev = self[target].inputs[inp_index]; - self[target].inputs[inp_index] = with; - let (entry, hash) = self.lookup.entry(target.key(&self.values), &self.values); - match entry { - hash_map::RawEntryMut::Occupied(other) => { - let rpl = other.get_key_value().0.value; - self[target].inputs[inp_index] = prev; - self.lookup.insert(target.key(&self.values), target, &self.values); - self.replace(target, rpl); - rpl - } - hash_map::RawEntryMut::Vacant(slot) => { - slot.insert(crate::ctx_map::Key { value: target, hash }, ()); - let index = self[prev].outputs.iter().position(|&o| o == target).unwrap(); - self[prev].outputs.swap_remove(index); - self[with].outputs.push(target); - self.remove(prev); - target - } - } - } - } - - #[track_caller] - fn unlock_remove(&mut self, id: Nid) -> bool { - self.unlock(id); - self.remove(id) - } - - fn iter(&self) -> impl DoubleEndedIterator { - self.values.iter().enumerate().filter_map(|(i, s)| Some((i as _, s.as_ref().ok()?))) - } - - #[expect(clippy::format_in_format_args)] - fn basic_blocks_instr(&self, out: &mut String, node: Nid) -> core::fmt::Result { - match self[node].kind { - Kind::Assert { .. } | Kind::Start => unreachable!("{} {out}", self[node].kind), - Kind::End => return Ok(()), - Kind::If => write!(out, " if: "), - Kind::Region | Kind::Loop => writeln!(out, " goto: {node}"), - Kind::Return { .. } => write!(out, " ret: "), - Kind::Die => write!(out, " die: "), - Kind::CInt { value } => write!(out, "cint: #{value:<4}"), - Kind::Phi => write!(out, " phi: "), - Kind::Arg => write!( - out, - " arg: {:<5}", - self[VOID].outputs.iter().position(|&n| n == node).unwrap() - 2 - ), - Kind::BinOp { op } | Kind::UnOp { op } => { - write!(out, "{:>4}: ", op.name()) - } - Kind::Call { func, args: _ } => { - write!(out, "call: {func} {} ", self[node].depth.get()) - } - Kind::Global { global } => write!(out, "glob: {global:<5}"), - Kind::Entry => write!(out, "ctrl: {:<5}", "entry"), - Kind::Then => write!(out, "ctrl: {:<5}", "then"), - Kind::Else => write!(out, "ctrl: {:<5}", "else"), - Kind::Stck => write!(out, "stck: "), - Kind::Load => write!(out, "load: "), - Kind::Stre => write!(out, "stre: "), - Kind::Mem => write!(out, " mem: "), - Kind::Loops => write!(out, "loops: "), - Kind::Join => write!(out, "join: "), - }?; - - if self[node].kind != Kind::Loop && self[node].kind != Kind::Region { - writeln!( - out, - " {:<3} {:<14} {}", - node, - format!("{:?}", self[node].inputs), - format!("{:?}", self[node].outputs) - )?; - } - - Ok(()) - } - - fn basic_blocks_low( - &self, - out: &mut String, - mut node: Nid, - visited: &mut BitSet, - ) -> core::fmt::Result { - let iter = |nodes: &Nodes, node| nodes[node].outputs.clone().into_iter().rev(); - while visited.set(node) { - match self[node].kind { - Kind::Start => { - writeln!(out, "start: {}", self[node].depth.get())?; - let mut cfg_index = Nid::MAX; - for o in iter(self, node) { - self.basic_blocks_instr(out, o)?; - if self[o].kind.is_cfg() { - cfg_index = o; - } - } - node = cfg_index; - } - Kind::End => break, - Kind::If => { - self.basic_blocks_low(out, self[node].outputs[0], visited)?; - node = self[node].outputs[1]; - } - Kind::Region => { - writeln!( - out, - "region{node}: {} {} {:?}", - self[node].depth.get(), - self[node].loop_depth.get(), - self[node].inputs - )?; - let mut cfg_index = Nid::MAX; - for o in iter(self, node) { - self.basic_blocks_instr(out, o)?; - if self.is_cfg(o) { - cfg_index = o; - } - } - node = cfg_index; - } - Kind::Loop => { - writeln!( - out, - "loop{node}: {} {} {:?}", - self[node].depth.get(), - self[node].loop_depth.get(), - self[node].outputs - )?; - let mut cfg_index = Nid::MAX; - for o in iter(self, node) { - self.basic_blocks_instr(out, o)?; - if self.is_cfg(o) { - cfg_index = o; - } - } - node = cfg_index; - } - Kind::Return { .. } | Kind::Die => { - node = self[node].outputs[0]; - } - Kind::Then | Kind::Else | Kind::Entry => { - writeln!( - out, - "b{node}: {} {} {:?}", - self[node].depth.get(), - self[node].loop_depth.get(), - self[node].outputs - )?; - let mut cfg_index = Nid::MAX; - for o in iter(self, node) { - self.basic_blocks_instr(out, o)?; - if self.is_cfg(o) { - cfg_index = o; - } - } - node = cfg_index; - } - Kind::Call { .. } => { - let mut cfg_index = Nid::MAX; - let mut print_ret = true; - for o in iter(self, node) { - if self[o].inputs[0] == node - && (self[node].outputs[0] != o || mem::take(&mut print_ret)) - { - self.basic_blocks_instr(out, o)?; - } - if self.is_cfg(o) { - cfg_index = o; - } - } - node = cfg_index; - } - _ => unreachable!(), - } - } - - Ok(()) - } - - fn basic_blocks(&self) { - let mut out = String::new(); - let mut visited = BitSet::default(); - self.basic_blocks_low(&mut out, VOID, &mut visited).unwrap(); - log::info!("{out}"); - } - - pub fn is_cfg(&self, o: Nid) -> bool { - self[o].kind.is_cfg() - } - - fn check_final_integrity(&self, disp: ty::Display) { - if !cfg!(debug_assertions) { - return; - } - - let mut failed = false; - for (id, node) in self.iter() { - if self.is_locked(id) { - log::error!("{} {} {:?}", node.lock_rc.get(), 0, node.kind); - failed = true; - } - if !matches!(node.kind, Kind::End | Kind::Mem | Kind::Arg | Kind::Loops) - && node.outputs.is_empty() - { - log::error!("outputs are empry {id} {:?}", node); - failed = true; - } - if node.inputs.first() == Some(&NEVER) && id != NEVER { - log::error!("is unreachable but still present {id} {:?}", node.kind); - failed = true; - } - if node.outputs.contains(&id) && !matches!(node.kind, Kind::Loop | Kind::End) { - log::error!("node depends on it self and its not a loop {id} {:?}", node); - failed = true; - } - } - - if failed { - self.graphviz_in_browser(disp); - panic!() - } - } - - fn check_loop_depth_integrity(&self, disp: ty::Display) { - if !cfg!(debug_assertions) { - return; - } - - let mut failed = false; - for &loob in self[LOOPS].outputs.iter() { - let mut stack = vec![self[loob].inputs[1]]; - let mut seen = BitSet::default(); - seen.set(loob); - let depth = self.loop_depth(loob, None); - while let Some(nid) = stack.pop() { - if seen.set(nid) { - if depth > self.loop_depth(nid, None) { - failed = true; - log::error!("{depth} {} {nid} {:?}", self.loop_depth(nid, None), self[nid]); - } - - match self[nid].kind { - Kind::Loop | Kind::Region => { - stack.extend(&self[nid].inputs[..2]); - } - _ => stack.push(self[nid].inputs[0]), - } - } - } - } - - if failed { - self.graphviz_in_browser(disp); - panic!() - } - } - fn load_loop_var(&mut self, index: usize, var: &mut Variable, loops: &mut [Loop]) { if var.value() != VOID { return; @@ -2059,273 +102,55 @@ impl Nodes { aclass.last_store.set(lvar.last_store.get(), self); } - fn assert_dominance(&self, nd: Nid, min: Nid, check_outputs: bool, scheds: Option<&[Nid]>) { - if !cfg!(debug_assertions) { - return; + fn merge_scopes( + &mut self, + loops: &mut [Loop], + ctrl: &StrongRef, + to: &mut Scope, + from: &mut Scope, + tys: &Types, + ) { + for (i, (to_value, from_value)) in to.vars.iter_mut().zip(from.vars.iter_mut()).enumerate() + { + debug_assert_eq!(to_value.ty, from_value.ty); + if to_value.value() != from_value.value() { + self.load_loop_var(i, from_value, loops); + self.load_loop_var(i, to_value, loops); + if to_value.value() != from_value.value() { + debug_assert!(!to_value.ptr); + debug_assert!(!from_value.ptr); + let inps = [ctrl.get(), from_value.value(), to_value.value()]; + to_value + .set_value_remove(self.new_node(from_value.ty, Kind::Phi, inps, tys), self); + } + } } - let node = &self[nd]; - for &i in &node.inputs[1..] { - let dom = self.idom(i, scheds); - debug_assert!( - self.dominates(dom, min, scheds), - "{dom} {min} {node:?} {:?}", - self.basic_blocks() - ); - } - if check_outputs { - for &o in node.outputs.iter() { - let dom = self.use_block(nd, o, scheds); - debug_assert!( - self.dominates(min, dom, scheds), - "{min} {dom} {node:?} {:?}", - self.basic_blocks() - ); + for (i, (to_class, from_class)) in + to.aclasses.iter_mut().zip(from.aclasses.iter_mut()).enumerate() + { + if to_class.last_store.get() != from_class.last_store.get() { + self.load_loop_aclass(i, from_class, loops); + self.load_loop_aclass(i, to_class, loops); + if to_class.last_store.get() != from_class.last_store.get() { + let inps = [ctrl.get(), from_class.last_store.get(), to_class.last_store.get()]; + to_class + .last_store + .set_remove(self.new_node(ty::Id::VOID, Kind::Phi, inps, tys), self); + } } } } - pub fn dominates(&self, dominator: Nid, mut dominated: Nid, scheds: Option<&[Nid]>) -> bool { - loop { - if dominator == dominated { - break true; - } - - debug_assert!(dominated != VOID); - - if self.idepth(dominator, scheds) > self.idepth(dominated, scheds) { - break false; - } - - dominated = self.idom(dominated, scheds); - } + fn new_const_lit(&mut self, ty: ty::Id, value: impl Into) -> Value { + Value::new(self.new_const(ty, value)).ty(ty) } - pub fn this_or_delegates<'a>(&'a self, source: Nid, target: &'a Nid) -> (Nid, &'a [Nid]) { - if self.is_unlocked(*target) { - (source, core::slice::from_ref(target)) - } else { - (*target, self[*target].outputs.as_slice()) - } - } - - pub fn is_hard_zero(&self, nid: Nid) -> bool { - self[nid].kind == Kind::CInt { value: 0 } - && self[nid].outputs.iter().all(|&n| self[n].kind != Kind::Phi) - } - - fn add_trigger(&mut self, blocker: Nid, target: Nid) { - if !self[blocker].peep_triggers.contains(&target) { - self[blocker].peep_triggers.push(target); - } + fn new_node_lit(&mut self, ty: ty::Id, kind: Kind, inps: impl Into, tys: &Types) -> Value { + Value::new(self.new_node(ty, kind, inps, tys)).ty(ty) } } -enum CondOptRes { - Unknown, - Known { value: bool, pin: Option }, -} - -impl ops::Index for Nodes { - type Output = Node; - - fn index(&self, index: Nid) -> &Self::Output { - self.values[index as usize].as_ref().unwrap_or_else(|(_, bt)| panic!("{index} {bt:#?}")) - } -} - -impl ops::IndexMut for Nodes { - fn index_mut(&mut self, index: Nid) -> &mut Self::Output { - self.values[index as usize].as_mut().unwrap_or_else(|(_, bt)| panic!("{index} {bt:#?}")) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum AssertKind { - NullCheck, - UnwrapCheck, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] -#[repr(u8)] -pub enum Kind { - #[default] - Start, - // [ctrl] - Entry, - // [VOID] - Mem, - // [VOID] - Loops, - // [terms...] - End, - // [ctrl, cond] - If, - Then, - Else, - // [lhs, rhs] - Region, - // [entry, back] - Loop, - // [ctrl, ?value] - Return { - file: ty::Module, - }, - // [ctrl] - Die, - // [ctrl] - CInt { - value: i64, - }, - // [ctrl, lhs, rhs] - Phi, - Arg, - // [ctrl, oper] - UnOp { - op: lexer::TokenKind, - }, - // [ctrl, lhs, rhs] - BinOp { - op: lexer::TokenKind, - }, - // [ctrl] - Global { - global: ty::Global, - }, - // [ctrl, ...args] - Call { - func: ty::Func, - args: ty::Tuple, - }, - // [ctrl, cond, value] - Assert { - kind: AssertKind, - pos: Pos, - }, - // [ctrl] - Stck, - // [ctrl, memory] - Load, - // [ctrl, value, memory] - Stre, - // [ctrl, a, b] - Join, -} - -impl Kind { - pub fn is_call(&self) -> bool { - matches!(self, Kind::Call { .. }) - } - - fn is_eca(&self) -> bool { - matches!(self, Kind::Call { func: ty::Func::ECA, .. }) - } - - fn is_pinned(&self) -> bool { - self.is_cfg() || self.is_at_start() || matches!(self, Self::Phi | Kind::Assert { .. }) - } - - fn is_at_start(&self) -> bool { - matches!(self, Self::Arg | Self::Mem | Self::Loops | Self::Entry) - } - - pub fn is_cfg(&self) -> bool { - matches!( - self, - Self::Start - | Self::End - | Self::Return { .. } - | Self::Die - | Self::Entry - | Self::Then - | Self::Else - | Self::Call { .. } - | Self::If - | Self::Region - | Self::Loop - ) - } - - fn ends_basic_block(&self) -> bool { - matches!(self, Self::Return { .. } | Self::If | Self::End | Self::Die) - } - - pub fn starts_basic_block(&self) -> bool { - matches!(self, Self::Region | Self::Loop | Self::Start | Kind::Then | Kind::Else) - } - - fn is_peeped(&self) -> bool { - !matches!(self, Self::End | Self::Arg | Self::Mem | Self::Loops) - } -} - -impl fmt::Display for Kind { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Kind::CInt { value } => write!(f, "#{value}"), - Kind::Entry => write!(f, "ctrl[entry]"), - Kind::Then => write!(f, "ctrl[then]"), - Kind::Else => write!(f, "ctrl[else]"), - Kind::BinOp { op } => write!(f, "{op}"), - Kind::Call { func, .. } => write!(f, "call {func}"), - slf => write!(f, "{slf:?}"), - } - } -} - -#[derive(Debug, Default, Clone)] -pub struct Node { - pub kind: Kind, - pub inputs: Vc, - pub outputs: Vc, - pub peep_triggers: Vc, - pub clobbers: BitSet, - pub ty: ty::Id, - pub pos: Pos, - - depth: Cell, - lock_rc: Cell, - loop_depth: Cell, - aclass: AClassId, -} - -impl Node { - fn is_dangling(&self) -> bool { - self.outputs.is_empty() && self.lock_rc.get() == 0 && self.kind != Kind::Arg - } - - fn key(&self) -> (Kind, &[Nid], ty::Id) { - (self.kind, &self.inputs, self.ty) - } - - fn is_lazy_phi(&self, loob: Nid) -> bool { - self.kind == Kind::Phi && self.inputs[2] == 0 && self.inputs[0] == loob - } - - fn is_not_gvnd(&self) -> bool { - (self.kind == Kind::Phi && self.inputs[2] == 0) - || matches!(self.kind, Kind::Arg | Kind::Stck | Kind::Stre) - || self.kind.is_cfg() - } - - fn is_mem(&self) -> bool { - matches!(self.kind, Kind::Stre | Kind::Load | Kind::Stck) - } - - pub fn is_data_phi(&self) -> bool { - self.kind == Kind::Phi && self.ty != ty::Id::VOID - } - - pub fn has_no_value(&self) -> bool { - (self.kind.is_cfg() && (!self.kind.is_call() || self.ty == ty::Id::VOID)) - || matches!(self.kind, Kind::Stre) - } -} - -type LoopDepth = u16; -type LockRc = u16; -type IDomDepth = u16; - #[derive(Clone, Copy)] pub enum CtLoopState { Terminated, @@ -3322,6 +1147,14 @@ impl<'a> Codegen<'a> { let mut lhs = self.ptr_expr_ctx(left, ctx)?; self.implicit_unwrap(left.pos(), &mut lhs); + fn is_scalar_op(op: TokenKind, ty: ty::Id) -> bool { + ty.is_pointer() + || ty.is_integer() + || ty == ty::Id::BOOL + || (ty == ty::Id::TYPE && matches!(op, TokenKind::Eq | TokenKind::Ne)) + || (ty.is_float() && op.is_supported_float_op()) + } + match lhs.ty.expand() { ty::Kind::Struct(s) if op.is_homogenous() => { debug_assert!(lhs.ptr); @@ -3350,7 +1183,7 @@ impl<'a> Codegen<'a> { self.struct_fold_op(left.pos(), op, binding_op, s, lhs.id, rhs.id) .or(Value::NEVER) } - _ if op.is_scalar_op(lhs.ty) => { + _ if is_scalar_op(op, lhs.ty) => { self.strip_ptr(&mut lhs); self.ci.nodes.lock(lhs.id); let rhs = self.expr_ctx(right, Ctx::default().with_ty(lhs.ty)); @@ -6046,16 +3879,6 @@ impl<'a> Codegen<'a> { } } -impl TokenKind { - fn is_scalar_op(self, ty: ty::Id) -> bool { - ty.is_pointer() - || ty.is_integer() - || ty == ty::Id::BOOL - || (ty == ty::Id::TYPE && matches!(self, TokenKind::Eq | TokenKind::Ne)) - || (ty.is_float() && self.is_supported_float_op()) - } -} - #[derive(Clone, Copy, Default)] struct TyScope { file: Module, diff --git a/lang/tests/son_tests_storing_into_nullable_struct.txt b/lang/tests/son_tests_storing_into_nullable_struct.txt index 938ee472..713870e4 100644 --- a/lang/tests/son_tests_storing_into_nullable_struct.txt +++ b/lang/tests/son_tests_storing_into_nullable_struct.txt @@ -50,21 +50,43 @@ optional: ADDI64 r254, r254, 16d JALA r0, r31, 0a optionala: - ADDI64 r254, r254, -48d - CP r17, r1 - ADDI64 r14, r254, 8d - ADDI64 r15, r254, 16d - ST r14, r254, 16a, 8h - LI64 r16, 1d - ST r16, r254, 24a, 8h - ADDI64 r13, r254, 0d - ST r13, r254, 32a, 8h + ADDI64 r254, r254, -128d + ST r31, r254, 64a, 64h + CP r38, r1 + ADDI64 r33, r254, 48d + JAL r31, r0, :some_index + ST r1, r33, 0a, 16h + LD r34, r254, 48a, 1h + ANDI r34, r34, 255d + JNE r34, r0, :0 + UN + 0: ADDI64 r34, r254, 8d + ADDI64 r35, r254, 16d + LRA r36, r0, :heap + LD r33, r254, 56a, 8h + ST r34, r254, 16a, 8h + LI64 r37, 1d + ADD64 r33, r33, r36 + ST r37, r254, 24a, 8h + LD r33, r33, 0a, 1h + ADDI64 r32, r254, 0d + ST r32, r254, 32a, 8h + ANDI r33, r33, 255d ST r0, r254, 0a, 8h ST r0, r254, 8a, 8h - ST r0, r254, 40a, 8h - BMC r15, r17, 32h - ADDI64 r254, r254, 48d + ST r33, r254, 40a, 8h + BMC r35, r38, 32h + LD r31, r254, 64a, 64h + ADDI64 r254, r254, 128d JALA r0, r31, 0a -code size: 556 +some_index: + ADDI64 r254, r254, -16d + LI8 r13, 1b + ST r13, r254, 0a, 1h + ST r0, r254, 8a, 8h + LD r1, r254, 0a, 16h + ADDI64 r254, r254, 16d + JALA r0, r31, 0a +code size: 769 ret: 100 status: Ok(())