From e4da0ca0e0375b892df7686e41c994d3aef550b8 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Thu, 13 Apr 2023 17:38:14 -0700 Subject: [PATCH 1/2] Support compilation of individual functions before serializing whole module, to keep memory usage down --- src/backend/mod.rs | 35 ++++++++++++++++------------------- src/interp.rs | 1 + src/ir/display.rs | 11 +++++++++++ src/ir/func.rs | 23 +++++++++++++++++------ 4 files changed, 45 insertions(+), 25 deletions(-) diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 9b42cce..328f92d 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -48,7 +48,7 @@ impl<'a> WasmFuncBackend<'a> { }) } - pub fn compile(&self) -> Result { + pub fn compile(&self) -> Result> { let mut func = wasm_encoder::Function::new( self.locals .locals @@ -76,7 +76,12 @@ impl<'a> WasmFuncBackend<'a> { log::debug!("Compiled to:\n{:?}\n", func); - Ok(func) + let mut bytes = vec![]; + { + use wasm_encoder::Encode; + func.encode(&mut bytes); + } + Ok(bytes) } fn lower_block(&self, block: &WasmBlock<'_>, func: &mut wasm_encoder::Function) { @@ -590,7 +595,9 @@ pub fn compile(module: &Module<'_>) -> anyhow::Result> { for (func, func_decl) in module.funcs.entries().skip(num_func_imports) { match func_decl { FuncDecl::Import(_, _) => anyhow::bail!("Import comes after func with body: {}", func), - FuncDecl::Lazy(sig, _, _) | FuncDecl::Body(sig, _, _) => { + FuncDecl::Lazy(sig, _, _) + | FuncDecl::Body(sig, _, _) + | FuncDecl::Compiled(sig, _, _) => { funcs.function(sig.index() as u32); } FuncDecl::None => panic!("FuncDecl::None at compilation time"), @@ -695,10 +702,6 @@ pub fn compile(module: &Module<'_>) -> anyhow::Result> { into_mod.section(&elem); let mut code = wasm_encoder::CodeSection::new(); - enum FuncOrRawBytes<'a> { - Func(wasm_encoder::Function), - Raw(&'a [u8]), - } let bodies = module .funcs @@ -706,33 +709,27 @@ pub fn compile(module: &Module<'_>) -> anyhow::Result> { .skip(num_func_imports) .collect::>() .par_iter() - .map(|(func, func_decl)| -> Result { + .map(|(func, func_decl)| -> Result<_> { match func_decl { FuncDecl::Lazy(_, _name, reader) => { let data = &module.orig_bytes[reader.range()]; - Ok(FuncOrRawBytes::Raw(data)) + Ok(Cow::Borrowed(data)) } + FuncDecl::Compiled(_, _name, bytes) => Ok(Cow::Borrowed(&bytes[..])), FuncDecl::Body(_, name, body) => { log::debug!("Compiling {} \"{}\"", func, name); WasmFuncBackend::new(body)? .compile() - .map(|f| FuncOrRawBytes::Func(f)) + .map(|bytes| Cow::Owned(bytes)) } FuncDecl::Import(_, _) => unreachable!("Should have skipped imports"), FuncDecl::None => panic!("FuncDecl::None at compilation time"), } }) - .collect::>>>()?; + .collect::>>()?; for body in bodies { - match body { - FuncOrRawBytes::Func(f) => { - code.function(&f); - } - FuncOrRawBytes::Raw(bytes) => { - code.raw(bytes); - } - } + code.raw(&body[..]); } into_mod.section(&code); diff --git a/src/interp.rs b/src/interp.rs index 9417597..0765bae 100644 --- a/src/interp.rs +++ b/src/interp.rs @@ -90,6 +90,7 @@ impl InterpContext { pub fn call(&mut self, module: &Module<'_>, func: Func, args: &[ConstVal]) -> InterpResult { let body = match &module.funcs[func] { FuncDecl::Lazy(..) => panic!("Un-expanded function"), + FuncDecl::Compiled(..) => panic!("Already-compiled function"), FuncDecl::Import(..) => { let import = &module.imports[func.index()]; assert_eq!(import.kind, ImportKind::Func(func)); diff --git a/src/ir/display.rs b/src/ir/display.rs index 841829c..61132cf 100644 --- a/src/ir/display.rs +++ b/src/ir/display.rs @@ -252,6 +252,17 @@ impl<'a> Display for ModuleDisplay<'a> { )?; writeln!(f, " # raw bytes (length {})", reader.range().len())?; } + FuncDecl::Compiled(sig, name, bytes) => { + writeln!( + f, + " {} \"{}\": {} = # {}", + func, + name, + sig, + sig_strs.get(&sig).unwrap() + )?; + writeln!(f, " # already compiled (length {})", bytes.len())?; + } FuncDecl::Import(sig, name) => { writeln!( f, diff --git a/src/ir/func.rs b/src/ir/func.rs index a06c2a4..f6b7a7a 100644 --- a/src/ir/func.rs +++ b/src/ir/func.rs @@ -1,4 +1,5 @@ use super::{Block, FunctionBodyDisplay, Local, Module, Signature, Type, Value, ValueDef}; +use crate::backend::WasmFuncBackend; use crate::cfg::CFGInfo; use crate::entity::{EntityRef, EntityVec, PerEntity}; use crate::frontend::parse_body; @@ -16,6 +17,8 @@ pub enum FuncDecl<'a> { Lazy(Signature, String, wasmparser::FunctionBody<'a>), /// A modified or new function body that requires compilation. Body(Signature, String, FunctionBody), + /// A compiled function body (was IR, has been collapsed back to bytecode). + Compiled(Signature, String, Vec), /// A placeholder. #[default] None, @@ -27,6 +30,7 @@ impl<'a> FuncDecl<'a> { FuncDecl::Import(sig, ..) => *sig, FuncDecl::Lazy(sig, ..) => *sig, FuncDecl::Body(sig, ..) => *sig, + FuncDecl::Compiled(sig, ..) => *sig, FuncDecl::None => panic!("No signature for FuncDecl::None"), } } @@ -76,18 +80,20 @@ impl<'a> FuncDecl<'a> { pub fn name(&self) -> &str { match self { - FuncDecl::Body(_, name, _) | FuncDecl::Lazy(_, name, _) | FuncDecl::Import(_, name) => { - &name[..] - } + FuncDecl::Body(_, name, _) + | FuncDecl::Lazy(_, name, _) + | FuncDecl::Import(_, name) + | FuncDecl::Compiled(_, name, _) => &name[..], FuncDecl::None => panic!("No name for FuncDecl::None"), } } pub fn set_name(&mut self, new_name: &str) { match self { - FuncDecl::Body(_, name, _) | FuncDecl::Lazy(_, name, _) | FuncDecl::Import(_, name) => { - *name = new_name.to_owned() - } + FuncDecl::Body(_, name, _) + | FuncDecl::Lazy(_, name, _) + | FuncDecl::Import(_, name) + | FuncDecl::Compiled(_, name, _) => *name = new_name.to_owned(), FuncDecl::None => panic!("No name for FuncDecl::None"), } } @@ -429,6 +435,11 @@ impl FunctionBody { Ok(()) } + + pub fn compile(&self) -> Result> { + let backend = WasmFuncBackend::new(self)?; + backend.compile() + } } #[derive(Clone, Debug, Default)] From 9d1ef7f396946ba15ce3bb6d8212d7fcb8fef2f1 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Thu, 13 Apr 2023 17:38:45 -0700 Subject: [PATCH 2/2] Version 0.0.15. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1fdfa13..1170148 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "waffle" -version = "0.0.14" +version = "0.0.15" description = "Wasm Analysis Framework For Lightweight Experiments" authors = ["Chris Fallin "] license = "Apache-2.0 WITH LLVM-exception"