From 0f4ff918d2acd934790d72fa747361ed42c382cb Mon Sep 17 00:00:00 2001 From: Jakub Doka Date: Sun, 13 Oct 2024 12:25:12 +0200 Subject: [PATCH] fixing bugs and improving memory consumption --- Cargo.lock | 152 ++++++++++++++++++++++++++++++++++++- Cargo.toml | 4 +- depell/README.md | 2 +- depell/src/index.js | 145 ++++++++++++++++++----------------- depell/src/main.rs | 4 +- depell/wasm-hbc/src/lib.rs | 12 ++- depell/wasm-rt/src/lib.rs | 4 + lang/src/codegen.rs | 32 ++++---- lang/src/fmt.rs | 16 ++-- lang/src/fs.rs | 2 +- lang/src/lib.rs | 11 ++- lang/src/parser.rs | 76 +++++++++++++------ lang/src/son.rs | 5 +- xtask/Cargo.toml | 4 + xtask/src/main.rs | 35 ++++++++- 15 files changed, 368 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3faf19..054cc95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ - "gimli", + "gimli 0.31.1", ] [[package]] @@ -35,6 +35,12 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +[[package]] +name = "anyhow" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" + [[package]] name = "async-trait" version = "0.1.83" @@ -46,6 +52,12 @@ dependencies = [ "syn", ] +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + [[package]] name = "axum" version = "0.7.7" @@ -175,6 +187,18 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -246,12 +270,29 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" +dependencies = [ + "fallible-iterator 0.2.0", + "indexmap 1.9.3", + "stable_deref_trait", +] + [[package]] name = "gimli" version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -259,6 +300,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", + "serde", ] [[package]] @@ -306,6 +348,12 @@ dependencies = [ "memmap2", ] +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.9" @@ -408,12 +456,45 @@ dependencies = [ "tower-service", ] +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +dependencies = [ + "equivalent", + "hashbrown 0.15.0", + "serde", +] + [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + [[package]] name = "libc" version = "0.2.159" @@ -574,7 +655,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ "bitflags", - "fallible-iterator", + "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", @@ -605,6 +686,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + [[package]] name = "serde" version = "1.0.210" @@ -681,6 +768,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "syn" version = "2.0.79" @@ -815,12 +908,49 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walrus" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68aa3c7b80be75c8458fc087453e5a31a226cfffede2e9b932393b2ea1c624a" +dependencies = [ + "anyhow", + "gimli 0.26.2", + "id-arena", + "leb128", + "log", + "walrus-macro", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "walrus-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ad39ff894c43c9649fa724cdde9a6fc50b855d517ef071a93e5df82fe51d3" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-encoder" +version = "0.212.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501940df4418b8929eb6d52f1aade1fdd15a5b86c92453cb696e3c906bd3fc33" +dependencies = [ + "leb128", +] + [[package]] name = "wasm-hbc" version = "0.1.0" @@ -846,6 +976,20 @@ dependencies = [ "log", ] +[[package]] +name = "wasmparser" +version = "0.212.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d28bc49ba1e5c5b61ffa7a2eace10820443c4b7d1c0b144109261d14570fdf8" +dependencies = [ + "ahash", + "bitflags", + "hashbrown 0.14.5", + "indexmap 2.6.0", + "semver", + "serde", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -922,6 +1066,10 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "xtask" version = "0.1.0" +dependencies = [ + "anyhow", + "walrus", +] [[package]] name = "zerocopy" diff --git a/Cargo.toml b/Cargo.toml index ef76769..e949af7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,8 +23,8 @@ hbjit = { path = "jit" } [profile.release] lto = true -#debug = true -strip = true +debug = true +#strip = true codegen-units = 1 panic = "abort" diff --git a/depell/README.md b/depell/README.md index a5ccaa7..b5576bf 100644 --- a/depell/README.md +++ b/depell/README.md @@ -5,6 +5,6 @@ Depell is a website that allows users to import/post/run hblang code and create ## Local Development ```bash -cargo xtask watch-depell +cargo xtask watch-depell-debug # browser http://localhost:8080 ``` diff --git a/depell/src/index.js b/depell/src/index.js index 431de93..e906c55 100644 --- a/depell/src/index.js +++ b/depell/src/index.js @@ -18,32 +18,90 @@ function compileCode(instance, code, fuel) { let { INPUT, INPUT_LEN, LOG_MESSAGES, LOG_MESSAGES_LEN, - PANIC_MESSAGE, PANIC_MESSAGE_LEN, memory, compile_and_run, } = instance.exports; if (!(true + && memory instanceof WebAssembly.Memory && INPUT instanceof WebAssembly.Global && INPUT_LEN instanceof WebAssembly.Global && LOG_MESSAGES instanceof WebAssembly.Global && LOG_MESSAGES_LEN instanceof WebAssembly.Global - && memory instanceof WebAssembly.Memory && typeof compile_and_run === "function" - )) console.log(instance.exports), never(); + )) never(); const dw = new DataView(memory.buffer); dw.setUint32(INPUT_LEN.value, code.length, true); new Uint8Array(memory.buffer, INPUT.value).set(code); + runWasmFunction(instance, compile_and_run, fuel); + return bufToString(memory, LOG_MESSAGES, LOG_MESSAGES_LEN); +} + +/**@type{WebAssembly.Instance}*/ let fmtInstance; +/**@type{Promise}*/ let fmtInstaceFuture; +/** @param {string} code @param {"fmt" | "minify"} action + * @returns {Promise} */ +async function modifyCode(code, action) { + fmtInstaceFuture ??= WebAssembly.instantiateStreaming(fetch("/hbfmt.wasm"), {}); + fmtInstance ??= (await fmtInstaceFuture).instance; + + let { + INPUT, INPUT_LEN, + OUTPUT, OUTPUT_LEN, + memory, fmt, minify + } = fmtInstance.exports; + + if (!(true + && memory instanceof WebAssembly.Memory + && INPUT instanceof WebAssembly.Global + && INPUT_LEN instanceof WebAssembly.Global + && OUTPUT instanceof WebAssembly.Global + && OUTPUT_LEN instanceof WebAssembly.Global + && typeof fmt === "function" + && typeof minify === "function" + )) never(); + + if (action !== "fmt") { + INPUT = OUTPUT; + INPUT_LEN = OUTPUT_LEN; + } + + let dw = new DataView(memory.buffer); + dw.setUint32(INPUT_LEN.value, code.length, true); + new Uint8Array(memory.buffer, INPUT.value).set(new TextEncoder().encode(code)); + + return runWasmFunction(fmtInstance, action === "fmt" ? fmt : minify) ? + bufToString(memory, OUTPUT, OUTPUT_LEN) : "invalid code"; +} + + +/** @param {WebAssembly.Instance} instance @param {CallableFunction} func @param {any[]} args + * @returns {boolean} */ +function runWasmFunction(instance, func, ...args) { + const prev = performance.now(); + const { PANIC_MESSAGE, PANIC_MESSAGE_LEN, memory, stack_pointer } = instance.exports; + if (!(true + && memory instanceof WebAssembly.Memory + && stack_pointer instanceof WebAssembly.Global + )) never(); + const ptr = stack_pointer.value; try { - compile_and_run(fuel); - return bufToString(memory, LOG_MESSAGES, LOG_MESSAGES_LEN); - } catch (e) { - if (PANIC_MESSAGE instanceof WebAssembly.Global - && PANIC_MESSAGE_LEN instanceof WebAssembly.Global) { - console.error(e, bufToString(memory, PANIC_MESSAGE, PANIC_MESSAGE_LEN)); + func(...args); + return true; + } catch (error) { + if (error instanceof WebAssembly.RuntimeError && error.message == "unreachable") { + if (PANIC_MESSAGE instanceof WebAssembly.Global + && PANIC_MESSAGE_LEN instanceof WebAssembly.Global) { + console.error(bufToString(memory, PANIC_MESSAGE, PANIC_MESSAGE_LEN), error); + } + } else { + console.error(error); } - return bufToString(memory, LOG_MESSAGES, LOG_MESSAGES_LEN); + stack_pointer.value = ptr; + return false; + } finally { + console.log("compiletion took:", performance.now() - prev); } } @@ -78,61 +136,6 @@ function bufToString(mem, ptr, len) { return res; } -/**@type{WebAssembly.Instance}*/ let fmtInstance; -/**@type{Promise}*/ let fmtInstaceFuture; -/** @param {string} code @param {"fmt" | "minify"} action - * @returns {Promise} */ -async function modifyCode(code, action) { - fmtInstaceFuture ??= WebAssembly.instantiateStreaming(fetch("/hbfmt.wasm"), {}); - fmtInstance ??= (await fmtInstaceFuture).instance; - - let { - INPUT, INPUT_LEN, - OUTPUT, OUTPUT_LEN, - PANIC_MESSAGE, PANIC_MESSAGE_LEN, - memory, fmt, minify - } = fmtInstance.exports; - - if (!(true - && INPUT instanceof WebAssembly.Global - && INPUT_LEN instanceof WebAssembly.Global - && OUTPUT instanceof WebAssembly.Global - && OUTPUT_LEN instanceof WebAssembly.Global - && memory instanceof WebAssembly.Memory - && typeof fmt === "function" - && typeof minify === "function" - )) never(); - - if (action !== "fmt") { - INPUT = OUTPUT; - INPUT_LEN = OUTPUT_LEN; - } - - let dw = new DataView(memory.buffer); - dw.setUint32(INPUT_LEN.value, code.length, true); - new Uint8Array(memory.buffer, INPUT.value) - .set(new TextEncoder().encode(code)); - - try { - if (action === "fmt") fmt(); else minify(); - let result = new TextDecoder() - .decode(new Uint8Array(memory.buffer, OUTPUT.value, - dw.getUint32(OUTPUT_LEN.value, true))); - return result; - } catch (e) { - if (PANIC_MESSAGE instanceof WebAssembly.Global - && PANIC_MESSAGE_LEN instanceof WebAssembly.Global) { - let message = new TextDecoder() - .decode(new Uint8Array(memory.buffer, PANIC_MESSAGE.value, - dw.getUint32(PANIC_MESSAGE_LEN.value, true))); - console.error(message, e); - } else { - console.error(e); - } - return undefined; - } -} - /** @param {HTMLElement} target */ function wireUp(target) { execApply(target); @@ -144,7 +147,7 @@ function wireUp(target) { /** @type {{ [key: string]: (content: string) => Promise | string }} */ const applyFns = { timestamp: (content) => new Date(parseInt(content) * 1000).toLocaleString(), - fmt: (content) => modifyCode(content, "fmt").then(c => c ?? "post has invalid code"), + fmt: (content) => modifyCode(content, "fmt").then(c => c), }; /** @param {HTMLElement} target */ @@ -156,16 +159,20 @@ async function bindCodeEdit(target) { const hbc = await getHbcInstance(); - const debounce = 0; + const debounce = 100; let timeout = 0; edit.addEventListener("input", () => { if (timeout) clearTimeout(timeout); timeout = setTimeout(() => { - const buf = packPosts([{ path: "local.hb", code: edit.value }]); + const buf = packPosts([ + { path: "local.hb", code: edit.value }, + { path: "lam.hb", code: "foo:=10" }, + ]); errors.textContent = compileCode(hbc, buf, 1); timeout = 0; }, debounce); }); + edit.dispatchEvent(new InputEvent("input")); } /** @param {HTMLElement} target */ @@ -191,8 +198,10 @@ function bindTextareaAutoResize(target) { textarea.style.height = (textarea.scrollHeight - padding) + "px"; textarea.style.overflowY = "hidden"; textarea.addEventListener("input", function() { + let top = window.scrollY; textarea.style.height = "auto"; textarea.style.height = (textarea.scrollHeight - padding) + "px"; + window.scrollTo({ top }); }); textarea.onkeydown = (ev) => { diff --git a/depell/src/main.rs b/depell/src/main.rs index 3d08559..2ef8997 100644 --- a/depell/src/main.rs +++ b/depell/src/main.rs @@ -264,7 +264,7 @@ impl PublicPage for Login { if let Some(e) = error {
e
} - @@ -402,7 +402,7 @@ async fn base(body: impl FnOnce(&mut String), session: Option<&Session>) -> Html
|f|{body(f)}
- + }) } diff --git a/depell/wasm-hbc/src/lib.rs b/depell/wasm-hbc/src/lib.rs index 98efa5a..c42d4b7 100644 --- a/depell/wasm-hbc/src/lib.rs +++ b/depell/wasm-hbc/src/lib.rs @@ -9,7 +9,8 @@ use { extern crate alloc; -wasm_rt::decl_runtime!(128 * 8 * 1024, 1024 * 4); +const ARENA_CAP: usize = 128 * 16 * 1024; +wasm_rt::decl_runtime!(ARENA_CAP, 1024 * 4); const MAX_INPUT_SIZE: usize = 32 * 4 * 1024; wasm_rt::decl_buffer!(MAX_INPUT_SIZE, MAX_INPUT, INPUT, INPUT_LEN); @@ -26,6 +27,8 @@ unsafe fn compile_and_run(mut fuel: usize) { code: &'a mut str, } + let mut root = 0; + let files = { let mut input_bytes = core::slice::from_raw_parts_mut(core::ptr::addr_of_mut!(INPUT).cast::(), INPUT_LEN); @@ -42,7 +45,10 @@ unsafe fn compile_and_run(mut fuel: usize) { input_bytes = rest; } + let root_path = files[root].path; hblang::quad_sort(&mut files, |a, b| a.path.cmp(b.path)); + root = files.binary_search_by_key(&root_path, |p| p.path).unwrap(); + files }; @@ -67,7 +73,7 @@ unsafe fn compile_and_run(mut fuel: usize) { let mut ct = { let mut c = Codegen::default(); c.files = files; - c.generate(); + c.generate(root as FileId); c.assemble_comptime() }; @@ -94,4 +100,6 @@ unsafe fn compile_and_run(mut fuel: usize) { } } } + + log::error!("memory consumption: {}b / {}b", ALLOCATOR.used(), ARENA_CAP); } diff --git a/depell/wasm-rt/src/lib.rs b/depell/wasm-rt/src/lib.rs index c4afaa6..3337f20 100644 --- a/depell/wasm-rt/src/lib.rs +++ b/depell/wasm-rt/src/lib.rs @@ -116,6 +116,10 @@ impl ArenaAllocator { pub unsafe fn reset(&self) { (*self.head.get()) = self.arena.get().cast::().add(SIZE); } + + pub fn used(&self) -> usize { + unsafe { self.arena.get() as usize + SIZE - (*self.head.get()) as usize } + } } unsafe impl Sync for ArenaAllocator {} diff --git a/lang/src/codegen.rs b/lang/src/codegen.rs index b51419e..e834976 100644 --- a/lang/src/codegen.rs +++ b/lang/src/codegen.rs @@ -253,8 +253,8 @@ mod reg { ) } - pub fn free(&mut self, reg: Id) { - if reg.1.is_some() { + pub fn free(&mut self, mut reg: Id) { + if reg.1.take().is_some() { self.free.push(reg.0); core::mem::forget(reg); } @@ -730,9 +730,9 @@ pub struct Codegen { } impl Codegen { - pub fn generate(&mut self) { + pub fn generate(&mut self, root: FileId) { self.ci.emit_entry_prelude(); - self.find_or_declare(0, 0, Err("main"), ""); + self.find_or_declare(0, root, Err("main"), ""); self.make_func_reachable(0); self.complete_call_graph(); } @@ -798,13 +798,7 @@ impl Codegen { } else { let values = captured .iter() - .map(|&id| E::Ident { - pos: 0, - is_ct: false, - id, - name: "booodab", - is_first: false, - }) + .map(|&id| E::Ident { pos: 0, is_ct: false, id, is_first: false }) .map(|expr| self.expr(&expr)) .collect::>>()?; let values_size = @@ -1236,7 +1230,7 @@ impl Codegen { self.ci.revert(checkpoint); match self.ty(target).expand() { ty::Kind::Module(idx) => { - match self.find_or_declare(target.pos(), idx, Err(field), "") { + match self.find_or_declare(pos, idx, Err(field), "") { ty::Kind::Global(idx) => self.handle_global(idx), e => Some(Value::ty(e.compress())), } @@ -1418,8 +1412,14 @@ impl Codegen { let loc = var.value.loc.as_ref(); Some(Value { ty: self.ci.vars[var_index].value.ty, loc }) } - E::Ident { id, name, .. } => { - match self.find_or_declare(ident::pos(id), self.ci.file, Ok(id), name) { + E::Ident { id, .. } => { + let cfile = self.cfile().clone(); + match self.find_or_declare( + ident::pos(id), + self.ci.file, + Ok(id), + cfile.ident_str(id), + ) { ty::Kind::Global(id) => self.handle_global(id), tk => Some(Value::ty(tk.compress())), } @@ -2465,7 +2465,7 @@ impl Codegen { let f = self.files[file as usize].clone(); let Some((expr, ident)) = f.find_decl(name) else { match name { - Ok(_) => self.report(pos, format_args!("undefined indentifier: {lit_name}")), + Ok(_) => self.report(pos, format_args!("undefined identifier: {lit_name}")), Err("main") => self.report(pos, format_args!("missing main function")), Err(name) => self.report(pos, format_args!("undefined indentifier: {name}")), } @@ -2774,7 +2774,7 @@ mod tests { let mut codegen = super::Codegen { files: crate::test_parse_files(ident, input), ..Default::default() }; - codegen.generate(); + codegen.generate(0); let mut out = Vec::new(); codegen.assemble(&mut out); diff --git a/lang/src/fmt.rs b/lang/src/fmt.rs index 8265c1c..dcbb760 100644 --- a/lang/src/fmt.rs +++ b/lang/src/fmt.rs @@ -1,9 +1,10 @@ use { crate::{ - lexer::{self, TokenKind}, + ident, + lexer::{self, Lexer, TokenKind}, parser::{self, CommentOr, CtorField, Expr, Poser, Radix, StructField}, }, - core::fmt, + core::{fmt, usize}, }; pub fn display_radix(radix: Radix, mut value: u64, buf: &mut [u8; 64]) -> &str { @@ -254,7 +255,7 @@ impl<'a> Formatter<'a> { fields, |s: &mut Self, CtorField { name, value, .. }: &_, f| { f.write_str(name)?; - if !matches!(value, Expr::Ident { name: n, .. } if name == n) { + if !matches!(value, &Expr::Ident { id, .. } if *name == &self.source[ident::range(id)]) { f.write_str(": ")?; s.fmt(value, f)?; } @@ -331,11 +332,12 @@ impl<'a> Formatter<'a> { self.fmt(val, f) } Expr::Return { val: None, .. } => f.write_str("return"), - Expr::Ident { name, is_ct: true, .. } => { - f.write_str("$")?; - f.write_str(name) + Expr::Ident { pos, is_ct, .. } => { + if is_ct { + f.write_str("$")?; + } + f.write_str(&self.source[Lexer::restore(self.source, pos).eat().range()]) } - Expr::Ident { name, is_ct: false, .. } => f.write_str(name), Expr::Block { stmts, .. } => { f.write_str("{")?; self.fmt_list(f, true, "}", "", stmts, Self::fmt) diff --git a/lang/src/fs.rs b/lang/src/fs.rs index 3384b69..ca19088 100644 --- a/lang/src/fs.rs +++ b/lang/src/fs.rs @@ -89,7 +89,7 @@ pub fn run_compiler(root_file: &str, options: Options, out: &mut Vec) -> std let mut codegen = codegen::Codegen::default(); codegen.files = parsed; - codegen.generate(); + codegen.generate(0); if options.dump_asm { codegen .disasm(unsafe { std::mem::transmute::<&mut Vec, &mut String>(out) }) diff --git a/lang/src/lib.rs b/lang/src/lib.rs index bfd0fea..8ca7853 100644 --- a/lang/src/lib.rs +++ b/lang/src/lib.rs @@ -231,9 +231,8 @@ mod ident { (ident >> LEN_BITS).saturating_sub(1) } - pub fn new(pos: u32, len: u32) -> u32 { - debug_assert!(len < (1 << LEN_BITS)); - ((pos + 1) << LEN_BITS) | len + pub fn new(pos: u32, len: u32) -> Option { + (len < (1 << LEN_BITS)).then_some(((pos + 1) << LEN_BITS) | len) } pub fn range(ident: u32) -> core::ops::Range { @@ -777,7 +776,7 @@ impl IdentInterner { match entry { hash_map::RawEntryMut::Occupied(o) => o.get_key_value().0.value, hash_map::RawEntryMut::Vacant(v) => { - let id = ident::new(self.strings.len() as _, ident.len() as _); + let id = ident::new(self.strings.len() as _, ident.len() as _).unwrap(); self.strings.push_str(ident); v.insert(ctx_map::Key { hash, value: id }, ()); id @@ -1032,12 +1031,12 @@ impl Types { .map(|f| { let name = if f.file != u32::MAX { let file = &files[f.file as usize]; - let Expr::BinOp { left: &Expr::Ident { name, .. }, .. } = + let Expr::BinOp { left: &Expr::Ident { id, .. }, .. } = f.expr.get(file).unwrap() else { unreachable!() }; - name + file.ident_str(id) } else { "target_fn" }; diff --git a/lang/src/parser.rs b/lang/src/parser.rs index df81b0c..1d224ae 100644 --- a/lang/src/parser.rs +++ b/lang/src/parser.rs @@ -232,7 +232,9 @@ impl<'a, 'b> Parser<'a, 'b> { { Some((i, elem)) => (i, elem, false), None => { - let id = ident::new(token.start, name.len() as _); + let Some(id) = ident::new(token.start, name.len() as _) else { + self.report(token.start, "identifier can at most have 64 characters"); + }; self.ctx.idents.push(ScopeIdent { ident: id, declared: false, @@ -345,8 +347,7 @@ impl<'a, 'b> Parser<'a, 'b> { }, T::Ident | T::CtIdent => { let (id, is_first) = self.resolve_ident(token); - let name = self.tok_str(token); - E::Ident { pos, is_ct: token.kind == T::CtIdent, name, id, is_first } + E::Ident { pos, is_ct: token.kind == T::CtIdent, id, is_first } } T::If => E::If { pos, @@ -501,7 +502,7 @@ impl<'a, 'b> Parser<'a, 'b> { s.expr() } else { let (id, is_first) = s.resolve_ident(name_tok); - Expr::Ident { pos: name_tok.start, is_ct: false, id, name, is_first } + Expr::Ident { pos: name_tok.start, is_ct: false, id, is_first } }, } }), @@ -716,7 +717,7 @@ generate_expr! { is_ct: bool, is_first: bool, id: Ident, - name: &'a str, + //name: &'a str, }, /// `LIST('{', [';'], '}', Expr)` Block { @@ -819,10 +820,12 @@ generate_expr! { } impl Expr<'_> { - pub fn declares(&self, iden: Result) -> Option { + pub fn declares(&self, iden: Result, source: &str) -> Option { match *self { - Self::Ident { id, name, .. } if iden == Ok(id) || iden == Err(name) => Some(id), - Self::Ctor { fields, .. } => fields.iter().find_map(|f| f.value.declares(iden)), + Self::Ident { id, .. } if iden == Ok(id) || iden == Err(&source[ident::range(id)]) => { + Some(id) + } + Self::Ctor { fields, .. } => fields.iter().find_map(|f| f.value.declares(iden, source)), _ => None, } } @@ -972,7 +975,9 @@ impl AstInner<[Symbol]> { } fn new(file: Box, path: &str, ctx: &mut ParserCtx, loader: Loader) -> NonNull { - let arena = Arena::with_capacity(file.len() * SOURCE_TO_AST_FACTOR); + let arena = Arena::with_capacity( + SOURCE_TO_AST_FACTOR * file.bytes().filter(|b| !b.is_ascii_whitespace()).count(), + ); let exprs = unsafe { core::mem::transmute(Parser::parse(ctx, &file, path, loader, &arena)) }; @@ -1059,7 +1064,9 @@ impl Ast { pub fn find_decl(&self, id: Result) -> Option<(&Expr, Ident)> { self.exprs().iter().find_map(|expr| match expr { - Expr::BinOp { left, op: TokenKind::Decl, .. } => left.declares(id).map(|id| (expr, id)), + Expr::BinOp { left, op: TokenKind::Decl, .. } => { + left.declares(id, &self.file).map(|id| (expr, id)) + } _ => None, }) } @@ -1224,10 +1231,7 @@ impl Arena { .unwrap(); let ptr = self.alloc_low(layout); unsafe { - ptr.cast::().copy_from_nonoverlapping( - NonNull::from(&expr).cast(), - layout.size() / core::mem::size_of::(), - ) + ptr.cast::().copy_from_nonoverlapping(NonNull::from(&expr).cast(), layout.size()) }; unsafe { ptr.cast::>().as_ref() } } @@ -1250,14 +1254,35 @@ impl Arena { return ptr; } - unsafe { - core::ptr::write( - chunk, - ArenaChunk::new( - 1024 * 4 - core::mem::size_of::(), - core::ptr::read(chunk), - ), - ); + const EXPANSION_ALLOC: usize = 1024 * 4 - core::mem::size_of::(); + + if layout.size() > EXPANSION_ALLOC { + let next_ptr = chunk.next_ptr(); + if next_ptr.is_null() { + unsafe { + core::ptr::write( + chunk, + ArenaChunk::new( + layout.size() + layout.align() - 1 + core::mem::size_of::(), + Default::default(), + ), + ); + } + } else { + unsafe { + let chunk = ArenaChunk::new( + layout.size() + layout.align() - 1 + core::mem::size_of::(), + core::ptr::read(next_ptr), + ); + let alloc = chunk.base.add(chunk.base.align_offset(layout.align())); + core::ptr::write(next_ptr, chunk); + return NonNull::new_unchecked(alloc); + } + } + } else { + unsafe { + core::ptr::write(chunk, ArenaChunk::new(EXPANSION_ALLOC, core::ptr::read(chunk))); + } } chunk.alloc(layout).unwrap() @@ -1300,11 +1325,16 @@ impl ArenaChunk { return None; } unsafe { self.end = self.end.sub(size) }; + debug_assert!(self.end >= self.base, "{:?} {:?}", self.end, self.base); unsafe { Some(NonNull::new_unchecked(self.end)) } } fn next(&self) -> Option<&Self> { - unsafe { self.base.cast::().sub(1).as_ref() } + unsafe { self.next_ptr().as_ref() } + } + + fn next_ptr(&self) -> *mut Self { + unsafe { self.base.cast::().sub(1) } } fn contains(&self, arg: *mut u8) -> bool { diff --git a/lang/src/son.rs b/lang/src/son.rs index 60b8c0d..7eb0bbb 100644 --- a/lang/src/son.rs +++ b/lang/src/son.rs @@ -1243,9 +1243,10 @@ impl Codegen { [VOID], )) } - Expr::Call { func: &Expr::Ident { pos, id, name, .. }, args, .. } => { + Expr::Call { func: &Expr::Ident { pos, id, .. }, args, .. } => { self.ci.call_count += 1; - let func = self.find_or_declare(pos, self.ci.file, Some(id), name); + let cfile = self.cfile().clone(); + let func = self.find_or_declare(pos, self.ci.file, Some(id), cfile.ident_str(id)); let ty::Kind::Func(func) = func else { self.report( pos, diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 1444750..095818f 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -3,3 +3,7 @@ name = "xtask" version = "0.1.0" edition = "2021" +[dependencies] +anyhow = "1.0.89" +walrus = "0.22.0" + diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 7c683d7..afc8a71 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -2,11 +2,13 @@ mod utils; use { crate::utils::IterExt, + anyhow::Context, std::{ fs::File, io::{self, BufRead, BufReader, BufWriter, Seek, Write}, path::Path, }, + walrus::{ir::Value, ConstExpr, GlobalKind, ValType}, }; fn root() -> &'static Path { @@ -29,7 +31,25 @@ fn exec(mut cmd: std::process::Command) -> io::Result<()> { Ok(()) } -fn build_wasm_blob(name: &str, debug: bool) -> io::Result<()> { +fn insert_stack_pointer_shim(file: impl AsRef) -> anyhow::Result<()> { + let mut module = walrus::Module::from_file(file.as_ref())?; + + let global = module + .globals + .iter() + .find(|g| g.ty == ValType::I32 && g.mutable) + .filter(|g| match g.kind { + GlobalKind::Local(ConstExpr::Value(Value::I32(n))) => n != 0, + _ => false, + }) + .context("binary is missing a stak pointer")?; + + module.exports.add("stack_pointer", global.id()); + + module.emit_wasm_file(file.as_ref()) +} + +fn build_wasm_blob(name: &str, debug: bool) -> anyhow::Result<()> { let mut c = build_cmd(if debug { "cargo wasm-build-debug" } else { "cargo wasm-build" }); c.arg(format!("wasm-{name}")); exec(c)?; @@ -39,14 +59,15 @@ fn build_wasm_blob(name: &str, debug: bool) -> io::Result<()> { exec(build_cmd(format!("wasm-opt -Oz {out_path} -o {out_path}")))?; } exec(build_cmd(format!("cp {out_path} depell/src/{name}.wasm")))?; + insert_stack_pointer_shim(format!("depell/src/{name}.wasm"))?; exec(build_cmd(format!("gzip -f depell/src/{name}.wasm")))?; Ok(()) } -fn main() -> io::Result<()> { +fn main() -> anyhow::Result<()> { let args = std::env::args().skip(1).collect::>(); match args[0].as_str() { - "fmt" => fmt(args[1] == "-r" || args[1] == "--renumber"), + "fmt" => fmt(args[1] == "-r" || args[1] == "--renumber").context(""), "build-depell-debug" => { build_wasm_blob("hbfmt", true)?; build_wasm_blob("hbc", true)?; @@ -61,12 +82,18 @@ fn main() -> io::Result<()> { exec(build_cmd("gzip -k -f depell/src/index.css"))?; Ok(()) } - "watch-depell" => { + "watch-depell-debug" => { let mut c = build_cmd("cargo watch --why"); c.arg("--exec=xtask build-depell-debug").arg("--exec=run -p depell"); exec(c)?; Ok(()) } + "watch-depell" => { + let mut c = build_cmd("cargo watch --why"); + c.arg("--exec=xtask build-depell").arg("--exec=run -p depell"); + exec(c)?; + Ok(()) + } _ => Ok(()), } }