AbleOS/holey-bytes
3
3
Fork 5
Code Issues 2 Pull requests Packages Projects Releases Wiki Activity

fixing bugs and improving memory consumption

Browse source
This commit is contained in:
Jakub Doka 2024-10-13 12:25:12 +02:00
parent 6d7e726066
commit 0f4ff918d2
No known key found for this signature in database
GPG key ID: C6E9A89936B8C143
15 changed files with 368 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

152
Cargo.lock generated
View file

@ -8,7 +8,7 @@ version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1"
dependencies = [ dependencies = [
"gimli", "gimli 0.31.1",
] ]
[[package]] [[package]]
@ -35,6 +35,12 @@ version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
[[package]]
name = "anyhow"
version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
[[package]] [[package]]
name = "async-trait" name = "async-trait"
version = "0.1.83" version = "0.1.83"
@ -46,6 +52,12 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "autocfg"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]] [[package]]
name = "axum" name = "axum"
version = "0.7.7" version = "0.7.7"
@ -175,6 +187,18 @@ dependencies = [
"powerfmt", "powerfmt",
] ]
[[package]]
name = "equivalent"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]] [[package]]
name = "fallible-iterator" name = "fallible-iterator"
version = "0.3.0" version = "0.3.0"
@ -246,12 +270,29 @@ dependencies = [
"wasi", "wasi",
] ]
[[package]]
name = "gimli"
version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
dependencies = [
"fallible-iterator 0.2.0",
"indexmap 1.9.3",
"stable_deref_trait",
]
[[package]] [[package]]
name = "gimli" name = "gimli"
version = "0.31.1" version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.14.5" version = "0.14.5"
@ -259,6 +300,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
dependencies = [ dependencies = [
"ahash", "ahash",
"serde",
] ]
[[package]] [[package]]
@ -306,6 +348,12 @@ dependencies = [
"memmap2", "memmap2",
] ]
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]] [[package]]
name = "hermit-abi" name = "hermit-abi"
version = "0.3.9" version = "0.3.9"
@ -408,12 +456,45 @@ dependencies = [
"tower-service", "tower-service",
] ]
[[package]]
name = "id-arena"
version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005"
[[package]]
name = "indexmap"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
]
[[package]]
name = "indexmap"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da"
dependencies = [
"equivalent",
"hashbrown 0.15.0",
"serde",
]
[[package]] [[package]]
name = "itoa" name = "itoa"
version = "1.0.11" version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
[[package]]
name = "leb128"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.159" version = "0.2.159"
@ -574,7 +655,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"fallible-iterator", "fallible-iterator 0.3.0",
"fallible-streaming-iterator", "fallible-streaming-iterator",
"hashlink", "hashlink",
"libsqlite3-sys", "libsqlite3-sys",
@ -605,6 +686,12 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "semver"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.210" version = "1.0.210"
@ -681,6 +768,12 @@ dependencies = [
"windows-sys", "windows-sys",
] ]
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.79" version = "2.0.79"
@ -815,12 +908,49 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "walrus"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d68aa3c7b80be75c8458fc087453e5a31a226cfffede2e9b932393b2ea1c624a"
dependencies = [
"anyhow",
"gimli 0.26.2",
"id-arena",
"leb128",
"log",
"walrus-macro",
"wasm-encoder",
"wasmparser",
]
[[package]]
name = "walrus-macro"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ad39ff894c43c9649fa724cdde9a6fc50b855d517ef071a93e5df82fe51d3"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "wasi" name = "wasi"
version = "0.11.0+wasi-snapshot-preview1" version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-encoder"
version = "0.212.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "501940df4418b8929eb6d52f1aade1fdd15a5b86c92453cb696e3c906bd3fc33"
dependencies = [
"leb128",
]
[[package]] [[package]]
name = "wasm-hbc" name = "wasm-hbc"
version = "0.1.0" version = "0.1.0"
@ -846,6 +976,20 @@ dependencies = [
"log", "log",
] ]
[[package]]
name = "wasmparser"
version = "0.212.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d28bc49ba1e5c5b61ffa7a2eace10820443c4b7d1c0b144109261d14570fdf8"
dependencies = [
"ahash",
"bitflags",
"hashbrown 0.14.5",
"indexmap 2.6.0",
"semver",
"serde",
]
[[package]] [[package]]
name = "windows-sys" name = "windows-sys"
version = "0.52.0" version = "0.52.0"
@ -922,6 +1066,10 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]] [[package]]
name = "xtask" name = "xtask"
version = "0.1.0" version = "0.1.0"
dependencies = [
"anyhow",
"walrus",
]
[[package]] [[package]]
name = "zerocopy" name = "zerocopy"

4
Cargo.toml
View file

@ -23,8 +23,8 @@ hbjit = { path = "jit" }
[profile.release] [profile.release]
lto = true lto = true
#debug = true debug = true
strip = true #strip = true
codegen-units = 1 codegen-units = 1
panic = "abort" panic = "abort"

2
depell/README.md
View file

@ -5,6 +5,6 @@ Depell is a website that allows users to import/post/run hblang code and create
## Local Development ## Local Development
```bash ```bash
cargo xtask watch-depell cargo xtask watch-depell-debug
# browser http://localhost:8080 # browser http://localhost:8080
``` ```

141
depell/src/index.js
View file

@ -18,32 +18,90 @@ function compileCode(instance, code, fuel) {
let { let {
INPUT, INPUT_LEN, INPUT, INPUT_LEN,
LOG_MESSAGES, LOG_MESSAGES_LEN, LOG_MESSAGES, LOG_MESSAGES_LEN,
PANIC_MESSAGE, PANIC_MESSAGE_LEN,
memory, compile_and_run, memory, compile_and_run,
} = instance.exports; } = instance.exports;
if (!(true if (!(true
&& memory instanceof WebAssembly.Memory
&& INPUT instanceof WebAssembly.Global && INPUT instanceof WebAssembly.Global
&& INPUT_LEN instanceof WebAssembly.Global && INPUT_LEN instanceof WebAssembly.Global
&& LOG_MESSAGES instanceof WebAssembly.Global && LOG_MESSAGES instanceof WebAssembly.Global
&& LOG_MESSAGES_LEN instanceof WebAssembly.Global && LOG_MESSAGES_LEN instanceof WebAssembly.Global
&& memory instanceof WebAssembly.Memory
&& typeof compile_and_run === "function" && typeof compile_and_run === "function"
)) console.log(instance.exports), never(); )) never();
const dw = new DataView(memory.buffer); const dw = new DataView(memory.buffer);
dw.setUint32(INPUT_LEN.value, code.length, true); dw.setUint32(INPUT_LEN.value, code.length, true);
new Uint8Array(memory.buffer, INPUT.value).set(code); new Uint8Array(memory.buffer, INPUT.value).set(code);
try { runWasmFunction(instance, compile_and_run, fuel);
compile_and_run(fuel);
return bufToString(memory, LOG_MESSAGES, LOG_MESSAGES_LEN); return bufToString(memory, LOG_MESSAGES, LOG_MESSAGES_LEN);
} catch (e) { }
/**@type{WebAssembly.Instance}*/ let fmtInstance;
/**@type{Promise<WebAssembly.WebAssemblyInstantiatedSource>}*/ let fmtInstaceFuture;
/** @param {string} code @param {"fmt" | "minify"} action
* @returns {Promise<string>} */
async function modifyCode(code, action) {
fmtInstaceFuture ??= WebAssembly.instantiateStreaming(fetch("/hbfmt.wasm"), {});
fmtInstance ??= (await fmtInstaceFuture).instance;
let {
INPUT, INPUT_LEN,
OUTPUT, OUTPUT_LEN,
memory, fmt, minify
} = fmtInstance.exports;
if (!(true
&& memory instanceof WebAssembly.Memory
&& INPUT instanceof WebAssembly.Global
&& INPUT_LEN instanceof WebAssembly.Global
&& OUTPUT instanceof WebAssembly.Global
&& OUTPUT_LEN instanceof WebAssembly.Global
&& typeof fmt === "function"
&& typeof minify === "function"
)) never();
if (action !== "fmt") {
INPUT = OUTPUT;
INPUT_LEN = OUTPUT_LEN;
}
let dw = new DataView(memory.buffer);
dw.setUint32(INPUT_LEN.value, code.length, true);
new Uint8Array(memory.buffer, INPUT.value).set(new TextEncoder().encode(code));
return runWasmFunction(fmtInstance, action === "fmt" ? fmt : minify) ?
bufToString(memory, OUTPUT, OUTPUT_LEN) : "invalid code";
}
/** @param {WebAssembly.Instance} instance @param {CallableFunction} func @param {any[]} args
* @returns {boolean} */
function runWasmFunction(instance, func, ...args) {
const prev = performance.now();
const { PANIC_MESSAGE, PANIC_MESSAGE_LEN, memory, stack_pointer } = instance.exports;
if (!(true
&& memory instanceof WebAssembly.Memory
&& stack_pointer instanceof WebAssembly.Global
)) never();
const ptr = stack_pointer.value;
try {
func(...args);
return true;
} catch (error) {
if (error instanceof WebAssembly.RuntimeError && error.message == "unreachable") {
if (PANIC_MESSAGE instanceof WebAssembly.Global if (PANIC_MESSAGE instanceof WebAssembly.Global
&& PANIC_MESSAGE_LEN instanceof WebAssembly.Global) { && PANIC_MESSAGE_LEN instanceof WebAssembly.Global) {
console.error(e, bufToString(memory, PANIC_MESSAGE, PANIC_MESSAGE_LEN)); console.error(bufToString(memory, PANIC_MESSAGE, PANIC_MESSAGE_LEN), error);
} }
return bufToString(memory, LOG_MESSAGES, LOG_MESSAGES_LEN); } else {
console.error(error);
}
stack_pointer.value = ptr;
return false;
} finally {
console.log("compiletion took:", performance.now() - prev);
} }
} }
@ -78,61 +136,6 @@ function bufToString(mem, ptr, len) {
return res; return res;
} }
/**@type{WebAssembly.Instance}*/ let fmtInstance;
/**@type{Promise<WebAssembly.WebAssemblyInstantiatedSource>}*/ let fmtInstaceFuture;
/** @param {string} code @param {"fmt" | "minify"} action
* @returns {Promise<string | undefined>} */
async function modifyCode(code, action) {
fmtInstaceFuture ??= WebAssembly.instantiateStreaming(fetch("/hbfmt.wasm"), {});
fmtInstance ??= (await fmtInstaceFuture).instance;
let {
INPUT, INPUT_LEN,
OUTPUT, OUTPUT_LEN,
PANIC_MESSAGE, PANIC_MESSAGE_LEN,
memory, fmt, minify
} = fmtInstance.exports;
if (!(true
&& INPUT instanceof WebAssembly.Global
&& INPUT_LEN instanceof WebAssembly.Global
&& OUTPUT instanceof WebAssembly.Global
&& OUTPUT_LEN instanceof WebAssembly.Global
&& memory instanceof WebAssembly.Memory
&& typeof fmt === "function"
&& typeof minify === "function"
)) never();
if (action !== "fmt") {
INPUT = OUTPUT;
INPUT_LEN = OUTPUT_LEN;
}
let dw = new DataView(memory.buffer);
dw.setUint32(INPUT_LEN.value, code.length, true);
new Uint8Array(memory.buffer, INPUT.value)
.set(new TextEncoder().encode(code));
try {
if (action === "fmt") fmt(); else minify();
let result = new TextDecoder()
.decode(new Uint8Array(memory.buffer, OUTPUT.value,
dw.getUint32(OUTPUT_LEN.value, true)));
return result;
} catch (e) {
if (PANIC_MESSAGE instanceof WebAssembly.Global
&& PANIC_MESSAGE_LEN instanceof WebAssembly.Global) {
let message = new TextDecoder()
.decode(new Uint8Array(memory.buffer, PANIC_MESSAGE.value,
dw.getUint32(PANIC_MESSAGE_LEN.value, true)));
console.error(message, e);
} else {
console.error(e);
}
return undefined;
}
}
/** @param {HTMLElement} target */ /** @param {HTMLElement} target */
function wireUp(target) { function wireUp(target) {
execApply(target); execApply(target);
@ -144,7 +147,7 @@ function wireUp(target) {
/** @type {{ [key: string]: (content: string) => Promise<string> | string }} */ /** @type {{ [key: string]: (content: string) => Promise<string> | string }} */
const applyFns = { const applyFns = {
timestamp: (content) => new Date(parseInt(content) * 1000).toLocaleString(), timestamp: (content) => new Date(parseInt(content) * 1000).toLocaleString(),
fmt: (content) => modifyCode(content, "fmt").then(c => c ?? "post has invalid code"), fmt: (content) => modifyCode(content, "fmt").then(c => c),
}; };
/** @param {HTMLElement} target */ /** @param {HTMLElement} target */
@ -156,16 +159,20 @@ async function bindCodeEdit(target) {
const hbc = await getHbcInstance(); const hbc = await getHbcInstance();
const debounce = 0; const debounce = 100;
let timeout = 0; let timeout = 0;
edit.addEventListener("input", () => { edit.addEventListener("input", () => {
if (timeout) clearTimeout(timeout); if (timeout) clearTimeout(timeout);
timeout = setTimeout(() => { timeout = setTimeout(() => {
const buf = packPosts([{ path: "local.hb", code: edit.value }]); const buf = packPosts([
{ path: "local.hb", code: edit.value },
{ path: "lam.hb", code: "foo:=10" },
]);
errors.textContent = compileCode(hbc, buf, 1); errors.textContent = compileCode(hbc, buf, 1);
timeout = 0; timeout = 0;
}, debounce); }, debounce);
}); });
edit.dispatchEvent(new InputEvent("input"));
} }
/** @param {HTMLElement} target */ /** @param {HTMLElement} target */
@ -191,8 +198,10 @@ function bindTextareaAutoResize(target) {
textarea.style.height = (textarea.scrollHeight - padding) + "px"; textarea.style.height = (textarea.scrollHeight - padding) + "px";
textarea.style.overflowY = "hidden"; textarea.style.overflowY = "hidden";
textarea.addEventListener("input", function() { textarea.addEventListener("input", function() {
let top = window.scrollY;
textarea.style.height = "auto"; textarea.style.height = "auto";
textarea.style.height = (textarea.scrollHeight - padding) + "px"; textarea.style.height = (textarea.scrollHeight - padding) + "px";
window.scrollTo({ top });
}); });
textarea.onkeydown = (ev) => { textarea.onkeydown = (ev) => {

4
depell/src/main.rs
View file

@ -264,7 +264,7 @@ impl PublicPage for Login {
if let Some(e) = error { <div class="error">e</div> } if let Some(e) = error { <div class="error">e</div> }
<input name="name" type="text" autocomplete="name" placeholder="name" value=name <input name="name" type="text" autocomplete="name" placeholder="name" value=name
required maxlength=MAX_NAME_LENGTH> required maxlength=MAX_NAME_LENGTH>
<input name="password" type="password" autocomplete="password" placeholder="password" <input name="password" type="password" autocomplete="current-password" placeholder="password"
value=password> value=password>
<input type="submit" value="submit"> <input type="submit" value="submit">
</form> </form>
@ -402,7 +402,7 @@ async fn base(body: impl FnOnce(&mut String), session: Option<&Session>) -> Html
<main>|f|{body(f)}</main> <main>|f|{body(f)}</main>
</body> </body>
<script src="https://unpkg.com/htmx.org@2.0.3" integrity="sha384-0895/pl2MU10Hqc6jd4RvrthNlDiE9U1tWmX7WRESftEDRosgxNsQG/Ze9YMRzHq" crossorigin="anonymous"></script> <script src="https://unpkg.com/htmx.org@2.0.3" integrity="sha384-0895/pl2MU10Hqc6jd4RvrthNlDiE9U1tWmX7WRESftEDRosgxNsQG/Ze9YMRzHq" crossorigin="anonymous"></script>
<script src="/index.js"></script> <script type="module" src="/index.js"></script>
</html> </html>
}) })
} }

12
depell/wasm-hbc/src/lib.rs
View file

@ -9,7 +9,8 @@ use {
extern crate alloc; extern crate alloc;
wasm_rt::decl_runtime!(128 * 8 * 1024, 1024 * 4); const ARENA_CAP: usize = 128 * 16 * 1024;
wasm_rt::decl_runtime!(ARENA_CAP, 1024 * 4);
const MAX_INPUT_SIZE: usize = 32 * 4 * 1024; const MAX_INPUT_SIZE: usize = 32 * 4 * 1024;
wasm_rt::decl_buffer!(MAX_INPUT_SIZE, MAX_INPUT, INPUT, INPUT_LEN); wasm_rt::decl_buffer!(MAX_INPUT_SIZE, MAX_INPUT, INPUT, INPUT_LEN);
@ -26,6 +27,8 @@ unsafe fn compile_and_run(mut fuel: usize) {
code: &'a mut str, code: &'a mut str,
} }
let mut root = 0;
let files = { let files = {
let mut input_bytes = let mut input_bytes =
core::slice::from_raw_parts_mut(core::ptr::addr_of_mut!(INPUT).cast::<u8>(), INPUT_LEN); core::slice::from_raw_parts_mut(core::ptr::addr_of_mut!(INPUT).cast::<u8>(), INPUT_LEN);
@ -42,7 +45,10 @@ unsafe fn compile_and_run(mut fuel: usize) {
input_bytes = rest; input_bytes = rest;
} }
let root_path = files[root].path;
hblang::quad_sort(&mut files, |a, b| a.path.cmp(b.path)); hblang::quad_sort(&mut files, |a, b| a.path.cmp(b.path));
root = files.binary_search_by_key(&root_path, |p| p.path).unwrap();
files files
}; };
@ -67,7 +73,7 @@ unsafe fn compile_and_run(mut fuel: usize) {
let mut ct = { let mut ct = {
let mut c = Codegen::default(); let mut c = Codegen::default();
c.files = files; c.files = files;
c.generate(); c.generate(root as FileId);
c.assemble_comptime() c.assemble_comptime()
}; };
@ -94,4 +100,6 @@ unsafe fn compile_and_run(mut fuel: usize) {
} }
} }
} }
log::error!("memory consumption: {}b / {}b", ALLOCATOR.used(), ARENA_CAP);
} }

4
depell/wasm-rt/src/lib.rs
View file

@ -116,6 +116,10 @@ impl<const SIZE: usize> ArenaAllocator<SIZE> {
pub unsafe fn reset(&self) { pub unsafe fn reset(&self) {
(*self.head.get()) = self.arena.get().cast::<u8>().add(SIZE); (*self.head.get()) = self.arena.get().cast::<u8>().add(SIZE);
} }
pub fn used(&self) -> usize {
unsafe { self.arena.get() as usize + SIZE - (*self.head.get()) as usize }
}
} }
unsafe impl<const SIZE: usize> Sync for ArenaAllocator<SIZE> {} unsafe impl<const SIZE: usize> Sync for ArenaAllocator<SIZE> {}

32
lang/src/codegen.rs
View file

@ -253,8 +253,8 @@ mod reg {
) )
} }
pub fn free(&mut self, reg: Id) { pub fn free(&mut self, mut reg: Id) {
if reg.1.is_some() { if reg.1.take().is_some() {
self.free.push(reg.0); self.free.push(reg.0);
core::mem::forget(reg); core::mem::forget(reg);
} }
@ -730,9 +730,9 @@ pub struct Codegen {
} }
impl Codegen { impl Codegen {
pub fn generate(&mut self) { pub fn generate(&mut self, root: FileId) {
self.ci.emit_entry_prelude(); self.ci.emit_entry_prelude();
self.find_or_declare(0, 0, Err("main"), ""); self.find_or_declare(0, root, Err("main"), "");
self.make_func_reachable(0); self.make_func_reachable(0);
self.complete_call_graph(); self.complete_call_graph();
} }
@ -798,13 +798,7 @@ impl Codegen {
} else { } else {
let values = captured let values = captured
.iter() .iter()
.map(|&id| E::Ident { .map(|&id| E::Ident { pos: 0, is_ct: false, id, is_first: false })
pos: 0,
is_ct: false,
id,
name: "booodab",
is_first: false,
})
.map(|expr| self.expr(&expr)) .map(|expr| self.expr(&expr))
.collect::<Option<Vec<_>>>()?; .collect::<Option<Vec<_>>>()?;
let values_size = let values_size =
@ -1236,7 +1230,7 @@ impl Codegen {
self.ci.revert(checkpoint); self.ci.revert(checkpoint);
match self.ty(target).expand() { match self.ty(target).expand() {
ty::Kind::Module(idx) => { ty::Kind::Module(idx) => {
match self.find_or_declare(target.pos(), idx, Err(field), "") { match self.find_or_declare(pos, idx, Err(field), "") {
ty::Kind::Global(idx) => self.handle_global(idx), ty::Kind::Global(idx) => self.handle_global(idx),
e => Some(Value::ty(e.compress())), e => Some(Value::ty(e.compress())),
} }
@ -1418,8 +1412,14 @@ impl Codegen {
let loc = var.value.loc.as_ref(); let loc = var.value.loc.as_ref();
Some(Value { ty: self.ci.vars[var_index].value.ty, loc }) Some(Value { ty: self.ci.vars[var_index].value.ty, loc })
} }
E::Ident { id, name, .. } => { E::Ident { id, .. } => {
match self.find_or_declare(ident::pos(id), self.ci.file, Ok(id), name) { let cfile = self.cfile().clone();
match self.find_or_declare(
ident::pos(id),
self.ci.file,
Ok(id),
cfile.ident_str(id),
) {
ty::Kind::Global(id) => self.handle_global(id), ty::Kind::Global(id) => self.handle_global(id),
tk => Some(Value::ty(tk.compress())), tk => Some(Value::ty(tk.compress())),
} }
@ -2465,7 +2465,7 @@ impl Codegen {
let f = self.files[file as usize].clone(); let f = self.files[file as usize].clone();
let Some((expr, ident)) = f.find_decl(name) else { let Some((expr, ident)) = f.find_decl(name) else {
match name { match name {
Ok(_) => self.report(pos, format_args!("undefined indentifier: {lit_name}")), Ok(_) => self.report(pos, format_args!("undefined identifier: {lit_name}")),
Err("main") => self.report(pos, format_args!("missing main function")), Err("main") => self.report(pos, format_args!("missing main function")),
Err(name) => self.report(pos, format_args!("undefined indentifier: {name}")), Err(name) => self.report(pos, format_args!("undefined indentifier: {name}")),
} }
@ -2774,7 +2774,7 @@ mod tests {
let mut codegen = let mut codegen =
super::Codegen { files: crate::test_parse_files(ident, input), ..Default::default() }; super::Codegen { files: crate::test_parse_files(ident, input), ..Default::default() };
codegen.generate(); codegen.generate(0);
let mut out = Vec::new(); let mut out = Vec::new();
codegen.assemble(&mut out); codegen.assemble(&mut out);

14
lang/src/fmt.rs
View file

@ -1,9 +1,10 @@
use { use {
crate::{ crate::{
lexer::{self, TokenKind}, ident,
lexer::{self, Lexer, TokenKind},
parser::{self, CommentOr, CtorField, Expr, Poser, Radix, StructField}, parser::{self, CommentOr, CtorField, Expr, Poser, Radix, StructField},
}, },
core::fmt, core::{fmt, usize},
}; };
pub fn display_radix(radix: Radix, mut value: u64, buf: &mut [u8; 64]) -> &str { pub fn display_radix(radix: Radix, mut value: u64, buf: &mut [u8; 64]) -> &str {
@ -254,7 +255,7 @@ impl<'a> Formatter<'a> {
fields, fields,
|s: &mut Self, CtorField { name, value, .. }: &_, f| { |s: &mut Self, CtorField { name, value, .. }: &_, f| {
f.write_str(name)?; f.write_str(name)?;
if !matches!(value, Expr::Ident { name: n, .. } if name == n) { if !matches!(value, &Expr::Ident { id, .. } if *name == &self.source[ident::range(id)]) {
f.write_str(": ")?; f.write_str(": ")?;
s.fmt(value, f)?; s.fmt(value, f)?;
} }
@ -331,11 +332,12 @@ impl<'a> Formatter<'a> {
self.fmt(val, f) self.fmt(val, f)
} }
Expr::Return { val: None, .. } => f.write_str("return"), Expr::Return { val: None, .. } => f.write_str("return"),
Expr::Ident { name, is_ct: true, .. } => { Expr::Ident { pos, is_ct, .. } => {
if is_ct {
f.write_str("$")?; f.write_str("$")?;
f.write_str(name)
} }
Expr::Ident { name, is_ct: false, .. } => f.write_str(name), f.write_str(&self.source[Lexer::restore(self.source, pos).eat().range()])
}
Expr::Block { stmts, .. } => { Expr::Block { stmts, .. } => {
f.write_str("{")?; f.write_str("{")?;
self.fmt_list(f, true, "}", "", stmts, Self::fmt) self.fmt_list(f, true, "}", "", stmts, Self::fmt)

2
lang/src/fs.rs
View file

@ -89,7 +89,7 @@ pub fn run_compiler(root_file: &str, options: Options, out: &mut Vec<u8>) -> std
let mut codegen = codegen::Codegen::default(); let mut codegen = codegen::Codegen::default();
codegen.files = parsed; codegen.files = parsed;
codegen.generate(); codegen.generate(0);
if options.dump_asm { if options.dump_asm {
codegen codegen
.disasm(unsafe { std::mem::transmute::<&mut Vec<u8>, &mut String>(out) }) .disasm(unsafe { std::mem::transmute::<&mut Vec<u8>, &mut String>(out) })

11
lang/src/lib.rs
View file

@ -231,9 +231,8 @@ mod ident {
(ident >> LEN_BITS).saturating_sub(1) (ident >> LEN_BITS).saturating_sub(1)
} }
pub fn new(pos: u32, len: u32) -> u32 { pub fn new(pos: u32, len: u32) -> Option<u32> {
debug_assert!(len < (1 << LEN_BITS)); (len < (1 << LEN_BITS)).then_some(((pos + 1) << LEN_BITS) | len)
((pos + 1) << LEN_BITS) | len
} }
pub fn range(ident: u32) -> core::ops::Range<usize> { pub fn range(ident: u32) -> core::ops::Range<usize> {
@ -777,7 +776,7 @@ impl IdentInterner {
match entry { match entry {
hash_map::RawEntryMut::Occupied(o) => o.get_key_value().0.value, hash_map::RawEntryMut::Occupied(o) => o.get_key_value().0.value,
hash_map::RawEntryMut::Vacant(v) => { hash_map::RawEntryMut::Vacant(v) => {
let id = ident::new(self.strings.len() as _, ident.len() as _); let id = ident::new(self.strings.len() as _, ident.len() as _).unwrap();
self.strings.push_str(ident); self.strings.push_str(ident);
v.insert(ctx_map::Key { hash, value: id }, ()); v.insert(ctx_map::Key { hash, value: id }, ());
id id
@ -1032,12 +1031,12 @@ impl Types {
.map(|f| { .map(|f| {
let name = if f.file != u32::MAX { let name = if f.file != u32::MAX {
let file = &files[f.file as usize]; let file = &files[f.file as usize];
let Expr::BinOp { left: &Expr::Ident { name, .. }, .. } = let Expr::BinOp { left: &Expr::Ident { id, .. }, .. } =
f.expr.get(file).unwrap() f.expr.get(file).unwrap()
else { else {
unreachable!() unreachable!()
}; };
name file.ident_str(id)
} else { } else {
"target_fn" "target_fn"
}; };

64
lang/src/parser.rs
View file

@ -232,7 +232,9 @@ impl<'a, 'b> Parser<'a, 'b> {
{ {
Some((i, elem)) => (i, elem, false), Some((i, elem)) => (i, elem, false),
None => { None => {
let id = ident::new(token.start, name.len() as _); let Some(id) = ident::new(token.start, name.len() as _) else {
self.report(token.start, "identifier can at most have 64 characters");
};
self.ctx.idents.push(ScopeIdent { self.ctx.idents.push(ScopeIdent {
ident: id, ident: id,
declared: false, declared: false,
@ -345,8 +347,7 @@ impl<'a, 'b> Parser<'a, 'b> {
}, },
T::Ident | T::CtIdent => { T::Ident | T::CtIdent => {
let (id, is_first) = self.resolve_ident(token); let (id, is_first) = self.resolve_ident(token);
let name = self.tok_str(token); E::Ident { pos, is_ct: token.kind == T::CtIdent, id, is_first }
E::Ident { pos, is_ct: token.kind == T::CtIdent, name, id, is_first }
} }
T::If => E::If { T::If => E::If {
pos, pos,
@ -501,7 +502,7 @@ impl<'a, 'b> Parser<'a, 'b> {
s.expr() s.expr()
} else { } else {
let (id, is_first) = s.resolve_ident(name_tok); let (id, is_first) = s.resolve_ident(name_tok);
Expr::Ident { pos: name_tok.start, is_ct: false, id, name, is_first } Expr::Ident { pos: name_tok.start, is_ct: false, id, is_first }
}, },
} }
}), }),
@ -716,7 +717,7 @@ generate_expr! {
is_ct: bool, is_ct: bool,
is_first: bool, is_first: bool,
id: Ident, id: Ident,
name: &'a str, //name: &'a str,
}, },
/// `LIST('{', [';'], '}', Expr)` /// `LIST('{', [';'], '}', Expr)`
Block { Block {
@ -819,10 +820,12 @@ generate_expr! {
} }
impl Expr<'_> { impl Expr<'_> {
pub fn declares(&self, iden: Result<Ident, &str>) -> Option<Ident> { pub fn declares(&self, iden: Result<Ident, &str>, source: &str) -> Option<Ident> {
match *self { match *self {
Self::Ident { id, name, .. } if iden == Ok(id) || iden == Err(name) => Some(id), Self::Ident { id, .. } if iden == Ok(id) || iden == Err(&source[ident::range(id)]) => {
Self::Ctor { fields, .. } => fields.iter().find_map(|f| f.value.declares(iden)), Some(id)
}
Self::Ctor { fields, .. } => fields.iter().find_map(|f| f.value.declares(iden, source)),
_ => None, _ => None,
} }
} }
@ -972,7 +975,9 @@ impl AstInner<[Symbol]> {
} }
fn new(file: Box<str>, path: &str, ctx: &mut ParserCtx, loader: Loader) -> NonNull<Self> { fn new(file: Box<str>, path: &str, ctx: &mut ParserCtx, loader: Loader) -> NonNull<Self> {
let arena = Arena::with_capacity(file.len() * SOURCE_TO_AST_FACTOR); let arena = Arena::with_capacity(
SOURCE_TO_AST_FACTOR * file.bytes().filter(|b| !b.is_ascii_whitespace()).count(),
);
let exprs = let exprs =
unsafe { core::mem::transmute(Parser::parse(ctx, &file, path, loader, &arena)) }; unsafe { core::mem::transmute(Parser::parse(ctx, &file, path, loader, &arena)) };
@ -1059,7 +1064,9 @@ impl Ast {
pub fn find_decl(&self, id: Result<Ident, &str>) -> Option<(&Expr, Ident)> { pub fn find_decl(&self, id: Result<Ident, &str>) -> Option<(&Expr, Ident)> {
self.exprs().iter().find_map(|expr| match expr { self.exprs().iter().find_map(|expr| match expr {
Expr::BinOp { left, op: TokenKind::Decl, .. } => left.declares(id).map(|id| (expr, id)), Expr::BinOp { left, op: TokenKind::Decl, .. } => {
left.declares(id, &self.file).map(|id| (expr, id))
}
_ => None, _ => None,
}) })
} }
@ -1224,10 +1231,7 @@ impl Arena {
.unwrap(); .unwrap();
let ptr = self.alloc_low(layout); let ptr = self.alloc_low(layout);
unsafe { unsafe {
ptr.cast::<usize>().copy_from_nonoverlapping( ptr.cast::<u8>().copy_from_nonoverlapping(NonNull::from(&expr).cast(), layout.size())
NonNull::from(&expr).cast(),
layout.size() / core::mem::size_of::<usize>(),
)
}; };
unsafe { ptr.cast::<Expr<'a>>().as_ref() } unsafe { ptr.cast::<Expr<'a>>().as_ref() }
} }
@ -1250,15 +1254,36 @@ impl Arena {
return ptr; return ptr;
} }
const EXPANSION_ALLOC: usize = 1024 * 4 - core::mem::size_of::<ArenaChunk>();
if layout.size() > EXPANSION_ALLOC {
let next_ptr = chunk.next_ptr();
if next_ptr.is_null() {
unsafe { unsafe {
core::ptr::write( core::ptr::write(
chunk, chunk,
ArenaChunk::new( ArenaChunk::new(
1024 * 4 - core::mem::size_of::<ArenaChunk>(), layout.size() + layout.align() - 1 + core::mem::size_of::<ArenaChunk>(),
core::ptr::read(chunk), Default::default(),
), ),
); );
} }
} else {
unsafe {
let chunk = ArenaChunk::new(
layout.size() + layout.align() - 1 + core::mem::size_of::<ArenaChunk>(),
core::ptr::read(next_ptr),
);
let alloc = chunk.base.add(chunk.base.align_offset(layout.align()));
core::ptr::write(next_ptr, chunk);
return NonNull::new_unchecked(alloc);
}
}
} else {
unsafe {
core::ptr::write(chunk, ArenaChunk::new(EXPANSION_ALLOC, core::ptr::read(chunk)));
}
}
chunk.alloc(layout).unwrap() chunk.alloc(layout).unwrap()
} }
@ -1300,11 +1325,16 @@ impl ArenaChunk {
return None; return None;
} }
unsafe { self.end = self.end.sub(size) }; unsafe { self.end = self.end.sub(size) };
debug_assert!(self.end >= self.base, "{:?} {:?}", self.end, self.base);
unsafe { Some(NonNull::new_unchecked(self.end)) } unsafe { Some(NonNull::new_unchecked(self.end)) }
} }
fn next(&self) -> Option<&Self> { fn next(&self) -> Option<&Self> {
unsafe { self.base.cast::<Self>().sub(1).as_ref() } unsafe { self.next_ptr().as_ref() }
}
fn next_ptr(&self) -> *mut Self {
unsafe { self.base.cast::<Self>().sub(1) }
} }
fn contains(&self, arg: *mut u8) -> bool { fn contains(&self, arg: *mut u8) -> bool {

5
lang/src/son.rs
View file

@ -1243,9 +1243,10 @@ impl Codegen {
[VOID], [VOID],
)) ))
} }
Expr::Call { func: &Expr::Ident { pos, id, name, .. }, args, .. } => { Expr::Call { func: &Expr::Ident { pos, id, .. }, args, .. } => {
self.ci.call_count += 1; self.ci.call_count += 1;
let func = self.find_or_declare(pos, self.ci.file, Some(id), name); let cfile = self.cfile().clone();
let func = self.find_or_declare(pos, self.ci.file, Some(id), cfile.ident_str(id));
let ty::Kind::Func(func) = func else { let ty::Kind::Func(func) = func else {
self.report( self.report(
pos, pos,

4
xtask/Cargo.toml
View file

@ -3,3 +3,7 @@ name = "xtask"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2021"
[dependencies]
anyhow = "1.0.89"
walrus = "0.22.0"

35
xtask/src/main.rs
View file

@ -2,11 +2,13 @@ mod utils;
use { use {
crate::utils::IterExt, crate::utils::IterExt,
anyhow::Context,
std::{ std::{
fs::File, fs::File,
io::{self, BufRead, BufReader, BufWriter, Seek, Write}, io::{self, BufRead, BufReader, BufWriter, Seek, Write},
path::Path, path::Path,
}, },
walrus::{ir::Value, ConstExpr, GlobalKind, ValType},
}; };
fn root() -> &'static Path { fn root() -> &'static Path {
@ -29,7 +31,25 @@ fn exec(mut cmd: std::process::Command) -> io::Result<()> {
Ok(()) Ok(())
} }
fn build_wasm_blob(name: &str, debug: bool) -> io::Result<()> { fn insert_stack_pointer_shim(file: impl AsRef<str>) -> anyhow::Result<()> {
let mut module = walrus::Module::from_file(file.as_ref())?;
let global = module
.globals
.iter()
.find(|g| g.ty == ValType::I32 && g.mutable)
.filter(|g| match g.kind {
GlobalKind::Local(ConstExpr::Value(Value::I32(n))) => n != 0,
_ => false,
})
.context("binary is missing a stak pointer")?;
module.exports.add("stack_pointer", global.id());
module.emit_wasm_file(file.as_ref())
}
fn build_wasm_blob(name: &str, debug: bool) -> anyhow::Result<()> {
let mut c = build_cmd(if debug { "cargo wasm-build-debug" } else { "cargo wasm-build" }); let mut c = build_cmd(if debug { "cargo wasm-build-debug" } else { "cargo wasm-build" });
c.arg(format!("wasm-{name}")); c.arg(format!("wasm-{name}"));
exec(c)?; exec(c)?;
@ -39,14 +59,15 @@ fn build_wasm_blob(name: &str, debug: bool) -> io::Result<()> {
exec(build_cmd(format!("wasm-opt -Oz {out_path} -o {out_path}")))?; exec(build_cmd(format!("wasm-opt -Oz {out_path} -o {out_path}")))?;
} }
exec(build_cmd(format!("cp {out_path} depell/src/{name}.wasm")))?; exec(build_cmd(format!("cp {out_path} depell/src/{name}.wasm")))?;
insert_stack_pointer_shim(format!("depell/src/{name}.wasm"))?;
exec(build_cmd(format!("gzip -f depell/src/{name}.wasm")))?; exec(build_cmd(format!("gzip -f depell/src/{name}.wasm")))?;
Ok(()) Ok(())
} }
fn main() -> io::Result<()> { fn main() -> anyhow::Result<()> {
let args = std::env::args().skip(1).collect::<Vec<_>>(); let args = std::env::args().skip(1).collect::<Vec<_>>();
match args[0].as_str() { match args[0].as_str() {
"fmt" => fmt(args[1] == "-r" || args[1] == "--renumber"), "fmt" => fmt(args[1] == "-r" || args[1] == "--renumber").context(""),
"build-depell-debug" => { "build-depell-debug" => {
build_wasm_blob("hbfmt", true)?; build_wasm_blob("hbfmt", true)?;
build_wasm_blob("hbc", true)?; build_wasm_blob("hbc", true)?;
@ -61,12 +82,18 @@ fn main() -> io::Result<()> {
exec(build_cmd("gzip -k -f depell/src/index.css"))?; exec(build_cmd("gzip -k -f depell/src/index.css"))?;
Ok(()) Ok(())
} }
"watch-depell" => { "watch-depell-debug" => {
let mut c = build_cmd("cargo watch --why"); let mut c = build_cmd("cargo watch --why");
c.arg("--exec=xtask build-depell-debug").arg("--exec=run -p depell"); c.arg("--exec=xtask build-depell-debug").arg("--exec=run -p depell");
exec(c)?; exec(c)?;
Ok(()) Ok(())
} }
"watch-depell" => {
let mut c = build_cmd("cargo watch --why");
c.arg("--exec=xtask build-depell").arg("--exec=run -p depell");
exec(c)?;
Ok(())
}
_ => Ok(()), _ => Ok(()),
} }
} }