From dc2e0cc5b36f84f9d5bb82b6f48e6b29869746d5 Mon Sep 17 00:00:00 2001 From: Jakub Doka Date: Mon, 14 Oct 2024 13:25:38 +0200 Subject: [PATCH] implementing dependency fetching --- Cargo.lock | 111 +++++++++++++++++++++++++++++ depell/Cargo.toml | 3 + depell/src/index.css | 4 ++ depell/src/index.js | 157 +++++++++++++++++++++++------------------- depell/src/main.rs | 115 ++++++++++++++++++++++++++----- depell/src/schema.sql | 1 + lang/src/codegen.rs | 1 + lang/src/lexer.rs | 2 +- 8 files changed, 306 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 054cc954..017915dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,18 @@ version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures", + "password-hash", +] + [[package]] name = "async-trait" version = "0.1.83" @@ -128,12 +140,36 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -164,14 +200,36 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cpufeatures" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "depell" version = "0.1.0" dependencies = [ + "argon2", "axum", "getrandom", + "hblang", "htmlm", "log", + "rand_core", "rusqlite", "serde", "time", @@ -187,6 +245,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -259,6 +328,16 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -587,6 +666,17 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -635,6 +725,15 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "regalloc2" version = "0.10.2" @@ -774,6 +873,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.79" @@ -890,6 +995,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-ident" version = "1.0.13" diff --git a/depell/Cargo.toml b/depell/Cargo.toml index 6fd1ad9c..0af1108a 100644 --- a/depell/Cargo.toml +++ b/depell/Cargo.toml @@ -4,10 +4,13 @@ version = "0.1.0" edition = "2021" [dependencies] +argon2 = "0.5.3" axum = "0.7.7" getrandom = "0.2.15" +hblang.workspace = true htmlm = "0.5.0" log = "0.4.22" +rand_core = { version = "0.6.4", features = ["getrandom"] } rusqlite = { version = "0.32.1", features = ["bundled"] } serde = { version = "1.0.210", features = ["derive"] } time = "0.3.36" diff --git a/depell/src/index.css b/depell/src/index.css index c88004a8..ea7c5d02 100644 --- a/depell/src/index.css +++ b/depell/src/index.css @@ -109,6 +109,10 @@ div#code-editor { display: flex; position: relative; + textarea { + flex: 1; + } + span#code-size { position: absolute; right: 2px; diff --git a/depell/src/index.js b/depell/src/index.js index f49e5d99..db02a55f 100644 --- a/depell/src/index.js +++ b/depell/src/index.js @@ -80,7 +80,6 @@ function modifyCode(instance, code, action) { /** @param {WebAssembly.Instance} instance @param {CallableFunction} func @param {any[]} args * @returns {boolean} */ function runWasmFunction(instance, func, ...args) { - //const prev = performance.now(); const { PANIC_MESSAGE, PANIC_MESSAGE_LEN, memory, stack_pointer } = instance.exports; if (!(true && memory instanceof WebAssembly.Memory @@ -91,18 +90,16 @@ function runWasmFunction(instance, func, ...args) { func(...args); return true; } catch (error) { - if (error instanceof WebAssembly.RuntimeError && error.message == "unreachable") { - if (PANIC_MESSAGE instanceof WebAssembly.Global - && PANIC_MESSAGE_LEN instanceof WebAssembly.Global) { - console.error(bufToString(memory, PANIC_MESSAGE, PANIC_MESSAGE_LEN), error); - } + if (error instanceof WebAssembly.RuntimeError + && error.message == "unreachable" + && PANIC_MESSAGE instanceof WebAssembly.Global + && PANIC_MESSAGE_LEN instanceof WebAssembly.Global) { + console.error(bufToString(memory, PANIC_MESSAGE, PANIC_MESSAGE_LEN), error); } else { console.error(error); } stack_pointer.value = ptr; return false; - } finally { - //console.log("compiletion took:", performance.now() - prev); } } @@ -143,27 +140,31 @@ function wireUp(target) { } const importRe = /@use\s*\(\s*"(([^"]|\\")+)"\s*\)/g; -/** @param {string} code @param {string[]} matches @returns {string[]} */ -function findImports(code, matches) { - matches.length = 0; +const prevRoots = new Set(); +/** @param {string} code @param {string[]} roots @param {Post[]} buf @returns {void} */ +function loadCachedPackages(code, roots, buf) { + buf[0].code = code; + + roots.length = 0; + let changed = false; for (const match of code.matchAll(importRe)) { - matches.push(match[1]); + changed ||= !prevRoots.has(match[1]); + roots.push(match[1]); } - matches.sort(); + if (!changed) return; + buf.length = 1; + prevRoots.clear(); - let c = 0; - for (let i = 1; i < matches.length; i++) { - if (matches[c] != matches[i]) { - matches[++c] = matches[i]; + for (let imp = roots.pop(); imp !== undefined; imp = roots.pop()) { + if (prevRoots.has(imp)) continue; prevRoots.add(imp); + buf.push({ path: imp, code: localStorage.getItem("package-" + imp) ?? never() }); + for (const match of buf[buf.length - 1].code.matchAll(importRe)) { + roots.push(match[1]); } } - matches.length = Math.min(matches.length, c + 1); - - return matches; } - /** @param {HTMLElement} target */ async function bindCodeEdit(target) { const edit = target.querySelector("#code-edit"); @@ -180,8 +181,8 @@ async function bindCodeEdit(target) { if (Number.isNaN(MAX_CODE_SIZE)) never(); const hbc = await getHbcInstance(), fmt = await getFmtInstance(); - const prevImports = []; - const matches = []; + let importDiff = new Set(); + const keyBuf = []; /**@type{Post[]}*/ const packages = [{ path: "local.hb", code: "" }]; const debounce = 100; @@ -189,59 +190,67 @@ async function bindCodeEdit(target) { let cancelation = undefined; let timeout = 0; - edit.addEventListener("input", () => { - if (timeout) clearTimeout(timeout); - timeout = setTimeout(() => { - prevImports.length = 0; prevImports.push(...matches); - const imports = findImports(edit.value, matches); - let changed = imports.length !== prevImports.length; - for (let i = 0; i < imports.length && !changed; i++) { - changed ||= imports[i] !== prevImports[i]; - } + const onInput = () => { + importDiff.clear(); + for (const match of edit.value.matchAll(importRe)) { + if (localStorage["package-" + match[1]]) continue; + importDiff.add(match[1]); + } - if (changed && imports.length !== 0) { - if (cancelation) cancelation.abort(); - cancelation = new AbortController(); - errors.textContent = "fetching: " + imports.join(", "); - fetch(`/code`, { - method: "POST", - signal: cancelation.signal, - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(imports), - }).then(async e => { - try { - const json = await e.json(); - if (e.status == 200) { - packages.length = 1; - packages.push(...json); + if (importDiff.size !== 0) { + if (cancelation) cancelation.abort(); + cancelation = new AbortController(); + + keyBuf.length = 0; + keyBuf.push(...importDiff.keys()); + + errors.textContent = "fetching: " + keyBuf.join(", "); + + fetch(`/code`, { + method: "POST", + signal: cancelation.signal, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(keyBuf), + }).then(async e => { + try { + const json = await e.json(); + if (e.status == 200) { + for (const key in json) localStorage["package-" + key] = json[key]; + const missing = keyBuf.filter(i => json[i] === undefined); + if (missing.length !== 0) { + errors.textContent = "failed to fetch: " + missing.join(", "); + } else { cancelation = undefined; edit.dispatchEvent(new InputEvent("input")); - } else { - errors.textContent = "failed to fetch: " + json.join(", "); } - } catch (er) { - errors.textContent = "completely failed to fetch (" - + e.status + "): " + imports.join(", "); - console.error(e); } - }); - } + } catch (er) { + errors.textContent = "completely failed to fetch (" + + e.status + "): " + keyBuf.join(", "); + console.error(e, er); + } + }); + } - if (cancelation && imports.length !== 0) { - return; - } + if (cancelation && importDiff.size !== 0) { + return; + } - packages[0].code = edit.value; + loadCachedPackages(edit.value, keyBuf, packages); - errors.textContent = compileCode(hbc, packages, 1); - const minified_size = modifyCode(fmt, edit.value, "minify")?.length; - if (minified_size) { - codeSize.textContent = (MAX_CODE_SIZE - minified_size) + ""; - const perc = Math.min(100, Math.floor(100 * (minified_size / MAX_CODE_SIZE))); - codeSize.style.color = `color-mix(in srgb, white, var(--error) ${perc}%)`; - } - timeout = 0; - }, debounce); + errors.textContent = compileCode(hbc, packages, 1); + const minified_size = modifyCode(fmt, edit.value, "minify")?.length; + if (minified_size) { + codeSize.textContent = (MAX_CODE_SIZE - minified_size) + ""; + const perc = Math.min(100, Math.floor(100 * (minified_size / MAX_CODE_SIZE))); + codeSize.style.color = `color-mix(in srgb, white, var(--error) ${perc}%)`; + } + timeout = 0; + }; + + edit.addEventListener("input", () => { + if (timeout) clearTimeout(timeout); + timeout = setTimeout(onInput, debounce) }); edit.dispatchEvent(new InputEvent("input")); } @@ -339,9 +348,19 @@ if (window.location.hostname === 'localhost') { })() } -document.body.addEventListener('htmx:afterSwap', (ev) => { +document.body.addEventListener('htmx:afterSettle', (ev) => { if (!(ev.target instanceof HTMLElement)) never(); wireUp(ev.target); }); +getFmtInstance().then(inst => { + document.body.addEventListener('htmx:configRequest', (ev) => { + const details = ev['detail']; + if (details.path === "/post" && details.verb === "post") { + details.parameters['code'] = modifyCode(inst, details.parameters['code'], "minify"); + } + }); +}); + + wireUp(document.body); diff --git a/depell/src/main.rs b/depell/src/main.rs index 7d147838..c57df4e0 100644 --- a/depell/src/main.rs +++ b/depell/src/main.rs @@ -1,4 +1,6 @@ +#![feature(iter_collect_into)] use { + argon2::{password_hash::SaltString, PasswordVerifier}, axum::{ body::Bytes, http::{header::COOKIE, request::Parts}, @@ -6,8 +8,13 @@ use { }, core::fmt, htmlm::{html, write_html}, + rand_core::OsRng, serde::{Deserialize, Serialize}, - std::{fmt::Write, net::Ipv4Addr}, + std::{ + collections::{HashMap, HashSet}, + fmt::Write, + net::Ipv4Addr, + }, }; const MAX_NAME_LENGTH: usize = 32; @@ -54,6 +61,7 @@ async fn amain() { .route("/post", get(Post::page)) .route("/post-view", get(Post::get)) .route("/post", post(Post::post)) + .route("/code", post(fetch_code)) .route("/login", get(Login::page)) .route("/login-view", get(Login::get)) .route("/login", post(Login::post)) @@ -82,7 +90,7 @@ trait PublicPage: Default { fn render(self) -> String { let mut str = String::new(); - Self::default().render_to_buf(&mut str); + self.render_to_buf(&mut str); str } @@ -100,7 +108,7 @@ trait Page: Default { fn render(self, session: &Session) -> String { let mut str = String::new(); - Self::default().render_to_buf(session, &mut str); + self.render_to_buf(session, &mut str); str } @@ -118,6 +126,30 @@ trait Page: Default { } } +async fn fetch_code( + axum::Json(paths): axum::Json>, +) -> axum::Json> { + let mut deps = HashMap::::new(); + db::with(|db| { + for path in &paths { + let Some((author, name)) = path.split_once('/') else { continue }; + db.fetch_deps + .query_map((name, author), |r| { + Ok(( + r.get::<_, String>(1)? + "/" + r.get_ref(0)?.as_str()?, + r.get::<_, String>(2)?, + )) + }) + .inspect_err(|e| log::error!("{e}")) + .into_iter() + .flatten() + .filter_map(|r| r.inspect_err(|e| log::error!("{e}")).ok()) + .collect_into(&mut deps); + } + }); + axum::Json(deps) +} + #[derive(Default)] struct Index; @@ -148,14 +180,14 @@ impl Page for Post { fn render_to_buf(self, session: &Session, buf: &mut String) { let Self { name, code, error, .. } = self; write_html! { (buf) -
+ if let Some(e) = error {
e
}
- + MAX_CODE_LENGTH
@@ -182,6 +214,18 @@ impl Post { log::error!("{e}"); Some("internal server error") }); + return; + } + + for (author, name) in hblang::lexer::Lexer::uses(&data.code) + .filter_map(|v| v.split_once('/')) + .collect::>() + { + if let Err(e) = db.create_import.insert((author, name, &session.name, &data.name)) { + log::error!("{e}"); + data.error = Some("internal server error"); + return; + }; } }); @@ -211,7 +255,7 @@ impl fmt::Display for Post { 
code
if *timestamp == 0 { - } } @@ -252,7 +296,20 @@ impl Page for Profile { } } -#[derive(Serialize, Deserialize, Default)] +fn hash_password(password: &str) -> String { + use argon2::PasswordHasher; + argon2::Argon2::default() + .hash_password(password.as_bytes(), &SaltString::generate(&mut OsRng)) + .unwrap() + .to_string() +} + +fn verify_password(hash: &str, password: &str) -> Result<(), argon2::password_hash::Error> { + argon2::Argon2::default() + .verify_password(password.as_bytes(), &argon2::PasswordHash::new(hash)?) +} + +#[derive(Serialize, Deserialize, Default, Debug)] struct Login { name: String, password: String, @@ -264,7 +321,7 @@ impl PublicPage for Login { fn render_to_buf(self, buf: &mut String) { let Login { name, password, error } = self; write_html! { (buf) - + if let Some(e) = error {
e
} @@ -282,9 +339,9 @@ impl Login { ) -> Result, Html> { // TODO: hash password let mut id = [0u8; 32]; - db::with(|db| match db.authenticate.query((&data.name, &data.password)) { - Ok(mut r) => { - if r.next().map_or(true, |v| v.is_none()) { + db::with(|db| match db.authenticate.query_row((&data.name,), |r| r.get::<_, String>(1)) { + Ok(hash) => { + if verify_password(&hash, &data.password).is_err() { data.error = Some("invalid credentials"); } else { getrandom::getrandom(&mut id).unwrap(); @@ -294,13 +351,17 @@ impl Login { } } } + Err(rusqlite::Error::QueryReturnedNoRows) => { + data.error = Some("invalid credentials"); + } Err(e) => { - log::error!("{e}"); + log::error!("foo {e}"); data.error = Some("internal server error"); } }); if data.error.is_some() { + log::error!("what {:?}", data); Err(Html(data.render())) } else { Ok(AppendHeaders([ @@ -335,7 +396,7 @@ impl PublicPage for Signup { fn render_to_buf(self, buf: &mut String) { let Signup { name, new_password, confirm_password, error } = self; write_html! { (buf) - + if let Some(e) = error {
e
} @@ -353,7 +414,7 @@ impl Signup { async fn post(axum::Form(mut data): axum::Form) -> Result> { db::with(|db| { // TODO: hash passwords - match db.register.insert((&data.name, &data.new_password)) { + match db.register.insert((&data.name, hash_password(&data.new_password))) { Ok(_) => {} Err(rusqlite::Error::SqliteFailure(e, _)) if e.code == rusqlite::ErrorCode::ConstraintViolation => @@ -498,7 +559,7 @@ mod db { macro_rules! gen_queries { ($vis:vis struct $name:ident { - $($qname:ident: $code:literal,)* + $($qname:ident: $code:expr,)* }) => { #[allow(dead_code)] $vis struct $name<'a> { @@ -518,12 +579,25 @@ mod db { gen_queries! { pub struct Queries { register: "INSERT INTO user (name, password_hash) VALUES(?, ?)", - authenticate: "SELECT name, password_hash FROM user WHERE name = ? AND password_hash = ?", + authenticate: "SELECT name, password_hash FROM user WHERE name = ?", login: "INSERT OR REPLACE INTO session (id, username, expiration) VALUES(?, ?, ?)", logout: "DELETE FROM session WHERE id = ?", get_session: "SELECT username, expiration FROM session WHERE id = ?", get_user_posts: "SELECT author, name, timestamp, code FROM post WHERE author = ?", create_post: "INSERT INTO post (name, author, timestamp, code) VALUES(?, ?, ?, ?)", + fetch_deps: " + WITH RECURSIVE roots(name, author, code) AS ( + SELECT name, author, code FROM post WHERE name = ? AND author = ? + UNION ALL + SELECT post.name, post.author, post.code FROM + post JOIN import ON post.name = import.to_name + AND post.author = import.to_author + JOIN roots ON import.from_name = roots.name + AND import.from_author = roots.author + ) SELECT * FROM roots; + ", + create_import: "INSERT INTO import(to_author, to_name, from_author, from_name) + VALUES(?, ?, ?, ?)", } } @@ -569,7 +643,12 @@ impl log::Log for Logger { fn log(&self, record: &log::Record) { if self.enabled(record.metadata()) { - eprintln!("{} - {}", record.module_path().unwrap_or("=="), record.args()); + eprintln!( + "{} {:?} - {}", + record.module_path().unwrap_or("=="), + record.line(), + record.args() + ); } } diff --git a/depell/src/schema.sql b/depell/src/schema.sql index 95297151..cf9c7672 100644 --- a/depell/src/schema.sql +++ b/depell/src/schema.sql @@ -49,3 +49,4 @@ CREATE TABLE IF NOT EXISTS run( FOREIGN KEY (runner) REFERENCES user(name), PRIMARY KEY (code_name, code_author, runner) ); + diff --git a/lang/src/codegen.rs b/lang/src/codegen.rs index 4429c213..57a79c2f 100644 --- a/lang/src/codegen.rs +++ b/lang/src/codegen.rs @@ -749,6 +749,7 @@ impl Codegen { pub fn generate(&mut self, root: FileId) { self.ci.emit_entry_prelude(); + self.ci.file = root; self.find_or_declare(0, root, Err("main"), ""); self.make_func_reachable(0); self.complete_call_graph(); diff --git a/lang/src/lexer.rs b/lang/src/lexer.rs index 2370d557..79b3ba32 100644 --- a/lang/src/lexer.rs +++ b/lang/src/lexer.rs @@ -376,7 +376,7 @@ impl<'a> Lexer<'a> { Self::restore(input, 0) } - pub fn imports(input: &'a str) -> impl Iterator { + pub fn uses(input: &'a str) -> impl Iterator { let mut s = Self::new(input); core::iter::from_fn(move || loop { let t = s.eat();