implementing dependency fetching

This commit is contained in:
Jakub Doka 2024-10-14 13:25:38 +02:00
parent c9b85f9004
commit dc2e0cc5b3
No known key found for this signature in database
GPG key ID: C6E9A89936B8C143
8 changed files with 306 additions and 88 deletions

111
Cargo.lock generated
View file

@ -41,6 +41,18 @@ version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
[[package]]
name = "argon2"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072"
dependencies = [
"base64ct",
"blake2",
"cpufeatures",
"password-hash",
]
[[package]]
name = "async-trait"
version = "0.1.83"
@ -128,12 +140,36 @@ dependencies = [
"windows-targets",
]
[[package]]
name = "base64ct"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
[[package]]
name = "bitflags"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
[[package]]
name = "blake2"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
dependencies = [
"digest",
]
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]]
name = "bumpalo"
version = "3.16.0"
@ -164,14 +200,36 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cpufeatures"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0"
dependencies = [
"libc",
]
[[package]]
name = "crypto-common"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "depell"
version = "0.1.0"
dependencies = [
"argon2",
"axum",
"getrandom",
"hblang",
"htmlm",
"log",
"rand_core",
"rusqlite",
"serde",
"time",
@ -187,6 +245,17 @@ dependencies = [
"powerfmt",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
"subtle",
]
[[package]]
name = "equivalent"
version = "1.0.1"
@ -259,6 +328,16 @@ dependencies = [
"pin-utils",
]
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.15"
@ -587,6 +666,17 @@ version = "1.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
[[package]]
name = "password-hash"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166"
dependencies = [
"base64ct",
"rand_core",
"subtle",
]
[[package]]
name = "percent-encoding"
version = "2.3.1"
@ -635,6 +725,15 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "regalloc2"
version = "0.10.2"
@ -774,6 +873,12 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "subtle"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
version = "2.0.79"
@ -890,6 +995,12 @@ dependencies = [
"once_cell",
]
[[package]]
name = "typenum"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]]
name = "unicode-ident"
version = "1.0.13"

View file

@ -4,10 +4,13 @@ version = "0.1.0"
edition = "2021"
[dependencies]
argon2 = "0.5.3"
axum = "0.7.7"
getrandom = "0.2.15"
hblang.workspace = true
htmlm = "0.5.0"
log = "0.4.22"
rand_core = { version = "0.6.4", features = ["getrandom"] }
rusqlite = { version = "0.32.1", features = ["bundled"] }
serde = { version = "1.0.210", features = ["derive"] }
time = "0.3.36"

View file

@ -109,6 +109,10 @@ div#code-editor {
display: flex;
position: relative;
textarea {
flex: 1;
}
span#code-size {
position: absolute;
right: 2px;

View file

@ -80,7 +80,6 @@ function modifyCode(instance, code, action) {
/** @param {WebAssembly.Instance} instance @param {CallableFunction} func @param {any[]} args
* @returns {boolean} */
function runWasmFunction(instance, func, ...args) {
//const prev = performance.now();
const { PANIC_MESSAGE, PANIC_MESSAGE_LEN, memory, stack_pointer } = instance.exports;
if (!(true
&& memory instanceof WebAssembly.Memory
@ -91,18 +90,16 @@ function runWasmFunction(instance, func, ...args) {
func(...args);
return true;
} catch (error) {
if (error instanceof WebAssembly.RuntimeError && error.message == "unreachable") {
if (PANIC_MESSAGE instanceof WebAssembly.Global
&& PANIC_MESSAGE_LEN instanceof WebAssembly.Global) {
console.error(bufToString(memory, PANIC_MESSAGE, PANIC_MESSAGE_LEN), error);
}
if (error instanceof WebAssembly.RuntimeError
&& error.message == "unreachable"
&& PANIC_MESSAGE instanceof WebAssembly.Global
&& PANIC_MESSAGE_LEN instanceof WebAssembly.Global) {
console.error(bufToString(memory, PANIC_MESSAGE, PANIC_MESSAGE_LEN), error);
} else {
console.error(error);
}
stack_pointer.value = ptr;
return false;
} finally {
//console.log("compiletion took:", performance.now() - prev);
}
}
@ -143,27 +140,31 @@ function wireUp(target) {
}
const importRe = /@use\s*\(\s*"(([^"]|\\")+)"\s*\)/g;
/** @param {string} code @param {string[]} matches @returns {string[]} */
function findImports(code, matches) {
matches.length = 0;
const prevRoots = new Set();
/** @param {string} code @param {string[]} roots @param {Post[]} buf @returns {void} */
function loadCachedPackages(code, roots, buf) {
buf[0].code = code;
roots.length = 0;
let changed = false;
for (const match of code.matchAll(importRe)) {
matches.push(match[1]);
changed ||= !prevRoots.has(match[1]);
roots.push(match[1]);
}
matches.sort();
if (!changed) return;
buf.length = 1;
prevRoots.clear();
let c = 0;
for (let i = 1; i < matches.length; i++) {
if (matches[c] != matches[i]) {
matches[++c] = matches[i];
for (let imp = roots.pop(); imp !== undefined; imp = roots.pop()) {
if (prevRoots.has(imp)) continue; prevRoots.add(imp);
buf.push({ path: imp, code: localStorage.getItem("package-" + imp) ?? never() });
for (const match of buf[buf.length - 1].code.matchAll(importRe)) {
roots.push(match[1]);
}
}
matches.length = Math.min(matches.length, c + 1);
return matches;
}
/** @param {HTMLElement} target */
async function bindCodeEdit(target) {
const edit = target.querySelector("#code-edit");
@ -180,8 +181,8 @@ async function bindCodeEdit(target) {
if (Number.isNaN(MAX_CODE_SIZE)) never();
const hbc = await getHbcInstance(), fmt = await getFmtInstance();
const prevImports = [];
const matches = [];
let importDiff = new Set();
const keyBuf = [];
/**@type{Post[]}*/
const packages = [{ path: "local.hb", code: "" }];
const debounce = 100;
@ -189,59 +190,67 @@ async function bindCodeEdit(target) {
let cancelation = undefined;
let timeout = 0;
edit.addEventListener("input", () => {
if (timeout) clearTimeout(timeout);
timeout = setTimeout(() => {
prevImports.length = 0; prevImports.push(...matches);
const imports = findImports(edit.value, matches);
let changed = imports.length !== prevImports.length;
for (let i = 0; i < imports.length && !changed; i++) {
changed ||= imports[i] !== prevImports[i];
}
const onInput = () => {
importDiff.clear();
for (const match of edit.value.matchAll(importRe)) {
if (localStorage["package-" + match[1]]) continue;
importDiff.add(match[1]);
}
if (changed && imports.length !== 0) {
if (cancelation) cancelation.abort();
cancelation = new AbortController();
errors.textContent = "fetching: " + imports.join(", ");
fetch(`/code`, {
method: "POST",
signal: cancelation.signal,
headers: { "Content-Type": "application/json" },
body: JSON.stringify(imports),
}).then(async e => {
try {
const json = await e.json();
if (e.status == 200) {
packages.length = 1;
packages.push(...json);
if (importDiff.size !== 0) {
if (cancelation) cancelation.abort();
cancelation = new AbortController();
keyBuf.length = 0;
keyBuf.push(...importDiff.keys());
errors.textContent = "fetching: " + keyBuf.join(", ");
fetch(`/code`, {
method: "POST",
signal: cancelation.signal,
headers: { "Content-Type": "application/json" },
body: JSON.stringify(keyBuf),
}).then(async e => {
try {
const json = await e.json();
if (e.status == 200) {
for (const key in json) localStorage["package-" + key] = json[key];
const missing = keyBuf.filter(i => json[i] === undefined);
if (missing.length !== 0) {
errors.textContent = "failed to fetch: " + missing.join(", ");
} else {
cancelation = undefined;
edit.dispatchEvent(new InputEvent("input"));
} else {
errors.textContent = "failed to fetch: " + json.join(", ");
}
} catch (er) {
errors.textContent = "completely failed to fetch ("
+ e.status + "): " + imports.join(", ");
console.error(e);
}
});
}
} catch (er) {
errors.textContent = "completely failed to fetch ("
+ e.status + "): " + keyBuf.join(", ");
console.error(e, er);
}
});
}
if (cancelation && imports.length !== 0) {
return;
}
if (cancelation && importDiff.size !== 0) {
return;
}
packages[0].code = edit.value;
loadCachedPackages(edit.value, keyBuf, packages);
errors.textContent = compileCode(hbc, packages, 1);
const minified_size = modifyCode(fmt, edit.value, "minify")?.length;
if (minified_size) {
codeSize.textContent = (MAX_CODE_SIZE - minified_size) + "";
const perc = Math.min(100, Math.floor(100 * (minified_size / MAX_CODE_SIZE)));
codeSize.style.color = `color-mix(in srgb, white, var(--error) ${perc}%)`;
}
timeout = 0;
}, debounce);
errors.textContent = compileCode(hbc, packages, 1);
const minified_size = modifyCode(fmt, edit.value, "minify")?.length;
if (minified_size) {
codeSize.textContent = (MAX_CODE_SIZE - minified_size) + "";
const perc = Math.min(100, Math.floor(100 * (minified_size / MAX_CODE_SIZE)));
codeSize.style.color = `color-mix(in srgb, white, var(--error) ${perc}%)`;
}
timeout = 0;
};
edit.addEventListener("input", () => {
if (timeout) clearTimeout(timeout);
timeout = setTimeout(onInput, debounce)
});
edit.dispatchEvent(new InputEvent("input"));
}
@ -339,9 +348,19 @@ if (window.location.hostname === 'localhost') {
})()
}
document.body.addEventListener('htmx:afterSwap', (ev) => {
document.body.addEventListener('htmx:afterSettle', (ev) => {
if (!(ev.target instanceof HTMLElement)) never();
wireUp(ev.target);
});
getFmtInstance().then(inst => {
document.body.addEventListener('htmx:configRequest', (ev) => {
const details = ev['detail'];
if (details.path === "/post" && details.verb === "post") {
details.parameters['code'] = modifyCode(inst, details.parameters['code'], "minify");
}
});
});
wireUp(document.body);

View file

@ -1,4 +1,6 @@
#![feature(iter_collect_into)]
use {
argon2::{password_hash::SaltString, PasswordVerifier},
axum::{
body::Bytes,
http::{header::COOKIE, request::Parts},
@ -6,8 +8,13 @@ use {
},
core::fmt,
htmlm::{html, write_html},
rand_core::OsRng,
serde::{Deserialize, Serialize},
std::{fmt::Write, net::Ipv4Addr},
std::{
collections::{HashMap, HashSet},
fmt::Write,
net::Ipv4Addr,
},
};
const MAX_NAME_LENGTH: usize = 32;
@ -54,6 +61,7 @@ async fn amain() {
.route("/post", get(Post::page))
.route("/post-view", get(Post::get))
.route("/post", post(Post::post))
.route("/code", post(fetch_code))
.route("/login", get(Login::page))
.route("/login-view", get(Login::get))
.route("/login", post(Login::post))
@ -82,7 +90,7 @@ trait PublicPage: Default {
fn render(self) -> String {
let mut str = String::new();
Self::default().render_to_buf(&mut str);
self.render_to_buf(&mut str);
str
}
@ -100,7 +108,7 @@ trait Page: Default {
fn render(self, session: &Session) -> String {
let mut str = String::new();
Self::default().render_to_buf(session, &mut str);
self.render_to_buf(session, &mut str);
str
}
@ -118,6 +126,30 @@ trait Page: Default {
}
}
async fn fetch_code(
axum::Json(paths): axum::Json<Vec<String>>,
) -> axum::Json<HashMap<String, String>> {
let mut deps = HashMap::<String, String>::new();
db::with(|db| {
for path in &paths {
let Some((author, name)) = path.split_once('/') else { continue };
db.fetch_deps
.query_map((name, author), |r| {
Ok((
r.get::<_, String>(1)? + "/" + r.get_ref(0)?.as_str()?,
r.get::<_, String>(2)?,
))
})
.inspect_err(|e| log::error!("{e}"))
.into_iter()
.flatten()
.filter_map(|r| r.inspect_err(|e| log::error!("{e}")).ok())
.collect_into(&mut deps);
}
});
axum::Json(deps)
}
#[derive(Default)]
struct Index;
@ -148,14 +180,14 @@ impl Page for Post {
fn render_to_buf(self, session: &Session, buf: &mut String) {
let Self { name, code, error, .. } = self;
write_html! { (buf)
<form id="postForm" "hx-post"="/post" "hx-swap"="outherHTML">
<form id="postForm" "hx-post"="/post" "hx-swap"="outerHTML">
if let Some(e) = error { <div class="error">e</div> }
<input name="author" type="text" value={session.name} hidden>
<input name="name" type="text" placeholder="name" value=name
required maxlength=MAX_POSTNAME_LENGTH>
<div id="code-editor">
<textarea id="code-edit" name="code" placeholder="code" rows=1 required
style="flex: 1">code</textarea>
<textarea id="code-edit" name="code" placeholder="code" rows=1
required>code</textarea>
<span id="code-size">MAX_CODE_LENGTH</span>
</div>
<input type="submit" value="submit">
@ -182,6 +214,18 @@ impl Post {
log::error!("{e}");
Some("internal server error")
});
return;
}
for (author, name) in hblang::lexer::Lexer::uses(&data.code)
.filter_map(|v| v.split_once('/'))
.collect::<HashSet<_>>()
{
if let Err(e) = db.create_import.insert((author, name, &session.name, &data.name)) {
log::error!("{e}");
data.error = Some("internal server error");
return;
};
}
});
@ -211,7 +255,7 @@ impl fmt::Display for Post {
</div>
<pre apply="fmt">code</pre>
if *timestamp == 0 {
<button "hx-get"="/post" "hx-swap"="outherHTML"
<button "hx-get"="/post" "hx-swap"="outerHTML"
"hx-target"="[preview]">"edit"</button>
}
</div> }
@ -252,7 +296,20 @@ impl Page for Profile {
}
}
#[derive(Serialize, Deserialize, Default)]
fn hash_password(password: &str) -> String {
use argon2::PasswordHasher;
argon2::Argon2::default()
.hash_password(password.as_bytes(), &SaltString::generate(&mut OsRng))
.unwrap()
.to_string()
}
fn verify_password(hash: &str, password: &str) -> Result<(), argon2::password_hash::Error> {
argon2::Argon2::default()
.verify_password(password.as_bytes(), &argon2::PasswordHash::new(hash)?)
}
#[derive(Serialize, Deserialize, Default, Debug)]
struct Login {
name: String,
password: String,
@ -264,7 +321,7 @@ impl PublicPage for Login {
fn render_to_buf(self, buf: &mut String) {
let Login { name, password, error } = self;
write_html! { (buf)
<form "hx-post"="/login" "hx-swap"="outherHTML">
<form "hx-post"="/login" "hx-swap"="outerHTML">
if let Some(e) = error { <div class="error">e</div> }
<input name="name" type="text" autocomplete="name" placeholder="name" value=name
required maxlength=MAX_NAME_LENGTH>
@ -282,9 +339,9 @@ impl Login {
) -> Result<AppendHeaders<[(&'static str, String); 2]>, Html<String>> {
// TODO: hash password
let mut id = [0u8; 32];
db::with(|db| match db.authenticate.query((&data.name, &data.password)) {
Ok(mut r) => {
if r.next().map_or(true, |v| v.is_none()) {
db::with(|db| match db.authenticate.query_row((&data.name,), |r| r.get::<_, String>(1)) {
Ok(hash) => {
if verify_password(&hash, &data.password).is_err() {
data.error = Some("invalid credentials");
} else {
getrandom::getrandom(&mut id).unwrap();
@ -294,13 +351,17 @@ impl Login {
}
}
}
Err(rusqlite::Error::QueryReturnedNoRows) => {
data.error = Some("invalid credentials");
}
Err(e) => {
log::error!("{e}");
log::error!("foo {e}");
data.error = Some("internal server error");
}
});
if data.error.is_some() {
log::error!("what {:?}", data);
Err(Html(data.render()))
} else {
Ok(AppendHeaders([
@ -335,7 +396,7 @@ impl PublicPage for Signup {
fn render_to_buf(self, buf: &mut String) {
let Signup { name, new_password, confirm_password, error } = self;
write_html! { (buf)
<form "hx-post"="/signup" "hx-swap"="outherHTML">
<form "hx-post"="/signup" "hx-swap"="outerHTML">
if let Some(e) = error { <div class="error">e</div> }
<input name="name" type="text" autocomplete="name" placeholder="name" value=name
maxlength=MAX_NAME_LENGTH required>
@ -353,7 +414,7 @@ impl Signup {
async fn post(axum::Form(mut data): axum::Form<Self>) -> Result<Redirect, Html<String>> {
db::with(|db| {
// TODO: hash passwords
match db.register.insert((&data.name, &data.new_password)) {
match db.register.insert((&data.name, hash_password(&data.new_password))) {
Ok(_) => {}
Err(rusqlite::Error::SqliteFailure(e, _))
if e.code == rusqlite::ErrorCode::ConstraintViolation =>
@ -498,7 +559,7 @@ mod db {
macro_rules! gen_queries {
($vis:vis struct $name:ident {
$($qname:ident: $code:literal,)*
$($qname:ident: $code:expr,)*
}) => {
#[allow(dead_code)]
$vis struct $name<'a> {
@ -518,12 +579,25 @@ mod db {
gen_queries! {
pub struct Queries {
register: "INSERT INTO user (name, password_hash) VALUES(?, ?)",
authenticate: "SELECT name, password_hash FROM user WHERE name = ? AND password_hash = ?",
authenticate: "SELECT name, password_hash FROM user WHERE name = ?",
login: "INSERT OR REPLACE INTO session (id, username, expiration) VALUES(?, ?, ?)",
logout: "DELETE FROM session WHERE id = ?",
get_session: "SELECT username, expiration FROM session WHERE id = ?",
get_user_posts: "SELECT author, name, timestamp, code FROM post WHERE author = ?",
create_post: "INSERT INTO post (name, author, timestamp, code) VALUES(?, ?, ?, ?)",
fetch_deps: "
WITH RECURSIVE roots(name, author, code) AS (
SELECT name, author, code FROM post WHERE name = ? AND author = ?
UNION ALL
SELECT post.name, post.author, post.code FROM
post JOIN import ON post.name = import.to_name
AND post.author = import.to_author
JOIN roots ON import.from_name = roots.name
AND import.from_author = roots.author
) SELECT * FROM roots;
",
create_import: "INSERT INTO import(to_author, to_name, from_author, from_name)
VALUES(?, ?, ?, ?)",
}
}
@ -569,7 +643,12 @@ impl log::Log for Logger {
fn log(&self, record: &log::Record) {
if self.enabled(record.metadata()) {
eprintln!("{} - {}", record.module_path().unwrap_or("=="), record.args());
eprintln!(
"{} {:?} - {}",
record.module_path().unwrap_or("=="),
record.line(),
record.args()
);
}
}

View file

@ -49,3 +49,4 @@ CREATE TABLE IF NOT EXISTS run(
FOREIGN KEY (runner) REFERENCES user(name),
PRIMARY KEY (code_name, code_author, runner)
);

View file

@ -749,6 +749,7 @@ impl Codegen {
pub fn generate(&mut self, root: FileId) {
self.ci.emit_entry_prelude();
self.ci.file = root;
self.find_or_declare(0, root, Err("main"), "");
self.make_func_reachable(0);
self.complete_call_graph();

View file

@ -376,7 +376,7 @@ impl<'a> Lexer<'a> {
Self::restore(input, 0)
}
pub fn imports(input: &'a str) -> impl Iterator<Item = &'a str> {
pub fn uses(input: &'a str) -> impl Iterator<Item = &'a str> {
let mut s = Self::new(input);
core::iter::from_fn(move || loop {
let t = s.eat();