renaming directories, reducing temporary allocations during parsing

This commit is contained in:
Jakub Doka 2024-10-10 15:48:08 +02:00
parent a538c0ddb0
commit f0ae65606d
No known key found for this signature in database
GPG key ID: C6E9A89936B8C143
105 changed files with 142 additions and 77 deletions

2
.gitignore vendored
View file

@ -1,5 +1,5 @@
/target /target
/hbbytecode/src/instrs.rs /bytecode/src/instrs.rs
/.rgignore /.rgignore
rustc-ice-* rustc-ice-*
db.sqlite db.sqlite

16
Cargo.lock generated
View file

@ -271,10 +271,6 @@ dependencies = [
name = "hbbytecode" name = "hbbytecode"
version = "0.1.0" version = "0.1.0"
[[package]]
name = "hbjit"
version = "0.1.0"
[[package]] [[package]]
name = "hblang" name = "hblang"
version = "0.1.0" version = "0.1.0"
@ -309,18 +305,18 @@ checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
[[package]] [[package]]
name = "htmlm" name = "htmlm"
version = "0.3.0" version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91eeb833c1f091f72c22c2d213d8acb12b953cea20fd911a2f636e806858a391" checksum = "a95b97f2c13991e486bf95be6d19c6c3d1fef4f8ec1e298e40aaf98769789295"
dependencies = [ dependencies = [
"htmlm-macro", "htmlm-macro",
] ]
[[package]] [[package]]
name = "htmlm-macro" name = "htmlm-macro"
version = "0.1.0" version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3607a5606a3432058fa765bc4dec74ab8d103282cf16f35df7be838a674ebaa9" checksum = "7f881f4929b944a9566f12d8ac3bf9881325c77c11b9b0adcdc6944018b55ac7"
[[package]] [[package]]
name = "http" name = "http"
@ -809,6 +805,10 @@ version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-hbc"
version = "0.1.0"
[[package]] [[package]]
name = "wasm-hbfmt" name = "wasm-hbfmt"
version = "0.1.0" version = "0.1.0"

View file

@ -1,19 +1,27 @@
[workspace] [workspace]
resolver = "2" resolver = "2"
members = [ members = [
"hbbytecode", "bytecode",
"hbvm", "vm",
"hbxrt", "xrt",
"xtask", "xtask",
"hblang", "lang",
"hbjit",
"depell", "depell",
"depell/wasm-hbfmt" "depell/wasm-fmt",
"depell/wasm-hbc",
] ]
[workspace.dependencies]
hbbytecode = { path = "bytecode", default-features = false }
hbvm = { path = "vm" }
hbxrt = { path = "xrt" }
hblang = { path = "lang", default-features = false }
hbjit = { path = "jit" }
[profile.release] [profile.release]
lto = true lto = true
debug = true debug = true
#strip = true
codegen-units = 1 codegen-units = 1
panic = "abort" panic = "abort"

View file

@ -159,7 +159,7 @@ fn comma_sep(items: impl Iterator<Item = String>) -> String {
} }
fn instructions() -> impl Iterator<Item = [&'static str; 4]> { fn instructions() -> impl Iterator<Item = [&'static str; 4]> {
include_str!("../hbbytecode/instructions.in") include_str!("instructions.in")
.lines() .lines()
.filter_map(|line| line.strip_suffix(';')) .filter_map(|line| line.strip_suffix(';'))
.map(|line| line.splitn(4, ',').map(str::trim).next_chunk().unwrap()) .map(|line| line.splitn(4, ',').map(str::trim).next_chunk().unwrap())

View file

@ -6,7 +6,7 @@ edition = "2021"
[dependencies] [dependencies]
axum = "0.7.7" axum = "0.7.7"
getrandom = "0.2.15" getrandom = "0.2.15"
htmlm = "0.3.0" htmlm = "0.5.0"
log = "0.4.22" log = "0.4.22"
rusqlite = "0.32.1" rusqlite = "0.32.1"
serde = { version = "1.0.210", features = ["derive"] } serde = { version = "1.0.210", features = ["derive"] }

View file

@ -1,12 +1,13 @@
use { use {
axum::{ axum::{
body::Bytes,
http::{header::COOKIE, request::Parts}, http::{header::COOKIE, request::Parts},
response::{AppendHeaders, Html}, response::{AppendHeaders, Html},
}, },
core::fmt, core::fmt,
htmlm::{html, write_html}, htmlm::{html, write_html},
serde::{Deserialize, Serialize}, serde::{Deserialize, Serialize},
std::net::Ipv4Addr, std::{fmt::Write, net::Ipv4Addr},
}; };
const MAX_NAME_LENGTH: usize = 32; const MAX_NAME_LENGTH: usize = 32;
@ -16,6 +17,17 @@ const SESSION_DURATION_SECS: u64 = 60 * 60;
type Redirect<const COUNT: usize = 1> = AppendHeaders<[(&'static str, &'static str); COUNT]>; type Redirect<const COUNT: usize = 1> = AppendHeaders<[(&'static str, &'static str); COUNT]>;
macro_rules! static_asset {
($mime:literal, $body:literal) => {
get(|| async {
axum::http::Response::builder()
.header("content-type", $mime)
.body(axum::body::Body::from(Bytes::from_static(include_bytes!($body))))
.unwrap()
})
};
}
async fn amain() { async fn amain() {
use axum::routing::{delete, get, post}; use axum::routing::{delete, get, post};
@ -30,15 +42,10 @@ async fn amain() {
.route("/", get(Index::page)) .route("/", get(Index::page))
.route( .route(
"/hbfmt.wasm", "/hbfmt.wasm",
get(|| async move { static_asset!(
axum::http::Response::builder() "application/wasm",
.header("content-type", "application/wasm") "../../target/wasm32-unknown-unknown/small/wasm_hbfmt.wasm"
.body(axum::body::Body::from( ),
include_bytes!("../../target/wasm32-unknown-unknown/small/wasm_hbfmt.wasm")
.to_vec(),
))
.unwrap()
}),
) )
.route("/index-view", get(Index::get)) .route("/index-view", get(Index::get))
.route("/feed", get(Index::page)) .route("/feed", get(Index::page))
@ -71,19 +78,31 @@ async fn amain() {
} }
trait PublicPage: Default { trait PublicPage: Default {
fn render(self) -> String; fn render_to_buf(self, buf: &mut String);
fn render(self) -> String {
let mut str = String::new();
Self::default().render_to_buf(&mut str);
str
}
async fn get() -> Html<String> { async fn get() -> Html<String> {
Html(Self::default().render()) Html(Self::default().render())
} }
async fn page(session: Option<Session>) -> Html<String> { async fn page(session: Option<Session>) -> Html<String> {
base(Self::default().render(), session).await base(|s| Self::default().render_to_buf(s), session.as_ref()).await
} }
} }
trait Page: Default { trait Page: Default {
fn render(self, session: &Session) -> String; fn render_to_buf(self, session: &Session, buf: &mut String);
fn render(self, session: &Session) -> String {
let mut str = String::new();
Self::default().render_to_buf(session, &mut str);
str
}
async fn get(session: Session) -> Html<String> { async fn get(session: Session) -> Html<String> {
Html(Self::default().render(&session)) Html(Self::default().render(&session))
@ -91,7 +110,9 @@ trait Page: Default {
async fn page(session: Option<Session>) -> Result<Html<String>, axum::response::Redirect> { async fn page(session: Option<Session>) -> Result<Html<String>, axum::response::Redirect> {
match session { match session {
Some(session) => Ok(base(Self::default().render(&session), Some(session)).await), Some(session) => {
Ok(base(|f| Self::default().render_to_buf(&session, f), Some(&session)).await)
}
None => Err(axum::response::Redirect::permanent("/login")), None => Err(axum::response::Redirect::permanent("/login")),
} }
} }
@ -101,8 +122,8 @@ trait Page: Default {
struct Index; struct Index;
impl PublicPage for Index { impl PublicPage for Index {
fn render(self) -> String { fn render_to_buf(self, buf: &mut String) {
include_str!("welcome-page.html").to_string() buf.push_str(include_str!("welcome-page.html"));
} }
} }
@ -124,9 +145,9 @@ struct Post {
} }
impl Page for Post { impl Page for Post {
fn render(self, session: &Session) -> String { fn render_to_buf(self, session: &Session, buf: &mut String) {
let Self { name, code, error, .. } = self; let Self { name, code, error, .. } = self;
html! { write_html! { (buf)
<form id="postForm" "hx-post"="/post" "hx-swap"="outherHTML"> <form id="postForm" "hx-post"="/post" "hx-swap"="outherHTML">
if let Some(e) = error { <div class="error">e</div> } if let Some(e) = error { <div class="error">e</div> }
<input name="author" type="text" value={session.name} hidden> <input name="author" type="text" value={session.name} hidden>
@ -197,7 +218,7 @@ impl fmt::Display for Post {
struct Profile; struct Profile;
impl Page for Profile { impl Page for Profile {
fn render(self, session: &Session) -> String { fn render_to_buf(self, session: &Session, buf: &mut String) {
db::with(|db| { db::with(|db| {
let iter = db let iter = db
.get_user_posts .get_user_posts
@ -214,7 +235,7 @@ impl Page for Profile {
.into_iter() .into_iter()
.flatten() .flatten()
.filter_map(|p| p.inspect_err(|e| log::error!("{e}")).ok()); .filter_map(|p| p.inspect_err(|e| log::error!("{e}")).ok());
html! { write_html! { (buf)
for post in iter { for post in iter {
!{post} !{post}
} else { } else {
@ -235,9 +256,9 @@ struct Login {
} }
impl PublicPage for Login { impl PublicPage for Login {
fn render(self) -> String { fn render_to_buf(self, buf: &mut String) {
let Login { name, password, error } = self; let Login { name, password, error } = self;
html! { write_html! { (buf)
<form "hx-post"="/login" "hx-swap"="outherHTML"> <form "hx-post"="/login" "hx-swap"="outherHTML">
if let Some(e) = error { <div class="error">e</div> } if let Some(e) = error { <div class="error">e</div> }
<input name="name" type="text" autocomplete="name" placeholder="name" value=name <input name="name" type="text" autocomplete="name" placeholder="name" value=name
@ -306,9 +327,9 @@ struct Signup {
} }
impl PublicPage for Signup { impl PublicPage for Signup {
fn render(self) -> String { fn render_to_buf(self, buf: &mut String) {
let Signup { name, new_password, confirm_password, error } = self; let Signup { name, new_password, confirm_password, error } = self;
html! { write_html! { (buf)
<form "hx-post"="/signup" "hx-swap"="outherHTML"> <form "hx-post"="/signup" "hx-swap"="outherHTML">
if let Some(e) = error { <div class="error">e</div> } if let Some(e) = error { <div class="error">e</div> }
<input name="name" type="text" autocomplete="name" placeholder="name" value=name <input name="name" type="text" autocomplete="name" placeholder="name" value=name
@ -349,11 +370,11 @@ impl Signup {
} }
} }
async fn base(body: String, session: Option<Session>) -> Html<String> { async fn base(body: impl FnOnce(&mut String), session: Option<&Session>) -> Html<String> {
let username = session.map(|s| s.name); let username = session.map(|s| &s.name);
Html(htmlm::html! { Html(html! {
"<!DOCTIPE>" "<!DOCTYPE html>"
<html lang="en"> <html lang="en">
<head> <head>
<style>!{include_str!("index.css")}</style> <style>!{include_str!("index.css")}</style>
@ -377,7 +398,7 @@ async fn base(body: String, session: Option<Session>) -> Html<String> {
</section> </section>
</nav> </nav>
<section id="post-form"></section> <section id="post-form"></section>
<main>!{body}</main> <main>|f|{body(f)}</main>
</body> </body>
<script src="https://unpkg.com/htmx.org@2.0.3" integrity="sha384-0895/pl2MU10Hqc6jd4RvrthNlDiE9U1tWmX7WRESftEDRosgxNsQG/Ze9YMRzHq" crossorigin="anonymous"></script> <script src="https://unpkg.com/htmx.org@2.0.3" integrity="sha384-0895/pl2MU10Hqc6jd4RvrthNlDiE9U1tWmX7WRESftEDRosgxNsQG/Ze9YMRzHq" crossorigin="anonymous"></script>
<script>!{include_str!("index.js")}</script> <script>!{include_str!("index.js")}</script>

View file

@ -7,4 +7,4 @@ edition = "2021"
crate-type = ["cdylib"] crate-type = ["cdylib"]
[dependencies] [dependencies]
hblang = { version = "0.1.0", path = "../../hblang", default-features = false, features = ["no_log"] } hblang = { workspace = true, features = ["no_log"] }

View file

@ -120,7 +120,8 @@ unsafe extern "C" fn fmt() {
let arena = let arena =
hblang::parser::Arena::with_capacity(code.len() * hblang::parser::SOURCE_TO_AST_FACTOR); hblang::parser::Arena::with_capacity(code.len() * hblang::parser::SOURCE_TO_AST_FACTOR);
let mut ctx = ParserCtx::default(); let mut ctx = ParserCtx::default();
let exprs = hblang::parser::Parser::parse(&mut ctx, code, "source.hb", &|_, _| Ok(0), &arena); let exprs =
hblang::parser::Parser::parse(&mut ctx, code, "source.hb", &mut |_, _| Ok(0), &arena);
let mut f = Write(&mut OUTPUT[..]); let mut f = Write(&mut OUTPUT[..]);
hblang::fmt::fmt_file(exprs, code, &mut f).unwrap(); hblang::fmt::fmt_file(exprs, code, &mut f).unwrap();

View file

@ -1,5 +1,5 @@
[package] [package]
name = "hbjit" name = "wasm-hbc"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2021"

View file

@ -0,0 +1,14 @@
pub fn add(left: u64, right: u64) -> u64 {
left + right
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn it_works() {
let result = add(2, 2);
assert_eq!(result, 4);
}
}

View file

@ -1,3 +0,0 @@
fn main() {
println!("Hello, world!");
}

View file

@ -9,8 +9,8 @@ path = "src/main.rs"
[dependencies] [dependencies]
hashbrown = { version = "0.15.0", default-features = false, features = ["raw-entry"] } hashbrown = { version = "0.15.0", default-features = false, features = ["raw-entry"] }
hbbytecode = { version = "0.1.0", path = "../hbbytecode" } hbbytecode = { workspace = true, features = ["disasm"] }
hbvm = { path = "../hbvm", features = ["nightly"] } hbvm = { workspace = true, features = ["nightly"] }
log = { version = "0.4.22", features = ["release_max_level_error"] } log = { version = "0.4.22", features = ["release_max_level_error"] }
[dependencies.regalloc2] [dependencies.regalloc2]

View file

@ -449,7 +449,7 @@ pub mod test {
let len = crate::fmt::minify(&mut minned); let len = crate::fmt::minify(&mut minned);
minned.truncate(len); minned.truncate(len);
let ast = parser::Ast::new(ident, minned, &mut ParserCtx::default(), &|_, _| Ok(0)); let ast = parser::Ast::new(ident, minned, &mut ParserCtx::default(), &mut |_, _| Ok(0));
log::error!( log::error!(
"{} / {} = {} | {} / {} = {}", "{} / {} = {} | {} / {} = {}",
ast.mem.size(), ast.mem.size(),

View file

@ -188,13 +188,14 @@ impl<T> TaskQueueInner<T> {
} }
pub fn parse_from_fs(extra_threads: usize, root: &str) -> io::Result<Vec<Ast>> { pub fn parse_from_fs(extra_threads: usize, root: &str) -> io::Result<Vec<Ast>> {
fn resolve(path: &str, from: &str) -> Result<PathBuf, CantLoadFile> { fn resolve(path: &str, from: &str, tmp: &mut PathBuf) -> Result<PathBuf, CantLoadFile> {
let path = match Path::new(from).parent() { tmp.clear();
Some(parent) => PathBuf::from_iter([parent, Path::new(path)]), match Path::new(from).parent() {
None => PathBuf::from(path), Some(parent) => tmp.extend([parent, Path::new(path)]),
None => tmp.push(path),
}; };
path.canonicalize().map_err(|source| CantLoadFile { path, source }) tmp.canonicalize().map_err(|source| CantLoadFile { path: std::mem::take(tmp), source })
} }
#[derive(Debug)] #[derive(Debug)]
@ -227,7 +228,7 @@ pub fn parse_from_fs(extra_threads: usize, root: &str) -> io::Result<Vec<Ast>> {
let tasks = TaskQueue::<Task>::new(extra_threads + 1); let tasks = TaskQueue::<Task>::new(extra_threads + 1);
let ast = Mutex::new(Vec::<io::Result<Ast>>::new()); let ast = Mutex::new(Vec::<io::Result<Ast>>::new());
let loader = |path: &str, from: &str| { let loader = |path: &str, from: &str, tmp: &mut _| {
if path.starts_with("rel:") { if path.starts_with("rel:") {
return Err(io::Error::new( return Err(io::Error::new(
io::ErrorKind::Other, io::ErrorKind::Other,
@ -236,17 +237,17 @@ pub fn parse_from_fs(extra_threads: usize, root: &str) -> io::Result<Vec<Ast>> {
)); ));
} }
let physiscal_path = resolve(path, from)?; let mut physiscal_path = resolve(path, from, tmp)?;
let id = { let id = {
let mut seen = seen.lock().unwrap(); let mut seen = seen.lock().unwrap();
let len = seen.len(); let len = seen.len();
match seen.entry(physiscal_path.clone()) { match seen.entry(physiscal_path) {
hash_map::Entry::Occupied(entry) => { hash_map::Entry::Occupied(entry) => {
return Ok(*entry.get()); return Ok(*entry.get());
} }
hash_map::Entry::Vacant(entry) => { hash_map::Entry::Vacant(entry) => {
entry.insert(len as _); physiscal_path = entry.insert_entry(len as _).key().clone();
len as u32 len as u32
} }
} }
@ -263,22 +264,23 @@ pub fn parse_from_fs(extra_threads: usize, root: &str) -> io::Result<Vec<Ast>> {
Ok(id) Ok(id)
}; };
let execute_task = |ctx: &mut _, (_, path): Task| { let execute_task = |ctx: &mut _, (_, path): Task, tmp: &mut _| {
let path = path.to_str().ok_or_else(|| { let path = path.to_str().ok_or_else(|| {
io::Error::new( io::Error::new(
io::ErrorKind::InvalidData, io::ErrorKind::InvalidData,
format!("path contains invalid characters: {}", display_rel_path(&path)), format!("path contains invalid characters: {}", display_rel_path(&path)),
) )
})?; })?;
Ok(Ast::new(path, std::fs::read_to_string(path)?, ctx, &|path, from| { Ok(Ast::new(path, std::fs::read_to_string(path)?, ctx, &mut |path, from| {
loader(path, from).map_err(|e| e.to_string()) loader(path, from, tmp).map_err(|e| e.to_string())
})) }))
}; };
let thread = || { let thread = || {
let mut ctx = ParserCtx::default(); let mut ctx = ParserCtx::default();
let mut tmp = PathBuf::new();
while let Some(task @ (indx, ..)) = tasks.pop() { while let Some(task @ (indx, ..)) = tasks.pop() {
let res = execute_task(&mut ctx, task); let res = execute_task(&mut ctx, task, &mut tmp);
let mut ast = ast.lock().unwrap(); let mut ast = ast.lock().unwrap();
let len = ast.len().max(indx as usize + 1); let len = ast.len().max(indx as usize + 1);
ast.resize_with(len, || Err(io::ErrorKind::InvalidData.into())); ast.resize_with(len, || Err(io::ErrorKind::InvalidData.into()));
@ -286,7 +288,9 @@ pub fn parse_from_fs(extra_threads: usize, root: &str) -> io::Result<Vec<Ast>> {
} }
}; };
let path = Path::new(root).canonicalize()?; let path = Path::new(root).canonicalize().map_err(|e| {
io::Error::new(e.kind(), format!("can't canonicalize root file path ({root})"))
})?;
seen.lock().unwrap().insert(path.clone(), 0); seen.lock().unwrap().insert(path.clone(), 0);
tasks.push((0, path)); tasks.push((0, path));

View file

@ -1348,7 +1348,7 @@ fn test_parse_files(ident: &'static str, input: &'static str) -> Vec<parser::Ast
fmt::test::format(ident, input[last_start..].trim()); fmt::test::format(ident, input[last_start..].trim());
module_map.push((last_module_name, input[last_start..].trim())); module_map.push((last_module_name, input[last_start..].trim()));
let loader = |path: &str, _: &str| { let mut loader = |path: &str, _: &str| {
module_map module_map
.iter() .iter()
.position(|&(name, _)| name == path) .position(|&(name, _)| name == path)
@ -1359,7 +1359,7 @@ fn test_parse_files(ident: &'static str, input: &'static str) -> Vec<parser::Ast
let mut ctx = parser::ParserCtx::default(); let mut ctx = parser::ParserCtx::default();
module_map module_map
.iter() .iter()
.map(|&(path, content)| parser::Ast::new(path, content.to_owned(), &mut ctx, &loader)) .map(|&(path, content)| parser::Ast::new(path, content.to_owned(), &mut ctx, &mut loader))
.collect() .collect()
} }

View file

@ -23,7 +23,7 @@ pub type Symbols = Vec<Symbol>;
pub type FileId = u32; pub type FileId = u32;
pub type IdentIndex = u16; pub type IdentIndex = u16;
pub type LoaderError = String; pub type LoaderError = String;
pub type Loader<'a> = &'a (dyn Fn(&str, &str) -> Result<FileId, LoaderError> + 'a); pub type Loader<'a> = &'a mut (dyn FnMut(&str, &str) -> Result<FileId, LoaderError> + 'a);
pub const SOURCE_TO_AST_FACTOR: usize = 7 * (core::mem::size_of::<usize>() / 4) + 1; pub const SOURCE_TO_AST_FACTOR: usize = 7 * (core::mem::size_of::<usize>() / 4) + 1;
@ -1068,7 +1068,7 @@ impl Ast {
impl Default for Ast { impl Default for Ast {
fn default() -> Self { fn default() -> Self {
Self(AstInner::new("".into(), "", &mut ParserCtx::default(), &no_loader)) Self(AstInner::new("".into(), "", &mut ParserCtx::default(), &mut no_loader))
} }
} }

View file

@ -9,4 +9,4 @@ alloc = []
nightly = [] nightly = []
[dependencies] [dependencies]
hbbytecode = { path = "../hbbytecode", default-features = false } hbbytecode = { workspace = true }

Some files were not shown because too many files have changed in this diff Show more