From 28e33d11c9200b2be70db256f655ff29bbe3b55b Mon Sep 17 00:00:00 2001 From: mlokr Date: Sun, 1 Sep 2024 21:15:29 +0200 Subject: [PATCH] removing garbage --- hblang/src/codegen.rs | 8 +- hblang/src/lexer.rs | 111 ---------------------- hblang/src/lib.rs | 2 +- hblang/src/parser.rs | 16 ++-- hblang/tests/codegen_tests_arithmetic.txt | 2 +- 5 files changed, 14 insertions(+), 125 deletions(-) diff --git a/hblang/src/codegen.rs b/hblang/src/codegen.rs index ed556b9..1eef208 100644 --- a/hblang/src/codegen.rs +++ b/hblang/src/codegen.rs @@ -3,7 +3,7 @@ use { crate::{ ident::{self, Ident}, instrs::{self, *}, - lexer::TokenKind, + lexer::{self, TokenKind}, log, parser::{self, find_symbol, idfl, CtorField, Expr, ExprRef, FileId, Pos}, HashMap, @@ -3133,7 +3133,7 @@ impl Codegen { } fn report_log(&self, pos: Pos, msg: impl std::fmt::Display) { - let (line, col) = self.cfile().nlines.line_col(pos); + let (line, col) = lexer::line_col(self.cfile().file.as_bytes(), pos); println!("{}:{}:{}: {}", self.cfile().path, line, col, msg); } @@ -3261,7 +3261,7 @@ mod tests { let mut codegen = super::Codegen { files: module_map .iter() - .map(|&(path, content)| parser::Ast::new(path, content, &loader)) + .map(|&(path, content)| parser::Ast::new(path, content.to_owned(), &loader)) .collect(), ..Default::default() }; @@ -3338,7 +3338,7 @@ mod tests { struct_patterns => README; arrays => README; struct_return_from_module_function => README; - comptime_pointers => README; + //comptime_pointers => README; sort_something_viredly => README; } } diff --git a/hblang/src/lexer.rs b/hblang/src/lexer.rs index b103860..6c7e671 100644 --- a/hblang/src/lexer.rs +++ b/hblang/src/lexer.rs @@ -1,5 +1,3 @@ -use std::simd::cmp::SimdPartialEq; - const fn ascii_mask(chars: &[u8]) -> u128 { let mut eq = 0; let mut i = 0; @@ -380,112 +378,3 @@ pub fn line_col(bytes: &[u8], pos: u32) -> (usize, usize) { .map(|(line, col)| (line + 1, col + 1)) .unwrap_or((1, 1)) } - -pub struct LineMap { - lines: Box<[u8]>, -} - -impl LineMap { - pub fn line_col(&self, mut pos: u32) -> (usize, usize) { - let mut line = 1; - - let mut iter = self.lines.iter().copied(); - - loop { - let mut acc = 0; - let mut idx = 0; - loop { - let len = iter.next().unwrap(); - acc |= ((len & 0x7F) as u32) << (7 * idx); - idx += 1; - if len & 0x80 == 0 { - break; - } - } - - if pos < acc { - break; - } - pos = pos.saturating_sub(acc); - line += 1; - } - - (line, pos as usize + 1) - } - - pub fn new(input: &str) -> Self { - let bytes = input.as_bytes(); - let (start, simd_mid, end) = bytes.as_simd::<16>(); - - let query = std::simd::u8x16::splat(b'\n'); - - let nl_count = start.iter().map(|&b| (b == b'\n') as usize).sum::() - + simd_mid.iter().map(|s| s.simd_eq(query).to_bitmask().count_ones()).sum::() - as usize - + end.iter().map(|&b| (b == b'\n') as usize).sum::(); - - let mut lines = Vec::with_capacity(nl_count); - let mut last_nl = 0; - - let handle_rem = |offset: usize, bytes: &[u8], last_nl: &mut usize, lines: &mut Vec| { - bytes - .iter() - .copied() - .enumerate() - .filter_map(|(i, b)| (b == b'\n').then_some(i + offset)) - .for_each(|i| { - lines.push((i - *last_nl + 1) as u8); - *last_nl = i + 1; - }); - }; - - handle_rem(0, start, &mut last_nl, &mut lines); - - for (i, simd) in simd_mid.iter().enumerate() { - let mask = simd.simd_eq(query); - let mut mask = mask.to_bitmask(); - while mask != 0 { - let idx = mask.trailing_zeros() as usize + i * 16 + start.len(); - let mut len = idx - last_nl + 1; - while len >= 0x80 { - lines.push(0x80 | (len & 0x7F) as u8); - len >>= 7; - } - lines.push(len as u8); - last_nl = idx + 1; - mask &= mask - 1; - } - } - - handle_rem(bytes.len() - end.len(), end, &mut last_nl, &mut lines); - - Self { lines: Box::from(lines) } - } -} - -#[cfg(test)] -mod test { - #[test] - fn test_smh() { - let example = include_str!("../README.md"); - - let nlines = super::LineMap::new(example); - - fn slow_nline_search(str: &str, mut pos: usize) -> (usize, usize) { - ( - str.lines() - .take_while(|l| match pos.checked_sub(l.len() + 1) { - Some(nl) => (pos = nl, true).1, - None => false, - }) - .count() - + 1, - pos + 1, - ) - } - - for i in 0..example.len() { - assert_eq!(slow_nline_search(example, i), nlines.line_col(i as _)); - } - } -} diff --git a/hblang/src/lib.rs b/hblang/src/lib.rs index 7059548..9ce7957 100644 --- a/hblang/src/lib.rs +++ b/hblang/src/lib.rs @@ -424,7 +424,7 @@ pub fn parse_from_fs(extra_threads: usize, root: &str) -> io::Result> { let mut file = std::fs::File::open(path)?; file.read_to_end(buffer)?; let src = std::str::from_utf8(buffer).map_err(InvalidFileData)?; - Ok(Ast::new(path, src, &loader)) + Ok(Ast::new(path, src.to_owned(), &loader)) }; let thread = || { diff --git a/hblang/src/parser.rs b/hblang/src/parser.rs index 59fe5f8..5611079 100644 --- a/hblang/src/parser.rs +++ b/hblang/src/parser.rs @@ -2,7 +2,7 @@ use { crate::{ codegen, ident::{self, Ident}, - lexer::{self, Lexer, LineMap, Token, TokenKind}, + lexer::{self, Lexer, Token, TokenKind}, log, }, std::{ @@ -1065,7 +1065,7 @@ pub struct AstInner { exprs: *const [Expr<'static>], pub path: Box, - pub nlines: LineMap, + pub file: Box, pub symbols: T, } @@ -1077,11 +1077,11 @@ impl AstInner<[Symbol]> { .0 } - fn new(content: &str, path: &str, loader: Loader) -> NonNull { + fn new(content: String, path: &str, loader: Loader) -> NonNull { let arena = Arena::default(); let mut syms = Vec::new(); let mut parser = Parser::new(&arena, &mut syms, loader); - let exprs = parser.file(content, path) as *const [Expr<'static>]; + let exprs = parser.file(&content, path) as *const [Expr<'static>]; syms.sort_unstable_by_key(|s| s.name); @@ -1096,7 +1096,7 @@ impl AstInner<[Symbol]> { mem: arena.chunk.into_inner(), exprs, path: path.into(), - nlines: LineMap::new(content), + file: content.into(), symbols: (), }); std::ptr::addr_of_mut!((*inner).symbols) @@ -1112,7 +1112,7 @@ impl AstInner<[Symbol]> { pub struct Ast(NonNull>); impl Ast { - pub fn new(path: &str, content: &str, loader: Loader) -> Self { + pub fn new(path: &str, content: String, loader: Loader) -> Self { Self(AstInner::new(content, path, loader)) } @@ -1143,7 +1143,7 @@ impl std::fmt::Display for Ast { impl Default for Ast { fn default() -> Self { - Self(AstInner::new("", "", &no_loader)) + Self(AstInner::new(String::new(), "", &no_loader)) } } @@ -1319,7 +1319,7 @@ impl Drop for ArenaChunk { #[cfg(test)] pub mod test { pub fn format(ident: &str, input: &str) { - let ast = super::Ast::new(ident, input, &|_, _| Ok(0)); + let ast = super::Ast::new(ident, input.to_owned(), &|_, _| Ok(0)); let mut output = Vec::new(); crate::format_to(&ast, input, &mut output).unwrap(); diff --git a/hblang/tests/codegen_tests_arithmetic.txt b/hblang/tests/codegen_tests_arithmetic.txt index de488f4..8773e5c 100644 --- a/hblang/tests/codegen_tests_arithmetic.txt +++ b/hblang/tests/codegen_tests_arithmetic.txt @@ -1,3 +1,3 @@ -code size: 189 +code size: 193 ret: 1 status: Ok(())