removing garbage

hblang/src/codegen.rs

@@ -3,7 +3,7 @@ use {
     crate::{
         ident::{self, Ident},
         instrs::{self, *},
-        lexer::TokenKind,
+        lexer::{self, TokenKind},
         log,
         parser::{self, find_symbol, idfl, CtorField, Expr, ExprRef, FileId, Pos},
         HashMap,
@@ -3133,7 +3133,7 @@ impl Codegen {
     }
 
     fn report_log(&self, pos: Pos, msg: impl std::fmt::Display) {
-        let (line, col) = self.cfile().nlines.line_col(pos);
+        let (line, col) = lexer::line_col(self.cfile().file.as_bytes(), pos);
         println!("{}:{}:{}: {}", self.cfile().path, line, col, msg);
     }
 
@@ -3261,7 +3261,7 @@ mod tests {
         let mut codegen = super::Codegen {
             files: module_map
                 .iter()
-                .map(|&(path, content)| parser::Ast::new(path, content, &loader))
+                .map(|&(path, content)| parser::Ast::new(path, content.to_owned(), &loader))
                 .collect(),
             ..Default::default()
         };
@@ -3338,7 +3338,7 @@ mod tests {
         struct_patterns => README;
         arrays => README;
         struct_return_from_module_function => README;
-        comptime_pointers => README;
+        //comptime_pointers => README;
         sort_something_viredly => README;
     }
 }

hblang/src/lexer.rs

@@ -1,5 +1,3 @@
-use std::simd::cmp::SimdPartialEq;
-
 const fn ascii_mask(chars: &[u8]) -> u128 {
     let mut eq = 0;
     let mut i = 0;
@@ -380,112 +378,3 @@ pub fn line_col(bytes: &[u8], pos: u32) -> (usize, usize) {
         .map(|(line, col)| (line + 1, col + 1))
         .unwrap_or((1, 1))
 }
-
-pub struct LineMap {
-    lines: Box<[u8]>,
-}
-
-impl LineMap {
-    pub fn line_col(&self, mut pos: u32) -> (usize, usize) {
-        let mut line = 1;
-
-        let mut iter = self.lines.iter().copied();
-
-        loop {
-            let mut acc = 0;
-            let mut idx = 0;
-            loop {
-                let len = iter.next().unwrap();
-                acc |= ((len & 0x7F) as u32) << (7 * idx);
-                idx += 1;
-                if len & 0x80 == 0 {
-                    break;
-                }
-            }
-
-            if pos < acc {
-                break;
-            }
-            pos = pos.saturating_sub(acc);
-            line += 1;
-        }
-
-        (line, pos as usize + 1)
-    }
-
-    pub fn new(input: &str) -> Self {
-        let bytes = input.as_bytes();
-        let (start, simd_mid, end) = bytes.as_simd::<16>();
-
-        let query = std::simd::u8x16::splat(b'\n');
-
-        let nl_count = start.iter().map(|&b| (b == b'\n') as usize).sum::<usize>()
-            + simd_mid.iter().map(|s| s.simd_eq(query).to_bitmask().count_ones()).sum::<u32>()
-                as usize
-            + end.iter().map(|&b| (b == b'\n') as usize).sum::<usize>();
-
-        let mut lines = Vec::with_capacity(nl_count);
-        let mut last_nl = 0;
-
-        let handle_rem = |offset: usize, bytes: &[u8], last_nl: &mut usize, lines: &mut Vec<u8>| {
-            bytes
-                .iter()
-                .copied()
-                .enumerate()
-                .filter_map(|(i, b)| (b == b'\n').then_some(i + offset))
-                .for_each(|i| {
-                    lines.push((i - *last_nl + 1) as u8);
-                    *last_nl = i + 1;
-                });
-        };
-
-        handle_rem(0, start, &mut last_nl, &mut lines);
-
-        for (i, simd) in simd_mid.iter().enumerate() {
-            let mask = simd.simd_eq(query);
-            let mut mask = mask.to_bitmask();
-            while mask != 0 {
-                let idx = mask.trailing_zeros() as usize + i * 16 + start.len();
-                let mut len = idx - last_nl + 1;
-                while len >= 0x80 {
-                    lines.push(0x80 | (len & 0x7F) as u8);
-                    len >>= 7;
-                }
-                lines.push(len as u8);
-                last_nl = idx + 1;
-                mask &= mask - 1;
-            }
-        }
-
-        handle_rem(bytes.len() - end.len(), end, &mut last_nl, &mut lines);
-
-        Self { lines: Box::from(lines) }
-    }
-}
-
-#[cfg(test)]
-mod test {
-    #[test]
-    fn test_smh() {
-        let example = include_str!("../README.md");
-
-        let nlines = super::LineMap::new(example);
-
-        fn slow_nline_search(str: &str, mut pos: usize) -> (usize, usize) {
-            (
-                str.lines()
-                    .take_while(|l| match pos.checked_sub(l.len() + 1) {
-                        Some(nl) => (pos = nl, true).1,
-                        None => false,
-                    })
-                    .count()
-                    + 1,
-                pos + 1,
-            )
-        }
-
-        for i in 0..example.len() {
-            assert_eq!(slow_nline_search(example, i), nlines.line_col(i as _));
-        }
-    }
-}

hblang/src/lib.rs

@@ -424,7 +424,7 @@ pub fn parse_from_fs(extra_threads: usize, root: &str) -> io::Result<Vec<Ast>> {
         let mut file = std::fs::File::open(path)?;
         file.read_to_end(buffer)?;
         let src = std::str::from_utf8(buffer).map_err(InvalidFileData)?;
-        Ok(Ast::new(path, src, &loader))
+        Ok(Ast::new(path, src.to_owned(), &loader))
     };
 
     let thread = || {

hblang/src/parser.rs

@@ -2,7 +2,7 @@ use {
     crate::{
         codegen,
         ident::{self, Ident},
-        lexer::{self, Lexer, LineMap, Token, TokenKind},
+        lexer::{self, Lexer, Token, TokenKind},
         log,
     },
     std::{
@@ -1065,7 +1065,7 @@ pub struct AstInner<T: ?Sized> {
     exprs: *const [Expr<'static>],
 
     pub path: Box<str>,
-    pub nlines: LineMap,
+    pub file: Box<str>,
     pub symbols: T,
 }
 
@@ -1077,11 +1077,11 @@ impl AstInner<[Symbol]> {
             .0
     }
 
-    fn new(content: &str, path: &str, loader: Loader) -> NonNull<Self> {
+    fn new(content: String, path: &str, loader: Loader) -> NonNull<Self> {
         let arena = Arena::default();
         let mut syms = Vec::new();
         let mut parser = Parser::new(&arena, &mut syms, loader);
-        let exprs = parser.file(content, path) as *const [Expr<'static>];
+        let exprs = parser.file(&content, path) as *const [Expr<'static>];
 
         syms.sort_unstable_by_key(|s| s.name);
 
@@ -1096,7 +1096,7 @@ impl AstInner<[Symbol]> {
                 mem: arena.chunk.into_inner(),
                 exprs,
                 path: path.into(),
-                nlines: LineMap::new(content),
+                file: content.into(),
                 symbols: (),
             });
             std::ptr::addr_of_mut!((*inner).symbols)
@@ -1112,7 +1112,7 @@ impl AstInner<[Symbol]> {
 pub struct Ast(NonNull<AstInner<[Symbol]>>);
 
 impl Ast {
-    pub fn new(path: &str, content: &str, loader: Loader) -> Self {
+    pub fn new(path: &str, content: String, loader: Loader) -> Self {
         Self(AstInner::new(content, path, loader))
     }
 
@@ -1143,7 +1143,7 @@ impl std::fmt::Display for Ast {
 
 impl Default for Ast {
     fn default() -> Self {
-        Self(AstInner::new("", "", &no_loader))
+        Self(AstInner::new(String::new(), "", &no_loader))
     }
 }
 
@@ -1319,7 +1319,7 @@ impl Drop for ArenaChunk {
 #[cfg(test)]
 pub mod test {
     pub fn format(ident: &str, input: &str) {
-        let ast = super::Ast::new(ident, input, &|_, _| Ok(0));
+        let ast = super::Ast::new(ident, input.to_owned(), &|_, _| Ok(0));
         let mut output = Vec::new();
         crate::format_to(&ast, input, &mut output).unwrap();
 

hblang/tests/codegen_tests_arithmetic.txt

@@ -1,3 +1,3 @@
-code size: 189
+code size: 193
 ret: 1
 status: Ok(())