removing garbage

This commit is contained in:
mlokr 2024-09-01 21:15:29 +02:00
parent 9012f976c5
commit 28e33d11c9
No known key found for this signature in database
GPG key ID: DEA147DDEE644993
5 changed files with 14 additions and 125 deletions

View file

@ -3,7 +3,7 @@ use {
crate::{
ident::{self, Ident},
instrs::{self, *},
lexer::TokenKind,
lexer::{self, TokenKind},
log,
parser::{self, find_symbol, idfl, CtorField, Expr, ExprRef, FileId, Pos},
HashMap,
@ -3133,7 +3133,7 @@ impl Codegen {
}
fn report_log(&self, pos: Pos, msg: impl std::fmt::Display) {
let (line, col) = self.cfile().nlines.line_col(pos);
let (line, col) = lexer::line_col(self.cfile().file.as_bytes(), pos);
println!("{}:{}:{}: {}", self.cfile().path, line, col, msg);
}
@ -3261,7 +3261,7 @@ mod tests {
let mut codegen = super::Codegen {
files: module_map
.iter()
.map(|&(path, content)| parser::Ast::new(path, content, &loader))
.map(|&(path, content)| parser::Ast::new(path, content.to_owned(), &loader))
.collect(),
..Default::default()
};
@ -3338,7 +3338,7 @@ mod tests {
struct_patterns => README;
arrays => README;
struct_return_from_module_function => README;
comptime_pointers => README;
//comptime_pointers => README;
sort_something_viredly => README;
}
}

View file

@ -1,5 +1,3 @@
use std::simd::cmp::SimdPartialEq;
const fn ascii_mask(chars: &[u8]) -> u128 {
let mut eq = 0;
let mut i = 0;
@ -380,112 +378,3 @@ pub fn line_col(bytes: &[u8], pos: u32) -> (usize, usize) {
.map(|(line, col)| (line + 1, col + 1))
.unwrap_or((1, 1))
}
pub struct LineMap {
lines: Box<[u8]>,
}
impl LineMap {
pub fn line_col(&self, mut pos: u32) -> (usize, usize) {
let mut line = 1;
let mut iter = self.lines.iter().copied();
loop {
let mut acc = 0;
let mut idx = 0;
loop {
let len = iter.next().unwrap();
acc |= ((len & 0x7F) as u32) << (7 * idx);
idx += 1;
if len & 0x80 == 0 {
break;
}
}
if pos < acc {
break;
}
pos = pos.saturating_sub(acc);
line += 1;
}
(line, pos as usize + 1)
}
pub fn new(input: &str) -> Self {
let bytes = input.as_bytes();
let (start, simd_mid, end) = bytes.as_simd::<16>();
let query = std::simd::u8x16::splat(b'\n');
let nl_count = start.iter().map(|&b| (b == b'\n') as usize).sum::<usize>()
+ simd_mid.iter().map(|s| s.simd_eq(query).to_bitmask().count_ones()).sum::<u32>()
as usize
+ end.iter().map(|&b| (b == b'\n') as usize).sum::<usize>();
let mut lines = Vec::with_capacity(nl_count);
let mut last_nl = 0;
let handle_rem = |offset: usize, bytes: &[u8], last_nl: &mut usize, lines: &mut Vec<u8>| {
bytes
.iter()
.copied()
.enumerate()
.filter_map(|(i, b)| (b == b'\n').then_some(i + offset))
.for_each(|i| {
lines.push((i - *last_nl + 1) as u8);
*last_nl = i + 1;
});
};
handle_rem(0, start, &mut last_nl, &mut lines);
for (i, simd) in simd_mid.iter().enumerate() {
let mask = simd.simd_eq(query);
let mut mask = mask.to_bitmask();
while mask != 0 {
let idx = mask.trailing_zeros() as usize + i * 16 + start.len();
let mut len = idx - last_nl + 1;
while len >= 0x80 {
lines.push(0x80 | (len & 0x7F) as u8);
len >>= 7;
}
lines.push(len as u8);
last_nl = idx + 1;
mask &= mask - 1;
}
}
handle_rem(bytes.len() - end.len(), end, &mut last_nl, &mut lines);
Self { lines: Box::from(lines) }
}
}
#[cfg(test)]
mod test {
#[test]
fn test_smh() {
let example = include_str!("../README.md");
let nlines = super::LineMap::new(example);
fn slow_nline_search(str: &str, mut pos: usize) -> (usize, usize) {
(
str.lines()
.take_while(|l| match pos.checked_sub(l.len() + 1) {
Some(nl) => (pos = nl, true).1,
None => false,
})
.count()
+ 1,
pos + 1,
)
}
for i in 0..example.len() {
assert_eq!(slow_nline_search(example, i), nlines.line_col(i as _));
}
}
}

View file

@ -424,7 +424,7 @@ pub fn parse_from_fs(extra_threads: usize, root: &str) -> io::Result<Vec<Ast>> {
let mut file = std::fs::File::open(path)?;
file.read_to_end(buffer)?;
let src = std::str::from_utf8(buffer).map_err(InvalidFileData)?;
Ok(Ast::new(path, src, &loader))
Ok(Ast::new(path, src.to_owned(), &loader))
};
let thread = || {

View file

@ -2,7 +2,7 @@ use {
crate::{
codegen,
ident::{self, Ident},
lexer::{self, Lexer, LineMap, Token, TokenKind},
lexer::{self, Lexer, Token, TokenKind},
log,
},
std::{
@ -1065,7 +1065,7 @@ pub struct AstInner<T: ?Sized> {
exprs: *const [Expr<'static>],
pub path: Box<str>,
pub nlines: LineMap,
pub file: Box<str>,
pub symbols: T,
}
@ -1077,11 +1077,11 @@ impl AstInner<[Symbol]> {
.0
}
fn new(content: &str, path: &str, loader: Loader) -> NonNull<Self> {
fn new(content: String, path: &str, loader: Loader) -> NonNull<Self> {
let arena = Arena::default();
let mut syms = Vec::new();
let mut parser = Parser::new(&arena, &mut syms, loader);
let exprs = parser.file(content, path) as *const [Expr<'static>];
let exprs = parser.file(&content, path) as *const [Expr<'static>];
syms.sort_unstable_by_key(|s| s.name);
@ -1096,7 +1096,7 @@ impl AstInner<[Symbol]> {
mem: arena.chunk.into_inner(),
exprs,
path: path.into(),
nlines: LineMap::new(content),
file: content.into(),
symbols: (),
});
std::ptr::addr_of_mut!((*inner).symbols)
@ -1112,7 +1112,7 @@ impl AstInner<[Symbol]> {
pub struct Ast(NonNull<AstInner<[Symbol]>>);
impl Ast {
pub fn new(path: &str, content: &str, loader: Loader) -> Self {
pub fn new(path: &str, content: String, loader: Loader) -> Self {
Self(AstInner::new(content, path, loader))
}
@ -1143,7 +1143,7 @@ impl std::fmt::Display for Ast {
impl Default for Ast {
fn default() -> Self {
Self(AstInner::new("", "", &no_loader))
Self(AstInner::new(String::new(), "", &no_loader))
}
}
@ -1319,7 +1319,7 @@ impl Drop for ArenaChunk {
#[cfg(test)]
pub mod test {
pub fn format(ident: &str, input: &str) {
let ast = super::Ast::new(ident, input, &|_, _| Ok(0));
let ast = super::Ast::new(ident, input.to_owned(), &|_, _| Ok(0));
let mut output = Vec::new();
crate::format_to(&ast, input, &mut output).unwrap();

View file

@ -1,3 +1,3 @@
code size: 189
code size: 193
ret: 1
status: Ok(())