fix: fix string being splitted

README.md

@@ -2,20 +2,19 @@
 another lisp dialect
 
 ```lisp
-(fun factorial [x]
+(fun factorial (x)
     (if (<= x 1)
         1
         (* x (factorial (- x 1)))))
 
-(def times 7)
 (do
-    (print (factorial times)))
+    (print (factorial 7)))
 ```
 
 Compliation flow:
 ```
-Input(file) -> Lexer -> Parser -> Interpret
-  String       Token     Expr         IO
+Input(file) -> Parser -> Interpret(TODO)
+  String       SExprs          IO
                      |->  Compile(TODO)
                               File 
 ```
@@ -23,9 +22,8 @@ Input(file) -> Lexer -> Parser -> Interpret
 Progress:
 - [X] Lexer & Parser
 - [ ] Syntax checker & Type checker
-- [X] Interpreter
+- [ ] Interpreter
 - [ ] Compiler
 
 Problems:
-- Parser only detect the first error.
 - Parser can't detect `(()))` syntax error.

example/hello.vl

@@ -1,2 +1,2 @@
 (print "hi")
-(print "hello")
+(print "Hello, World!")

src/parser.rs

@@ -86,9 +86,11 @@ impl Parser {
 }
 
 pub fn tokenize(str: &str) -> Vec<String> {
-    str.replace("(", " ( ")
-        .replace(")", " ) ")
-        .split_whitespace()
-        .map(|s| s.to_string())
-        .collect()
+    let regex = Regex::new(r###"[\s,]*(~@|[\[\]{}()'`~^@]|"(?:\\.|[^\\"])*"?|;.*|[^\s\[\]{}('"`,;)]+)"###).unwrap();
+    let mut res = vec![];
+    for cap in regex.captures_iter(str) {
+        if cap[1].starts_with(";") { continue; }
+        res.push(String::from(&cap[1]));
+    }
+    res
 }