A bit of work on the parser, schema syntax, compilation/interpretation, and so on

This commit is contained in:
DOOM1EM8Cover 2023-06-27 03:15:54 +01:00
parent 9eebd7675c
commit 304e521971
21 changed files with 477 additions and 93 deletions

View file

@ -8,6 +8,14 @@ edition = "2021"
[dependencies]
anyhow = "1.0.71"
once_cell = "1.17.1"
panik-handler = "0.1.0"
pest = "2.6.0"
pest_derive = "2.6.0"
rmp = "0.8.11"
rmp-serde = "1.1.1"
serde = "1.0.164"
serde_derive = "1.0.164"
[dev-dependencies]
pretty_assertions = "1.3.0"

View file

@ -2,15 +2,6 @@ namespace examples.thing.simple
const AGREEMENT_KEY: string = "agreement_key"
@min_chars=0 max_chars=1024
validator StringBounds {
assert_error(
value.length > params.max_chars or value.length < params.min_chars,
"String {value.name} must be more than {params.min_chars} and less than {params.max_chars}"
)
}
@min_chars=16 max_chars=60
validator PasswordCheck {
assert_valid(

View file

@ -1,10 +1,11 @@
// Standard Uses
// Local Uses
use crate::ir::unit::{Protocol, Unit};
//use crate::ir::unit::{Protocol, Unit};
// External Uses
/*
fn unit_to_code(unit: Unit) -> String {
let mut code = String::new();
@ -73,3 +74,4 @@ fn protocol_to_code(protocol: Protocol) -> String {
code.to_owned()
}
*/

View file

@ -1,6 +1,8 @@
// IDL grammar
schema = _{
namespace
COMMENT* ~ MULTILINE_COMMENT*
~ WS?
~ namespace
~ (
COMMENT | import
| settings | constant
@ -25,13 +27,12 @@ constant = {
}
settings = {
WS ~ parameter*
~ "settings" ~ WS? ~ id? ~ WS?
WS? ~ "settings" ~ WS? ~ id? ~ WS?
~ "{" ~ WS? ~ parameter* ~ WS? ~ "}"
}
enumeration = {
WS ~ parameter*
WS? ~ property*
~ "enum" ~ WS ~ id ~ WS?
~ "{" ~ WS? ~ enum_variant+ ~ WS? ~ "}"
~ WS?
@ -41,7 +42,7 @@ enum_variant = {
}
validator = {
WS ~ parameter*
WS? ~ property*
~ "validator" ~ WS ~ id ~ WS?
~ "{" ~ WS? ~ function_call* ~ WS? ~ "}"
~ WS?
@ -77,14 +78,15 @@ boolean_operator = {
}
structure = {
WS? ~ parameter*
WS? ~ docstring? ~ property*
~ "struct" ~ WS ~ id ~ WS?
~ "{" ~ WS? ~ (constant | field)+ ~ WS? ~ "}"
~ WS?
}
field = {
WS? ~ property*
~ (index ~ "#")? ~ (WS? ~ requirement)?
// ~ (index ~ "#")?
~ (WS? ~ requirement)?
~ WS? ~ id ~ WS? ~ ":" ~ WS? ~ kind
~ (WS? ~ "=" ~ WS? ~ value)? ~ WS?
}
@ -92,7 +94,7 @@ index = @{ digit }
requirement = { "optional" }
error = {
WS? ~ property*
WS? ~ docstring? ~ property*
~ "error" ~ WS ~ id ~ WS?
~ "{" ~ WS?
~ (parameter | field)+
@ -100,19 +102,19 @@ error = {
}
protocol = {
WS? ~ property* ~ WS?
WS? ~ docstring? ~ property*
~ "protocol" ~ WS ~ id ~ WS?
~ "{" ~ WS? ~ function* ~ WS? ~ "}"
}
function = {
WS? ~ property* ~ WS?
WS? ~ property*
~ (index ~ WS? ~ "#" ~ WS?)?
~ (asynchronous ~ WS?)?
~ (direction ~ WS?)?
~ "function" ~ WS ~ id ~ WS?
~ "(" ~ WS? ~ argument* ~ WS? ~ ")"
~ (WS? ~ "->" ~ WS? ~ returns+)?
~ (WS? ~ ":" ~ WS? ~ parameter+)?
// ~ (WS? ~ ":" ~ WS? ~ parameter+)?
~ (WS? ~ "!" ~ WS? ~ throw+)?
}
direction = { "client" | "server" }
@ -167,8 +169,12 @@ instantiation = {
~ "(" ~ domain ~ ")"
}
docstring = {
"///" ~ (!NEWLINE ~ ANY)* ~ NEWLINE
}
value = {
"true" | "false"
"true" | "false" | digit
| string | string_interpolated
| instantiation
| domain | domain_namespaced
@ -197,5 +203,12 @@ alpha = { 'a'..'z' | 'A'..'Z' }
digit = { '0'..'9' }
WS = _{ (" " | "\t" | "\n")+ }
COMMENT = _{ "//" ~ (!NEWLINE ~ ANY)* ~ NEWLINE }
// MULTILINE_COMMENT = _{ "/*" ~ (MULTILINE_COMMENT | !"*/" ~ ANY)* ~ "*/" }
COMMENT = _{
!"///" ~
"//" ~ (!NEWLINE ~ ANY)* ~ NEWLINE
}
MULTILINE_COMMENT = _{
"/*"
~ (MULTILINE_COMMENT | !"*/" ~ ANY)*
~ "*/"
}

View file

@ -1,11 +1,13 @@
// Standard Uses
// Local Uses
use crate::ir::new_unit::Unit;
use crate::ir::new_unit::{Unit, UnitIndex, VUnit, WholeUnit};
use crate::ir::primitive;
use crate::ir::primitive::TypeValue;
// External Uses
use anyhow::Result;
use pest::iterators::Pair;
use pest::{iterators::Pair, Parser};
#[derive(Parser)]
@ -13,24 +15,205 @@ use pest::iterators::Pair;
pub struct IDLParser;
/*
pub fn parse_into_unit(content: &str) -> Result<Vec<Unit>> {
pub fn from_path(path: &str) -> Result<WholeUnit> {
let raw = std::fs::read_to_string(path).unwrap();
let vunit = parse_into_unit(raw.clone().as_str()).unwrap();
let index = UnitIndex::Index { path: path.to_string(), source: raw };
Ok((index, vunit))
}
pub fn parse_into_unit(content: &str) -> Result<VUnit> {
let pairs = IDLParser::parse(Rule::schema, content)?;
let mut units = vec![];
let mut unit = vec![];
for pair in pairs { units.push(parse_inner(pair).unwrap()) }
for pair in pairs { unit.push(parse_inner(pair).unwrap()) }
Ok(units)
Ok(unit)
}
#[allow(unused)]
pub fn parse_inner(pairs: Pair<Rule>) -> Result<Unit> {
match pairs.as_rule() {
Rule::schema {
/*
Rule::schema => {
todo!()
},
*/
Rule::namespace => {
Ok(Unit::Namespace(pairs.into_inner().as_str().to_owned()))
},
Rule::import => {
Ok(Unit::Import(pairs.into_inner().as_str().to_owned()))
}
Rule::constant => {
let pairs = pairs.into_inner();
let mut docstring: Option<String> = None;
let mut id: Option<String> = None;
let mut kind: Option<TypeValue> = None;
let mut default_value: Option<String> = None;
for pair in pairs {
match pair.as_rule() {
Rule::docstring => docstring = Some(pair.as_str().to_owned()),
Rule::id => id = Some(pair.as_str().to_owned()),
Rule::kind => kind = Some(primitive::to_type(pair.as_str())),
Rule::value => default_value = to_value_other(pair),
(r) => panic!("Rule not implemented on 'constant': {:?}", r)
}
}
Ok(Unit::Constant {
docstring,
name: id.ok_or("Id is not present").unwrap(),
kind: kind.ok_or("Type is not present").unwrap(),
default_value,
})
},
Rule::settings => {
let mut pairs = pairs.into_inner();
let mut docstring: Option<String> = None;
let mut name: Option<String> = None;
let mut parameters: Vec<Unit> = vec![];
for pair in pairs {
match pair.as_rule() {
Rule::docstring => docstring = Some(pair.as_str().to_owned()),
Rule::id => name = Some(pair.as_str().to_owned()),
Rule::parameter => parameters.push(to_parameter(pair)),
(r) => panic!("Rule not implemented on 'settings': {:?}", r)
}
}
Ok(Unit::Settings {
docstring,
name: name.unwrap(),
parameters,
})
},
Rule::enumeration => {
let mut pairs = pairs.into_inner();
let mut docstring: Option<String> = None;
let mut name: Option<String> = None;
let mut variants: Vec<Unit> = vec![];
for pair in pairs {
match pair.as_rule() {
Rule::docstring => docstring = Some(pair.as_str().to_owned()),
Rule::id => name = Some(pair.as_str().to_owned()),
Rule::enum_variant => {
let mut inner = pair.into_inner();
let name = inner.next().unwrap().as_str().to_string();
let kind = inner.next().map(|s| s.as_str().to_string());
variants.push(Unit::EnumVariant {
name, kind,
});
},
(r) => panic!("Rule not implemented on 'enumeration': {:?}", r)
}
}
Ok(Unit::Enum {
docstring, name: name.unwrap(),
variants,
})
},
Rule::structure => {
let mut pairs = pairs.into_inner();
let mut docstring: Option<String> = None;
let mut parameters: Vec<Unit> = vec![];
let mut name: Option<String> = None;
let mut fields: Vec<Unit> = vec![];
for pair in pairs {
match pair.as_rule() {
Rule::docstring => docstring = Some(pair.as_str().to_owned()),
Rule::id => name = Some(pair.as_str().to_owned()),
Rule::parameter => parameters.push(to_parameter(pair)),
Rule::field => fields.push(to_field(pair)),
(r) => panic!("Rule not implemented on 'structure': {:?}", r)
}
}
Ok(Unit::Struct {
docstring, parameters,
name: name.unwrap(), fields: vec![],
})
}
(r) => panic!("Rule not implemented: {:?}", r)
}
}
/*
pub fn to_parameters(mut pairs: Pairs<Rule>) -> Vec<Unit> {
let mut params = vec![];
let t = pairs.as_str();
while let Some(pair) = pairs.next() {
let temp = pair.as_str();
params.push(to_parameter(pair.into_inner()));
}
params
}
*/
pub fn to_parameter(mut pair: Pair<Rule>) -> Unit {
let mut pairs = pair.into_inner();
Unit::Parameter {
name: pairs.next().unwrap().as_str().to_string(),
default_value: pairs.next().unwrap().as_str().to_string()
}
}
pub fn to_value_other(pair: Pair<Rule>) -> Option<String> {
let inner = pair.into_inner().next().unwrap();
match inner.as_rule() {
Rule::string => Some(inner.as_str().to_string()),
Rule::string_interpolated => Some(inner.as_str().to_string()),
Rule::digit => Some(inner.as_str().to_string()),
r => panic!("Rule not implemented in 'value': {:?}", r)
}
// "".to_string()
}
pub fn to_field(pair: Pair<Rule>) -> Unit {
let mut pairs = pair.into_inner();
let mut docstring: Option<String> = None;
let mut parameters: Vec<Unit> = vec![];
let mut optional: bool = false;
let mut name: Option<String> = None;
let mut kind: Option<String> = None;
let mut default_value: Option<String> = None;
for pair in pairs {
match pair.as_rule() {
Rule::docstring => docstring = Some(pair.as_str().to_owned()),
Rule::parameter => parameters.push(to_parameter(pair)),
Rule::requirement => optional = true,
Rule::id => name = Some(pair.as_str().to_owned()),
Rule::kind => kind = Some(pair.as_str().to_owned()),
Rule::value => default_value = Some(pair.as_str().to_owned()),
r => panic!("Rule not implemented in 'field': {:?}", r)
}
}
Unit::Field {
docstring, parameters,
optional,
name: name.unwrap(), kind: kind.unwrap(), default_value,
}
}

View file

@ -3,6 +3,7 @@
// Local Uses
use crate::ir::primitive;
use crate::ir::primitive::TypeValue;
use crate::ir::unit::{
Argument, Const, Direction, Enum, EnumVariant,
Error, Field, Function, Parameter,
@ -19,7 +20,7 @@ use pest::Parser;
#[grammar = "idl/idl.pest"]
pub struct IDLParser;
/*
pub fn parse_into_unit(raw: &str) -> Result<Unit> {
let parse = IDLParser::parse(Rule::schema, raw)?;
@ -173,9 +174,9 @@ fn parse_inner(record: Pair<Rule>) -> Result<Unit> {
inner
}
*/
fn to_parameters(mut pairs: Pairs<Rule>) -> Vec<Parameter> {
pub fn to_parameters(mut pairs: Pairs<Rule>) -> Vec<Parameter> {
let mut params = vec![];
while let Some(pair) = pairs.next() {
@ -185,7 +186,7 @@ fn to_parameters(mut pairs: Pairs<Rule>) -> Vec<Parameter> {
params
}
fn to_parameter(mut pairs: Pairs<Rule>) -> Parameter {
pub fn to_parameter(mut pairs: Pairs<Rule>) -> Parameter {
Parameter {
id: pairs.next().unwrap().as_str().to_owned(),
value: pairs.next().unwrap().as_str().as_bytes().to_vec()

View file

@ -1,6 +1,6 @@
// Relative Modules
mod sanitizer;
mod context;
pub mod context;
mod serialization;
pub mod unit;
pub mod new_unit;

View file

@ -1,7 +1,43 @@
// Standard Uses
// Local Uses
use crate::ir::context::Context;
use crate::ir::new_unit::{Unit, VUnit};
// External Uses
use anyhow::{Result, bail};
pub fn interpret_unit(context: Context) -> Result<VUnit> {
let interpreted = vec![];
for unit in &context.main.1 {
match unit {
// Unit::Tag(_) => {}
Unit::Namespace(n) => {}
Unit::Import(i) => {
let relative_unit = context.find_unit(i)?;
bail!(format!("Could not find in namespace {}", i))
}
/*
Unit::Constant { .. } => {}
Unit::Enum { .. } => {}
Unit::EnumVariant { .. } => {}
Unit::Property { .. } => {}
Unit::Parameter { .. } => {}
Unit::Settings { .. } => {}
Unit::Struct { .. } => {}
Unit::Protocol { .. } => {}
Unit::Function { .. } => {}
Unit::Error { .. } => {}
Unit::Validator { .. } => {}
Unit::Field { .. } => {}
*/
r => panic!("Left to impl: {:?}", r)
}
}
Ok(interpreted)
}

View file

@ -3,22 +3,34 @@
// Local Uses
use crate::ir::unit::Unit;
use crate::ir::serialization::Origin;
use crate::ir::new_unit::{UnitIndex, VUnit, WholeUnit};
// External Uses
use anyhow::{Result, bail};
pub struct Context {
pub units: Vec<(Origin, Unit)>
pub main: WholeUnit,
pub relative_units: Vec<WholeUnit>
}
#[allow(unused)]
impl Context {
fn add_unit(self, origin: Origin, unit: Unit) {
pub fn with_main(main: WholeUnit) -> Self{
Context { main, relative_units: vec![] }
}
pub(crate) fn add_relative_unit(mut self, whole_unit: WholeUnit) {
self.relative_units.push(whole_unit)
}
pub(crate) fn sanitize_units(self) {
todo!()
}
fn sanitize_units(self) {
pub(crate) fn find_unit(&self, import: &str) -> Result<&WholeUnit> {
todo!()
}
}

View file

@ -1,19 +1,20 @@
// Standard Uses
// Local Uses
use crate::ir::primitive::{Primitive, TypeValue};
use crate::ir::primitive::TypeValue;
// External Uses
pub type OrderIndex = u16;
#[derive(Debug, Eq, PartialEq)]
#[derive(Debug, Eq, PartialEq, Deserialize, Serialize)]
pub enum Direction { Client, Server, Both }
#[derive(Debug, Eq, PartialEq)]
#[derive(Debug, Eq, PartialEq, Deserialize, Serialize)]
pub enum Unit {
/*
Schema {
tag: String,
namespace: String,
@ -26,34 +27,53 @@ pub enum Unit {
errors: Vec<Unit>,
protocols: Vec<Unit>
},
*/
Tag(String),
Namespace(String),
Import(String),
Constant {
docstring: Option<String>,
name: String,
kind: String,
default_value: String,
kind: TypeValue, // String
default_value: Option<String>,
},
Enum {
docstring: Option<String>,
name: String,
variants: Vec<Unit>
},
EnumVariant {
name: String,
kind: Option<String>
},
Property {
name: String,
values: Vec<String>
},
Parameter {
name: String,
default_value: String
},
Settings {
docstring: Option<String>,
name: String,
parameters: Vec<Unit>,
},
Struct {
name: String,
docstring: Option<String>,
parameters: Vec<Unit>,
functions: Vec<Unit>
name: String,
fields: Vec<Unit>,
},
Protocol {
name: String,
docstring: String,
parameters: Vec<Unit>,
name: String,
functions: Vec<Unit>
},
Function {
index: OrderIndex,
docstring: String,
// index: OrderIndex,
name: String,
synchronous: bool,
direction: Direction,
@ -62,17 +82,40 @@ pub enum Unit {
throws: Vec<Unit>
},
Error {
name: String,
docstring: String,
parameters: Vec<Unit>,
name: String,
fields: Vec<Unit>
},
Validator {
docstring: String,
parameters: Vec<Unit>,
name: String,
message: String,
},
Field {
index: OrderIndex,
docstring: Option<String>,
parameters: Vec<Unit>,
// index: OrderIndex,
optional: bool,
name: String,
kind: String,
default_value: Option<String>,
}
}
#[derive(PartialEq, Debug)]
pub enum UnitIndex {
Source(String),
Index {
path: String,
source: String,
// nodes: Vec<Unit>
},
Node {
index: u32,
start_position: u32, length: u32
}
}
pub type VUnit = Vec<Unit>;
pub type WholeUnit = (UnitIndex, VUnit);

View file

@ -4,7 +4,6 @@
// External Uses
#[derive(Debug, Eq, PartialEq)]
#[repr(u8)]
pub enum Primitive {
@ -27,7 +26,7 @@ pub enum Primitive {
Namespaced(String)
}
#[derive(Debug, Eq, PartialEq)]
#[derive(Debug, Eq, PartialEq, Deserialize, Serialize)]
#[repr(u8)]
pub enum TypeValue {
U8 = 0, U16, U32, U64, U128,
@ -93,4 +92,3 @@ fn to_value(type_: &TypeValue, value: &str) -> Vec<u8> {
TypeValue::Namespaced => Vec::from([value.parse::<u8>().unwrap()]),
}
}

View file

@ -8,7 +8,5 @@ use crate::ir::context::Context;
#[allow(unused)]
fn sanitize_context_units(context: &Context) {
for (_, unit) in &context.units {
}
todo!()
}

View file

@ -2,8 +2,13 @@
extern crate pest;
#[macro_use]
extern crate pest_derive;
extern crate serde;
#[macro_use]
extern crate serde_derive;
extern crate rmp_serde as rmps;
// Relative Modules
pub mod ir;
pub mod codegen;
pub mod idl;
pub mod stdlib;

14
src/stdlib/lang_items.rs Normal file
View file

@ -0,0 +1,14 @@
// Standard Uses
// Local Uses
use crate::ir::new_unit::Unit;
use crate::idl::parser::from_path;
// External Uses
/*
pub const LANGUAGE_ITEMS: [Vec<Unit>] = *[
from_path("src/stdlib/validators/string_bounds.ids").unwrap()
];
*/

10
src/stdlib/mod.rs Normal file
View file

@ -0,0 +1,10 @@
// Relative Modules
pub mod lang_items;
pub mod validators;
// Standard Uses
// Local Uses
// External Uses

View file

View file

@ -0,0 +1,12 @@
namespace std::validators::StringBounds
/// Checks if a string length is between a minimum and a maximum
/// @min_chars: Minimum length of the string
/// @max_chars: Maximum length of the string
@min_chars=0 max_chars=1024
validator StringBounds {
assert_equal(
value.length > params.max_chars or value.length < params.min_chars,
"String {value.name} must be more than {params.min_chars} and less than {params.max_chars}"
)
}

View file

@ -1 +1 @@
mod pest;
mod unit;

View file

@ -1,26 +0,0 @@
// Standard Uses
// Local Uses
// External Uses
use comline::idl;
use comline::idl::constants::SCHEMA_EXTENSION;
use comline::ir::unit::Unit;
#[test]
fn from_raw_to_unit() {
let raw = std::fs::read_to_string(
format!("tests/idl/simple.{}", SCHEMA_EXTENSION)
).unwrap();
let unit = idl::pest::parse_into_unit(raw.as_str()).unwrap();
assert_eq!(
unit, Unit::Items(vec![
Unit::Namespace("".to_string()),
]
)
)
}

View file

@ -1,27 +1,39 @@
// Simple Schema
namespace tests.idl.simple
const POWER: u8 = 10
const DEFAULT_NAME: str = f"flower {POWER}"
import std::validators::StringBounds
const POWER: u8 = 1
const DEFAULT_NAME: str = f"flower power: {POWER}"
settings Test {
forbid_indexing=True
forbid_optional_indexing=True
}
enum EncryptionMode {
None
Encrypt
}
/// A message that can be sent through the mail protocol
struct Message {
1# name: str = DEFAULT_NAME
2# encryption_mode: EncryptionMode = default
@validators=[StringBounds(min_chars=3 max_chars=12)]
2# optional recipient: str = "bee"
}
/// Throw when sending a message to a missing recipient
error RecipientNotFoundError {
message = f"Recipient with name {self.recipient} not found"
message = "Recipient with name {self.recipient} not found"
@validators=[StringBounds(min_chars=8 max_chars=16)]
1# recipient: str
}
/// Mail API for receiving and sending emails
@provider=Any
protocol Mail {

72
tests/idl/unit.rs Normal file
View file

@ -0,0 +1,72 @@
// Standard Uses
// Local Uses
// External Uses
use comline::{idl, ir};
use comline::idl::constants::SCHEMA_EXTENSION;
use comline::ir::new_unit::Unit;
use panik_handler;
use pretty_assertions::{assert_eq, assert_ne};
use comline::ir::compiler;
use comline::ir::primitive::TypeValue;
#[test]
fn from_raw_to_unit() {
panik_handler::install();
let raw = std::fs::read_to_string(
format!("tests/idl/simple.{}", SCHEMA_EXTENSION)
).unwrap();
let unit = idl::parser::parse_into_unit(raw.as_str()).unwrap();
assert_eq!(
unit, vec![
Unit::Namespace("tests.idl.simple".to_string()),
Unit::Import("std::validators::StringBounds".to_string()),
Unit::Constant {
docstring: None, name: "POWER".to_string(),
kind: TypeValue::U8, default_value: Some("1".to_string()),
},
Unit::Constant {
docstring: None, name: "DEFAULT_NAME".to_string(),
kind: TypeValue::String, default_value: Some("f\"flower power: {POWER}\"".to_string()),
},
Unit::Settings {
docstring: None, name: "Test".to_string(),
parameters: vec![
Unit::Parameter {
name: "forbid_indexing".to_string(),
default_value: "True".to_string(),
},
Unit::Parameter {
name: "forbid_optional_indexing".to_string(),
default_value: "True".to_string(),
}
],
},
Unit::Enum {
docstring: None, name: "EncryptionMode".to_string(),
variants: vec![
Unit::EnumVariant { name: "None".to_string(), kind: None },
Unit::EnumVariant { name: "Encrypt".to_string(), kind: None }
]
}
]
)
}
#[test]
fn compile_unit() {
let unit = idl::parser::from_path(
format!("tests/idl/simple.{}", SCHEMA_EXTENSION).as_str()
).unwrap();
let context = ir::context::Context::with_main(unit);
let interpreted = compiler::interpret_unit(context);
println!("{:?}", interpreted);
}