Deukhoofd
/
SeraphScript
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Loads of additional parsing work

This commit is contained in:
Deukhoofd 2022-04-02 23:30:05 +02:00
parent a1663ff032
commit 75f8720c54
Signed by: Deukhoofd
GPG Key ID: F63E044490819F6F
12 changed files with 1101 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
grammar.ebnf
View File

@ -48,7 +48,7 @@ exprpreop ::= '-' | '+' | '!' | '++' | '--' | '~' | '@';
arglist ::= '(' [identifier ':'] assign {',' [identifier ':'] assign} ')';
funccall ::= scope identifier arglist;
constructcall ::= type arglist;
varaccess ::= scope | identifier;
varaccess ::= scope identifier;
cast ::= 'cast' '<' type '>' '(' assign ')';
literal ::= number | string | 'true' | 'false' | 'null';
typemod ::= ['&' ['in' | 'out' | 'inout']];

2
src/defines.rs
View File

@ -2,3 +2,5 @@
pub type LiteralInt = i64;
/// The size floating point numbers use internally to store literals and do compile time calculations.
pub type LiteralFloat = f64;
pub type PointerSize = usize;

5
src/lib.rs
View File

@ -6,8 +6,11 @@
#![feature(iter_advance_by)]
#![feature(backtrace)]
pub(crate) mod defines;
extern crate core;
pub mod defines;
pub mod logger;
pub mod modifiers;
pub mod parsing;
pub(crate) mod prim_type;
pub mod span;

1
src/parsing/lexer/lex_tokens.rs
View File

@ -19,6 +19,7 @@ pub enum TokenType {
Colon,
ColonColon,
Comma,
Dot,
OpenBracket,
CloseBracket,
OpenCurlyBracket,

1
src/parsing/lexer/mod.rs
View File

@ -449,6 +449,7 @@ pub fn lex(s: &str, log: &mut dyn FnMut(Message, Span)) -> Vec<LexToken> {
'[' => lex_and_consume(&mut chars, TT::OpenBlockBracket, &mut add_token),
']' => lex_and_consume(&mut chars, TT::CloseBlockBracket, &mut add_token),
',' => lex_and_consume(&mut chars, TT::Comma, &mut add_token),
'.' => lex_and_consume(&mut chars, TT::Dot, &mut add_token),
'0'..'9' => lex_numeric(&mut chars, &mut add_token),
'a'..'z' | 'A'..'Z' | '_' => lex_keyword_or_identifier(&mut chars, &mut add_token),

2
src/parsing/mod.rs
View File

@ -1,2 +1,2 @@
pub mod lexer;
pub mod parser;
pub mod parser;

824
src/parsing/parser/mod.rs
View File

@ -1,5 +1,6 @@
pub mod parsed_statement;
pub mod parsed_type_modifier;
mod parser_operators;
#[cfg(test)]
mod parser_tests;
@ -8,6 +9,8 @@ use crate::logger::messages::Message;
use crate::logger::messages::Message::UnexpectedToken;
use crate::modifiers::{FieldModifier, TypeModifier};
use crate::parsing::parser::parsed_type_modifier::ParsedTypeModifier;
use crate::parsing::parser::parser_operators::{BinaryOperator, PreOperator};
use crate::prim_type::PrimitiveType;
use crate::span::Span;
use enumflags2::BitFlags;
use parsed_statement::ParsedStatement;
@ -15,18 +18,30 @@ use parsed_statement::ParsedStatement;
struct ParseReader<'a> {
tokens: &'a Vec<LexToken>,
position: usize,
peek_distance: usize,
}
impl<'a> ParseReader<'a> {
pub fn peek(&mut self) -> &LexToken {
let t = self.tokens.get(self.position + self.peek_distance);
self.peek_distance += 1;
let t = self.tokens.get(self.position);
match t {
None => self.tokens.last().unwrap(),
Some(v) => {
if v.token_type == TokenType::WhiteSpace {
self.peek()
self.peek_ahead(1)
} else {
v
}
}
}
}
fn peek_ahead(&mut self, index: usize) -> &LexToken {
let t = self.tokens.get(self.position + index);
match t {
None => self.tokens.last().unwrap(),
Some(v) => {
if v.token_type == TokenType::WhiteSpace {
self.peek_ahead(index + 1)
} else {
v
}
@ -37,7 +52,6 @@ impl<'a> ParseReader<'a> {
pub fn next(&mut self) -> &LexToken {
let t = self.tokens.get(self.position);
self.position += 1;
self.peek_distance = 0;
match t {
None => self.tokens.last().unwrap(),
Some(v) => {
@ -50,11 +64,6 @@ impl<'a> ParseReader<'a> {
}
}
pub fn move_to_peek(&mut self) {
self.position += self.peek_distance;
self.peek_distance = 0;
}
pub fn consume(
&mut self,
expected: TokenType,
@ -73,22 +82,15 @@ impl<'a> ParseReader<'a> {
n
}
#[inline(always)]
pub fn reset_peek(&mut self) {
self.peek_distance = 0;
}
pub fn create_inner(&self) -> ParseReader<'a> {
ParseReader {
tokens: self.tokens,
position: self.position,
peek_distance: self.peek_distance,
}
}
pub fn set_from_inner(&mut self, inner: &ParseReader) {
self.position = inner.position;
self.peek_distance = inner.peek_distance;
}
}
@ -97,11 +99,38 @@ pub fn parse(tokens: Vec<LexToken>, log: &mut dyn FnMut(Message, Span)) -> Box<P
let mut reader = ParseReader {
tokens: &tokens,
position: 0,
peek_distance: 0,
};
parse_script(&mut reader, log)
}
fn parse_identifier(
reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
can_fail: bool,
) -> Option<String> {
let identifier_token = reader.peek();
let s: String;
match &identifier_token.token_type {
TokenType::Identifier(i) => {
s = i.to_string();
reader.next();
}
_ => {
if !can_fail {
log(
Message::UnexpectedToken {
found: identifier_token.token_type.clone(),
expected: vec![TokenType::Identifier(String::new())],
},
identifier_token.span,
);
}
return None;
}
}
Some(s)
}
fn parse_script(
reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
@ -121,6 +150,8 @@ fn parse_script(
_ => {
if let Some(s) = parse_interface(reader, log) {
vec.push(*s);
} else if let Some(s) = parse_var(reader, log) {
vec.push(*s);
} else {
log(
UnexpectedToken {
@ -138,36 +169,12 @@ fn parse_script(
Box::new(ParsedStatement::Script { statements: vec })
}
fn parse_identifier(
reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Option<String> {
let identifier_token = reader.next();
let s: String;
match &identifier_token.token_type {
TokenType::Identifier(i) => {
s = i.to_string();
}
_ => {
log(
Message::UnexpectedToken {
found: identifier_token.token_type.clone(),
expected: vec![TokenType::Identifier(String::new())],
},
identifier_token.span,
);
return None;
}
}
Some(s)
}
fn parse_namespace(
reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Box<ParsedStatement> {
reader.next(); // Consume namespace
let identifier = parse_identifier(reader, log);
let identifier = parse_identifier(reader, log, false);
if identifier.is_none() {
return Box::new(ParsedStatement::Invalid);
}
@ -181,15 +188,15 @@ fn parse_namespace(
}
fn parse_interface(
reader: &mut ParseReader,
outer_reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
let mut type_mod: BitFlags<TypeModifier> = BitFlags::empty();
let identifier: Option<String>;
let mut has_interface_keyword = false;
reader.reset_peek();
let mut reader = outer_reader.create_inner();
loop {
match &reader.peek().token_type {
match &reader.next().token_type {
TokenType::ExternalKeyword => type_mod |= TypeModifier::External,
TokenType::SharedKeyword => type_mod |= TypeModifier::Shared,
TokenType::Identifier(s) => {
@ -207,7 +214,6 @@ fn parse_interface(
if !has_interface_keyword {
return None;
}
reader.move_to_peek();
let mut statements: Vec<ParsedStatement> = Vec::new();
let mut inherits: Vec<String> = Vec::new();
loop {
@ -255,11 +261,10 @@ fn parse_interface(
}
}
}
reader.reset_peek();
if reader.peek().token_type == TokenType::Colon {
reader.next();
loop {
let inherit_identifier = parse_identifier(reader, log);
let inherit_identifier = parse_identifier(&mut reader, log, false);
if inherit_identifier.is_none() {
continue;
}
@ -276,7 +281,7 @@ fn parse_interface(
if reader.peek().token_type == TokenType::CloseCurlyBracket {
break;
}
let prop = parse_virtprop(reader, log);
let prop = parse_virtprop(&mut reader, log);
if prop.is_none() {
break;
}
@ -284,7 +289,7 @@ fn parse_interface(
}
reader.consume(TokenType::CloseCurlyBracket, log);
reader.move_to_peek();
outer_reader.set_from_inner(&reader);
if identifier.is_none() {
return Some(Box::new(ParsedStatement::Invalid));
}
@ -297,7 +302,56 @@ fn parse_interface(
}))
}
fn parse_statblock(
fn parse_interface_method(
_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_typedef(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_import(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_enum(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_mixin(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_class(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_funcdef(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_func(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
@ -330,11 +384,11 @@ fn parse_virtprop(
}
}
let mut is_handle = false;
if reader.peek().token_type == TokenType::Ampersand {
if reader.peek().token_type == TokenType::AtSymbol {
reader.next();
is_handle = true;
}
let identifier = parse_identifier(&mut reader, log);
let identifier = parse_identifier(&mut reader, log, false);
identifier.as_ref()?;
let next = reader.next();
if next.token_type != TokenType::OpenCurlyBracket {
@ -416,6 +470,487 @@ fn parse_virtprop(
}))
}
fn parse_paramlist(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_funcattr(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_statblock(
reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
// FIXME
if reader.peek().token_type != TokenType::OpenCurlyBracket {
return None;
}
None
}
fn parse_var(
outer_reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
// var ::= ['private'|'protected'] type identifier
let mut reader = outer_reader.create_inner();
let mut field_mod: BitFlags<FieldModifier> = BitFlags::empty();
let property_type: Option<Box<ParsedStatement>>;
loop {
let t = reader.peek();
match t.token_type {
TokenType::PrivateKeyword => {
field_mod |= FieldModifier::Private;
reader.next();
}
TokenType::ProtectedKeyword => {
field_mod |= FieldModifier::Protected;
reader.next();
}
_ => {
property_type = parse_type(&mut reader, log);
property_type.as_ref()?;
break;
}
}
}
if property_type.is_none() {
return None;
}
let identifier = parse_identifier(&mut reader, log, false);
identifier.as_ref()?;
let mut assignment: Option<Box<ParsedStatement>> = None;
// FIXME: Implement arglist
// [( '=' (initlist | expr)) | arglist]
if reader.peek().token_type == TokenType::Equals {
// Eat the equals token.
reader.next();
// micro-optimization so we don't go into the initlist if not needed.
if reader.peek().token_type == TokenType::OpenCurlyBracket {
if let Some(s) = parse_initlist(&mut reader, log) {
assignment = Some(s);
}
}
if assignment.is_none() {
assignment = parse_expr(&mut reader, log);
}
}
// {',' identifier [( '=' (initlist | expr)) | arglist]} ';';
outer_reader.set_from_inner(&reader);
Some(Box::new(ParsedStatement::Var {
modifier: field_mod,
var_type: property_type.unwrap(),
identifier: identifier.unwrap(),
assignment,
}))
}
fn parse_statement(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_switch(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_case(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_try(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_dowhile(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_while(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_for(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_if(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_break(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_continue(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_exprstat(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_return(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_ternary(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_expr(
reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
// expr ::= exprterm {(mathop | compop | logicop | bitop) exprterm};
let expr_term = parse_exprterm(reader, log);
expr_term.as_ref()?;
let mut binary_operand = parse_mathop(reader, log);
if binary_operand.is_none() {
binary_operand = parse_compop(reader, log);
}
if binary_operand.is_none() {
binary_operand = parse_logicop(reader, log);
}
if binary_operand.is_none() {
binary_operand = parse_bitop(reader, log);
}
if binary_operand.is_some() {
let expr_term2 = parse_exprterm(reader, log);
// FIXME: deal with empty expr_term2
return Some(Box::new(ParsedStatement::BinaryExpr {
left: expr_term.unwrap(),
operator: binary_operand.unwrap(),
right: expr_term2.unwrap(),
}));
}
expr_term
}
fn parse_exprterm(
outer_reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
let mut reader = outer_reader.create_inner();
// exprterm ::= ([type '='] initlist) | ({exprpreop} exprvalue {exprpostop});
let expr_type = parse_type(&mut reader, log);
if expr_type.is_some() {
outer_reader.consume(TokenType::Equals, log);
}
let mut init_list = None;
if reader.peek().token_type == TokenType::OpenCurlyBracket {
init_list = parse_initlist(&mut reader, log);
}
if let Some(initlist) = init_list {
outer_reader.set_from_inner(&reader);
return Some(Box::new(ParsedStatement::ExprTermInitList {
statement_type: expr_type,
initlist,
}));
}
// Reset to initial here.
reader = outer_reader.create_inner();
let mut expr_pre_op = Vec::new();
while let Some(s) = parse_exprpreop(&mut reader, log) {
expr_pre_op.push(s);
}
let expr_value = parse_exprvalue(&mut reader, log);
expr_value.as_ref()?;
let mut real_value = expr_value.unwrap();
let real_value = parse_exprpostop(&mut reader, log, real_value);
outer_reader.set_from_inner(&reader);
Some(Box::new(ParsedStatement::ExprTerm {
pre_op: expr_pre_op,
exp_value: real_value,
}))
}
fn parse_exprpostop(
reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
base: Box<ParsedStatement>,
) -> Box<ParsedStatement> {
// exprpostop ::= ('.' (funccall | identifier)) | ('[' [identifier ':'] assign {',' [identifier ':' assign} ']') | arglist | '++' | '--';
// FIXME: This needs to be implemented.
// ('.' (funccall | identifier))
if reader.peek().token_type == TokenType::Dot {
reader.next();
}
// ('[' [identifier ':'] assign {',' [identifier ':' assign} ']')
// arglist
// '++'
// '--'
base
}
fn parse_exprvalue(
outer_reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
// exprvalue ::= 'void' | constructcall | funccall | varaccess | cast | literal | '(' assign ')' | lambda;
if outer_reader.peek().token_type == TokenType::VoidKeyword {
outer_reader.next();
return Some(Box::new(ParsedStatement::ExprVoidValue {}));
}
let mut reader = outer_reader.create_inner();
if let Some(s) = parse_constructcall(&mut reader, log) {
outer_reader.set_from_inner(&reader);
return Some(s);
}
if let Some(s) = parse_funccall(&mut reader, log) {
outer_reader.set_from_inner(&reader);
return Some(s);
}
if let Some(s) = parse_varaccess(&mut reader, log) {
outer_reader.set_from_inner(&reader);
return Some(s);
}
if let Some(s) = parse_cast(&mut reader, log) {
outer_reader.set_from_inner(&reader);
return Some(s);
}
if let Some(s) = parse_literal(&mut reader, log) {
outer_reader.set_from_inner(&reader);
return Some(s);
}
if reader.peek().token_type == TokenType::OpenBracket {
reader.next();
let inner = parse_assign(&mut reader, log);
// FIXME: deal with None value
reader.consume(TokenType::CloseBracket, log);
outer_reader.set_from_inner(&reader);
return inner;
}
if let Some(s) = parse_lambda(&mut reader, log) {
outer_reader.set_from_inner(&reader);
return Some(s);
}
None
}
fn parse_lambda(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_typemod(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_literal(
reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
let value = match &reader.peek().token_type {
TokenType::IntegerLiteral(v) => Some(Box::new(ParsedStatement::IntegerLiteral(*v))),
TokenType::FloatLiteral(v) => Some(Box::new(ParsedStatement::FloatLiteral(*v))),
TokenType::TrueKeyword => Some(Box::new(ParsedStatement::BoolLiteral(true))),
TokenType::FalseKeyword => Some(Box::new(ParsedStatement::BoolLiteral(false))),
// FIXME: String copy, too slow.
TokenType::StringLiteral(v) => {
Some(Box::new(ParsedStatement::StringLiteral(v.to_string())))
}
_ => None,
};
if value.is_some() {
reader.next();
}
value
}
fn parse_cast(
reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
// cast ::= 'cast' '<' type '>' '(' assign ')';
if reader.peek().token_type != TokenType::CastKeyword {
return None;
}
reader.next();
reader.consume(TokenType::LessThan, log);
let cast_type = parse_type(reader, log);
// FIXME: deal with no cast type
reader.consume(TokenType::GreaterThan, log);
reader.consume(TokenType::OpenBracket, log);
let assign = parse_assign(reader, log);
// FIXME: deal with no assign
reader.consume(TokenType::CloseBracket, log);
Some(Box::new(ParsedStatement::Cast {
cast_type: cast_type.unwrap(),
assign: assign.unwrap(),
}))
}
fn parse_varaccess(
outer_reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
// scope identifier
let mut reader = outer_reader.create_inner();
let scope = parse_scope(&mut reader, log);
let identifier = parse_identifier(&mut reader, log, true);
identifier.as_ref()?;
outer_reader.set_from_inner(&reader);
Some(Box::new(ParsedStatement::VarAccess {
scope,
identifier: identifier.unwrap(),
}))
}
fn parse_constructcall(
outer_reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
// type arglist
let mut reader = outer_reader.create_inner();
let construct_type = parse_type(&mut reader, log);
construct_type.as_ref()?;
let arg_list = parse_arglist(&mut reader, log);
// FIXME: deal with None value for arg list
outer_reader.set_from_inner(&reader);
Some(Box::new(ParsedStatement::ConstructCall {
statement_type: construct_type,
arglist: arg_list.unwrap(),
}))
}
fn parse_funccall(
outer_reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
// scope identifier arglist
let mut reader = outer_reader.create_inner();
let scope = parse_scope(&mut reader, log);
let identifier = parse_identifier(&mut reader, log, true);
identifier.as_ref()?;
let arg_list = parse_arglist(&mut reader, log);
arg_list.as_ref()?;
outer_reader.set_from_inner(&reader);
Some(Box::new(ParsedStatement::FuncCall {
scope,
identifier: identifier.unwrap(),
arglist: arg_list.unwrap(),
}))
}
fn parse_arglist(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_exprpreop(
reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<PreOperator> {
// exprpreop ::= '-' | '+' | '!' | '++' | '--' | '~' | '@';
let op = match reader.peek().token_type {
TokenType::Minus => Some(PreOperator::Negative),
TokenType::Plus => Some(PreOperator::Identity),
TokenType::ExclamationMark => Some(PreOperator::Negation),
TokenType::PlusPlus => Some(PreOperator::Increment),
TokenType::MinusMinus => Some(PreOperator::Decrement),
TokenType::Tilde => Some(PreOperator::Complement),
TokenType::AtSymbol => Some(PreOperator::HandleOf),
_ => None,
};
if op.is_some() {
reader.next();
}
op
}
fn parse_initlist(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_assign(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_type(
reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
@ -427,9 +962,8 @@ fn parse_type(
is_const = true;
}
let scope = parse_scope(reader, log);
let identifier = parse_identifier(reader, log);
// if none, we already logged an error. Return None.
identifier.as_ref()?;
let datatype = parse_datatype(reader, log);
datatype.as_ref()?;
// TODO: Generics
@ -442,7 +976,7 @@ fn parse_type(
reader.consume(TokenType::CloseBlockBracket, log);
modifiers.push(ParsedTypeModifier::Array);
}
TokenType::Ampersand => {
TokenType::AtSymbol => {
reader.next();
if reader.peek().token_type == TokenType::ConstKeyword {
reader.next();
@ -458,7 +992,7 @@ fn parse_type(
Some(Box::new(ParsedStatement::Type {
is_const,
scope,
identifier: identifier.unwrap(),
datatype: datatype.unwrap(),
modifiers,
}))
}
@ -467,6 +1001,7 @@ fn parse_scope(
outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
// scope ::= ['::'] {identifier '::'} [identifier ['<' type {',' type} '>'] '::']
let mut reader = outer_reader.create_inner();
let is_global = reader.peek().token_type == TokenType::ColonColon;
if is_global {
@ -500,3 +1035,174 @@ fn parse_scope(
}))
}
}
fn parse_datatype(
reader: &mut ParseReader,
log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
if let Some(identifier) = parse_identifier(reader, log, true) {
return Some(Box::new(ParsedStatement::DataTypeIdentifier { identifier }));
}
let key = reader.peek();
let res = match &key.token_type {
TokenType::AutoKeyword => Some(Box::new(ParsedStatement::DataTypeAuto {})),
TokenType::VoidKeyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::Void,
})),
TokenType::Int8Keyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::Int8,
})),
TokenType::Int16Keyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::Int16,
})),
TokenType::Int32Keyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::Int32,
})),
TokenType::Int64Keyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::Int64,
})),
TokenType::Uint8Keyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::UInt8,
})),
TokenType::Uint16Keyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::UInt16,
})),
TokenType::Uint32Keyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::UInt32,
})),
TokenType::Uint64Keyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::UInt64,
})),
TokenType::FloatKeyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::Float,
})),
TokenType::DoubleKeyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::Double,
})),
TokenType::BoolKeyword => Some(Box::new(ParsedStatement::DataTypePrimType {
prim_type: PrimitiveType::Bool,
})),
_ => None,
};
if res.is_some() {
reader.next();
}
res
}
fn parse_primtype(
_outer_reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<Box<ParsedStatement>> {
unimplemented!();
}
fn parse_bitop(
reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<BinaryOperator> {
// bitop ::= '&' | '|' | '^' | '<<' | '>>' | '>>>';
let operand = match reader.peek().token_type {
TokenType::Ampersand => Some(BinaryOperator::BitwiseAnd),
TokenType::VerticalLine => Some(BinaryOperator::BitwiseOr),
TokenType::Roof => Some(BinaryOperator::BitwiseXor),
TokenType::LeftLeft => Some(BinaryOperator::LeftShift),
TokenType::RightRight => Some(BinaryOperator::RightShift),
TokenType::RightRightRight => Some(BinaryOperator::ArithmeticRightShift),
_ => None,
};
if operand.is_some() {
reader.next();
}
operand
}
fn parse_mathop(
reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<BinaryOperator> {
// mathop := '+' | '-' | '*' | '/' | '%' | '**';
let operand = match reader.peek().token_type {
TokenType::Plus => Some(BinaryOperator::Addition),
TokenType::Minus => Some(BinaryOperator::Subtraction),
TokenType::Star => Some(BinaryOperator::Multiplication),
TokenType::Slash => Some(BinaryOperator::Division),
TokenType::Percent => Some(BinaryOperator::Modulo),
TokenType::StarStar => Some(BinaryOperator::Exponentiation),
_ => None,
};
if operand.is_some() {
reader.next();
}
operand
}
fn parse_compop(
reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<BinaryOperator> {
// compop ::= '==' | '!=' | '<' | '<=' | '>' | '>=' | 'is' | '!is';
let operand = match reader.peek().token_type {
TokenType::EqualsEquals => Some(BinaryOperator::Equality),
TokenType::NotEquals => Some(BinaryOperator::Inequality),
TokenType::LessThan => Some(BinaryOperator::LessThan),
TokenType::LessThanEquals => Some(BinaryOperator::LessThanEquals),
TokenType::GreaterThan => Some(BinaryOperator::GreaterThan),
TokenType::GreaterThanEquals => Some(BinaryOperator::GreaterThanEquals),
TokenType::IsKeyword => Some(BinaryOperator::IdentityEquality),
TokenType::NotIsKeyword => Some(BinaryOperator::IdentityInequality),
_ => None,
};
if operand.is_some() {
reader.next();
}
operand
}
fn parse_logicop(
reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<BinaryOperator> {
// logicop ::= '&&' | '||' | '^^' | 'and' | 'or' | 'xor';
let operand = match reader.peek().token_type {
TokenType::AmpersandAmpersand => Some(BinaryOperator::And),
TokenType::LineLine => Some(BinaryOperator::Or),
TokenType::RoofRoof => Some(BinaryOperator::Xor),
TokenType::AndKeyword => Some(BinaryOperator::And),
TokenType::OrKeyword => Some(BinaryOperator::Or),
TokenType::XorKeyword => Some(BinaryOperator::Xor),
_ => None,
};
if operand.is_some() {
reader.next();
}
operand
}
fn parse_assignop(
reader: &mut ParseReader,
_log: &mut dyn FnMut(Message, Span),
) -> Option<BinaryOperator> {
// assignop ::= '=' | '+=' | '-=' | '*=' | '/=' | '|=' | '&=' | '^=' | '%=' | '**=' | '<<=' | '>>=' | '>>>=';
let operand = match reader.peek().token_type {
TokenType::Equals => Some(BinaryOperator::Assignment),
TokenType::PlusEquals => Some(BinaryOperator::AssignmentAddition),
TokenType::MinusEquals => Some(BinaryOperator::AssignmentSubtraction),
TokenType::StarEquals => Some(BinaryOperator::AssignmentMultiplication),
TokenType::PercentEquals => Some(BinaryOperator::AssignmentModulo),
TokenType::StarStarEquals => Some(BinaryOperator::AssignmentExponentiation),
TokenType::SlashEquals => Some(BinaryOperator::AssignmentDivision),
TokenType::LineEquals => Some(BinaryOperator::AssignmentBitwiseOr),
TokenType::AmpersandEquals => Some(BinaryOperator::AssignmentBitwiseAnd),
TokenType::RoofEquals => Some(BinaryOperator::AssignmentBitwiseXor),
TokenType::LeftLeftEquals => Some(BinaryOperator::AssignmentLeftShift),
TokenType::RightRightEquals => Some(BinaryOperator::AssignmentRightShift),
TokenType::RightRightRightEquals => Some(BinaryOperator::AssignmentArithmeticRightShift),
_ => None,
};
if operand.is_some() {
reader.next();
}
operand
}

57
src/parsing/parser/parsed_statement.rs
View File

@ -1,7 +1,11 @@
use super::parsed_type_modifier::ParsedTypeModifier;
use crate::defines::{LiteralFloat, LiteralInt};
use crate::modifiers::{FieldModifier, TypeModifier};
use crate::parsing::parser::parser_operators::{BinaryOperator, PreOperator};
use crate::prim_type::PrimitiveType;
use enumflags2::BitFlags;
#[derive(PartialEq, Debug)]
pub enum ParsedStatement {
Invalid,
Script {
@ -36,10 +40,61 @@ pub enum ParsedStatement {
is_set_const: bool,
set_statement: Option<Box<ParsedStatement>>,
},
DataTypeIdentifier {
identifier: String,
},
DataTypeAuto {},
DataTypePrimType {
prim_type: PrimitiveType,
},
Type {
is_const: bool,
scope: Option<Box<ParsedStatement>>,
identifier: String,
datatype: Box<ParsedStatement>,
modifiers: Vec<ParsedTypeModifier>,
},
Var {
modifier: BitFlags<FieldModifier>,
var_type: Box<ParsedStatement>,
identifier: String,
assignment: Option<Box<ParsedStatement>>,
},
ExprTermInitList {
statement_type: Option<Box<ParsedStatement>>,
initlist: Box<ParsedStatement>,
},
ExprTerm {
pre_op: Vec<PreOperator>,
exp_value: Box<ParsedStatement>,
},
ExprPreOp {
operator: PreOperator,
},
ExprVoidValue {},
ConstructCall {
statement_type: Option<Box<ParsedStatement>>,
arglist: Box<ParsedStatement>,
},
FuncCall {
scope: Option<Box<ParsedStatement>>,
identifier: String,
arglist: Box<ParsedStatement>,
},
VarAccess {
scope: Option<Box<ParsedStatement>>,
identifier: String,
},
Cast {
cast_type: Box<ParsedStatement>,
assign: Box<ParsedStatement>,
},
IntegerLiteral(LiteralInt),
FloatLiteral(LiteralFloat),
BoolLiteral(bool),
StringLiteral(String),
BinaryExpr {
left: Box<ParsedStatement>,
operator: BinaryOperator,
right: Box<ParsedStatement>,
},
}

1
src/parsing/parser/parsed_type_modifier.rs
View File

@ -1,3 +1,4 @@
#[derive(Eq, PartialEq, Debug)]
pub enum ParsedTypeModifier {
Array,
Handle,

55
src/parsing/parser/parser_operators.rs Normal file
View File

@ -0,0 +1,55 @@
#[derive(Eq, PartialEq, Debug)]
pub enum PreOperator {
Negative,
Identity,
Negation,
Increment,
Decrement,
Complement,
HandleOf,
}
#[derive(Eq, PartialEq, Debug)]
pub enum BinaryOperator {
// math op
Addition,
Subtraction,
Multiplication,
Division,
Modulo,
Exponentiation,
// comp op
Equality,
Inequality,
LessThan,
LessThanEquals,
GreaterThan,
GreaterThanEquals,
IdentityEquality,
IdentityInequality,
// logic op
And,
Or,
Xor,
// bit op
BitwiseAnd,
BitwiseOr,
BitwiseXor,
LeftShift,
RightShift,
ArithmeticRightShift,
// assign op
Assignment,
AssignmentAddition,
AssignmentSubtraction,
AssignmentMultiplication,
AssignmentDivision,
AssignmentModulo,
AssignmentExponentiation,
AssignmentBitwiseAnd,
AssignmentBitwiseOr,
AssignmentBitwiseXor,
AssignmentLeftShift,
AssignmentRightShift,
AssignmentArithmeticRightShift,
}

184
src/parsing/parser/parser_tests.rs
View File

@ -2,6 +2,8 @@ use super::parse;
use super::parsed_statement::ParsedStatement;
use crate::modifiers::TypeModifier;
use crate::parsing::lexer::lex_tokens::{LexToken, TokenType};
use crate::parsing::parser::parsed_statement::ParsedStatement::DataTypeAuto;
use crate::parsing::parser::parser_operators::{BinaryOperator, PreOperator};
use crate::span::Span;
use enumflags2::BitFlags;
@ -179,13 +181,17 @@ fn test_interface_with_virtprop() {
{
assert_eq!(*field_mod, BitFlags::empty());
if let ParsedStatement::Type {
is_const,
identifier,
..
is_const, datatype, ..
} = property_type.as_ref()
{
assert!(!is_const);
assert_eq!(identifier, "bar");
if let ParsedStatement::DataTypeIdentifier { identifier, .. } =
datatype.as_ref()
{
assert_eq!(identifier, "bar");
} else {
unreachable!()
}
} else {
unreachable!()
}
@ -207,3 +213,173 @@ fn test_interface_with_virtprop() {
unreachable!();
}
}
#[test]
fn test_assign_to_global_variable() {
let script = parse(
create_tokens(vec![
TokenType::AutoKeyword,
TokenType::WhiteSpace,
TokenType::Identifier("foo".to_string()),
TokenType::WhiteSpace,
TokenType::Equals,
TokenType::WhiteSpace,
TokenType::IntegerLiteral(100),
TokenType::EndOfFile,
]),
&mut |_message, _span| {
std::panic::panic_any(_message.stringify());
},
);
if let ParsedStatement::Script { statements } = script.as_ref() {
assert_eq!(1, statements.len());
if let ParsedStatement::Var {
modifier,
var_type,
identifier,
assignment,
} = &statements[0]
{
assert!(modifier.is_empty());
assert_eq!(
*var_type.as_ref(),
ParsedStatement::Type {
is_const: false,
scope: None,
datatype: Box::from(DataTypeAuto {}),
modifiers: vec![]
}
);
assert_eq!(identifier, "foo");
assert!(assignment.is_some());
assert_eq!(
*(assignment.as_ref().unwrap().as_ref()),
ParsedStatement::ExprTerm {
pre_op: vec![],
exp_value: Box::from(ParsedStatement::IntegerLiteral(100)),
}
);
} else {
unreachable!()
}
} else {
unreachable!();
}
}
#[test]
fn test_assign_negative_to_global_variable() {
let script = parse(
create_tokens(vec![
TokenType::AutoKeyword,
TokenType::WhiteSpace,
TokenType::Identifier("foo".to_string()),
TokenType::WhiteSpace,
TokenType::Equals,
TokenType::WhiteSpace,
TokenType::Minus,
TokenType::IntegerLiteral(100),
TokenType::EndOfFile,
]),
&mut |_message, _span| {
std::panic::panic_any(_message.stringify());
},
);
if let ParsedStatement::Script { statements } = script.as_ref() {
assert_eq!(1, statements.len());
if let ParsedStatement::Var {
modifier,
var_type,
identifier,
assignment,
} = &statements[0]
{
assert!(modifier.is_empty());
assert_eq!(
*var_type.as_ref(),
ParsedStatement::Type {
is_const: false,
scope: None,
datatype: Box::from(DataTypeAuto {}),
modifiers: vec![]
}
);
assert_eq!(identifier, "foo");
assert!(assignment.is_some());
assert_eq!(
*(assignment.as_ref().unwrap().as_ref()),
ParsedStatement::ExprTerm {
pre_op: vec![PreOperator::Negative],
exp_value: Box::from(ParsedStatement::IntegerLiteral(100)),
}
);
} else {
unreachable!()
}
} else {
unreachable!();
}
}
#[test]
fn test_assign_addition_to_global_variable() {
let script = parse(
create_tokens(vec![
TokenType::AutoKeyword,
TokenType::WhiteSpace,
TokenType::Identifier("foo".to_string()),
TokenType::WhiteSpace,
TokenType::Equals,
TokenType::WhiteSpace,
TokenType::IntegerLiteral(100),
TokenType::Plus,
TokenType::Minus,
TokenType::IntegerLiteral(20),
TokenType::EndOfFile,
]),
&mut |_message, _span| {
std::panic::panic_any(_message.stringify());
},
);
if let ParsedStatement::Script { statements } = script.as_ref() {
assert_eq!(1, statements.len());
if let ParsedStatement::Var {
modifier,
var_type,
identifier,
assignment,
} = &statements[0]
{
assert!(modifier.is_empty());
assert_eq!(
*var_type.as_ref(),
ParsedStatement::Type {
is_const: false,
scope: None,
datatype: Box::from(DataTypeAuto {}),
modifiers: vec![]
}
);
assert_eq!(identifier, "foo");
assert!(assignment.is_some());
assert_eq!(
*(assignment.as_ref().unwrap().as_ref()),
ParsedStatement::BinaryExpr {
left: Box::new(ParsedStatement::ExprTerm {
pre_op: vec![],
exp_value: Box::new(ParsedStatement::IntegerLiteral(100)),
}),
operator: BinaryOperator::Addition,
right: Box::new(ParsedStatement::ExprTerm {
pre_op: vec![PreOperator::Negative],
exp_value: Box::new(ParsedStatement::IntegerLiteral(20)),
}),
}
);
} else {
unreachable!()
}
} else {
unreachable!();
}
}

34
src/prim_type.rs Normal file
View File

@ -0,0 +1,34 @@
use crate::defines::PointerSize;
#[derive(Eq, PartialEq, Debug)]
pub enum PrimitiveType {
Void,
Int8,
Int16,
Int32,
Int64,
UInt8,
UInt16,
UInt32,
UInt64,
Float,
Double,
Bool,
}
pub fn get_primitive_type_byte_size(t: PrimitiveType) -> PointerSize {
match t {
PrimitiveType::Void => 0,
PrimitiveType::Int8 => 1,
PrimitiveType::Int16 => 2,
PrimitiveType::Int32 => 4,
PrimitiveType::Int64 => 8,
PrimitiveType::UInt8 => 1,
PrimitiveType::UInt16 => 2,
PrimitiveType::UInt32 => 4,
PrimitiveType::UInt64 => 8,
PrimitiveType::Float => 4,
PrimitiveType::Double => 8,
PrimitiveType::Bool => 1,
}
}