From 75f8720c54c712532b6a89cf92f361e64ed9d25c Mon Sep 17 00:00:00 2001 From: Deukhoofd Date: Sat, 2 Apr 2022 23:30:05 +0200 Subject: [PATCH] Loads of additional parsing work --- grammar.ebnf | 2 +- src/defines.rs | 2 + src/lib.rs | 5 +- src/parsing/lexer/lex_tokens.rs | 1 + src/parsing/lexer/mod.rs | 1 + src/parsing/mod.rs | 2 +- src/parsing/parser/mod.rs | 824 +++++++++++++++++++-- src/parsing/parser/parsed_statement.rs | 57 +- src/parsing/parser/parsed_type_modifier.rs | 1 + src/parsing/parser/parser_operators.rs | 55 ++ src/parsing/parser/parser_tests.rs | 184 ++++- src/prim_type.rs | 34 + 12 files changed, 1101 insertions(+), 67 deletions(-) create mode 100644 src/parsing/parser/parser_operators.rs create mode 100644 src/prim_type.rs diff --git a/grammar.ebnf b/grammar.ebnf index 203477d..87f243f 100644 --- a/grammar.ebnf +++ b/grammar.ebnf @@ -48,7 +48,7 @@ exprpreop ::= '-' | '+' | '!' | '++' | '--' | '~' | '@'; arglist ::= '(' [identifier ':'] assign {',' [identifier ':'] assign} ')'; funccall ::= scope identifier arglist; constructcall ::= type arglist; -varaccess ::= scope | identifier; +varaccess ::= scope identifier; cast ::= 'cast' '<' type '>' '(' assign ')'; literal ::= number | string | 'true' | 'false' | 'null'; typemod ::= ['&' ['in' | 'out' | 'inout']]; diff --git a/src/defines.rs b/src/defines.rs index 8df6423..5e42d6f 100644 --- a/src/defines.rs +++ b/src/defines.rs @@ -2,3 +2,5 @@ pub type LiteralInt = i64; /// The size floating point numbers use internally to store literals and do compile time calculations. pub type LiteralFloat = f64; + +pub type PointerSize = usize; diff --git a/src/lib.rs b/src/lib.rs index 9d5abf5..608c100 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,8 +6,11 @@ #![feature(iter_advance_by)] #![feature(backtrace)] -pub(crate) mod defines; +extern crate core; + +pub mod defines; pub mod logger; pub mod modifiers; pub mod parsing; +pub(crate) mod prim_type; pub mod span; diff --git a/src/parsing/lexer/lex_tokens.rs b/src/parsing/lexer/lex_tokens.rs index 22c1e7a..8385fe5 100644 --- a/src/parsing/lexer/lex_tokens.rs +++ b/src/parsing/lexer/lex_tokens.rs @@ -19,6 +19,7 @@ pub enum TokenType { Colon, ColonColon, Comma, + Dot, OpenBracket, CloseBracket, OpenCurlyBracket, diff --git a/src/parsing/lexer/mod.rs b/src/parsing/lexer/mod.rs index 840d9d9..be4fa45 100644 --- a/src/parsing/lexer/mod.rs +++ b/src/parsing/lexer/mod.rs @@ -449,6 +449,7 @@ pub fn lex(s: &str, log: &mut dyn FnMut(Message, Span)) -> Vec { '[' => lex_and_consume(&mut chars, TT::OpenBlockBracket, &mut add_token), ']' => lex_and_consume(&mut chars, TT::CloseBlockBracket, &mut add_token), ',' => lex_and_consume(&mut chars, TT::Comma, &mut add_token), + '.' => lex_and_consume(&mut chars, TT::Dot, &mut add_token), '0'..'9' => lex_numeric(&mut chars, &mut add_token), 'a'..'z' | 'A'..'Z' | '_' => lex_keyword_or_identifier(&mut chars, &mut add_token), diff --git a/src/parsing/mod.rs b/src/parsing/mod.rs index 591a845..6054fb1 100644 --- a/src/parsing/mod.rs +++ b/src/parsing/mod.rs @@ -1,2 +1,2 @@ pub mod lexer; -pub mod parser; \ No newline at end of file +pub mod parser; diff --git a/src/parsing/parser/mod.rs b/src/parsing/parser/mod.rs index 5a537aa..4d926d5 100644 --- a/src/parsing/parser/mod.rs +++ b/src/parsing/parser/mod.rs @@ -1,5 +1,6 @@ pub mod parsed_statement; pub mod parsed_type_modifier; +mod parser_operators; #[cfg(test)] mod parser_tests; @@ -8,6 +9,8 @@ use crate::logger::messages::Message; use crate::logger::messages::Message::UnexpectedToken; use crate::modifiers::{FieldModifier, TypeModifier}; use crate::parsing::parser::parsed_type_modifier::ParsedTypeModifier; +use crate::parsing::parser::parser_operators::{BinaryOperator, PreOperator}; +use crate::prim_type::PrimitiveType; use crate::span::Span; use enumflags2::BitFlags; use parsed_statement::ParsedStatement; @@ -15,18 +18,30 @@ use parsed_statement::ParsedStatement; struct ParseReader<'a> { tokens: &'a Vec, position: usize, - peek_distance: usize, } impl<'a> ParseReader<'a> { pub fn peek(&mut self) -> &LexToken { - let t = self.tokens.get(self.position + self.peek_distance); - self.peek_distance += 1; + let t = self.tokens.get(self.position); match t { None => self.tokens.last().unwrap(), Some(v) => { if v.token_type == TokenType::WhiteSpace { - self.peek() + self.peek_ahead(1) + } else { + v + } + } + } + } + + fn peek_ahead(&mut self, index: usize) -> &LexToken { + let t = self.tokens.get(self.position + index); + match t { + None => self.tokens.last().unwrap(), + Some(v) => { + if v.token_type == TokenType::WhiteSpace { + self.peek_ahead(index + 1) } else { v } @@ -37,7 +52,6 @@ impl<'a> ParseReader<'a> { pub fn next(&mut self) -> &LexToken { let t = self.tokens.get(self.position); self.position += 1; - self.peek_distance = 0; match t { None => self.tokens.last().unwrap(), Some(v) => { @@ -50,11 +64,6 @@ impl<'a> ParseReader<'a> { } } - pub fn move_to_peek(&mut self) { - self.position += self.peek_distance; - self.peek_distance = 0; - } - pub fn consume( &mut self, expected: TokenType, @@ -73,22 +82,15 @@ impl<'a> ParseReader<'a> { n } - #[inline(always)] - pub fn reset_peek(&mut self) { - self.peek_distance = 0; - } - pub fn create_inner(&self) -> ParseReader<'a> { ParseReader { tokens: self.tokens, position: self.position, - peek_distance: self.peek_distance, } } pub fn set_from_inner(&mut self, inner: &ParseReader) { self.position = inner.position; - self.peek_distance = inner.peek_distance; } } @@ -97,11 +99,38 @@ pub fn parse(tokens: Vec, log: &mut dyn FnMut(Message, Span)) -> Box

Option { + let identifier_token = reader.peek(); + let s: String; + match &identifier_token.token_type { + TokenType::Identifier(i) => { + s = i.to_string(); + reader.next(); + } + _ => { + if !can_fail { + log( + Message::UnexpectedToken { + found: identifier_token.token_type.clone(), + expected: vec![TokenType::Identifier(String::new())], + }, + identifier_token.span, + ); + } + return None; + } + } + Some(s) +} + fn parse_script( reader: &mut ParseReader, log: &mut dyn FnMut(Message, Span), @@ -121,6 +150,8 @@ fn parse_script( _ => { if let Some(s) = parse_interface(reader, log) { vec.push(*s); + } else if let Some(s) = parse_var(reader, log) { + vec.push(*s); } else { log( UnexpectedToken { @@ -138,36 +169,12 @@ fn parse_script( Box::new(ParsedStatement::Script { statements: vec }) } -fn parse_identifier( - reader: &mut ParseReader, - log: &mut dyn FnMut(Message, Span), -) -> Option { - let identifier_token = reader.next(); - let s: String; - match &identifier_token.token_type { - TokenType::Identifier(i) => { - s = i.to_string(); - } - _ => { - log( - Message::UnexpectedToken { - found: identifier_token.token_type.clone(), - expected: vec![TokenType::Identifier(String::new())], - }, - identifier_token.span, - ); - return None; - } - } - Some(s) -} - fn parse_namespace( reader: &mut ParseReader, log: &mut dyn FnMut(Message, Span), ) -> Box { reader.next(); // Consume namespace - let identifier = parse_identifier(reader, log); + let identifier = parse_identifier(reader, log, false); if identifier.is_none() { return Box::new(ParsedStatement::Invalid); } @@ -181,15 +188,15 @@ fn parse_namespace( } fn parse_interface( - reader: &mut ParseReader, + outer_reader: &mut ParseReader, log: &mut dyn FnMut(Message, Span), ) -> Option> { let mut type_mod: BitFlags = BitFlags::empty(); let identifier: Option; let mut has_interface_keyword = false; - reader.reset_peek(); + let mut reader = outer_reader.create_inner(); loop { - match &reader.peek().token_type { + match &reader.next().token_type { TokenType::ExternalKeyword => type_mod |= TypeModifier::External, TokenType::SharedKeyword => type_mod |= TypeModifier::Shared, TokenType::Identifier(s) => { @@ -207,7 +214,6 @@ fn parse_interface( if !has_interface_keyword { return None; } - reader.move_to_peek(); let mut statements: Vec = Vec::new(); let mut inherits: Vec = Vec::new(); loop { @@ -255,11 +261,10 @@ fn parse_interface( } } } - reader.reset_peek(); if reader.peek().token_type == TokenType::Colon { reader.next(); loop { - let inherit_identifier = parse_identifier(reader, log); + let inherit_identifier = parse_identifier(&mut reader, log, false); if inherit_identifier.is_none() { continue; } @@ -276,7 +281,7 @@ fn parse_interface( if reader.peek().token_type == TokenType::CloseCurlyBracket { break; } - let prop = parse_virtprop(reader, log); + let prop = parse_virtprop(&mut reader, log); if prop.is_none() { break; } @@ -284,7 +289,7 @@ fn parse_interface( } reader.consume(TokenType::CloseCurlyBracket, log); - reader.move_to_peek(); + outer_reader.set_from_inner(&reader); if identifier.is_none() { return Some(Box::new(ParsedStatement::Invalid)); } @@ -297,7 +302,56 @@ fn parse_interface( })) } -fn parse_statblock( +fn parse_interface_method( + _reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_typedef( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_import( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_enum( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_mixin( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_class( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_funcdef( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_func( _outer_reader: &mut ParseReader, _log: &mut dyn FnMut(Message, Span), ) -> Option> { @@ -330,11 +384,11 @@ fn parse_virtprop( } } let mut is_handle = false; - if reader.peek().token_type == TokenType::Ampersand { + if reader.peek().token_type == TokenType::AtSymbol { reader.next(); is_handle = true; } - let identifier = parse_identifier(&mut reader, log); + let identifier = parse_identifier(&mut reader, log, false); identifier.as_ref()?; let next = reader.next(); if next.token_type != TokenType::OpenCurlyBracket { @@ -416,6 +470,487 @@ fn parse_virtprop( })) } +fn parse_paramlist( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_funcattr( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_statblock( + reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + // FIXME + if reader.peek().token_type != TokenType::OpenCurlyBracket { + return None; + } + None +} + +fn parse_var( + outer_reader: &mut ParseReader, + log: &mut dyn FnMut(Message, Span), +) -> Option> { + // var ::= ['private'|'protected'] type identifier + + let mut reader = outer_reader.create_inner(); + let mut field_mod: BitFlags = BitFlags::empty(); + let property_type: Option>; + loop { + let t = reader.peek(); + match t.token_type { + TokenType::PrivateKeyword => { + field_mod |= FieldModifier::Private; + reader.next(); + } + TokenType::ProtectedKeyword => { + field_mod |= FieldModifier::Protected; + reader.next(); + } + _ => { + property_type = parse_type(&mut reader, log); + property_type.as_ref()?; + break; + } + } + } + if property_type.is_none() { + return None; + } + + let identifier = parse_identifier(&mut reader, log, false); + identifier.as_ref()?; + + let mut assignment: Option> = None; + + // FIXME: Implement arglist + // [( '=' (initlist | expr)) | arglist] + if reader.peek().token_type == TokenType::Equals { + // Eat the equals token. + reader.next(); + // micro-optimization so we don't go into the initlist if not needed. + if reader.peek().token_type == TokenType::OpenCurlyBracket { + if let Some(s) = parse_initlist(&mut reader, log) { + assignment = Some(s); + } + } + if assignment.is_none() { + assignment = parse_expr(&mut reader, log); + } + } + + // {',' identifier [( '=' (initlist | expr)) | arglist]} ';'; + + outer_reader.set_from_inner(&reader); + Some(Box::new(ParsedStatement::Var { + modifier: field_mod, + var_type: property_type.unwrap(), + identifier: identifier.unwrap(), + assignment, + })) +} + +fn parse_statement( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_switch( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_case( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_try( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_dowhile( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_while( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_for( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_if( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_break( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_continue( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_exprstat( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_return( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_ternary( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_expr( + reader: &mut ParseReader, + log: &mut dyn FnMut(Message, Span), +) -> Option> { + // expr ::= exprterm {(mathop | compop | logicop | bitop) exprterm}; + + let expr_term = parse_exprterm(reader, log); + expr_term.as_ref()?; + + let mut binary_operand = parse_mathop(reader, log); + if binary_operand.is_none() { + binary_operand = parse_compop(reader, log); + } + if binary_operand.is_none() { + binary_operand = parse_logicop(reader, log); + } + if binary_operand.is_none() { + binary_operand = parse_bitop(reader, log); + } + + if binary_operand.is_some() { + let expr_term2 = parse_exprterm(reader, log); + // FIXME: deal with empty expr_term2 + return Some(Box::new(ParsedStatement::BinaryExpr { + left: expr_term.unwrap(), + operator: binary_operand.unwrap(), + right: expr_term2.unwrap(), + })); + } + + expr_term +} + +fn parse_exprterm( + outer_reader: &mut ParseReader, + log: &mut dyn FnMut(Message, Span), +) -> Option> { + let mut reader = outer_reader.create_inner(); + // exprterm ::= ([type '='] initlist) | ({exprpreop} exprvalue {exprpostop}); + let expr_type = parse_type(&mut reader, log); + if expr_type.is_some() { + outer_reader.consume(TokenType::Equals, log); + } + + let mut init_list = None; + if reader.peek().token_type == TokenType::OpenCurlyBracket { + init_list = parse_initlist(&mut reader, log); + } + + if let Some(initlist) = init_list { + outer_reader.set_from_inner(&reader); + return Some(Box::new(ParsedStatement::ExprTermInitList { + statement_type: expr_type, + initlist, + })); + } + + // Reset to initial here. + reader = outer_reader.create_inner(); + let mut expr_pre_op = Vec::new(); + while let Some(s) = parse_exprpreop(&mut reader, log) { + expr_pre_op.push(s); + } + let expr_value = parse_exprvalue(&mut reader, log); + expr_value.as_ref()?; + + let mut real_value = expr_value.unwrap(); + + let real_value = parse_exprpostop(&mut reader, log, real_value); + + outer_reader.set_from_inner(&reader); + Some(Box::new(ParsedStatement::ExprTerm { + pre_op: expr_pre_op, + exp_value: real_value, + })) +} + +fn parse_exprpostop( + reader: &mut ParseReader, + log: &mut dyn FnMut(Message, Span), + base: Box, +) -> Box { + // exprpostop ::= ('.' (funccall | identifier)) | ('[' [identifier ':'] assign {',' [identifier ':' assign} ']') | arglist | '++' | '--'; + + // FIXME: This needs to be implemented. + + // ('.' (funccall | identifier)) + if reader.peek().token_type == TokenType::Dot { + reader.next(); + } + // ('[' [identifier ':'] assign {',' [identifier ':' assign} ']') + // arglist + // '++' + // '--' + + base +} + +fn parse_exprvalue( + outer_reader: &mut ParseReader, + log: &mut dyn FnMut(Message, Span), +) -> Option> { + // exprvalue ::= 'void' | constructcall | funccall | varaccess | cast | literal | '(' assign ')' | lambda; + if outer_reader.peek().token_type == TokenType::VoidKeyword { + outer_reader.next(); + return Some(Box::new(ParsedStatement::ExprVoidValue {})); + } + let mut reader = outer_reader.create_inner(); + if let Some(s) = parse_constructcall(&mut reader, log) { + outer_reader.set_from_inner(&reader); + return Some(s); + } + if let Some(s) = parse_funccall(&mut reader, log) { + outer_reader.set_from_inner(&reader); + return Some(s); + } + if let Some(s) = parse_varaccess(&mut reader, log) { + outer_reader.set_from_inner(&reader); + return Some(s); + } + if let Some(s) = parse_cast(&mut reader, log) { + outer_reader.set_from_inner(&reader); + return Some(s); + } + if let Some(s) = parse_literal(&mut reader, log) { + outer_reader.set_from_inner(&reader); + return Some(s); + } + if reader.peek().token_type == TokenType::OpenBracket { + reader.next(); + let inner = parse_assign(&mut reader, log); + // FIXME: deal with None value + reader.consume(TokenType::CloseBracket, log); + outer_reader.set_from_inner(&reader); + return inner; + } + if let Some(s) = parse_lambda(&mut reader, log) { + outer_reader.set_from_inner(&reader); + return Some(s); + } + + None +} + +fn parse_lambda( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_typemod( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_literal( + reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + let value = match &reader.peek().token_type { + TokenType::IntegerLiteral(v) => Some(Box::new(ParsedStatement::IntegerLiteral(*v))), + TokenType::FloatLiteral(v) => Some(Box::new(ParsedStatement::FloatLiteral(*v))), + TokenType::TrueKeyword => Some(Box::new(ParsedStatement::BoolLiteral(true))), + TokenType::FalseKeyword => Some(Box::new(ParsedStatement::BoolLiteral(false))), + // FIXME: String copy, too slow. + TokenType::StringLiteral(v) => { + Some(Box::new(ParsedStatement::StringLiteral(v.to_string()))) + } + _ => None, + }; + if value.is_some() { + reader.next(); + } + value +} + +fn parse_cast( + reader: &mut ParseReader, + log: &mut dyn FnMut(Message, Span), +) -> Option> { + // cast ::= 'cast' '<' type '>' '(' assign ')'; + if reader.peek().token_type != TokenType::CastKeyword { + return None; + } + reader.next(); + reader.consume(TokenType::LessThan, log); + let cast_type = parse_type(reader, log); + // FIXME: deal with no cast type + reader.consume(TokenType::GreaterThan, log); + reader.consume(TokenType::OpenBracket, log); + let assign = parse_assign(reader, log); + // FIXME: deal with no assign + reader.consume(TokenType::CloseBracket, log); + + Some(Box::new(ParsedStatement::Cast { + cast_type: cast_type.unwrap(), + assign: assign.unwrap(), + })) +} + +fn parse_varaccess( + outer_reader: &mut ParseReader, + log: &mut dyn FnMut(Message, Span), +) -> Option> { + // scope identifier + let mut reader = outer_reader.create_inner(); + let scope = parse_scope(&mut reader, log); + let identifier = parse_identifier(&mut reader, log, true); + identifier.as_ref()?; + outer_reader.set_from_inner(&reader); + Some(Box::new(ParsedStatement::VarAccess { + scope, + identifier: identifier.unwrap(), + })) +} + +fn parse_constructcall( + outer_reader: &mut ParseReader, + log: &mut dyn FnMut(Message, Span), +) -> Option> { + // type arglist + let mut reader = outer_reader.create_inner(); + let construct_type = parse_type(&mut reader, log); + construct_type.as_ref()?; + let arg_list = parse_arglist(&mut reader, log); + // FIXME: deal with None value for arg list + + outer_reader.set_from_inner(&reader); + Some(Box::new(ParsedStatement::ConstructCall { + statement_type: construct_type, + arglist: arg_list.unwrap(), + })) +} + +fn parse_funccall( + outer_reader: &mut ParseReader, + log: &mut dyn FnMut(Message, Span), +) -> Option> { + // scope identifier arglist + let mut reader = outer_reader.create_inner(); + let scope = parse_scope(&mut reader, log); + let identifier = parse_identifier(&mut reader, log, true); + identifier.as_ref()?; + + let arg_list = parse_arglist(&mut reader, log); + arg_list.as_ref()?; + + outer_reader.set_from_inner(&reader); + Some(Box::new(ParsedStatement::FuncCall { + scope, + identifier: identifier.unwrap(), + arglist: arg_list.unwrap(), + })) +} + +fn parse_arglist( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_exprpreop( + reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option { + // exprpreop ::= '-' | '+' | '!' | '++' | '--' | '~' | '@'; + + let op = match reader.peek().token_type { + TokenType::Minus => Some(PreOperator::Negative), + TokenType::Plus => Some(PreOperator::Identity), + TokenType::ExclamationMark => Some(PreOperator::Negation), + TokenType::PlusPlus => Some(PreOperator::Increment), + TokenType::MinusMinus => Some(PreOperator::Decrement), + TokenType::Tilde => Some(PreOperator::Complement), + TokenType::AtSymbol => Some(PreOperator::HandleOf), + _ => None, + }; + + if op.is_some() { + reader.next(); + } + op +} + +fn parse_initlist( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_assign( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + fn parse_type( reader: &mut ParseReader, log: &mut dyn FnMut(Message, Span), @@ -427,9 +962,8 @@ fn parse_type( is_const = true; } let scope = parse_scope(reader, log); - let identifier = parse_identifier(reader, log); - // if none, we already logged an error. Return None. - identifier.as_ref()?; + let datatype = parse_datatype(reader, log); + datatype.as_ref()?; // TODO: Generics @@ -442,7 +976,7 @@ fn parse_type( reader.consume(TokenType::CloseBlockBracket, log); modifiers.push(ParsedTypeModifier::Array); } - TokenType::Ampersand => { + TokenType::AtSymbol => { reader.next(); if reader.peek().token_type == TokenType::ConstKeyword { reader.next(); @@ -458,7 +992,7 @@ fn parse_type( Some(Box::new(ParsedStatement::Type { is_const, scope, - identifier: identifier.unwrap(), + datatype: datatype.unwrap(), modifiers, })) } @@ -467,6 +1001,7 @@ fn parse_scope( outer_reader: &mut ParseReader, _log: &mut dyn FnMut(Message, Span), ) -> Option> { + // scope ::= ['::'] {identifier '::'} [identifier ['<' type {',' type} '>'] '::'] let mut reader = outer_reader.create_inner(); let is_global = reader.peek().token_type == TokenType::ColonColon; if is_global { @@ -500,3 +1035,174 @@ fn parse_scope( })) } } + +fn parse_datatype( + reader: &mut ParseReader, + log: &mut dyn FnMut(Message, Span), +) -> Option> { + if let Some(identifier) = parse_identifier(reader, log, true) { + return Some(Box::new(ParsedStatement::DataTypeIdentifier { identifier })); + } + let key = reader.peek(); + let res = match &key.token_type { + TokenType::AutoKeyword => Some(Box::new(ParsedStatement::DataTypeAuto {})), + TokenType::VoidKeyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::Void, + })), + TokenType::Int8Keyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::Int8, + })), + TokenType::Int16Keyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::Int16, + })), + TokenType::Int32Keyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::Int32, + })), + TokenType::Int64Keyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::Int64, + })), + TokenType::Uint8Keyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::UInt8, + })), + TokenType::Uint16Keyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::UInt16, + })), + TokenType::Uint32Keyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::UInt32, + })), + TokenType::Uint64Keyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::UInt64, + })), + TokenType::FloatKeyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::Float, + })), + TokenType::DoubleKeyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::Double, + })), + TokenType::BoolKeyword => Some(Box::new(ParsedStatement::DataTypePrimType { + prim_type: PrimitiveType::Bool, + })), + _ => None, + }; + if res.is_some() { + reader.next(); + } + res +} + +fn parse_primtype( + _outer_reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option> { + unimplemented!(); +} + +fn parse_bitop( + reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option { + // bitop ::= '&' | '|' | '^' | '<<' | '>>' | '>>>'; + let operand = match reader.peek().token_type { + TokenType::Ampersand => Some(BinaryOperator::BitwiseAnd), + TokenType::VerticalLine => Some(BinaryOperator::BitwiseOr), + TokenType::Roof => Some(BinaryOperator::BitwiseXor), + TokenType::LeftLeft => Some(BinaryOperator::LeftShift), + TokenType::RightRight => Some(BinaryOperator::RightShift), + TokenType::RightRightRight => Some(BinaryOperator::ArithmeticRightShift), + _ => None, + }; + if operand.is_some() { + reader.next(); + } + operand +} + +fn parse_mathop( + reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option { + // mathop := '+' | '-' | '*' | '/' | '%' | '**'; + let operand = match reader.peek().token_type { + TokenType::Plus => Some(BinaryOperator::Addition), + TokenType::Minus => Some(BinaryOperator::Subtraction), + TokenType::Star => Some(BinaryOperator::Multiplication), + TokenType::Slash => Some(BinaryOperator::Division), + TokenType::Percent => Some(BinaryOperator::Modulo), + TokenType::StarStar => Some(BinaryOperator::Exponentiation), + _ => None, + }; + if operand.is_some() { + reader.next(); + } + operand +} + +fn parse_compop( + reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option { + // compop ::= '==' | '!=' | '<' | '<=' | '>' | '>=' | 'is' | '!is'; + let operand = match reader.peek().token_type { + TokenType::EqualsEquals => Some(BinaryOperator::Equality), + TokenType::NotEquals => Some(BinaryOperator::Inequality), + TokenType::LessThan => Some(BinaryOperator::LessThan), + TokenType::LessThanEquals => Some(BinaryOperator::LessThanEquals), + TokenType::GreaterThan => Some(BinaryOperator::GreaterThan), + TokenType::GreaterThanEquals => Some(BinaryOperator::GreaterThanEquals), + TokenType::IsKeyword => Some(BinaryOperator::IdentityEquality), + TokenType::NotIsKeyword => Some(BinaryOperator::IdentityInequality), + _ => None, + }; + if operand.is_some() { + reader.next(); + } + operand +} + +fn parse_logicop( + reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option { + // logicop ::= '&&' | '||' | '^^' | 'and' | 'or' | 'xor'; + let operand = match reader.peek().token_type { + TokenType::AmpersandAmpersand => Some(BinaryOperator::And), + TokenType::LineLine => Some(BinaryOperator::Or), + TokenType::RoofRoof => Some(BinaryOperator::Xor), + TokenType::AndKeyword => Some(BinaryOperator::And), + TokenType::OrKeyword => Some(BinaryOperator::Or), + TokenType::XorKeyword => Some(BinaryOperator::Xor), + _ => None, + }; + if operand.is_some() { + reader.next(); + } + operand +} + +fn parse_assignop( + reader: &mut ParseReader, + _log: &mut dyn FnMut(Message, Span), +) -> Option { + // assignop ::= '=' | '+=' | '-=' | '*=' | '/=' | '|=' | '&=' | '^=' | '%=' | '**=' | '<<=' | '>>=' | '>>>='; + let operand = match reader.peek().token_type { + TokenType::Equals => Some(BinaryOperator::Assignment), + TokenType::PlusEquals => Some(BinaryOperator::AssignmentAddition), + TokenType::MinusEquals => Some(BinaryOperator::AssignmentSubtraction), + TokenType::StarEquals => Some(BinaryOperator::AssignmentMultiplication), + TokenType::PercentEquals => Some(BinaryOperator::AssignmentModulo), + TokenType::StarStarEquals => Some(BinaryOperator::AssignmentExponentiation), + + TokenType::SlashEquals => Some(BinaryOperator::AssignmentDivision), + TokenType::LineEquals => Some(BinaryOperator::AssignmentBitwiseOr), + TokenType::AmpersandEquals => Some(BinaryOperator::AssignmentBitwiseAnd), + TokenType::RoofEquals => Some(BinaryOperator::AssignmentBitwiseXor), + TokenType::LeftLeftEquals => Some(BinaryOperator::AssignmentLeftShift), + TokenType::RightRightEquals => Some(BinaryOperator::AssignmentRightShift), + TokenType::RightRightRightEquals => Some(BinaryOperator::AssignmentArithmeticRightShift), + _ => None, + }; + if operand.is_some() { + reader.next(); + } + operand +} diff --git a/src/parsing/parser/parsed_statement.rs b/src/parsing/parser/parsed_statement.rs index a7a06a8..e28ab07 100644 --- a/src/parsing/parser/parsed_statement.rs +++ b/src/parsing/parser/parsed_statement.rs @@ -1,7 +1,11 @@ use super::parsed_type_modifier::ParsedTypeModifier; +use crate::defines::{LiteralFloat, LiteralInt}; use crate::modifiers::{FieldModifier, TypeModifier}; +use crate::parsing::parser::parser_operators::{BinaryOperator, PreOperator}; +use crate::prim_type::PrimitiveType; use enumflags2::BitFlags; +#[derive(PartialEq, Debug)] pub enum ParsedStatement { Invalid, Script { @@ -36,10 +40,61 @@ pub enum ParsedStatement { is_set_const: bool, set_statement: Option>, }, + DataTypeIdentifier { + identifier: String, + }, + DataTypeAuto {}, + DataTypePrimType { + prim_type: PrimitiveType, + }, Type { is_const: bool, scope: Option>, - identifier: String, + datatype: Box, modifiers: Vec, }, + Var { + modifier: BitFlags, + var_type: Box, + identifier: String, + assignment: Option>, + }, + ExprTermInitList { + statement_type: Option>, + initlist: Box, + }, + ExprTerm { + pre_op: Vec, + exp_value: Box, + }, + ExprPreOp { + operator: PreOperator, + }, + ExprVoidValue {}, + ConstructCall { + statement_type: Option>, + arglist: Box, + }, + FuncCall { + scope: Option>, + identifier: String, + arglist: Box, + }, + VarAccess { + scope: Option>, + identifier: String, + }, + Cast { + cast_type: Box, + assign: Box, + }, + IntegerLiteral(LiteralInt), + FloatLiteral(LiteralFloat), + BoolLiteral(bool), + StringLiteral(String), + BinaryExpr { + left: Box, + operator: BinaryOperator, + right: Box, + }, } diff --git a/src/parsing/parser/parsed_type_modifier.rs b/src/parsing/parser/parsed_type_modifier.rs index 4ad2bc8..bcd4671 100644 --- a/src/parsing/parser/parsed_type_modifier.rs +++ b/src/parsing/parser/parsed_type_modifier.rs @@ -1,3 +1,4 @@ +#[derive(Eq, PartialEq, Debug)] pub enum ParsedTypeModifier { Array, Handle, diff --git a/src/parsing/parser/parser_operators.rs b/src/parsing/parser/parser_operators.rs new file mode 100644 index 0000000..958569f --- /dev/null +++ b/src/parsing/parser/parser_operators.rs @@ -0,0 +1,55 @@ +#[derive(Eq, PartialEq, Debug)] +pub enum PreOperator { + Negative, + Identity, + Negation, + Increment, + Decrement, + Complement, + HandleOf, +} + +#[derive(Eq, PartialEq, Debug)] +pub enum BinaryOperator { + // math op + Addition, + Subtraction, + Multiplication, + Division, + Modulo, + Exponentiation, + // comp op + Equality, + Inequality, + LessThan, + LessThanEquals, + GreaterThan, + GreaterThanEquals, + IdentityEquality, + IdentityInequality, + // logic op + And, + Or, + Xor, + // bit op + BitwiseAnd, + BitwiseOr, + BitwiseXor, + LeftShift, + RightShift, + ArithmeticRightShift, + // assign op + Assignment, + AssignmentAddition, + AssignmentSubtraction, + AssignmentMultiplication, + AssignmentDivision, + AssignmentModulo, + AssignmentExponentiation, + AssignmentBitwiseAnd, + AssignmentBitwiseOr, + AssignmentBitwiseXor, + AssignmentLeftShift, + AssignmentRightShift, + AssignmentArithmeticRightShift, +} diff --git a/src/parsing/parser/parser_tests.rs b/src/parsing/parser/parser_tests.rs index 296f548..8389864 100644 --- a/src/parsing/parser/parser_tests.rs +++ b/src/parsing/parser/parser_tests.rs @@ -2,6 +2,8 @@ use super::parse; use super::parsed_statement::ParsedStatement; use crate::modifiers::TypeModifier; use crate::parsing::lexer::lex_tokens::{LexToken, TokenType}; +use crate::parsing::parser::parsed_statement::ParsedStatement::DataTypeAuto; +use crate::parsing::parser::parser_operators::{BinaryOperator, PreOperator}; use crate::span::Span; use enumflags2::BitFlags; @@ -179,13 +181,17 @@ fn test_interface_with_virtprop() { { assert_eq!(*field_mod, BitFlags::empty()); if let ParsedStatement::Type { - is_const, - identifier, - .. + is_const, datatype, .. } = property_type.as_ref() { assert!(!is_const); - assert_eq!(identifier, "bar"); + if let ParsedStatement::DataTypeIdentifier { identifier, .. } = + datatype.as_ref() + { + assert_eq!(identifier, "bar"); + } else { + unreachable!() + } } else { unreachable!() } @@ -207,3 +213,173 @@ fn test_interface_with_virtprop() { unreachable!(); } } + +#[test] +fn test_assign_to_global_variable() { + let script = parse( + create_tokens(vec![ + TokenType::AutoKeyword, + TokenType::WhiteSpace, + TokenType::Identifier("foo".to_string()), + TokenType::WhiteSpace, + TokenType::Equals, + TokenType::WhiteSpace, + TokenType::IntegerLiteral(100), + TokenType::EndOfFile, + ]), + &mut |_message, _span| { + std::panic::panic_any(_message.stringify()); + }, + ); + if let ParsedStatement::Script { statements } = script.as_ref() { + assert_eq!(1, statements.len()); + if let ParsedStatement::Var { + modifier, + var_type, + identifier, + assignment, + } = &statements[0] + { + assert!(modifier.is_empty()); + assert_eq!( + *var_type.as_ref(), + ParsedStatement::Type { + is_const: false, + scope: None, + datatype: Box::from(DataTypeAuto {}), + modifiers: vec![] + } + ); + assert_eq!(identifier, "foo"); + assert!(assignment.is_some()); + assert_eq!( + *(assignment.as_ref().unwrap().as_ref()), + ParsedStatement::ExprTerm { + pre_op: vec![], + exp_value: Box::from(ParsedStatement::IntegerLiteral(100)), + } + ); + } else { + unreachable!() + } + } else { + unreachable!(); + } +} + +#[test] +fn test_assign_negative_to_global_variable() { + let script = parse( + create_tokens(vec![ + TokenType::AutoKeyword, + TokenType::WhiteSpace, + TokenType::Identifier("foo".to_string()), + TokenType::WhiteSpace, + TokenType::Equals, + TokenType::WhiteSpace, + TokenType::Minus, + TokenType::IntegerLiteral(100), + TokenType::EndOfFile, + ]), + &mut |_message, _span| { + std::panic::panic_any(_message.stringify()); + }, + ); + if let ParsedStatement::Script { statements } = script.as_ref() { + assert_eq!(1, statements.len()); + if let ParsedStatement::Var { + modifier, + var_type, + identifier, + assignment, + } = &statements[0] + { + assert!(modifier.is_empty()); + assert_eq!( + *var_type.as_ref(), + ParsedStatement::Type { + is_const: false, + scope: None, + datatype: Box::from(DataTypeAuto {}), + modifiers: vec![] + } + ); + assert_eq!(identifier, "foo"); + assert!(assignment.is_some()); + assert_eq!( + *(assignment.as_ref().unwrap().as_ref()), + ParsedStatement::ExprTerm { + pre_op: vec![PreOperator::Negative], + exp_value: Box::from(ParsedStatement::IntegerLiteral(100)), + } + ); + } else { + unreachable!() + } + } else { + unreachable!(); + } +} + +#[test] +fn test_assign_addition_to_global_variable() { + let script = parse( + create_tokens(vec![ + TokenType::AutoKeyword, + TokenType::WhiteSpace, + TokenType::Identifier("foo".to_string()), + TokenType::WhiteSpace, + TokenType::Equals, + TokenType::WhiteSpace, + TokenType::IntegerLiteral(100), + TokenType::Plus, + TokenType::Minus, + TokenType::IntegerLiteral(20), + TokenType::EndOfFile, + ]), + &mut |_message, _span| { + std::panic::panic_any(_message.stringify()); + }, + ); + if let ParsedStatement::Script { statements } = script.as_ref() { + assert_eq!(1, statements.len()); + if let ParsedStatement::Var { + modifier, + var_type, + identifier, + assignment, + } = &statements[0] + { + assert!(modifier.is_empty()); + assert_eq!( + *var_type.as_ref(), + ParsedStatement::Type { + is_const: false, + scope: None, + datatype: Box::from(DataTypeAuto {}), + modifiers: vec![] + } + ); + assert_eq!(identifier, "foo"); + assert!(assignment.is_some()); + assert_eq!( + *(assignment.as_ref().unwrap().as_ref()), + ParsedStatement::BinaryExpr { + left: Box::new(ParsedStatement::ExprTerm { + pre_op: vec![], + exp_value: Box::new(ParsedStatement::IntegerLiteral(100)), + }), + operator: BinaryOperator::Addition, + right: Box::new(ParsedStatement::ExprTerm { + pre_op: vec![PreOperator::Negative], + exp_value: Box::new(ParsedStatement::IntegerLiteral(20)), + }), + } + ); + } else { + unreachable!() + } + } else { + unreachable!(); + } +} diff --git a/src/prim_type.rs b/src/prim_type.rs new file mode 100644 index 0000000..b854f02 --- /dev/null +++ b/src/prim_type.rs @@ -0,0 +1,34 @@ +use crate::defines::PointerSize; + +#[derive(Eq, PartialEq, Debug)] +pub enum PrimitiveType { + Void, + Int8, + Int16, + Int32, + Int64, + UInt8, + UInt16, + UInt32, + UInt64, + Float, + Double, + Bool, +} + +pub fn get_primitive_type_byte_size(t: PrimitiveType) -> PointerSize { + match t { + PrimitiveType::Void => 0, + PrimitiveType::Int8 => 1, + PrimitiveType::Int16 => 2, + PrimitiveType::Int32 => 4, + PrimitiveType::Int64 => 8, + PrimitiveType::UInt8 => 1, + PrimitiveType::UInt16 => 2, + PrimitiveType::UInt32 => 4, + PrimitiveType::UInt64 => 8, + PrimitiveType::Float => 4, + PrimitiveType::Double => 8, + PrimitiveType::Bool => 1, + } +}