add first working version
This commit is contained in:
@@ -0,0 +1,401 @@
|
||||
/**
|
||||
* EFTL Token Types
|
||||
*/
|
||||
export enum TokenType {
|
||||
// Block delimiters
|
||||
EFTL_OPEN = 'EFTL_OPEN', // [EFTL]
|
||||
EFTL_CLOSE = 'EFTL_CLOSE', // [/EFTL]
|
||||
|
||||
// Variable
|
||||
VAR_OPEN = 'VAR_OPEN', // [VAR ...]
|
||||
VAR_CLOSE = 'VAR_CLOSE', // [/VAR]
|
||||
|
||||
// Control structures
|
||||
IF_OPEN = 'IF_OPEN', // [IF]
|
||||
IF_CLOSE = 'IF_CLOSE', // [/IF]
|
||||
CONDITION_OPEN = 'CONDITION_OPEN', // [CONDITION]
|
||||
CONDITION_CLOSE = 'CONDITION_CLOSE', // [/CONDITION]
|
||||
THEN_OPEN = 'THEN_OPEN', // [THEN]
|
||||
THEN_CLOSE = 'THEN_CLOSE', // [/THEN]
|
||||
ELSE_OPEN = 'ELSE_OPEN', // [ELSE]
|
||||
ELSE_CLOSE = 'ELSE_CLOSE', // [/ELSE]
|
||||
ELSE_IF_OPEN = 'ELSE_IF_OPEN', // [ELSE IF]
|
||||
ELSE_IF_CLOSE = 'ELSE_IF_CLOSE', // [/ELSE IF]
|
||||
WHILE_OPEN = 'WHILE_OPEN', // [WHILE]
|
||||
WHILE_CLOSE = 'WHILE_CLOSE', // [/WHILE]
|
||||
DO_OPEN = 'DO_OPEN', // [DO]
|
||||
DO_CLOSE = 'DO_CLOSE', // [/DO]
|
||||
|
||||
// Functions
|
||||
TAG_OPEN = 'TAG_OPEN', // [TAG]
|
||||
TAG_CLOSE = 'TAG_CLOSE', // [/TAG]
|
||||
SPLIT_OPEN = 'SPLIT_OPEN', // [SPLIT ...]
|
||||
SPLIT_CLOSE = 'SPLIT_CLOSE', // [/SPLIT]
|
||||
TRIM_OPEN = 'TRIM_OPEN', // [TRIM]
|
||||
TRIM_CLOSE = 'TRIM_CLOSE', // [/TRIM]
|
||||
CONTAINS_OPEN = 'CONTAINS_OPEN', // [CONTAINS ...]
|
||||
CONTAINS_CLOSE = 'CONTAINS_CLOSE', // [/CONTAINS]
|
||||
FORMAT_OPEN = 'FORMAT_OPEN', // [FORMAT ...]
|
||||
FORMAT_CLOSE = 'FORMAT_CLOSE', // [/FORMAT]
|
||||
VALUE_OF = 'VALUE_OF', // [VALUE_OF ... /]
|
||||
SIZE_OF = 'SIZE_OF', // [SIZE_OF ... /]
|
||||
HEADER = 'HEADER', // [HEADER ... /]
|
||||
|
||||
// Expressions
|
||||
EXPR_OPEN = 'EXPR_OPEN', // [%
|
||||
EXPR_CLOSE = 'EXPR_CLOSE', // %]
|
||||
EXPR_OUTPUT_OPEN = 'EXPR_OUTPUT_OPEN', // [%=
|
||||
|
||||
// Comments
|
||||
COMMENT_OPEN = 'COMMENT_OPEN', // [!--
|
||||
COMMENT_CLOSE = 'COMMENT_CLOSE', // --]
|
||||
|
||||
// Content
|
||||
TEXT = 'TEXT', // Plain text
|
||||
IDENTIFIER = 'IDENTIFIER', // Variable names
|
||||
STRING = 'STRING', // "..."
|
||||
NUMBER = 'NUMBER', // 123, 45.67
|
||||
|
||||
// Other
|
||||
EOF = 'EOF',
|
||||
ERROR = 'ERROR'
|
||||
}
|
||||
|
||||
export interface Token {
|
||||
type: TokenType;
|
||||
value: string;
|
||||
line: number;
|
||||
column: number;
|
||||
length: number;
|
||||
}
|
||||
|
||||
export interface TokenizerError {
|
||||
message: string;
|
||||
line: number;
|
||||
column: number;
|
||||
length: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* EFTL Tokenizer - converts source text into tokens
|
||||
*/
|
||||
export class EftlTokenizer {
|
||||
private source: string;
|
||||
private pos: number = 0;
|
||||
private line: number = 1;
|
||||
private column: number = 1;
|
||||
private tokens: Token[] = [];
|
||||
private errors: TokenizerError[] = [];
|
||||
|
||||
constructor(source: string) {
|
||||
this.source = source;
|
||||
}
|
||||
|
||||
tokenize(): { tokens: Token[]; errors: TokenizerError[] } {
|
||||
this.tokens = [];
|
||||
this.errors = [];
|
||||
this.pos = 0;
|
||||
this.line = 1;
|
||||
this.column = 1;
|
||||
|
||||
while (this.pos < this.source.length) {
|
||||
this.scanToken();
|
||||
}
|
||||
|
||||
this.tokens.push({
|
||||
type: TokenType.EOF,
|
||||
value: '',
|
||||
line: this.line,
|
||||
column: this.column,
|
||||
length: 0
|
||||
});
|
||||
|
||||
return { tokens: this.tokens, errors: this.errors };
|
||||
}
|
||||
|
||||
private scanToken(): void {
|
||||
const startLine = this.line;
|
||||
const startColumn = this.column;
|
||||
|
||||
// Check for EFTL constructs starting with [
|
||||
if (this.peek() === '[') {
|
||||
if (this.match('[EFTL]')) {
|
||||
this.addToken(TokenType.EFTL_OPEN, '[EFTL]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/EFTL]')) {
|
||||
this.addToken(TokenType.EFTL_CLOSE, '[/EFTL]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[!--')) {
|
||||
this.addToken(TokenType.COMMENT_OPEN, '[!--', startLine, startColumn);
|
||||
this.scanComment();
|
||||
return;
|
||||
}
|
||||
if (this.match('[%=')) {
|
||||
this.addToken(TokenType.EXPR_OUTPUT_OPEN, '[%=', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[%')) {
|
||||
this.addToken(TokenType.EXPR_OPEN, '[%', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[VAR')) {
|
||||
this.scanVarOpen(startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/VAR]')) {
|
||||
this.addToken(TokenType.VAR_CLOSE, '[/VAR]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[IF]')) {
|
||||
this.addToken(TokenType.IF_OPEN, '[IF]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/IF]')) {
|
||||
this.addToken(TokenType.IF_CLOSE, '[/IF]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[CONDITION]')) {
|
||||
this.addToken(TokenType.CONDITION_OPEN, '[CONDITION]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/CONDITION]')) {
|
||||
this.addToken(TokenType.CONDITION_CLOSE, '[/CONDITION]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[THEN]')) {
|
||||
this.addToken(TokenType.THEN_OPEN, '[THEN]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/THEN]')) {
|
||||
this.addToken(TokenType.THEN_CLOSE, '[/THEN]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[ELSE IF]')) {
|
||||
this.addToken(TokenType.ELSE_IF_OPEN, '[ELSE IF]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/ELSE IF]')) {
|
||||
this.addToken(TokenType.ELSE_IF_CLOSE, '[/ELSE IF]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[ELSE]')) {
|
||||
this.addToken(TokenType.ELSE_OPEN, '[ELSE]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/ELSE]')) {
|
||||
this.addToken(TokenType.ELSE_CLOSE, '[/ELSE]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[WHILE]')) {
|
||||
this.addToken(TokenType.WHILE_OPEN, '[WHILE]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/WHILE]')) {
|
||||
this.addToken(TokenType.WHILE_CLOSE, '[/WHILE]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[DO]')) {
|
||||
this.addToken(TokenType.DO_OPEN, '[DO]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/DO]')) {
|
||||
this.addToken(TokenType.DO_CLOSE, '[/DO]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[TAG]')) {
|
||||
this.addToken(TokenType.TAG_OPEN, '[TAG]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/TAG]')) {
|
||||
this.addToken(TokenType.TAG_CLOSE, '[/TAG]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[SPLIT')) {
|
||||
this.scanTagWithAttributes(TokenType.SPLIT_OPEN, 'SPLIT', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/SPLIT]')) {
|
||||
this.addToken(TokenType.SPLIT_CLOSE, '[/SPLIT]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[TRIM]')) {
|
||||
this.addToken(TokenType.TRIM_OPEN, '[TRIM]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/TRIM]')) {
|
||||
this.addToken(TokenType.TRIM_CLOSE, '[/TRIM]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[CONTAINS')) {
|
||||
this.scanTagWithAttributes(TokenType.CONTAINS_OPEN, 'CONTAINS', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/CONTAINS]')) {
|
||||
this.addToken(TokenType.CONTAINS_CLOSE, '[/CONTAINS]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[FORMAT')) {
|
||||
this.scanTagWithAttributes(TokenType.FORMAT_OPEN, 'FORMAT', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[/FORMAT]')) {
|
||||
this.addToken(TokenType.FORMAT_CLOSE, '[/FORMAT]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[VALUE_OF')) {
|
||||
this.scanSelfClosingTag(TokenType.VALUE_OF, 'VALUE_OF', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[SIZE_OF')) {
|
||||
this.scanSelfClosingTag(TokenType.SIZE_OF, 'SIZE_OF', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
if (this.match('[HEADER')) {
|
||||
this.scanSelfClosingTag(TokenType.HEADER, 'HEADER', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Check for expression close
|
||||
if (this.match('%]')) {
|
||||
this.addToken(TokenType.EXPR_CLOSE, '%]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check for comment close
|
||||
if (this.match('--]')) {
|
||||
this.addToken(TokenType.COMMENT_CLOSE, '--]', startLine, startColumn);
|
||||
return;
|
||||
}
|
||||
|
||||
// Plain text - consume until we hit a special character
|
||||
this.scanText(startLine, startColumn);
|
||||
}
|
||||
|
||||
private scanComment(): void {
|
||||
const startLine = this.line;
|
||||
const startColumn = this.column;
|
||||
let content = '';
|
||||
|
||||
while (this.pos < this.source.length) {
|
||||
if (this.match('--]')) {
|
||||
this.addToken(TokenType.COMMENT_CLOSE, '--]', this.line, this.column - 3);
|
||||
return;
|
||||
}
|
||||
content += this.advance();
|
||||
}
|
||||
|
||||
// Unclosed comment
|
||||
this.errors.push({
|
||||
message: 'Unclosed comment: expected "--]"',
|
||||
line: startLine,
|
||||
column: startColumn,
|
||||
length: content.length
|
||||
});
|
||||
}
|
||||
|
||||
private scanVarOpen(startLine: number, startColumn: number): void {
|
||||
let value = '[VAR';
|
||||
// Scan until we find ] or ]
|
||||
while (this.pos < this.source.length && this.peek() !== ']') {
|
||||
value += this.advance();
|
||||
}
|
||||
if (this.peek() === ']') {
|
||||
value += this.advance();
|
||||
}
|
||||
this.addToken(TokenType.VAR_OPEN, value, startLine, startColumn);
|
||||
}
|
||||
|
||||
private scanTagWithAttributes(type: TokenType, tagName: string, startLine: number, startColumn: number): void {
|
||||
let value = '[' + tagName;
|
||||
while (this.pos < this.source.length && this.peek() !== ']') {
|
||||
value += this.advance();
|
||||
}
|
||||
if (this.peek() === ']') {
|
||||
value += this.advance();
|
||||
}
|
||||
this.addToken(type, value, startLine, startColumn);
|
||||
}
|
||||
|
||||
private scanSelfClosingTag(type: TokenType, tagName: string, startLine: number, startColumn: number): void {
|
||||
let value = '[' + tagName;
|
||||
while (this.pos < this.source.length) {
|
||||
const ch = this.advance();
|
||||
value += ch;
|
||||
if (value.endsWith('/]')) {
|
||||
break;
|
||||
}
|
||||
if (ch === ']') {
|
||||
// Not self-closing, error
|
||||
this.errors.push({
|
||||
message: `Expected self-closing tag: [${tagName} ... /]`,
|
||||
line: startLine,
|
||||
column: startColumn,
|
||||
length: value.length
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
this.addToken(type, value, startLine, startColumn);
|
||||
}
|
||||
|
||||
private scanText(startLine: number, startColumn: number): void {
|
||||
let text = '';
|
||||
while (this.pos < this.source.length) {
|
||||
const ch = this.peek();
|
||||
// Stop at special characters
|
||||
if (ch === '[' || (ch === '%' && this.peekNext() === ']') || (ch === '-' && this.peekNext() === '-' && this.peekAt(2) === ']')) {
|
||||
break;
|
||||
}
|
||||
text += this.advance();
|
||||
}
|
||||
if (text.length > 0) {
|
||||
this.addToken(TokenType.TEXT, text, startLine, startColumn);
|
||||
}
|
||||
}
|
||||
|
||||
private peek(): string {
|
||||
return this.source[this.pos] || '\0';
|
||||
}
|
||||
|
||||
private peekNext(): string {
|
||||
return this.source[this.pos + 1] || '\0';
|
||||
}
|
||||
|
||||
private peekAt(offset: number): string {
|
||||
return this.source[this.pos + offset] || '\0';
|
||||
}
|
||||
|
||||
private advance(): string {
|
||||
const ch = this.source[this.pos++];
|
||||
if (ch === '\n') {
|
||||
this.line++;
|
||||
this.column = 1;
|
||||
} else {
|
||||
this.column++;
|
||||
}
|
||||
return ch;
|
||||
}
|
||||
|
||||
private match(expected: string): boolean {
|
||||
if (this.source.substring(this.pos, this.pos + expected.length) === expected) {
|
||||
for (let i = 0; i < expected.length; i++) {
|
||||
this.advance();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private addToken(type: TokenType, value: string, line: number, column: number): void {
|
||||
this.tokens.push({
|
||||
type,
|
||||
value,
|
||||
line,
|
||||
column,
|
||||
length: value.length
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user