402 lines
14 KiB
TypeScript
402 lines
14 KiB
TypeScript
/**
|
|
* EFTL Token Types
|
|
*/
|
|
export enum TokenType {
|
|
// Block delimiters
|
|
EFTL_OPEN = 'EFTL_OPEN', // [EFTL]
|
|
EFTL_CLOSE = 'EFTL_CLOSE', // [/EFTL]
|
|
|
|
// Variable
|
|
VAR_OPEN = 'VAR_OPEN', // [VAR ...]
|
|
VAR_CLOSE = 'VAR_CLOSE', // [/VAR]
|
|
|
|
// Control structures
|
|
IF_OPEN = 'IF_OPEN', // [IF]
|
|
IF_CLOSE = 'IF_CLOSE', // [/IF]
|
|
CONDITION_OPEN = 'CONDITION_OPEN', // [CONDITION]
|
|
CONDITION_CLOSE = 'CONDITION_CLOSE', // [/CONDITION]
|
|
THEN_OPEN = 'THEN_OPEN', // [THEN]
|
|
THEN_CLOSE = 'THEN_CLOSE', // [/THEN]
|
|
ELSE_OPEN = 'ELSE_OPEN', // [ELSE]
|
|
ELSE_CLOSE = 'ELSE_CLOSE', // [/ELSE]
|
|
ELSE_IF_OPEN = 'ELSE_IF_OPEN', // [ELSE IF]
|
|
ELSE_IF_CLOSE = 'ELSE_IF_CLOSE', // [/ELSE IF]
|
|
WHILE_OPEN = 'WHILE_OPEN', // [WHILE]
|
|
WHILE_CLOSE = 'WHILE_CLOSE', // [/WHILE]
|
|
DO_OPEN = 'DO_OPEN', // [DO]
|
|
DO_CLOSE = 'DO_CLOSE', // [/DO]
|
|
|
|
// Functions
|
|
TAG_OPEN = 'TAG_OPEN', // [TAG]
|
|
TAG_CLOSE = 'TAG_CLOSE', // [/TAG]
|
|
SPLIT_OPEN = 'SPLIT_OPEN', // [SPLIT ...]
|
|
SPLIT_CLOSE = 'SPLIT_CLOSE', // [/SPLIT]
|
|
TRIM_OPEN = 'TRIM_OPEN', // [TRIM]
|
|
TRIM_CLOSE = 'TRIM_CLOSE', // [/TRIM]
|
|
CONTAINS_OPEN = 'CONTAINS_OPEN', // [CONTAINS ...]
|
|
CONTAINS_CLOSE = 'CONTAINS_CLOSE', // [/CONTAINS]
|
|
FORMAT_OPEN = 'FORMAT_OPEN', // [FORMAT ...]
|
|
FORMAT_CLOSE = 'FORMAT_CLOSE', // [/FORMAT]
|
|
VALUE_OF = 'VALUE_OF', // [VALUE_OF ... /]
|
|
SIZE_OF = 'SIZE_OF', // [SIZE_OF ... /]
|
|
HEADER = 'HEADER', // [HEADER ... /]
|
|
|
|
// Expressions
|
|
EXPR_OPEN = 'EXPR_OPEN', // [%
|
|
EXPR_CLOSE = 'EXPR_CLOSE', // %]
|
|
EXPR_OUTPUT_OPEN = 'EXPR_OUTPUT_OPEN', // [%=
|
|
|
|
// Comments
|
|
COMMENT_OPEN = 'COMMENT_OPEN', // [!--
|
|
COMMENT_CLOSE = 'COMMENT_CLOSE', // --]
|
|
|
|
// Content
|
|
TEXT = 'TEXT', // Plain text
|
|
IDENTIFIER = 'IDENTIFIER', // Variable names
|
|
STRING = 'STRING', // "..."
|
|
NUMBER = 'NUMBER', // 123, 45.67
|
|
|
|
// Other
|
|
EOF = 'EOF',
|
|
ERROR = 'ERROR'
|
|
}
|
|
|
|
export interface Token {
|
|
type: TokenType;
|
|
value: string;
|
|
line: number;
|
|
column: number;
|
|
length: number;
|
|
}
|
|
|
|
export interface TokenizerError {
|
|
message: string;
|
|
line: number;
|
|
column: number;
|
|
length: number;
|
|
}
|
|
|
|
/**
|
|
* EFTL Tokenizer - converts source text into tokens
|
|
*/
|
|
export class EftlTokenizer {
|
|
private source: string;
|
|
private pos: number = 0;
|
|
private line: number = 1;
|
|
private column: number = 1;
|
|
private tokens: Token[] = [];
|
|
private errors: TokenizerError[] = [];
|
|
|
|
constructor(source: string) {
|
|
this.source = source;
|
|
}
|
|
|
|
tokenize(): { tokens: Token[]; errors: TokenizerError[] } {
|
|
this.tokens = [];
|
|
this.errors = [];
|
|
this.pos = 0;
|
|
this.line = 1;
|
|
this.column = 1;
|
|
|
|
while (this.pos < this.source.length) {
|
|
this.scanToken();
|
|
}
|
|
|
|
this.tokens.push({
|
|
type: TokenType.EOF,
|
|
value: '',
|
|
line: this.line,
|
|
column: this.column,
|
|
length: 0
|
|
});
|
|
|
|
return { tokens: this.tokens, errors: this.errors };
|
|
}
|
|
|
|
private scanToken(): void {
|
|
const startLine = this.line;
|
|
const startColumn = this.column;
|
|
|
|
// Check for EFTL constructs starting with [
|
|
if (this.peek() === '[') {
|
|
if (this.match('[EFTL]')) {
|
|
this.addToken(TokenType.EFTL_OPEN, '[EFTL]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/EFTL]')) {
|
|
this.addToken(TokenType.EFTL_CLOSE, '[/EFTL]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[!--')) {
|
|
this.addToken(TokenType.COMMENT_OPEN, '[!--', startLine, startColumn);
|
|
this.scanComment();
|
|
return;
|
|
}
|
|
if (this.match('[%=')) {
|
|
this.addToken(TokenType.EXPR_OUTPUT_OPEN, '[%=', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[%')) {
|
|
this.addToken(TokenType.EXPR_OPEN, '[%', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[VAR')) {
|
|
this.scanVarOpen(startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/VAR]')) {
|
|
this.addToken(TokenType.VAR_CLOSE, '[/VAR]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[IF]')) {
|
|
this.addToken(TokenType.IF_OPEN, '[IF]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/IF]')) {
|
|
this.addToken(TokenType.IF_CLOSE, '[/IF]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[CONDITION]')) {
|
|
this.addToken(TokenType.CONDITION_OPEN, '[CONDITION]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/CONDITION]')) {
|
|
this.addToken(TokenType.CONDITION_CLOSE, '[/CONDITION]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[THEN]')) {
|
|
this.addToken(TokenType.THEN_OPEN, '[THEN]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/THEN]')) {
|
|
this.addToken(TokenType.THEN_CLOSE, '[/THEN]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[ELSE IF]')) {
|
|
this.addToken(TokenType.ELSE_IF_OPEN, '[ELSE IF]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/ELSE IF]')) {
|
|
this.addToken(TokenType.ELSE_IF_CLOSE, '[/ELSE IF]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[ELSE]')) {
|
|
this.addToken(TokenType.ELSE_OPEN, '[ELSE]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/ELSE]')) {
|
|
this.addToken(TokenType.ELSE_CLOSE, '[/ELSE]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[WHILE]')) {
|
|
this.addToken(TokenType.WHILE_OPEN, '[WHILE]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/WHILE]')) {
|
|
this.addToken(TokenType.WHILE_CLOSE, '[/WHILE]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[DO]')) {
|
|
this.addToken(TokenType.DO_OPEN, '[DO]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/DO]')) {
|
|
this.addToken(TokenType.DO_CLOSE, '[/DO]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[TAG]')) {
|
|
this.addToken(TokenType.TAG_OPEN, '[TAG]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/TAG]')) {
|
|
this.addToken(TokenType.TAG_CLOSE, '[/TAG]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[SPLIT')) {
|
|
this.scanTagWithAttributes(TokenType.SPLIT_OPEN, 'SPLIT', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/SPLIT]')) {
|
|
this.addToken(TokenType.SPLIT_CLOSE, '[/SPLIT]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[TRIM]')) {
|
|
this.addToken(TokenType.TRIM_OPEN, '[TRIM]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/TRIM]')) {
|
|
this.addToken(TokenType.TRIM_CLOSE, '[/TRIM]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[CONTAINS')) {
|
|
this.scanTagWithAttributes(TokenType.CONTAINS_OPEN, 'CONTAINS', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/CONTAINS]')) {
|
|
this.addToken(TokenType.CONTAINS_CLOSE, '[/CONTAINS]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[FORMAT')) {
|
|
this.scanTagWithAttributes(TokenType.FORMAT_OPEN, 'FORMAT', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[/FORMAT]')) {
|
|
this.addToken(TokenType.FORMAT_CLOSE, '[/FORMAT]', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[VALUE_OF')) {
|
|
this.scanSelfClosingTag(TokenType.VALUE_OF, 'VALUE_OF', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[SIZE_OF')) {
|
|
this.scanSelfClosingTag(TokenType.SIZE_OF, 'SIZE_OF', startLine, startColumn);
|
|
return;
|
|
}
|
|
if (this.match('[HEADER')) {
|
|
this.scanSelfClosingTag(TokenType.HEADER, 'HEADER', startLine, startColumn);
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Check for expression close
|
|
if (this.match('%]')) {
|
|
this.addToken(TokenType.EXPR_CLOSE, '%]', startLine, startColumn);
|
|
return;
|
|
}
|
|
|
|
// Check for comment close
|
|
if (this.match('--]')) {
|
|
this.addToken(TokenType.COMMENT_CLOSE, '--]', startLine, startColumn);
|
|
return;
|
|
}
|
|
|
|
// Plain text - consume until we hit a special character
|
|
this.scanText(startLine, startColumn);
|
|
}
|
|
|
|
private scanComment(): void {
|
|
const startLine = this.line;
|
|
const startColumn = this.column;
|
|
let content = '';
|
|
|
|
while (this.pos < this.source.length) {
|
|
if (this.match('--]')) {
|
|
this.addToken(TokenType.COMMENT_CLOSE, '--]', this.line, this.column - 3);
|
|
return;
|
|
}
|
|
content += this.advance();
|
|
}
|
|
|
|
// Unclosed comment
|
|
this.errors.push({
|
|
message: 'Unclosed comment: expected "--]"',
|
|
line: startLine,
|
|
column: startColumn,
|
|
length: content.length
|
|
});
|
|
}
|
|
|
|
private scanVarOpen(startLine: number, startColumn: number): void {
|
|
let value = '[VAR';
|
|
// Scan until we find ] or ]
|
|
while (this.pos < this.source.length && this.peek() !== ']') {
|
|
value += this.advance();
|
|
}
|
|
if (this.peek() === ']') {
|
|
value += this.advance();
|
|
}
|
|
this.addToken(TokenType.VAR_OPEN, value, startLine, startColumn);
|
|
}
|
|
|
|
private scanTagWithAttributes(type: TokenType, tagName: string, startLine: number, startColumn: number): void {
|
|
let value = '[' + tagName;
|
|
while (this.pos < this.source.length && this.peek() !== ']') {
|
|
value += this.advance();
|
|
}
|
|
if (this.peek() === ']') {
|
|
value += this.advance();
|
|
}
|
|
this.addToken(type, value, startLine, startColumn);
|
|
}
|
|
|
|
private scanSelfClosingTag(type: TokenType, tagName: string, startLine: number, startColumn: number): void {
|
|
let value = '[' + tagName;
|
|
while (this.pos < this.source.length) {
|
|
const ch = this.advance();
|
|
value += ch;
|
|
if (value.endsWith('/]')) {
|
|
break;
|
|
}
|
|
if (ch === ']') {
|
|
// Not self-closing, error
|
|
this.errors.push({
|
|
message: `Expected self-closing tag: [${tagName} ... /]`,
|
|
line: startLine,
|
|
column: startColumn,
|
|
length: value.length
|
|
});
|
|
break;
|
|
}
|
|
}
|
|
this.addToken(type, value, startLine, startColumn);
|
|
}
|
|
|
|
private scanText(startLine: number, startColumn: number): void {
|
|
let text = '';
|
|
while (this.pos < this.source.length) {
|
|
const ch = this.peek();
|
|
// Stop at special characters
|
|
if (ch === '[' || (ch === '%' && this.peekNext() === ']') || (ch === '-' && this.peekNext() === '-' && this.peekAt(2) === ']')) {
|
|
break;
|
|
}
|
|
text += this.advance();
|
|
}
|
|
if (text.length > 0) {
|
|
this.addToken(TokenType.TEXT, text, startLine, startColumn);
|
|
}
|
|
}
|
|
|
|
private peek(): string {
|
|
return this.source[this.pos] || '\0';
|
|
}
|
|
|
|
private peekNext(): string {
|
|
return this.source[this.pos + 1] || '\0';
|
|
}
|
|
|
|
private peekAt(offset: number): string {
|
|
return this.source[this.pos + offset] || '\0';
|
|
}
|
|
|
|
private advance(): string {
|
|
const ch = this.source[this.pos++];
|
|
if (ch === '\n') {
|
|
this.line++;
|
|
this.column = 1;
|
|
} else {
|
|
this.column++;
|
|
}
|
|
return ch;
|
|
}
|
|
|
|
private match(expected: string): boolean {
|
|
if (this.source.substring(this.pos, this.pos + expected.length) === expected) {
|
|
for (let i = 0; i < expected.length; i++) {
|
|
this.advance();
|
|
}
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private addToken(type: TokenType, value: string, line: number, column: number): void {
|
|
this.tokens.push({
|
|
type,
|
|
value,
|
|
line,
|
|
column,
|
|
length: value.length
|
|
});
|
|
}
|
|
}
|