From 3c0695f8f294e93f62ce8ddcf580ac71fbf75e2d Mon Sep 17 00:00:00 2001 From: Pier Paolo MAMMI Date: Wed, 19 Aug 2026 14:12:48 +0200 Subject: [PATCH] refine tag management --- package-lock.json | 18 +++- package.json | 4 +- src/parser.ts | 123 +++++++++++++++++++++++++ syntaxes/eftl.tmLanguage.json | 6 +- test/grammar.test.js | 77 ++++++++++++++++ test/parser.test.js | 165 ++++++++++++++++++++++++++++++++-- 6 files changed, 380 insertions(+), 13 deletions(-) create mode 100644 test/grammar.test.js diff --git a/package-lock.json b/package-lock.json index 4be48f1..cbcd5fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,9 @@ "@types/node": "^20.10.0", "@types/vscode": "^1.75.0", "eslint": "^9.39.2", - "typescript": "^5.3.0" + "typescript": "^5.3.0", + "vscode-oniguruma": "^2.0.1", + "vscode-textmate": "^9.3.2" }, "engines": { "vscode": "^1.75.0" @@ -1217,6 +1219,20 @@ "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==", "license": "MIT" }, + "node_modules/vscode-oniguruma": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-2.0.1.tgz", + "integrity": "sha512-poJU8iHIWnC3vgphJnrLZyI3YdqRlR27xzqDmpPXYzA93R4Gk8z7T6oqDzDoHjoikA2aS82crdXFkjELCdJsjQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/vscode-textmate": { + "version": "9.3.2", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.3.2.tgz", + "integrity": "sha512-n2uGbUcrjhUEBH16uGA0TvUfhWwliFZ1e3+pTjrkim1Mt7ydB41lV08aUvsi70OlzDWp6X7Bx3w/x3fAXIsN0Q==", + "dev": true, + "license": "MIT" + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index decfd35..f472f85 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,9 @@ "@types/node": "^20.10.0", "@types/vscode": "^1.75.0", "eslint": "^9.39.2", - "typescript": "^5.3.0" + "typescript": "^5.3.0", + "vscode-oniguruma": "^2.0.1", + "vscode-textmate": "^9.3.2" }, "repository": { "type": "git", diff --git a/src/parser.ts b/src/parser.ts index 8da550a..bc65cab 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -17,6 +17,13 @@ export interface ParserError { length: number; } +type IfPhase = 'condition' | 'then' | 'branches' | 'afterElse'; + +interface IfContext { + token: Token; + phase: IfPhase; +} + /** * EFTL Parser - validates structure and reports errors */ @@ -25,6 +32,8 @@ export class EftlParser { private pos: number = 0; private errors: ParserError[] = []; private blockStack: { type: TokenType; token: Token }[] = []; + private ifContexts: IfContext[] = []; + private eftlRootCount: number = 0; constructor(tokens: Token[]) { this.tokens = tokens; @@ -34,6 +43,8 @@ export class EftlParser { this.pos = 0; this.errors = []; this.blockStack = []; + this.ifContexts = []; + this.eftlRootCount = 0; while (!this.isAtEnd()) { this.parseTopLevel(); @@ -50,14 +61,32 @@ export class EftlParser { }); } + if (this.eftlRootCount === 0) { + const eof = this.current(); + this.errors.push({ + message: 'Missing required [EFTL] root block', + line: eof.line, + column: eof.column, + length: eof.length + }); + } + return { errors: this.errors }; } private parseTopLevel(): void { const token = this.current(); + this.validateOutsideEftl(token); + this.validateDirectIfContent(token); switch (token.type) { case TokenType.EFTL_OPEN: + if (this.blockStack.some(block => block.type === TokenType.EFTL_OPEN)) { + this.addError(token, 'EFTL root blocks cannot be nested'); + } else if (this.eftlRootCount > 0) { + this.addError(token, 'Only one [EFTL] root block is allowed'); + } + this.eftlRootCount++; this.pushBlock(token); this.advance(); break; @@ -68,6 +97,14 @@ export class EftlParser { break; case TokenType.VAR_OPEN: + if (this.blockStack.some(block => block.type === TokenType.VAR_OPEN)) { + this.errors.push({ + message: 'VAR tags cannot be nested inside another VAR tag', + line: token.line, + column: token.column, + length: token.length + }); + } this.pushBlock(token); this.advance(); break; @@ -78,12 +115,17 @@ export class EftlParser { break; case TokenType.IF_OPEN: + this.ifContexts.push({ token, phase: 'condition' }); this.pushBlock(token); this.advance(); break; case TokenType.IF_CLOSE: + const closesIf = this.blockStack[this.blockStack.length - 1]?.type === TokenType.IF_OPEN; this.popBlock(TokenType.IF_OPEN, token); + if (closesIf) { + this.ifContexts.pop(); + } this.advance(); break; @@ -255,6 +297,87 @@ export class EftlParser { } } + private validateOutsideEftl(token: Token): void { + if (this.blockStack.some(block => block.type === TokenType.EFTL_OPEN)) { + return; + } + + if (token.type === TokenType.EFTL_OPEN || token.type === TokenType.EFTL_CLOSE || + token.type === TokenType.COMMENT_OPEN || token.type === TokenType.COMMENT_CLOSE || + token.type === TokenType.EOF || (token.type === TokenType.TEXT && token.value.trim() === '')) { + return; + } + + this.addError(token, 'Only comments and whitespace are allowed outside the [EFTL] root block'); + } + + private validateDirectIfContent(token: Token): void { + const parent = this.blockStack[this.blockStack.length - 1]; + if (!parent || parent.type !== TokenType.IF_OPEN) { + return; + } + + const context = this.ifContexts[this.ifContexts.length - 1]; + if (!context || context.token !== parent.token) { + return; + } + + if (token.type === TokenType.TEXT && token.value.trim() === '') { + return; + } + + if (token.type === TokenType.IF_OPEN) { + this.addError(token, 'IF cannot be nested directly inside another IF; place it inside a branch block'); + return; + } + + if (context.phase === 'condition' && token.type === TokenType.CONDITION_OPEN) { + context.phase = 'then'; + return; + } + + if (context.phase === 'then' && token.type === TokenType.THEN_OPEN) { + context.phase = 'branches'; + return; + } + + if (context.phase === 'branches' && token.type === TokenType.ELSE_IF_OPEN) { + return; + } + + if (context.phase === 'branches' && token.type === TokenType.ELSE_OPEN) { + context.phase = 'afterElse'; + return; + } + + if (token.type === TokenType.IF_CLOSE) { + if (context.phase === 'condition') { + this.addError(token, 'Expected [CONDITION] immediately after [IF]'); + } else if (context.phase === 'then') { + this.addError(token, 'Expected [THEN] immediately after [/CONDITION]'); + } + return; + } + + const expected = context.phase === 'condition' + ? '[CONDITION]' + : context.phase === 'then' + ? '[THEN]' + : context.phase === 'branches' + ? '[ELSE IF], [ELSE], or [/IF]' + : '[/IF]'; + this.addError(token, `Expected ${expected} as the next direct child of [IF], got ${token.value}`); + } + + private addError(token: Token, message: string): void { + this.errors.push({ + message, + line: token.line, + column: token.column, + length: token.length + }); + } + private pushBlock(token: Token): void { if (token.value.endsWith('/]')) { return; // It's self-closing, don't push to stack diff --git a/syntaxes/eftl.tmLanguage.json b/syntaxes/eftl.tmLanguage.json index a80c2fb..98d94b2 100644 --- a/syntaxes/eftl.tmLanguage.json +++ b/syntaxes/eftl.tmLanguage.json @@ -174,7 +174,7 @@ "patterns": [ { "name": "meta.tag.block.eftl", - "begin": "(\\[)(TAG)(?:\\s+[^\\x00]*?)?(? new oniguruma.OnigScanner(patterns), + createOnigString: value => new oniguruma.OnigString(value) + }), + loadGrammar: async scopeName => scopeName === 'source.eftl' + ? textmate.parseRawGrammar( + fs.readFileSync('syntaxes/eftl.tmLanguage.json', 'utf8'), + 'eftl.tmLanguage.json' + ) + : null + }); + + return registry.loadGrammar('source.eftl'); +} + +test('highlights nested VAR blocks, self-closing functions, and WHILE attributes', async () => { + const grammar = await loadGrammar(); + const source = [ + '[EFTL]', + '[VAR name="count" type="number"][SIZE_OF varname="items" /][/VAR]', + '[WHILE threshold="99"][CONDITION][% count > 0 %][/CONDITION][DO]', + '[VAR name="current" type="string"][VALUE_OF varname="items" index="count" /][/VAR]', + '[/DO][/WHILE]', + '[IF][CONDITION][% count == 0 %][/CONDITION][THEN]', + '[VAR name="empty" type="boolean"][% empty = true; %][/VAR]', + '[/THEN][/IF]', + '[/EFTL]' + ]; + let ruleStack = textmate.INITIAL; + const variableTokens = []; + const selfClosingFunctionTokens = []; + const thresholdTokens = []; + const thresholdValueTokens = []; + const whileTokens = []; + + for (const line of source) { + const result = grammar.tokenizeLine(line, ruleStack); + ruleStack = result.ruleStack; + + for (const token of result.tokens) { + const text = line.slice(token.startIndex, token.endIndex); + if (text === 'VAR' && line[token.startIndex - 1] !== '/') variableTokens.push(token); + if (text === 'SIZE_OF' || text === 'VALUE_OF') selfClosingFunctionTokens.push(token); + if (text === 'threshold') thresholdTokens.push(token); + if (text === '"99"') thresholdValueTokens.push(token); + if (text === 'WHILE') whileTokens.push(token); + } + } + + assert.equal(variableTokens.length, 3); + for (const token of variableTokens) { + assert.ok(token.scopes.includes('storage.type.variable.eftl')); + } + + for (const token of selfClosingFunctionTokens) { + assert.ok(token.scopes.includes('meta.function.eftl')); + assert.ok(!token.scopes.includes('meta.function.block.eftl')); + } + + assert.equal(thresholdTokens.length, 1); + assert.ok(thresholdTokens[0].scopes.includes('entity.other.attribute-name.eftl')); + assert.equal(thresholdValueTokens.length, 1); + assert.ok(thresholdValueTokens[0].scopes.includes('string.quoted.double.eftl')); + assert.equal(whileTokens.length, 1); + assert.ok(whileTokens[0].scopes.includes('keyword.control.loop.eftl')); +}); \ No newline at end of file diff --git a/test/parser.test.js b/test/parser.test.js index 4f0b476..aa13107 100644 --- a/test/parser.test.js +++ b/test/parser.test.js @@ -10,24 +10,45 @@ function parse(source) { return new EftlParser(tokens).parse().errors; } +function messages(source) { + return parse(source).map(error => error.message); +} + test('accepts a correctly closed VAR tag', () => { - assert.deepEqual(parse('[VAR][/VAR]'), []); + assert.deepEqual(parse('[EFTL][VAR][/VAR][/EFTL]'), []); }); -test('reports a VAR tag without its closing tag', () => { - assert.deepEqual(parse('[VAR]'), [{ - message: 'Unclosed tag: [VAR] (expected [/VAR])', +test('accepts sibling VAR tags', () => { + assert.deepEqual(parse('[EFTL][VAR][/VAR][VAR][/VAR][/EFTL]'), []); +}); + +test('reports a VAR nested directly inside another VAR', () => { + assert.deepEqual(parse('[EFTL][VAR][VAR][/VAR][/VAR][/EFTL]'), [{ + message: 'VAR tags cannot be nested inside another VAR tag', line: 1, - column: 1, + column: 12, length: 5 }]); }); -test('reports an unexpected closing tag', () => { - const errors = parse('[/VAR]'); +test('reports a VAR nested indirectly inside another VAR', () => { + assert.deepEqual(parse('[EFTL][VAR][SPLIT][VAR][/VAR][/SPLIT][/VAR][/EFTL]'), [{ + message: 'VAR tags cannot be nested inside another VAR tag', + line: 1, + column: 19, + length: 5 + }]); +}); + +test('reports a VAR tag without its closing tag', () => { + assert.ok(messages('[EFTL][VAR]').includes('Unclosed tag: [VAR] (expected [/VAR])')); +}); + +test('reports a mismatched closing tag', () => { + const errors = parse('[EFTL][/VAR][/EFTL]'); assert.equal(errors.length, 1); - assert.match(errors[0].message, /Unexpected closing tag/); + assert.equal(errors[0].message, 'Mismatched closing tag: expected [/EFTL], got [/VAR]'); }); test('reports every unclosed nested tag at its opening token', () => { @@ -45,4 +66,132 @@ test('reports every unclosed nested tag at its opening token', () => { length: 5 } ]); +}); + +test('requires exactly one EFTL root block', () => { + assert.deepEqual(parse(''), [{ + message: 'Missing required [EFTL] root block', + line: 1, + column: 1, + length: 0 + }]); + + assert.deepEqual(parse('[EFTL][/EFTL]\n[EFTL][/EFTL]'), [{ + message: 'Only one [EFTL] root block is allowed', + line: 2, + column: 1, + length: 6 + }]); +}); + +test('allows comments and multiline whitespace around the EFTL root', () => { + assert.deepEqual(parse(' \n[!-- before --]\n[EFTL][/EFTL]\n[!-- after --]\n '), []); +}); + +test('rejects instructions outside the EFTL root', () => { + assert.deepEqual(messages('[VAR][/VAR][EFTL][/EFTL]'), [ + 'Only comments and whitespace are allowed outside the [EFTL] root block', + 'Only comments and whitespace are allowed outside the [EFTL] root block' + ]); +}); + +test('rejects a nested EFTL root', () => { + assert.deepEqual(parse('[EFTL][EFTL][/EFTL][/EFTL]'), [{ + message: 'EFTL root blocks cannot be nested', + line: 1, + column: 7, + length: 6 + }]); +}); + +test('accepts CONDITION, THEN, repeated ELSE IF, and one final ELSE in an IF', () => { + const source = [ + '[EFTL][IF]', + ' [CONDITION][% true %][/CONDITION]', + ' [THEN][/THEN]', + ' [ELSE IF][CONDITION][% false %][/CONDITION][THEN][/THEN][/ELSE IF]', + ' [ELSE IF][CONDITION][% false %][/CONDITION][THEN][/THEN][/ELSE IF]', + ' [ELSE][/ELSE]', + '[/IF][/EFTL]' + ].join('\n'); + + assert.deepEqual(parse(source), []); +}); + +test('allows an IF nested inside a THEN block', () => { + const source = '[EFTL][IF][CONDITION][/CONDITION][THEN]' + + '[IF][CONDITION][/CONDITION][THEN][/THEN][/IF]' + + '[/THEN][/IF][/EFTL]'; + + assert.deepEqual(parse(source), []); +}); + +test('rejects an IF nested directly inside another IF', () => { + const source = '[EFTL][IF]' + + '[IF][CONDITION][/CONDITION][THEN][/THEN][/IF]' + + '[CONDITION][/CONDITION][THEN][/THEN][/IF][/EFTL]'; + + assert.deepEqual(messages(source), [ + 'IF cannot be nested directly inside another IF; place it inside a branch block' + ]); +}); + +test('requires CONDITION followed by THEN as the first direct IF children', () => { + assert.deepEqual(messages('[EFTL][IF][/IF][/EFTL]'), [ + 'Expected [CONDITION] immediately after [IF]' + ]); + + assert.deepEqual(messages('[EFTL][IF][THEN][/THEN][CONDITION][/CONDITION][THEN][/THEN][/IF][/EFTL]'), [ + 'Expected [CONDITION] as the next direct child of [IF], got [THEN]' + ]); + + assert.deepEqual(messages('[EFTL][IF][CONDITION][/CONDITION][/IF][/EFTL]'), [ + 'Expected [THEN] immediately after [/CONDITION]' + ]); +}); + +test('allows only whitespace between direct IF child blocks', () => { + const source = '[EFTL][IF][CONDITION][/CONDITION]' + + '[VAR][/VAR][!-- comment --]text' + + '[THEN][/THEN][/IF][/EFTL]'; + + assert.deepEqual(messages(source), [ + 'Expected [THEN] as the next direct child of [IF], got [VAR]', + 'Expected [THEN] as the next direct child of [IF], got [!--', + 'Expected [THEN] as the next direct child of [IF], got text' + ]); +}); + +test('allows ELSE IF only before the optional ELSE', () => { + const source = '[EFTL][IF][CONDITION][/CONDITION][THEN][/THEN]' + + '[ELSE][/ELSE][ELSE IF][/ELSE IF][/IF][/EFTL]'; + + assert.deepEqual(messages(source), [ + 'Expected [/IF] as the next direct child of [IF], got [ELSE IF]' + ]); + + assert.deepEqual(messages( + '[EFTL][IF][CONDITION][/CONDITION][THEN][/THEN][ELSE][/ELSE][ELSE][/ELSE][/IF][/EFTL]' + ), [ + 'Expected [/IF] as the next direct child of [IF], got [ELSE]' + ]); +}); + +test('accepts WHILE with an optional string threshold attribute', () => { + assert.deepEqual(parse('[EFTL][WHILE][/WHILE][/EFTL]'), []); + assert.deepEqual(parse('[EFTL][WHILE threshold="99"][/WHILE][/EFTL]'), []); + + const { tokens } = new EftlTokenizer('[WHILE threshold="99"][/WHILE]').tokenize(); + assert.equal(tokens[0].type, 'WHILE_OPEN'); + assert.equal(tokens[0].value, '[WHILE threshold="99"]'); +}); + +test('rejects invalid WHILE attributes', () => { + const expectedMessage = 'WHILE accepts only the optional string attribute threshold="..."'; + + assert.deepEqual(messages('[EFTL][WHILE threshold=99][/WHILE][/EFTL]'), [expectedMessage]); + assert.deepEqual(messages('[EFTL][WHILE limit="99"][/WHILE][/EFTL]'), [expectedMessage]); + assert.deepEqual(messages( + '[EFTL][WHILE threshold="10" threshold="20"][/WHILE][/EFTL]' + ), [expectedMessage]); }); \ No newline at end of file