refine tag management

package-lock.json

@@ -16,7 +16,9 @@
         "@types/node": "^20.10.0",
         "@types/vscode": "^1.75.0",
         "eslint": "^9.39.2",
-        "typescript": "^5.3.0"
+        "typescript": "^5.3.0",
+        "vscode-oniguruma": "^2.0.1",
+        "vscode-textmate": "^9.3.2"
       },
       "engines": {
         "vscode": "^1.75.0"
@@ -1217,6 +1219,20 @@
       "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==",
       "license": "MIT"
     },
+    "node_modules/vscode-oniguruma": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-2.0.1.tgz",
+      "integrity": "sha512-poJU8iHIWnC3vgphJnrLZyI3YdqRlR27xzqDmpPXYzA93R4Gk8z7T6oqDzDoHjoikA2aS82crdXFkjELCdJsjQ==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/vscode-textmate": {
+      "version": "9.3.2",
+      "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.3.2.tgz",
+      "integrity": "sha512-n2uGbUcrjhUEBH16uGA0TvUfhWwliFZ1e3+pTjrkim1Mt7ydB41lV08aUvsi70OlzDWp6X7Bx3w/x3fAXIsN0Q==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

package.json

@@ -70,7 +70,9 @@
     "@types/node": "^20.10.0",
     "@types/vscode": "^1.75.0",
     "eslint": "^9.39.2",
-    "typescript": "^5.3.0"
+    "typescript": "^5.3.0",
+    "vscode-oniguruma": "^2.0.1",
+    "vscode-textmate": "^9.3.2"
   },
   "repository": {
     "type": "git",

src/parser.ts

@@ -17,6 +17,13 @@ export interface ParserError {
     length: number;
 }
 
+type IfPhase = 'condition' | 'then' | 'branches' | 'afterElse';
+
+interface IfContext {
+    token: Token;
+    phase: IfPhase;
+}
+
 /**
  * EFTL Parser - validates structure and reports errors
  */
@@ -25,6 +32,8 @@ export class EftlParser {
     private pos: number = 0;
     private errors: ParserError[] = [];
     private blockStack: { type: TokenType; token: Token }[] = [];
+    private ifContexts: IfContext[] = [];
+    private eftlRootCount: number = 0;
 
     constructor(tokens: Token[]) {
         this.tokens = tokens;
@@ -34,6 +43,8 @@ export class EftlParser {
         this.pos = 0;
         this.errors = [];
         this.blockStack = [];
+        this.ifContexts = [];
+        this.eftlRootCount = 0;
 
         while (!this.isAtEnd()) {
             this.parseTopLevel();
@@ -50,14 +61,32 @@ export class EftlParser {
             });
         }
 
+        if (this.eftlRootCount === 0) {
+            const eof = this.current();
+            this.errors.push({
+                message: 'Missing required [EFTL] root block',
+                line: eof.line,
+                column: eof.column,
+                length: eof.length
+            });
+        }
+
         return { errors: this.errors };
     }
 
     private parseTopLevel(): void {
         const token = this.current();
+        this.validateOutsideEftl(token);
+        this.validateDirectIfContent(token);
 
         switch (token.type) {
             case TokenType.EFTL_OPEN:
+                if (this.blockStack.some(block => block.type === TokenType.EFTL_OPEN)) {
+                    this.addError(token, 'EFTL root blocks cannot be nested');
+                } else if (this.eftlRootCount > 0) {
+                    this.addError(token, 'Only one [EFTL] root block is allowed');
+                }
+                this.eftlRootCount++;
                 this.pushBlock(token);
                 this.advance();
                 break;
@@ -68,6 +97,14 @@ export class EftlParser {
                 break;
 
             case TokenType.VAR_OPEN:
+                if (this.blockStack.some(block => block.type === TokenType.VAR_OPEN)) {
+                    this.errors.push({
+                        message: 'VAR tags cannot be nested inside another VAR tag',
+                        line: token.line,
+                        column: token.column,
+                        length: token.length
+                    });
+                }
                 this.pushBlock(token);
                 this.advance();
                 break;
@@ -78,12 +115,17 @@ export class EftlParser {
                 break;
 
             case TokenType.IF_OPEN:
+                this.ifContexts.push({ token, phase: 'condition' });
                 this.pushBlock(token);
                 this.advance();
                 break;
 
             case TokenType.IF_CLOSE:
+                const closesIf = this.blockStack[this.blockStack.length - 1]?.type === TokenType.IF_OPEN;
                 this.popBlock(TokenType.IF_OPEN, token);
+                if (closesIf) {
+                    this.ifContexts.pop();
+                }
                 this.advance();
                 break;
 
@@ -255,6 +297,87 @@ export class EftlParser {
         }
     }
 
+    private validateOutsideEftl(token: Token): void {
+        if (this.blockStack.some(block => block.type === TokenType.EFTL_OPEN)) {
+            return;
+        }
+
+        if (token.type === TokenType.EFTL_OPEN || token.type === TokenType.EFTL_CLOSE ||
+            token.type === TokenType.COMMENT_OPEN || token.type === TokenType.COMMENT_CLOSE ||
+            token.type === TokenType.EOF || (token.type === TokenType.TEXT && token.value.trim() === '')) {
+            return;
+        }
+
+        this.addError(token, 'Only comments and whitespace are allowed outside the [EFTL] root block');
+    }
+
+    private validateDirectIfContent(token: Token): void {
+        const parent = this.blockStack[this.blockStack.length - 1];
+        if (!parent || parent.type !== TokenType.IF_OPEN) {
+            return;
+        }
+
+        const context = this.ifContexts[this.ifContexts.length - 1];
+        if (!context || context.token !== parent.token) {
+            return;
+        }
+
+        if (token.type === TokenType.TEXT && token.value.trim() === '') {
+            return;
+        }
+
+        if (token.type === TokenType.IF_OPEN) {
+            this.addError(token, 'IF cannot be nested directly inside another IF; place it inside a branch block');
+            return;
+        }
+
+        if (context.phase === 'condition' && token.type === TokenType.CONDITION_OPEN) {
+            context.phase = 'then';
+            return;
+        }
+
+        if (context.phase === 'then' && token.type === TokenType.THEN_OPEN) {
+            context.phase = 'branches';
+            return;
+        }
+
+        if (context.phase === 'branches' && token.type === TokenType.ELSE_IF_OPEN) {
+            return;
+        }
+
+        if (context.phase === 'branches' && token.type === TokenType.ELSE_OPEN) {
+            context.phase = 'afterElse';
+            return;
+        }
+
+        if (token.type === TokenType.IF_CLOSE) {
+            if (context.phase === 'condition') {
+                this.addError(token, 'Expected [CONDITION] immediately after [IF]');
+            } else if (context.phase === 'then') {
+                this.addError(token, 'Expected [THEN] immediately after [/CONDITION]');
+            }
+            return;
+        }
+
+        const expected = context.phase === 'condition'
+            ? '[CONDITION]'
+            : context.phase === 'then'
+                ? '[THEN]'
+                : context.phase === 'branches'
+                    ? '[ELSE IF], [ELSE], or [/IF]'
+                    : '[/IF]';
+        this.addError(token, `Expected ${expected} as the next direct child of [IF], got ${token.value}`);
+    }
+
+    private addError(token: Token, message: string): void {
+        this.errors.push({
+            message,
+            line: token.line,
+            column: token.column,
+            length: token.length
+        });
+    }
+
     private pushBlock(token: Token): void {
         if (token.value.endsWith('/]')) {
             return; // It's self-closing, don't push to stack

syntaxes/eftl.tmLanguage.json

@@ -174,7 +174,7 @@
       "patterns": [
         {
           "name": "meta.tag.block.eftl",
-          "begin": "(\\[)(TAG)(?:\\s+[^\\x00]*?)?(?<!/)(\\])",
+          "begin": "(\\[)(TAG)(?:\\s+[^\\]]*?)?(?<!/)(\\])",
           "end": "(\\[/)(TAG)(\\])",
           "beginCaptures": {
             "1": { "name": "entity.name.tag.eftl" },
@@ -261,7 +261,7 @@
       "patterns": [
         {
           "name": "meta.variable.block.eftl",
-          "begin": "(\\[)(VAR)(?:\\s+[^\\x00]*?)?(?<!/)(\\])",
+          "begin": "(\\[)(VAR)(?:\\s+[^\\]]*?)?(?<!/)(\\])",
           "end": "(\\[/)(VAR)(\\])",
           "beginCaptures": {
             "1": { "name": "storage.type.variable.eftl" },
@@ -301,7 +301,7 @@
       "patterns": [
         {
           "name": "meta.function.block.eftl",
-          "begin": "(\\[)(VALUE_OF|SIZE_OF|SPLIT|FORMAT|CONTAINS|TRIM)(?:\\s+[^\\x00]*?)?(?<!/)(\\])",
+          "begin": "(\\[)(VALUE_OF|SIZE_OF|SPLIT|FORMAT|CONTAINS|TRIM)(?:\\s+[^\\]]*?)?(?<!/)(\\])",
           "end": "(\\[/)(VALUE_OF|SIZE_OF|SPLIT|FORMAT|CONTAINS|TRIM)(\\])",
           "beginCaptures": {
             "1": { "name": "support.function.eftl" },

test/grammar.test.js

@@ -0,0 +1,77 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const test = require('node:test');
+
+const oniguruma = require('vscode-oniguruma');
+const textmate = require('vscode-textmate');
+
+async function loadGrammar() {
+    await oniguruma.loadWASM(fs.readFileSync(require.resolve('vscode-oniguruma/release/onig.wasm')).buffer);
+
+    const registry = new textmate.Registry({
+        onigLib: Promise.resolve({
+            createOnigScanner: patterns => new oniguruma.OnigScanner(patterns),
+            createOnigString: value => new oniguruma.OnigString(value)
+        }),
+        loadGrammar: async scopeName => scopeName === 'source.eftl'
+            ? textmate.parseRawGrammar(
+                fs.readFileSync('syntaxes/eftl.tmLanguage.json', 'utf8'),
+                'eftl.tmLanguage.json'
+            )
+            : null
+    });
+
+    return registry.loadGrammar('source.eftl');
+}
+
+test('highlights nested VAR blocks, self-closing functions, and WHILE attributes', async () => {
+    const grammar = await loadGrammar();
+    const source = [
+        '[EFTL]',
+        '[VAR name="count" type="number"][SIZE_OF varname="items" /][/VAR]',
+        '[WHILE threshold="99"][CONDITION][% count > 0 %][/CONDITION][DO]',
+        '[VAR name="current" type="string"][VALUE_OF varname="items" index="count" /][/VAR]',
+        '[/DO][/WHILE]',
+        '[IF][CONDITION][% count == 0 %][/CONDITION][THEN]',
+        '[VAR name="empty" type="boolean"][% empty = true; %][/VAR]',
+        '[/THEN][/IF]',
+        '[/EFTL]'
+    ];
+    let ruleStack = textmate.INITIAL;
+    const variableTokens = [];
+    const selfClosingFunctionTokens = [];
+    const thresholdTokens = [];
+    const thresholdValueTokens = [];
+    const whileTokens = [];
+
+    for (const line of source) {
+        const result = grammar.tokenizeLine(line, ruleStack);
+        ruleStack = result.ruleStack;
+
+        for (const token of result.tokens) {
+            const text = line.slice(token.startIndex, token.endIndex);
+            if (text === 'VAR' && line[token.startIndex - 1] !== '/') variableTokens.push(token);
+            if (text === 'SIZE_OF' || text === 'VALUE_OF') selfClosingFunctionTokens.push(token);
+            if (text === 'threshold') thresholdTokens.push(token);
+            if (text === '"99"') thresholdValueTokens.push(token);
+            if (text === 'WHILE') whileTokens.push(token);
+        }
+    }
+
+    assert.equal(variableTokens.length, 3);
+    for (const token of variableTokens) {
+        assert.ok(token.scopes.includes('storage.type.variable.eftl'));
+    }
+
+    for (const token of selfClosingFunctionTokens) {
+        assert.ok(token.scopes.includes('meta.function.eftl'));
+        assert.ok(!token.scopes.includes('meta.function.block.eftl'));
+    }
+
+    assert.equal(thresholdTokens.length, 1);
+    assert.ok(thresholdTokens[0].scopes.includes('entity.other.attribute-name.eftl'));
+    assert.equal(thresholdValueTokens.length, 1);
+    assert.ok(thresholdValueTokens[0].scopes.includes('string.quoted.double.eftl'));
+    assert.equal(whileTokens.length, 1);
+    assert.ok(whileTokens[0].scopes.includes('keyword.control.loop.eftl'));
+});

test/parser.test.js

@@ -10,24 +10,45 @@ function parse(source) {
     return new EftlParser(tokens).parse().errors;
 }
 
+function messages(source) {
+    return parse(source).map(error => error.message);
+}
+
 test('accepts a correctly closed VAR tag', () => {
-    assert.deepEqual(parse('[VAR][/VAR]'), []);
+    assert.deepEqual(parse('[EFTL][VAR][/VAR][/EFTL]'), []);
 });
 
-test('reports a VAR tag without its closing tag', () => {
+test('accepts sibling VAR tags', () => {
-    assert.deepEqual(parse('[VAR]'), [{
+    assert.deepEqual(parse('[EFTL][VAR][/VAR][VAR][/VAR][/EFTL]'), []);
-        message: 'Unclosed tag: [VAR] (expected [/VAR])',
+});
+
+test('reports a VAR nested directly inside another VAR', () => {
+    assert.deepEqual(parse('[EFTL][VAR][VAR][/VAR][/VAR][/EFTL]'), [{
+        message: 'VAR tags cannot be nested inside another VAR tag',
         line: 1,
-        column: 1,
+        column: 12,
         length: 5
     }]);
 });
 
-test('reports an unexpected closing tag', () => {
+test('reports a VAR nested indirectly inside another VAR', () => {
-    const errors = parse('[/VAR]');
+    assert.deepEqual(parse('[EFTL][VAR][SPLIT][VAR][/VAR][/SPLIT][/VAR][/EFTL]'), [{
+        message: 'VAR tags cannot be nested inside another VAR tag',
+        line: 1,
+        column: 19,
+        length: 5
+    }]);
+});
+
+test('reports a VAR tag without its closing tag', () => {
+    assert.ok(messages('[EFTL][VAR]').includes('Unclosed tag: [VAR] (expected [/VAR])'));
+});
+
+test('reports a mismatched closing tag', () => {
+    const errors = parse('[EFTL][/VAR][/EFTL]');
 
     assert.equal(errors.length, 1);
-    assert.match(errors[0].message, /Unexpected closing tag/);
+    assert.equal(errors[0].message, 'Mismatched closing tag: expected [/EFTL], got [/VAR]');
 });
 
 test('reports every unclosed nested tag at its opening token', () => {
@@ -45,4 +66,132 @@ test('reports every unclosed nested tag at its opening token', () => {
             length: 5
         }
     ]);
+});
+
+test('requires exactly one EFTL root block', () => {
+    assert.deepEqual(parse(''), [{
+        message: 'Missing required [EFTL] root block',
+        line: 1,
+        column: 1,
+        length: 0
+    }]);
+
+    assert.deepEqual(parse('[EFTL][/EFTL]\n[EFTL][/EFTL]'), [{
+        message: 'Only one [EFTL] root block is allowed',
+        line: 2,
+        column: 1,
+        length: 6
+    }]);
+});
+
+test('allows comments and multiline whitespace around the EFTL root', () => {
+    assert.deepEqual(parse(' \n[!-- before --]\n[EFTL][/EFTL]\n[!-- after --]\n '), []);
+});
+
+test('rejects instructions outside the EFTL root', () => {
+    assert.deepEqual(messages('[VAR][/VAR][EFTL][/EFTL]'), [
+        'Only comments and whitespace are allowed outside the [EFTL] root block',
+        'Only comments and whitespace are allowed outside the [EFTL] root block'
+    ]);
+});
+
+test('rejects a nested EFTL root', () => {
+    assert.deepEqual(parse('[EFTL][EFTL][/EFTL][/EFTL]'), [{
+        message: 'EFTL root blocks cannot be nested',
+        line: 1,
+        column: 7,
+        length: 6
+    }]);
+});
+
+test('accepts CONDITION, THEN, repeated ELSE IF, and one final ELSE in an IF', () => {
+    const source = [
+        '[EFTL][IF]',
+        '  [CONDITION][% true %][/CONDITION]',
+        '  [THEN][/THEN]',
+        '  [ELSE IF][CONDITION][% false %][/CONDITION][THEN][/THEN][/ELSE IF]',
+        '  [ELSE IF][CONDITION][% false %][/CONDITION][THEN][/THEN][/ELSE IF]',
+        '  [ELSE][/ELSE]',
+        '[/IF][/EFTL]'
+    ].join('\n');
+
+    assert.deepEqual(parse(source), []);
+});
+
+test('allows an IF nested inside a THEN block', () => {
+    const source = '[EFTL][IF][CONDITION][/CONDITION][THEN]'
+        + '[IF][CONDITION][/CONDITION][THEN][/THEN][/IF]'
+        + '[/THEN][/IF][/EFTL]';
+
+    assert.deepEqual(parse(source), []);
+});
+
+test('rejects an IF nested directly inside another IF', () => {
+    const source = '[EFTL][IF]'
+        + '[IF][CONDITION][/CONDITION][THEN][/THEN][/IF]'
+        + '[CONDITION][/CONDITION][THEN][/THEN][/IF][/EFTL]';
+
+    assert.deepEqual(messages(source), [
+        'IF cannot be nested directly inside another IF; place it inside a branch block'
+    ]);
+});
+
+test('requires CONDITION followed by THEN as the first direct IF children', () => {
+    assert.deepEqual(messages('[EFTL][IF][/IF][/EFTL]'), [
+        'Expected [CONDITION] immediately after [IF]'
+    ]);
+
+    assert.deepEqual(messages('[EFTL][IF][THEN][/THEN][CONDITION][/CONDITION][THEN][/THEN][/IF][/EFTL]'), [
+        'Expected [CONDITION] as the next direct child of [IF], got [THEN]'
+    ]);
+
+    assert.deepEqual(messages('[EFTL][IF][CONDITION][/CONDITION][/IF][/EFTL]'), [
+        'Expected [THEN] immediately after [/CONDITION]'
+    ]);
+});
+
+test('allows only whitespace between direct IF child blocks', () => {
+    const source = '[EFTL][IF][CONDITION][/CONDITION]'
+        + '[VAR][/VAR][!-- comment --]text'
+        + '[THEN][/THEN][/IF][/EFTL]';
+
+    assert.deepEqual(messages(source), [
+        'Expected [THEN] as the next direct child of [IF], got [VAR]',
+        'Expected [THEN] as the next direct child of [IF], got [!--',
+        'Expected [THEN] as the next direct child of [IF], got text'
+    ]);
+});
+
+test('allows ELSE IF only before the optional ELSE', () => {
+    const source = '[EFTL][IF][CONDITION][/CONDITION][THEN][/THEN]'
+        + '[ELSE][/ELSE][ELSE IF][/ELSE IF][/IF][/EFTL]';
+
+    assert.deepEqual(messages(source), [
+        'Expected [/IF] as the next direct child of [IF], got [ELSE IF]'
+    ]);
+
+    assert.deepEqual(messages(
+        '[EFTL][IF][CONDITION][/CONDITION][THEN][/THEN][ELSE][/ELSE][ELSE][/ELSE][/IF][/EFTL]'
+    ), [
+        'Expected [/IF] as the next direct child of [IF], got [ELSE]'
+    ]);
+});
+
+test('accepts WHILE with an optional string threshold attribute', () => {
+    assert.deepEqual(parse('[EFTL][WHILE][/WHILE][/EFTL]'), []);
+    assert.deepEqual(parse('[EFTL][WHILE threshold="99"][/WHILE][/EFTL]'), []);
+
+    const { tokens } = new EftlTokenizer('[WHILE threshold="99"][/WHILE]').tokenize();
+    assert.equal(tokens[0].type, 'WHILE_OPEN');
+    assert.equal(tokens[0].value, '[WHILE threshold="99"]');
+});
+
+test('rejects invalid WHILE attributes', () => {
+    const expectedMessage = 'WHILE accepts only the optional string attribute threshold="..."';
+
+    assert.deepEqual(messages('[EFTL][WHILE threshold=99][/WHILE][/EFTL]'), [expectedMessage]);
+    assert.deepEqual(messages('[EFTL][WHILE limit="99"][/WHILE][/EFTL]'), [expectedMessage]);
+    assert.deepEqual(messages(
+        '[EFTL][WHILE threshold="10" threshold="20"][/WHILE][/EFTL]'
+    ), [expectedMessage]);
 });