add extract-elx script from external repo

2026-05-15 13:24:04 +02:00
parent fb06f7c5eb
commit 82fa329675
5 changed files with 0 additions and 0 deletions
@@ -0,0 +1,684 @@
+#!/usr/bin/env node
+
+/**
+ * extract-elx.js
+ * 
+ * Extracts root properties from ElixForms .elx JSON files into separate folders
+ * for better version control and diff visibility.
+ * 
+ * Usage:
+ *   node extract-elx.js <input-file-path> [--overwrite] [--clean]
+ * 
+ * Output:
+ *   Creates a folder named after the input file (without extension) containing:
+ *   - Subfolders for each root property
+ *   - .xml files for XML strings (pretty-printed)
+ *   - .json files for objects and metadata
+ *   - .txt files for plain text strings
+ *   - _manifest.json with extraction metadata
+ */
+
+import { promises as fs } from 'fs';
+import { join as _join, dirname, basename as _basename, extname } from 'path';
+import { execSync } from 'child_process';
+import { existsSync } from 'fs';
+import crc32 from 'crc-32';
+import xmlFormatter from 'xml-formatter';
+import { XMLParser } from 'fast-xml-parser';
+
+// ============================================================================
+// CONFIGURATION
+// ============================================================================
+
+const CONFIG = {
+  XML_INDENT: 2,
+  JSON_INDENT: 2,
+  MANIFEST_FILE: '_manifest.json',
+  INDEX_FILE: 'index',
+  ITEM_PREFIX: 'item',
+  INTERNALS_SUFFIX: '_internals',
+  VERSION: '0.0.1',
+};
+
+// ============================================================================
+// UTILITY FUNCTIONS
+// ============================================================================
+
+/**
+ * Detect if a string is XML content
+ */
+function isXmlContent(str) {
+  if (typeof str !== 'string') return false;
+  const trimmed = str.trim();
+  return trimmed.startsWith('<?xml') || trimmed.startsWith('<');
+}
+
+/**
+ * Pretty-print XML with indentation using xml-formatter
+ */
+function prettifyXml(xmlString, indent = 2) {
+  try {
+    const formatted = xmlFormatter(xmlString, {
+      indentation: ' '.repeat(indent),
+      collapseContent: false,
+      lineSeparator: '\n',
+      filter: () => true, // Keep all whitespace nodes
+    });
+    return formatted;
+  } catch (error) {
+    console.error(`⚠️  Warning: Failed to format XML: ${error.message}`);
+    // Return original string if formatting fails
+    return xmlString;
+  }
+}
+
+/**
+ * Pretty-print JSON with indentation
+ */
+function prettifyJson(obj, indent = 2) {
+  return JSON.stringify(obj, null, indent);
+}
+
+/**
+ * Escape a string for use in RegExp
+ */
+/**
+ * Sanitize a string for safe file names.
+ */
+function sanitizeFileName(value) {
+  return value
+    .trim()
+    .replace(/\s+/g, '_')
+    .replace(/[^a-zA-Z0-9._-]/g, '_')
+    .replace(/_+/g, '_')
+    .replace(/^[_-]+|[_-]+$/g, '');
+}
+
+const XML_PARSE_OPTIONS = {
+  ignoreAttributes: false,
+  attributeNamePrefix: '@_',
+  textNodeName: '#text',
+  parseTagValue: false,
+  trimValues: false,
+};
+
+function parseXmlDocument(xmlString) {
+  if (typeof xmlString !== 'string') return null;
+  try {
+    return new XMLParser(XML_PARSE_OPTIONS).parse(xmlString);
+  } catch (error) {
+    return null;
+  }
+}
+
+function getXmlNodeText(node) {
+  if (node == null) return '';
+  if (typeof node === 'string') return node;
+  if (typeof node === 'object') {
+    if ('#text' in node) {
+      return String(node['#text']);
+    }
+    if (Object.keys(node).length === 0) {
+      return '';
+    }
+  }
+  return '';
+}
+
+function toArray(value) {
+  if (value == null) return [];
+  return Array.isArray(value) ? value : [value];
+}
+
+/**
+ * Extract the text content of a named COL element inside a SECTION with a given title.
+ */
+function extractXmlValueFromSection(xmlString, sectionTitle, colName) {
+  const document = parseXmlDocument(xmlString);
+  if (!document || !document.EXT) return '';
+
+  const sections = toArray(document.EXT.SECTION);
+  for (const section of sections) {
+    const titleValue = section['@_title'] || section.title || '';
+    if (String(titleValue).trim() === sectionTitle) {
+      return String(getXmlNodeText(section[colName] || '')).trim();
+    }
+  }
+
+  return '';
+}
+
+/**
+ * Compare string values with numeric fallback.
+ */
+function compareSortKeys(a, b) {
+  const valueA = `${a}`.trim();
+  const valueB = `${b}`.trim();
+
+  const numericA = parseFloat(valueA);
+  const numericB = parseFloat(valueB);
+
+  if (!Number.isNaN(numericA) && !Number.isNaN(numericB)) {
+    return numericA - numericB;
+  }
+
+  return valueA.localeCompare(valueB, undefined, { numeric: true, sensitivity: 'base' });
+}
+
+/**
+ * Determine a sort key for a property array item.
+ */
+function getArraySortKey(propertyName, item) {
+  if (propertyName === 'moduleConfig' && typeof item === 'string') {
+    return extractXmlValueFromSection(item, 'Step', 'COL0005');
+  }
+  if (propertyName === 'moduleTab' && typeof item === 'string') {
+    return extractXmlValueFromSection(item, 'Config', 'COL0006');
+  }
+
+  return null;
+}
+
+/**
+ * Get the output file name for a property array item.
+ */
+function getArrayItemName(propertyName, item, index, totalCount) {
+  var itemSuffix = null;
+
+  if (propertyName === 'formSchemaData' && item && typeof item === 'object' && item.id != null) {
+    itemSuffix = String(item.id);
+  }
+
+  if (propertyName === 'moduleConfig' && typeof item === 'string') {
+    const rawOrderValue = extractXmlValueFromSection(item, 'Step', 'COL0005');
+    if (rawOrderValue) {
+      const sanitized = sanitizeFileName(rawOrderValue);
+      if (sanitized) {
+        itemSuffix = sanitized;
+      }
+    }
+  }
+
+  if (propertyName === 'moduleTab' && typeof item === 'string') {
+    const rawOrderValue = extractXmlValueFromSection(item, 'Config', 'COL0006');
+    if (rawOrderValue) {
+      const sanitized = sanitizeFileName(rawOrderValue);
+      if (sanitized) {
+        itemSuffix = sanitized;
+      }
+    }
+  }
+
+  if (propertyName === 'moduleUserValidation' && typeof item === 'string') {
+    const rawOrderValue = extractXmlValueFromSection(item, 'System', 'COL0001');
+    if (rawOrderValue) {
+      const sanitized = sanitizeFileName(rawOrderValue);
+      if (sanitized) {
+        itemSuffix = sanitized;
+      }
+    }
+  }
+
+  if (propertyName === 'moduleValidation' && typeof item === 'string') {
+    const rawOrderValue = extractXmlValueFromSection(item, 'System', 'COL0004');
+    if (rawOrderValue) {
+      const sanitized = sanitizeFileName(rawOrderValue);
+      if (sanitized) {
+        itemSuffix = sanitized;
+      }
+    }
+  }
+
+  if (propertyName === 'translationsTabMapper' && item && typeof item === 'object' && item.order != null) {
+    itemSuffix = String(item.order);
+  }
+  
+  if (propertyName === 'validationTypes' && typeof item === 'string') {
+    const rawOrderValue = extractXmlValueFromSection(item, 'Dati', 'COL0003');
+    if (rawOrderValue) {
+      const sanitized = sanitizeFileName(rawOrderValue);
+      if (sanitized) {
+        itemSuffix = sanitized;
+      }
+    }
+  }
+
+  if (propertyName === 'translationsValidationTabMapper' && item && typeof item === 'object' && item.code != null) {
+    itemSuffix = String(item.code);
+  }
+
+  if (!itemSuffix) {
+    itemSuffix = padIndex(index, totalCount);
+  }
+
+  return `${propertyName}_${CONFIG.ITEM_PREFIX}_${itemSuffix}`;
+}
+
+/**
+ * Get file extension based on content type
+ */
+function getFileExtension(content) {
+  if (typeof content === 'string') {
+    return isXmlContent(content) ? 'xml' : 'txt';
+  }
+  return 'json';
+}
+
+/**
+ * Pad number with zeros for consistent file naming
+ */
+function padIndex(index, totalCount) {
+  const maxDigits = totalCount.toString().length;
+  return index.toString().padStart(maxDigits, '0');
+}
+
+/**
+ * Format file size for display
+ */
+function formatFileSize(bytes) {
+  if (bytes === 0) return '0 B';
+  const k = 1024;
+  const sizes = ['B', 'KB', 'MB'];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+}
+
+// ============================================================================
+// FILE OPERATIONS
+// ============================================================================
+
+async function calculateFileCrc32(filePath) {
+  try {
+    const buffer = await fs.readFile(filePath);
+    return (crc32.buf(buffer) >>> 0).toString(16).toUpperCase().padStart(8, '0');
+  } catch (error) {
+    throw new Error(`Failed to calculate CRC32: ${error.message}`);
+  }
+}
+
+/**
+ * Read and parse the input JSON file
+ */
+async function readInputFile(filePath) {
+  try {
+    const content = await fs.readFile(filePath, 'utf-8');
+    const data = JSON.parse(content);
+    return data;
+  } catch (error) {
+    if (error instanceof SyntaxError) {
+      throw new Error(`Invalid JSON: ${error.message}`);
+    }
+    throw new Error(`Failed to read file: ${error.message}`);
+  }
+}
+
+/**
+ * Create output directory for extracted properties
+ */
+async function createOutputDirectory(outputPath, allowOverwrite = false, cleanExisting = false) {
+  if (!allowOverwrite && !cleanExisting && existsSync(outputPath)) {
+    throw new Error(
+      `Output directory already exists: ${outputPath}\n` +
+      `Use --overwrite flag to replace it or --clean flag to remove its contents.`
+    );
+  }
+
+  // Optionally remove existing directory
+  if (cleanExisting) {
+    console.log(`   Removing existing directory contents: ${outputPath}`);
+    await fs.rm(`${outputPath}`, { recursive: true, force: true });
+  } else if (allowOverwrite) {
+    console.log(`   Overwriting existing directory: ${outputPath}`);
+  }
+
+  try {
+    await fs.mkdir(outputPath, { recursive: true });
+  } catch (error) {
+    throw new Error(`Failed to create output directory: ${error.message}`);
+  }
+}
+
+/**
+ * Save a string to file with appropriate extension
+ */
+async function saveStringProperty(dirPath, filename, content, options = {}) {
+  const ext = getFileExtension(content);
+  const filepath = _join(dirPath, `${filename}.${ext}`);
+
+  let output = content;
+
+  // Pretty-print XML if detected
+  if (ext === 'xml' && !options.skipFormat) {
+    output = prettifyXml(content, CONFIG.XML_INDENT);
+  }
+
+  await fs.writeFile(filepath, output, 'utf-8');
+  return { filepath, size: output.length, type: ext };
+}
+
+/**
+ * Save a JSON object to file
+ */
+async function saveJsonProperty(dirPath, filename, obj, options = {}) {
+  const filepath = _join(dirPath, `${filename}.json`);
+  const output = prettifyJson(obj, CONFIG.JSON_INDENT);
+
+  await fs.writeFile(filepath, output, 'utf-8');
+  return { filepath, size: output.length, type: 'json' };
+}
+
+/**
+ * Create subdirectory for a property
+ */
+async function createPropertyDirectory(outputPath, propertyName) {
+  const dirPath = _join(outputPath, propertyName);
+  await fs.mkdir(dirPath, { recursive: true });
+  return dirPath;
+}
+
+/**
+ * Save backup of original file
+ */
+async function saveBackup(outputPath, data) {
+  const backupPath = _join(outputPath, CONFIG.BACKUP_SUFFIX);
+  const output = prettifyJson(data, CONFIG.JSON_INDENT);
+  await fs.writeFile(backupPath, output, 'utf-8');
+  return backupPath;
+}
+
+/**
+ * Create CRC32 checksum file for integrity verification
+ */
+async function createCrc32Checksum(outputPath, inputPath) {
+  const fileCrc32 = await calculateFileCrc32(inputPath);
+  const checksumPath = _join(outputPath, `crc32.txt`);
+  await fs.writeFile(checksumPath, `${fileCrc32}\n`, 'utf-8');
+  return { checksumPath, fileCrc32 };
+}
+
+// ============================================================================
+// PROPERTY PROCESSING
+// ============================================================================
+
+/**
+ * Process a string property
+ */
+async function processStringProperty(outputPath, propertyName, content) {
+  const result = {
+    property: propertyName,
+    type: 'string',
+    contentType: isXmlContent(content) ? 'xml' : 'text',
+    size: content.length,
+    files: [],
+  };
+
+  const dirPath = await createPropertyDirectory(outputPath, propertyName);
+  const fileInfo = await saveStringProperty(dirPath, CONFIG.INDEX_FILE, content);
+  result.files.push(fileInfo.filepath);
+
+  return result;
+}
+
+/**
+ * Process an array property
+ */
+async function processArrayProperty(outputPath, propertyName, arr) {
+  const result = {
+    property: propertyName,
+    type: 'array',
+    itemCount: arr.length,
+    itemTypes: {},
+    files: [],
+  };
+
+  const dirPath = await createPropertyDirectory(outputPath, propertyName);
+
+  // Create index.json with metadata
+  const indexData = {
+    count: arr.length,
+    itemTypes: [],
+    extractedAt: new Date().toISOString(),
+    order: propertyName,
+  };
+
+  const items = arr.map((item, originalIndex) => ({
+    item,
+    originalIndex,
+    sortKey: getArraySortKey(propertyName, item),
+  }));
+
+  const hasSortKey = items.some(entry => entry.sortKey !== null && entry.sortKey !== '');
+  if (hasSortKey) {
+    items.sort((a, b) => {
+      const keyA = a.sortKey ?? '';
+      const keyB = b.sortKey ?? '';
+      const compare = compareSortKeys(keyA, keyB);
+      return compare !== 0 ? compare : a.originalIndex - b.originalIndex;
+    });
+  }
+
+  const usedNames = new Set();
+
+  for (let sortedIndex = 0; sortedIndex < items.length; sortedIndex++) {
+    const { item, originalIndex } = items[sortedIndex];
+    const itemNameBase = getArrayItemName(propertyName, item, sortedIndex, arr.length);
+    let itemName = itemNameBase;
+    let duplicateCounter = 1;
+
+    while (usedNames.has(itemName)) {
+      itemName = `${itemNameBase}_${duplicateCounter}`;
+      duplicateCounter += 1;
+    }
+    usedNames.add(itemName);
+
+    const itemType = Array.isArray(item) ? 'array' : typeof item;
+
+    // Track item type distribution
+    if (!result.itemTypes[itemType]) {
+      result.itemTypes[itemType] = 0;
+    }
+    result.itemTypes[itemType]++;
+
+    let fileInfo;
+    if (typeof item === 'string') {
+      fileInfo = await saveStringProperty(dirPath, itemName, item);
+      indexData.itemTypes.push({ index: originalIndex, type: fileInfo.type, fileName: `${itemName}.${fileInfo.type}` });
+    } else if (typeof item === 'object' && item !== null) {
+      fileInfo = await saveJsonProperty(dirPath, itemName, item);
+      indexData.itemTypes.push({ index: originalIndex, type: 'json', fileName: `${itemName}.json` });
+      // Additional processing for some items (e.g. extract XML from strings inside objects)
+      for (const [key, value] of Object.entries(item)) {
+        if (typeof value === 'string' && isXmlContent(value)) {
+          const xmlFileName = `${itemName}_${key}`;
+          const xmlFileInfo = await saveStringProperty(dirPath, xmlFileName, value);
+          result.files.push(xmlFileInfo.filepath);
+        }
+      }
+    } else {
+      fileInfo = await saveJsonProperty(dirPath, itemName, item);
+      indexData.itemTypes.push({ index: originalIndex, type: typeof item, fileName: `${itemName}.json` });
+    }
+
+    result.files.push(fileInfo.filepath);
+  }
+
+  // Save index metadata
+  const indexPath = _join(dirPath, `${CONFIG.INDEX_FILE}.json`);
+  const indexOutput = prettifyJson(indexData, CONFIG.JSON_INDENT);
+  await fs.writeFile(indexPath, indexOutput, 'utf-8');
+  result.files.unshift(indexPath); // Put index first in list
+
+  return result;
+}
+
+/**
+ * Process an object property
+ */
+async function processObjectProperty(outputPath, propertyName, obj) {
+  const result = {
+    property: propertyName,
+    type: 'object',
+    keyCount: Object.keys(obj).length,
+    files: [],
+  };
+
+  const dirPath = await createPropertyDirectory(outputPath, propertyName);
+  const filepath = _join(dirPath, `${CONFIG.INDEX_FILE}.json`);
+  const output = prettifyJson(obj, CONFIG.JSON_INDENT);
+
+  await fs.writeFile(filepath, output, 'utf-8');
+  result.files.push(filepath);
+
+  return result;
+}
+
+/**
+ * Process a single root property
+ */
+async function processProperty(outputPath, propertyName, value) {
+  if (typeof value === 'string') {
+    return await processStringProperty(outputPath, propertyName, value);
+  } else if (Array.isArray(value)) {
+    return await processArrayProperty(outputPath, propertyName, value);
+  } else if (typeof value === 'object' && value !== null) {
+    return await processObjectProperty(outputPath, propertyName, value);
+  } else {
+    // Primitive value (number, boolean, null)
+    const result = {
+      property: propertyName,
+      type: typeof value,
+      value: value,
+      files: [],
+    };
+
+    const dirPath = await createPropertyDirectory(outputPath, propertyName);
+    const filepath = _join(dirPath, `${CONFIG.INDEX_FILE}.json`);
+    await fs.writeFile(filepath, prettifyJson(value, CONFIG.JSON_INDENT), 'utf-8');
+    result.files.push(filepath);
+
+    return result;
+  }
+}
+
+// ============================================================================
+// MANIFEST GENERATION
+// ============================================================================
+
+/**
+ * Create manifest file with extraction metadata
+ */
+async function createManifest(outputPath, data, processedProperties) {
+  const manifest = {
+    extractedAt: new Date().toISOString(),
+    scriptVersion: CONFIG.VERSION,
+    properties: {},
+    summary: {
+      totalProperties: processedProperties.length,
+      totalFiles: processedProperties.reduce((sum, p) => sum + p.files.length, 0),
+    },
+  };
+
+  for (const prop of processedProperties) {
+    manifest.properties[prop.property] = {
+      type: prop.type,
+      ...(prop.itemCount !== undefined && { itemCount: prop.itemCount }),
+      ...(prop.itemTypes && Object.keys(prop.itemTypes).length > 0 && { itemTypes: prop.itemTypes }),
+      ...(prop.keyCount !== undefined && { keyCount: prop.keyCount }),
+      fileCount: prop.files.length,
+    };
+  }
+
+  const manifestPath = _join(outputPath, CONFIG.MANIFEST_FILE);
+  const output = prettifyJson(manifest, CONFIG.JSON_INDENT);
+  await fs.writeFile(manifestPath, output, 'utf-8');
+
+  return manifestPath;
+}
+
+// ============================================================================
+// MAIN EXECUTION
+// ============================================================================
+
+async function main() {
+  try {
+    // Parse command-line arguments
+    const args = process.argv.slice(2);
+
+    if (args.length === 0) {
+      console.error('Usage: node extract-elx.js <input-file-path> [--overwrite] [--clean]');
+      console.error('Example: node extract-elx.js elxforms_4951.elx');
+      process.exit(1);
+    }
+
+    const inputPath = args[0];
+    const allowOverwrite = args.includes('--overwrite');
+    const cleanExisting = args.includes('--clean');
+
+    // Validate input file
+    if (!existsSync(inputPath)) {
+      throw new Error(`File not found: ${inputPath}`);
+    }
+
+    // Determine output directory
+    const dir = dirname(inputPath);
+    const basename = _basename(inputPath, extname(inputPath));
+    const outputPath = _join(dir, `${basename}${CONFIG.INTERNALS_SUFFIX}`);
+
+    console.log('\n📁 ElixForms JSON Extractor\n');
+    console.log(`📖 Reading: ${inputPath}`);
+
+    // Read input file
+    const data = await readInputFile(inputPath);
+    const fileStats = await fs.stat(inputPath);
+    console.log(`   Size: ${formatFileSize(fileStats.size)}`);
+    
+    // Create output directory
+    console.log(`\n📂 Creating output: ${outputPath}`);
+    await createOutputDirectory(outputPath, allowOverwrite, cleanExisting);
+    
+    // Create CRC32 checksum file for integrity verification
+    console.log(`\n🐾 Creating checksum file...`);
+    const crc32 = await createCrc32Checksum(outputPath, inputPath);
+    console.log(`\n   Checksum file: ${crc32.checksumPath}`);
+    //console.log(`   CRC32: ${crc32.fileCrc32}`);
+
+    // Process each root property
+    console.log('\n⚙️ Processing properties:\n');
+    const processedProperties = [];
+    const rootProps = Object.keys(data);
+
+    for (const propName of rootProps) {
+      process.stdout.write(`   ${propName}... `);
+      const propResult = await processProperty(outputPath, propName, data[propName]);
+      processedProperties.push(propResult);
+
+      let typeLabel = propResult.type;
+      if (propResult.itemCount !== undefined) {
+        typeLabel += ` (${propResult.itemCount} items)`;
+      } else if (propResult.keyCount !== undefined) {
+        typeLabel += ` (${propResult.keyCount} keys)`;
+      }
+
+      console.log(`✓ ${typeLabel} → ${propResult.files.length} file(s)`);
+    }
+
+    // Create manifest
+    console.log('\n📋 Creating manifest...');
+    await createManifest(outputPath, data, processedProperties);
+
+    // Summary
+    const totalFiles = processedProperties.reduce((sum, p) => sum + p.files.length, 0);
+    console.log(`\n✅ Extraction complete!\n`);
+    console.log(`   Properties extracted: ${rootProps.length}`);
+    console.log(`   Total files created: ${totalFiles + 1} (includes manifest)\n`);
+
+    console.log(`📍 Output location: ${outputPath}\n`);
+
+  } catch (error) {
+    console.error(`\n❌ Error: ${error.message}\n`);
+    process.exit(1);
+  }
+}
+
+// Run main function
+main();