From 1b09da3731c67abb1bc3f02a5944e1f536693616 Mon Sep 17 00:00:00 2001 From: Pier-Paolo Mammi Date: Tue, 12 May 2026 14:47:37 +0200 Subject: [PATCH] add extract-elx.js script to ease elx files versioning --- scripts/extract-elx/extract-elx.js | 626 +++++++++++++++++++++++++++++ 1 file changed, 626 insertions(+) create mode 100644 scripts/extract-elx/extract-elx.js diff --git a/scripts/extract-elx/extract-elx.js b/scripts/extract-elx/extract-elx.js new file mode 100644 index 0000000..f95b0d7 --- /dev/null +++ b/scripts/extract-elx/extract-elx.js @@ -0,0 +1,626 @@ +#!/usr/bin/env node + +/** + * extract-elx.js + * + * Extracts root properties from ElixForms .elx JSON files into separate folders + * for better version control and diff visibility. + * + * Usage: + * node extract-elx.js [--overwrite] + * + * Output: + * Creates a folder named after the input file (without extension) containing: + * - Subfolders for each root property + * - .xml files for XML strings (pretty-printed) + * - .json files for objects and metadata + * - .txt files for plain text strings + * - _manifest.json with extraction metadata + * - .elx.original.json backup + */ + +const fs = require('fs').promises; +const path = require('path'); +const { execSync } = require('child_process'); +const { existsSync } = require('fs'); +const { XMLParser } = require('fast-xml-parser'); + +// Import xml-formatter (try multiple locations) +let xmlFormatter; +try { + // Try local node_modules first + xmlFormatter = require('xml-formatter'); +} catch (err) { + try { + // Try to find npm global prefix and load from there + const npmPrefix = execSync('npm config get prefix', { encoding: 'utf-8' }).trim(); + const globalXmlFormatter = path.join(npmPrefix, 'node_modules', 'xml-formatter'); + xmlFormatter = require(globalXmlFormatter); + } catch (err2) { + console.error( + '\nāŒ Missing dependency: xml-formatter\n' + + 'Please install globally with: npm install -g xml-formatter\n' + + 'Or locally in the current directory with: npm install xml-formatter\n' + ); + process.exit(1); + } +} + +// ============================================================================ +// CONFIGURATION +// ============================================================================ + +const CONFIG = { + XML_INDENT: 2, + JSON_INDENT: 2, + BACKUP_SUFFIX: '.elx.original.json', + MANIFEST_FILE: '_manifest.json', + INDEX_FILE: 'index', + ITEM_PREFIX: 'item', + ALLOW_OVERWRITE: false, +}; + +// ============================================================================ +// UTILITY FUNCTIONS +// ============================================================================ + +/** + * Detect if a string is XML content + */ +function isXmlContent(str) { + if (typeof str !== 'string') return false; + const trimmed = str.trim(); + return trimmed.startsWith(' true, // Keep all whitespace nodes + }); + return formatted; + } catch (error) { + console.error(`āš ļø Warning: Failed to format XML: ${error.message}`); + // Return original string if formatting fails + return xmlString; + } +} + +/** + * Pretty-print JSON with indentation + */ +function prettifyJson(obj, indent = 2) { + return JSON.stringify(obj, null, indent); +} + +/** + * Escape a string for use in RegExp + */ +/** + * Sanitize a string for safe file names. + */ +function sanitizeFileName(value) { + return value + .trim() + .replace(/\s+/g, '_') + .replace(/[^a-zA-Z0-9._-]/g, '_') + .replace(/_+/g, '_') + .replace(/^[_-]+|[_-]+$/g, ''); +} + +const XML_PARSE_OPTIONS = { + ignoreAttributes: false, + attributeNamePrefix: '@_', + textNodeName: '#text', + parseTagValue: false, + trimValues: false, +}; + +const xmlParser = new XMLParser(XML_PARSE_OPTIONS); + +function parseXmlDocument(xmlString) { + if (typeof xmlString !== 'string') return null; + try { + return xmlParser.parse(xmlString); + } catch (error) { + return null; + } +} + +function getXmlNodeText(node) { + if (node == null) return ''; + if (typeof node === 'string') return node; + if (typeof node === 'object') { + if ('#text' in node) { + return String(node['#text']); + } + if (Object.keys(node).length === 0) { + return ''; + } + } + return ''; +} + +function toArray(value) { + if (value == null) return []; + return Array.isArray(value) ? value : [value]; +} + +/** + * Extract the text content of a named COL element inside a SECTION with a given title. + */ +function extractXmlValueFromSection(xmlString, sectionTitle, colName) { + const document = parseXmlDocument(xmlString); + if (!document || !document.EXT) return ''; + + const sections = toArray(document.EXT.SECTION); + for (const section of sections) { + const titleValue = section['@_title'] || section.title || ''; + if (String(titleValue).trim() === sectionTitle) { + return String(getXmlNodeText(section[colName] || '')).trim(); + } + } + + return ''; +} + +/** + * Compare string values with numeric fallback. + */ +function compareSortKeys(a, b) { + const valueA = `${a}`.trim(); + const valueB = `${b}`.trim(); + + const numericA = parseFloat(valueA); + const numericB = parseFloat(valueB); + + if (!Number.isNaN(numericA) && !Number.isNaN(numericB)) { + return numericA - numericB; + } + + return valueA.localeCompare(valueB, undefined, { numeric: true, sensitivity: 'base' }); +} + +/** + * Determine a sort key for a property array item. + */ +function getArraySortKey(propertyName, item) { + if (propertyName === 'moduleConfig' && typeof item === 'string') { + return extractXmlValueFromSection(item, 'Step', 'COL0005'); + } + if (propertyName === 'moduleTab' && typeof item === 'string') { + return extractXmlValueFromSection(item, 'Config', 'COL0006'); + } + + return null; +} + +/** + * Get the output file name for a property array item. + */ +function getArrayItemName(propertyName, item, index, totalCount) { + if (propertyName === 'translationsTabMapper' && item && typeof item === 'object' && item.order != null) { + return `${CONFIG.ITEM_PREFIX}_${String(item.order)}`; + } + + if (propertyName === 'formSchemaData' && item && typeof item === 'object' && item.id != null) { + return `schemaId_${String(item.id)}`; + } + + if (propertyName === 'moduleTab' && typeof item === 'string') { + const rawOrderValue = extractXmlValueFromSection(item, 'Config', 'COL0006'); + if (rawOrderValue) { + const sanitized = sanitizeFileName(rawOrderValue); + if (sanitized) { + return `moduleTabItem_${sanitized}`; + } + } + } + + return `${CONFIG.ITEM_PREFIX}_${padIndex(index, totalCount)}`; +} + +/** + * Get file extension based on content type + */ +function getFileExtension(content) { + if (typeof content === 'string') { + return isXmlContent(content) ? 'xml' : 'txt'; + } + return 'json'; +} + +/** + * Pad number with zeros for consistent file naming + */ +function padIndex(index, totalCount) { + const maxDigits = totalCount.toString().length; + return index.toString().padStart(maxDigits, '0'); +} + +/** + * Format file size for display + */ +function formatFileSize(bytes) { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; +} + +// ============================================================================ +// FILE OPERATIONS +// ============================================================================ + +/** + * Read and parse the input JSON file + */ +async function readInputFile(filePath) { + try { + const content = await fs.readFile(filePath, 'utf-8'); + const data = JSON.parse(content); + return data; + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Invalid JSON: ${error.message}`); + } + throw new Error(`Failed to read file: ${error.message}`); + } +} + +/** + * Create output directory for extracted properties + */ +async function createOutputDirectory(outputPath, allowOverwrite = false) { + if (existsSync(outputPath)) { + if (!allowOverwrite) { + throw new Error( + `Output directory already exists: ${outputPath}\n` + + `Use --overwrite flag to replace it.` + ); + } + // Optionally remove existing directory + console.log(`Removing existing directory: ${outputPath}`); + } + + try { + await fs.mkdir(outputPath, { recursive: true }); + } catch (error) { + throw new Error(`Failed to create output directory: ${error.message}`); + } +} + +/** + * Save a string to file with appropriate extension + */ +async function saveStringProperty(dirPath, filename, content, options = {}) { + const ext = getFileExtension(content); + const filepath = path.join(dirPath, `${filename}.${ext}`); + + let output = content; + + // Pretty-print XML if detected + if (ext === 'xml' && !options.skipFormat) { + output = prettifyXml(content, CONFIG.XML_INDENT); + } + + await fs.writeFile(filepath, output, 'utf-8'); + return { filepath, size: output.length, type: ext }; +} + +/** + * Save a JSON object to file + */ +async function saveJsonProperty(dirPath, filename, obj, options = {}) { + const filepath = path.join(dirPath, `${filename}.json`); + const output = prettifyJson(obj, CONFIG.JSON_INDENT); + + await fs.writeFile(filepath, output, 'utf-8'); + return { filepath, size: output.length, type: 'json' }; +} + +/** + * Create subdirectory for a property + */ +async function createPropertyDirectory(outputPath, propertyName) { + const dirPath = path.join(outputPath, propertyName); + await fs.mkdir(dirPath, { recursive: true }); + return dirPath; +} + +/** + * Save backup of original file + */ +async function saveBackup(outputPath, data) { + const backupPath = path.join(outputPath, CONFIG.BACKUP_SUFFIX); + const output = prettifyJson(data, CONFIG.JSON_INDENT); + await fs.writeFile(backupPath, output, 'utf-8'); + return backupPath; +} + +// ============================================================================ +// PROPERTY PROCESSING +// ============================================================================ + +/** + * Process a string property + */ +async function processStringProperty(outputPath, propertyName, content) { + const result = { + property: propertyName, + type: 'string', + contentType: isXmlContent(content) ? 'xml' : 'text', + size: content.length, + files: [], + }; + + const dirPath = await createPropertyDirectory(outputPath, propertyName); + const fileInfo = await saveStringProperty(dirPath, CONFIG.INDEX_FILE, content); + result.files.push(fileInfo.filepath); + + return result; +} + +/** + * Process an array property + */ +async function processArrayProperty(outputPath, propertyName, arr) { + const result = { + property: propertyName, + type: 'array', + itemCount: arr.length, + itemTypes: {}, + files: [], + }; + + const dirPath = await createPropertyDirectory(outputPath, propertyName); + + // Create index.json with metadata + const indexData = { + count: arr.length, + itemTypes: [], + extractedAt: new Date().toISOString(), + order: propertyName, + }; + + const items = arr.map((item, originalIndex) => ({ + item, + originalIndex, + sortKey: getArraySortKey(propertyName, item), + })); + + const hasSortKey = items.some(entry => entry.sortKey !== null && entry.sortKey !== ''); + if (hasSortKey) { + items.sort((a, b) => { + const keyA = a.sortKey ?? ''; + const keyB = b.sortKey ?? ''; + const compare = compareSortKeys(keyA, keyB); + return compare !== 0 ? compare : a.originalIndex - b.originalIndex; + }); + } + + const usedNames = new Set(); + + for (let sortedIndex = 0; sortedIndex < items.length; sortedIndex++) { + const { item, originalIndex } = items[sortedIndex]; + const itemNameBase = getArrayItemName(propertyName, item, sortedIndex, arr.length); + let itemName = itemNameBase; + let duplicateCounter = 1; + + while (usedNames.has(itemName)) { + itemName = `${itemNameBase}_${duplicateCounter}`; + duplicateCounter += 1; + } + usedNames.add(itemName); + + const itemType = Array.isArray(item) ? 'array' : typeof item; + + // Track item type distribution + if (!result.itemTypes[itemType]) { + result.itemTypes[itemType] = 0; + } + result.itemTypes[itemType]++; + + let fileInfo; + if (typeof item === 'string') { + fileInfo = await saveStringProperty(dirPath, itemName, item); + indexData.itemTypes.push({ index: originalIndex, type: fileInfo.type, fileName: `${itemName}.${fileInfo.type}` }); + } else if (typeof item === 'object' && item !== null) { + fileInfo = await saveJsonProperty(dirPath, itemName, item); + indexData.itemTypes.push({ index: originalIndex, type: 'json', fileName: `${itemName}.json` }); + } else { + fileInfo = await saveJsonProperty(dirPath, itemName, item); + indexData.itemTypes.push({ index: originalIndex, type: typeof item, fileName: `${itemName}.json` }); + } + + result.files.push(fileInfo.filepath); + } + + // Save index metadata + const indexPath = path.join(dirPath, `${CONFIG.INDEX_FILE}.json`); + const indexOutput = prettifyJson(indexData, CONFIG.JSON_INDENT); + await fs.writeFile(indexPath, indexOutput, 'utf-8'); + result.files.unshift(indexPath); // Put index first in list + + return result; +} + +/** + * Process an object property + */ +async function processObjectProperty(outputPath, propertyName, obj) { + const result = { + property: propertyName, + type: 'object', + keyCount: Object.keys(obj).length, + files: [], + }; + + const dirPath = await createPropertyDirectory(outputPath, propertyName); + const filepath = path.join(dirPath, `${CONFIG.INDEX_FILE}.json`); + const output = prettifyJson(obj, CONFIG.JSON_INDENT); + + await fs.writeFile(filepath, output, 'utf-8'); + result.files.push(filepath); + + return result; +} + +/** + * Process a single root property + */ +async function processProperty(outputPath, propertyName, value) { + if (typeof value === 'string') { + return await processStringProperty(outputPath, propertyName, value); + } else if (Array.isArray(value)) { + return await processArrayProperty(outputPath, propertyName, value); + } else if (typeof value === 'object' && value !== null) { + return await processObjectProperty(outputPath, propertyName, value); + } else { + // Primitive value (number, boolean, null) + const result = { + property: propertyName, + type: typeof value, + value: value, + files: [], + }; + + const dirPath = await createPropertyDirectory(outputPath, propertyName); + const filepath = path.join(dirPath, `${CONFIG.INDEX_FILE}.json`); + await fs.writeFile(filepath, prettifyJson(value, CONFIG.JSON_INDENT), 'utf-8'); + result.files.push(filepath); + + return result; + } +} + +// ============================================================================ +// MANIFEST GENERATION +// ============================================================================ + +/** + * Create manifest file with extraction metadata + */ +async function createManifest(outputPath, data, processedProperties) { + const manifest = { + extractedAt: new Date().toISOString(), + scriptVersion: '1.0.0', + properties: {}, + summary: { + totalProperties: processedProperties.length, + totalFiles: processedProperties.reduce((sum, p) => sum + p.files.length, 0), + }, + }; + + for (const prop of processedProperties) { + manifest.properties[prop.property] = { + type: prop.type, + ...(prop.itemCount !== undefined && { itemCount: prop.itemCount }), + ...(prop.itemTypes && Object.keys(prop.itemTypes).length > 0 && { itemTypes: prop.itemTypes }), + ...(prop.keyCount !== undefined && { keyCount: prop.keyCount }), + fileCount: prop.files.length, + }; + } + + const manifestPath = path.join(outputPath, CONFIG.MANIFEST_FILE); + const output = prettifyJson(manifest, CONFIG.JSON_INDENT); + await fs.writeFile(manifestPath, output, 'utf-8'); + + return manifestPath; +} + +// ============================================================================ +// MAIN EXECUTION +// ============================================================================ + +async function main() { + try { + // Parse command-line arguments + const args = process.argv.slice(2); + + if (args.length === 0) { + console.error('Usage: node extract-elx.js [--overwrite]'); + console.error('Example: node extract-elx.js elxforms_4951.elx'); + process.exit(1); + } + + const inputPath = args[0]; + const allowOverwrite = args.includes('--overwrite'); + + // Validate input file + if (!existsSync(inputPath)) { + throw new Error(`File not found: ${inputPath}`); + } + + // Determine output directory + const dir = path.dirname(inputPath); + const basename = path.basename(inputPath, path.extname(inputPath)); + const outputPath = path.join(dir, basename); + + console.log('\nšŸ“ ElixForms JSON Extractor\n'); + console.log(`šŸ“– Reading: ${inputPath}`); + + // Read input file + const data = await readInputFile(inputPath); + const fileStats = await fs.stat(inputPath); + console.log(` Size: ${formatFileSize(fileStats.size)}`); + + // Create output directory + console.log(`\nšŸ“‚ Creating output: ${outputPath}`); + await createOutputDirectory(outputPath, allowOverwrite); + + // Save backup + console.log(' Saving backup...'); + await saveBackup(outputPath, data); + + // Process each root property + console.log('\nāš™ļø Processing properties:\n'); + const processedProperties = []; + const rootProps = Object.keys(data).filter(name => name !== 'powershellScript'); + + for (const propName of rootProps) { + process.stdout.write(` ${propName}... `); + const propResult = await processProperty(outputPath, propName, data[propName]); + processedProperties.push(propResult); + + let typeLabel = propResult.type; + if (propResult.itemCount !== undefined) { + typeLabel += ` (${propResult.itemCount} items)`; + } else if (propResult.keyCount !== undefined) { + typeLabel += ` (${propResult.keyCount} keys)`; + } + + console.log(`āœ“ ${typeLabel} ā†’ ${propResult.files.length} file(s)`); + } + + if (data.hasOwnProperty('powershellScript')) { + console.log(' powershellScript... skipped'); + } + + // Create manifest + console.log('\nšŸ“‹ Creating manifest...'); + await createManifest(outputPath, data, processedProperties); + + // Summary + const totalFiles = processedProperties.reduce((sum, p) => sum + p.files.length, 0); + console.log(`\nāœ… Extraction complete!\n`); + console.log(` Properties extracted: ${rootProps.length}`); + console.log(` Total files created: ${totalFiles + 1} (includes manifest)\n`); + + console.log(`šŸ“ Output location: ${outputPath}\n`); + + } catch (error) { + console.error(`\nāŒ Error: ${error.message}\n`); + process.exit(1); + } +} + +// Run main function +main();