Files

691 lines
21 KiB
JavaScript

#!/usr/bin/env node
/**
* extract-elx.js
*
* Extracts root properties from ElixForms .elx JSON files into separate folders
* for better version control and diff visibility.
*
* Usage:
* node extract-elx.js <input-file-path> [--overwrite] [--clean]
*
* Output:
* Creates a folder named after the input file (without extension) containing:
* - Subfolders for each root property
* - .xml files for XML strings (pretty-printed)
* - .json files for objects and metadata
* - .txt files for plain text strings
* - _manifest.json with extraction metadata
*/
import { promises as fs } from 'fs';
import { join as _join, dirname, basename as _basename, extname } from 'path';
import { execSync } from 'child_process';
import { existsSync } from 'fs';
import crc32 from 'crc-32';
import xmlFormatter from 'xml-formatter';
import { XMLParser } from 'fast-xml-parser';
// ============================================================================
// CONFIGURATION
// ============================================================================
const CONFIG = {
XML_INDENT: 2,
JSON_INDENT: 2,
MANIFEST_FILE: '_manifest.json',
INDEX_FILE: 'index',
ITEM_PREFIX: 'item',
INTERNALS_SUFFIX: '_internals',
VERSION: '0.0.1',
};
// ============================================================================
// UTILITY FUNCTIONS
// ============================================================================
/**
* Detect if a string is XML content
*/
function isXmlContent(str) {
if (typeof str !== 'string') return false;
const trimmed = str.trim();
return trimmed.startsWith('<?xml') || trimmed.startsWith('<');
}
/**
* Pretty-print XML with indentation using xml-formatter
*/
function prettifyXml(xmlString, indent = 2) {
try {
const formatted = xmlFormatter(xmlString, {
indentation: ' '.repeat(indent),
collapseContent: false,
lineSeparator: '\n',
filter: () => true, // Keep all whitespace nodes
attributeQuotes: 'double',
forceSelfClosingEmptyTag: true,
ignoredPaths: [], // Do not ignore any paths
strictMode: false,
throwOnFailure: false,
whiteSpaceAtEndOfSelfclosingTag: false,
});
return formatted;
} catch (error) {
console.error(`⚠️ Warning: Failed to format XML: ${error.message}`);
// Return original string if formatting fails
return xmlString;
}
}
/**
* Pretty-print JSON with indentation
*/
function prettifyJson(obj, indent = 2) {
return JSON.stringify(obj, null, indent);
}
/**
* Escape a string for use in RegExp
*/
/**
* Sanitize a string for safe file names.
*/
function sanitizeFileName(value) {
return value
.trim()
.replace(/\s+/g, '_')
.replace(/[^a-zA-Z0-9._-]/g, '_')
.replace(/_+/g, '_')
.replace(/^[_-]+|[_-]+$/g, '');
}
const XML_PARSE_OPTIONS = {
ignoreAttributes: false,
attributeNamePrefix: '@_',
textNodeName: '#text',
parseTagValue: false,
trimValues: false,
};
function parseXmlDocument(xmlString) {
if (typeof xmlString !== 'string') return null;
try {
return new XMLParser(XML_PARSE_OPTIONS).parse(xmlString);
} catch (error) {
return null;
}
}
function getXmlNodeText(node) {
if (node == null) return '';
if (typeof node === 'string') return node;
if (typeof node === 'object') {
if ('#text' in node) {
return String(node['#text']);
}
if (Object.keys(node).length === 0) {
return '';
}
}
return '';
}
function toArray(value) {
if (value == null) return [];
return Array.isArray(value) ? value : [value];
}
/**
* Extract the text content of a named COL element inside a SECTION with a given title.
*/
function extractXmlValueFromSection(xmlString, sectionTitle, colName) {
const document = parseXmlDocument(xmlString);
if (!document || !document.EXT) return '';
const sections = toArray(document.EXT.SECTION);
for (const section of sections) {
const titleValue = section['@_title'] || section.title || '';
if (String(titleValue).trim() === sectionTitle) {
return String(getXmlNodeText(section[colName] || '')).trim();
}
}
return '';
}
/**
* Compare string values with numeric fallback.
*/
function compareSortKeys(a, b) {
const valueA = `${a}`.trim();
const valueB = `${b}`.trim();
const numericA = parseFloat(valueA);
const numericB = parseFloat(valueB);
if (!Number.isNaN(numericA) && !Number.isNaN(numericB)) {
return numericA - numericB;
}
return valueA.localeCompare(valueB, undefined, { numeric: true, sensitivity: 'base' });
}
/**
* Determine a sort key for a property array item.
*/
function getArraySortKey(propertyName, item) {
if (propertyName === 'moduleConfig' && typeof item === 'string') {
return extractXmlValueFromSection(item, 'Step', 'COL0005');
}
if (propertyName === 'moduleTab' && typeof item === 'string') {
return extractXmlValueFromSection(item, 'Config', 'COL0006');
}
return null;
}
/**
* Get the output file name for a property array item.
*/
function getArrayItemName(propertyName, item, index, totalCount) {
var itemSuffix = null;
if (propertyName === 'formSchemaData' && item && typeof item === 'object' && item.id != null) {
itemSuffix = String(item.id);
}
if (propertyName === 'moduleConfig' && typeof item === 'string') {
const rawOrderValue = extractXmlValueFromSection(item, 'Step', 'COL0005');
if (rawOrderValue) {
const sanitized = sanitizeFileName(rawOrderValue);
if (sanitized) {
itemSuffix = sanitized;
}
}
}
if (propertyName === 'moduleTab' && typeof item === 'string') {
const rawOrderValue = extractXmlValueFromSection(item, 'Config', 'COL0006');
if (rawOrderValue) {
const sanitized = sanitizeFileName(rawOrderValue);
if (sanitized) {
itemSuffix = sanitized;
}
}
}
if (propertyName === 'moduleUserValidation' && typeof item === 'string') {
const rawOrderValue = extractXmlValueFromSection(item, 'System', 'COL0001');
if (rawOrderValue) {
const sanitized = sanitizeFileName(rawOrderValue);
if (sanitized) {
itemSuffix = sanitized;
}
}
}
if (propertyName === 'moduleValidation' && typeof item === 'string') {
const rawOrderValue = extractXmlValueFromSection(item, 'System', 'COL0004');
if (rawOrderValue) {
const sanitized = sanitizeFileName(rawOrderValue);
if (sanitized) {
itemSuffix = sanitized;
}
}
}
if (propertyName === 'translationsTabMapper' && item && typeof item === 'object' && item.order != null) {
itemSuffix = String(item.order);
}
if (propertyName === 'validationTypes' && typeof item === 'string') {
const rawOrderValue = extractXmlValueFromSection(item, 'Dati', 'COL0003');
if (rawOrderValue) {
const sanitized = sanitizeFileName(rawOrderValue);
if (sanitized) {
itemSuffix = sanitized;
}
}
}
if (propertyName === 'translationsValidationTabMapper' && item && typeof item === 'object' && item.code != null) {
itemSuffix = String(item.code);
}
if (!itemSuffix) {
itemSuffix = padIndex(index, totalCount);
}
return `${propertyName}_${CONFIG.ITEM_PREFIX}_${itemSuffix}`;
}
/**
* Get file extension based on content type
*/
function getFileExtension(content) {
if (typeof content === 'string') {
return isXmlContent(content) ? 'xml' : 'txt';
}
return 'json';
}
/**
* Pad number with zeros for consistent file naming
*/
function padIndex(index, totalCount) {
const maxDigits = totalCount.toString().length;
return index.toString().padStart(maxDigits, '0');
}
/**
* Format file size for display
*/
function formatFileSize(bytes) {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
// ============================================================================
// FILE OPERATIONS
// ============================================================================
async function calculateFileCrc32(filePath) {
try {
const buffer = await fs.readFile(filePath);
return (crc32.buf(buffer) >>> 0).toString(16).toUpperCase().padStart(8, '0');
} catch (error) {
throw new Error(`Failed to calculate CRC32: ${error.message}`);
}
}
/**
* Read and parse the input JSON file
*/
async function readInputFile(filePath) {
try {
const content = await fs.readFile(filePath, 'utf-8');
const data = JSON.parse(content);
return data;
} catch (error) {
if (error instanceof SyntaxError) {
throw new Error(`Invalid JSON: ${error.message}`);
}
throw new Error(`Failed to read file: ${error.message}`);
}
}
/**
* Create output directory for extracted properties
*/
async function createOutputDirectory(outputPath, allowOverwrite = false, cleanExisting = false) {
if (!allowOverwrite && !cleanExisting && existsSync(outputPath)) {
throw new Error(
`Output directory already exists: ${outputPath}\n` +
`Use --overwrite flag to replace it or --clean flag to remove its contents.`
);
}
// Optionally remove existing directory
if (cleanExisting) {
console.log(` Removing existing directory contents: ${outputPath}`);
await fs.rm(`${outputPath}`, { recursive: true, force: true });
} else if (allowOverwrite) {
console.log(` Overwriting existing directory: ${outputPath}`);
}
try {
await fs.mkdir(outputPath, { recursive: true });
} catch (error) {
throw new Error(`Failed to create output directory: ${error.message}`);
}
}
/**
* Save a string to file with appropriate extension
*/
async function saveStringProperty(dirPath, filename, content, options = {}) {
const ext = getFileExtension(content);
const filepath = _join(dirPath, `${filename}.${ext}`);
let output = content;
// Pretty-print XML if detected
if (ext === 'xml' && !options.skipFormat) {
output = prettifyXml(content, CONFIG.XML_INDENT);
}
await fs.writeFile(filepath, output, 'utf-8');
return { filepath, size: output.length, type: ext };
}
/**
* Save a JSON object to file
*/
async function saveJsonProperty(dirPath, filename, obj, options = {}) {
const filepath = _join(dirPath, `${filename}.json`);
const output = prettifyJson(obj, CONFIG.JSON_INDENT);
await fs.writeFile(filepath, output, 'utf-8');
return { filepath, size: output.length, type: 'json' };
}
/**
* Create subdirectory for a property
*/
async function createPropertyDirectory(outputPath, propertyName) {
const dirPath = _join(outputPath, propertyName);
await fs.mkdir(dirPath, { recursive: true });
return dirPath;
}
/**
* Save backup of original file
*/
async function saveBackup(outputPath, data) {
const backupPath = _join(outputPath, CONFIG.BACKUP_SUFFIX);
const output = prettifyJson(data, CONFIG.JSON_INDENT);
await fs.writeFile(backupPath, output, 'utf-8');
return backupPath;
}
/**
* Create CRC32 checksum file for integrity verification
*/
async function createCrc32Checksum(outputPath, inputPath) {
const fileCrc32 = await calculateFileCrc32(inputPath);
const checksumPath = _join(outputPath, `crc32.txt`);
await fs.writeFile(checksumPath, `${fileCrc32}\n`, 'utf-8');
return { checksumPath, fileCrc32 };
}
// ============================================================================
// PROPERTY PROCESSING
// ============================================================================
/**
* Process a string property
*/
async function processStringProperty(outputPath, propertyName, content) {
const result = {
property: propertyName,
type: 'string',
contentType: isXmlContent(content) ? 'xml' : 'text',
size: content.length,
files: [],
};
const dirPath = await createPropertyDirectory(outputPath, propertyName);
const fileInfo = await saveStringProperty(dirPath, CONFIG.INDEX_FILE, content);
result.files.push(fileInfo.filepath);
return result;
}
/**
* Process an array property
*/
async function processArrayProperty(outputPath, propertyName, arr) {
const result = {
property: propertyName,
type: 'array',
itemCount: arr.length,
itemTypes: {},
files: [],
};
const dirPath = await createPropertyDirectory(outputPath, propertyName);
// Create index.json with metadata
const indexData = {
count: arr.length,
itemTypes: [],
//extractedAt: new Date().toISOString(),
order: propertyName,
};
const items = arr.map((item, originalIndex) => ({
item,
originalIndex,
sortKey: getArraySortKey(propertyName, item),
}));
const hasSortKey = items.some(entry => entry.sortKey !== null && entry.sortKey !== '');
if (hasSortKey) {
items.sort((a, b) => {
const keyA = a.sortKey ?? '';
const keyB = b.sortKey ?? '';
const compare = compareSortKeys(keyA, keyB);
return compare !== 0 ? compare : a.originalIndex - b.originalIndex;
});
}
const usedNames = new Set();
for (let sortedIndex = 0; sortedIndex < items.length; sortedIndex++) {
const { item, originalIndex } = items[sortedIndex];
const itemNameBase = getArrayItemName(propertyName, item, sortedIndex, arr.length);
let itemName = itemNameBase;
let duplicateCounter = 1;
while (usedNames.has(itemName)) {
itemName = `${itemNameBase}_${duplicateCounter}`;
duplicateCounter += 1;
}
usedNames.add(itemName);
const itemType = Array.isArray(item) ? 'array' : typeof item;
// Track item type distribution
if (!result.itemTypes[itemType]) {
result.itemTypes[itemType] = 0;
}
result.itemTypes[itemType]++;
let fileInfo;
if (typeof item === 'string') {
fileInfo = await saveStringProperty(dirPath, itemName, item);
indexData.itemTypes.push({ index: sortedIndex, type: fileInfo.type, fileName: `${itemName}.${fileInfo.type}` });
} else if (typeof item === 'object' && item !== null) {
fileInfo = await saveJsonProperty(dirPath, itemName, item);
indexData.itemTypes.push({ index: sortedIndex, type: 'json', fileName: `${itemName}.json` });
// Additional processing for some items (e.g. extract XML from strings inside objects)
for (const [key, value] of Object.entries(item)) {
if (typeof value === 'string' && isXmlContent(value)) {
const xmlFileName = `${itemName}_${key}`;
const xmlFileInfo = await saveStringProperty(dirPath, xmlFileName, value);
result.files.push(xmlFileInfo.filepath);
}
}
} else {
fileInfo = await saveJsonProperty(dirPath, itemName, item);
indexData.itemTypes.push({ index: sortedIndex, type: typeof item, fileName: `${itemName}.json` });
}
result.files.push(fileInfo.filepath);
}
// Save index metadata
const indexPath = _join(dirPath, `${CONFIG.INDEX_FILE}.json`);
const indexOutput = prettifyJson(indexData, CONFIG.JSON_INDENT);
await fs.writeFile(indexPath, indexOutput, 'utf-8');
result.files.unshift(indexPath); // Put index first in list
return result;
}
/**
* Process an object property
*/
async function processObjectProperty(outputPath, propertyName, obj) {
const result = {
property: propertyName,
type: 'object',
keyCount: Object.keys(obj).length,
files: [],
};
const dirPath = await createPropertyDirectory(outputPath, propertyName);
const filepath = _join(dirPath, `${CONFIG.INDEX_FILE}.json`);
const output = prettifyJson(obj, CONFIG.JSON_INDENT);
await fs.writeFile(filepath, output, 'utf-8');
result.files.push(filepath);
return result;
}
/**
* Process a single root property
*/
async function processProperty(outputPath, propertyName, value) {
if (typeof value === 'string') {
return await processStringProperty(outputPath, propertyName, value);
} else if (Array.isArray(value)) {
return await processArrayProperty(outputPath, propertyName, value);
} else if (typeof value === 'object' && value !== null) {
return await processObjectProperty(outputPath, propertyName, value);
} else {
// Primitive value (number, boolean, null)
const result = {
property: propertyName,
type: typeof value,
value: value,
files: [],
};
const dirPath = await createPropertyDirectory(outputPath, propertyName);
const filepath = _join(dirPath, `${CONFIG.INDEX_FILE}.json`);
await fs.writeFile(filepath, prettifyJson(value, CONFIG.JSON_INDENT), 'utf-8');
result.files.push(filepath);
return result;
}
}
// ============================================================================
// MANIFEST GENERATION
// ============================================================================
/**
* Create manifest file with extraction metadata
*/
async function createManifest(outputPath, data, processedProperties) {
const manifest = {
//extractedAt: new Date().toISOString(),
scriptVersion: CONFIG.VERSION,
properties: {},
summary: {
totalProperties: processedProperties.length,
totalFiles: processedProperties.reduce((sum, p) => sum + p.files.length, 0),
},
};
for (const prop of processedProperties) {
manifest.properties[prop.property] = {
type: prop.type,
...(prop.itemCount !== undefined && { itemCount: prop.itemCount }),
...(prop.itemTypes && Object.keys(prop.itemTypes).length > 0 && { itemTypes: prop.itemTypes }),
...(prop.keyCount !== undefined && { keyCount: prop.keyCount }),
fileCount: prop.files.length,
};
}
const manifestPath = _join(outputPath, CONFIG.MANIFEST_FILE);
const output = prettifyJson(manifest, CONFIG.JSON_INDENT);
await fs.writeFile(manifestPath, output, 'utf-8');
return manifestPath;
}
// ============================================================================
// MAIN EXECUTION
// ============================================================================
async function main() {
try {
// Parse command-line arguments
const args = process.argv.slice(2);
if (args.length === 0) {
console.error('Usage: node extract-elx.js <input-file-path> [--overwrite] [--clean]');
console.error('Example: node extract-elx.js elxforms_4951.elx');
process.exit(1);
}
const inputPath = args[0];
const allowOverwrite = args.includes('--overwrite');
const cleanExisting = args.includes('--clean');
// Validate input file
if (!existsSync(inputPath)) {
throw new Error(`File not found: ${inputPath}`);
}
// Determine output directory
const dir = dirname(inputPath);
const basename = _basename(inputPath, extname(inputPath));
const outputPath = _join(dir, `${basename}${CONFIG.INTERNALS_SUFFIX}`);
console.log('\n📁 ElixForms JSON Extractor\n');
console.log(`📖 Reading: ${inputPath}`);
// Read input file
const data = await readInputFile(inputPath);
const fileStats = await fs.stat(inputPath);
console.log(` Size: ${formatFileSize(fileStats.size)}`);
// Create output directory
console.log(`\n📂 Creating output: ${outputPath}`);
await createOutputDirectory(outputPath, allowOverwrite, cleanExisting);
// Create CRC32 checksum file for integrity verification
console.log(`\n🐾 Creating checksum file...`);
const crc32 = await createCrc32Checksum(outputPath, inputPath);
console.log(`\n Checksum file: ${crc32.checksumPath}`);
//console.log(` CRC32: ${crc32.fileCrc32}`);
// Process each root property
console.log('\n⚙️ Processing properties:\n');
const processedProperties = [];
const rootProps = Object.keys(data);
for (const propName of rootProps) {
process.stdout.write(` ${propName}... `);
const propResult = await processProperty(outputPath, propName, data[propName]);
processedProperties.push(propResult);
let typeLabel = propResult.type;
if (propResult.itemCount !== undefined) {
typeLabel += ` (${propResult.itemCount} items)`;
} else if (propResult.keyCount !== undefined) {
typeLabel += ` (${propResult.keyCount} keys)`;
}
console.log(`✓ ${typeLabel}${propResult.files.length} file(s)`);
}
// Create manifest
console.log('\n📋 Creating manifest...');
await createManifest(outputPath, data, processedProperties);
// Summary
const totalFiles = processedProperties.reduce((sum, p) => sum + p.files.length, 0);
console.log(`\n✅ Extraction complete!\n`);
console.log(` Properties extracted: ${rootProps.length}`);
console.log(` Total files created: ${totalFiles + 1} (includes manifest)\n`);
console.log(`📍 Output location: ${outputPath}\n`);
} catch (error) {
console.error(`\n❌ Error: ${error.message}\n`);
process.exit(1);
}
}
// Run main function
main();