#!/usr/bin/env node import { readFile, writeFile } from 'node:fs/promises'; import { basename } from 'node:path'; import { fileURLToPath } from 'node:url'; const ALLOWED_TABLES = new Set([ 'puzzle_work_profile', 'custom_world_profile', 'match3d_work_profile', 'square_hole_work_profile', 'big_fish_work_profile', 'visual_novel_work_profile', ]); const WORK_TABLE_TYPES = { puzzle_work_profile: 'puzzle', custom_world_profile: 'customWorld', match3d_work_profile: 'match3d', square_hole_work_profile: 'squareHole', big_fish_work_profile: 'bigFish', visual_novel_work_profile: 'visualNovel', }; const TABLE_OUTPUT_ORDER = [ 'puzzle_work_profile', 'custom_world_profile', 'match3d_work_profile', 'square_hole_work_profile', 'big_fish_work_profile', 'visual_novel_work_profile', ]; const WORK_TYPES = ['puzzle', 'customWorld', 'match3d', 'squareHole', 'bigFish', 'visualNovel']; const SHORT_TEXT_LIMIT = 120; const LONG_TEXT_LIMIT = 500; const SENSITIVE_PATTERN = /(token|secret|password|passwd|phone|wallet|credential|authorization|auth[_-]?key|api[_-]?key)/giu; class StableMapper { constructor(prefix) { this.prefix = prefix; this.values = new Map(); } map(value) { if (value === undefined || value === null || value === '') return value; const key = String(value); if (!this.values.has(key)) { this.values.set( key, `${this.prefix}-${String(this.values.size + 1).padStart(3, '0')}`, ); } return this.values.get(key); } } function createContext() { return { user: new StableMapper('user'), session: new StableMapper('session'), author: new StableMapper('author'), authorCode: new StableMapper('author-code'), publicWorkCode: new StableMapper('public-work-code'), coverAsset: new StableMapper('asset'), work: new StableMapper('work'), profile: new StableMapper('profile'), }; } function createWorkTypeBuckets() { return Object.fromEntries(WORK_TYPES.map((type) => [type, []])); } function unwrapSpacetimeOption(value) { if ( value && typeof value === 'object' && !Array.isArray(value) && Object.keys(value).length === 1 ) { if (Object.prototype.hasOwnProperty.call(value, 'some')) return value.some; if (Object.prototype.hasOwnProperty.call(value, 'none')) return undefined; } return value; } function truncateText(value, limit) { if (value === undefined || value === null) return value; const text = String(value).replace(/\s+/g, ' ').trim(); if (text.length <= limit) return text; return `${text.slice(0, limit)}…`; } function redactSensitiveText(value) { if (value === undefined || value === null) return value; return String(value).replace(SENSITIVE_PATTERN, '[redacted]'); } function sanitizeCoverImageSrc(value) { const unwrapped = unwrapSpacetimeOption(value); if (unwrapped === undefined || unwrapped === null || unwrapped === '') return unwrapped; const text = String(unwrapped); if (text.startsWith('data:image/')) return '[redacted-data-image]'; let withoutQuery = text.split('?')[0].split('#')[0]; if (withoutQuery.length > 180) withoutQuery = `${withoutQuery.slice(0, 180)}…`; return withoutQuery; } function sanitizeLargeJson(value) { const unwrapped = unwrapSpacetimeOption(value); if (unwrapped === undefined || unwrapped === null) return unwrapped; if (typeof unwrapped === 'string') { return truncateText(redactSensitiveText(unwrapped), LONG_TEXT_LIMIT); } try { return truncateText(redactSensitiveText(JSON.stringify(unwrapped)), LONG_TEXT_LIMIT); } catch { return truncateText(redactSensitiveText(String(unwrapped)), LONG_TEXT_LIMIT); } } function firstDefined(row, keys) { for (const key of keys) { if (row[key] !== undefined && row[key] !== null) return row[key]; } return undefined; } function sanitizeShortField(row, sanitized, key) { if (row[key] !== undefined) { sanitized[key] = truncateText(unwrapSpacetimeOption(row[key]), SHORT_TEXT_LIMIT); } } function sanitizeWorkRow(row, ctx) { const sanitized = {}; const profileId = unwrapSpacetimeOption(firstDefined(row, ['profile_id', 'profileId'])); const workId = unwrapSpacetimeOption(firstDefined(row, ['work_id', 'workId'])); if (profileId !== undefined) sanitized.profile_id = ctx.profile.map(profileId); if (workId !== undefined) sanitized.work_id = ctx.work.map(workId); if (row.owner_user_id !== undefined) { sanitized.owner_user_id = ctx.user.map(unwrapSpacetimeOption(row.owner_user_id)); } if (row.user_id !== undefined) sanitized.user_id = ctx.user.map(unwrapSpacetimeOption(row.user_id)); if (row.author_display_name !== undefined) { sanitized.author_display_name = ctx.author.map(unwrapSpacetimeOption(row.author_display_name)); } if (row.public_work_code !== undefined) { sanitized.public_work_code = ctx.publicWorkCode.map(unwrapSpacetimeOption(row.public_work_code)); } if (row.author_public_user_code !== undefined) { sanitized.author_public_user_code = ctx.authorCode.map( unwrapSpacetimeOption(row.author_public_user_code), ); } if (row.cover_asset_id !== undefined) { sanitized.cover_asset_id = ctx.coverAsset.map(unwrapSpacetimeOption(row.cover_asset_id)); } if (row.cover_image_src !== undefined) sanitized.cover_image_src = sanitizeCoverImageSrc(row.cover_image_src); for (const key of [ 'title', 'work_title', 'level_name', 'world_name', 'summary', 'summary_text', 'description', 'work_description', 'subtitle', ]) { sanitizeShortField(row, sanitized, key); } for (const key of ['levels_json', 'profile_payload_json', 'anchor_pack_json', 'theme_tags_json']) { if (row[key] !== undefined) sanitized[key] = sanitizeLargeJson(row[key]); } const passthroughKeys = [ 'publication_status', 'publicationStatus', 'play_count', 'playCount', 'like_count', 'likeCount', 'remix_count', 'remixCount', 'updated_at', 'created_at', 'published_at', 'visibility', 'status', 'category', 'tags', ]; for (const key of passthroughKeys) { if (row[key] !== undefined) sanitized[key] = unwrapSpacetimeOption(row[key]); } return sanitized; } function normalizeWork(tableName, row) { const type = WORK_TABLE_TYPES[tableName]; return { type, workId: row.work_id, profileId: row.profile_id, ownerUserId: row.owner_user_id, publicWorkCode: row.public_work_code, title: row.title ?? row.work_title ?? row.level_name ?? row.world_name, subtitle: row.subtitle ?? row.summary_text ?? row.summary ?? row.work_description ?? row.description, publicationStatus: row.publicationStatus ?? row.publication_status ?? row.status, playCount: row.playCount ?? row.play_count ?? 0, likeCount: row.likeCount ?? row.like_count ?? 0, remixCount: row.remixCount ?? row.remix_count ?? 0, coverImageSrc: row.cover_image_src, updatedAt: row.updated_at, }; } function toRowsByTable(input) { const tables = Array.isArray(input?.tables) ? input.tables : []; const result = new Map(); for (const table of tables) { if (!ALLOWED_TABLES.has(table?.name)) continue; result.set(table.name, Array.isArray(table.rows) ? table.rows : []); } return result; } export function extractWorksListData(input, options = {}) { const ctx = createContext(); const rowsByTable = toRowsByTable(input); const outputTables = {}; const counts = {}; const profileIds = createWorkTypeBuckets(); const workIds = createWorkTypeBuckets(); const normalizedWorks = []; for (const tableName of TABLE_OUTPUT_ORDER) { const sourceRows = rowsByTable.get(tableName); if (!sourceRows) continue; const sanitizedRows = sourceRows.map((row) => sanitizeWorkRow(row, ctx)); outputTables[tableName] = sanitizedRows; counts[tableName] = sanitizedRows.length; const type = WORK_TABLE_TYPES[tableName]; if (type) { for (const row of sanitizedRows) { if (row.profile_id) profileIds[type].push(row.profile_id); if (row.work_id) workIds[type].push(row.work_id); normalizedWorks.push(normalizeWork(tableName, row)); } } } return { source: options.source ?? 'unknown', generatedAt: options.generatedAt ?? new Date().toISOString(), counts, tables: outputTables, profileIds, workIds, normalizedWorks, }; } function createSampleOutput(output, maxRowsPerTable = 3) { const tables = {}; const counts = {}; const allowedWorkIds = new Set(); const allowedProfileIds = new Set(); for (const [tableName, rows] of Object.entries(output.tables)) { tables[tableName] = rows.slice(0, maxRowsPerTable); counts[tableName] = tables[tableName].length; const type = WORK_TABLE_TYPES[tableName]; if (type) { for (const row of tables[tableName]) { if (row.work_id) allowedWorkIds.add(row.work_id); if (row.profile_id) allowedProfileIds.add(row.profile_id); } } } const profileIds = Object.fromEntries( Object.entries(output.profileIds).map(([type, ids]) => [ type, ids.filter((id) => allowedProfileIds.has(id)).slice(0, maxRowsPerTable), ]), ); const workIds = Object.fromEntries( Object.entries(output.workIds).map(([type, ids]) => [ type, ids.filter((id) => allowedWorkIds.has(id)).slice(0, maxRowsPerTable), ]), ); const normalizedWorks = output.normalizedWorks .filter((work) => allowedWorkIds.has(work.workId) || allowedProfileIds.has(work.profileId)) .slice(0, maxRowsPerTable * 6); return { ...output, counts, tables, profileIds, workIds, normalizedWorks, }; } function parseArgs(argv) { const args = {}; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === '--input' || arg === '--output' || arg === '--sample-output') { const value = argv[index + 1]; if (!value || value.startsWith('--')) throw new Error(`${arg} requires a value`); args[arg.slice(2)] = value; index += 1; } else if (arg === '--help' || arg === '-h') { args.help = true; } else { throw new Error(`Unknown argument: ${arg}`); } } return args; } function usage() { return 'Usage: node scripts/loadtest/extract-works-list-data.mjs --input --output [--sample-output ]'; } export async function runCli(argv = process.argv.slice(2)) { const args = parseArgs(argv); if (args.help) { console.log(usage()); return; } if (!args.input) throw new Error('Missing required --input. ' + usage()); if (!args.output) throw new Error('Missing required --output. ' + usage()); const raw = await readFile(args.input, 'utf8'); const migration = JSON.parse(raw); const output = extractWorksListData(migration, { source: basename(args.input) }); await writeFile(args.output, `${JSON.stringify(output, null, 2)}\n`, 'utf8'); if (args['sample-output']) { const sample = createSampleOutput(output); await writeFile(args['sample-output'], `${JSON.stringify(sample, null, 2)}\n`, 'utf8'); } console.log( `works-list extracted: source=${output.source}, tables=${Object.keys(output.tables).length}, normalizedWorks=${output.normalizedWorks.length}`, ); for (const [tableName, count] of Object.entries(output.counts)) { console.log(` ${tableName}: ${count}`); } } const isDirectRun = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]; if (isDirectRun) { runCli().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; }); }