/** * Git Registry Sync * * Pulls CTP files from a remote GitHub repository (the "community registry") * and upserts them into the local database. * * The registry repo is expected to contain CTP JSON files at: * ///.ctp.json * * Configuration: * REGISTRY_REPO — GitHub repo URL, e.g. https://github.com/org/clicktrack-registry * REGISTRY_BRANCH — branch to pull from (default: main) */ import { execSync } from "child_process"; import { existsSync, readdirSync, readFileSync, statSync } from "fs"; import { join } from "path"; import { tmpdir } from "os"; import { validateCTP } from "../ctp/validate"; import { upsertSong, insertTempoMap, withTransaction, query } from "../db/client"; const REGISTRY_REPO = process.env.REGISTRY_REPO ?? ""; const REGISTRY_BRANCH = process.env.REGISTRY_BRANCH ?? "main"; const CLONE_DIR = join(tmpdir(), "clicktrack-registry"); // ─── Types ──────────────────────────────────────────────────────────────────── export interface SyncResult { recordsAdded: number; recordsUpdated: number; errors: Array<{ file: string; error: string }>; status: "success" | "partial" | "failed"; } // ─── Helpers ────────────────────────────────────────────────────────────────── function cloneOrPull(): void { if (existsSync(join(CLONE_DIR, ".git"))) { execSync(`git -C "${CLONE_DIR}" fetch origin "${REGISTRY_BRANCH}" --depth=1 -q`, { stdio: "pipe", }); execSync(`git -C "${CLONE_DIR}" reset --hard "origin/${REGISTRY_BRANCH}" -q`, { stdio: "pipe", }); } else { execSync( `git clone --depth=1 --branch "${REGISTRY_BRANCH}" "${REGISTRY_REPO}" "${CLONE_DIR}"`, { stdio: "pipe" } ); } } /** Recursively finds all .ctp.json files under a directory. */ function findCtpFiles(dir: string): string[] { const results: string[] = []; for (const entry of readdirSync(dir)) { const full = join(dir, entry); if (statSync(full).isDirectory()) { results.push(...findCtpFiles(full)); } else if (entry.endsWith(".ctp.json")) { results.push(full); } } return results; } // ─── Main sync function ─────────────────────────────────────────────────────── export async function syncRegistry(): Promise { if (!REGISTRY_REPO) { return { recordsAdded: 0, recordsUpdated: 0, errors: [{ file: "", error: "REGISTRY_REPO is not configured" }], status: "failed", }; } let recordsAdded = 0; let recordsUpdated = 0; const errors: SyncResult["errors"] = []; try { cloneOrPull(); } catch (err) { const message = err instanceof Error ? err.message : String(err); await logSync(0, 0, "failed", `Git clone/pull failed: ${message}`); return { recordsAdded: 0, recordsUpdated: 0, errors: [{ file: "", error: message }], status: "failed" }; } const files = findCtpFiles(CLONE_DIR); for (const filePath of files) { let raw: unknown; try { raw = JSON.parse(readFileSync(filePath, "utf-8")); } catch (err) { errors.push({ file: filePath, error: `JSON parse error: ${err}` }); continue; } const validation = validateCTP(raw); if (!validation.success) { errors.push({ file: filePath, error: `CTP validation failed: ${validation.errors.message}`, }); continue; } const doc = validation.data; try { await withTransaction(async (client) => { // Ensure the song exists in the songs table if (doc.metadata.mbid) { await upsertSong({ mbid: doc.metadata.mbid, title: doc.metadata.title, artist: doc.metadata.artist, duration_seconds: doc.metadata.duration_seconds, acousticbrainz_bpm: null, acousticbrainz_time_sig_num: null, source: "registry", }); // Check if a matching tempo map already exists (same mbid + contributor) const existing = await client.query( `SELECT id FROM tempo_maps WHERE song_mbid = $1 AND contributed_by = $2 LIMIT 1`, [doc.metadata.mbid, doc.metadata.contributed_by] ); if (existing.rowCount && existing.rowCount > 0) { await client.query( `UPDATE tempo_maps SET ctp_data = $1, updated_at = NOW() WHERE song_mbid = $2 AND contributed_by = $3`, [JSON.stringify(raw), doc.metadata.mbid, doc.metadata.contributed_by] ); recordsUpdated++; } else { await insertTempoMap({ song_mbid: doc.metadata.mbid, ctp_data: raw as Record, contributed_by: doc.metadata.contributed_by, }); recordsAdded++; } } }); } catch (err) { errors.push({ file: filePath, error: `DB upsert error: ${err}` }); } } const status: SyncResult["status"] = errors.length === 0 ? "success" : recordsAdded + recordsUpdated > 0 ? "partial" : "failed"; await logSync(recordsAdded, recordsUpdated, status, errors.length > 0 ? JSON.stringify(errors.slice(0, 5)) : undefined); return { recordsAdded, recordsUpdated, errors, status }; } async function logSync( added: number, updated: number, status: string, message?: string ): Promise { await query( `INSERT INTO registry_sync_log (records_added, records_updated, status, message) VALUES ($1, $2, $3, $4)`, [added, updated, status, message ?? null] ); }