clicktrack/lib/registry/sync.ts

/**
 * Git Registry Sync
 *
 * Pulls CTP files from a remote git repository (the "community registry")
 * served over HTTPS. Compatible with any self-hosted git server
 * (Gitea, Forgejo, GitLab CE, Gogs, etc.) or any public git host.
 *
 * The registry repo is expected to contain CTP JSON files at:
 *   <repo-root>/<artist-initial>/<artist-slug>/<recording-mbid>.ctp.json
 *
 * Configuration:
 *   REGISTRY_REPO — HTTPS URL of the registry repo,
 *                   e.g. https://git.yourdomain.com/org/clicktrack-registry
 *                   To authenticate, embed credentials in the URL:
 *                   https://user:token@git.yourdomain.com/org/clicktrack-registry
 *   REGISTRY_BRANCH — branch to pull from (default: main)
 */

import { execSync } from "child_process";
import { existsSync, readdirSync, readFileSync, statSync } from "fs";
import { join } from "path";
import { tmpdir } from "os";
import { validateCTP } from "../ctp/validate";
import { upsertSong, insertTempoMap, withTransaction, query } from "../db/client";

const REGISTRY_REPO = process.env.REGISTRY_REPO ?? "";
const REGISTRY_BRANCH = process.env.REGISTRY_BRANCH ?? "main";
const CLONE_DIR = join(tmpdir(), "clicktrack-registry");

// ─── Types ────────────────────────────────────────────────────────────────────

export interface SyncResult {
  recordsAdded: number;
  recordsUpdated: number;
  errors: Array<{ file: string; error: string }>;
  status: "success" | "partial" | "failed";
}

// ─── Helpers ──────────────────────────────────────────────────────────────────

function cloneOrPull(): void {
  if (existsSync(join(CLONE_DIR, ".git"))) {
    execSync(`git -C "${CLONE_DIR}" fetch origin "${REGISTRY_BRANCH}" --depth=1 -q`, {
      stdio: "pipe",
    });
    execSync(`git -C "${CLONE_DIR}" reset --hard "origin/${REGISTRY_BRANCH}" -q`, {
      stdio: "pipe",
    });
  } else {
    execSync(
      `git clone --depth=1 --branch "${REGISTRY_BRANCH}" "${REGISTRY_REPO}" "${CLONE_DIR}"`,
      { stdio: "pipe" }
    );
  }
}

/** Recursively finds all .ctp.json files under a directory. */
function findCtpFiles(dir: string): string[] {
  const results: string[] = [];
  for (const entry of readdirSync(dir)) {
    const full = join(dir, entry);
    if (statSync(full).isDirectory()) {
      results.push(...findCtpFiles(full));
    } else if (entry.endsWith(".ctp.json")) {
      results.push(full);
    }
  }
  return results;
}

// ─── Main sync function ───────────────────────────────────────────────────────

export async function syncRegistry(): Promise<SyncResult> {
  if (!REGISTRY_REPO) {
    return {
      recordsAdded: 0,
      recordsUpdated: 0,
      errors: [{ file: "", error: "REGISTRY_REPO is not configured" }],
      status: "failed",
    };
  }

  let recordsAdded = 0;
  let recordsUpdated = 0;
  const errors: SyncResult["errors"] = [];

  try {
    cloneOrPull();
  } catch (err) {
    const message = err instanceof Error ? err.message : String(err);
    await logSync(0, 0, "failed", `Git clone/pull failed: ${message}`);
    return { recordsAdded: 0, recordsUpdated: 0, errors: [{ file: "", error: message }], status: "failed" };
  }

  const files = findCtpFiles(CLONE_DIR);

  for (const filePath of files) {
    let raw: unknown;
    try {
      raw = JSON.parse(readFileSync(filePath, "utf-8"));
    } catch (err) {
      errors.push({ file: filePath, error: `JSON parse error: ${err}` });
      continue;
    }

    const validation = validateCTP(raw);
    if (!validation.success) {
      errors.push({
        file: filePath,
        error: `CTP validation failed: ${validation.errors.message}`,
      });
      continue;
    }

    const doc = validation.data;

    try {
      await withTransaction(async (client) => {
        // Ensure the song exists in the songs table
        if (doc.metadata.mbid) {
          await upsertSong({
            mbid: doc.metadata.mbid,
            title: doc.metadata.title,
            artist: doc.metadata.artist,
            duration_seconds: doc.metadata.duration_seconds,
            acousticbrainz_bpm: null,
            acousticbrainz_time_sig_num: null,
            source: "registry",
          });

          // Check if a matching tempo map already exists (same mbid + contributor)
          const existing = await client.query(
            `SELECT id FROM tempo_maps
             WHERE song_mbid = $1 AND contributed_by = $2
             LIMIT 1`,
            [doc.metadata.mbid, doc.metadata.contributed_by]
          );

          if (existing.rowCount && existing.rowCount > 0) {
            await client.query(
              `UPDATE tempo_maps SET ctp_data = $1, updated_at = NOW()
               WHERE song_mbid = $2 AND contributed_by = $3`,
              [JSON.stringify(raw), doc.metadata.mbid, doc.metadata.contributed_by]
            );
            recordsUpdated++;
          } else {
            await insertTempoMap({
              song_mbid: doc.metadata.mbid,
              ctp_data: raw as Record<string, unknown>,
              contributed_by: doc.metadata.contributed_by,
            });
            recordsAdded++;
          }
        }
      });
    } catch (err) {
      errors.push({ file: filePath, error: `DB upsert error: ${err}` });
    }
  }

  const status: SyncResult["status"] =
    errors.length === 0 ? "success" : recordsAdded + recordsUpdated > 0 ? "partial" : "failed";

  await logSync(recordsAdded, recordsUpdated, status, errors.length > 0 ? JSON.stringify(errors.slice(0, 5)) : undefined);

  return { recordsAdded, recordsUpdated, errors, status };
}

async function logSync(
  added: number,
  updated: number,
  status: string,
  message?: string
): Promise<void> {
  await query(
    `INSERT INTO registry_sync_log (records_added, records_updated, status, message)
     VALUES ($1, $2, $3, $4)`,
    [added, updated, status, message ?? null]
  );
}