refactor: sync languages for import using a worker

db/scripts/schema.sql

@@ -2,7 +2,7 @@
 -- PostgreSQL database dump
 --
 
-\restrict c94gLPI6ezO1vBMLBMfOfYjDbpargAbfZojWXlj17MycdwqgZaVDh4Fq687emKQ
+\restrict Vr7pCDALsHJAnIofCw21egcssPsjFyuBsjcQc8LqN2ef1IGjTc1lCERXabetfOh
 
 -- Dumped from database version 14.22 (Debian 14.22-1.pgdg13+1)
 -- Dumped by pg_dump version 14.22 (Debian 14.22-1.pgdg13+1)
@@ -409,6 +409,17 @@ SET default_tablespace = '';
 
 SET default_table_access_method = heap;
 
+--
+-- Name: ai_gloss_language; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.ai_gloss_language (
+    code text NOT NULL,
+    name text NOT NULL,
+    created_at timestamp with time zone DEFAULT now() NOT NULL
+);
+
+
 --
 -- Name: book; Type: TABLE; Schema: public; Owner: -
 --
@@ -1124,6 +1135,14 @@ ALTER TABLE ONLY public.weekly_contribution_statistics ALTER COLUMN id SET DEFAU
 ALTER TABLE ONLY public.weekly_gloss_statistics ALTER COLUMN id SET DEFAULT nextval('public.weekly_gloss_statistics_id_seq'::regclass);
 
 
+--
+-- Name: ai_gloss_language ai_gloss_language_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.ai_gloss_language
+    ADD CONSTRAINT ai_gloss_language_pkey PRIMARY KEY (code);
+
+
 --
 -- Name: book_completion_progress book_completion_progress_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -2036,5 +2055,5 @@ ALTER TABLE ONLY public.word
 -- PostgreSQL database dump complete
 --
 
-\unrestrict c94gLPI6ezO1vBMLBMfOfYjDbpargAbfZojWXlj17MycdwqgZaVDh4Fq687emKQ
+\unrestrict Vr7pCDALsHJAnIofCw21egcssPsjFyuBsjcQc8LqN2ef1IGjTc1lCERXabetfOh
 

package.json

@@ -34,7 +34,6 @@
     "@google-cloud/translate": "9.3.0",
     "@gracious.tech/fetch-client": "0.8.9",
     "@headlessui/react": "2.2.9",
-    "@isaacs/ttlcache": "2.1.4",
     "@octokit/rest": "22.0.1",
     "@tailwindcss/vite": "4.2.0",
     "@tanstack/eslint-plugin-query": "5.91.5",

src/db.ts

@@ -32,6 +32,7 @@ import {
   WordTable,
 } from "./modules/bible-core/db/schema";
 import {
+  AIGlossLanguageTable,
   FootnoteTable,
   GlossEventTable,
   GlossHistoryTable,
@@ -50,6 +51,7 @@ import {
 } from "./modules/reporting/db/schema";
 
 export interface Database {
+  ai_gloss_language: AIGlossLanguageTable;
   book: BookTable;
   book_completion_progress: BookCompletionProgressTable;
   book_word_map: BookWordMapView;

src/modules/translation/data-access/aiGlossLanguageRepository.test.ts

@@ -0,0 +1,73 @@
+import { getDb } from "@/db";
+import { initializeDatabase } from "@/tests/vitest/dbUtils";
+import { describe, expect, test } from "vitest";
+import { aiGlossLanguageRepository } from "./aiGlossLanguageRepository";
+
+initializeDatabase();
+
+describe("upsertAll", () => {
+  test("inserts all provided languages", async () => {
+    await aiGlossLanguageRepository.upsertAll([
+      { code: "eng", name: "English" },
+      { code: "spa", name: "Spanish" },
+    ]);
+
+    const languages = await getDb()
+      .selectFrom("ai_gloss_language")
+      .orderBy("code")
+      .selectAll()
+      .execute();
+
+    expect(languages).toEqual([
+      {
+        code: "eng",
+        name: "English",
+        created_at: expect.toBeNow(),
+      },
+      {
+        code: "spa",
+        name: "Spanish",
+        created_at: expect.toBeNow(),
+      },
+    ]);
+  });
+
+  test("updates existing names while preserving created_at", async () => {
+    await aiGlossLanguageRepository.upsertAll([
+      { code: "eng", name: "English" },
+    ]);
+
+    const inserted = await getDb()
+      .selectFrom("ai_gloss_language")
+      .where("code", "=", "eng")
+      .selectAll()
+      .executeTakeFirstOrThrow();
+
+    await aiGlossLanguageRepository.upsertAll([
+      { code: "eng", name: "English Updated" },
+    ]);
+
+    const updated = await getDb()
+      .selectFrom("ai_gloss_language")
+      .where("code", "=", "eng")
+      .selectAll()
+      .executeTakeFirstOrThrow();
+
+    expect(updated).toEqual({
+      code: "eng",
+      name: "English Updated",
+      created_at: inserted.created_at,
+    });
+  });
+
+  test("does nothing for empty input", async () => {
+    await aiGlossLanguageRepository.upsertAll([]);
+
+    const languages = await getDb()
+      .selectFrom("ai_gloss_language")
+      .selectAll()
+      .execute();
+
+    expect(languages).toEqual([]);
+  });
+});

src/modules/translation/data-access/aiGlossLanguageRepository.ts

@@ -0,0 +1,24 @@
+import { getDb } from "@/db";
+
+export interface UpsertAIGlossLanguageInput {
+  code: string;
+  name: string;
+}
+
+export const aiGlossLanguageRepository = {
+  async upsertAll(languages: Array<UpsertAIGlossLanguageInput>): Promise<void> {
+    if (languages.length === 0) {
+      return;
+    }
+
+    await getDb()
+      .insertInto("ai_gloss_language")
+      .values(languages)
+      .onConflict((oc) =>
+        oc.column("code").doUpdateSet((eb) => ({
+          name: eb.ref("excluded.name"),
+        })),
+      )
+      .execute();
+  },
+};

src/modules/translation/db/migrations/ai-gloss-language.schema.sql

@@ -0,0 +1,9 @@
+begin;
+
+create table ai_gloss_language (
+  code text primary key,
+  name text not null,
+  created_at timestamptz not null default now()
+);
+
+commit;

src/modules/translation/db/schema.ts

@@ -65,6 +65,12 @@ export interface MachineGlossModelTable {
   code: string;
 }
 
+export interface AIGlossLanguageTable {
+  code: string;
+  name: string;
+  created_at: Generated<Date>;
+}
+
 export interface GlossEventTable {
   id: string;
   phrase_id: number;

src/modules/translation/jobs/jobType.ts

@@ -1,3 +1,4 @@
 export const TRANSLATION_JOB_TYPES = {
   IMPORT_AI_GLOSSES: "import_ai_glosses",
+  SYNC_AI_GLOSS_LANGUAGES: "sync_ai_gloss_languages",
 };

src/modules/translation/jobs/syncAIGlossLanguages.ts

@@ -0,0 +1,38 @@
+import { logger } from "@/logging";
+import { Job } from "@/shared/jobs/model";
+import { aiGlossImportService } from "../data-access/aiGlossImportService";
+import { aiGlossLanguageRepository } from "../data-access/aiGlossLanguageRepository";
+import { TRANSLATION_JOB_TYPES } from "./jobType";
+
+export async function syncAIGlossLanguages(job: Job<void>) {
+  const jobLogger = logger.child({
+    job: {
+      id: job.id,
+      type: job.type,
+    },
+  });
+
+  if (job.type !== TRANSLATION_JOB_TYPES.SYNC_AI_GLOSS_LANGUAGES) {
+    jobLogger.error(
+      `received job type ${job.type}, expected ${TRANSLATION_JOB_TYPES.SYNC_AI_GLOSS_LANGUAGES}`,
+    );
+    throw new Error(
+      `Expected job type ${TRANSLATION_JOB_TYPES.SYNC_AI_GLOSS_LANGUAGES}, but received ${job.type}`,
+    );
+  }
+
+  jobLogger.info("Starting sync of AI gloss languages");
+
+  const languages = await aiGlossImportService.getAvailableLanguages();
+  await aiGlossLanguageRepository.upsertAll(
+    languages.map((language) => ({
+      code: language.code,
+      name: language.name,
+    })),
+  );
+
+  jobLogger.info(
+    { languageCount: languages.length },
+    "Synced AI gloss languages",
+  );
+}

src/shared/jobs/jobMap.ts

@@ -10,6 +10,7 @@ import { exportGlossesChildJob } from "@/modules/export/jobs/exportGlossesChildJ
 import { exportGlossesFinalizeJob } from "@/modules/export/jobs/exportGlossesFinalizeJob";
 import { TRANSLATION_JOB_TYPES } from "@/modules/translation/jobs/jobType";
 import { importAIGlosses } from "@/modules/translation/jobs/importAIGlosses";
+import { syncAIGlossLanguages } from "@/modules/translation/jobs/syncAIGlossLanguages";
 
 export type JobHandler<Payload, Data = unknown> = (
   job: Job<Payload, Data>,
@@ -55,6 +56,10 @@ const jobMap: Record<string, JobMapEntry<any, any>> = {
     handler: importAIGlosses,
     timeout: 60 * 15, // 15 minutes
   },
+  [TRANSLATION_JOB_TYPES.SYNC_AI_GLOSS_LANGUAGES]: {
+    handler: syncAIGlossLanguages,
+    timeout: 60 * 5, // 5 minutes
+  },
 };
 
 export default jobMap;

src/ui/admin/readModels/getAIGlossImportLanguagesReadModel.ts

@@ -1,25 +1,16 @@
-import { TTLCache } from "@isaacs/ttlcache";
-import {
-  aiGlossImportService,
-  type Language,
-} from "@/modules/translation/data-access/aiGlossImportService";
+import { getDb } from "@/db";
 
-const CACHE_KEY = "ai-gloss-import-languages";
-const THREE_HOURS_MS = 3 * 60 * 60 * 1000;
-
-const aiGlossImportLanguagesCache = new TTLCache<string, Array<Language>>({
-  ttl: THREE_HOURS_MS,
-});
-
-export async function getAIGlossImportLanguagesReadModel() {
-  const cachedLanguages = aiGlossImportLanguagesCache.get(CACHE_KEY, {
-    checkAgeOnGet: true,
-  });
-  if (cachedLanguages) {
-    return cachedLanguages;
-  }
+export interface AIGlossImportLanguageReadModel {
+  code: string;
+  name: string;
+}
 
-  const languages = await aiGlossImportService.getAvailableLanguages();
-  aiGlossImportLanguagesCache.set(CACHE_KEY, languages);
-  return languages;
+export async function getAIGlossImportLanguagesReadModel(): Promise<
+  Array<AIGlossImportLanguageReadModel>
+> {
+  return getDb()
+    .selectFrom("ai_gloss_language")
+    .select(["code", "name"])
+    .orderBy("name")
+    .execute();
 }

src/ui/admin/routes/_main.jobs.tsx

@@ -17,6 +17,7 @@ import { useSuspenseQuery } from "@tanstack/react-query";
 import { createFileRoute } from "@tanstack/react-router";
 import { withDocumentTitle } from "@/documentTitle";
 import { getActiveJobs } from "@/ui/admin/serverFns/getActiveJobs";
+import { TRANSLATION_JOB_TYPES } from "@/modules/translation/jobs/jobType";
 
 export const Route = createFileRoute("/_main/admin/_main/jobs")({
   head: () => withDocumentTitle("Jobs | Admin"),
@@ -58,6 +59,14 @@ function AdminJobsView() {
         >
           Recompute Language Progress
         </ServerAction>
+        <ServerAction
+          actionData={{
+            type: TRANSLATION_JOB_TYPES.SYNC_AI_GLOSS_LANGUAGES,
+          }}
+          action={queueJobAction}
+        >
+          Sync AI Gloss Languages
+        </ServerAction>
 
         <div className="mt-8 max-w-5xl">
           <h2 className="text-xl font-bold mb-3">GitHub Export</h2>