diff --git a/db/scripts/data.dump b/db/scripts/data.dump index 196115b4..31f1f28c 100644 Binary files a/db/scripts/data.dump and b/db/scripts/data.dump differ diff --git a/db/scripts/schema.sql b/db/scripts/schema.sql index a53cacb8..5aa4ee6c 100644 --- a/db/scripts/schema.sql +++ b/db/scripts/schema.sql @@ -2,7 +2,7 @@ -- PostgreSQL database dump -- -\restrict c94gLPI6ezO1vBMLBMfOfYjDbpargAbfZojWXlj17MycdwqgZaVDh4Fq687emKQ +\restrict Vr7pCDALsHJAnIofCw21egcssPsjFyuBsjcQc8LqN2ef1IGjTc1lCERXabetfOh -- Dumped from database version 14.22 (Debian 14.22-1.pgdg13+1) -- Dumped by pg_dump version 14.22 (Debian 14.22-1.pgdg13+1) @@ -409,6 +409,17 @@ SET default_tablespace = ''; SET default_table_access_method = heap; +-- +-- Name: ai_gloss_language; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.ai_gloss_language ( + code text NOT NULL, + name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + -- -- Name: book; Type: TABLE; Schema: public; Owner: - -- @@ -1124,6 +1135,14 @@ ALTER TABLE ONLY public.weekly_contribution_statistics ALTER COLUMN id SET DEFAU ALTER TABLE ONLY public.weekly_gloss_statistics ALTER COLUMN id SET DEFAULT nextval('public.weekly_gloss_statistics_id_seq'::regclass); +-- +-- Name: ai_gloss_language ai_gloss_language_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.ai_gloss_language + ADD CONSTRAINT ai_gloss_language_pkey PRIMARY KEY (code); + + -- -- Name: book_completion_progress book_completion_progress_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -2036,5 +2055,5 @@ ALTER TABLE ONLY public.word -- PostgreSQL database dump complete -- -\unrestrict c94gLPI6ezO1vBMLBMfOfYjDbpargAbfZojWXlj17MycdwqgZaVDh4Fq687emKQ +\unrestrict Vr7pCDALsHJAnIofCw21egcssPsjFyuBsjcQc8LqN2ef1IGjTc1lCERXabetfOh diff --git a/package-lock.json b/package-lock.json index 2e3d505a..d5f8cfde 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,6 @@ "@google-cloud/translate": "9.3.0", "@gracious.tech/fetch-client": "0.8.9", "@headlessui/react": "2.2.9", - "@isaacs/ttlcache": "2.1.4", "@octokit/rest": "22.0.1", "@tailwindcss/vite": "4.2.0", "@tanstack/eslint-plugin-query": "5.91.5", @@ -3677,16 +3676,6 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/@isaacs/ttlcache": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@isaacs/ttlcache/-/ttlcache-2.1.4.tgz", - "integrity": "sha512-7kMz0BJpMvgAMkyglums7B2vtrn5g0a0am77JY0GjkZZNetOBCFn7AG7gKCwT0QPiXyxW7YIQSgtARknUEOcxQ==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=12" - } - }, "node_modules/@jest/schemas": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", diff --git a/package.json b/package.json index 72bbcbe8..8a6e9777 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,6 @@ "@google-cloud/translate": "9.3.0", "@gracious.tech/fetch-client": "0.8.9", "@headlessui/react": "2.2.9", - "@isaacs/ttlcache": "2.1.4", "@octokit/rest": "22.0.1", "@tailwindcss/vite": "4.2.0", "@tanstack/eslint-plugin-query": "5.91.5", diff --git a/src/db.ts b/src/db.ts index 5c7833fc..47e05f09 100644 --- a/src/db.ts +++ b/src/db.ts @@ -32,6 +32,7 @@ import { WordTable, } from "./modules/bible-core/db/schema"; import { + AIGlossLanguageTable, FootnoteTable, GlossEventTable, GlossHistoryTable, @@ -50,6 +51,7 @@ import { } from "./modules/reporting/db/schema"; export interface Database { + ai_gloss_language: AIGlossLanguageTable; book: BookTable; book_completion_progress: BookCompletionProgressTable; book_word_map: BookWordMapView; diff --git a/src/modules/translation/data-access/aiGlossLanguageRepository.test.ts b/src/modules/translation/data-access/aiGlossLanguageRepository.test.ts new file mode 100644 index 00000000..aaa0b5c4 --- /dev/null +++ b/src/modules/translation/data-access/aiGlossLanguageRepository.test.ts @@ -0,0 +1,73 @@ +import { getDb } from "@/db"; +import { initializeDatabase } from "@/tests/vitest/dbUtils"; +import { describe, expect, test } from "vitest"; +import { aiGlossLanguageRepository } from "./aiGlossLanguageRepository"; + +initializeDatabase(); + +describe("upsertAll", () => { + test("inserts all provided languages", async () => { + await aiGlossLanguageRepository.upsertAll([ + { code: "eng", name: "English" }, + { code: "spa", name: "Spanish" }, + ]); + + const languages = await getDb() + .selectFrom("ai_gloss_language") + .orderBy("code") + .selectAll() + .execute(); + + expect(languages).toEqual([ + { + code: "eng", + name: "English", + created_at: expect.toBeNow(), + }, + { + code: "spa", + name: "Spanish", + created_at: expect.toBeNow(), + }, + ]); + }); + + test("updates existing names while preserving created_at", async () => { + await aiGlossLanguageRepository.upsertAll([ + { code: "eng", name: "English" }, + ]); + + const inserted = await getDb() + .selectFrom("ai_gloss_language") + .where("code", "=", "eng") + .selectAll() + .executeTakeFirstOrThrow(); + + await aiGlossLanguageRepository.upsertAll([ + { code: "eng", name: "English Updated" }, + ]); + + const updated = await getDb() + .selectFrom("ai_gloss_language") + .where("code", "=", "eng") + .selectAll() + .executeTakeFirstOrThrow(); + + expect(updated).toEqual({ + code: "eng", + name: "English Updated", + created_at: inserted.created_at, + }); + }); + + test("does nothing for empty input", async () => { + await aiGlossLanguageRepository.upsertAll([]); + + const languages = await getDb() + .selectFrom("ai_gloss_language") + .selectAll() + .execute(); + + expect(languages).toEqual([]); + }); +}); diff --git a/src/modules/translation/data-access/aiGlossLanguageRepository.ts b/src/modules/translation/data-access/aiGlossLanguageRepository.ts new file mode 100644 index 00000000..d055f8b3 --- /dev/null +++ b/src/modules/translation/data-access/aiGlossLanguageRepository.ts @@ -0,0 +1,24 @@ +import { getDb } from "@/db"; + +export interface UpsertAIGlossLanguageInput { + code: string; + name: string; +} + +export const aiGlossLanguageRepository = { + async upsertAll(languages: Array): Promise { + if (languages.length === 0) { + return; + } + + await getDb() + .insertInto("ai_gloss_language") + .values(languages) + .onConflict((oc) => + oc.column("code").doUpdateSet((eb) => ({ + name: eb.ref("excluded.name"), + })), + ) + .execute(); + }, +}; diff --git a/src/modules/translation/db/migrations/ai-gloss-language.schema.sql b/src/modules/translation/db/migrations/ai-gloss-language.schema.sql new file mode 100644 index 00000000..a41ace95 --- /dev/null +++ b/src/modules/translation/db/migrations/ai-gloss-language.schema.sql @@ -0,0 +1,9 @@ +begin; + +create table ai_gloss_language ( + code text primary key, + name text not null, + created_at timestamptz not null default now() +); + +commit; diff --git a/src/modules/translation/db/schema.ts b/src/modules/translation/db/schema.ts index 9b931fc2..6030feb4 100644 --- a/src/modules/translation/db/schema.ts +++ b/src/modules/translation/db/schema.ts @@ -65,6 +65,12 @@ export interface MachineGlossModelTable { code: string; } +export interface AIGlossLanguageTable { + code: string; + name: string; + created_at: Generated; +} + export interface GlossEventTable { id: string; phrase_id: number; diff --git a/src/modules/translation/jobs/jobType.ts b/src/modules/translation/jobs/jobType.ts index cb5d2ef6..1d3249dc 100644 --- a/src/modules/translation/jobs/jobType.ts +++ b/src/modules/translation/jobs/jobType.ts @@ -1,3 +1,4 @@ export const TRANSLATION_JOB_TYPES = { IMPORT_AI_GLOSSES: "import_ai_glosses", + SYNC_AI_GLOSS_LANGUAGES: "sync_ai_gloss_languages", }; diff --git a/src/modules/translation/jobs/syncAIGlossLanguages.ts b/src/modules/translation/jobs/syncAIGlossLanguages.ts new file mode 100644 index 00000000..a62e7891 --- /dev/null +++ b/src/modules/translation/jobs/syncAIGlossLanguages.ts @@ -0,0 +1,38 @@ +import { logger } from "@/logging"; +import { Job } from "@/shared/jobs/model"; +import { aiGlossImportService } from "../data-access/aiGlossImportService"; +import { aiGlossLanguageRepository } from "../data-access/aiGlossLanguageRepository"; +import { TRANSLATION_JOB_TYPES } from "./jobType"; + +export async function syncAIGlossLanguages(job: Job) { + const jobLogger = logger.child({ + job: { + id: job.id, + type: job.type, + }, + }); + + if (job.type !== TRANSLATION_JOB_TYPES.SYNC_AI_GLOSS_LANGUAGES) { + jobLogger.error( + `received job type ${job.type}, expected ${TRANSLATION_JOB_TYPES.SYNC_AI_GLOSS_LANGUAGES}`, + ); + throw new Error( + `Expected job type ${TRANSLATION_JOB_TYPES.SYNC_AI_GLOSS_LANGUAGES}, but received ${job.type}`, + ); + } + + jobLogger.info("Starting sync of AI gloss languages"); + + const languages = await aiGlossImportService.getAvailableLanguages(); + await aiGlossLanguageRepository.upsertAll( + languages.map((language) => ({ + code: language.code, + name: language.name, + })), + ); + + jobLogger.info( + { languageCount: languages.length }, + "Synced AI gloss languages", + ); +} diff --git a/src/shared/jobs/jobMap.ts b/src/shared/jobs/jobMap.ts index ed58c674..90813cfb 100644 --- a/src/shared/jobs/jobMap.ts +++ b/src/shared/jobs/jobMap.ts @@ -10,6 +10,7 @@ import { exportGlossesChildJob } from "@/modules/export/jobs/exportGlossesChildJ import { exportGlossesFinalizeJob } from "@/modules/export/jobs/exportGlossesFinalizeJob"; import { TRANSLATION_JOB_TYPES } from "@/modules/translation/jobs/jobType"; import { importAIGlosses } from "@/modules/translation/jobs/importAIGlosses"; +import { syncAIGlossLanguages } from "@/modules/translation/jobs/syncAIGlossLanguages"; export type JobHandler = ( job: Job, @@ -55,6 +56,10 @@ const jobMap: Record> = { handler: importAIGlosses, timeout: 60 * 15, // 15 minutes }, + [TRANSLATION_JOB_TYPES.SYNC_AI_GLOSS_LANGUAGES]: { + handler: syncAIGlossLanguages, + timeout: 60 * 5, // 5 minutes + }, }; export default jobMap; diff --git a/src/ui/admin/readModels/getAIGlossImportLanguagesReadModel.ts b/src/ui/admin/readModels/getAIGlossImportLanguagesReadModel.ts index 0e02a4e1..79f39146 100644 --- a/src/ui/admin/readModels/getAIGlossImportLanguagesReadModel.ts +++ b/src/ui/admin/readModels/getAIGlossImportLanguagesReadModel.ts @@ -1,25 +1,16 @@ -import { TTLCache } from "@isaacs/ttlcache"; -import { - aiGlossImportService, - type Language, -} from "@/modules/translation/data-access/aiGlossImportService"; +import { getDb } from "@/db"; -const CACHE_KEY = "ai-gloss-import-languages"; -const THREE_HOURS_MS = 3 * 60 * 60 * 1000; - -const aiGlossImportLanguagesCache = new TTLCache>({ - ttl: THREE_HOURS_MS, -}); - -export async function getAIGlossImportLanguagesReadModel() { - const cachedLanguages = aiGlossImportLanguagesCache.get(CACHE_KEY, { - checkAgeOnGet: true, - }); - if (cachedLanguages) { - return cachedLanguages; - } +export interface AIGlossImportLanguageReadModel { + code: string; + name: string; +} - const languages = await aiGlossImportService.getAvailableLanguages(); - aiGlossImportLanguagesCache.set(CACHE_KEY, languages); - return languages; +export async function getAIGlossImportLanguagesReadModel(): Promise< + Array +> { + return getDb() + .selectFrom("ai_gloss_language") + .select(["code", "name"]) + .orderBy("name") + .execute(); } diff --git a/src/ui/admin/routes/_main.jobs.tsx b/src/ui/admin/routes/_main.jobs.tsx index 157142c1..c4e8a87a 100644 --- a/src/ui/admin/routes/_main.jobs.tsx +++ b/src/ui/admin/routes/_main.jobs.tsx @@ -17,6 +17,7 @@ import { useSuspenseQuery } from "@tanstack/react-query"; import { createFileRoute } from "@tanstack/react-router"; import { withDocumentTitle } from "@/documentTitle"; import { getActiveJobs } from "@/ui/admin/serverFns/getActiveJobs"; +import { TRANSLATION_JOB_TYPES } from "@/modules/translation/jobs/jobType"; export const Route = createFileRoute("/_main/admin/_main/jobs")({ head: () => withDocumentTitle("Jobs | Admin"), @@ -58,6 +59,14 @@ function AdminJobsView() { > Recompute Language Progress + + Sync AI Gloss Languages +

GitHub Export