diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 1acf7bdc60..e927dfee72 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -54,6 +54,12 @@ jobs: run: bun scripts/lint/no-unauth-routes.ts - name: Check unsafe type casts run: bun check:casts:strict + - name: Check for duplicate @packrat/utils implementations + run: bun check:utils + - name: Check @packrat/utils provenance manifest + run: bun check:provenance:strict + - name: Check code duplication (jscpd) + run: bun check:duplication - name: Check types run: bun check-types - name: Run Expo Doctor diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000000..bcd1b7a05a --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,289 @@ +name: Coverage + +on: + push: + branches: ["main", "development"] + paths: + - "package.json" + - "bun.lock" + - "apps/**" + - "packages/**" + - "scripts/lint/coverage-ratchet.ts" + - "scripts/lint/coverage-baseline-update.ts" + - "scripts/lint/no-weak-assertions.ts" + - "scripts/vitest.config.ts" + - "coverage-baselines.json" + - ".github/workflows/coverage.yml" + pull_request: + branches: ["**"] + paths: + - "package.json" + - "bun.lock" + - "apps/**" + - "packages/**" + - "scripts/lint/coverage-ratchet.ts" + - "scripts/lint/coverage-baseline-update.ts" + - "scripts/lint/no-weak-assertions.ts" + - "scripts/vitest.config.ts" + - "coverage-baselines.json" + - ".github/workflows/coverage.yml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: write # for baseline auto-commit on main + pull-requests: write # for vitest-coverage-report-action comments + +jobs: + # One coverage run per tracked workspace. Uploads the coverage-summary.json + # as an artifact for the ratchet job to aggregate. + coverage: + name: Coverage (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # `summary_path` / `final_path` are repo-relative (used by artifact + # upload + ratchet restore). `summary_relative` / `final_relative` + # are relative to `working_directory` (used by the coverage report + # action, which joins them with working_directory internally). + - name: packages/api + artifact_slug: packages-api + test_command: bun run --cwd packages/api test:unit:coverage + summary_path: packages/api/coverage/unit/coverage-summary.json + final_path: packages/api/coverage/unit/coverage-final.json + summary_relative: ./coverage/unit/coverage-summary.json + final_relative: ./coverage/unit/coverage-final.json + vite_config_path: ./vitest.unit.config.ts + working_directory: ./packages/api + - name: apps/expo + artifact_slug: apps-expo + test_command: bun run --cwd apps/expo test:coverage + summary_path: apps/expo/coverage/unit/coverage-summary.json + final_path: apps/expo/coverage/unit/coverage-final.json + summary_relative: ./coverage/unit/coverage-summary.json + final_relative: ./coverage/unit/coverage-final.json + vite_config_path: ./vitest.config.ts + working_directory: ./apps/expo + - name: packages/mcp + artifact_slug: packages-mcp + test_command: bun run --cwd packages/mcp test --coverage + summary_path: packages/mcp/coverage/coverage-summary.json + final_path: packages/mcp/coverage/coverage-final.json + summary_relative: ./coverage/coverage-summary.json + final_relative: ./coverage/coverage-final.json + vite_config_path: ./vitest.config.ts + working_directory: ./packages/mcp + - name: packages/analytics + artifact_slug: packages-analytics + test_command: bun run --cwd packages/analytics test --coverage + summary_path: packages/analytics/coverage/coverage-summary.json + final_path: packages/analytics/coverage/coverage-final.json + summary_relative: ./coverage/coverage-summary.json + final_relative: ./coverage/coverage-final.json + vite_config_path: ./vitest.config.ts + working_directory: ./packages/analytics + - name: packages/overpass + artifact_slug: packages-overpass + test_command: bun run --cwd packages/overpass test --coverage + summary_path: packages/overpass/coverage/coverage-summary.json + final_path: packages/overpass/coverage/coverage-final.json + summary_relative: ./coverage/coverage-summary.json + final_relative: ./coverage/coverage-final.json + vite_config_path: ./vitest.config.ts + working_directory: ./packages/overpass + - name: packages/units + artifact_slug: packages-units + test_command: bun run --cwd packages/units test --coverage + summary_path: packages/units/coverage/coverage-summary.json + final_path: packages/units/coverage/coverage-final.json + summary_relative: ./coverage/coverage-summary.json + final_relative: ./coverage/coverage-final.json + vite_config_path: ./vitest.config.ts + working_directory: ./packages/units + - name: packages/utils + artifact_slug: packages-utils + test_command: bun run --cwd packages/utils test --coverage + summary_path: packages/utils/coverage/coverage-summary.json + final_path: packages/utils/coverage/coverage-final.json + summary_relative: ./coverage/coverage-summary.json + final_relative: ./coverage/coverage-final.json + vite_config_path: ./vitest.config.ts + working_directory: ./packages/utils + steps: + - uses: actions/checkout@v6 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + env: + PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN: ${{ secrets.PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN }} + run: bun install --frozen-lockfile + + - name: Run coverage for ${{ matrix.name }} + run: ${{ matrix.test_command }} + + - name: Report coverage on PR + if: always() && github.event_name == 'pull_request' + uses: davelosert/vitest-coverage-report-action@v2 + with: + name: ${{ matrix.name }} + json-summary-path: ${{ matrix.summary_relative }} + json-final-path: ${{ matrix.final_relative }} + vite-config-path: ${{ matrix.vite_config_path }} + working-directory: ${{ matrix.working_directory }} + + - name: Upload coverage summary artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-summary-${{ matrix.artifact_slug }} + path: ${{ matrix.summary_path }} + if-no-files-found: error + retention-days: 7 + + # Aggregate every workspace's coverage-summary.json and run the ratchet. + # Fails the workflow if any workspace dropped below its baseline. + ratchet: + name: Coverage Ratchet + runs-on: ubuntu-latest + needs: coverage + if: always() + steps: + - uses: actions/checkout@v6 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + env: + PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN: ${{ secrets.PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN }} + run: bun install --frozen-lockfile + + - name: Download all coverage summaries + uses: actions/download-artifact@v4 + with: + pattern: coverage-summary-* + path: artifacts + + - name: Restore summaries to their workspace paths + run: | + set -euo pipefail + # Each artifact arrives as artifacts/coverage-summary-/ + # actions/download-artifact@v4 unzips a single-file artifact into a directory + # named after the artifact, preserving the source file's basename. + # Copy each back to its expected location so coverage-baselines.json's + # summaryPath entries resolve. + declare -A targets=( + [packages-api]=packages/api/coverage/unit/coverage-summary.json + [apps-expo]=apps/expo/coverage/unit/coverage-summary.json + [packages-mcp]=packages/mcp/coverage/coverage-summary.json + [packages-analytics]=packages/analytics/coverage/coverage-summary.json + [packages-overpass]=packages/overpass/coverage/coverage-summary.json + [packages-units]=packages/units/coverage/coverage-summary.json + ) + for slug in "${!targets[@]}"; do + target="${targets[$slug]}" + src_dir="artifacts/coverage-summary-${slug}" + if [ ! -d "$src_dir" ]; then + echo "::warning::missing artifact for $slug — coverage job may have failed" + continue + fi + mkdir -p "$(dirname "$target")" + # Find the single JSON file inside (path may be flat or preserved). + src_file=$(find "$src_dir" -name 'coverage-summary.json' | head -n1) + if [ -z "$src_file" ]; then + echo "::warning::no coverage-summary.json inside artifacts/coverage-summary-${slug}" + continue + fi + cp "$src_file" "$target" + echo "restored $slug → $target" + done + + - name: Run coverage ratchet + run: bun check:coverage + + # On a green push to main, auto-bump coverage-baselines.json upward. + # Never runs on PRs — PRs cannot edit the baseline file silently. + bump-baseline: + name: Bump Coverage Baselines + runs-on: ubuntu-latest + needs: ratchet + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + steps: + - uses: actions/checkout@v6 + with: + # Need full token to push the auto-commit back to main. + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + env: + PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN: ${{ secrets.PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN }} + run: bun install --frozen-lockfile + + - name: Download all coverage summaries + uses: actions/download-artifact@v4 + with: + pattern: coverage-summary-* + path: artifacts + + - name: Restore summaries to their workspace paths + run: | + set -euo pipefail + declare -A targets=( + [packages-api]=packages/api/coverage/unit/coverage-summary.json + [apps-expo]=apps/expo/coverage/unit/coverage-summary.json + [packages-mcp]=packages/mcp/coverage/coverage-summary.json + [packages-analytics]=packages/analytics/coverage/coverage-summary.json + [packages-overpass]=packages/overpass/coverage/coverage-summary.json + [packages-units]=packages/units/coverage/coverage-summary.json + ) + for slug in "${!targets[@]}"; do + target="${targets[$slug]}" + src_dir="artifacts/coverage-summary-${slug}" + if [ ! -d "$src_dir" ]; then + continue + fi + mkdir -p "$(dirname "$target")" + src_file=$(find "$src_dir" -name 'coverage-summary.json' | head -n1) + if [ -n "$src_file" ]; then + cp "$src_file" "$target" + fi + done + + - name: Compute baseline updates + run: bun check:coverage:update + + - name: Commit baseline updates + uses: stefanzweifel/git-auto-commit-action@v6 + with: + commit_message: "chore(coverage): bump baselines after green main" + file_pattern: coverage-baselines.json + + # The scripts test suite — verifies the ratchet and assertion-lint analyzers + # themselves on every PR that touches them or their tests. + scripts-tests: + name: Scripts Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + - name: Install dependencies + env: + PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN: ${{ secrets.PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN }} + run: bun install --frozen-lockfile + - name: Run scripts test suite + run: bun test:scripts diff --git a/.github/workflows/eas-update.yml b/.github/workflows/eas-update.yml index 5ccf9ed910..4a9c9c584b 100644 --- a/.github/workflows/eas-update.yml +++ b/.github/workflows/eas-update.yml @@ -8,6 +8,9 @@ on: required: false type: string +permissions: + contents: read + jobs: update: name: Publish EAS Update diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 3faee06299..0000000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: Unit Tests - -on: - push: - branches: ["main", "development"] - paths: - - "package.json" - - "bun.lock" - - "packages/api/src/**" - - "packages/api/package.json" - - "packages/api/vitest.unit.config.ts" - - "apps/expo/package.json" - - "apps/expo/vitest.config.ts" - - "apps/expo/utils/**" - - "apps/expo/lib/utils/**" - - "apps/expo/features/**/utils/**" - - ".github/workflows/unit-tests.yml" - pull_request: - branches: ["**"] - paths: - - "package.json" - - "bun.lock" - - "packages/api/src/**" - - "packages/api/package.json" - - "packages/api/vitest.unit.config.ts" - - "apps/expo/package.json" - - "apps/expo/vitest.config.ts" - - "apps/expo/utils/**" - - "apps/expo/lib/utils/**" - - "apps/expo/features/**/utils/**" - - ".github/workflows/unit-tests.yml" - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - contents: read - pull-requests: write - -jobs: - api-unit-tests: - name: API Unit Tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6 - - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - - name: Install dependencies - env: - PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN: ${{ secrets.PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN }} - run: bun install --frozen-lockfile - - - name: Run API unit tests - run: bun run --cwd packages/api test:unit:coverage - - - name: Report API coverage - if: always() - uses: davelosert/vitest-coverage-report-action@v2 - with: - name: API Unit Tests Coverage - json-summary-path: ./coverage/unit/coverage-summary.json - json-final-path: ./coverage/unit/coverage-final.json - vite-config-path: ./vitest.unit.config.ts - working-directory: ./packages/api - - expo-unit-tests: - name: Expo Unit Tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6 - - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - - name: Install dependencies - env: - PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN: ${{ secrets.PACKRAT_NATIVEWIND_UI_GITHUB_TOKEN }} - run: bun install --frozen-lockfile - - - name: Run Expo unit tests - run: bun run --cwd apps/expo test:coverage - - - name: Report Expo coverage - if: always() - uses: davelosert/vitest-coverage-report-action@v2 - with: - name: Expo Unit Tests Coverage - json-summary-path: ./coverage/unit/coverage-summary.json - json-final-path: ./coverage/unit/coverage-final.json - vite-config-path: ./vitest.config.ts - working-directory: ./apps/expo diff --git a/.jscpd.json b/.jscpd.json new file mode 100644 index 0000000000..611a5051aa --- /dev/null +++ b/.jscpd.json @@ -0,0 +1,26 @@ +{ + "threshold": 7, + "minTokens": 50, + "absolute": false, + "gitignore": true, + "reporters": ["console"], + "format": ["typescript", "tsx"], + "ignore": [ + "**/node_modules/**", + "**/dist/**", + "**/build/**", + "**/.next/**", + "**/.expo/**", + "**/.wrangler/**", + "**/*.test.ts", + "**/*.test.tsx", + "**/*.spec.ts", + "**/*.spec.tsx", + "**/*.gen.ts", + "**/src/codegen/**", + "**/drizzle/**", + "**/coverage/**", + "**/ios/**", + "**/android/**" + ] +} diff --git a/CLAUDE.md b/CLAUDE.md index be81d88ee7..ea6e0fabaa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,9 +43,13 @@ bun format # Biome format --write bun check # Biome check (no auto-fix, CI mode) bun check-types # tsc --noEmit -# Testing -bun test:api:unit # API unit tests (Vitest + Cloudflare pool) -bun test:expo # Expo tests (Vitest) +# Testing — see docs/testing.md for the full policy +bun test:api:unit # API unit tests (Vitest, Node env, deps mocked) +bun test:expo # Expo pure-TS tests (Vitest) +bun test:mcp # MCP package tests +bun test:scripts # scripts/lint analyzer tests (ratchet + assertion lint) +bun check:coverage # Coverage ratchet — fails on regression vs coverage-baselines.json +bun lint:weak-assertions # Catches assertion-free tests, bare .toBeDefined / .toHaveBeenCalled, oversized snapshots # Dependencies bun install # Install all workspaces (takes 120s+, never cancel) @@ -56,6 +60,10 @@ bun fix:deps # manypkg auto-fix dependency issues bun bump # Bump monorepo version ``` +## Testing Policy (summary) + +PackRat enforces coverage at two layers: each workspace's `vitest.config.ts` declares per-metric thresholds (mostly 95%+; `packages/units` 100%, `packages/{analytics,overpass}` 80%), and a **coverage ratchet** (`bun check:coverage` against `coverage-baselines.json`) blocks any PR that lowers a workspace's coverage. An **assertion-strength lint** (`bun lint:weak-assertions`) flags coverage-theater patterns (assertion-free tests, bare `.toBeDefined()`, bare `.toHaveBeenCalled()`, oversized snapshots). `packages/api` integration tests still run (`api-tests.yml`) but are not coverage-counted — V8 instrumentation is unsupported under the Cloudflare Workers pool. Full policy and patterns: **`docs/testing.md`**. + ## Code Style Enforced by **Biome 2.0** via lefthook pre-commit hook: @@ -244,6 +252,24 @@ Defined in root `tsconfig.json`: - Migrations: Drizzle Kit (`drizzle-kit`) - Embeddings: pgvector with 1536 dimensions +### Migration discipline (read before touching `packages/api/drizzle/`) + +1. **Always generate via drizzle-kit.** Edit `packages/api/src/db/schema.ts` (or `packages/db/src/schema.ts` for the shared workspace), then run from the API package: + + ```bash + cd packages/api && bun run db:generate + ``` + + Drizzle-kit emits a random-name file like `0048_loud_squirrel_girl.sql`. That random name is fine — keep it. The naming convention here is "whatever drizzle-kit gives you." + +2. **Do not rename a generated migration file.** The `meta/_journal.json` `tag` field, the migration SQL filename, and the snapshot filename all encode the migration identity together. Renaming any one of them (even with corresponding journal edits) makes the migration look hand-authored and creates drift that future drizzle-kit operations can mis-handle. + +3. **Do not hand-edit `meta/_journal.json`, `meta/*_snapshot.json`, or the generated SQL.** If the generated migration is wrong, fix the schema, delete the bad migration + snapshot + journal entry, and regenerate. Do not patch around it. + +4. **Collapse additive changes into one migration when they ship together** — fewer snapshot files in the diff, easier to revert as a unit. Splitting only makes sense when migrations need to land in separate releases. + +5. **Verify after generating.** Run `bunx drizzle-kit check` from `packages/api/` — it validates the snapshot chain is internally consistent. Run before pushing. + ## EAS Build Profiles | Profile | Use | Distribution | diff --git a/README.md b/README.md index d076380f6f..256b317f8f 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ So pack your bags, grab your friends, and get ready for your next adventure with [![view - Documentation](https://img.shields.io/badge/view-Documentation-blue?style=for-the-badge)](/docs/ "Go to project documentation") -[![view - Testing Guide](https://img.shields.io/badge/view-Testing_Guide-green?style=for-the-badge)](/TESTING.md "Go to testing documentation") +[![view - Testing Guide](https://img.shields.io/badge/view-Testing_Guide-green?style=for-the-badge)](/docs/testing.md "Go to testing documentation") diff --git a/TESTING.md b/TESTING.md deleted file mode 100644 index 74b5939779..0000000000 --- a/TESTING.md +++ /dev/null @@ -1,453 +0,0 @@ -# Unit Testing Guide for PackRat - -This document outlines testing standards and patterns used in the PackRat codebase. - -## Overview - -PackRat uses **Vitest** as its primary testing framework across both API and Expo layers. This guide demonstrates the patterns established in our unit test suite. - ---- - -## Testing Infrastructure - -### API Layer (packages/api) - -**Configuration Files:** -- `vitest.config.ts` - Full integration tests with PostgreSQL + Cloudflare Workers -- `vitest.unit.config.ts` - Pure unit tests with mocked dependencies (recommended for most unit tests) - -**Commands:** -```bash -# From packages/api -bun test # Full integration tests (requires Docker) -bun test:unit # Unit tests only (no database) -bun test:unit:coverage # Unit tests with coverage report - -# From monorepo root -bun test:api:unit -``` - -**Coverage Configuration:** -- **Provider:** v8 -- **Reports:** text, lcov, html -- **Directory:** `packages/api/coverage/unit/` -- **Target:** 80%+ coverage for critical paths - -### Expo Layer (apps/expo) - -**Configuration File:** -- `vitest.config.ts` - Node environment for pure utility functions - -**Commands:** -```bash -# From apps/expo -bun test # Run utility tests -bun test:coverage # With coverage report - -# From monorepo root -bun test:expo -``` - -**Coverage Configuration:** -- **Provider:** v8 -- **Reports:** text, lcov, html -- **Directory:** `apps/expo/coverage/unit/` -- **Target:** 75%+ coverage for utility functions - -**Note:** Currently limited to pure utility functions. React Native hooks and components require additional setup (e.g., @testing-library/react-native). - ---- - -## Test Patterns - -### Pattern 1: Service Tests with Mocked Dependencies - -**Example:** `/packages/api/src/services/__tests__/catalogService.test.ts` - -```typescript -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { CatalogService } from '../catalogService'; -import * as embeddingService from '@packrat/api/services/embeddingService'; - -// Module-level mocks (hoisted) -vi.mock('@packrat/api/db', () => ({ - createDb: vi.fn(), - createDbClient: vi.fn(), -})); - -vi.mock('@packrat/api/services/embeddingService', () => ({ - generateEmbedding: vi.fn(), - generateManyEmbeddings: vi.fn(), -})); - -// Test suite -describe('CatalogService', () => { - let service: CatalogService; - - beforeEach(() => { - vi.clearAllMocks(); - service = new CatalogService(makeEnv(), false); - }); - - describe('vectorSearch', () => { - beforeEach(() => { - vi.mocked(embeddingService.generateEmbedding) - .mockResolvedValue(new Array(1536).fill(0.1)); - }); - - it('returns empty result for empty query string', async () => { - const result = await service.vectorSearch('', 10, 0); - - expect(result).toEqual({ - items: [], - total: 0, - limit: 10, - offset: 0, - nextOffset: 10, - }); - expect(embeddingService.generateEmbedding).not.toHaveBeenCalled(); - }); - }); -}); -``` - -**Key Points:** -- Use `vi.mock()` for module-level mocks (these are hoisted) -- Import mocked modules for type-safe access (e.g., `import * as embeddingService`) -- Use `vi.mocked()` for type-safe mock assertions -- Clear mocks in `beforeEach()` for test isolation - -### Pattern 2: API Service Tests with Fetch Mocking - -**Example:** `/packages/api/src/services/__tests__/weatherService.test.ts` - -```typescript -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { WeatherService } from '../weatherService'; - -describe('WeatherService', () => { - let service: WeatherService; - let fetchMock: ReturnType; - - beforeEach(() => { - vi.clearAllMocks(); - mockContext = makeMockContext(); - service = new WeatherService(mockContext); - - // Mock global fetch - fetchMock = vi.fn(); - global.fetch = fetchMock; - }); - - describe('getWeatherForLocation', () => { - it('returns formatted weather data for valid location', async () => { - const mockResponse = { - main: { temp: 72.5, humidity: 65 }, - weather: [{ main: 'Clear' }], - wind: { speed: 8.3 }, - }; - - fetchMock.mockResolvedValue({ - ok: true, - json: async () => mockResponse, - }); - - const result = await service.getWeatherForLocation('San Francisco'); - - expect(result.temperature).toBe(73); // Rounded - expect(result.conditions).toBe('Clear'); - }); - }); -}); -``` - -**Key Points:** -- Mock `global.fetch` in `beforeEach()` for fresh state -- Use `mockResolvedValue` for successful responses -- Test both success and error paths -- Verify API was called with correct parameters - -### Pattern 3: Pure Utility Function Tests - -**Example:** `/apps/expo/features/packs/utils/__tests__/convertToGrams.test.ts` - -```typescript -import { describe, expect, it } from 'vitest'; -import { convertToGrams } from '../convertToGrams'; - -describe('convertToGrams', () => { - describe('metric conversions', () => { - it('returns same value for grams', () => { - expect(convertToGrams(100, 'g')).toBe(100); - expect(convertToGrams(0, 'g')).toBe(0); - expect(convertToGrams(1, 'g')).toBe(1); - }); - - it('converts kilograms to grams correctly', () => { - expect(convertToGrams(1, 'kg')).toBe(1000); - expect(convertToGrams(2.5, 'kg')).toBe(2500); - }); - }); - - describe('edge cases', () => { - it('handles zero weight', () => { - expect(convertToGrams(0, 'kg')).toBe(0); - }); - - it('returns original value for unknown units', () => { - expect(convertToGrams(100, 'invalid')).toBe(100); - }); - }); -}); -``` - -**Key Points:** -- No mocking needed for pure functions -- Group related tests with nested `describe()` blocks -- Test edge cases (zero, negative, invalid input) -- Use `toBe()` for exact values, `toBeCloseTo()` for floating point -- Test real-world scenarios to ensure practical correctness - ---- - -## Best Practices - -### 1. Test Organization - -```typescript -describe('ServiceName', () => { - // Setup - let service: ServiceName; - - beforeEach(() => { - // Reset mocks and create fresh instances - }); - - describe('methodName', () => { - // Group related tests - - describe('when condition', () => { - // Nested context-specific tests - }); - }); -}); -``` - -### 2. Mock Isolation - -```typescript -beforeEach(() => { - vi.clearAllMocks(); // Reset all mock state - // Re-create service instances - // Set default mock return values -}); -``` - -### 3. Input Validation Tests - -Always test: -- ✅ Valid inputs (happy path) -- ✅ Invalid inputs (error paths) -- ✅ Edge cases (empty, null, undefined, zero, negative) -- ✅ Boundary conditions (min/max values) - -### 4. Floating Point Comparisons - -```typescript -// ❌ Don't use exact equality for floats -expect(convertToGrams(1, 'oz')).toBe(28.3495); - -// ✅ Use toBeCloseTo() with appropriate precision -expect(convertToGrams(1, 'oz')).toBeCloseTo(28.3495, 4); -``` - -### 5. Async Testing - -```typescript -// Test async functions -it('handles async operation', async () => { - const result = await service.fetchData(); - expect(result).toBeDefined(); -}); - -// Test error handling -it('throws on invalid input', async () => { - await expect(service.process(null)).rejects.toThrow('Invalid input'); -}); -``` - -### 6. Mock Configuration Patterns - -```typescript -// Default behavior for all tests in describe block -beforeEach(() => { - mockFunction.mockResolvedValue(defaultValue); -}); - -// Override for specific test -it('handles special case', async () => { - mockFunction.mockResolvedValueOnce(specialValue); - // ... -}); -``` - ---- - -## Coverage Guidelines - -### What to Test (Priority Order) - -1. **Critical Business Logic** - - Payment processing - - User authentication - - Data validation - - Core algorithms - -2. **Public APIs** - - All exported functions - - All route handlers - - Service methods - -3. **Edge Cases** - - Null/undefined handling - - Empty collections - - Boundary values - - Invalid inputs - -4. **Error Paths** - - Exception handling - - Validation errors - - Network failures - - Database errors - -### What NOT to Test - -- Third-party library internals -- Simple getters/setters with no logic -- Generated code -- Configuration files -- Type definitions - ---- - -## Running Tests in CI - -### API Tests (GitHub Actions) - -```yaml -- name: Run API Unit Tests - run: | - cd packages/api - bun test:unit --coverage -``` - -### Expo Tests (GitHub Actions) - -```yaml -- name: Run Expo Tests - run: | - cd apps/expo - bun test --coverage -``` - -### Coverage Reports - -Coverage reports are generated in: -- API: `packages/api/coverage/unit/` -- Expo: `apps/expo/coverage/unit/` - -Open `index.html` to view detailed coverage reports locally. - ---- - -## Troubleshooting - -### "Cannot access before initialization" Error - -**Problem:** Trying to use a variable declared after `vi.mock()` - -**Solution:** Import the mocked module after the mock declaration - -```typescript -// ❌ Won't work - hoisting issue -const mockFn = vi.fn(); -vi.mock('./module', () => ({ fn: mockFn })); - -// ✅ Works - import after mock -vi.mock('./module', () => ({ fn: vi.fn() })); -import * as module from './module'; -// Use vi.mocked(module.fn) in tests -``` - -### Mock Not Resetting Between Tests - -**Problem:** Mock state persists across tests - -**Solution:** Always call `vi.clearAllMocks()` in `beforeEach()` - -```typescript -beforeEach(() => { - vi.clearAllMocks(); // Resets all mock history and implementations -}); -``` - -### Floating Point Precision Errors - -**Problem:** `expect(0.1 + 0.2).toBe(0.3)` fails due to floating point arithmetic - -**Solution:** Use `toBeCloseTo()` with appropriate precision - -```typescript -expect(0.1 + 0.2).toBeCloseTo(0.3, 10); // 10 decimal places -``` - ---- - -## Resources - -- [Vitest Documentation](https://vitest.dev/) -- [Testing Best Practices](https://testingjavascript.com/) -- [Mocking in Vitest](https://vitest.dev/guide/mocking.html) -- [Coverage Configuration](https://vitest.dev/guide/coverage.html) - ---- - -## Test Statistics (Current) - -### API Layer -- **Test Files:** 8 -- **Tests:** 101 passing -- **Coverage Target:** 80%+ - -### Expo Layer -- **Test Files:** 8 -- **Tests:** 93 passing (excluding pre-existing failures) -- **Coverage Target:** 75%+ - -### Recent Additions -- ✅ `CatalogService` - Vector search, batch operations, input validation -- ✅ `WeatherService` - API calls, error handling, data transformations -- ✅ `convertToGrams` - Unit conversions, edge cases, real-world scenarios -- ✅ `convertFromGrams` - Reverse conversions, precision handling - ---- - -## Contributing - -When adding new features: - -1. **Write tests first** (TDD approach) or alongside implementation -2. **Aim for 80%+ coverage** for new code -3. **Test all code paths** including error cases -4. **Use existing patterns** from this guide -5. **Update this document** if introducing new patterns - -When fixing bugs: - -1. **Write a failing test** that reproduces the bug -2. **Fix the bug** until the test passes -3. **Verify** no regressions with full test suite - ---- - -*Last Updated: 2026-04-01* diff --git a/apps/admin/components/analytics/catalog-analytics.tsx b/apps/admin/components/analytics/catalog-analytics.tsx index 3ca0fa8461..f55c1599f0 100644 --- a/apps/admin/components/analytics/catalog-analytics.tsx +++ b/apps/admin/components/analytics/catalog-analytics.tsx @@ -1,5 +1,6 @@ 'use client'; +import { safeJsonStringify } from '@packrat/utils'; import { Badge } from '@packrat/web-ui/components/badge'; import { Button } from '@packrat/web-ui/components/button'; import { @@ -146,7 +147,7 @@ function EtlJobFailuresDialog({ jobId, totalInvalid }: { jobId: string; totalInv raw data
-                            {JSON.stringify(s.rawData, null, 2)}
+                            {safeJsonStringify(s.rawData, null, 2)}
                           
)} diff --git a/apps/admin/components/raw-object-dialog.tsx b/apps/admin/components/raw-object-dialog.tsx index 023aa8922c..da6dece999 100644 --- a/apps/admin/components/raw-object-dialog.tsx +++ b/apps/admin/components/raw-object-dialog.tsx @@ -1,5 +1,6 @@ 'use client'; +import { safeJsonStringify } from '@packrat/utils'; import { Button } from '@packrat/web-ui/components/button'; import { Dialog, @@ -36,7 +37,7 @@ export function RawObjectDialog({ label, data }: RawObjectDialogProps) {
-            {JSON.stringify(data, null, 2)}
+            {safeJsonStringify(data, null, 2)}
           
diff --git a/apps/admin/package.json b/apps/admin/package.json index a1e728920c..f6b2d1c46f 100644 --- a/apps/admin/package.json +++ b/apps/admin/package.json @@ -1,6 +1,6 @@ { "name": "packrat-admin-app", - "version": "2.0.26", + "version": "2.0.27", "private": true, "scripts": { "build": "next build", @@ -17,6 +17,7 @@ "@packrat/app": "workspace:*", "@packrat/guards": "workspace:*", "@packrat/schemas": "workspace:*", + "@packrat/utils": "workspace:*", "@packrat/web-ui": "workspace:*", "@radix-ui/react-alert-dialog": "catalog:", "@radix-ui/react-avatar": "catalog:", diff --git a/apps/expo/app.config.ts b/apps/expo/app.config.ts index 478019087e..daacf21f60 100644 --- a/apps/expo/app.config.ts +++ b/apps/expo/app.config.ts @@ -37,7 +37,7 @@ export default (): ExpoConfig => { name: getAppName(), slug: 'packrat', - version: '2.0.26', + version: '2.0.27', scheme: 'packrat', web: { bundler: 'metro', diff --git a/apps/expo/atoms/atomWithAsyncStorage.ts b/apps/expo/atoms/atomWithAsyncStorage.ts index 49060418e7..aa9be68b42 100644 --- a/apps/expo/atoms/atomWithAsyncStorage.ts +++ b/apps/expo/atoms/atomWithAsyncStorage.ts @@ -1,4 +1,5 @@ import { isFunction } from '@packrat/guards'; +import { safeJsonParse, safeJsonStringify } from '@packrat/utils'; import AsyncStorage from '@react-native-async-storage/async-storage'; import { atom } from 'jotai'; @@ -14,7 +15,7 @@ export const atomWithAsyncStorage = ({ baseAtom.onMount = (setValue) => { (async () => { const item = await AsyncStorage.getItem(key); - setValue(item ? JSON.parse(item) : initialValue); + setValue(item ? safeJsonParse(item) : initialValue); })(); }; @@ -23,7 +24,7 @@ export const atomWithAsyncStorage = ({ (get, set, update) => { const nextValue = isFunction(update) ? update(get(baseAtom)) : update; set(baseAtom, nextValue); - AsyncStorage.setItem(key, JSON.stringify(nextValue)); + AsyncStorage.setItem(key, safeJsonStringify(nextValue)); }, ); diff --git a/apps/expo/atoms/atomWithKvStorage.ts b/apps/expo/atoms/atomWithKvStorage.ts index 886827e7d7..7ffaafadb1 100644 --- a/apps/expo/atoms/atomWithKvStorage.ts +++ b/apps/expo/atoms/atomWithKvStorage.ts @@ -1,4 +1,5 @@ import { isFunction } from '@packrat/guards'; +import { safeJsonParse, safeJsonStringify } from '@packrat/utils'; import Storage from 'expo-sqlite/kv-store'; import { atom } from 'jotai'; @@ -8,7 +9,7 @@ export const atomWithKvStorage = ({ key, initialValue }: { key: string; initi baseAtom.onMount = (setValue) => { (async () => { const item = await Storage.getItem(key); - setValue(item ? JSON.parse(item) : initialValue); + setValue(item ? safeJsonParse(item) : initialValue); })(); }; @@ -19,7 +20,7 @@ export const atomWithKvStorage = ({ key, initialValue }: { key: string; initi set(baseAtom, nextValue); - Storage.setItem(key, JSON.stringify(nextValue)); + Storage.setItem(key, safeJsonStringify(nextValue)); }, ); diff --git a/apps/expo/atoms/atomWithSecureStorage.ts b/apps/expo/atoms/atomWithSecureStorage.ts index da3c3ddcfe..043e9b24a4 100644 --- a/apps/expo/atoms/atomWithSecureStorage.ts +++ b/apps/expo/atoms/atomWithSecureStorage.ts @@ -1,4 +1,5 @@ import { isFunction } from '@packrat/guards'; +import { safeJsonParse, safeJsonStringify } from '@packrat/utils'; import * as SecureStore from 'expo-app/lib/secureStore'; import { atom } from 'jotai'; @@ -14,7 +15,7 @@ export const atomWithSecureStorage = ({ baseAtom.onMount = (setValue) => { (async () => { const item = await SecureStore.getItemAsync(key); - setValue(item ? JSON.parse(item) : initialValue); + setValue(item ? safeJsonParse(item) : initialValue); })(); }; @@ -23,7 +24,7 @@ export const atomWithSecureStorage = ({ (get, set, update) => { const nextValue = isFunction(update) ? update(get(baseAtom)) : update; set(baseAtom, nextValue); - SecureStore.setItemAsync(key, JSON.stringify(nextValue)); + SecureStore.setItemAsync(key, safeJsonStringify(nextValue)); }, ); diff --git a/apps/expo/atoms/atomWithSecureStorage.web.ts b/apps/expo/atoms/atomWithSecureStorage.web.ts index f54b9176f3..d851a14ed3 100644 --- a/apps/expo/atoms/atomWithSecureStorage.web.ts +++ b/apps/expo/atoms/atomWithSecureStorage.web.ts @@ -1,4 +1,5 @@ import { isFunction } from '@packrat/guards'; +import { safeJsonParse, safeJsonStringify } from '@packrat/utils'; import { atom } from 'jotai'; /** @@ -19,7 +20,7 @@ export const atomWithSecureStorage = ({ baseAtom.onMount = (setValue) => { try { const item = localStorage.getItem(key); - setValue(item !== null ? JSON.parse(item) : initialValue); + setValue(item !== null ? safeJsonParse(item, { strict: true }) : initialValue); } catch { setValue(initialValue); } @@ -31,7 +32,7 @@ export const atomWithSecureStorage = ({ const nextValue = isFunction(update) ? (update as (prev: T) => T)(get(baseAtom)) : update; set(baseAtom, nextValue); try { - localStorage.setItem(key, JSON.stringify(nextValue)); + localStorage.setItem(key, safeJsonStringify(nextValue)); } catch { // Ignore storage errors } diff --git a/apps/expo/features/ai/atoms/chatStorageAtoms.ts b/apps/expo/features/ai/atoms/chatStorageAtoms.ts index 7c9a90a856..a6bbde5692 100644 --- a/apps/expo/features/ai/atoms/chatStorageAtoms.ts +++ b/apps/expo/features/ai/atoms/chatStorageAtoms.ts @@ -1,5 +1,6 @@ import type { UIMessage } from '@ai-sdk/react'; import { isObject, isString } from '@packrat/guards'; +import { safeJsonParse, safeJsonStringify } from '@packrat/utils'; import AsyncStorage from '@react-native-async-storage/async-storage'; export type ChatContext = { @@ -64,7 +65,7 @@ export async function loadChatMessages(context: ChatContext): Promise { try { const key = getChatStorageKey(context); - await AsyncStorage.setItem(key, JSON.stringify(messages)); + await AsyncStorage.setItem(key, safeJsonStringify(messages)); } catch (error) { console.error('Failed to save chat messages:', error); } diff --git a/apps/expo/features/ai/components/ToolInvocationRenderer.tsx b/apps/expo/features/ai/components/ToolInvocationRenderer.tsx index e3b8630816..e3135e0363 100644 --- a/apps/expo/features/ai/components/ToolInvocationRenderer.tsx +++ b/apps/expo/features/ai/components/ToolInvocationRenderer.tsx @@ -1,4 +1,5 @@ import { isString } from '@packrat/guards'; +import { safeJsonParse } from '@packrat/utils'; import type { ToolUIPart } from 'ai'; import type { CatalogItemsTool } from './CatalogItemsGenerativeUI'; import { CatalogItemsGenerativeUI } from './CatalogItemsGenerativeUI'; @@ -22,7 +23,7 @@ export function ToolInvocationRenderer({ toolInvocation }: ToolInvocationRendere // Normalize it here once so all GenUI components receive a plain object. const normalizedInvocation = toolInvocation.state === 'output-available' && isString(toolInvocation.output) - ? { ...toolInvocation, output: JSON.parse(toolInvocation.output) } + ? { ...toolInvocation, output: safeJsonParse(toolInvocation.output) } : toolInvocation; // safe-cast: each case branch narrows toolInvocation.type to the discriminant literal; the diff --git a/apps/expo/features/ai/lib/CustomChatTransport.ts b/apps/expo/features/ai/lib/CustomChatTransport.ts index de0392d0c1..25ed9af7db 100644 --- a/apps/expo/features/ai/lib/CustomChatTransport.ts +++ b/apps/expo/features/ai/lib/CustomChatTransport.ts @@ -1,5 +1,6 @@ import type { UIMessage } from '@ai-sdk/react'; import { isString } from '@packrat/guards'; +import { safeJsonStringify } from '@packrat/utils'; import { type ChatRequestOptions, type ChatTransport, @@ -64,7 +65,7 @@ export class CustomChatTransport implements ChatTransport { if (error instanceof Error) { return error.message; } - return JSON.stringify(error); + return safeJsonStringify(error); }, }); } diff --git a/apps/expo/features/ai/lib/appleModelWrapper.ts b/apps/expo/features/ai/lib/appleModelWrapper.ts index 842cb1ddfd..b43c81b042 100644 --- a/apps/expo/features/ai/lib/appleModelWrapper.ts +++ b/apps/expo/features/ai/lib/appleModelWrapper.ts @@ -24,6 +24,7 @@ */ import { isString } from '@packrat/guards'; +import { safeJsonStringify } from '@packrat/utils'; // biome-ignore lint/suspicious/noExplicitAny: Apple model type is unknown at this layer type AnyModel = any; @@ -96,7 +97,7 @@ export class AppleModelWrapper { toolCallId, toolName: part.toolName, // Apple may return input as an object; the spec requires a JSON string - input: isString(part.input) ? part.input : JSON.stringify(part.input), + input: isString(part.input) ? part.input : safeJsonStringify(part.input), providerExecuted: true, }); } else if (part.type === 'tool-result') { diff --git a/apps/expo/features/ai/lib/llamaToolsWrapper.ts b/apps/expo/features/ai/lib/llamaToolsWrapper.ts index 28f9a8ca4a..efbc8de427 100644 --- a/apps/expo/features/ai/lib/llamaToolsWrapper.ts +++ b/apps/expo/features/ai/lib/llamaToolsWrapper.ts @@ -15,6 +15,7 @@ import { generateId } from '@ai-sdk/provider-utils'; import { isString } from '@packrat/guards'; +import { safeJsonStringify } from '@packrat/utils'; import type { LlamaLanguageModel } from '@react-native-ai/llama'; // Minimal structural slice of LanguageModelV2CallOptions we need @@ -63,13 +64,13 @@ function toolResultOutputToString(output: any): string { return String(output.value ?? ''); case 'json': case 'error-json': - return JSON.stringify(output.value); + return safeJsonStringify(output.value); case 'content': return (output.value ?? []) .map((p: { type: string; text?: string }) => (p.type === 'text' ? (p.text ?? '') : '')) .join(''); default: - return JSON.stringify(output); + return safeJsonStringify(output); } } @@ -114,7 +115,7 @@ function convertPromptToLlamaMessages(prompt: Prompt): LlamaMessage[] { id: p.toolCallId, function: { name: p.toolName, - arguments: isString(p.input) ? p.input : JSON.stringify(p.input), + arguments: isString(p.input) ? p.input : safeJsonStringify(p.input), }, })), }); diff --git a/apps/expo/features/auth/hooks/useAuthActions.ts b/apps/expo/features/auth/hooks/useAuthActions.ts index 6c95b74d2f..c4a1dd8653 100644 --- a/apps/expo/features/auth/hooks/useAuthActions.ts +++ b/apps/expo/features/auth/hooks/useAuthActions.ts @@ -1,4 +1,5 @@ import { asBoolean, asString } from '@packrat/guards'; +import { safeJsonParse } from '@packrat/utils'; import AsyncStorage from '@react-native-async-storage/async-storage'; import { GoogleSignin, @@ -27,7 +28,7 @@ import { function redirect(route: string) { try { - const parsedRoute: Href = JSON.parse(route); + const parsedRoute = safeJsonParse(route, { strict: true }); return router.dismissTo(parsedRoute); } catch { router.dismissTo(route as Href); // safe-cast: Href = string | HrefObject; string literal branch failed JSON.parse so plain string is the correct type here diff --git a/apps/expo/features/catalog/lib/normalizeDescription.ts b/apps/expo/features/catalog/lib/normalizeDescription.ts index 9a6d68d035..8e939e3a51 100644 --- a/apps/expo/features/catalog/lib/normalizeDescription.ts +++ b/apps/expo/features/catalog/lib/normalizeDescription.ts @@ -1,3 +1,5 @@ +import { safeJsonParse } from '@packrat/utils'; + const DETAILS_ARRAY_RE = /^Details:\s*(\[[\s\S]*\])$/; export function normalizeDescription(description: string | null | undefined): string | null { @@ -5,7 +7,7 @@ export function normalizeDescription(description: string | null | undefined): st const match = description.match(DETAILS_ARRAY_RE); if (match?.[1]) { try { - const items = JSON.parse(match[1]) as string[]; + const items = safeJsonParse(match[1], { strict: true }) as string[]; return items.join('. '); } catch { // fall through diff --git a/apps/expo/features/catalog/types.ts b/apps/expo/features/catalog/types.ts index 8b2e07e5a8..583f633c28 100644 --- a/apps/expo/features/catalog/types.ts +++ b/apps/expo/features/catalog/types.ts @@ -49,4 +49,5 @@ export interface CatalogItemInput { }>; } -export type CatalogItemWithPackItemFields = CatalogItem & Partial; +export type CatalogItemWithPackItemFields = CatalogItem & + Partial>; diff --git a/apps/expo/features/pack-templates/components/FeaturedPacksSection.tsx b/apps/expo/features/pack-templates/components/FeaturedPacksSection.tsx index 71bce69524..420bb1d06c 100644 --- a/apps/expo/features/pack-templates/components/FeaturedPacksSection.tsx +++ b/apps/expo/features/pack-templates/components/FeaturedPacksSection.tsx @@ -1,8 +1,8 @@ +import { isArray } from '@packrat/guards'; import { Text } from '@packrat/ui/nativewindui'; import { WeightBadge } from 'expo-app/components/initial/WeightBadge'; import { useTranslation } from 'expo-app/lib/hooks/useTranslation'; import { useRouter } from 'expo-router'; -import { isArray } from 'radash'; import { useMemo } from 'react'; import { Image, Platform, Pressable, ScrollView, View } from 'react-native'; import { usePackTemplates } from '../hooks'; diff --git a/apps/expo/features/trail-conditions/hooks/useTrailConditionReports.ts b/apps/expo/features/trail-conditions/hooks/useTrailConditionReports.ts index 5ac2853fc0..3e80a03b66 100644 --- a/apps/expo/features/trail-conditions/hooks/useTrailConditionReports.ts +++ b/apps/expo/features/trail-conditions/hooks/useTrailConditionReports.ts @@ -1,4 +1,5 @@ import { useSelector } from '@legendapp/state/react'; +import { safeJsonParse, safeJsonStringify } from '@packrat/utils'; import AsyncStorage from '@react-native-async-storage/async-storage'; import { useQuery } from '@tanstack/react-query'; import { userStore } from 'expo-app/features/auth/store/user'; @@ -26,7 +27,7 @@ async function writeCachedReports({ try { await AsyncStorage.setItem( cacheKey({ userId: opts.userId, trailName: opts.trailName }), - JSON.stringify(reports), + safeJsonStringify(reports), ); } catch { // Best-effort — swallow write errors silently @@ -43,7 +44,7 @@ async function readCachedReports(opts: { cacheKey({ userId: opts.userId, trailName: opts.trailName }), ); // safe-cast: JSON.parse returns unknown; data was written as TrailConditionReport[] earlier - if (raw) return JSON.parse(raw) as TrailConditionReport[]; + if (raw) return safeJsonParse(raw, { strict: true }) as TrailConditionReport[]; } catch { // Corrupt or missing cache — ignore } diff --git a/apps/expo/features/weather/screens/LocationSearchScreen.tsx b/apps/expo/features/weather/screens/LocationSearchScreen.tsx index f0365f95a7..33ece35d33 100644 --- a/apps/expo/features/weather/screens/LocationSearchScreen.tsx +++ b/apps/expo/features/weather/screens/LocationSearchScreen.tsx @@ -1,4 +1,5 @@ import { Text } from '@packrat/ui/nativewindui'; +import { safeJsonParse, safeJsonStringify } from '@packrat/utils'; import AsyncStorage from '@react-native-async-storage/async-storage'; import { Icon } from 'expo-app/components/Icon'; import { SearchInput } from 'expo-app/components/SearchInput'; @@ -54,7 +55,7 @@ export default function LocationSearchScreen() { try { const storedSearches = await AsyncStorage.getItem(RECENT_SEARCHES_KEY); if (storedSearches) { - setRecentSearches(JSON.parse(storedSearches)); + setRecentSearches(safeJsonParse(storedSearches, { strict: true })); } } catch (err) { console.error('Error loading recent searches:', err); @@ -76,14 +77,14 @@ export default function LocationSearchScreen() { ].slice(0, 5); // Keep only 5 most recent setRecentSearches(updatedSearches); - await AsyncStorage.setItem(RECENT_SEARCHES_KEY, JSON.stringify(updatedSearches)); + await AsyncStorage.setItem(RECENT_SEARCHES_KEY, safeJsonStringify(updatedSearches)); return; } // Add new search term to the beginning and limit to 5 const updatedSearches = [searchTerm, ...recentSearches].slice(0, 5); setRecentSearches(updatedSearches); - await AsyncStorage.setItem(RECENT_SEARCHES_KEY, JSON.stringify(updatedSearches)); + await AsyncStorage.setItem(RECENT_SEARCHES_KEY, safeJsonStringify(updatedSearches)); } catch (err) { console.error('Error saving recent search:', err); } diff --git a/apps/expo/lib/api/packrat.ts b/apps/expo/lib/api/packrat.ts index 09d318ce2c..9f576e3ce5 100644 --- a/apps/expo/lib/api/packrat.ts +++ b/apps/expo/lib/api/packrat.ts @@ -1,5 +1,6 @@ import { createApiClient } from '@packrat/api-client'; import { fromZod } from '@packrat/guards'; +import { safeJsonParse } from '@packrat/utils'; import { store } from 'expo-app/atoms/store'; import { needsReauthAtom } from 'expo-app/features/auth/atoms/authAtoms'; import { getApiBaseUrl } from 'expo-app/lib/api/getBaseUrl'; @@ -17,7 +18,7 @@ const CookieStoreSchema = z.record(z.object({ value: z.string() })); // HTTPS servers (remote dev/prod) prefix the cookie name with __Secure-; HTTP (local) does not. function parseSessionToken(cookieJson: string | null): string | null { if (!cookieJson) return null; - const cookies = fromZod(CookieStoreSchema)(JSON.parse(cookieJson)); + const cookies = fromZod(CookieStoreSchema)(safeJsonParse(cookieJson)); if (!cookies) return null; return ( cookies['better-auth.session_token']?.value ?? diff --git a/apps/expo/package.json b/apps/expo/package.json index bcf5e879e7..3804baa6d8 100644 --- a/apps/expo/package.json +++ b/apps/expo/package.json @@ -1,6 +1,6 @@ { "name": "packrat-expo-app", - "version": "2.0.26", + "version": "2.0.27", "private": true, "main": "expo-router/entry", "scripts": { @@ -65,6 +65,7 @@ "@packrat/types": "workspace:*", "@packrat/ui": "workspace:*", "@packrat/units": "workspace:*", + "@packrat/utils": "workspace:*", "@react-native-ai/apple": "~0.10.0", "@react-native-ai/llama": "~0.10.0", "@react-native-async-storage/async-storage": "2.2.0", @@ -135,7 +136,6 @@ "llama.rn": "0.10.1", "nanoid": "^5.1.9", "nativewind": "^4.2.3", - "radash": "catalog:", "react": "catalog:", "react-dom": "catalog:", "react-i18next": "^17.0.4", diff --git a/apps/expo/polyfills.ts b/apps/expo/polyfills.ts index 0932b107d9..b66a0b968a 100644 --- a/apps/expo/polyfills.ts +++ b/apps/expo/polyfills.ts @@ -1,4 +1,5 @@ import 'react-native-get-random-values'; +import { isString } from '@packrat/guards'; import structuredClone from '@ungap/structured-clone'; import { BackHandler, Platform } from 'react-native'; @@ -17,7 +18,7 @@ if (Platform.OS === 'web') { if (__DEV__) { const _origError = console.error.bind(console); console.error = (...args: unknown[]) => { - if (typeof args[0] === 'string' && args[0].includes('Unexpected text node')) return; + if (isString(args[0]) && args[0].includes('Unexpected text node')) return; _origError(...args); }; } diff --git a/apps/expo/utils/storage.ts b/apps/expo/utils/storage.ts index abbab9a7da..bd94702fe8 100644 --- a/apps/expo/utils/storage.ts +++ b/apps/expo/utils/storage.ts @@ -2,16 +2,9 @@ import AsyncStorage from '@react-native-async-storage/async-storage'; import type { WeatherLocation } from 'expo-app/features/weather/types'; import { createJSONStorage } from 'jotai/utils'; -// Create a storage adapter for Jotai that uses AsyncStorage -export const asyncStorage = createJSONStorage(() => ({ - getItem: async (key: string) => { - const value = await AsyncStorage.getItem(key); - return value ? JSON.parse(value) : null; - }, - setItem: async (key: string, value: unknown) => { - await AsyncStorage.setItem(key, JSON.stringify(value)); - }, - removeItem: async (key: string) => { - await AsyncStorage.removeItem(key); - }, -})); +// Jotai storage adapter backed by AsyncStorage. `createJSONStorage` owns the +// JSON (de)serialization, so the backing storage must be a *string* storage — +// AsyncStorage already is one. (The previous hand-rolled adapter parsed inside +// the string-storage layer, which only type-checked because JSON.parse returns +// `any`; it was the wrong shape for createJSONStorage.) +export const asyncStorage = createJSONStorage(() => AsyncStorage); diff --git a/apps/guides/app/dev/generate/page.tsx b/apps/guides/app/dev/generate/page.tsx index 120915e590..ebca110cc0 100644 --- a/apps/guides/app/dev/generate/page.tsx +++ b/apps/guides/app/dev/generate/page.tsx @@ -2,6 +2,7 @@ import { guideEnv } from '@packrat/env/next'; import { assertEnum } from '@packrat/guards'; +import { safeJsonStringify } from '@packrat/utils'; import { Badge } from '@packrat/web-ui/components/badge'; import { Button } from '@packrat/web-ui/components/button'; import { @@ -145,7 +146,7 @@ export default function GeneratePage() { headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({ + body: safeJsonStringify({ title, description, categories: selectedCategories, @@ -198,7 +199,7 @@ export default function GeneratePage() { headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({ + body: safeJsonStringify({ count: batchCount, categories: batchCategories.length > 0 ? batchCategories : undefined, }), diff --git a/apps/guides/package.json b/apps/guides/package.json index a4497178e8..8ca6cc3f82 100644 --- a/apps/guides/package.json +++ b/apps/guides/package.json @@ -1,6 +1,6 @@ { "name": "packrat-guides-app", - "version": "2.0.26", + "version": "2.0.27", "private": true, "scripts": { "build": "bun run build-content && bun run generate-og-images && next build", @@ -31,6 +31,7 @@ "@packrat/env": "workspace:*", "@packrat/guards": "workspace:*", "@packrat/schemas": "workspace:*", + "@packrat/utils": "workspace:*", "@packrat/web-ui": "workspace:*", "@radix-ui/react-accordion": "catalog:", "@radix-ui/react-alert-dialog": "catalog:", diff --git a/apps/guides/scripts/build-content.ts b/apps/guides/scripts/build-content.ts index ef2e267c6f..5c6c27d65d 100644 --- a/apps/guides/scripts/build-content.ts +++ b/apps/guides/scripts/build-content.ts @@ -2,6 +2,7 @@ // It processes your MDX files and outputs JSON that can be imported // in your static site +import { safeJsonStringify } from '@packrat/utils'; import fs from 'fs'; import matter from 'gray-matter'; import type { Post } from 'guides-app/lib/types'; @@ -61,11 +62,11 @@ async function buildContent() { const contentFile = `// This file is auto-generated. Do not edit manually. import type { Post } from './types'; -export const posts: Post[] = ${JSON.stringify(posts, null, 2)}; +export const posts: Post[] = ${safeJsonStringify(posts, null, 2)}; -export const postContent: Record = ${JSON.stringify(postContent, null, 2)}; +export const postContent: Record = ${safeJsonStringify(postContent, null, 2)}; -export const categories: string[] = ${JSON.stringify(categories, null, 2)}; +export const categories: string[] = ${safeJsonStringify(categories, null, 2)}; `; fs.writeFileSync(outputFile, contentFile); diff --git a/apps/guides/scripts/generate-content.ts b/apps/guides/scripts/generate-content.ts index f43a57b881..72590d9de7 100644 --- a/apps/guides/scripts/generate-content.ts +++ b/apps/guides/scripts/generate-content.ts @@ -1,5 +1,6 @@ import { openai } from '@ai-sdk/openai'; import { arrayIncludes, objectEntries } from '@packrat/guards'; +import { safeJsonParse } from '@packrat/utils'; import { generateText } from 'ai'; import chalk from 'chalk'; import { format } from 'date-fns'; @@ -267,7 +268,9 @@ async function generateTopicIdeas({ try { // Extract JSON from the text response which might contain markdown code blocks const jsonText = extractJsonFromText(text); - const ideas = JSON.parse(jsonText); + const ideas = safeJsonParse< + Array<{ title: string; description: string; categories: string[]; difficulty: string }> + >(jsonText, { strict: true }); // Transform the data to match our internal format return ideas.map( diff --git a/apps/landing/package.json b/apps/landing/package.json index 11d718de32..4de0f59e23 100644 --- a/apps/landing/package.json +++ b/apps/landing/package.json @@ -1,6 +1,6 @@ { "name": "packrat-landing-app", - "version": "2.0.26", + "version": "2.0.27", "private": true, "scripts": { "build": "bun run generate-og-images && next build", diff --git a/apps/trails/lib/auth.ts b/apps/trails/lib/auth.ts index 3d95a64e1b..1cb8889d00 100644 --- a/apps/trails/lib/auth.ts +++ b/apps/trails/lib/auth.ts @@ -4,6 +4,7 @@ import { safeLocalStorage } from '@packrat/app/browser'; import { fromZod, isString } from '@packrat/guards'; +import { safeJsonParse, safeJsonStringify } from '@packrat/utils'; import z from 'zod'; const ACCESS_KEY = 'access_token'; @@ -12,7 +13,7 @@ const REFRESH_KEY = 'refresh_token'; function parseToken(raw: string | null): string | null { if (!raw) return null; try { - const parsed = JSON.parse(raw); + const parsed = safeJsonParse(raw, { strict: true }); return isString(parsed) ? parsed : null; } catch { // Not JSON-encoded — return as-is (raw JWT) @@ -54,13 +55,13 @@ export const UserInfoSchema = z.object({ export type UserInfo = z.infer; export function setUser(user: UserInfo): void { - safeLocalStorage.setItem({ key: 'user', value: JSON.stringify(user) }); + safeLocalStorage.setItem({ key: 'user', value: safeJsonStringify(user) }); } export function getUser(): UserInfo | null { try { const raw = safeLocalStorage.getItem('user'); - return raw ? (fromZod(UserInfoSchema)(JSON.parse(raw)) ?? null) : null; + return raw ? (fromZod(UserInfoSchema)(safeJsonParse(raw, { strict: true })) ?? null) : null; } catch { return null; } diff --git a/apps/trails/package.json b/apps/trails/package.json index 39a5fb7bb1..b84b4dd185 100644 --- a/apps/trails/package.json +++ b/apps/trails/package.json @@ -1,6 +1,6 @@ { "name": "packrat-trails-app", - "version": "2.0.26", + "version": "2.0.27", "private": true, "scripts": { "build": "bun run generate-og-images && next build", @@ -19,6 +19,7 @@ "@packrat/app": "workspace:*", "@packrat/guards": "workspace:*", "@packrat/overpass": "workspace:*", + "@packrat/utils": "workspace:*", "@packrat/web-ui": "workspace:*", "@radix-ui/react-dialog": "catalog:", "@radix-ui/react-label": "catalog:", diff --git a/apps/web/app/auth/page.tsx b/apps/web/app/auth/page.tsx index 6c989613c1..b41f613cbb 100644 --- a/apps/web/app/auth/page.tsx +++ b/apps/web/app/auth/page.tsx @@ -1,5 +1,6 @@ 'use client'; import { webEnv } from '@packrat/env/web'; +import { safeJsonStringify } from '@packrat/utils'; import { useMutation } from '@tanstack/react-query'; import { useRouter } from 'next/navigation'; import type React from 'react'; @@ -14,7 +15,7 @@ function useLoginMutation() { const res = await fetch(`${API_BASE}/api/auth/sign-in/email`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), + body: safeJsonStringify(body), }); if (!res.ok) throw new Error('Login failed'); return res.json() as Promise<{ token?: string; user?: unknown }>; @@ -34,7 +35,7 @@ function useRegisterMutation() { const res = await fetch(`${API_BASE}/api/auth/sign-up/email`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ email: body.email, password: body.password, name }), + body: safeJsonStringify({ email: body.email, password: body.password, name }), }); if (!res.ok) throw new Error('Registration failed'); return res.json(); diff --git a/apps/web/components/screens/catalog-screen.tsx b/apps/web/components/screens/catalog-screen.tsx index fc3624e485..916d9115ef 100644 --- a/apps/web/components/screens/catalog-screen.tsx +++ b/apps/web/components/screens/catalog-screen.tsx @@ -55,7 +55,9 @@ export function CatalogScreen() { // Client-side weight class filtering (since the API doesn't support it) const filteredItems = React.useMemo(() => { if (selectedWeightClass === 'all') return allItems; - return allItems.filter((item) => weightClass(item.weight) === selectedWeightClass); + return allItems.filter( + (item) => item.weight !== null && weightClass(item.weight) === selectedWeightClass, + ); }, [allItems, selectedWeightClass]); const activeFilterCount = (selectedCategory ? 1 : 0) + (selectedWeightClass !== 'all' ? 1 : 0); @@ -267,13 +269,15 @@ function CatalogCard({ onAdd: () => void; onSave: () => void; }) { - const wc = weightClass(item.weight); + const wc = item.weight === null ? null : weightClass(item.weight); const wcStyles = { ultralight: 'bg-[#30d158]/15 text-[#30d158]', lightweight: 'bg-[#ff9f0a]/15 text-[#ff9f0a]', standard: 'bg-muted text-muted-foreground', + unknown: 'bg-muted text-muted-foreground', }; - const wcLabels = { ultralight: 'UL', lightweight: 'LW', standard: 'STD' }; + const wcLabels = { ultralight: 'UL', lightweight: 'LW', standard: 'STD', unknown: 'N/A' }; + const weightClassKey = wc ?? 'unknown'; return (
@@ -290,9 +294,12 @@ function CatalogCard({ )} - {wcLabels[wc]} + {wcLabels[weightClassKey]} @@ -309,7 +316,9 @@ function CatalogCard({ {/* Weight + Rating + Price */}
- {fw(item.weight)} + + {item.weight === null ? 'Unknown' : fw(item.weight)} + {item.ratingValue && ( @@ -400,7 +409,9 @@ function GearDetailModal({

Weight

-

{fw(item.weight)}

+

+ {item.weight === null ? 'Unknown' : fw(item.weight)} +

{item.price && (
diff --git a/apps/web/components/screens/packs-screen.tsx b/apps/web/components/screens/packs-screen.tsx index 8237ada2da..c0e4e8192b 100644 --- a/apps/web/components/screens/packs-screen.tsx +++ b/apps/web/components/screens/packs-screen.tsx @@ -632,7 +632,7 @@ function AddItemSlideOver({ packId, body: { name: item.name, - weight: item.weight, + weight: item.weight ?? 0, weightUnit: 'g', catalogItemId: item.id, }, @@ -648,7 +648,9 @@ function AddItemSlideOver({

{item.seller}

-

{fw(item.weight)}

+

+ {item.weight === null ? 'Unknown' : fw(item.weight)} +

{item.price &&

${item.price}

}
diff --git a/apps/web/lib/types.ts b/apps/web/lib/types.ts index 5cc46ffdaa..42114e95fd 100644 --- a/apps/web/lib/types.ts +++ b/apps/web/lib/types.ts @@ -117,8 +117,8 @@ export type CatalogItem = { name: string; productUrl: string; sku: string; - weight: number; - weightUnit: string; + weight: number | null; + weightUnit: string | null; description: string | null; categories: string[] | null; images: string[] | null; diff --git a/apps/web/package.json b/apps/web/package.json index 631b065306..d2c6bc91ed 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -14,6 +14,7 @@ "@packrat/app": "workspace:*", "@packrat/env": "workspace:*", "@packrat/guards": "workspace:*", + "@packrat/utils": "workspace:*", "@packrat/web-ui": "workspace:*", "@tanstack/react-query": "catalog:", "@tanstack/react-query-devtools": "catalog:", diff --git a/ast-grep-rules/PARITY.md b/ast-grep-rules/PARITY.md new file mode 100644 index 0000000000..6e61ca0f72 --- /dev/null +++ b/ast-grep-rules/PARITY.md @@ -0,0 +1,56 @@ +# ast-grep parity with retired regex lint scripts + +Maps each behavior of the old `scripts/lint/no-raw-typeof.ts` and +`scripts/lint/no-raw-regex.ts` to the ast-grep rules that replace them. The old +`.ts` scripts are left in place until the orchestrator confirms this proof. + +## no-raw-typeof.ts → `no-raw-typeof.yml` (+ `no-raw-typeof-tsx.yml`) + +| Old-script behavior | ast-grep coverage | +|---|---| +| Flags `typeof X === ` / `!==` | `rule.any` of `typeof $X === $T` / `typeof $X !== $T` | +| Primitive set: string,number,boolean,object,function,undefined,symbol,bigint | `constraints.T.regex` exactly that set, quote-anchored | +| Exempt globals window/document/globalThis/Bun/navigator/process | `constraints.X.not.regex` for those identifiers | +| Scanned only `apps/` + `packages/`, skipped node_modules/dist/build/.wrangler | `ignores` for `scripts/**`, `.github/**`, mocks; ast-grep already skips ignored/build dirs via repo .gitignore | +| Exempt `packages/guards/**` | `ignores: **/packages/guards/**` (plus `packages/utils/**`) | +| Exempt `*.test`/`*.spec` files | `ignores` for `*.test.{ts,tsx}` / `*.spec.{ts,tsx}` | +| Only matched `.ts/.tsx/.cts/.mts` | typescript-language rule + tsx-language twin for `.tsx` | + +**Stricter than the old script (real bugs the line-regex MISSED):** the old +identifier regex `[A-Za-z_][A-Za-z0-9_.]*` did not match optional chaining or +bracket access, so it silently skipped `typeof options?.md5 === 'string'` +(packages/api/src/services/r2-bucket.ts) and `typeof entry[0] === 'string'` +(packages/api-client/src/index.ts). ast-grep's `$X` metavar matches any +expression, catching all of these. The `.tsx` twin also catches +`packages/web-ui/src/components/chart.tsx`. All migrated (see report). + +## no-raw-regex.ts → `no-raw-regex.yml` (+ `no-raw-regex-tsx.yml`) + +| Old-script behavior | ast-grep coverage | +|---|---| +| Flags `new RegExp(...)` | `pattern: new RegExp($$$ARGS)` | +| Flags `.replace/.replaceAll/.match/.matchAll/.test/.split/.search(/.../)` | method `any` + `has: {field: arguments, has: {kind: regex}}` (top-level regex literal arg only — mirrors the old `(/` heuristic, no over-match into nested calls) | +| Scope apps/+packages/ non-test | same `ignores` set as typeof | +| Allowlist enrichment.ts + alltrails.ts | `ignores` entries for both files | +| Biome `performance/useTopLevelRegex` covers the strict AST case | noted in rule `message` | + +## no-raw-json (new) → `no-raw-json*.yml` + +Not part of parity (no old script). `severity: warning` so CI is not gated. +`JSON.parse($X)`→`safeJsonParse($X)` and single-arg `JSON.stringify($X)`→ +`safeJsonStringify($X)` carry autofix `fix:`. Multi-arg stringify is flagged without +autofix (no clean 1:1 rewrite). Import insertion is out of scope. + +## no-primitive-cast (new) → `no-primitive-cast*.yml` (+ `-tsx` twin) + +Not part of parity (no old script). Complements `packages/checks/check-type-casts.ts`, +which deliberately exempts single-word lowercase types (`if (LOWERCASE_TYPE.test(castType)) continue;`) +and so never flags `as string` / `as number` / `as boolean`. This rule fills that gap: +it matches an as-expression to a primitive type (`$X as string|number|boolean`) and steers +authors to a `@packrat/guards` narrow (`toString`/`toNumber`/`toBoolean`, `as*` aliases) or an +explicit coercion (`String`/`Number`/`toFloat`/`toInt`). No double-reporting — the two checks +cover disjoint cast shapes (named types vs. primitives). `as const` / `as unknown` / `as T[]` +are not primitive single-type assertions and are naturally excluded by the pattern. Same +`ignores` scope as the typeof rules (guards/utils/tooling/test files). `severity: warning` +because ~63 primitive casts already exist (mostly `apps/expo`); burn the backlog down, then +promote to `error`. diff --git a/ast-grep-rules/no-primitive-cast-tsx.yml b/ast-grep-rules/no-primitive-cast-tsx.yml new file mode 100644 index 0000000000..14df591109 --- /dev/null +++ b/ast-grep-rules/no-primitive-cast-tsx.yml @@ -0,0 +1,24 @@ +id: no-primitive-cast-tsx +# TSX variant of no-primitive-cast. The `typescript` language parser does NOT +# match .tsx files, so .tsx app code (apps/expo, apps/trails, apps/guides, +# apps/landing) needs this `tsx`-language twin. Same rule/ignores as +# no-primitive-cast.yml. +language: tsx +severity: warning +message: "Don't assert with `as string` / `as number` / `as boolean` — assertions lie to the type-checker at boundaries. Use a `@packrat/guards` narrow (`toString`/`toNumber`/`toBoolean`, or the `asString`/`asNumber`/`asBoolean` aliases) and handle the `undefined` case, or coerce explicitly (e.g. `String(x)`, `Number(x)`, `toFloat`/`toInt` from @packrat/utils). `as const` / `as unknown` are fine and not flagged. Annotate a genuinely unavoidable cast with `// safe-cast:`." +ignores: + - "**/packages/guards/**" + - "**/packages/utils/**" + - "scripts/**" + - ".github/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" +rule: + any: + - pattern: $X as string + - pattern: $X as number + - pattern: $X as boolean diff --git a/ast-grep-rules/no-primitive-cast.yml b/ast-grep-rules/no-primitive-cast.yml new file mode 100644 index 0000000000..6592831e37 --- /dev/null +++ b/ast-grep-rules/no-primitive-cast.yml @@ -0,0 +1,24 @@ +id: no-primitive-cast +language: typescript +# warning, not error: the repo already has ~63 primitive casts (mostly apps/expo), +# so gating CI would break the build. Burn the backlog down, then promote to error. +severity: warning +message: "Don't assert with `as string` / `as number` / `as boolean` — assertions lie to the type-checker at boundaries. Use a `@packrat/guards` narrow (`toString`/`toNumber`/`toBoolean`, or the `asString`/`asNumber`/`asBoolean` aliases) and handle the `undefined` case, or coerce explicitly (e.g. `String(x)`, `Number(x)`, `toFloat`/`toInt` from @packrat/utils). `as const` / `as unknown` are fine and not flagged. Annotate a genuinely unavoidable cast with `// safe-cast:`." +ignores: + # Mirrors no-raw-typeof scope: guards/utils are the canonical narrow homes, + # tooling dirs and non-production files are out of scope. + - "**/packages/guards/**" + - "**/packages/utils/**" + - "scripts/**" + - ".github/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" +rule: + any: + - pattern: $X as string + - pattern: $X as number + - pattern: $X as boolean diff --git a/ast-grep-rules/no-raw-json-stringify-multi-tsx.yml b/ast-grep-rules/no-raw-json-stringify-multi-tsx.yml new file mode 100644 index 0000000000..ad9e98d52e --- /dev/null +++ b/ast-grep-rules/no-raw-json-stringify-multi-tsx.yml @@ -0,0 +1,23 @@ +id: no-raw-json-stringify-multi-tsx +# TSX variant of no-raw-json-stringify-multi (multi-arg, no autofix). +language: tsx +severity: error +message: "Prefer safeJsonStringify from @packrat/utils over raw JSON.stringify (multi-arg: replacer/space need manual migration)." +ignores: + - "**/test/**" + - "**/playwright/**" + - "**/e2e/**" + - "scripts/**" + - ".github/**" + - "**/packages/utils/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" +rule: + all: + - pattern: JSON.stringify($$$ARGS) + - not: + pattern: JSON.stringify($X) diff --git a/ast-grep-rules/no-raw-json-stringify-multi.yml b/ast-grep-rules/no-raw-json-stringify-multi.yml new file mode 100644 index 0000000000..9145bed7dc --- /dev/null +++ b/ast-grep-rules/no-raw-json-stringify-multi.yml @@ -0,0 +1,27 @@ +id: no-raw-json-stringify-multi +language: typescript +# WARNING (not error): multi-argument JSON.stringify (replacer / space). Flagged +# for the separate JSON-migration unit but intentionally NO autofix — there is +# no clean 1:1 rewrite, so the orchestrator's codemod handles these by hand. +# The single-arg form is covered (with autofix) by no-raw-json-stringify.yml; +# the `not` constraint below keeps the two rules from double-reporting. +severity: error +message: "Prefer safeJsonStringify from @packrat/utils over raw JSON.stringify (multi-arg: replacer/space need manual migration)." +ignores: + - "**/test/**" + - "**/playwright/**" + - "**/e2e/**" + - "scripts/**" + - ".github/**" + - "**/packages/utils/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" +rule: + all: + - pattern: JSON.stringify($$$ARGS) + - not: + pattern: JSON.stringify($X) diff --git a/ast-grep-rules/no-raw-json-stringify-tsx.yml b/ast-grep-rules/no-raw-json-stringify-tsx.yml new file mode 100644 index 0000000000..5a5c669948 --- /dev/null +++ b/ast-grep-rules/no-raw-json-stringify-tsx.yml @@ -0,0 +1,21 @@ +id: no-raw-json-stringify-tsx +# TSX variant of no-raw-json-stringify (single-arg, autofixable). +language: tsx +severity: error +message: "Prefer safeJsonStringify from @packrat/utils over raw JSON.stringify." +ignores: + - "**/test/**" + - "**/playwright/**" + - "**/e2e/**" + - "scripts/**" + - ".github/**" + - "**/packages/utils/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" +rule: + pattern: JSON.stringify($X) +fix: safeJsonStringify($X) diff --git a/ast-grep-rules/no-raw-json-stringify.yml b/ast-grep-rules/no-raw-json-stringify.yml new file mode 100644 index 0000000000..388d1f6bbf --- /dev/null +++ b/ast-grep-rules/no-raw-json-stringify.yml @@ -0,0 +1,25 @@ +id: no-raw-json-stringify +language: typescript +# WARNING (not error): see no-raw-json.yml. Covers the single-argument form of +# JSON.stringify, which has a clean 1:1 autofix (JSON.stringify($X) -> +# safeJsonStringify($X)). Multi-arg calls (replacer / space) are handled by +# no-raw-json-stringify-multi.yml, which has no autofix. Import insertion is out +# of scope here (the orchestrator's codemod handles it). +severity: error +message: "Prefer safeJsonStringify from @packrat/utils over raw JSON.stringify." +ignores: + - "**/test/**" + - "**/playwright/**" + - "**/e2e/**" + - "scripts/**" + - ".github/**" + - "**/packages/utils/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" +rule: + pattern: JSON.stringify($X) +fix: safeJsonStringify($X) diff --git a/ast-grep-rules/no-raw-json-tsx.yml b/ast-grep-rules/no-raw-json-tsx.yml new file mode 100644 index 0000000000..00ce0a60d9 --- /dev/null +++ b/ast-grep-rules/no-raw-json-tsx.yml @@ -0,0 +1,21 @@ +id: no-raw-json-tsx +# TSX variant of no-raw-json (the `typescript` parser does not match .tsx). +language: tsx +severity: error +message: "Prefer safeJsonParse from @packrat/utils over raw JSON.parse. Raw JSON.parse throws on malformed input and returns `any`; safeJsonParse returns a typed result." +ignores: + - "**/test/**" + - "**/playwright/**" + - "**/e2e/**" + - "scripts/**" + - ".github/**" + - "**/packages/utils/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" +rule: + pattern: JSON.parse($X) +fix: safeJsonParse($X) diff --git a/ast-grep-rules/no-raw-json.yml b/ast-grep-rules/no-raw-json.yml new file mode 100644 index 0000000000..21fa34f5bc --- /dev/null +++ b/ast-grep-rules/no-raw-json.yml @@ -0,0 +1,24 @@ +id: no-raw-json +language: typescript +# WARNING (not error): the repo-wide JSON migration (~156 sites) is a separate +# unit handled by the orchestrator. This rule surfaces JSON.parse call sites +# without failing CI yet. The `fix` rewrites JSON.parse($X) -> safeJsonParse($X); +# import insertion is out of scope here (the orchestrator's codemod handles it). +severity: error +message: "Prefer safeJsonParse from @packrat/utils over raw JSON.parse. Raw JSON.parse throws on malformed input and returns `any`; safeJsonParse returns a typed result." +ignores: + - "**/test/**" + - "**/playwright/**" + - "**/e2e/**" + - "scripts/**" + - ".github/**" + - "**/packages/utils/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" +rule: + pattern: JSON.parse($X) +fix: safeJsonParse($X) diff --git a/ast-grep-rules/no-raw-regex-tsx.yml b/ast-grep-rules/no-raw-regex-tsx.yml new file mode 100644 index 0000000000..0a7e0d7314 --- /dev/null +++ b/ast-grep-rules/no-raw-regex-tsx.yml @@ -0,0 +1,34 @@ +id: no-raw-regex-tsx +# TSX variant of no-raw-regex (the `typescript` parser does not match .tsx). +language: tsx +severity: error +message: "Prefer magic-regexp over a raw regex literal or `new RegExp(...)`. Raw regex is error-prone (missing escapes, unintended capture groups, poor readability); magic-regexp gives a typed, composable builder. See packages/analytics/src/core/enrichment.ts for a reference. Biome's performance/useTopLevelRegex covers the stricter top-level check." +ignores: + - "scripts/**" + - ".github/**" + - "**/packages/guards/**" + - "**/packages/utils/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" + - "**/packages/analytics/src/core/enrichment.ts" + - "**/packages/api/src/routes/alltrails.ts" +rule: + any: + - pattern: new RegExp($$$ARGS) + - all: + - any: + - { pattern: $OBJ.replace($$$) } + - { pattern: $OBJ.replaceAll($$$) } + - { pattern: $OBJ.match($$$) } + - { pattern: $OBJ.matchAll($$$) } + - { pattern: $OBJ.test($$$) } + - { pattern: $OBJ.split($$$) } + - { pattern: $OBJ.search($$$) } + - has: + field: arguments + has: + kind: regex diff --git a/ast-grep-rules/no-raw-regex.yml b/ast-grep-rules/no-raw-regex.yml new file mode 100644 index 0000000000..083a210424 --- /dev/null +++ b/ast-grep-rules/no-raw-regex.yml @@ -0,0 +1,38 @@ +id: no-raw-regex +language: typescript +severity: error +message: "Prefer magic-regexp over a raw regex literal or `new RegExp(...)`. Raw regex is error-prone (missing escapes, unintended capture groups, poor readability); magic-regexp gives a typed, composable builder. See packages/analytics/src/core/enrichment.ts for a reference. Biome's performance/useTopLevelRegex covers the stricter top-level check." +ignores: + # Mirror the old scripts/lint/no-raw-regex.ts scope: apps/ + packages/ only, + # non-test. Tooling dirs were never scanned; guards/utils are exempt homes. + - "scripts/**" + - ".github/**" + - "**/packages/guards/**" + - "**/packages/utils/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" + # File-level allowlist carried over from the old script: + # enrichment.ts is the magic-regexp reference; alltrails.ts builds a regex + # from a dynamic `property` arg and cannot be a static magic-regexp constant. + - "**/packages/analytics/src/core/enrichment.ts" + - "**/packages/api/src/routes/alltrails.ts" +rule: + any: + - pattern: new RegExp($$$ARGS) + - all: + - any: + - { pattern: $OBJ.replace($$$) } + - { pattern: $OBJ.replaceAll($$$) } + - { pattern: $OBJ.match($$$) } + - { pattern: $OBJ.matchAll($$$) } + - { pattern: $OBJ.test($$$) } + - { pattern: $OBJ.split($$$) } + - { pattern: $OBJ.search($$$) } + - has: + field: arguments + has: + kind: regex diff --git a/ast-grep-rules/no-raw-typeof-tsx.yml b/ast-grep-rules/no-raw-typeof-tsx.yml new file mode 100644 index 0000000000..e789c0720c --- /dev/null +++ b/ast-grep-rules/no-raw-typeof-tsx.yml @@ -0,0 +1,28 @@ +id: no-raw-typeof-tsx +# TSX variant of no-raw-typeof. The `typescript` language parser does NOT match +# .tsx files, so .tsx app code (apps/expo, apps/guides, apps/landing) needs this +# `tsx`-language twin. Same rule/constraints/ignores as no-raw-typeof.yml. +language: tsx +severity: error +message: "Use a guard from @packrat/guards (isString/isNumber/isBoolean/…) instead of a raw `typeof` primitive check. Raw typeof is allowed only inside packages/guards and packages/utils, and for global availability checks (window/document/globalThis/Bun/navigator/process)." +ignores: + - "**/packages/guards/**" + - "**/packages/utils/**" + - "scripts/**" + - ".github/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" +rule: + any: + - pattern: typeof $X === $T + - pattern: typeof $X !== $T +constraints: + T: + regex: "^['\"](string|number|boolean|object|function|undefined|symbol|bigint)['\"]$" + X: + not: + regex: "^(window|document|globalThis|Bun|navigator|process)$" diff --git a/ast-grep-rules/no-raw-typeof.yml b/ast-grep-rules/no-raw-typeof.yml new file mode 100644 index 0000000000..a55006959b --- /dev/null +++ b/ast-grep-rules/no-raw-typeof.yml @@ -0,0 +1,28 @@ +id: no-raw-typeof +language: typescript +severity: error +message: "Use a guard from @packrat/guards (isString/isNumber/isBoolean/…) instead of a raw `typeof` primitive check. Raw typeof is allowed only inside packages/guards and packages/utils, and for global availability checks (window/document/globalThis/Bun/navigator/process)." +ignores: + # The old scripts/lint/no-raw-typeof.ts only scanned apps/ + packages/. + # Everything below mirrors that scope: tooling dirs and non-production files + # the old script never walked, plus the canonical guard/util homes. + - "**/packages/guards/**" + - "**/packages/utils/**" + - "scripts/**" + - ".github/**" + - "**/mocks/**" + - "**/__mocks__/**" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" +rule: + any: + - pattern: typeof $X === $T + - pattern: typeof $X !== $T +constraints: + T: + regex: "^['\"](string|number|boolean|object|function|undefined|symbol|bigint)['\"]$" + X: + not: + regex: "^(window|document|globalThis|Bun|navigator|process)$" diff --git a/ast-grep-tests/__snapshots__/no-primitive-cast-snapshot.yml b/ast-grep-tests/__snapshots__/no-primitive-cast-snapshot.yml new file mode 100644 index 0000000000..d8cdc24625 --- /dev/null +++ b/ast-grep-tests/__snapshots__/no-primitive-cast-snapshot.yml @@ -0,0 +1,38 @@ +id: no-primitive-cast +snapshots: + const a = x as string;: + labels: + - source: x as string + style: primary + start: 10 + end: 21 + const b = y as number;: + labels: + - source: y as number + style: primary + start: 10 + end: 21 + const c = z as boolean;: + labels: + - source: z as boolean + style: primary + start: 10 + end: 22 + const d = (s as unknown) as string;: + labels: + - source: (s as unknown) as string + style: primary + start: 10 + end: 34 + const e = params.lat as string;: + labels: + - source: params.lat as string + style: primary + start: 10 + end: 30 + const f = Number.parseFloat(params.lon as string);: + labels: + - source: params.lon as string + style: primary + start: 28 + end: 48 diff --git a/ast-grep-tests/__snapshots__/no-primitive-cast-tsx-snapshot.yml b/ast-grep-tests/__snapshots__/no-primitive-cast-tsx-snapshot.yml new file mode 100644 index 0000000000..87d4ef840f --- /dev/null +++ b/ast-grep-tests/__snapshots__/no-primitive-cast-tsx-snapshot.yml @@ -0,0 +1,14 @@ +id: no-primitive-cast-tsx +snapshots: + const C = () => : + labels: + - source: id as string + style: primary + start: 23 + end: 35 + const D = () => : + labels: + - source: count as number + style: primary + start: 22 + end: 37 diff --git a/ast-grep-tests/__snapshots__/no-raw-regex-snapshot.yml b/ast-grep-tests/__snapshots__/no-raw-regex-snapshot.yml new file mode 100644 index 0000000000..d06fb892ad --- /dev/null +++ b/ast-grep-tests/__snapshots__/no-raw-regex-snapshot.yml @@ -0,0 +1,112 @@ +id: no-raw-regex +snapshots: + const a = new RegExp('x'): + labels: + - source: new RegExp('x') + style: primary + start: 10 + end: 25 + const b = new RegExp(pattern, 'g'): + labels: + - source: new RegExp(pattern, 'g') + style: primary + start: 10 + end: 34 + const c = s.replace(/a/, 'b'): + labels: + - source: s.replace(/a/, 'b') + style: primary + start: 10 + end: 29 + - source: /a/ + style: secondary + start: 20 + end: 23 + - source: /a/ + style: secondary + start: 20 + end: 23 + const d = s.replaceAll(/a/g, 'b'): + labels: + - source: s.replaceAll(/a/g, 'b') + style: primary + start: 10 + end: 33 + - source: /a/g + style: secondary + start: 23 + end: 27 + - source: /a/g + style: secondary + start: 23 + end: 27 + const e = s.match(/x/): + labels: + - source: s.match(/x/) + style: primary + start: 10 + end: 22 + - source: /x/ + style: secondary + start: 18 + end: 21 + - source: /x/ + style: secondary + start: 18 + end: 21 + const f = s.matchAll(/x/g): + labels: + - source: s.matchAll(/x/g) + style: primary + start: 10 + end: 26 + - source: /x/g + style: secondary + start: 21 + end: 25 + - source: /x/g + style: secondary + start: 21 + end: 25 + const g = s.test(/y/): + labels: + - source: s.test(/y/) + style: primary + start: 10 + end: 21 + - source: /y/ + style: secondary + start: 17 + end: 20 + - source: /y/ + style: secondary + start: 17 + end: 20 + const h = s.split(/,/): + labels: + - source: s.split(/,/) + style: primary + start: 10 + end: 22 + - source: /,/ + style: secondary + start: 18 + end: 21 + - source: /,/ + style: secondary + start: 18 + end: 21 + const i = s.search(/z/): + labels: + - source: s.search(/z/) + style: primary + start: 10 + end: 23 + - source: /z/ + style: secondary + start: 19 + end: 22 + - source: /z/ + style: secondary + start: 19 + end: 22 diff --git a/ast-grep-tests/__snapshots__/no-raw-typeof-snapshot.yml b/ast-grep-tests/__snapshots__/no-raw-typeof-snapshot.yml new file mode 100644 index 0000000000..bbde6d22f5 --- /dev/null +++ b/ast-grep-tests/__snapshots__/no-raw-typeof-snapshot.yml @@ -0,0 +1,68 @@ +id: no-raw-typeof +snapshots: + const n = typeof y === 'number': + labels: + - source: typeof y === 'number' + style: primary + start: 10 + end: 31 + if (typeof b === 'boolean') {}: + labels: + - source: typeof b === 'boolean' + style: primary + start: 4 + end: 26 + if (typeof big === 'bigint') {}: + labels: + - source: typeof big === 'bigint' + style: primary + start: 4 + end: 27 + if (typeof entry[0] === 'string') {}: + labels: + - source: typeof entry[0] === 'string' + style: primary + start: 4 + end: 32 + if (typeof f === 'function') {}: + labels: + - source: typeof f === 'function' + style: primary + start: 4 + end: 27 + if (typeof options?.md5 === 'string') {}: + labels: + - source: typeof options?.md5 === 'string' + style: primary + start: 4 + end: 36 + if (typeof s === 'symbol') {}: + labels: + - source: typeof s === 'symbol' + style: primary + start: 4 + end: 25 + if (typeof u === 'undefined') {}: + labels: + - source: typeof u === 'undefined' + style: primary + start: 4 + end: 28 + if (typeof x !== 'string') {}: + labels: + - source: typeof x !== 'string' + style: primary + start: 4 + end: 25 + if (typeof x === 'string') {}: + labels: + - source: typeof x === 'string' + style: primary + start: 4 + end: 25 + if (typeof z === 'object') {}: + labels: + - source: typeof z === 'object' + style: primary + start: 4 + end: 25 diff --git a/ast-grep-tests/__snapshots__/no-raw-typeof-tsx-snapshot.yml b/ast-grep-tests/__snapshots__/no-raw-typeof-tsx-snapshot.yml new file mode 100644 index 0000000000..0a89e4c99a --- /dev/null +++ b/ast-grep-tests/__snapshots__/no-raw-typeof-tsx-snapshot.yml @@ -0,0 +1,14 @@ +id: no-raw-typeof-tsx +snapshots: + 'const C = () => (typeof x === ''string'' ? : )': + labels: + - source: typeof x === 'string' + style: primary + start: 17 + end: 38 + 'const D = () => (typeof n !== ''number'' ? : )': + labels: + - source: typeof n !== 'number' + style: primary + start: 17 + end: 38 diff --git a/ast-grep-tests/no-primitive-cast-test.yml b/ast-grep-tests/no-primitive-cast-test.yml new file mode 100644 index 0000000000..111b28b16a --- /dev/null +++ b/ast-grep-tests/no-primitive-cast-test.yml @@ -0,0 +1,20 @@ +id: no-primitive-cast +valid: + # `as const` and `as unknown` are required, safe casts — not flagged. + - "const a = [1, 2, 3] as const;" + - "const b = x as unknown;" + # Array-type casts are not the primitive single-type assertion we target. + - "const c = x as string[];" + # Non-primitive named-type casts are check-type-casts.ts's job, not this rule's. + - "const d = x as MyType;" + # The narrowing API itself — the recommended replacement, never a cast. + - "const e = toString(x);" +invalid: + - "const a = x as string;" + - "const b = y as number;" + - "const c = z as boolean;" + # Double-assertion to a primitive: the outer `as string` is still flagged. + - "const d = (s as unknown) as string;" + # Works on any expression, not just identifiers. + - "const e = params.lat as string;" + - "const f = Number.parseFloat(params.lon as string);" diff --git a/ast-grep-tests/no-primitive-cast-tsx-test.yml b/ast-grep-tests/no-primitive-cast-tsx-test.yml new file mode 100644 index 0000000000..c806199d3f --- /dev/null +++ b/ast-grep-tests/no-primitive-cast-tsx-test.yml @@ -0,0 +1,8 @@ +id: no-primitive-cast-tsx +valid: + - "const C = () => (cond ? : )" + - "const D = () => " +invalid: + # Proves .tsx code (JSX present) is covered by the tsx-language twin. + - "const C = () => " + - "const D = () => " diff --git a/ast-grep-tests/no-raw-regex-test.yml b/ast-grep-tests/no-raw-regex-test.yml new file mode 100644 index 0000000000..29a9edf63d --- /dev/null +++ b/ast-grep-tests/no-raw-regex-test.yml @@ -0,0 +1,23 @@ +id: no-raw-regex +valid: + # String methods called with NON-regex args are fine. + - "const a = s.split(',')" + - "const b = s.replace('a', 'b')" + - "const c = s.split(sep)" + - "const d = s.match(myRegexVar)" + # Regex nested inside a non-first/non-top-level arg is not the `.method(/` shape. + - "const e = foo.bar(x, baz(/z/))" + # magic-regexp builder usage. + - "const r = createRegExp(word)" +invalid: + # new RegExp(...) in any form. + - "const a = new RegExp('x')" + - "const b = new RegExp(pattern, 'g')" + # Raw regex literal passed to a string method (the old script's `.method(/` cases). + - "const c = s.replace(/a/, 'b')" + - "const d = s.replaceAll(/a/g, 'b')" + - "const e = s.match(/x/)" + - "const f = s.matchAll(/x/g)" + - "const g = s.test(/y/)" + - "const h = s.split(/,/)" + - "const i = s.search(/z/)" diff --git a/ast-grep-tests/no-raw-typeof-test.yml b/ast-grep-tests/no-raw-typeof-test.yml new file mode 100644 index 0000000000..2f4cc6b7a2 --- /dev/null +++ b/ast-grep-tests/no-raw-typeof-test.yml @@ -0,0 +1,27 @@ +id: no-raw-typeof +valid: + # Global availability checks are exempt (accessing undeclared globals throws). + - "if (typeof window !== 'undefined') {}" + - "if (typeof document === 'undefined') {}" + - "const g = typeof globalThis === 'object'" + - "if (typeof Bun !== 'undefined') {}" + - "if (typeof navigator === 'object') {}" + - "if (typeof process !== 'undefined') {}" + # typeof against a non-primitive-literal RHS is not a primitive narrowing. + - "if (typeof a === b) {}" + - "if (typeof a === SOME_CONST) {}" + # Already migrated to a guard predicate. + - "if (isString(x)) {}" +invalid: + - "if (typeof x === 'string') {}" + - "if (typeof x !== 'string') {}" + - "const n = typeof y === 'number'" + - "if (typeof z === 'object') {}" + - "if (typeof f === 'function') {}" + - "if (typeof b === 'boolean') {}" + - "if (typeof u === 'undefined') {}" + - "if (typeof s === 'symbol') {}" + - "if (typeof big === 'bigint') {}" + # Patterns the old line-regex MISSED but ast-grep catches: + - "if (typeof options?.md5 === 'string') {}" + - "if (typeof entry[0] === 'string') {}" diff --git a/ast-grep-tests/no-raw-typeof-tsx-test.yml b/ast-grep-tests/no-raw-typeof-tsx-test.yml new file mode 100644 index 0000000000..4b89f3aa06 --- /dev/null +++ b/ast-grep-tests/no-raw-typeof-tsx-test.yml @@ -0,0 +1,8 @@ +id: no-raw-typeof-tsx +valid: + - "const C = () => (typeof window !== 'undefined' ? : )" + - "const v = isString(x) ? : null" +invalid: + # Proves .tsx code (JSX present) is covered by the tsx-language twin. + - "const C = () => (typeof x === 'string' ? : )" + - "const D = () => (typeof n !== 'number' ? : )" diff --git a/biome.json b/biome.json index 5715215a55..ca6e143008 100644 --- a/biome.json +++ b/biome.json @@ -53,7 +53,31 @@ "useTopLevelRegex": "error" }, "style": { - "noNonNullAssertion": "error" + "noNonNullAssertion": "error", + "noRestrictedImports": { + "level": "error", + "options": { + "patterns": [ + { + "group": [ + "radashi", + "radashi/**", + "radash", + "radash/**", + "es-toolkit", + "es-toolkit/**", + "lodash", + "lodash/**", + "remeda", + "remeda/**", + "destr", + "safe-stable-stringify" + ], + "message": "Import utilities from @packrat/utils (or narrowing from @packrat/guards). These libs may only be imported inside packages/utils — the single curated utility surface. See docs/utils-policy.md." + } + ] + } + } }, "suspicious": { "noUnknownAtRules": "off" @@ -61,6 +85,16 @@ } }, "overrides": [ + { + "includes": ["packages/utils/**"], + "linter": { + "rules": { + "style": { + "noRestrictedImports": "off" + } + } + } + }, { "includes": [ "apps/expo/atoms/atomWith*.ts", diff --git a/bun.lock b/bun.lock index e13363a3da..9997add77a 100644 --- a/bun.lock +++ b/bun.lock @@ -5,6 +5,7 @@ "": { "name": "packrat-monorepo", "devDependencies": { + "@ast-grep/cli": "^0.43.0", "@biomejs/biome": "2.4.6", "@manypkg/cli": "^0.24.0", "@playwright/test": "^1.59.1", @@ -14,21 +15,24 @@ "@typescript/native-preview": "7.0.0-dev.20260527.2", "fs-extra": "^11.3.0", "glob": "^11.0.3", + "jscpd": "^4.2.4", "lefthook": "^1.11.14", "semver": "catalog:", "sort-package-json": "^3.6.1", + "ts-morph": "^28.0.0", "turbo": "^2.5.0", }, }, "apps/admin": { "name": "packrat-admin-app", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@elysiajs/eden": "catalog:", "@packrat/api-client": "workspace:*", "@packrat/app": "workspace:*", "@packrat/guards": "workspace:*", "@packrat/schemas": "workspace:*", + "@packrat/utils": "workspace:*", "@packrat/web-ui": "workspace:*", "@radix-ui/react-alert-dialog": "catalog:", "@radix-ui/react-avatar": "catalog:", @@ -71,7 +75,7 @@ }, "apps/expo": { "name": "packrat-expo-app", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@ai-sdk/react": "^3.0.170", "@better-auth/expo": "^1.6.9", @@ -91,6 +95,7 @@ "@packrat/types": "workspace:*", "@packrat/ui": "workspace:*", "@packrat/units": "workspace:*", + "@packrat/utils": "workspace:*", "@react-native-ai/apple": "~0.10.0", "@react-native-ai/llama": "~0.10.0", "@react-native-async-storage/async-storage": "2.2.0", @@ -161,7 +166,6 @@ "llama.rn": "0.10.1", "nanoid": "^5.1.9", "nativewind": "^4.2.3", - "radash": "catalog:", "react": "catalog:", "react-dom": "catalog:", "react-i18next": "^17.0.4", @@ -210,7 +214,7 @@ }, "apps/guides": { "name": "packrat-guides-app", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@ai-sdk/openai": "catalog:", "@elysiajs/eden": "catalog:", @@ -219,6 +223,7 @@ "@packrat/env": "workspace:*", "@packrat/guards": "workspace:*", "@packrat/schemas": "workspace:*", + "@packrat/utils": "workspace:*", "@packrat/web-ui": "workspace:*", "@radix-ui/react-accordion": "catalog:", "@radix-ui/react-alert-dialog": "catalog:", @@ -299,7 +304,7 @@ }, "apps/landing": { "name": "packrat-landing-app", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@emotion/is-prop-valid": "^1.3.1", "@hookform/resolvers": "catalog:", @@ -369,12 +374,13 @@ }, "apps/trails": { "name": "packrat-trails-app", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@packrat/api-client": "workspace:*", "@packrat/app": "workspace:*", "@packrat/guards": "workspace:*", "@packrat/overpass": "workspace:*", + "@packrat/utils": "workspace:*", "@packrat/web-ui": "workspace:*", "@radix-ui/react-dialog": "catalog:", "@radix-ui/react-label": "catalog:", @@ -415,6 +421,7 @@ "@packrat/app": "workspace:*", "@packrat/env": "workspace:*", "@packrat/guards": "workspace:*", + "@packrat/utils": "workspace:*", "@packrat/web-ui": "workspace:*", "@tanstack/react-query": "catalog:", "@tanstack/react-query-devtools": "catalog:", @@ -442,14 +449,14 @@ }, "packages/analytics": { "name": "@packrat/analytics", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@duckdb/node-api": "catalog:", "@packrat/env": "workspace:*", "@packrat/guards": "workspace:*", + "@packrat/utils": "workspace:*", "consola": "catalog:", "magic-regexp": "catalog:", - "radash": "catalog:", "zod": "catalog:", }, "devDependencies": { @@ -459,7 +466,7 @@ }, "packages/api": { "name": "@packrat/api", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@ai-sdk/google": "^3.0.64", "@ai-sdk/openai": "catalog:", @@ -480,7 +487,8 @@ "@packrat/schemas": "workspace:*", "@packrat/types": "workspace:*", "@packrat/units": "workspace:*", - "@sentry/cloudflare": "^10.0.0", + "@packrat/utils": "workspace:*", + "@sentry/cloudflare": "^10.37.0", "@sinclair/typebox": "^0.34.15", "@types/nodemailer": "^6.4.17", "ai": "catalog:", @@ -496,7 +504,6 @@ "linkedom": "^0.18.11", "nodemailer": "^6.10.0", "pg": "catalog:", - "radash": "catalog:", "resend": "^6.10.0", "workers-ai-provider": "^0.7.2", "ws": "catalog:", @@ -523,10 +530,11 @@ }, "packages/api-client": { "name": "@packrat/api-client", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@elysiajs/eden": "catalog:", "@packrat/guards": "workspace:*", + "@packrat/utils": "workspace:*", }, "devDependencies": { "@packrat/api": "workspace:*", @@ -541,7 +549,7 @@ }, "packages/app": { "name": "@packrat/app", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@packrat/api-client": "workspace:*", "@packrat/schemas": "workspace:*", @@ -562,11 +570,11 @@ }, "packages/checks": { "name": "@packrat/checks", - "version": "2.0.26", + "version": "2.0.27", }, "packages/cli": { "name": "@packrat/cli", - "version": "2.0.26", + "version": "2.0.27", "bin": { "packrat": "./src/index.ts", }, @@ -576,6 +584,7 @@ "@packrat/api-client": "workspace:*", "@packrat/env": "workspace:*", "@packrat/guards": "workspace:*", + "@packrat/utils": "workspace:*", "chalk": "catalog:", "citty": "^0.2.1", "cli-table3": "^0.6.5", @@ -590,21 +599,21 @@ }, "packages/config": { "name": "@packrat/config", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@packrat/guards": "workspace:*", }, }, "packages/constants": { "name": "@packrat/constants", - "version": "0.0.0", + "version": "2.0.27", "devDependencies": { "typescript": "catalog:", }, }, "packages/db": { "name": "@packrat/db", - "version": "0.0.0", + "version": "2.0.27", "dependencies": { "@packrat/constants": "workspace:*", "drizzle-orm": "catalog:", @@ -616,27 +625,28 @@ }, "packages/env": { "name": "@packrat/env", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "zod": "catalog:", }, }, "packages/guards": { "name": "@packrat/guards", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { - "radash": "catalog:", + "@packrat/utils": "workspace:*", "ts-extras": "catalog:", "zod": "catalog:", }, }, "packages/mcp": { "name": "@packrat/mcp", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@cloudflare/workers-oauth-provider": "^0.4.0", "@modelcontextprotocol/sdk": "^1.11.0", "@packrat/api-client": "workspace:*", + "@packrat/utils": "workspace:*", "agents": "^0.11.0", "magic-regexp": "catalog:", "zod": "catalog:", @@ -652,7 +662,7 @@ }, "packages/osm-db": { "name": "@packrat/osm-db", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@neondatabase/serverless": "catalog:", "drizzle-orm": "catalog:", @@ -666,7 +676,7 @@ }, "packages/osm-import": { "name": "@packrat/osm-import", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@packrat/env": "workspace:*", "pg": "catalog:", @@ -674,7 +684,7 @@ }, "packages/overpass": { "name": "@packrat/overpass", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@packrat/guards": "workspace:*", "zod": "catalog:", @@ -686,11 +696,12 @@ }, "packages/schemas": { "name": "@packrat/schemas", - "version": "0.0.0", + "version": "2.0.27", "dependencies": { "@packrat/constants": "workspace:*", "@packrat/db": "workspace:*", "@packrat/guards": "workspace:*", + "@packrat/utils": "workspace:*", "zod": "catalog:", }, "devDependencies": { @@ -699,7 +710,7 @@ }, "packages/types": { "name": "@packrat/types", - "version": "0.0.0", + "version": "2.0.27", "dependencies": { "@packrat/constants": "workspace:*", "@packrat/schemas": "workspace:*", @@ -714,14 +725,14 @@ }, "packages/ui": { "name": "@packrat/ui", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@packrat-ai/nativewindui": "2.0.3-2", }, }, "packages/units": { "name": "@packrat/units", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@packrat/constants": "workspace:*", "@packrat/guards": "workspace:*", @@ -731,9 +742,26 @@ "vitest": "catalog:", }, }, + "packages/utils": { + "name": "@packrat/utils", + "version": "2.0.27", + "dependencies": { + "destr": "^2.0.5", + "es-toolkit": "^1.47.0", + "lodash": "^4.18.1", + "radash": "catalog:", + "radashi": "^12.9.1", + "remeda": "^2.37.0", + "safe-stable-stringify": "^2.5.0", + }, + "devDependencies": { + "@types/lodash": "^4.17.24", + "vitest": "catalog:", + }, + }, "packages/web-ui": { "name": "@packrat/web-ui", - "version": "2.0.26", + "version": "2.0.27", "dependencies": { "@packrat/guards": "workspace:*", "@radix-ui/react-accordion": "catalog:", @@ -934,6 +962,22 @@ "@appium/types": ["@appium/types@1.4.0", "", { "dependencies": { "@appium/logger": "2.0.7", "@appium/schema": "1.1.1", "@appium/tsconfig": "1.1.2", "type-fest": "5.6.0" } }, "sha512-GeYnDMj1yOIFA8ujOHv0/ZKoZe42F9ldCVSlnEOheYnxqA5ueHGwRI11ifZoIfMBsq7hpU77MAzmu+v9NV1vig=="], + "@ast-grep/cli": ["@ast-grep/cli@0.43.0", "", { "dependencies": { "detect-libc": "2.1.2" }, "optionalDependencies": { "@ast-grep/cli-darwin-arm64": "0.43.0", "@ast-grep/cli-darwin-x64": "0.43.0", "@ast-grep/cli-linux-arm64-gnu": "0.43.0", "@ast-grep/cli-linux-x64-gnu": "0.43.0", "@ast-grep/cli-win32-arm64-msvc": "0.43.0", "@ast-grep/cli-win32-ia32-msvc": "0.43.0", "@ast-grep/cli-win32-x64-msvc": "0.43.0" }, "bin": { "sg": "sg", "ast-grep": "ast-grep" } }, "sha512-DGi6xXAOBJubGg9QWqTeW8PzKSGHWEOa3uxXspEfYf532yb3lHmNJAcKMl1d+O9Xs9bTcNeDLC8se+O+tirEFQ=="], + + "@ast-grep/cli-darwin-arm64": ["@ast-grep/cli-darwin-arm64@0.43.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0i63gSBbgriPaRpFsbP3yETxolHPK2JAZbpcGbFOytB7QTnKAguwhlKmIOkUGKfsCzYiEq1awY0EBmvjMONXOg=="], + + "@ast-grep/cli-darwin-x64": ["@ast-grep/cli-darwin-x64@0.43.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-SPkj00HGKpYdqReUmso2ftG5Xgd+bWNFH4i9fubLxsWzn4ey2G6sotPruaOtcrzxb9xH+8kmhN7KJfm1k8Atzg=="], + + "@ast-grep/cli-linux-arm64-gnu": ["@ast-grep/cli-linux-arm64-gnu@0.43.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-U8+2fkcY8sBxNHHBYZ33vTa7C97GFAB+Kj6gFzVMSqK8Ve1Aw+DxhVWD4i/PBryDdmWb4/erjGSkTQtdhVa2vg=="], + + "@ast-grep/cli-linux-x64-gnu": ["@ast-grep/cli-linux-x64-gnu@0.43.0", "", { "os": "linux", "cpu": "x64" }, "sha512-r/o9Mag6OZmGevY9OJjatuUKDOX1rSvgo29qSfxpMbIciiH3hkzEW/2w1xTPZI8xnM7iC+k+CkGoknmoXVTYGg=="], + + "@ast-grep/cli-win32-arm64-msvc": ["@ast-grep/cli-win32-arm64-msvc@0.43.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-oHa4ruD87xccnqFuR+Pmx6F/suHV0YtibuyZ6SxUqgpNJAFZiUNAiFblzhEgQ5gp03e8B012P/Yy/7GYOvxOLg=="], + + "@ast-grep/cli-win32-ia32-msvc": ["@ast-grep/cli-win32-ia32-msvc@0.43.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-RSzz9bKzSEutWgX7/g84guudEp75Q990CYpG/TBasdJP+U27zX8aA9d5p1+o/lF0hw3UoTYuFCGG8PU/QelfNQ=="], + + "@ast-grep/cli-win32-x64-msvc": ["@ast-grep/cli-win32-x64-msvc@0.43.0", "", { "os": "win32", "cpu": "x64" }, "sha512-/5dDD9B65vVuHCdVHN5+tIDquhE5s43S5CgEn88w1BgQaoayf+nRnUO4ZFez6SqyCGqAfa/yZdoaOQ9N2ySa1w=="], + "@aws-crypto/crc32": ["@aws-crypto/crc32@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg=="], "@aws-crypto/crc32c": ["@aws-crypto/crc32c@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag=="], @@ -1564,6 +1608,16 @@ "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + "@jscpd/badge-reporter": ["@jscpd/badge-reporter@4.2.4", "", { "dependencies": { "badgen": "^3.2.3", "colors": "^1.4.0", "fs-extra": "^11.2.0" } }, "sha512-g5vu05u0lX9rcHA0k3CptLfpOiuMzxh5+mUe2iYRAznTwH3ks6JAVAf9aPi5mBFttMCRiJh2zSt3xnSadHtMGg=="], + + "@jscpd/core": ["@jscpd/core@4.2.4", "", { "dependencies": { "eventemitter3": "^5.0.1" } }, "sha512-9V9YzmmhYg9682kFqi+n0KGOhXNSoqxHbuIP3i/l/oSd6upBOnnSeBWDZMGOenQRQnyKEtCIbnS9YFz+3B+siQ=="], + + "@jscpd/finder": ["@jscpd/finder@4.2.4", "", { "dependencies": { "@jscpd/core": "4.2.4", "@jscpd/tokenizer": "4.2.4", "blamer": "^1.0.6", "bytes": "^3.1.2", "cli-table3": "^0.6.5", "colors": "^1.4.0", "fast-glob": "^3.3.2", "fs-extra": "^11.2.0", "markdown-table": "^2.0.0", "pug": "^3.0.3" } }, "sha512-4LLEuAAmAraud/TAAlB5BByVdWfy7SYiPKacj5yEggpkNs0qsw2kiZ5EyU3LonB+/vntJJEDDpJMmvOeS58e0A=="], + + "@jscpd/html-reporter": ["@jscpd/html-reporter@4.2.4", "", { "dependencies": { "colors": "1.4.0", "fs-extra": "^11.2.0", "pug": "^3.0.3" } }, "sha512-6UljCTVGf7O+o6D6fs1zNBG+vR1PTn47W2mSgb5hzSrvNw60rLrVoAMZMnr/TeIEdd/OEgAu+icbdvvVBfnvJw=="], + + "@jscpd/tokenizer": ["@jscpd/tokenizer@4.2.4", "", { "dependencies": { "@jscpd/core": "4.2.4", "spark-md5": "^3.0.2" } }, "sha512-nM4kGyDvpcevt8t0zOsMQ82ShSc65c3LIQUHClTYwraiOGOmWgUQyen+JIiFCNF8eDCGR2Qa5iI5XBfGWYQzIg=="], + "@legendapp/state": ["@legendapp/state@3.0.0-beta.47", "", { "dependencies": { "use-sync-external-store": "^1.2.2" }, "peerDependencies": { "expo-sqlite": "^15.0.0" }, "optionalPeers": ["expo-sqlite"] }, "sha512-MPgPacXXSoAazAv7ulW/o0ZAtK4YHk3twvXZ241l2HqAHciHozb7tg5SMbEAc2HKUUfC3JBh+9+DXfMsYokLpQ=="], "@lhci/cli": ["@lhci/cli@0.14.0", "", { "dependencies": { "@lhci/utils": "0.14.0", "chrome-launcher": "^0.13.4", "compression": "^1.7.4", "debug": "^4.3.1", "express": "^4.17.1", "inquirer": "^6.3.1", "isomorphic-fetch": "^3.0.0", "lighthouse": "12.1.0", "lighthouse-logger": "1.2.0", "open": "^7.1.0", "proxy-agent": "^6.4.0", "tmp": "^0.1.0", "uuid": "^8.3.1", "yargs": "^15.4.1", "yargs-parser": "^13.1.2" }, "bin": { "lhci": "./src/cli.js" } }, "sha512-TxOH9pFBnmmN7Jmo2Aimxx5UhE8veqXpHfFJDMWsCVxkwh7mGxcAWchGl84mK139SZbbRmerqZ72c+h2nG9/QQ=="], @@ -1662,6 +1716,8 @@ "@packrat/units": ["@packrat/units@workspace:packages/units"], + "@packrat/utils": ["@packrat/utils@workspace:packages/utils"], + "@packrat/web-ui": ["@packrat/web-ui@workspace:packages/web-ui"], "@paulirish/trace_engine": ["@paulirish/trace_engine@0.0.23", "", {}, "sha512-2ym/q7HhC5K+akXkNV6Gip3oaHpbI6TsGjmcAsl7bcJ528MVbacPQeoauLFEeLXH4ulJvsxQwNDIg/kAEhFZxw=="], @@ -2152,6 +2208,8 @@ "@tootallnate/quickjs-emscripten": ["@tootallnate/quickjs-emscripten@0.23.0", "", {}, "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA=="], + "@ts-morph/common": ["@ts-morph/common@0.29.0", "", { "dependencies": { "minimatch": "^10.0.1", "path-browserify": "^1.0.1", "tinyglobby": "^0.2.14" } }, "sha512-35oUmphHbJvQ/+UTwFNme/t2p3FoKiGJ5auTjjpNTop2dyREspirjMy82PLSC1pnDJ8ah1GU98hwpVt64YXQsg=="], + "@tsconfig/node20": ["@tsconfig/node20@20.1.9", "", {}, "sha512-IjlTv1RsvnPtUcjTqtVsZExKVq+KQx4g5pCP5tI7rAs6Xesl2qFwSz/tPDBC4JajkL/MlezBu3gPUwqRHl+RIg=="], "@turbo/darwin-64": ["@turbo/darwin-64@2.9.16", "", { "os": "darwin", "cpu": "x64" }, "sha512-jLjApWTSNd7JZ5JaLYfelW1ytnGQOvB7ivl+2RD1xQvJTbi8I9gBjzcga7tDZVPyaxpl10YTfJt3BrYXR18KDw=="], @@ -2230,6 +2288,8 @@ "@types/leaflet": ["@types/leaflet@1.9.21", "", { "dependencies": { "@types/geojson": "*" } }, "sha512-TbAd9DaPGSnzp6QvtYngntMZgcRk+igFELwR2N99XZn7RXUdKgsXMR+28bUO0rPsWp8MIu/f47luLIQuSLYv/w=="], + "@types/lodash": ["@types/lodash@4.17.24", "", {}, "sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ=="], + "@types/mdast": ["@types/mdast@4.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="], "@types/mdx": ["@types/mdx@2.0.13", "", {}, "sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw=="], @@ -2248,6 +2308,8 @@ "@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="], + "@types/sarif": ["@types/sarif@2.1.7", "", {}, "sha512-kRz0VEkJqWLf1LLVN4pT1cg1Z9wAuvI6L97V3m2f5B76Tg8d413ddvLBPTEHAZJlnn4XSvu0FkZtViCQGVyrXQ=="], + "@types/stack-utils": ["@types/stack-utils@2.0.3", "", {}, "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw=="], "@types/triple-beam": ["@types/triple-beam@1.3.5", "", {}, "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw=="], @@ -2418,6 +2480,8 @@ "asap": ["asap@2.0.6", "", {}, "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA=="], + "assert-never": ["assert-never@1.4.0", "", {}, "sha512-5oJg84os6NMQNl27T9LnZkvvqzvAnHu03ShCnoj6bsJwS7L8AO4lf+C/XjK/nvzEqQB744moC6V128RucQd1jA=="], + "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], "ast-module-types": ["ast-module-types@6.0.2", "", {}, "sha512-6KuK/7nZ/2Qh7sGuVEiwxjCxzTY2Pdb5mTo5z1e6/J8BA0tvjR7G8vQJKrQMTqwmnA3UPEyKIFX4YUS1DO1Hvw=="], @@ -2472,6 +2536,10 @@ "babel-preset-jest": ["babel-preset-jest@29.6.3", "", { "dependencies": { "babel-plugin-jest-hoist": "^29.6.3", "babel-preset-current-node-syntax": "^1.0.0" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA=="], + "babel-walk": ["babel-walk@3.0.0-canary-5", "", { "dependencies": { "@babel/types": "^7.9.6" } }, "sha512-GAwkz0AihzY5bkwIY5QDR+LvsRQgB/B+1foMPvi0FZPMl5fjD7ICiznUiBdLYMH1QYe6vqu4gWYytZOccLouFw=="], + + "badgen": ["badgen@3.3.2", "", {}, "sha512-fbQwK9norfdzbdsoPwbLIAmgBXDGEme3jeIyqPAH7o6vp9lmuLHS7uXULvOiQ6XnMLkYNG4gDjILf74hgtTAug=="], + "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], "balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], @@ -2522,6 +2590,8 @@ "blake3-wasm": ["blake3-wasm@2.1.5", "", {}, "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g=="], + "blamer": ["blamer@1.0.7", "", { "dependencies": { "execa": "^4.0.0", "which": "^2.0.2" } }, "sha512-GbBStl/EVlSWkiJQBZps3H1iARBrC7vt++Jb/TTmCNu/jZ04VW7tSN1nScbFXBUy1AN+jzeL7Zep9sbQxLhXKA=="], + "bluebird": ["bluebird@3.7.2", "", {}, "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="], "body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="], @@ -2588,6 +2658,8 @@ "character-entities-legacy": ["character-entities-legacy@3.0.0", "", {}, "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ=="], + "character-parser": ["character-parser@2.2.0", "", { "dependencies": { "is-regex": "^1.0.3" } }, "sha512-+UqJQjFEFaTAs3bNsF2j2kEN1baG/zghZbdqoYEDxGZtJo9LBzl1A+m0D4n3qKx8N2FNv8/Xp6yV9mQmBuptaw=="], + "chardet": ["chardet@2.1.1", "", {}, "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ=="], "check-error": ["check-error@2.1.3", "", {}, "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA=="], @@ -2630,6 +2702,8 @@ "cmdk": ["cmdk@1.1.1", "", { "dependencies": { "@radix-ui/react-compose-refs": "^1.1.1", "@radix-ui/react-dialog": "^1.1.6", "@radix-ui/react-id": "^1.1.0", "@radix-ui/react-primitive": "^2.0.2" }, "peerDependencies": { "react": "^18 || ^19 || ^19.0.0-rc", "react-dom": "^18 || ^19 || ^19.0.0-rc" } }, "sha512-Vsv7kFaXm+ptHDMZ7izaRsP70GgrW9NBNGswt9OZaVBLlE0SNpDq8eu/VGXyF9r7M0azK3Wy7OlYXsuyYLFzHg=="], + "code-block-writer": ["code-block-writer@13.0.3", "", {}, "sha512-Oofo0pq3IKnsFtuHqSF7TqBfr71aeyZDVJ0HpmqB7FBM2qEigL0iPONSCZSO9pE9dZTAxANe5XHG9Uy0YMv8cg=="], + "color": ["color@4.2.3", "", { "dependencies": { "color-convert": "^2.0.1", "color-string": "^1.9.0" } }, "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A=="], "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], @@ -2638,11 +2712,13 @@ "color-string": ["color-string@1.9.1", "", { "dependencies": { "color-name": "^1.0.0", "simple-swizzle": "^0.2.2" } }, "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg=="], + "colors": ["colors@1.4.0", "", {}, "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA=="], + "combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="], "comma-separated-tokens": ["comma-separated-tokens@2.0.3", "", {}, "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg=="], - "commander": ["commander@12.1.0", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="], + "commander": ["commander@5.1.0", "", {}, "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg=="], "comment-json": ["comment-json@4.6.2", "", { "dependencies": { "array-timsort": "^1.0.3", "esprima": "^4.0.1" } }, "sha512-R2rze/hDX30uul4NZoIZ76ImSJLFxn/1/ZxtKC1L77y2X1k+yYu1joKbAtMA2Fg3hZrTOiw0I5mwVMo0cf250w=="], @@ -2668,6 +2744,8 @@ "console-control-strings": ["console-control-strings@1.1.0", "", {}, "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ=="], + "constantinople": ["constantinople@4.0.1", "", { "dependencies": { "@babel/parser": "^7.6.0", "@babel/types": "^7.6.1" } }, "sha512-vCrqcSIq4//Gx74TXXCGnHpulY1dskqLTFGDmhrGxzeXL8lF8kvXv6mpNWlJj1uD4DW23D4ljAqbY4RRaaUZIw=="], + "content-disposition": ["content-disposition@1.1.0", "", {}, "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g=="], "content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="], @@ -2798,6 +2876,8 @@ "dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="], + "destr": ["destr@2.0.5", "", {}, "sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA=="], + "destroy": ["destroy@1.2.0", "", {}, "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="], "detect-indent": ["detect-indent@7.0.2", "", {}, "sha512-y+8xyqdGLL+6sh0tVeHcfP/QDd8gUgbasolJJpY7NgeQGSZ739bDtSiaiDgtoicy+mtYB81dKLxO9xRhCyIB3A=="], @@ -2846,6 +2926,8 @@ "doctrine": ["doctrine@3.0.0", "", { "dependencies": { "esutils": "^2.0.2" } }, "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w=="], + "doctypes": ["doctypes@1.1.0", "", {}, "sha512-LLBi6pEqS6Do3EKQ3J0NqHWV5hhb78Pi8vvESYwyOy2c31ZEZVdtitdzsQsKb7878PEERhzUk0ftqGhG6Mz+pQ=="], + "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], @@ -2928,7 +3010,7 @@ "es-to-primitive": ["es-to-primitive@1.3.0", "", { "dependencies": { "is-callable": "^1.2.7", "is-date-object": "^1.0.5", "is-symbol": "^1.0.4" } }, "sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g=="], - "es-toolkit": ["es-toolkit@1.46.1", "", {}, "sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ=="], + "es-toolkit": ["es-toolkit@1.47.0", "", {}, "sha512-n1GuoD0WEQZMBk5tttoZSqwgyLx01oqa5XsBmCHwPyNe1S9jPBEmtR2pSgp2kJuWE3ciFZ6yRHmY4pM4C3OOkw=="], "esbuild": ["esbuild@0.27.3", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.27.3", "@esbuild/android-arm": "0.27.3", "@esbuild/android-arm64": "0.27.3", "@esbuild/android-x64": "0.27.3", "@esbuild/darwin-arm64": "0.27.3", "@esbuild/darwin-x64": "0.27.3", "@esbuild/freebsd-arm64": "0.27.3", "@esbuild/freebsd-x64": "0.27.3", "@esbuild/linux-arm": "0.27.3", "@esbuild/linux-arm64": "0.27.3", "@esbuild/linux-ia32": "0.27.3", "@esbuild/linux-loong64": "0.27.3", "@esbuild/linux-mips64el": "0.27.3", "@esbuild/linux-ppc64": "0.27.3", "@esbuild/linux-riscv64": "0.27.3", "@esbuild/linux-s390x": "0.27.3", "@esbuild/linux-x64": "0.27.3", "@esbuild/netbsd-arm64": "0.27.3", "@esbuild/netbsd-x64": "0.27.3", "@esbuild/openbsd-arm64": "0.27.3", "@esbuild/openbsd-x64": "0.27.3", "@esbuild/openharmony-arm64": "0.27.3", "@esbuild/sunos-x64": "0.27.3", "@esbuild/win32-arm64": "0.27.3", "@esbuild/win32-ia32": "0.27.3", "@esbuild/win32-x64": "0.27.3" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg=="], @@ -3006,6 +3088,8 @@ "exact-mirror": ["exact-mirror@0.2.7", "", { "peerDependencies": { "@sinclair/typebox": "^0.34.15" }, "optionalPeers": ["@sinclair/typebox"] }, "sha512-+MeEmDcLA4o/vjK2zujgk+1VTxPR4hdp23qLqkWfStbECtAq9gmsvQa3LW6z/0GXZyHJobrCnmy1cdeE7BjsYg=="], + "execa": ["execa@4.1.0", "", { "dependencies": { "cross-spawn": "^7.0.0", "get-stream": "^5.0.0", "human-signals": "^1.1.1", "is-stream": "^2.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^4.0.0", "onetime": "^5.1.0", "signal-exit": "^3.0.2", "strip-final-newline": "^2.0.0" } }, "sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA=="], + "exit-hook": ["exit-hook@2.2.1", "", {}, "sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw=="], "expect-type": ["expect-type@1.3.0", "", {}, "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA=="], @@ -3238,7 +3322,7 @@ "get-stdin": ["get-stdin@4.0.1", "", {}, "sha512-F5aQMywwJ2n85s4hJPTT9RPxGmubonuB10MNYo17/xph174n2MIR33HRguhzVag10O/npM7SPk73LMZNP+FaWw=="], - "get-stream": ["get-stream@9.0.1", "", { "dependencies": { "@sec-ant/readable-stream": "^0.4.1", "is-stream": "^4.0.1" } }, "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA=="], + "get-stream": ["get-stream@5.2.0", "", { "dependencies": { "pump": "^3.0.0" } }, "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA=="], "get-symbol-description": ["get-symbol-description@1.1.0", "", { "dependencies": { "call-bound": "^1.0.3", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6" } }, "sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg=="], @@ -3336,6 +3420,8 @@ "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], + "human-signals": ["human-signals@1.1.1", "", {}, "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw=="], + "hyphenate-style-name": ["hyphenate-style-name@1.1.0", "", {}, "sha512-WDC/ui2VVRrz3jOVi+XtjqkDjiVjTtFaAGiW37k6b+ohyQ5wYDOGkvCZa8+H0nx3gyvv0+BST9xuOgIyGQ00gw=="], "i": ["i@0.3.7", "", {}, "sha512-FYz4wlXgkQwIPqhzC5TdNMLSE5+GS1IIDJZY/1ZiEPCT2S3COUVZeT5OW4BmW4r5LHLQuOosSwsvnroG9GR59Q=="], @@ -3410,6 +3496,8 @@ "is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="], + "is-expression": ["is-expression@4.0.0", "", { "dependencies": { "acorn": "^7.1.1", "object-assign": "^4.1.1" } }, "sha512-zMIXX63sxzG3XrkHkrAPvm/OVZVSCPNkwMHU8oTX7/U3AL78I0QXCEICXUM13BIa8TYGZ68PiTKfQz3yaTNr4A=="], + "is-extendable": ["is-extendable@0.1.1", "", {}, "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw=="], "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], @@ -3530,12 +3618,18 @@ "js-library-detector": ["js-library-detector@6.7.0", "", {}, "sha512-c80Qupofp43y4cJ7+8TTDN/AsDwLi5oOm/plBrWI+iQt485vKXCco+yVmOwEgdo9VOdsYTuV0UlTeetVPTriXA=="], + "js-stringify": ["js-stringify@1.0.2", "", {}, "sha512-rtS5ATOo2Q5k1G+DADISilDA6lv79zIiwFd6CcjuIxGKLFm5C+RLImRscVap9k55i+MOZwgliw+NejvkLuGD5g=="], + "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], "js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="], "jsc-safe-url": ["jsc-safe-url@0.2.4", "", {}, "sha512-0wM3YBWtYePOjfyXQH5MWQ8H7sdk5EXSwZvmSLKk2RboVQ2Bu239jycHDz5J/8Blf3K0Qnoy2b6xD+z10MFB+Q=="], + "jscpd": ["jscpd@4.2.4", "", { "dependencies": { "@jscpd/badge-reporter": "4.2.4", "@jscpd/core": "4.2.4", "@jscpd/finder": "4.2.4", "@jscpd/html-reporter": "4.2.4", "@jscpd/tokenizer": "4.2.4", "colors": "^1.4.0", "commander": "^5.0.0", "fs-extra": "^11.2.0", "jscpd-sarif-reporter": "4.2.4" }, "bin": { "jscpd": "bin/jscpd" } }, "sha512-PSo2U0G8OxULayGyQMv7T/0ZQ+c3PPltdMOz/57v9Xnmq5xSIhh4cnZ0oYZPKqejy10aFwAbMVxqAlo24+PQ3g=="], + + "jscpd-sarif-reporter": ["jscpd-sarif-reporter@4.2.4", "", { "dependencies": { "colors": "^1.4.0", "fs-extra": "^11.2.0", "node-sarif-builder": "^3.4.0" } }, "sha512-JtX79kFSyAhqJh5TdLUcvtYJtJd1F8UW8b4Miaga+EIgUn2/nR0N2zWL9mH5cRXgbzLuQbbsw9kReUVIECApwQ=="], + "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], "jsftp": ["jsftp@2.1.3", "", { "dependencies": { "debug": "^3.1.0", "ftp-response-parser": "^1.0.1", "once": "^1.4.0", "parse-listing": "^1.1.3", "stream-combiner": "^0.2.2", "unorm": "^1.4.1" } }, "sha512-r79EVB8jaNAZbq8hvanL8e8JGu2ZNr2bXdHC4ZdQhRImpSPpnWwm5DYVzQ5QxJmtGtKhNNuvqGgbNaFl604fEQ=="], @@ -3558,6 +3652,8 @@ "jsonfile": ["jsonfile@6.2.1", "", { "dependencies": { "universalify": "^2.0.0" }, "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q=="], + "jstransformer": ["jstransformer@1.0.0", "", { "dependencies": { "is-promise": "^2.0.0", "promise": "^7.0.1" } }, "sha512-C9YK3Rf8q6VAPDCCU9fnqo3mAfOH6vUGnMcP4AQAYIEpWtfGLpwOTmZ+igtdK5y+VvI2n3CyYSzy4Qh34eq24A=="], + "jsx-ast-utils": ["jsx-ast-utils@3.3.5", "", { "dependencies": { "array-includes": "^3.1.6", "array.prototype.flat": "^1.3.1", "object.assign": "^4.1.4", "object.values": "^1.1.6" } }, "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ=="], "jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], @@ -3702,7 +3798,7 @@ "markdown-it": ["markdown-it@13.0.2", "", { "dependencies": { "argparse": "^2.0.1", "entities": "~3.0.1", "linkify-it": "^4.0.1", "mdurl": "^1.0.1", "uc.micro": "^1.0.5" }, "bin": { "markdown-it": "bin/markdown-it.js" } }, "sha512-FtwnEuuK+2yVU7goGn/MJ0WBZMM9ZPgU9spqlFs7/A/pDIUNSOQZhUgOqYCficIuR2QaFnrt8LHqBWsbTAoI5w=="], - "markdown-table": ["markdown-table@3.0.4", "", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="], + "markdown-table": ["markdown-table@2.0.0", "", { "dependencies": { "repeat-string": "^1.0.0" } }, "sha512-Ezda85ToJUBhM6WGaG6veasyym+Tbs3cMAw/ZhOPqXiYsr0jgocBV3j3nx+4lk47plLlIqjwuTm/ywVI+zjJ/A=="], "marky": ["marky@1.3.0", "", {}, "sha512-ocnPZQLNpvbedwTy9kNrQEsknEfgvcLMvOtz3sFeWApDq1MXH1TqkCIx58xlpESsfwQOnuBO9beyQuNGzVvuhQ=="], @@ -3852,7 +3948,7 @@ "mimetext": ["mimetext@3.0.28", "", { "dependencies": { "@babel/runtime": "^7.26.0", "@babel/runtime-corejs3": "^7.26.0", "js-base64": "^3.7.7", "mime-types": "^2.1.35" } }, "sha512-eQXpbNrtxLCjUtiVbR/qR09dbPgZ2o+KR1uA7QKqGhbn8QV7HIL16mXXsobBL4/8TqoYh1us31kfz+dNfCev9g=="], - "mimic-fn": ["mimic-fn@1.2.0", "", {}, "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ=="], + "mimic-fn": ["mimic-fn@2.1.0", "", {}, "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="], "miniflare": ["miniflare@4.20250906.0", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "acorn": "8.14.0", "acorn-walk": "8.3.2", "exit-hook": "2.2.1", "glob-to-regexp": "0.4.1", "sharp": "^0.33.5", "stoppable": "1.1.0", "undici": "^7.10.0", "workerd": "1.20250906.0", "ws": "8.18.0", "youch": "4.1.0-beta.10", "zod": "3.22.3" }, "bin": { "miniflare": "bootstrap.js" } }, "sha512-T/RWn1sa0ien80s6NjU+Un/tj12gR6wqScZoiLeMJDD4/fK0UXfnbWXJDubnUED8Xjm7RPQ5ESYdE+mhPmMtuQ=="], @@ -3920,6 +4016,8 @@ "node-releases": ["node-releases@2.0.44", "", {}, "sha512-5WUyunoPMsvvEhS8AxHtRzP+oA8UCkJ7YRxatWKjngndhDGLiqEVAQKWjFAiAiuL8zMRGzGSJxFnLetoa43qGQ=="], + "node-sarif-builder": ["node-sarif-builder@3.4.0", "", { "dependencies": { "@types/sarif": "^2.1.7", "fs-extra": "^11.1.1" } }, "sha512-tGnJW6OKRii9u/b2WiUViTJS+h7Apxx17qsMUjsUeNDiMMX5ZFf8F8Fcz7PAQ6omvOxHZtvDTmOYKJQwmfpjeg=="], + "node-source-walk": ["node-source-walk@7.0.2", "", { "dependencies": { "@babel/parser": "^7.29.0" } }, "sha512-71kFFjYaSshDTA8/a2HiTYPLdASWjLJxUyJxGE+ffxU+KhxSBtM9kiLUX+R2yooFdSFKMFpi4n3PFtDy6qXv8A=="], "nodemailer": ["nodemailer@6.10.1", "", {}, "sha512-Z+iLaBGVaSjbIzQ4pX6XV41HrooLsQ10ZWPUehGmuantvzWoDVBnmsdUcOIDM1t+yPor5pDhVlDESgOMEGxhHA=="], @@ -3930,6 +4028,8 @@ "npm-package-arg": ["npm-package-arg@11.0.3", "", { "dependencies": { "hosted-git-info": "^7.0.0", "proc-log": "^4.0.0", "semver": "^7.3.5", "validate-npm-package-name": "^5.0.0" } }, "sha512-sHGJy8sOC1YraBywpzQlIKBE4pBbGbiF95U6Auspzyem956E0+FtDtsx1ZxlOJkQCZ1AFXAY/yuvtFYrOxF+Bw=="], + "npm-run-path": ["npm-run-path@4.0.1", "", { "dependencies": { "path-key": "^3.0.0" } }, "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw=="], + "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], "nullthrows": ["nullthrows@1.1.1", "", {}, "sha512-2vPPEi+Z7WqML2jZYddDIfy5Dqb0r2fze2zTxNNknZaFpVHU3mFB3R+DWeJWGVx0ecvttSGlJTI+WG+8Z4cDWw=="], @@ -3968,7 +4068,7 @@ "one-time": ["one-time@1.0.0", "", { "dependencies": { "fn.name": "1.x.x" } }, "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g=="], - "onetime": ["onetime@2.0.1", "", { "dependencies": { "mimic-fn": "^1.0.0" } }, "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ=="], + "onetime": ["onetime@5.1.2", "", { "dependencies": { "mimic-fn": "^2.1.0" } }, "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg=="], "open": ["open@7.4.2", "", { "dependencies": { "is-docker": "^2.0.0", "is-wsl": "^2.1.1" } }, "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q=="], @@ -4040,6 +4140,8 @@ "path": ["path@0.12.7", "", { "dependencies": { "process": "^0.11.1", "util": "^0.10.3" } }, "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q=="], + "path-browserify": ["path-browserify@1.0.1", "", {}, "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g=="], + "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="], "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], @@ -4170,6 +4272,30 @@ "proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="], + "pug": ["pug@3.0.4", "", { "dependencies": { "pug-code-gen": "^3.0.4", "pug-filters": "^4.0.0", "pug-lexer": "^5.0.1", "pug-linker": "^4.0.0", "pug-load": "^3.0.0", "pug-parser": "^6.0.0", "pug-runtime": "^3.0.1", "pug-strip-comments": "^2.0.0" } }, "sha512-kFfq5mMzrS7+wrl5pLJzZEzemx34OQ0w4SARfhy/3yxTlhbstsudDwJzhf1hP02yHzbjoVMSXUj/Sz6RNfMyXg=="], + + "pug-attrs": ["pug-attrs@3.0.0", "", { "dependencies": { "constantinople": "^4.0.1", "js-stringify": "^1.0.2", "pug-runtime": "^3.0.0" } }, "sha512-azINV9dUtzPMFQktvTXciNAfAuVh/L/JCl0vtPCwvOA21uZrC08K/UnmrL+SXGEVc1FwzjW62+xw5S/uaLj6cA=="], + + "pug-code-gen": ["pug-code-gen@3.0.4", "", { "dependencies": { "constantinople": "^4.0.1", "doctypes": "^1.1.0", "js-stringify": "^1.0.2", "pug-attrs": "^3.0.0", "pug-error": "^2.1.0", "pug-runtime": "^3.0.1", "void-elements": "^3.1.0", "with": "^7.0.0" } }, "sha512-6okWYIKdasTyXICyEtvobmTZAVX57JkzgzIi4iRJlin8kmhG+Xry2dsus+Mun/nGCn6F2U49haHI5mkELXB14g=="], + + "pug-error": ["pug-error@2.1.0", "", {}, "sha512-lv7sU9e5Jk8IeUheHata6/UThZ7RK2jnaaNztxfPYUY+VxZyk/ePVaNZ/vwmH8WqGvDz3LrNYt/+gA55NDg6Pg=="], + + "pug-filters": ["pug-filters@4.0.0", "", { "dependencies": { "constantinople": "^4.0.1", "jstransformer": "1.0.0", "pug-error": "^2.0.0", "pug-walk": "^2.0.0", "resolve": "^1.15.1" } }, "sha512-yeNFtq5Yxmfz0f9z2rMXGw/8/4i1cCFecw/Q7+D0V2DdtII5UvqE12VaZ2AY7ri6o5RNXiweGH79OCq+2RQU4A=="], + + "pug-lexer": ["pug-lexer@5.0.1", "", { "dependencies": { "character-parser": "^2.2.0", "is-expression": "^4.0.0", "pug-error": "^2.0.0" } }, "sha512-0I6C62+keXlZPZkOJeVam9aBLVP2EnbeDw3An+k0/QlqdwH6rv8284nko14Na7c0TtqtogfWXcRoFE4O4Ff20w=="], + + "pug-linker": ["pug-linker@4.0.0", "", { "dependencies": { "pug-error": "^2.0.0", "pug-walk": "^2.0.0" } }, "sha512-gjD1yzp0yxbQqnzBAdlhbgoJL5qIFJw78juN1NpTLt/mfPJ5VgC4BvkoD3G23qKzJtIIXBbcCt6FioLSFLOHdw=="], + + "pug-load": ["pug-load@3.0.0", "", { "dependencies": { "object-assign": "^4.1.1", "pug-walk": "^2.0.0" } }, "sha512-OCjTEnhLWZBvS4zni/WUMjH2YSUosnsmjGBB1An7CsKQarYSWQ0GCVyd4eQPMFJqZ8w9xgs01QdiZXKVjk92EQ=="], + + "pug-parser": ["pug-parser@6.0.0", "", { "dependencies": { "pug-error": "^2.0.0", "token-stream": "1.0.0" } }, "sha512-ukiYM/9cH6Cml+AOl5kETtM9NR3WulyVP2y4HOU45DyMim1IeP/OOiyEWRr6qk5I5klpsBnbuHpwKmTx6WURnw=="], + + "pug-runtime": ["pug-runtime@3.0.1", "", {}, "sha512-L50zbvrQ35TkpHwv0G6aLSuueDRwc/97XdY8kL3tOT0FmhgG7UypU3VztfV/LATAvmUfYi4wNxSajhSAeNN+Kg=="], + + "pug-strip-comments": ["pug-strip-comments@2.0.0", "", { "dependencies": { "pug-error": "^2.0.0" } }, "sha512-zo8DsDpH7eTkPHCXFeAk1xZXJbyoTfdPlNR0bK7rpOMuhBYb0f5qUVCO1xlsitYd3w5FQTK7zpNVKb3rZoUrrQ=="], + + "pug-walk": ["pug-walk@2.0.0", "", {}, "sha512-yYELe9Q5q9IQhuvqsZNwA5hfPkMJ8u92bQLIMcsMxf/VADjNtEYptU+inlufAFYcWdHlwNfZOEnOOQrZrcyJCQ=="], + "pump": ["pump@3.0.4", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA=="], "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], @@ -4188,6 +4314,8 @@ "radash": ["radash@12.1.1", "", {}, "sha512-h36JMxKRqrAxVD8201FrCpyeNuUY9Y5zZwujr20fFO77tpUtGa6EZzfKw/3WaiBX95fq7+MpsuMLNdSnORAwSA=="], + "radashi": ["radashi@12.9.1", "", {}, "sha512-HCvrL1Ag7qnyH11UiSWQaEIiizJ7kldHjBw63aELoum7C8nQrSLqotLDuKKvoRPtO0w8azCzUQcL3yrU3lBksw=="], + "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="], "raw-body": ["raw-body@3.0.2", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" } }, "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA=="], @@ -4322,6 +4450,10 @@ "remark-stringify": ["remark-stringify@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-to-markdown": "^2.0.0", "unified": "^11.0.0" } }, "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw=="], + "remeda": ["remeda@2.37.0", "", {}, "sha512-wN6BXWua0t4o7vDamqc27J3VRxnokG9cDezsFN2nOnt2JD/IkJQHTYqM6UvmEctAZETAoviwEFQZJO3kZ4Ohew=="], + + "repeat-string": ["repeat-string@1.6.1", "", {}, "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w=="], + "repeating": ["repeating@2.0.1", "", { "dependencies": { "is-finite": "^1.0.0" } }, "sha512-ZqtSMuVybkISo2OWvqvm7iHSWngvdaW3IpsT9/uP8v4gMi591LY6h35wdOfvQdWCKFWZWm2Y1Opp4kV7vQKT6A=="], "require-directory": ["require-directory@2.1.1", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="], @@ -4474,6 +4606,8 @@ "space-separated-tokens": ["space-separated-tokens@2.0.2", "", {}, "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q=="], + "spark-md5": ["spark-md5@3.0.2", "", {}, "sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw=="], + "spawn-command": ["spawn-command@0.0.2", "", {}, "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ=="], "spdx-correct": ["spdx-correct@3.2.0", "", { "dependencies": { "spdx-expression-parse": "^3.0.0", "spdx-license-ids": "^3.0.0" } }, "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA=="], @@ -4552,6 +4686,8 @@ "strip-bom-string": ["strip-bom-string@1.0.0", "", {}, "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g=="], + "strip-final-newline": ["strip-final-newline@2.0.0", "", {}, "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA=="], + "strip-indent": ["strip-indent@1.0.1", "", { "dependencies": { "get-stdin": "^4.0.1" }, "bin": { "strip-indent": "cli.js" } }, "sha512-I5iQq6aFMM62fBEAIB/hXzwJD6EEZ0xEGCX2t7oXqaKPIRgt4WruAQ285BISgdkP+HLGWyeGmNJcpIwFeRYRUA=="], "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], @@ -4648,6 +4784,8 @@ "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], + "token-stream": ["token-stream@1.0.0", "", {}, "sha512-VSsyNPPW74RpHwR8Fc21uubwHY7wMDeJLys2IX5zJNih+OnAnaifKHo+1LHT7DAdloQ7apeaaWg8l7qnf/TnEg=="], + "token-types": ["token-types@6.1.2", "", { "dependencies": { "@borewit/text-codec": "^0.2.1", "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" } }, "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww=="], "toqr": ["toqr@0.1.1", "", {}, "sha512-FWAPzCIHZHnrE/5/w9MPk0kK25hSQSH2IKhYh9PyjS3SG/+IEMvlwIHbhz+oF7xl54I+ueZlVnMjyzdSwLmAwA=="], @@ -4674,6 +4812,8 @@ "ts-interface-checker": ["ts-interface-checker@0.1.13", "", {}, "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA=="], + "ts-morph": ["ts-morph@28.0.0", "", { "dependencies": { "@ts-morph/common": "~0.29.0", "code-block-writer": "^13.0.3" } }, "sha512-Wp3tnZ2bzwxyTZMtgWVzXDfm7lB1Drz+y9DmmYH/L702PQhPyVrp3pkou3yIz4qjS14GY9kcpmLiOOMvl8oG1g=="], + "tsconfck": ["tsconfck@3.1.6", "", { "peerDependencies": { "typescript": "^5.0.0" }, "optionalPeers": ["typescript"], "bin": { "tsconfck": "bin/tsconfck.js" } }, "sha512-ks6Vjr/jEw0P1gmOVwutM3B7fWxoWBL2KRDb1JfqGVawBmO5UsvmWOQFGHBPl5yxYz4eERr19E6L7NMv+Fej4w=="], "tsconfig-paths": ["tsconfig-paths@3.15.0", "", { "dependencies": { "@types/json5": "^0.0.29", "json5": "^1.0.2", "minimist": "^1.2.6", "strip-bom": "^3.0.0" } }, "sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg=="], @@ -4854,6 +4994,8 @@ "winston-transport": ["winston-transport@4.9.0", "", { "dependencies": { "logform": "^2.7.0", "readable-stream": "^3.6.2", "triple-beam": "^1.3.0" } }, "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A=="], + "with": ["with@7.0.2", "", { "dependencies": { "@babel/parser": "^7.9.6", "@babel/types": "^7.9.6", "assert-never": "^1.2.1", "babel-walk": "3.0.0-canary-5" } }, "sha512-RNGKj82nUPg3g5ygxkQl0R937xLyho1J24ItRCBTr/m1YnZkzJy1hUiHUJrc/VlsDQzsCnInEGSg3bci0Lmd4w=="], + "word-wrap": ["word-wrap@1.2.5", "", {}, "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA=="], "workerd": ["workerd@1.20260515.1", "", { "optionalDependencies": { "@cloudflare/workerd-darwin-64": "1.20260515.1", "@cloudflare/workerd-darwin-arm64": "1.20260515.1", "@cloudflare/workerd-linux-64": "1.20260515.1", "@cloudflare/workerd-linux-arm64": "1.20260515.1", "@cloudflare/workerd-windows-64": "1.20260515.1" }, "bin": { "workerd": "bin/workerd" } }, "sha512-MjKOJLcvU45xXedQowvuiHtJTxu4WTHYQeIlF7YmjuqhiI6dImTFxWCEoRQHiskztxuVSNEmdO7/0UfDu6OMnQ=="], @@ -4950,6 +5092,8 @@ "@appium/support/bplist-creator": ["bplist-creator@0.1.1", "", { "dependencies": { "stream-buffers": "2.2.x" } }, "sha512-Ese7052fdWrxp/vqSJkydgx/1MdBnNOCV2XVfbmdGWD2H6EYza+Q4pyYSuVSnCUD22hfI/BFI4jHaC3NLXLlJQ=="], + "@appium/support/get-stream": ["get-stream@9.0.1", "", { "dependencies": { "@sec-ant/readable-stream": "^0.4.1", "is-stream": "^4.0.1" } }, "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA=="], + "@appium/support/glob": ["glob@13.0.5", "", { "dependencies": { "minimatch": "^10.2.1", "minipass": "^7.1.2", "path-scurry": "^2.0.0" } }, "sha512-BzXxZg24Ibra1pbQ/zE7Kys4Ua1ks7Bn6pKLkVPZ9FZe4JQS6/Q7ef3LG1H+k7lUf5l4T3PLSyYyYJVYUvfgTw=="], "@appium/support/log-symbols": ["log-symbols@7.0.1", "", { "dependencies": { "is-unicode-supported": "^2.0.0", "yoctocolors": "^2.1.1" } }, "sha512-ja1E3yCr9i/0hmBVaM0bfwDjnGy8I/s6PP4DFp+yP+a+mrHO4Rm7DtmnqROTUkHIkqffC84YY7AeqX6oFk0WFg=="], @@ -5356,6 +5500,8 @@ "eslint-utils/eslint-visitor-keys": ["eslint-visitor-keys@1.3.0", "", {}, "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ=="], + "execa/signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], + "expo-modules-autolinking/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], "expo-modules-autolinking/commander": ["commander@7.2.0", "", {}, "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw=="], @@ -5378,8 +5524,6 @@ "external-editor/tmp": ["tmp@0.0.33", "", { "dependencies": { "os-tmpdir": "~1.0.2" } }, "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw=="], - "extract-zip/get-stream": ["get-stream@5.2.0", "", { "dependencies": { "pump": "^3.0.0" } }, "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA=="], - "extract-zip/yauzl": ["yauzl@2.10.0", "", { "dependencies": { "buffer-crc32": "~0.2.3", "fd-slicer": "~1.1.0" } }, "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g=="], "fast-glob/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], @@ -5398,8 +5542,6 @@ "ftp-response-parser/readable-stream": ["readable-stream@1.1.14", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.1", "isarray": "0.0.1", "string_decoder": "~0.10.x" } }, "sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ=="], - "get-stream/is-stream": ["is-stream@4.0.1", "", {}, "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A=="], - "get-uri/data-uri-to-buffer": ["data-uri-to-buffer@6.0.2", "", {}, "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw=="], "globals/type-fest": ["type-fest@0.20.2", "", {}, "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ=="], @@ -5422,6 +5564,10 @@ "inquirer/strip-ansi": ["strip-ansi@5.2.0", "", { "dependencies": { "ansi-regex": "^4.1.0" } }, "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA=="], + "is-expression/acorn": ["acorn@7.4.1", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A=="], + + "is-expression/object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], + "isomorphic-fetch/node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], "istanbul-lib-instrument/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], @@ -5444,6 +5590,10 @@ "jsftp/debug": ["debug@3.2.7", "", { "dependencies": { "ms": "^2.1.1" } }, "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ=="], + "jstransformer/is-promise": ["is-promise@2.2.2", "", {}, "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ=="], + + "jstransformer/promise": ["promise@7.3.1", "", { "dependencies": { "asap": "~2.0.3" } }, "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg=="], + "lazystream/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], "lighthouse/chrome-launcher": ["chrome-launcher@1.2.1", "", { "dependencies": { "@types/node": "*", "escape-string-regexp": "^4.0.0", "is-wsl": "^2.2.0", "lighthouse-logger": "^2.0.1" }, "bin": { "print-chrome-path": "bin/print-chrome-path.cjs" } }, "sha512-qmFR5PLMzHyuNJHwOloHPAHhbaNglkfeV/xDtt5b7xiFFyU1I+AZZX0PYseMuhenJSSirgxELYIbswcoc+5H4A=="], @@ -5480,6 +5630,8 @@ "mdast-util-find-and-replace/escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], + "mdast-util-gfm-table/markdown-table": ["markdown-table@3.0.4", "", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="], + "merge-options/is-plain-obj": ["is-plain-obj@2.1.0", "", {}, "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA=="], "method-override/debug": ["debug@3.1.0", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g=="], @@ -5536,6 +5688,8 @@ "postcss/nanoid": ["nanoid@3.3.12", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ=="], + "precinct/commander": ["commander@12.1.0", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="], + "pretty-format/ansi-styles": ["ansi-styles@5.2.0", "", {}, "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA=="], "pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], @@ -5546,12 +5700,16 @@ "proxy-agent/lru-cache": ["lru-cache@7.18.3", "", {}, "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="], + "pug-load/object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], + "rc/ini": ["ini@1.3.8", "", {}, "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="], "rc/strip-json-comments": ["strip-json-comments@2.0.1", "", {}, "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ=="], "react-devtools-core/ws": ["ws@7.5.10", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": "^5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ=="], + "react-native/commander": ["commander@12.1.0", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="], + "react-native/glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="], "react-native/ws": ["ws@7.5.10", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": "^5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ=="], @@ -5576,6 +5734,10 @@ "readdir-glob/minimatch": ["minimatch@5.1.9", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw=="], + "recharts/es-toolkit": ["es-toolkit@1.46.1", "", {}, "sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ=="], + + "restore-cursor/onetime": ["onetime@2.0.1", "", { "dependencies": { "mimic-fn": "^1.0.0" } }, "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ=="], + "restore-cursor/signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], "rimraf/glob": ["glob@13.0.6", "", { "dependencies": { "minimatch": "^10.2.2", "minipass": "^7.1.3", "path-scurry": "^2.0.2" } }, "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw=="], @@ -5668,6 +5830,8 @@ "@appium/support/axios/proxy-from-env": ["proxy-from-env@2.1.0", "", {}, "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA=="], + "@appium/support/get-stream/is-stream": ["is-stream@4.0.1", "", {}, "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A=="], + "@appium/support/log-symbols/is-unicode-supported": ["is-unicode-supported@2.1.0", "", {}, "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ=="], "@appium/support/plist/@xmldom/xmldom": ["@xmldom/xmldom@0.9.10", "", {}, "sha512-A9gOqLdi6cV4ibazAjcQufGj0B1y/vDqYrcuP6d/6x8P27gRS8643Dj9o1dEKtB6O7fwxb2FgBmJS2mX7gpvdw=="], @@ -6138,6 +6302,8 @@ "readdir-glob/minimatch/brace-expansion": ["brace-expansion@2.1.0", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w=="], + "restore-cursor/onetime/mimic-fn": ["mimic-fn@1.2.0", "", {}, "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ=="], + "tailwindcss/chokidar/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], "tailwindcss/chokidar/readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="], @@ -6494,8 +6660,6 @@ "agents/yargs/string-width/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], - "appium/ora/cli-cursor/restore-cursor/onetime": ["onetime@5.1.2", "", { "dependencies": { "mimic-fn": "^2.1.0" } }, "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg=="], - "appium/ora/cli-cursor/restore-cursor/signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], "archiver-utils/glob/jackspeak/@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], @@ -6534,8 +6698,6 @@ "@lhci/cli/yargs/find-up/locate-path/p-locate/p-limit": ["p-limit@2.3.0", "", { "dependencies": { "p-try": "^2.0.0" } }, "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w=="], - "appium/ora/cli-cursor/restore-cursor/onetime/mimic-fn": ["mimic-fn@2.1.0", "", {}, "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="], - "archiver-utils/glob/jackspeak/@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], "archiver-utils/glob/jackspeak/@isaacs/cliui/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], diff --git a/copilot-instructions.md b/copilot-instructions.md index 83baf28438..041d20b18e 100644 --- a/copilot-instructions.md +++ b/copilot-instructions.md @@ -88,10 +88,14 @@ cd apps/guides && bun dev - NEVER CANCEL: Takes ~1.4 seconds to start, set timeout to 30+ seconds - Runs on `http://localhost:3001` (if 3000 is taken) -#### **Testing** -- **API Unit Tests**: `bun test:api:unit` -- NEVER CANCEL: Takes ~5 seconds -- **Expo Tests**: `bun test:expo` -- runs Expo/React Native unit tests -- Tests run sequentially (`fileParallelism: false` in `packages/api/vitest.unit.config.ts`) to avoid database deadlocks +#### **Testing** — see `docs/testing.md` for the full policy +- **API Unit Tests**: `bun test:api:unit` -- Node env, deps mocked. Runtime varies with suite size +- **Expo Tests**: `bun test:expo` -- Vitest, pure-TS modules only (no native imports) +- **MCP Tests**: `bun test:mcp` +- **Scripts Tests**: `bun test:scripts` -- analyzer tests for the coverage ratchet and assertion lint +- **Coverage ratchet**: `bun check:coverage` -- compares each tracked workspace's `coverage/[unit/]coverage-summary.json` against `coverage-baselines.json` at the repo root. Fails CI on regression +- **Assertion-strength lint**: `bun lint:weak-assertions` -- catches assertion-free tests, bare `.toBeDefined()`, bare `.toHaveBeenCalled()`, oversized inline snapshots +- **Integration tests** (`bun run --cwd packages/api test`): require Docker (Postgres + neon-wsproxy), run sequentially (`fileParallelism: false`) to avoid database deadlocks. NOT coverage-counted (V8 unsupported under Cloudflare Workers pool) - Tests expect environment variables to be configured (see `.env.example`) #### **Build Commands** diff --git a/coverage-baselines.json b/coverage-baselines.json new file mode 100644 index 0000000000..2661502a58 --- /dev/null +++ b/coverage-baselines.json @@ -0,0 +1,67 @@ +{ + "_comment": "Coverage ratchet baselines — see docs/testing.md and scripts/lint/coverage-ratchet.ts. Each entry is a workspace's floor: a PR that drops any metric below the baseline (modulo epsilon) fails CI. CI on main auto-bumps these numbers upward via scripts/lint/coverage-baseline-update.ts after a green run.", + "_epsilon": 0.5, + "packages/api": { + "summaryPath": "packages/api/coverage/unit/coverage-summary.json", + "tier": "B", + "lines": 98.31, + "branches": 95.43, + "functions": 100, + "statements": 98.31, + "recordedAt": "2026-05-19" + }, + "apps/expo": { + "summaryPath": "apps/expo/coverage/unit/coverage-summary.json", + "tier": "C", + "lines": 97.36, + "branches": 95, + "functions": 100, + "statements": 97.36, + "recordedAt": "2026-05-19" + }, + "packages/mcp": { + "summaryPath": "packages/mcp/coverage/coverage-summary.json", + "tier": "B", + "lines": 98.87, + "branches": 98.38, + "functions": 100, + "statements": 98.87, + "recordedAt": "2026-05-19" + }, + "packages/analytics": { + "summaryPath": "packages/analytics/coverage/coverage-summary.json", + "tier": "B", + "lines": 84.48, + "branches": 83.33, + "functions": 89.13, + "statements": 84.48, + "recordedAt": "2026-05-19" + }, + "packages/overpass": { + "summaryPath": "packages/overpass/coverage/coverage-summary.json", + "tier": "A", + "lines": 100, + "branches": 95.65, + "functions": 100, + "statements": 100, + "recordedAt": "2026-05-19" + }, + "packages/units": { + "summaryPath": "packages/units/coverage/coverage-summary.json", + "tier": "A", + "lines": 100, + "branches": 100, + "functions": 100, + "statements": 100, + "recordedAt": "2026-05-19" + }, + "packages/utils": { + "summaryPath": "packages/utils/coverage/coverage-summary.json", + "tier": "A", + "lines": 100, + "branches": 100, + "functions": 100, + "statements": 100, + "recordedAt": "2026-05-31" + } +} diff --git a/docs/audits/2026-05-16-etl-audit.md b/docs/audits/2026-05-16-etl-audit.md new file mode 100644 index 0000000000..84f1d69449 --- /dev/null +++ b/docs/audits/2026-05-16-etl-audit.md @@ -0,0 +1,183 @@ +# ETL Pipeline Audit — 2026-05-16 + +## Summary + +The catalog ETL pipeline works end-to-end and has been hardened through a recent series of fixes (OOM, CPU-time budget, atomic counters, byte-range chunking), but it is not production-ready: chunking + a single shared `jobId` produces double-counted `totalProcessed`, mis-marks jobs `completed` after the first chunk finishes, and lacks any dead-letter / retry policy at the queue layer. Catastrophic per-message failures silently swallow errors in `processQueueBatch` (`try/catch` with `console.error` only), so the queue happily acks bad chunks. The retry endpoint also re-queues only the original object key, ignoring multi-chunk jobs entirely. + +**Top 3 risks**: (1) cross-chunk job-status race (any one chunk's completion marks the entire job `completed`), (2) consumer swallows errors so failed messages never retry/DLQ, (3) retry endpoint and stuck-job sweep are incompatible with byte-range chunking. + +## Architecture + +``` +POST /api/catalog/etl ── api-key auth + │ body: { filename, chunks[], source, scraperRevision } + ▼ +1. INSERT etl_jobs (status='running') +2. For each objectKey: R2.head() → split into 20 MB byte-range chunks +3. queueCatalogETL → ETL_QUEUE.sendBatch (one message per chunk, same jobId) + +ETL_QUEUE (max_batch_size=1, max_concurrency=1) + ▼ +processQueueBatch + ▼ +processCatalogETL ── per chunk + ├── R2.get(key, {range}) ── stream body + ├── if non-first chunk: GET first 4 KB → extract header → inject; skip partial row + ├── csv-parse stream w/ backpressure (parser.write returns false → wait 'drain') + ├── yield every 100 rows (setTimeout(1)) + ├── flush at BATCH_SIZE=100 rows: + │ valid → processValidItemsBatch → mergeBySku → embeddings → catalogService.upsert → updateEtlJobProgress + │ invalid → processLogsBatch → invalid_item_logs → updateEtlJobProgress + └── on success: UPDATE etl_jobs SET status='completed' ◀── PROBLEM with multi-chunk jobs + on throw: UPDATE etl_jobs SET status='failed' + rethrow +``` + +Counters are atomic per call (`COALESCE(col, 0) + n` in SQL). Job rows are not. + +## Findings + +### [P0] Multi-chunk jobs are marked `completed` after the first chunk finishes +- **What**: All chunks for a single source file share one `jobId`; each chunk independently sets `status='completed'` on success. +- **Where**: `packages/api/src/services/etl/processCatalogEtl.ts:188-191`; chunks created in `packages/api/src/routes/catalog/index.ts:182-200`. +- **Why it matters**: A 100 MB file becomes 5 chunks → 5 messages → the first message to finish flips the job to `completed`, even though 80% of rows haven't been processed yet. The dashboard, `success_rate`, and any downstream check ("is the catalog refresh done?") fire prematurely. Subsequent chunks continue to mutate `totalProcessed/totalValid/totalInvalid`, so the row reads as `completed` with rising counters. +- **Recommendation**: Track per-chunk completion. Two options: (a) add a `chunks_total` and `chunks_completed` column; only set `completed` when `chunks_completed = chunks_total`. (b) give each chunk its own jobId and group by a parent `batch_id`. Option (a) is the smaller change. + +### [P0] `processQueueBatch` swallows errors — failed chunks never retry or DLQ +- **What**: Per-message exceptions are caught and logged but never rethrown; CF Queues auto-acks every message in the batch. +- **Where**: `packages/api/src/services/etl/queue.ts:50-60`. +- **Why it matters**: A transient DB error, OpenAI 429, or R2 read failure permanently loses the chunk. The job is marked `failed` (good) but the message is acked (bad) — there is no retry, no dead-letter queue, and `wrangler.jsonc` does not declare a `dead_letter_queue` or `max_retries`. Combined with the multi-chunk issue above, a single failure can corrupt the job state while other chunks succeed and mark it `completed`. +- **Recommendation**: Rethrow in the catch (or call `message.retry()` explicitly on the specific message). Add `dead_letter_queue` and `max_retries: 3` to the ETL queue consumer in `wrangler.jsonc:76-82`. Process messages with `for...of` calling `message.ack()` / `message.retry()` explicitly so partial-batch semantics are correct even though `max_batch_size=1` today. + +### [P1] Retry endpoint discards multi-chunk structure +- **What**: `POST /admin/etl/:jobId/retry` re-queues exactly one chunk built from `v2/${source}/${filename}` with no chunking. +- **Where**: `packages/api/src/routes/admin/analytics/catalog.ts:434-450`. +- **Why it matters**: If the original job was chunked (20 MB+ files), retry blasts the entire file at one Worker invocation, blowing past the 300s CPU-time limit that prompted the chunking work in the first place. Result: retries of any large failed job silently re-fail. +- **Recommendation**: Re-run the same R2.head + chunk-split logic the producer endpoint uses (lines 182-200). Extract that to a shared helper so both call sites stay in sync. + +### [P1] Stuck-job sweep is wall-clock based and incompatible with serial chunked jobs +- **What**: `POST /admin/etl/reset-stuck` flips any job in `running` for >30 min to `failed`. +- **Where**: `packages/api/src/routes/admin/analytics/catalog.ts:384-403`. +- **Why it matters**: With `max_concurrency=1` and 20 MB chunks each consuming most of a 300s CPU budget, a 500 MB file produces 25 chunks at up to ~5 minutes each → comfortably past 30 minutes. Healthy long jobs will be marked `failed`. The trigger should be "no progress for N minutes" (e.g., `totalProcessed` unchanged), not "started >30 min ago". +- **Recommendation**: Add `lastProgressAt` updated on each `updateEtlJobProgress` call; sweep on `lastProgressAt < now - 15min`. Or check `completedAt IS NULL AND startedAt < now - 2h` for the absolute floor. + +### [P1] First-chunk header injection assumes the first 4 KB contains a complete header +- **What**: For non-first chunks, the parser fetches `bytes 0-4095` and uses `headerText.split('\n')[0]` as the header row. +- **Where**: `packages/api/src/services/etl/processCatalogEtl.ts:53-58`. +- **Why it matters**: If the header row exceeds 4 KB (wide CSVs with 30+ columns and long names — possible here given the catalog schema has 25+ fields), `split('\n')[0]` returns a *truncated* header, so `fieldMap` silently maps the last column wrong. There is also no validation that the slice actually contained a newline before `byteEnd=4095`. +- **Recommendation**: Loop the range request (or use a streaming `until newline` reader). At minimum, throw if no `\n` appears in the first 4 KB so the failure is loud, not silent. + +### [P1] Partial-row skip can drop a valid full row when chunk boundary lands on a newline +- **What**: `skipPartialRow` discards everything up to and including the first `\n` after `byteStart`. If `byteStart` happens to be the first byte *after* a newline (i.e., the previous chunk's last byte is `\n`), the producing chunk processed the full row, and this chunk correctly starts on a row boundary — but the skip logic still throws away the first whole row. +- **Where**: `packages/api/src/services/etl/processCatalogEtl.ts:95-108`. +- **Why it matters**: Off-by-one row drop at every chunk boundary in worst case (data loss, not just dup). For 25-chunk file → potentially 24 lost catalog items. No test covers the boundary-aligned case. +- **Recommendation**: When splitting chunks at line 195-196 of `routes/catalog/index.ts`, do not split on arbitrary 20 MB offsets — peek at R2 with a short range request and align `byteEnd` to a newline so the skip logic is unnecessary, *or* skip only when the previous byte (range `byteStart-1`) was non-newline. + +### [P1] CSV row spanning chunk boundary is never reassembled +- **What**: A row beginning before `byteEnd` and ending after will be cut in half. The producing chunk parses a truncated row (likely fails validation); the next chunk discards the tail. +- **Where**: `packages/api/src/services/etl/processCatalogEtl.ts:95-108` (skip logic), `routes/catalog/index.ts:182-200` (chunk creation). +- **Why it matters**: Every chunk boundary loses (or invalidates) one row. Symptom would be `totalInvalid` rising by ~N per N-chunk job, with field-shaped errors. Severity depends on row width vs 20 MB. +- **Recommendation**: Same as above — align chunk boundaries to row boundaries in the producer. Alternatively, the producing chunk should fetch ~64 KB beyond `byteEnd` to complete its final row, and the next chunk skip logic stays. + +### [P2] `console.log`/`console.error` only — no structured logging, no Sentry +- **What**: Every log uses `console.log` with emoji prefixes; no Sentry integration in ETL paths despite Sentry being a documented monitoring tool. +- **Where**: All ETL files; verified by `grep -rn "Sentry|captureException" packages/api/src/services/etl/` → no results. Same applies to `packages/api/src/`. +- **Why it matters**: A stuck job cannot be debugged without paging through CF Workers logs by hand. No correlation IDs (other than jobId), no per-chunk structured fields (`byteStart`, `rowsProcessed`, `elapsed_ms`), no error categorization. Failures in `processLogsBatch` are caught and `console.error`-ed without rethrow (`packages/api/src/services/etl/processLogsBatch.ts:25-27`) — invalid logs can fail to write and nobody knows. +- **Recommendation**: Add a thin logger (`logger.info({ jobId, chunk: { byteStart, byteEnd }, event: 'chunk_start' })`). Call `Sentry.captureException(err, { tags: { jobId, objectKey } })` in the `processCatalogETL` catch block. + +### [P2] `processLogsBatch` swallows DB errors silently +- **What**: Catch logs to console and returns normally — caller has no idea logs were dropped. +- **Where**: `packages/api/src/services/etl/processLogsBatch.ts:25-27`. +- **Why it matters**: Invalid-item logs are the *only* forensic record of what failed validation. If the INSERT fails (Neon hiccup, payload size, FK violation), we lose visibility forever. The `updateEtlJobProgress` call is also inside the try, so `totalInvalid`/`totalProcessed` will be undercounted. +- **Recommendation**: Rethrow. Let the outer ETL catch flip the job to `failed` — the alternative is silent data quality erosion. + +### [P2] Embedding failure path silently drops embeddings without marking it +- **What**: When `generateManyEmbeddings` throws, items are upserted with `embedding=undefined` (i.e., NULL) but the job still reports as fully successful. +- **Where**: `packages/api/src/services/etl/processValidItemsBatch.ts:52-63`. +- **Why it matters**: No metric distinguishes "successful with embeddings" from "successful but degraded". The `/admin/embeddings` route reports coverage but cannot attribute the drop to a specific job. A backfill is required to recover, and there is no automatic re-queue. +- **Recommendation**: Add a `totalEmbeddingFailures` column on `etl_jobs`, increment it in the fallback path, and surface in the admin dashboard. Optionally enqueue the affected SKUs into `EMBEDDINGS_QUEUE` from the fallback for automatic backfill. + +### [P2] `parser.end()` is called inside a fire-and-forget IIFE — errors are unhandled +- **What**: The async writer is invoked as `(async () => { ... })()` with no `.catch()`. Any stream read error or `parser.write` throw becomes an unhandled rejection. +- **Where**: `packages/api/src/services/etl/processCatalogEtl.ts:89-117`. +- **Why it matters**: In CF Workers, unhandled rejections can terminate the isolate. More commonly the outer `for await (const record of parser)` loop will just hang on a stalled parser if the writer rejected. The job will sit in `running` until the stuck-job sweep notices. +- **Recommendation**: Wrap in an explicit promise: `const writerPromise = (async () => { ... })().catch(err => parser.destroy(err));` and `await writerPromise` after the `for await` loop. Surface the error to the outer catch. + +### [P2] `setTimeout(resolve, 1)` every 100 rows is a fragile yield mechanism +- **What**: Used to yield to event loop / give GC a chance. +- **Where**: `packages/api/src/services/etl/processCatalogEtl.ts:120`. +- **Why it matters**: `setTimeout` consumes wall-clock budget. Workers have a 30s wall-clock per invocation (separate from `cpu_ms`). At 1ms × 600 yields per 60k-row chunk = 0.6s — fine today, but the comment mentions a previous "per-row yield hits the CF Worker wall-clock limit". The thresholds are tightly coupled and undocumented. +- **Recommendation**: Replace with `await scheduler.yield()` (CF supports it) or `await new Promise(setImmediate)`-equivalent. Add a unit test that verifies a 100k-row CSV completes within wall-clock. + +### [P2] `BATCH_SIZE = 100` is exported but reads inconsistent with comment/runtime +- **What**: `processCatalogEtl.ts:13` exports `BATCH_SIZE = 100`. The catalog OpenAI embedding API supports 1000+ per call, so this is conservative; meanwhile the queue's `batchSize` for `sendBatch` is hard-coded at 100 (`queue.ts:17`) for an unrelated reason (max batch size from CF). Reusing the symbol `100` for two different concepts is fragile. +- **Where**: `processCatalogEtl.ts:13`, `queue.ts:17`. +- **Recommendation**: Rename to `ITEM_FLUSH_BATCH_SIZE` and `CF_QUEUE_BATCH_SIZE`, hoist both to a shared constants file. + +### [P3] `mergeItemsBySku` logs change diff on every merge — unbounded console output +- **What**: Logs a `🔄 Merged SKU` line for every SKU collision with every changed field. +- **Where**: `packages/api/src/services/etl/mergeItemsBySku.ts:34-48`. +- **Why it matters**: On a 500 MB CSV with many duplicate SKUs across chunks, this can produce millions of log lines, polluting CF logs and possibly hitting `logpush` quotas. +- **Recommendation**: Aggregate into a single per-batch summary or gate behind a debug flag. + +### [P3] Validator: no URL scheme check, no length limits, no SKU charset rules +- **What**: `isValidUrl` allows any `new URL()`-parseable input (e.g., `mailto:`, `javascript:`, `file:`). +- **Where**: `packages/api/src/services/etl/CatalogItemValidator.ts:60-67`. +- **Why it matters**: `productUrl` is rendered in the mobile app and on the guides site. A scraper bug could inject `javascript:` URLs that survive to the UI. +- **Recommendation**: Restrict to `http:`/`https:`. Add length caps (`name` ≤ 500, `description` ≤ 50k, `sku` matches `[A-Za-z0-9_.\-/]+`). + +### [P3] Soft-delete is not handled by the upsert +- **What**: `catalogItems` has no `deletedAt` column (verified — grep returns nothing). CLAUDE.md notes "Soft deletes for all user content" but catalog items are scraper-controlled, so this may be intentional. However, an item that disappears from the source CSV is never marked unavailable. +- **Where**: `packages/api/src/db/schema.ts:132-215`; `packages/api/src/services/catalogService.ts:337-407`. +- **Why it matters**: The catalog grows monotonically. Discontinued products keep their `availability` from the last successful upsert. There is no "items present in last job but not in this one → mark out-of-stock" reconciliation. +- **Recommendation**: After a successful ETL, run `UPDATE catalog_items SET availability='OutOfStock' WHERE NOT EXISTS (SELECT 1 FROM catalog_item_etl_jobs WHERE catalog_item_id = id AND etl_job_id IN (last N jobs for this source))`. Or accept the limitation and document it. + +### [P3] No invalid-items retention policy +- **What**: `invalid_item_logs` grows forever; no TTL/sweep. +- **Where**: `packages/api/src/db/schema.ts:481-490`. +- **Why it matters**: Each bad row stores `raw_data` as JSONB plus an `errors` array — a single bad upload can write hundreds of MB to Neon. +- **Recommendation**: Add a scheduled task (or CF Cron Trigger) to drop logs >90 days. + +### [P3] No runbook / deploy docs +- **What**: No `docs/runbooks/etl.md`. `grep "etl|ETL"` in `README.md`/`docs/` returns only stale plan files. +- **Where**: N/A (missing). +- **Recommendation**: Write a 1-page runbook: how to trigger an ETL, how to inspect queue depth (`wrangler queues list/info packrat-etl-queue`), how to retry a failed job, how to drain the queue (`wrangler queues consumer remove`), how to interpret `success_rate`. Reference admin endpoints `/admin/etl/*`. + +## Test Coverage Gaps + +Tests cover the happy path with mocked R2 and globally-mocked DB. The following are **not** tested: + +- **Byte-range chunk processing** — no test sets `byteStart`/`byteEnd` in the message. The injected-header fetch, partial-row skip, and boundary off-by-ones (P1 above) are entirely uncovered. +- **Multi-message job (same jobId, multiple chunks)** — no integration test exercises the "two chunks complete sequentially" path, so the P0 premature-completion bug is invisible to CI. +- **Header > 4 KB** — see P1 finding. +- **Row spanning chunk boundary** — see P1 finding. +- **Embedding service failure path** — `processValidItemsBatch.test` mocks the rejection but does not assert that items were upserted without embeddings (the actual fallback behavior). +- **`processLogsBatch` DB failure** — no test for the swallowed-error case. +- **Backpressure** — `parser.write` returning `false` and waiting on `'drain'` is not unit-testable with the current mock (whole CSV emitted in one chunk). +- **Yield/wall-clock budget** — no test asserts a 100k-row CSV completes under wall-clock. +- **`processQueueBatch`** — no direct test; the per-message catch-and-swallow (P0) is untested. +- **Retry endpoint** — no integration test verifies the retry produces a new running job and a queue send. +- **Stuck-job sweep** — no test for the 30-minute cutoff. +- **Concurrent updates to same job row** — no race-condition test (e.g., two batches calling `updateEtlJobProgress` interleaved). Atomicity at the SQL level is good but a parallel-batch test would lock it in. +- **`mergeItemsBySku` cross-chunk SKU collisions** — merging happens within a single batch; SKUs duplicated across batches (or across chunks) hit the DB upsert path, not the merge path. No test for that. +- **Header injection — wrong column ordering** — what if the source CSV has a BOM, or quoted headers with commas inside? + +## Production Readiness Checklist + +- [ ] Multi-chunk job completion tracked correctly (chunks_total / chunks_completed columns) — addresses P0 #1 +- [ ] Queue consumer rethrows on per-message failure; DLQ + max_retries configured in `wrangler.jsonc` — addresses P0 #2 +- [ ] Retry endpoint chunks large files the same way the producer does — addresses P1 #1 +- [ ] Stuck-job sweep keyed on `lastProgressAt`, not `startedAt` — addresses P1 #2 +- [ ] Chunk boundaries aligned to row boundaries in the producer (or reassembly in the consumer) — addresses P1 #3 and P1 #4 +- [ ] Header injection validates first 4 KB contains a `\n`; tested with wide CSV — addresses P1 #5 +- [ ] Sentry integration in ETL paths with `jobId`/`objectKey` tags — addresses P2 #1 +- [ ] `processLogsBatch` rethrows on DB failure — addresses P2 #2 +- [ ] Embedding fallback tracked via counter and visible in admin dashboard — addresses P2 #3 +- [ ] Writer IIFE error attached to outer flow — addresses P2 #4 +- [ ] Yield mechanism uses `scheduler.yield()` and has a wall-clock test — addresses P2 #5 +- [ ] Rename ambiguous `BATCH_SIZE` constants — addresses P2 #6 +- [ ] `mergeItemsBySku` summary log instead of per-SKU — addresses P3 #1 +- [ ] Validator enforces `http(s):` scheme and length caps — addresses P3 #2 +- [ ] Discontinued-item reconciliation strategy chosen and documented — addresses P3 #3 +- [ ] `invalid_item_logs` retention policy — addresses P3 #4 +- [ ] Runbook checked in at `docs/runbooks/etl.md` — addresses P3 #5 +- [ ] Test coverage added for all gaps listed above diff --git a/docs/plans/2026-05-19-001-chore-coverage-ratchet-and-quality-gates-plan.md b/docs/plans/2026-05-19-001-chore-coverage-ratchet-and-quality-gates-plan.md new file mode 100644 index 0000000000..1c7a0b4af3 --- /dev/null +++ b/docs/plans/2026-05-19-001-chore-coverage-ratchet-and-quality-gates-plan.md @@ -0,0 +1,184 @@ +--- +type: plan +status: active +plan_type: chore +title: Coverage ratchet and assertion-strength gates +created: 2026-05-19 +worktree: .worktrees/chore/ramp-test-coverage +branch: chore/ramp-test-coverage +supersedes: docs/plans/2026-05-17-001-chore-test-coverage-ramp-and-ci-gate-plan.md +--- + +# chore: Coverage ratchet and assertion-strength gates + +## Summary + +A 2026-05-17 plan proposed a 9-unit, 4-phase ramp toward 95%+ coverage across the monorepo with a CI gate that blocks regressions. While that plan was being written, upstream `main` independently landed the threshold ramp itself — Vitest configs across `packages/api`, `apps/expo`, `packages/analytics`, `packages/mcp`, and `packages/overpass` now sit at 95/92/97/95 (or close), with refined exclude lists and added unit tests for middleware, image utils, and storage. **U2 of the original plan is therefore obsoleted by upstream.** + +What remains novel and ship-ready is the *enforcement* layer the original plan introduced: + +- **U6** — a coverage **ratchet** (`scripts/lint/coverage-ratchet.ts` + committed `coverage-baselines.json`) that fails CI if any tracked workspace drops below its baseline. Complements Vitest's per-config thresholds: thresholds enforce the floor, the ratchet enforces no-regression toward the tier target. +- **U9** — an **assertion-strength lint** (`scripts/lint/no-weak-assertions.ts`) that catches coverage theater patterns (assertion-free tests, bare `.toBeDefined()`, bare `.toHaveBeenCalled()`, oversized snapshots). +- **U1** — migrate the testing guide off the repo root (`TESTING.md` → `docs/testing.md`, per the "no random md in root" convention) and rewrite around the ratchet + lint, not the obsolete tier ramp. + +Future phases (consolidated coverage workflow / Stryker mutation testing / per-workspace backfills for the still-untested packages) are deferred to follow-up plans. + +--- + +## Problem Frame + +PackRat's Vitest configs now declare strict per-package coverage thresholds (largely 95%+), but the surrounding enforcement is thin: + +1. **No regression gate.** Vitest's `thresholds` block fails the build when a single config's run drops below its declared floor. There is nothing that fails CI when an existing workspace silently slides from 95% to 89% if the threshold is lowered as a "temporary unblock" — exactly the pattern the Elysia migration's PR-2083 history shows (`4c2c00d19 fix(ci): separate API type-check, lower coverage threshold`). A floor that can be edited by the same PR that drops below it is not a gate. +2. **No quality gate beyond line counts.** Coverage rewards *executing* code, not *asserting* on it. The standing failure mode is `expect(x).toBeDefined()` after a parse, or `expect(spy).toHaveBeenCalled()` without arg matching — both fully covered, both regression-blind. +3. **Doc hygiene.** `TESTING.md` lives at the repo root alongside `CLAUDE.md` and `README.md`. The repo convention is that only those last two belong at root; everything else goes under `docs/`. The testing guide itself was already out of date relative to upstream's threshold ramp. + +This plan adds the missing gates and the missing doc move. It does **not** lower or rewrite any existing Vitest threshold. + +--- + +## Scope Boundaries + +### In scope + +- A `scripts/lint/coverage-ratchet.ts` enforcement script + `scripts/lint/coverage-baseline-update.ts` (CI-only baseline bump) + committed `coverage-baselines.json` at the repo root. +- `bun check:coverage` / `bun check:coverage:update` package.json scripts. +- A unit test suite for the ratchet at `scripts/lint/__tests__/coverage-ratchet.test.ts`. +- `scripts/lint/no-weak-assertions.ts` and its unit test suite at `scripts/lint/__tests__/no-weak-assertions.test.ts`. +- `bun lint:weak-assertions` script and a `scripts/vitest.config.ts` for the scripts test suite. +- Migrating `TESTING.md` to `docs/testing.md` with content rewritten around current reality (upstream's 95%+ thresholds plus the new ratchet + lint). + +### Deferred to follow-up work + +- **Consolidated `.github/workflows/coverage.yml` matrix workflow** that runs every Tier A/B/C workspace, posts per-workspace PR comments, and invokes the ratchet. Out of scope for this PR — easier to land on top of the ratchet once the gate is in place. Tracked separately. +- **Stryker mutation testing** on `packages/api/src/{services,middleware,utils}/**` and the nightly workflow. Follow-up. +- **Backfilling tests in currently-untested workspaces** (`apps/{admin,trails,web}`, `packages/{guards,env,app,cli,checks,config,osm-db,osm-import,web-ui,api-client,ui}`). Some of these gained tests via upstream's work; the rest are a separate effort. +- **Wiring `bun lint:weak-assertions` into `scripts/check-all.ts`** as a blocking check. The lint currently surfaces a handful of real findings against the upstream codebase (mostly in `packages/analytics`); wiring into the gate requires a small cleanup PR first. + +### Out of scope + +- Lowering or rewriting any Vitest threshold upstream put in place. +- Adding Codecov / Coveralls integration. +- E2E / visual-regression / mutation testing. + +--- + +## Requirements + +| ID | Requirement | +|---|---| +| R1 | `coverage-baselines.json` lives at the repo root and records, per tracked workspace: `summaryPath`, four-metric baseline, and `recordedAt`. Updates happen via the baseline-update script on green merges to `main` — never manually edited in feature PRs. | +| R2 | `scripts/lint/coverage-ratchet.ts` exits non-zero on any metric dropping below the baseline (modulo `_epsilon` to absorb v8 jitter), on a missing summary file, or on a malformed summary. | +| R3 | The ratchet's analysis logic is testable in isolation. `scripts/lint/__tests__/coverage-ratchet.test.ts` covers happy path, regressions, missing summaries, malformed summaries, multi-metric regressions, and baseline parsing. | +| R4 | `scripts/lint/no-weak-assertions.ts` flags the four documented coverage-theater patterns (assertion-free tests, `only-tobedefined`, `bare-tohavebeencalled`, `large-snapshot`) and respects a file-level `// no-weak-assertions: disable` escape hatch. | +| R5 | The lint's analysis logic is testable. `scripts/lint/__tests__/no-weak-assertions.test.ts` covers each rule's positive/negative cases, the disable comment, expect-helper detection, and multi-violation files. | +| R6 | `bun check:coverage` runs the ratchet. `bun check:coverage:update` runs the baseline-update script. `bun lint:weak-assertions` runs the assertion lint. `bun test:scripts` runs both unit suites. | +| R7 | `TESTING.md` no longer exists at the repo root. `docs/testing.md` is the single canonical testing guide. `CLAUDE.md`, `README.md`, `copilot-instructions.md`, and any solutions doc that linked to the old path point to the new one. | +| R8 | Baselines for U6 are captured from real coverage runs against current upstream configs (not from the obsolete numbers in the superseded plan). | + +--- + +## Key Technical Decisions + +| Decision | Choice | Why | +|---|---|---| +| Threshold authority | **Keep upstream's existing Vitest thresholds; do not touch them.** The ratchet adds a second layer of enforcement on top. | Upstream already ramped to 95%+. Touching their numbers reopens decisions that were settled in PRs the merged into `main` between 2026-05-17 and 2026-05-19. | +| Ratchet implementation | **In-repo Bun script + committed `coverage-baselines.json`** | No external service, no new secrets, zero dependencies beyond the test runs that already happen. Mirrors `scripts/lint/no-duplicate-deps.ts` shape. | +| Baseline update flow | **CI-only on `main` post-merge** | Auto-commit baseline bumps after green runs so the floor only moves up. PRs cannot edit `coverage-baselines.json` to silently lower the gate. | +| Epsilon | **0.5 percentage points** | V8 coverage instrumentation has small run-to-run variance on identical code (observed ~0.16% on `packages/analytics` functions metric). Epsilon absorbs this without making the gate meaningless. | +| Assertion-strength rule strictness | **Flag only genuinely weak matchers** (`toBeDefined`, `toBeTruthy`, `toBeFalsy`, `.not.toBe{Undefined,Null}`). `toBeUndefined()` and `toBeNull()` alone are NOT flagged — they assert specific return values. | Avoids false positives on tests that legitimately assert "this returns null". | +| Helper-assertion detection | **Any call to `expect[A-Z][A-Za-z0-9]*(` counts as an assertion** (e.g., `expectUnauthorized(res)`, `expectJsonResponse(res)`). | `packages/api/test/` uses this convention extensively. Treating helpers as black-box assertions avoids flagging them as `assertion-free-test`. | +| Lint gate enable | **Add the command, defer wiring into `check-all.ts`** | Surfaces ~handful of real findings in current upstream code. Wiring into the blocking check requires a cleanup PR first; the script ships so contributors can run it manually until then. | +| Docs location | **`docs/testing.md`** | Per the "no random md in root" convention — `CLAUDE.md` and `README.md` are the only root markdown files. | + +--- + +## Implementation Units + +### U1. Migrate `TESTING.md` → `docs/testing.md` + +- **Goal:** Honor the "no random md in root" convention. Rewrite the testing guide around current reality (upstream's 95%+ thresholds + the ratchet + the lint). +- **Requirements:** R7 +- **Dependencies:** none +- **Files:** + - `TESTING.md` (delete) + - `docs/testing.md` (new — moved + rewritten) + - `CLAUDE.md` (update Testing section: link to new path, summarize ratchet + lint) + - `README.md` (badge link → `/docs/testing.md`) + - `copilot-instructions.md` (testing section: point at `docs/testing.md`, mention new scripts) + - `docs/solutions/ui-bugs/android-textinput-keyboard-focus-loss.md` (cross-ref link) +- **Approach:** The original `TESTING.md` content remains useful for patterns. The "Test Statistics" block at the bottom is out of date; replace it with a "Coverage Tier Model" matrix reflecting upstream's actual thresholds (api/expo/analytics/mcp at ~95/92/97/95). Add new sections for the ratchet and the assertion-strength lint. +- **Test scenarios:** Test expectation: none — documentation move with no behaviour change. Verify by grep: `rg -n 'TESTING\.md' .` returns no matches outside of `docs/plans/`. +- **Verification:** `TESTING.md` does not exist at root; `docs/testing.md` opens and renders; every `TESTING.md` reference in code, docs, and configs is updated. + +--- + +### U6. Coverage ratchet + baseline file + +- **Goal:** Add the regression-blocking gate. Vitest thresholds enforce the floor in each workspace; the ratchet ensures the floor cannot quietly slide down across PRs. +- **Requirements:** R1, R2, R3, R6, R8 +- **Dependencies:** none structurally; baselines must be captured from current upstream configs before commit +- **Files:** + - `coverage-baselines.json` (new — repo root) + - `scripts/lint/coverage-ratchet.ts` (new) + - `scripts/lint/coverage-baseline-update.ts` (new — CI-only) + - `scripts/lint/__tests__/coverage-ratchet.test.ts` (new — 13 scenarios) + - `package.json` (add `check:coverage` and `check:coverage:update`) +- **Approach:** + - Each workspace baseline carries: `summaryPath`, `tier`, four metric percentages, `recordedAt`. + - The ratchet reads `coverage-baselines.json` and each workspace's `coverage-summary.json`, compares per-metric with epsilon 0.5, and exits 0/1. + - Missing summary file → exit 1 ("silent skipping is exactly the regression mode this script exists to prevent"). + - Malformed summary (missing required `total` fields) → exit 1. + - The baseline-update script bumps numbers upward when current > baseline + epsilon; never lowers. Designed for `main`-only auto-commit, not for PR-time manual updates. + - Initial baselines captured fresh from `bun run --cwd test:coverage` against upstream's configs. +- **Patterns to follow:** `scripts/lint/no-duplicate-deps.ts` for CLI shape; existing `scripts/lint/__tests__/` once U9 lands for the test layout. +- **Test scenarios:** Per the ratchet test file: + - `compareWorkspace` returns `ok` / `improvement` / `regression` based on metric deltas. + - Tolerates noise below epsilon (default 0.5). + - Rejects drops just above epsilon. + - Reports multiple regressions in one workspace. + - `runRatchet` fails on missing or invalid summaries. + - `loadBaseline` parses workspace entries, honors `_epsilon`, falls back to default, and skips malformed entries. +- **Verification:** `bun check:coverage` runs cleanly on the captured baselines; `bun test:scripts` includes the 13 ratchet tests. + +--- + +### U9. Assertion-strength lint + +- **Goal:** Catch coverage theater (assertion-free tests, weak matchers, oversized snapshots, untyped mock calls) at lint time. +- **Requirements:** R4, R5, R6 +- **Dependencies:** none — cherry-picked from prior Phase 1 work +- **Files:** + - `scripts/lint/no-weak-assertions.ts` + - `scripts/lint/__tests__/no-weak-assertions.test.ts` (16 scenarios) + - `scripts/vitest.config.ts` (for `bun test:scripts`) + - `package.json` (add `lint:weak-assertions`, `test:scripts`) +- **Approach:** See Key Technical Decisions for the matcher-strictness and helper-detection rules. +- **Patterns to follow:** `scripts/lint/no-raw-typeof.ts` for shape. +- **Test scenarios:** Each of the four rules: positive case (flags), negative case (does not flag), edge cases (`it.todo`, helper assertions, disable comment, multi-violation file). +- **Verification:** `bun test:scripts` passes; `bun lint:weak-assertions` runs in under 5 seconds across the repo and surfaces only the small set of known-current findings. + +--- + +## Risk Analysis & Mitigation + +| Risk | Likelihood | Impact | Mitigation | +|---|---|---|---| +| Captured baselines too aggressive → first PR red | Medium | Medium | Epsilon 0.5 absorbs v8 jitter. Baselines captured from clean coverage runs, not from older numbers. | +| Lint flags too many existing tests | Confirmed (handful) | Low | Lint command exists but is NOT wired into the blocking `check-all.ts` until a separate cleanup PR. File-level `// no-weak-assertions: disable` escape hatch is available for grandfathered tests if needed. | +| Baseline file becomes a merge-conflict magnet | Low | Low | Updates happen only on `main` via the post-merge auto-commit. PR-level edits to `coverage-baselines.json` are not the workflow. | +| Future Vitest threshold change diverges from baseline | Low | Medium | Both layers gate independently. Vitest threshold is per-config; ratchet is per-baseline. A drop will trip whichever has a stricter floor — that's the right behaviour. Document the dual-layer model in `docs/testing.md`. | + +--- + +## Verification Strategy + +- **U1**: `rg -n 'TESTING\.md' .` returns no matches outside `docs/plans/`. `docs/testing.md` exists and renders. `bun check` passes. +- **U6**: `bun check:coverage` exits 0 on committed baselines; intentionally tweaking a baseline number downward makes a fresh coverage run trip the ratchet. `bun test:scripts` includes 13 ratchet tests, all passing. +- **U9**: `bun test:scripts` includes 16 assertion-lint tests, all passing. `bun lint:weak-assertions` runs in under 5 seconds and produces a stable list of findings. + +--- + +## Origin + +This plan supersedes `docs/plans/2026-05-17-001-chore-test-coverage-ramp-and-ci-gate-plan.md`. That document remains in place as historical context — its U2 (threshold ramp) was made obsolete by upstream work between 2026-05-17 and 2026-05-19, while its U1/U6/U9 carry forward unchanged in intent (only the baseline numbers were re-captured against the updated configs). diff --git a/docs/plans/2026-05-19-001-fix-etl-pipeline-audit-remediation-plan.md b/docs/plans/2026-05-19-001-fix-etl-pipeline-audit-remediation-plan.md new file mode 100644 index 0000000000..6df4f8b89c --- /dev/null +++ b/docs/plans/2026-05-19-001-fix-etl-pipeline-audit-remediation-plan.md @@ -0,0 +1,1063 @@ +--- +title: "fix: ETL pipeline audit remediation" +type: fix +status: superseded +supersededBy: docs/plans/2026-05-20-001-fix-etl-pipeline-workflows-migration-plan.md +supersededReason: "Pivoted execution engine from Cloudflare Queues + outbox to Cloudflare Workflows on 2026-05-20. Workflows natively provides the durable-step + idempotency + retry + state semantics that ~8 of the 15 units in this plan were manually reconstructing. The audit findings about CSV correctness, validator hardening, observability, retention, and runbook remain real and carry into the successor plan; the queue-as-state-machine subplot is dropped." +date: 2026-05-19 +deepened: 2026-05-19 +origin: docs/audits/2026-05-16-etl-audit.md +--- + +# fix: ETL pipeline audit remediation + +## Summary + +Remediate the catalog ETL pipeline against every finding in the 2026-05-16 audit (2 P0, 5 P1, 6 P2, 3 P3), correct two stale assumptions the audit made about Cloudflare runtime APIs, add bucket-vs-job reconciliation (both an admin-triggered tool and automatic post-job verification), and add a "re-ingest from the top" recovery path for jobs the buggy stuck-job sweep has already corrupted. Delivered as one master plan in four sequenced phases — schema + P0 blockers first, then chunking correctness, then observability + reconciliation, then hardening + runbook. + +--- + +## Problem Frame + +The pipeline ingests scraper CSVs from R2 (`packrat-scrapy-bucket`) into Neon Postgres via a Cloudflare Queue consumer. It is currently silently incorrect: live prod admin data (192 runs / 74 failed = 38% failure rate) shows seven large jobs from 2026-05-14 marked `failed` with identical `completedAt` timestamps — the wall-clock-based stuck-job sweep firing on healthy long jobs — while the dashboard reports `successRate: 100%` on those same failed jobs. Audit `docs/audits/2026-05-16-etl-audit.md` enumerates the structural causes: a single shared `jobId` across byte-range chunks lets the first finishing chunk flip the parent job to `completed`, per-message exceptions are swallowed (no DLQ, no retry), byte-range chunk boundaries silently drop or invalidate rows that span them, retries discard chunking entirely, and there is no Sentry / structured logging anywhere in the ETL path. + +The user's stated concern — *"some [data] is missing or falsely labeling as success"* — is corroborated on both ends: `completed` jobs can be premature (P0 #1), and `failed` jobs can be false failures (P1 #2). Either way the catalog count `totalItemsIngested: 304,431` cannot currently be trusted. + +--- + +## Requirements + +- R1. **No chunk causes premature job completion.** A multi-chunk job transitions to `completed` only when every chunk has succeeded. +- R2. **Per-message queue failures retry and ultimately DLQ.** No exception thrown by chunk processing is silently swallowed. +- R3. **Stuck-job sweep is progress-based, not wall-clock-based.** Healthy long-running jobs are not falsely marked `failed`. +- R4. **Chunk boundaries do not drop or invalidate rows.** Every row in the source CSV is processed exactly once. +- R5. **Retry / repair endpoints chunk the same way the producer does.** Retrying a large file does not single-shot it. +- R6. **CSV header injection for non-first chunks is correct or fails loudly.** No silent column misalignment. +- R7. **Every ETL job has post-ingestion verification.** R2 row count is compared to `totalProcessed` and the result is observable; significant deltas are surfaced. +- R8. **Operators can trigger a "from scratch" repair of any historical job** without invoking the original producer endpoint. +- R9. **Failures emit Sentry events with structured context.** Operators can debug a stuck job without paging through raw Worker logs. +- R10. **Embedding-fallback degradation is observable.** A job that completed without embeddings is distinguishable from a fully-successful one. +- R11. **Validator rejects unsafe URLs and oversize fields.** Mobile/web cannot be tricked into rendering `javascript:` URLs from the catalog. +- R12. **`invalid_item_logs` retention is bounded.** A bad upload cannot fill Neon storage indefinitely. +- R13. **A documented runbook exists for ETL operations.** A new on-caller can trigger / inspect / retry / drain without reading source. +- R14. **Test coverage exists for every behavior in R1–R12.** Specifically including the cases the global queue-mock in `packages/api/test/setup.ts` currently hides. + +--- + +## Scope Boundaries + +- The plan does not raise `max_concurrency` above 1 for the ETL queue. Concurrency bump is blocked on per-chunk idempotency keys that this plan introduces; the actual bump is a follow-up after this lands and bakes. +- The plan does not add a DLQ to the embeddings queue. ETL queue DLQ only. +- The plan does not migrate or rewrite the existing `etl_jobs` row data for the 7 historical jobs falsely marked `failed`. The repair-from-scratch endpoint introduced in U6 is the mechanism operators will use; the actual recovery run is operational, not a code unit. +- The plan does not change the producer endpoint's authentication, the source CSV schema, or the scraper revision pinning. +- The plan does not introduce a new ETL Worker — the current `packages/api` Elysia Worker continues to host both the HTTP routes and the queue consumer. +- The plan does not address `apps/landing` / `apps/guides` / `apps/expo` consumers of catalog data even when bucket-vs-job reconciliation finds drift. Surfacing inconsistencies is in scope; downstream cache invalidation is not. + +### Deferred to Follow-Up Work + +- **Concurrency bump on `packrat-etl-queue` consumer**: separate PR after this plan ships and per-chunk idempotency is verified in production for ≥2 weeks. +- **Embeddings-queue DLQ + retry policy**: separate plan; same shape as ETL DLQ work in U3, but a distinct surface. +- **Catalog reconciliation across multiple historical jobs**: only per-job reconciliation is in scope. Historical cross-source rollup ("did we lose 5% of the catalog last quarter?") is a separate analytics workstream. +- **Soft-delete / discontinued-item reconciliation** (audit P3 #3): documented as accepted limitation in the runbook (catalog is scraper-controlled, not user content). A future plan can add `availability='OutOfStock'` reconciliation if business requirements emerge. +- **CLI subcommand surface in `packages/cli/src/commands/admin/etl.ts`**: U12 wires the new admin endpoints into the existing CLI command file. Broader CLI ergonomics work is out of scope. + +--- + +## Context & Research + +### Relevant Code and Patterns + +- **Producer endpoint:** `packages/api/src/routes/catalog/index.ts:229-293` — `POST /catalog/etl`, R2 head + 20 MB chunking at `:253-271`. Chunk creation logic to extract into a shared helper used by U6. +- **Queue producer:** `packages/api/src/services/etl/queue.ts:6-41` — `queueCatalogETL`; uses `sendBatch` with `batchSize: 100` (CF queue per-call cap). +- **Queue consumer dispatch:** `packages/api/src/services/etl/queue.ts:43-61` — `processQueueBatch` with the swallowed catch at `:50-60`. **This is the core P0 #2 surface.** +- **Per-chunk processor:** `packages/api/src/services/etl/processCatalogEtl.ts` — header injection (`:50-58`), partial-row skip (`:95-108`), batch flush (`:120-187`), per-chunk completion (`:188-191`), per-chunk failure (`:201-204`). +- **Atomic counter pattern (mirror this):** `packages/api/src/services/etl/updateEtlJobProgress.ts:16-23` — `sql\`COALESCE(${col}, 0) + ${n}\``. New `chunks_completed` / `total_embedding_failures` increments use the same idiom; the "set status=completed when chunks_completed+1 == chunks_total" branch uses a single `UPDATE ... SET ... WHERE` with a `CASE` expression in the same transaction. +- **Embeddings queue pattern (mirror this):** `packages/api/src/services/catalogService.ts:461-507` — consumer rethrows on failure so CF Queue retries fire. ETL consumer must adopt the same shape. +- **Admin routing pattern:** `packages/api/src/routes/admin/index.ts:117-237` mounts the admin prefix; `:230-237` enforces `adminAuthGuard` on every sub-route. New endpoints in `packages/api/src/routes/admin/analytics/catalog.ts` inherit the guard. +- **R2 access (S3-API not Workers binding):** `packages/api/src/services/r2-bucket.ts:193-360` — `R2BucketService({ env, bucketType: 'catalog' })` wraps `@aws-sdk/client-s3` against the R2 S3 endpoint. `r2.head(key)` and `r2.get(key, { range: { offset, length } })` are the surface. Range format `bytes=offset-(offset+length-1)` at `:675-691`. +- **Schema location:** `packages/db/src/schema.ts:446-510` — `etlJobs`, `invalidItemLogs`, `catalogItemEtlJobs`, status enum at `:460`. **Audit cites a stale path (`packages/api/src/db/schema.ts`); the file was extracted into the `packages/db` package — see merge `b14f4dbd5`.** +- **Drizzle migration location:** `packages/api/drizzle/NNNN_.sql` + `meta/NNNN_snapshot.json` + `_journal.json`. Latest is `0047_cute_bloodscream.sql`; new migrations land at `0048` and `0049` (split per Drizzle Kit's enum-add constraint). Generated via `bun run --cwd packages/api db:generate`. Custom linter at `scripts/lint/check-drizzle-migrations.ts` runs in `lint:custom`. +- **Existing ETL integration test:** `packages/api/test/etl.test.ts` — mocks `R2BucketService` per-test, uses real Postgres via wsproxy at `localhost:5434`. Setup at `packages/api/test/setup.ts:535-572` globally mocks both `queueCatalogETL` and `processQueueBatch` (lines `:544-551`) — this is precisely *why* the per-message swallow in P0 #2 is invisible to CI today, and U14 must un-mock to cover it. +- **Wrangler config:** `packages/api/wrangler.jsonc:65-89` (prod queues) and `:161-194` (dev). Currently `max_batch_size: 1, max_concurrency: 1`, **no `dead_letter_queue`, no `max_retries`** on either consumer. Queue routing handler at `packages/api/src/index.ts:109-124`. +- **Admin CLI surface:** `packages/cli/src/commands/admin/etl.ts` already exists. New endpoints in U6 and U12 add corresponding subcommands. + +### Institutional Learnings + +- `docs/solutions/` has no prior ETL, Cloudflare Queues, R2 byte-range, or Sentry-in-Workers learnings — only an unrelated Better Auth CLI note and an Android UI bug. This remediation is greenfield from an institutional-knowledge standpoint, which makes it a strong `/ce-compound` target after each phase ships. + +### External References + +- **Cloudflare Queues — ack/retry semantics:** `message.ack()` / `message.retry({ delaySeconds })` / `ackAll()` / `retryAll()` documented at . Throwing fails the un-acked remainder of the batch. `retryDelaySeconds` max is 24h per . +- **Cloudflare Queues — DLQ:** `dead_letter_queue` (string name) + `max_retries` (default 3, max 100) in the consumer block per . +- **Cloudflare Workers Scheduler:** Only `scheduler.wait(ms)` is documented at . **`scheduler.yield()` does not exist** — the audit P2 #5 recommendation is wrong on this. Use `await scheduler.wait(0)` instead. +- **Wall-clock limit:** Queue consumer wall-clock cap is **15 minutes**, not 30 seconds, per . The audit's "30 s wall-clock" framing under P2 #5 is stale. +- **Sentry on Cloudflare:** Prefer the first-party `@sentry/cloudflare` over toucan-js. Wrap via `Sentry.withSentry(optsFn, { fetch, queue })` per . Queue instrumentation guidance at . +- **Drizzle enum-add limitation:** `ALTER TYPE … ADD VALUE` inside the same transaction as code that uses the new value fails. Split migrations. Tracked at . +- **R2 range reads with AWS SDK:** R2's S3 API fully supports the `Range` header — `GetObjectCommand({ Range: 'bytes=0-1023' })` behaves identically to S3 per . + +--- + +## Key Technical Decisions + +- **Track chunk completion via two new columns (`chunks_total`, `chunks_completed`) on the existing `etl_jobs` row, gated by a per-chunk idempotency table `etl_job_chunks(job_id, chunk_index, completed_at)` with PK on `(job_id, chunk_index)`.** Rationale: even at `max_concurrency: 1` today, Cloudflare Queues are *at-least-once* — a chunk whose DB writes succeed but whose ack is lost will be redelivered, which would double-increment a naive `chunks_completed = chunks_completed + 1` and either crash through `chunks_total` or transition the job to `completed` while a sibling chunk is still pending. The idempotency table makes the increment a deterministic side-effect of `INSERT … ON CONFLICT (job_id, chunk_index) DO NOTHING RETURNING 1`; the counter only bumps when the insert created a new row. This was originally scoped as a follow-up under "Deferred" but the deepening pass surfaced it as a correctness prerequisite — pulled forward into U1/U2. +- **No new `partial` enum value on `etl_job_status`.** Embedding-fallback degradation is observable via `total_embedding_failures > 0` on a `completed` row. Adding an enum value would force the audit P2 #3 split into two migrations (Drizzle Kit limitation) and complicate every admin filter without observable benefit. +- **Use `@sentry/cloudflare` (first-party), not toucan-js as the audit suggested.** Toucan still works but is no longer the recommended Sentry path on Workers as of 2026. `withSentry({ fetch, queue })` wraps both entry points in one call; no manual `waitUntil` plumbing needed. +- **Use `await scheduler.wait(0)` for yielding, not the non-existent `scheduler.yield()`.** Audit P2 #5 is corrected here. +- **Stuck-job sweep keyed on `last_progress_at < now() - interval '15 minutes'` AND `status = 'running'`,** not on `started_at`. The 15-min figure derives from the actual CF Queue consumer wall-clock cap (15 min), not the audit's stale 30 s/30 min framing. With per-chunk progress updates writing `last_progress_at`, any chunk making real progress is safe; only truly stalled jobs flip to `failed`. +- **Row-boundary alignment happens in the producer**, not the consumer. The producer's `r2.head(key)` flow does an extra small range read on each chunk-end region (e.g., 64 KB) to find the last `\n` and emits chunks with newline-aligned `byteEnd`. This eliminates both the partial-row skip bug (P1 #4) and the row-spanning-chunk bug (P1 #5) in one place. Consumer's `skipPartialRow` logic is removed. +- **CSV header re-read with bounded loop, not a fixed 4 KB slice.** For non-first chunks, the consumer fetches `[0, 4096)`, and if no `\n` appears, expands to `[0, 16384)`, then `[0, 65536)`. If still no newline, throw — header is malformed. Eliminates P1 #3 silent column misalignment. +- **Per-chunk idempotency key is `(jobId, chunkIndex)`** — added to `CatalogETLMessage`. Even though `max_concurrency: 1` means de-facto serialization today, threading the key now unblocks the future concurrency bump without another migration. +- **DLQ is a dedicated new queue `packrat-etl-dlq`** with a minimal consumer that captures the failure to Sentry, persists a row to a new `etl_dlq_events` table for forensics, and acks. The DLQ does *not* attempt to re-process — it's an event sink + visibility tool. +- **Reconciliation runs as both a manual admin endpoint and an automatic post-job step, with the automatic step on its own queue.** Manual endpoint stays synchronous (operator-explicit, scoped to one job). Automatic step is dispatched as a queue message to a new `packrat-etl-reconcile-queue` on the final-chunk completion transition, *not* via `ctx.waitUntil` — `waitUntil` shares the queue invocation's wall-clock budget, which for a multi-GB CSV exceeds the 15-min cap when added on top of the chunk's own processing time. The reconcile consumer streams the file in 100 MB byte-range windows with progress checkpointed to a transient column so retries resume. The consumer's `INSERT … RETURNING` includes `verified_at IS NULL` as an idempotency gate so a redelivered reconcile message is a no-op. Warning threshold remains `> max(10, ceil(0.01 * total_processed))`. +- **Repair-from-scratch endpoint creates a NEW `etl_jobs` row and links it to the old via a new nullable `superseded_by_job_id` column with `ON DELETE SET NULL` and a paired `superseded_at timestamp`.** No mutation of the old row's counters — preserves audit trail and lets the dashboard show "originally failed, repaired by job X". `ON DELETE SET NULL` (not `CASCADE`) so deleting one row never silently nukes a chain of repair attempts. A CHECK constraint prevents self-reference (`superseded_by_job_id != id`). The runbook procedure (U15) requires verifying R2 source presence + ETag match before invoking repair, so an overwritten source cannot silently re-ingest the wrong file. +- **Structured logger lives at `packages/api/src/utils/logger.ts`** as a thin wrapper around `console.*` for now, accepting a `LogContext` (jobId, chunkIndex, r2Key, etc.) and emitting JSON-prefixed lines. Sentry breadcrumbs piggyback on the same call surface. Not a full logger framework — that's a separate decision. + +--- + +## Open Questions + +### Resolved During Planning + +- **Should the chunk completion track go on `etl_jobs` columns alone, or be paired with a per-chunk idempotency table?** Resolved during deepening: both. `etl_jobs.{chunks_total, chunks_completed}` are the counters; `etl_job_chunks(job_id, chunk_index)` is the idempotency gate that makes the increment safe under at-least-once delivery. See Key Technical Decisions. +- **Should embedding-fallback get a new enum value `partial`?** Resolved: no — use `total_embedding_failures` counter on a `completed` row. +- **Toucan-js or `@sentry/cloudflare`?** Resolved: `@sentry/cloudflare`. See External References. +- **Wall-clock budget for the stuck-job sweep cutoff?** Resolved: `last_progress_at < now() - interval '15 minutes'`, matching the actual queue-consumer wall-clock cap. +- **Should the row-boundary alignment happen in producer or consumer?** Resolved: producer. Single source of truth for chunk boundaries. +- **Should auto-reconcile use `ctx.waitUntil` or its own queue?** Resolved during deepening: dedicated queue (`packrat-etl-reconcile-queue`) with resumable byte-range streaming. `waitUntil` shares the chunk consumer's wall-clock budget, which fails at multi-GB files. +- **Should the DLQ consumer's INSERT + status UPDATE be transactional?** Resolved during deepening: yes, single `db.transaction()`. Same for the sweep's UPDATE + sentinel-event INSERT. +- **Should the migration split into 0048a/0048b/0048c?** Resolved during deepening: no — at ~200 rows, the single-migration approach is fine. Splitting becomes correct when `etl_jobs` exceeds ~100k rows, and the migration header carries a comment to revisit at that scale. + +### Deferred to Implementation + +- **Exact Drizzle migration sequencing within Phase 1.** All six columns + the partial index + the new `etl_dlq_events` table can land in a single migration `0048` since none touch the enum. Whether to split `superseded_by_job_id` (added later in U6) into its own migration `0049` or include it in `0048` is decided at U1 implementation. Either way the enum stays untouched in this plan. +- **`@sentry/cloudflare` instrumentation depth for the queue consumer.** The exact `Sentry.startSpan` attributes per queue message (some attributes are conventional, some are CF-specific) get finalized when U8 lands. +- **Sentry sampling rate** for the queue consumer. Default to `tracesSampleRate: 0.1` and tune in production; not a plan-time decision. +- **Exact threshold for "significant" reconciliation delta** that triggers a Sentry warning vs informational event. Default: `> max(10, ceil(0.01 * total_processed))` rows of delta. Tunable in production. +- **Cron schedule for `invalid_item_logs` retention sweep.** Daily at 09:00 UTC unless ops has a quieter window. + +--- + +## High-Level Technical Design + +> *This illustrates the intended approach and is directional guidance for review, not implementation specification. The implementing agent should treat it as context, not code to reproduce.* + +```text +Producer ─── POST /catalog/etl ──┐ + │ + ▼ + ┌─────────────────────────────────────────────┐ + │ chunkCsvForR2(key) (NEW shared helper) │ + │ 1. r2.head(key) -> size │ + │ 2. for each 20 MB window: │ + │ peek (next 64 KB) to find last '\n' │ + │ emit chunk with byteEnd = newline-1 │ + │ 3. tag each chunk: { jobId, chunkIndex, │ + │ chunksTotal, byteRange } + └─────────────────────────────────────────────┘ + │ + INSERT etl_jobs + (status='running', + chunks_total=N, + chunks_completed=0) + │ + ETL_QUEUE.sendBatch(chunks) + │ + ▼ + ┌─────────────────────────────────────────────┐ + │ processQueueBatch (REWRITE) │ + │ for message of batch: │ + │ try { │ + │ processCatalogETL(msg) │ + │ message.ack() │ + │ } catch (err) { │ + │ Sentry.captureException(err, {...}) │ + │ message.retry({ delaySeconds: 30 }) │ + │ } │ + └─────────────────────────────────────────────┘ + │ + ▼ + ┌─────────────────────────────────────────────┐ + │ processCatalogETL (per chunk) │ + │ r2.get(key, range) -> stream │ + │ if chunkIndex > 0: re-fetch header │ + │ (expand 4K→16K→64K, throw if no '\n') │ + │ parse rows (csv-parse, backpressure) │ + │ per 100 rows: scheduler.wait(0) │ + │ flush valid -> processValidItemsBatch │ + │ (embedding fallback increments │ + │ total_embedding_failures atomically) │ + │ flush invalid -> processLogsBatch │ + │ (now RETHROWS on DB failure) │ + │ on success: │ + │ UPDATE etl_jobs │ + │ SET chunks_completed = chunks_completed+1, + │ last_progress_at = now(), │ + │ status = CASE │ + │ WHEN chunks_completed+1 │ + │ = chunks_total │ + │ THEN 'completed' │ + │ ELSE status │ + │ END │ + │ if completed (in same txn): │ + │ enqueue ReconcileMessage to │ + │ packrat-etl-reconcile-queue │ + └─────────────────────────────────────────────┘ + │ + (on completion transition) + ▼ + ┌─────────────────────────────────────────────┐ + │ processReconcileBatch │ + │ reconcileJob(jobId, resumeFromByte=0): │ + │ if verified_at IS NOT NULL: ack │ + │ stream 100 MB byte-range windows │ + │ checkpoint to │ + │ verified_row_count_partial │ + │ if budget low: throw ResumeError │ + │ (consumer re-enqueues) │ + │ on EOF: UPDATE verified_at, count │ + │ if delta > threshold: Sentry warning │ + └─────────────────────────────────────────────┘ + │ + (on any thrown error after retries) + ▼ + packrat-etl-dlq + │ + ▼ + ┌─────────────────────────────────────────────┐ + │ dlqConsumer │ + │ Sentry.captureException │ + │ INSERT etl_dlq_events │ + │ ack │ + └─────────────────────────────────────────────┘ + +Background (CF Cron): + stuck-job sweep: status='running' AND last_progress_at < now()-15min + -> status='failed', emit Sentry warning + invalid-log retention: DELETE FROM invalid_item_logs WHERE created_at < now()-90d +``` + +--- + +## Implementation Units + +### U1. Schema migration: chunk tracking, idempotency table, progress timestamp, embedding failures, reconciliation columns, DLQ events table, constraint hardening + +**Goal:** Add the columns, tables, indexes, and constraints that the rest of the plan reads and writes. Lands first so every subsequent unit can compile and migrate against a known schema. Single migration `0048` is acceptable at the current ~200-row scale of `etl_jobs`; splitting into multiple migrations is unnecessary engineering at this size (revisit if `etl_jobs` exceeds ~100k rows). + +**Requirements:** R1, R3, R7, R8, R10 + +**Dependencies:** None + +**Files:** +- Modify: `packages/db/src/schema.ts` (add columns to `etlJobs`; add new `etlJobChunks` table; add new `etlDlqEvents` table; add UNIQUE constraint to `catalogItemEtlJobs`; export all) +- Create: `packages/api/drizzle/0048_etl_chunking_and_observability.sql` +- Create: `packages/api/drizzle/meta/0048_snapshot.json` (generated) +- Modify: `packages/api/drizzle/meta/_journal.json` (generated) +- Test: `packages/api/test/db-schema-etl.test.ts` (new — schema smoke test asserting columns exist with expected defaults; uses the existing Docker Postgres wsproxy setup at `localhost:5434`) + +**Approach:** +- Columns added to `etl_jobs`: + - `chunks_total integer` (nullable — single-chunk legacy jobs leave it null) + - `chunks_completed integer DEFAULT 0 NOT NULL` + - `last_progress_at timestamp` (nullable initially; backfilled to `started_at` for legacy rows in the same migration) + - `total_embedding_failures integer DEFAULT 0 NOT NULL` + - `verified_at timestamp` (nullable) + - `verified_row_count integer` (nullable) + - `verified_row_count_partial integer` (nullable — checkpoint for resumable reconcile in U10) + - `superseded_by_job_id text` (nullable, FK to `etl_jobs.id` `ON DELETE SET NULL`) + - `superseded_at timestamp` (nullable — paired with `superseded_by_job_id` so the timeline survives even after FK cleanup) + - `source_etag text` (nullable — captured on producer insert from `r2.head(objectKey).etag`; U6's repair endpoint uses this for failure-closed source verification) + - `source_last_modified timestamp` (nullable — same capture; redundant with etag but cheap) +- CHECK constraints on `etl_jobs`: + - `etl_jobs_chunks_completed_lte_total CHECK (chunks_total IS NULL OR chunks_completed <= chunks_total)` — fail loudly on over-count. + - `etl_jobs_no_self_supersede CHECK (superseded_by_job_id IS NULL OR superseded_by_job_id <> id)` — prevent self-referential repair loop. +- New indexes on `etl_jobs`: + - Partial: `etl_jobs_running_progress_idx` on `(status, last_progress_at)` `WHERE status = 'running'` — for the U5 stuck-job sweep. + - Partial: `etl_jobs_unverified_idx` on `(verified_at)` `WHERE status = 'completed' AND verified_at IS NULL` — for the U10 watchdog scan. + - `etl_jobs_superseded_by_idx` on `(superseded_by_job_id)` — for the admin dashboard's "is this job superseded?" lookup. +- New table `etl_job_chunks` (per-chunk idempotency, see Key Technical Decisions): + - `job_id text NOT NULL` (FK to `etl_jobs.id` `ON DELETE CASCADE`) + - `chunk_index integer NOT NULL` + - `completed_at timestamp DEFAULT now() NOT NULL` + - `PRIMARY KEY (job_id, chunk_index)` +- New table `etl_dlq_events`: `id text PK`, `job_id text` (FK, nullable, `ON DELETE SET NULL`), `chunk_index integer`, `message_body jsonb`, `error_message text`, `error_stack text`, `attempts integer`, `source text` (one of `consumer`, `sweep`; defaults to `consumer`), `created_at timestamp DEFAULT now() NOT NULL`. Index on `created_at`. +- Modification to `catalog_item_etl_jobs`: add `UNIQUE (catalog_item_id, etl_job_id)` so a redelivered chunk's upsert can use `ON CONFLICT DO NOTHING` and not produce duplicate provenance rows. +- Backfill: `UPDATE etl_jobs SET last_progress_at = started_at WHERE last_progress_at IS NULL`. Safe — `etl_jobs` is ~200 rows; sub-100ms on Neon. +- Drizzle generator: `bun run --cwd packages/api db:generate` then verify the SQL file matches the design. **Verify Drizzle Kit emits `DEFAULT 0 NOT NULL` literally in the SQL** — Drizzle sometimes drops the SQL-side default and keeps only the JS-side, which would break inserts from in-flight old workers during a rolling deploy. **Do NOT touch the `etl_job_status` enum in this migration** — no new enum value is needed (see Key Technical Decisions). +- Drizzle Kit does not auto-emit `CONCURRENTLY` for indexes. At 200 rows the index build is instant so `CONCURRENTLY` is nice-to-have, not blocking. If the table grows >100k rows before this lands, hand-edit the generated SQL to use `CREATE INDEX CONCURRENTLY IF NOT EXISTS` and split each index into its own statement-breakpoint block. + +**Patterns to follow:** +- Existing `etl_jobs` definition at `packages/db/src/schema.ts:460-479` for column shape and import style. +- Migration `0027_past_madrox.sql` (added `scraper_revision` + index) for the "add column + partial index" pattern. +- `scripts/lint/check-drizzle-migrations.ts` runs in `lint:custom`; the new migration must pass it. + +**Test scenarios:** +- Happy path: After migration runs against a populated test DB, all 8 new `etl_jobs` columns are present with the documented defaults; `etl_job_chunks` and `etl_dlq_events` exist; the three new partial/normal indexes are queryable (`EXPLAIN SELECT ... WHERE status='running' ...` uses the running-progress index; the unverified index serves the watchdog). +- Happy path: `INSERT INTO etl_job_chunks (job_id, chunk_index) VALUES ('j1', 0)` succeeds; a duplicate insert returns no row via `ON CONFLICT DO NOTHING RETURNING 1` and the table still contains exactly one row. +- Edge case: Legacy rows have `chunks_total = NULL` and `last_progress_at` backfilled to `started_at`. +- Edge case: `chunks_completed DEFAULT 0` is correctly applied to existing rows (verify with a row that has `chunks_completed = 0` post-migration). The generated SQL must literally include `DEFAULT 0 NOT NULL` — assert via SQL `information_schema.columns`. +- Edge case: `UNIQUE (catalog_item_id, etl_job_id)` on `catalog_item_etl_jobs` prevents a duplicate-insert (returns conflict). +- Error path: Attempting to insert a row with `chunks_completed > chunks_total` violates the CHECK constraint and errors clearly. +- Error path: Attempting to set `superseded_by_job_id = id` violates the no-self-supersede CHECK. +- Error path: Re-running the migration on an already-migrated DB is a no-op (Drizzle's migration log handles this; smoke-test the up/down via `bun run --cwd packages/api db:migrate`). +- Edge case: Down-migration cleanly drops the new columns/tables on a DB with no Phase 2+ data. **Once Phase 2 ships and writes start landing in the new columns, the migration is forward-only** — document in the migration header comment. + +**Verification:** +- `bun run --cwd packages/api db:migrate` applies cleanly against a fresh Docker Postgres + against a Postgres seeded with current-prod-shape `etl_jobs` rows. +- `bunx drizzle-kit check` (run from `packages/api/`) validates the snapshot chain is internally consistent — run this before pushing any migration change. +- `bun lint:custom` passes on the new migration. +- `bun test:api:unit` includes the new schema test and it passes. + +--- + +### U2. P0 #1 fix: chunk-completion lifecycle in producer + consumer + +**Goal:** A multi-chunk job's `status` transitions to `completed` only after every chunk has finished. Premature completion eliminated. + +**Requirements:** R1 + +**Dependencies:** U1 + +**Files:** +- Modify: `packages/api/src/routes/catalog/index.ts` (producer endpoint sets `chunks_total` on `etl_jobs` insert and tags each `CatalogETLMessage` with `chunkIndex` and `chunksTotal`) +- Modify: `packages/api/src/services/etl/types.ts` (extend `CatalogETLMessage.data` with `chunkIndex: number` and `chunksTotal: number`; `byteStart`/`byteEnd` remain) +- Modify: `packages/api/src/services/etl/processCatalogEtl.ts` (rewrite the `:188-191` success-path UPDATE to use the `CASE` expression that flips status only when `chunks_completed + 1 = chunks_total`; also update `last_progress_at` on every counter write) +- Modify: `packages/api/src/services/etl/updateEtlJobProgress.ts` (include `last_progress_at: sql\`now()\`` in the update set so every progress write refreshes the sweep timestamp) +- Test: `packages/api/test/etl-chunk-completion.test.ts` (new) + +**Approach:** +- Producer: compute `chunks` first, then `INSERT etl_jobs (..., chunks_total) VALUES (..., ${chunks.length})` — a single round-trip including `chunks_total`. Then `sendBatch` with each message carrying `chunkIndex` 0..N-1 and `chunksTotal: N`. Setting `chunks_total` in the initial INSERT (rather than a separate follow-up UPDATE) eliminates a window where a chunk consumer could observe `chunks_total IS NULL` and silently fail the `chunks_completed + 1 = chunks_total` CASE comparison. +- Consumer success path runs inside a single Drizzle `db.transaction()`: + 1. `INSERT INTO etl_job_chunks (job_id, chunk_index) VALUES ($1, $2) ON CONFLICT (job_id, chunk_index) DO NOTHING RETURNING 1` — the idempotency gate. If no row returned, this is a redelivery; skip the increment, ack the message, return. + 2. If the insert created a row, run the atomic UPDATE: `UPDATE etl_jobs SET chunks_completed = chunks_completed + 1, last_progress_at = now(), status = CASE WHEN chunks_completed + 1 = chunks_total THEN 'completed' ELSE status END, completed_at = CASE WHEN chunks_completed + 1 = chunks_total THEN now() ELSE completed_at END WHERE id = $1 AND status = 'running' RETURNING status, chunks_completed, chunks_total`. + 3. The `WHERE status = 'running'` gate prevents clobbering a row the U5 sweep has already flipped to `failed` (status-flip-flop hazard). + 4. If the returned row shows the transition to `completed`, *and* this transaction was the one that created the chunk-row in step 1, send a message to `packrat-etl-reconcile-queue` (see U10) for the auto-reconcile. +- On per-chunk failure: the consumer no longer flips the parent job to `failed` immediately. Instead it lets the message throw / retry. The parent job only flips to `failed` via (a) DLQ consumer when retries are exhausted, or (b) the stuck-job sweep (U5). +- Single-chunk legacy jobs: when `chunks_total IS NULL`, the `etl_job_chunks` insert still gates the increment; legacy rows backfilled to `chunks_total = 1` migrate cleanly. Backwards-compatible with any in-flight legacy messages. +- The CHECK constraint `chunks_completed <= chunks_total` from U1 is the loud-failure safety net — if the idempotency gate ever leaks (e.g., a code bug bypasses the chunk-table insert), the next `UPDATE` errors with a constraint violation rather than silently corrupting the counter. + +**Patterns to follow:** +- Atomic SQL update idiom at `packages/api/src/services/etl/updateEtlJobProgress.ts:16-23`. +- Drizzle transaction shape: `await db.transaction(async (tx) => { ... })`. + +**Test scenarios:** +- Happy path: 5-chunk job; chunks 0..3 complete successfully → status remains `running` with `chunks_completed = 4`; chunk 4 completes → status flips to `completed`, `completed_at` set, `etl_job_chunks` has 5 rows. +- Happy path (idempotency): Chunk 2 succeeds, ack lost, CF redelivers → second attempt's `INSERT … ON CONFLICT DO NOTHING RETURNING` returns no row → increment is skipped → `chunks_completed` increments exactly once over the two deliveries. +- Edge case: Chunks complete out of order (chunk 3 finishes before chunk 1) → status flips only when all five have incremented; the `etl_job_chunks` rows record actual completion order. +- Edge case: Single-chunk legacy job (`chunks_total = 1`) → flips to `completed` on its one success; `etl_job_chunks` has 1 row. +- Edge case: Sweep flips job to `failed` mid-flight; the next chunk's UPDATE `WHERE … AND status = 'running'` returns zero rows → transaction sees the conflict, logs warning, lets the operator route to repair-from-scratch. +- Error path: One chunk throws; other chunks succeed → parent job stays `running` while CF Queue retries the failed chunk; if retries exhaust, DLQ consumer (U3) handles state transition. +- Error path: CHECK constraint trips (hypothetical leaked-idempotency bug) → UPDATE errors loudly, chunk retries, no silent corruption. +- Integration: With `R2BucketService` mocked to return a small CSV split into 3 chunks via `byteRange`, the full producer→queue→consumer cycle ends in exactly one `status=completed` transition for the parent job AND exactly one reconcile message enqueued. +- Integration (idempotency at scale): Replay every chunk message twice → `etl_job_chunks` has exactly `chunks_total` rows, counters match, status = `completed`. + +**Verification:** +- Re-running `etl.test.ts` plus the new test under `bun test:api` shows no `status='completed'` write until `chunks_completed = chunks_total`. +- A manual prod-shape replay (`POST /catalog/etl` against the dev Worker with a CSV that produces ≥3 chunks) shows the dashboard's `successRate` remain at the running state until all chunks finish. + +--- + +### U3. P0 #2 fix: explicit ack/retry + DLQ wiring + +**Goal:** No per-message exception is silently swallowed. Failures retry; exhausted retries land in a dedicated DLQ that emits Sentry events and persists for forensics. + +**Requirements:** R2, R9 + +**Dependencies:** U1 (for `etl_dlq_events` table) + +**Files:** +- Modify: `packages/api/src/services/etl/queue.ts` (rewrite `processQueueBatch` for explicit per-message ack/retry; remove the swallow at `:50-60`) +- Create: `packages/api/src/services/etl/processDlqEvent.ts` (DLQ consumer; INSERT into `etl_dlq_events`, capture Sentry exception, ack) +- Modify: `packages/api/src/index.ts` (extend the `queue()` switch at `:109-124` with arms for `packrat-etl-dlq` and `packrat-etl-dlq-dev`) +- Modify: `packages/api/wrangler.jsonc` (declare `packrat-etl-dlq` and `packrat-etl-dlq-dev` as producer + consumer; add `dead_letter_queue: "packrat-etl-dlq"` and `max_retries: 3` to the ETL consumer block at `:78-82` and dev equivalent at `:178-182`) +- Modify: `packages/api/src/services/etl/processCatalogEtl.ts` (when a chunk's processing throws, also UPDATE `last_progress_at` and increment a transient `last_error_at` if useful — see Approach for trade-off; primary work is removing the per-chunk `status='failed'` write at `:201-204` since the DLQ consumer is now responsible for state transition) +- Test: `packages/api/test/etl-queue-retry.test.ts` (new — covers the global-mock blind spot in `setup.ts:544-551`) + +**Approach:** +- Rewrite `processQueueBatch`: + ```text + for (const message of batch.messages) { + try { + await processCatalogETL({ message: message.body, env }); + message.ack(); + } catch (err) { + logger.error('etl.chunk.failed', { jobId, chunkIndex, err }); + Sentry.captureException(err, { tags: { jobId, chunkIndex, r2Key }, contexts: { queue: { messageId: message.id, attempts: message.attempts } } }); + message.retry({ delaySeconds: 30 }); + } + } + ``` + (Sentry wiring lives in U8; in U3 the call sites are added as no-ops that U8 fills in.) +- DLQ consumer reads from `packrat-etl-dlq` and, inside a single `db.transaction()`, performs: (1) `INSERT INTO etl_dlq_events (… source = 'consumer')` capturing `{ jobId, chunkIndex, message_body, error_message, error_stack, attempts }`, (2) `UPDATE etl_jobs SET status = 'failed', completed_at = now() WHERE id = $1 AND status = 'running'` — the `WHERE status = 'running'` clause is the no-op gate that prevents racing the U5 sweep. `Sentry.captureException` fires *before* the transaction (so the event survives even if the DB transaction rolls back) with tags `{ jobId, chunkIndex, r2Key }`. The `error_stack` field is contractually free of raw CSV row data — only structural error messages — to avoid accidental PII capture (documented at the call site). +- Wrangler config additions: + ```text + // producer + { "queue": "packrat-etl-dlq", "binding": "ETL_DLQ" } + // consumer + { "queue": "packrat-etl-dlq", "max_batch_size": 10, "max_batch_timeout": 30 } + // on the existing ETL consumer: + "dead_letter_queue": "packrat-etl-dlq", + "max_retries": 3 + ``` + Same shape applied to `*-dev` queues. +- The removal of the per-chunk `status='failed'` write at `processCatalogEtl.ts:201-204` is critical — leaving it would race with the DLQ consumer's state transition. +- `processCatalogETL` rethrows on any internal failure (it already does); no behavioral change other than the consumer's catch now retries instead of swallowing. + +**Patterns to follow:** +- Embeddings consumer pattern at `packages/api/src/services/catalogService.ts:461-507` for the rethrow shape. +- Existing `queue()` dispatch at `packages/api/src/index.ts:109-124` for the new DLQ arm. + +**Test scenarios:** +- Happy path: Single message processes successfully → `message.ack()` called exactly once; no retry; no DLQ row. +- Error path: Transient throw (simulated R2 5xx) → first call: `message.retry({ delaySeconds: 30 })` and no DLQ; second call succeeds → ack. Total DLQ rows = 0. +- Error path: Permanent throw (4 attempts all fail) → exhausts `max_retries: 3` → message routed to `packrat-etl-dlq` → DLQ consumer inserts row in `etl_dlq_events` with `attempts = 4`, captures Sentry, flips `etl_jobs.status = 'failed'`. +- Integration: Un-mock `processQueueBatch` (override `setup.ts:544-551` per-file with `vi.doUnmock`) and exercise the real consumer against an in-memory queue stub. +- Edge case: Two messages in a batch, first throws and second succeeds (this should not happen at `max_batch_size: 1` but the code path supports it) → first retries, second acks; no cross-contamination of state. + +**Verification:** +- New test passes with the per-message catch removed; passes with the catch present too (so the test actually proves the new behavior). +- `bun test:api` overall still green. +- Inspecting `packrat-etl-dlq` queue depth in `wrangler queues info packrat-etl-dlq-dev` after a forced failure shows zero (because the DLQ consumer drains immediately). + +--- + +### U4. Sweep cleanup: remove the broken wall-clock stuck-job sweep before U5 replaces it + +**Goal:** Take the existing `POST /admin/etl/reset-stuck` endpoint out of production rotation before U5's progress-based replacement lands, to stop new false-failures while the rest of Phase 2 ships. + +**Requirements:** R3 + +**Dependencies:** None (independent of U1; this is a code removal) + +**Files:** +- Modify: `packages/api/src/routes/admin/analytics/catalog.ts` (remove or guard the `POST /admin/etl/reset-stuck` route at `:384-409`; if removed, also remove from the OpenAPI spec) +- Modify: `packages/cli/src/commands/admin/etl.ts` (drop any subcommand wired to the removed endpoint) +- Test: `packages/api/test/admin-etl-routes.test.ts` (new or extend existing — assert the route returns 410 Gone or is absent) + +**Approach:** +- Two options, both acceptable: + - **Remove the route entirely.** Anyone calling it gets a 404. Cleanest. Recommended if no automation depends on it. + - **Replace the route body with a 410 Gone response** that links to the runbook (added in U15) and the new sweep design from U5. Use if there's any concern about external automation calling it. +- Existing endpoint logic at `:384-409` does `UPDATE etl_jobs SET status='failed' WHERE status='running' AND started_at < now() - interval '30 minutes'`. This is the SQL that wrongly failed the 7 jobs on 2026-05-14. +- This unit ships before U5 lands the replacement, so for a short window there is no automated sweep at all. Acceptable because stuck-job recovery in that window is operational (U15 runbook documents the manual SQL). + +**Patterns to follow:** +- Existing admin route removal pattern (none in repo as of this writing); fall back to standard Elysia route definition omission. + +**Test scenarios:** +- Happy path: `POST /admin/etl/reset-stuck` returns 410 (or 404 if removed) — test asserts on the chosen behavior. +- Edge case: Admin CLI subcommand for the old endpoint no longer exists (or returns a clear "removed, see runbook" message). + +**Verification:** +- `bun test:api` passes with the new assertion. +- Manual `curl` against dev Worker returns the chosen status code. + +--- + +### U5. P1 #2 fix: progress-based stuck-job sweep + +**Goal:** Replace the wall-clock-based sweep with one that uses `last_progress_at` so healthy long jobs (e.g., 50,100-row `evo` file) are not falsely failed. + +**Requirements:** R3 + +**Dependencies:** U1 (for `last_progress_at`), U2 (for the `last_progress_at` write-on-progress), U4 (so the old sweep is gone first) + +**Files:** +- Create: `packages/api/src/services/etl/sweepStuckJobs.ts` (the sweep function — pure DB logic, no HTTP) +- Modify: `packages/api/src/routes/admin/analytics/catalog.ts` (new `POST /admin/etl/sweep-stuck` endpoint that calls `sweepStuckJobs` and returns the affected rows; for manual triggering) +- Modify: `packages/api/wrangler.jsonc` (declare a CF Cron Trigger for the sweep, e.g., `*/5 * * * *`) +- Modify: `packages/api/src/index.ts` (add `scheduled()` handler that invokes `sweepStuckJobs` on the cron event; if a `scheduled` handler doesn't yet exist, add one) +- Test: `packages/api/test/etl-stuck-job-sweep.test.ts` (new) + +**Approach:** +- Sweep runs inside a single `db.transaction()`: + 1. `UPDATE etl_jobs SET status='failed', completed_at = now() WHERE status='running' AND COALESCE(last_progress_at, started_at) < now() - interval '15 minutes' RETURNING id, source, filename, started_at, last_progress_at, chunks_total, chunks_completed`. (The `COALESCE` defends against any legacy row that somehow escaped the U1 backfill.) + 2. For each returned row, `INSERT INTO etl_dlq_events (job_id, error_message, source) VALUES ($1, 'sweep:no_progress', 'sweep')` so the forensic table is the single source of truth for *every* failed transition — whether triggered by the consumer DLQ or by the sweep. `chunk_index = NULL` in sweep-sourced events. +- Returned rows also feed a Sentry warning event per affected job (`level: warning`, tags `{ jobId, source: 'sweep' }`, extra includes `chunks_completed/chunks_total` so the operator immediately sees how far the job got). +- 15-minute interval matches the CF Queue consumer wall-clock cap. Any chunk making real progress writes `last_progress_at = now()` (via U2's modification to `updateEtlJobProgress`), so this only catches truly stalled jobs. +- CF Cron Trigger every 5 minutes (configurable via env if needed). The cron handler is idempotent — the partial index from U1 keeps the query cheap even at thousands of jobs. Wrangler config shape: `"triggers": { "crons": ["*/5 * * * *"] }` — top-level `triggers` object wrapping a `crons` array, not a bare top-level `crons` key. +- Manual admin endpoint exists for on-demand sweep — useful during incident response. + +**Patterns to follow:** +- Admin route structure at `packages/api/src/routes/admin/analytics/catalog.ts` for the new endpoint. +- CF Cron Triggers config in `wrangler.jsonc` (the repo has none today — this is the first; reference ). + +**Test scenarios:** +- Happy path: Insert a job with `status='running'`, `last_progress_at = now() - 30min` → sweep flips it to `failed`. +- Edge case: Insert a job with `status='running'`, `last_progress_at = now() - 5min` → sweep leaves it alone (within budget). +- Edge case: Insert a job with `last_progress_at = NULL` (somehow — legacy row that escaped backfill) → COALESCE the column with `started_at` in the WHERE clause so it still gets evaluated. +- Edge case: 50,100-row job in progress — chunks write `last_progress_at = now()` every 100 rows → sweep never fires on it. +- Integration: Cron-event simulation calls the same code path as the admin endpoint; both return identical results for the same DB state. +- Error path: Sweep query fails (DB down) → caller observes the error; Sentry captures; cron does not silently mask. + +**Verification:** +- After running the sweep against a DB with the seeded test cases, exactly the long-stalled rows are affected. +- `bun test:api` includes the new test and passes. +- Dev cron schedule fires (`wrangler dev --test-scheduled`) and exercises the handler. + +--- + +### U6. P1 #1 fix: shared chunking helper + retry endpoint + repair-from-scratch endpoint + +**Goal:** Both retry and repair use the same producer chunking logic. The repair endpoint creates a brand-new `etl_jobs` row linked to the broken historical one — directly enabling the operational recovery of the 7 wrongly-`failed` jobs from 2026-05-14. + +**Requirements:** R5, R8 + +**Dependencies:** U1 (for `superseded_by_job_id`), U2 (for `chunks_total` write semantics) + +**Files:** +- Create: `packages/api/src/services/etl/chunkCsvForR2.ts` (extracted shared helper: takes `objectKey`, returns an array of `{ chunkIndex, chunksTotal, byteStart, byteEnd }` with newline-aligned boundaries — newline alignment itself ships in U7) +- Modify: `packages/api/src/routes/catalog/index.ts` (replace inline chunking at `:253-271` with a call to `chunkCsvForR2`) +- Modify: `packages/api/src/routes/admin/analytics/catalog.ts` (rewrite `POST /admin/etl/:jobId/retry` at `:413-470` to use `chunkCsvForR2`; add new `POST /admin/etl/:jobId/repair-from-scratch`) +- Modify: `packages/api/src/services/etl/queue.ts` (extend `queueCatalogETL` to accept pre-computed chunks rather than constructing them — or accept either, with the chunk-construction path migrating to the shared helper) +- Modify: `packages/cli/src/commands/admin/etl.ts` (add `retry ` subcommand if not present, plus new `repair-from-scratch ` subcommand) +- Test: `packages/api/test/etl-retry-repair.test.ts` (new) + +**Approach:** +- `chunkCsvForR2(objectKey, r2, options?)`: signature returns `Promise`. Calls `r2.head(objectKey)`, splits into 20 MB windows. Newline-alignment lives in U7 but the shape lands here so U7 is a fill-in. +- Retry endpoint (`POST /admin/etl/:jobId/retry`): looks up `(source, filename, scraperRevision)` from the existing job, generates a fresh `jobId`, INSERTs a new `etl_jobs` row with `chunks_total = chunkCsvForR2(...).length`, sets `superseded_by_job_id = ` on the new row only if the original is `failed`, sends batch. +- Repair-from-scratch (`POST /admin/etl/:jobId/repair-from-scratch`): same behavior as retry but always sets `superseded_by_job_id` and `superseded_at = now()` on the new row, and always re-reads the full file (even if the original was `completed`). Use case: an operator suspects a `completed` job is undercount; repair recreates from scratch. +- **R2 ETag verification (failure-closed)**: before creating the new job row, both endpoints call `r2.head(objectKey)` and compare the returned `etag` (and `lastModified`) against the original job's recorded values. If the original job has no `etag` stored (legacy rows), require an explicit `?force=true` query flag. If the `etag` differs (source was overwritten by a later scrape), return 409 Conflict with a clear message naming both etags — never silently re-ingest a different file under the same path. (This implies adding `source_etag text` and `source_last_modified timestamp` to `etl_jobs` — fold into U1's column list if not already, or capture as a follow-up here.) +- Both endpoints accept an optional `?dryRun=true` query that returns the planned chunk spec without enqueuing anything — operator preview. +- The 7 historical jobs from 2026-05-14 will be recovered by calling repair-from-scratch on each of them once Phase 1+2 ships. U15 runbook documents the operator procedure including the ETag verification step. + +**Patterns to follow:** +- Admin route structure at `packages/api/src/routes/admin/analytics/catalog.ts:178-235` for response shape. +- Existing retry endpoint at `:413-470` for the basic flow (just don't replicate the broken single-chunk behavior). + +**Test scenarios:** +- Happy path: Retry a failed job with a 50 MB source file → 3 chunks created via `chunkCsvForR2`, 3 messages sent, new `etl_jobs` row has `chunks_total = 3`, `superseded_by_job_id` matches original. +- Happy path: Repair-from-scratch a `completed` job with apparent undercount → new job created with `superseded_by_job_id` set; original row untouched. +- Edge case: Retry a single-chunk legacy job (file size < 20 MB) → 1 chunk, `chunks_total = 1`, behaves identically to the producer endpoint. +- Edge case: Retry on a job whose `filename` no longer exists in R2 → endpoint returns 404 with a clear message; no new `etl_jobs` row. +- Edge case: `?dryRun=true` returns the planned chunk spec; no DB writes, no queue sends. +- Integration: Repair-from-scratch on a 50,100-row file (the `evo` case) produces the expected ~3 chunks, all enqueued, and after the full pipeline completes the new job's `total_processed` matches the file's actual row count. +- Covers AE: the 7 jobs from 2026-05-14 can each be repaired by calling repair-from-scratch — verified manually post-deploy. + +**Verification:** +- Both endpoints documented in the OpenAPI spec emitted by `@elysiajs/openapi`. +- CLI subcommands invoke the endpoints with proper auth. +- `bun test:api` passes the new integration test. + +--- + +### U7. P1 #3 + P1 #4 + P1 #5 fix: row-boundary-aligned chunks + robust header injection + +**Goal:** No row is silently dropped, invalidated, or split across chunks. Wide-CSV headers (>4 KB) fail loudly instead of silently misaligning columns. + +**Requirements:** R4, R6 + +**Dependencies:** U6 (for `chunkCsvForR2`) + +**Files:** +- Modify: `packages/api/src/services/etl/chunkCsvForR2.ts` (implement newline alignment — for each 20 MB window, read the next 64 KB tail, find the last `\n`, snap `byteEnd` to the byte before that newline) +- Modify: `packages/api/src/services/etl/processCatalogEtl.ts` (remove `skipPartialRow` at `:95-108`; rewrite header injection at `:50-58` with a bounded expand loop 4K→16K→64K; throw a typed error if no newline in 64 KB) +- Test: `packages/api/test/etl-chunk-boundaries.test.ts` (new) + +**Approach:** +- Newline alignment in producer: + - For each chunk window `[start, start + 20MB)`: + - Read `[start + 20MB - 64KB, start + 20MB)`. + - Find the index of the last `\n` in that slice. + - If found: `byteEnd = (start + 20MB - 64KB) + lastNewlineIndex`. The next chunk's `byteStart = byteEnd + 1`. + - If not found in 64 KB (extremely unlikely with normal CSV row sizes): throw `ChunkBoundaryError` immediately, surfacing to Sentry and aborting the job creation. Caller is told the file has a row larger than 64 KB. + - Last chunk: `byteEnd = file.size - 1`. +- Header re-fetch in consumer (for `chunkIndex > 0`): + ```text + let headerSlice = await r2.get(key, { range: { offset: 0, length: 4096 }}).then(b => b.text()); + let nlIdx = headerSlice.indexOf('\n'); + if (nlIdx === -1) { + headerSlice = await r2.get(key, { range: { offset: 0, length: 16384 }}).then(b => b.text()); + nlIdx = headerSlice.indexOf('\n'); + } + if (nlIdx === -1) { + headerSlice = await r2.get(key, { range: { offset: 0, length: 65536 }}).then(b => b.text()); + nlIdx = headerSlice.indexOf('\n'); + } + if (nlIdx === -1) throw new EtlHeaderError(`No newline in first 64 KB of ${key} — malformed header`); + const headerRow = headerSlice.slice(0, nlIdx); + ``` +- Since chunks are now newline-aligned, `skipPartialRow` is no longer needed — the consumer can stream the chunk body directly into the parser after prepending the header. +- BOM handling: if the first byte of the header slice is `0xEF 0xBB 0xBF`, strip it before extracting the header row. Same treatment for the first chunk. + +**Patterns to follow:** +- R2 byte-range read pattern at `packages/api/src/services/etl/processCatalogEtl.ts:54, 71`. +- Typed-error pattern: extend whatever the repo uses for domain errors (typically `Error` subclasses in `packages/api/src/utils/errors.ts`). + +**Test scenarios:** +- Happy path: 5 MB file, 1 chunk → no boundary logic exercised; row count matches actual. +- Happy path: 60 MB file, 3 chunks; rows of varying width; all `byteEnd` values land immediately before a `\n`; total row count across chunks = file row count. +- Edge case: Chunk boundary lands exactly on a newline character (`source[byteEnd] === '\n'`) → still aligned; next chunk starts on next row; no dropped row. +- Edge case: Header row of 4500 bytes (just over 4 KB) → re-fetch expands to 16 KB, succeeds; columns mapped correctly. +- Edge case: Header row of 50 KB (one absurdly wide CSV) → re-fetch expands to 64 KB, succeeds. +- Edge case: BOM at start of file → stripped from header extraction in both chunk-0 and re-fetch paths. +- Error path: File with no newline in first 64 KB → throws `EtlHeaderError`; job marked `failed` via DLQ (U3). +- Error path: Row larger than 64 KB encountered at a chunk boundary → producer throws `ChunkBoundaryError`; no job created. +- Integration: A real CSV from prod (anonymized fixture in `packages/api/test/fixtures/`) splits into multiple chunks; sum of consumer-reported `totalProcessed` across all chunks equals `wc -l fixture.csv - 1` (subtract header). +- Covers AE: A 50,100-row file (the `evo` shape) ingested via the new chunking logic shows `total_processed = 50100`, `total_valid + total_invalid = 50100`, no missing rows. + +**Verification:** +- Manual run on a real prod fixture file with `wc -l` cross-check matches the job's `total_processed`. +- `bun test:api` passes the new fixture-driven test. +- Sentry catches the malformed-header case during the next dev exercise. + +--- + +### U8. Sentry wiring via `@sentry/cloudflare` + +**Goal:** Every uncaught exception in the API Worker — including queue-consumer paths — emits a Sentry event with structured tags. Operators can debug a stuck job without paging through raw Worker logs. + +**Requirements:** R9 + +**Dependencies:** None (independent; can start in parallel with Phase 1 but lands in Phase 3) + +**Files:** +- Modify: `packages/api/package.json` (add `@sentry/cloudflare` dependency; pin to a specific version) +- Modify: `packages/api/src/index.ts` (wrap the Worker default export with `Sentry.withSentry({...}, { fetch, queue })`; pass the Sentry options factory that reads `env.SENTRY_DSN`) +- Modify: `packages/api/src/utils/env-validation.ts` (no schema change — `SENTRY_DSN` is already declared at `:9, 94`; verify it's required vs optional and adjust accordingly so dev doesn't break without a DSN) +- Modify: `packages/api/wrangler.jsonc` (add `upload_source_maps: true` at the top level) +- Modify: `packages/api/src/services/etl/queue.ts` (fill in the `Sentry.captureException(...)` call site that U3 stubbed) +- Modify: `packages/api/src/services/etl/processCatalogEtl.ts` (Sentry breadcrumbs at chunk-start, batch-flush, and chunk-end; `Sentry.startSpan` around the chunk lifecycle) +- Create: `packages/api/src/utils/logger.ts` (the thin structured logger — accepts `LogContext`, emits JSON-prefixed `console.log` lines, also calls `Sentry.addBreadcrumb` when Sentry is initialized) +- Modify: All `packages/api/src/services/etl/*.ts` console calls migrated to `logger.{info,warn,error}` (mechanical change — sweeps across the ETL files) +- Test: `packages/api/test/sentry-instrumentation.test.ts` (new — mocks `@sentry/cloudflare` and asserts captureException/breadcrumb call shape) + +**Approach:** +- `withSentry({ fetch, queue })` wraps the existing default export at `packages/api/src/index.ts`. The Sentry options factory reads `env.SENTRY_DSN`, `env.ENVIRONMENT`, sets `tracesSampleRate: 0.1`. +- Queue consumer instrumentation per : + - `Sentry.startSpan({ op: 'queue.process', name: 'etl-chunk', attributes: { 'messaging.message.id': msg.id, 'messaging.message.retry.count': msg.attempts, 'jobId': msg.body.id, 'chunkIndex': msg.body.data.chunkIndex } }, async () => { ... })`. + - `Sentry.captureException(err, { tags: { jobId, chunkIndex, r2Key }, contexts: { queue: { messageId, attempts } } })` inside the catch. +- DLQ consumer (from U3) gets the same treatment. +- `logger.ts`: ~30 lines. Functions: `info(event, ctx)`, `warn(event, ctx)`, `error(event, ctx, err?)`. Emits a JSON line; if Sentry is initialized, also calls `Sentry.addBreadcrumb({ category: event, data: ctx, level })`. +- Source maps: `upload_source_maps: true` works with Wrangler 4.x and `compatibility_date: 2025-06-01`. + +**Patterns to follow:** +- No existing Sentry initialization in `packages/api` — this is the first. +- Reference Sentry-in-CF guidance: . + +**Test scenarios:** +- Happy path: Successful chunk → one `startSpan` invocation, breadcrumbs at chunk-start/flush/end, no `captureException`. +- Error path: Chunk throws → `captureException` called once with expected tags; span marks status `internal_error`. +- Edge case: `SENTRY_DSN` empty (dev without secret) → no Sentry calls fire; logger still emits lines; no crash. +- Edge case: Logger called before Sentry initialized (cold-start race) → graceful no-op on breadcrumb path; logger.info still emits the line. +- Integration: A real Sentry test project receives events from `bun api` dev-server when forced failures are triggered. + +**Verification:** +- Dev `bun api` cold start logs the Sentry init line. +- A forced chunk failure produces a Sentry event visible in the project. +- All `packages/api/src/services/etl/*.ts` files have zero `console.*` references (`grep -rn 'console\.' packages/api/src/services/etl/` returns nothing). + +--- + +### U9. P2 #2 + P2 #3 + P2 #4 fix: error propagation + embedding-failure observability + IIFE error handling + +**Goal:** Three related but smaller correctness issues that all share the theme "errors should not vanish silently." + +**Requirements:** R2, R10 + +**Dependencies:** U1 (for `total_embedding_failures`), U8 (so the new error sites can `Sentry.captureException`) + +**Files:** +- Modify: `packages/api/src/services/etl/processLogsBatch.ts` (rethrow on DB failure at `:25-27`; remove the swallow) +- Modify: `packages/api/src/services/etl/processValidItemsBatch.ts` (in the embedding-fallback path at `:52-63`, atomically increment `etl_jobs.total_embedding_failures` before upserting; surface a Sentry warning event with `jobId` and the affected SKU count; do not throw) +- Modify: `packages/api/src/services/etl/processCatalogEtl.ts` (wrap the writer IIFE at `:89-117` in an explicit promise: `const writerPromise = (async () => { ... })().catch(err => parser.destroy(err)); ...; await writerPromise.catch(err => { throw err })` so unhandled rejections become outer-flow throws) +- Modify: `packages/api/src/routes/admin/analytics/catalog.ts` (extend the admin job-list response to include `totalEmbeddingFailures` so dashboards can surface degradation) +- Test: `packages/api/test/etl-error-propagation.test.ts` (new) + +**Approach:** +- `processLogsBatch`: catch block currently logs and returns. Replace with `throw err`. The outer `processCatalogETL` catch already exists and the chunk will retry/DLQ correctly via U3. +- Embedding fallback: at `processValidItemsBatch.ts:52-63`, on `generateManyEmbeddings` throw: + ```text + await db.update(etlJobs).set({ totalEmbeddingFailures: sql`COALESCE(${etlJobs.totalEmbeddingFailures}, 0) + ${items.length}` }).where(eq(etlJobs.id, jobId)); + logger.warn('etl.embedding.fallback', { jobId, skuCount: items.length }); + Sentry.captureMessage('etl.embedding.fallback', { level: 'warning', tags: { jobId }, extra: { skuCount: items.length } }); + // continue with upsert; embedding stays NULL + ``` +- IIFE wrap pattern: + ```text + const writerPromise = (async () => { ... })() + .catch(err => { parser.destroy(err); throw err; }); + // ... for await loop ... + await writerPromise; + ``` + Any rejection in the writer now propagates to the outer try/catch in `processCatalogETL` and triggers retry/DLQ via U3. +- Admin response extension: extend the existing `GET /admin/analytics/catalog/etl` route's select shape to include `totalEmbeddingFailures` and update the response Zod schema if one is declared. + +**Patterns to follow:** +- Atomic update idiom at `packages/api/src/services/etl/updateEtlJobProgress.ts:16-23`. +- Admin route response shape at `packages/api/src/routes/admin/analytics/catalog.ts:178-235`. + +**Test scenarios:** +- Happy path (embedding fallback): Embedding service throws → SKUs upserted with `embedding=NULL`; `total_embedding_failures` increments by exactly `items.length`; Sentry warning event fires once per batch (not per SKU). +- Happy path (logs rethrow): `processLogsBatch` DB INSERT fails → exception propagates to outer catch → chunk retried by CF Queue. +- Happy path (IIFE wrap): Writer throws inside the async IIFE → parser destroyed; outer `for await` loop terminates; outer catch fires; chunk retried. +- Edge case: Multiple consecutive embedding batches in one chunk all fall back → counter increments cumulatively; Sentry warnings fire once per batch, not once per chunk. +- Edge case: Mixed batch — some SKUs embed, then fallback kicks in for the rest → counter increments only for the failed batch's SKU count. +- Integration: Admin endpoint response includes `totalEmbeddingFailures` field for every job; the prod-shape dashboard query still parses cleanly. + +**Verification:** +- New test passes with the rethrow / wrap / counter increments in place. +- `bun test:api` overall green. +- Dev admin endpoint `GET /admin/analytics/catalog/etl?limit=5` returns the new field. + +--- + +### U10. Reconciliation: admin endpoint + automatic post-job verification (via dedicated queue) + CLI subcommand + +**Goal:** Every ETL completion writes a verification row count; operators can also trigger reconciliation on any job on demand. Surfaces the user's "missing or falsely labeling" concern as a first-class observable signal. Auto-reconciliation runs on its own queue, not via `ctx.waitUntil`, so multi-GB files do not exceed the queue invocation's 15-min wall-clock. + +**Requirements:** R7 + +**Dependencies:** U1 (for `verified_at`, `verified_row_count`, `verified_row_count_partial`), U2 (for the completion transition that enqueues the reconcile message), U8 (for Sentry warnings on delta) + +**Files:** +- Create: `packages/api/src/services/etl/reconcileJob.ts` (pure function: given a `jobId` and optional `resumeFromByte`, stream the R2 source in 100 MB byte-range windows, count newlines, checkpoint progress, finalize verification on EOF, return delta) +- Create: `packages/api/src/services/etl/processReconcileBatch.ts` (queue consumer for `packrat-etl-reconcile-queue`; calls `reconcileJob`; handles retry/resume) +- Modify: `packages/api/src/services/etl/queue.ts` (extend producer to enqueue reconcile messages; type `ReconcileMessage { jobId: string; resumeFromByte?: number }`) +- Modify: `packages/api/src/services/etl/processCatalogEtl.ts` (on the final-chunk completion transition from U2, enqueue a `ReconcileMessage` to `packrat-etl-reconcile-queue` *inside the same transaction* as the status flip so a row can never transition to `completed` without an enqueued reconcile) +- Modify: `packages/api/src/index.ts` (extend the `queue()` switch with an arm for `packrat-etl-reconcile-queue` and `packrat-etl-reconcile-queue-dev`) +- Modify: `packages/api/wrangler.jsonc` (declare `packrat-etl-reconcile-queue` and `packrat-etl-reconcile-queue-dev` as producer + consumer with its own `dead_letter_queue: 'packrat-etl-dlq'` and `max_retries: 3`) +- Modify: `packages/api/src/routes/admin/analytics/catalog.ts` (add `POST /admin/etl/:jobId/reconcile` — calls `reconcileJob` synchronously; for small/medium files returns inline; for large files returns 202 Accepted and enqueues to the reconcile queue with the existing job id) +- Modify: `packages/cli/src/commands/admin/etl.ts` (add `reconcile ` subcommand) +- Modify: admin list endpoint response shape (include `verifiedAt`, `verifiedRowCount`, and `verifiedRowCountPartial` so the dashboard surfaces it) +- Test: `packages/api/test/etl-reconciliation.test.ts` (new) + +**Approach:** +- `reconcileJob(jobId, resumeFromByte = 0)`: + 1. Read `(filename, total_processed, verified_at, verified_row_count_partial)` from `etl_jobs`. If `verified_at IS NOT NULL`, return early — idempotent no-op for redelivered messages. + 2. `r2.head(key)` → `fileSize`. + 3. From `resumeFromByte` (or `verified_row_count_partial`'s checkpoint byte position if set), read 100 MB byte-range windows. For each window: + - Count `\n` bytes in the window. + - Add to running `lineCount`. + - On the last window, subtract 1 for the header row. + - Every 5 windows (500 MB processed) or when elapsed time > 10 min: `UPDATE etl_jobs SET verified_row_count_partial = $lineCount` (checkpoint), then throw a typed `ReconcileResumeError` carrying the current byte offset so the queue retry re-enqueues with `resumeFromByte` advanced. Wall-clock budget reset. + 4. On EOF: `UPDATE etl_jobs SET verified_at = now(), verified_row_count = $lineCount, verified_row_count_partial = NULL WHERE id = $1 AND verified_at IS NULL` (idempotency gate). + 5. Compute `delta = lineCount - total_processed`. If `abs(delta) > max(10, ceil(0.01 * lineCount))`: `Sentry.captureMessage('etl.reconciliation.delta', { level: 'warning', tags: { jobId }, extra: { delta, expected: lineCount, actual: total_processed } })`. + 6. Return `{ jobId, expectedRowCount: lineCount, actualRowCount: total_processed, delta, withinThreshold }`. +- `processReconcileBatch` (queue consumer): + - For each message: try `reconcileJob(msg.jobId, msg.resumeFromByte)` → on success `ack()`. On `ReconcileResumeError`: enqueue a new message with the advanced offset and `ack()` the current one. On any other error: `retry({ delaySeconds: 60 })`. +- Auto-trigger: in U2's completion transaction, after the status flip to `completed`, enqueue `{ jobId, resumeFromByte: 0 }` to `packrat-etl-reconcile-queue`. Because both writes are in the same transaction, a row can never be `completed` without an enqueued reconcile. +- Manual endpoint (`POST /admin/etl/:jobId/reconcile`): + - For files where `fileSize < 200 MB`: call `reconcileJob` synchronously and return the result inline. + - For files ≥ 200 MB: enqueue to `packrat-etl-reconcile-queue` and return 202 with a "poll the job for `verified_at`" message. + - Optional `?force=true` query: clear `verified_at` first and re-enqueue (operator override for a re-verify). +- CLI subcommand: `packrat-admin etl reconcile ` → wraps the endpoint, polls until `verifiedAt` is set or timeout. +- The 7 historical jobs from 2026-05-14 can each be reconciled retroactively via this endpoint *before* deciding to repair (U6). Confirms the suspicion that they processed partial data before being swept. + +**Patterns to follow:** +- Queue consumer pattern from U3 (per-message ack/retry, DLQ wired). +- Streaming-count pattern: `for await (const chunk of body)` and accumulate `chunk.filter(byte => byte === 0x0A).length`. + +**Test scenarios:** +- Happy path: Job with `total_processed = 100`, R2 file has 101 lines (100 rows + header) → delta = 0; `verified_at` set; no Sentry warning. +- Happy path: Job with `total_processed = 1000`, R2 file has 1006 lines (1005 rows + header) → delta = 5; within threshold; no warning. +- Edge case: Job with `total_processed = 50000`, R2 file has 50100 lines + header → delta = 100; threshold = `max(10, 500)` = 500; within threshold; no warning. (The 50,100 case stays informational.) +- Edge case (the real case): Job with `total_processed = 400`, R2 file has 50101 lines (50100 rows + header) — what the `campmor`-shape failures looked like → delta = 49700; way over threshold; Sentry warning fires. +- Edge case (resume): A 1.5 GB file forces three resume-error checkpoints; each resume picks up at the right byte offset; final `verified_row_count` matches the true row count. +- Edge case (idempotency): A redelivered reconcile message with `resumeFromByte = 0` against a job that already has `verified_at` set — `reconcileJob` returns early without re-reading the file. +- Error path: R2 object missing → `reconcileJob` throws a typed error; queue consumer retries with backoff; after exhausting `max_retries: 3`, the DLQ captures it. +- Edge case: Job with `total_processed = NULL` (legacy stuck-job-sweep casualty) → reconcileJob computes delta as `expected - 0 = expected`; the warning carries useful context for diagnosing the historical job. +- Integration: Auto-verify fires exactly once per job, enqueued atomically with the completion transition; it does not fire for intermediate chunk completions; it does not fire twice on a redelivered final chunk (idempotency comes from the `etl_job_chunks` gate in U2). + +**Verification:** +- New test passes. +- Calling the endpoint on a real dev-seeded job returns the documented shape (inline for small files, 202 + queued for large). +- The chunk-completion transaction either commits both the status flip and the reconcile enqueue, or neither (verify with a forced enqueue failure mid-transaction). + +--- + +### U11. Quality-of-life: scheduler.wait, BATCH_SIZE rename, mergeBySku log aggregation + +**Goal:** Three tiny correctness/cleanliness wins that share a maintenance flavor and ship together. + +**Requirements:** R9 (log volume), and audit P2 #5, P2 #6, P3 #1 + +**Dependencies:** U8 (for the logger surface used by the aggregated merge summary) + +**Files:** +- Modify: `packages/api/src/services/etl/processCatalogEtl.ts:120` (replace `setTimeout(resolve, 1)` with `await scheduler.wait(0)`) +- Create: `packages/api/src/services/etl/constants.ts` (new — exports `ITEM_FLUSH_BATCH_SIZE = 100` and `CF_QUEUE_BATCH_SIZE = 100`) +- Modify: `packages/api/src/services/etl/processCatalogEtl.ts:13` and `packages/api/src/services/etl/queue.ts:17` (import from the new constants module instead of declaring inline) +- Modify: `packages/api/src/services/etl/mergeItemsBySku.ts:34-48` (replace per-SKU `console.log` with a per-batch summary `logger.info('etl.merge.summary', { jobId, mergedSkuCount, totalChangedFields })`) +- Test: `packages/api/test/etl-yield-and-constants.test.ts` (new — minimal; mostly behavior-preservation) + +**Approach:** +- `await scheduler.wait(0)` is the documented Workers Scheduler API. `scheduler.yield()` does not exist (corrected from audit P2 #5). +- The constants module is dead-simple — two exports — but the rename surfaces intent at the call site and ends the ambiguity the audit flagged at P2 #6. +- The mergeBySku aggregation accumulates change counts across one batch (already a natural unit) and logs once at the end. No per-SKU lines. + +**Patterns to follow:** +- Module organization mirrors `packages/api/src/services/etl/types.ts` for a constants file. + +**Test scenarios:** +- Behavior preservation: A 10,000-row chunk completes at least as fast as before with `scheduler.wait(0)` (regression check, not a strict assertion). +- Happy path (merge log): A batch with 50 SKU merges → exactly one log line emitted, summarizing the batch. +- Edge case: A batch with 0 merges → no log line. + +**Verification:** +- `grep -rn "setTimeout\(.*1.*\)" packages/api/src/services/etl/` returns nothing. +- `grep -rn "BATCH_SIZE\s*=" packages/api/src/services/etl/` returns only the new constants. +- A real ETL run on dev with 1k duplicate SKUs produces 1 merge summary line, not 1000. + +--- + +### U12. Validator hardening: URL scheme + length caps + SKU charset + +**Goal:** Eliminate the audit P3 #2 attack surface — `javascript:` URLs and oversize fields cannot enter the catalog. + +**Requirements:** R11 + +**Dependencies:** None (independent; can land any time after Phase 1) + +**Files:** +- Modify: `packages/api/src/services/etl/CatalogItemValidator.ts` (rewrite `isValidUrl` at `:60-67`; add length caps and SKU regex) +- Test: `packages/api/test/etl-validator.test.ts` (new or extend existing) + +**Approach:** +- `isValidUrl`: parse with `new URL()`; reject any scheme other than `http:` and `https:`. Reject URLs longer than 2048 chars. +- Length caps (rejects, not truncates): `name ≤ 500`, `description ≤ 50000`, `brand ≤ 200`, `category ≤ 200`. +- SKU regex: `/^[A-Za-z0-9_.\-\/]+$/`; max length 200. +- Each rejection produces a structured invalid-item log entry with the specific reason — surfaces in the existing `/admin/etl/:jobId/failures` endpoint. + +**Patterns to follow:** +- Existing validator structure at `packages/api/src/services/etl/CatalogItemValidator.ts`. +- Invalid-log shape at `packages/api/src/services/etl/processLogsBatch.ts`. + +**Test scenarios:** +- Happy path: Valid `https://example.com/product/123` URL accepted. +- Error path: `javascript:alert(1)` URL rejected with reason `INVALID_URL_SCHEME`. +- Error path: `mailto:foo@bar` rejected with `INVALID_URL_SCHEME`. +- Error path: URL of 3000 chars rejected with `URL_TOO_LONG`. +- Edge case: Name of exactly 500 chars accepted; 501 chars rejected. +- Edge case: SKU `ABC-123_/test.sku` accepted; SKU `