Add Go based backend

.github/workflows/api-server.yml

@@ -0,0 +1,79 @@
+name: api-server
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'apps/api/**'
+      - '.github/workflows/api-server.yml'
+  pull_request:
+    paths:
+      - 'apps/api/**'
+      - '.github/workflows/api-server.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: railforless/tracky-server
+
+jobs:
+  test:
+    name: Run API tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: apps/api/go.mod
+
+      - name: Run Go tests
+        working-directory: ./apps/api
+        run: go test ./...
+
+  build:
+    name: Build & push server image
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Compute image tags
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,format=short,prefix=
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Build & push
+        uses: docker/build-push-action@v6
+        with:
+          context: ./apps/api
+          file: ./apps/api/Dockerfile.server
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          provenance: false

.gitignore

@@ -62,6 +62,14 @@ apps/web/out/
 
 # Go backend
 apps/api/bin/
+apps/api/server
+apps/api/sync-realtime
+apps/api/sync-static
+*.db
+*.db-shm
+*.db-wal
+*.pmtiles
+apps/api/debug_*
 
 # bun (not used, but just in case)
 bun.lockb

README.md

@@ -70,7 +70,8 @@ Tracky/
 ├── apps/
 │   ├── mobile/    Expo app (iOS + Android)
 │   ├── web/       Next.js landing page
-│   └── api/       Go API server
+│   ├── api/       Go API server + edge collector
+│   └── collector/ Cloudflare Worker hosting the collector container
 ├── packages/      Shared code (reserved, not yet populated)
 ├── package.json   Root workspace scripts
 └── pnpm-workspace.yaml
@@ -139,12 +140,34 @@ pnpm dev
 
 ### Run the backend
 
+The backend is split into two Go binaries plus a thin Cloudflare Worker that hosts the collector container.
+
+**On-prem server** ([apps/api/cmd/server](apps/api/cmd/server)) — accepts collector ingest, drains R2 backlog on outage recovery, serves WebSocket clients:
+
 ```bash
 cd apps/api
-go run ./cmd/api
+INGEST_SECRET=dev go run ./cmd/server
 ```
 
-The API server starts on port 8080 by default (configurable via `PORT` env var).
+Listens on `:8080` (override with `PORT`). Set the four `R2_*` vars in `cmd/server/.env.example` to enable the drainer.
+
+**Edge collector** ([apps/api/cmd/collector](apps/api/cmd/collector)) — polls every provider and ships per-tick snapshots:
+
+```bash
+cd apps/api
+# Mock mode (default): logs snapshots to stderr, no network destination.
+go run ./cmd/collector
+
+# Talk to a local server:
+INGEST_URL=http://localhost:8080 INGEST_SECRET=dev go run ./cmd/collector
+```
+
+**Collector Worker** ([apps/collector](apps/collector)) — only needed when validating the container/R2 path:
+
+```bash
+cd apps/collector
+npx wrangler dev
+```
 
 ### EAS builds
 
@@ -190,11 +213,43 @@ pnpm lint              # ESLint
 Backend scripts run from `apps/api/`:
 
 ```bash
-go run ./cmd/api       # Start the API server
-go test ./...          # Run all tests
-go build -o bin/api ./cmd/api  # Build binary
+go run ./cmd/server                   # Start the on-prem ingest + WS server
+go run ./cmd/collector                # Start the edge collector (mock emitter by default)
+go run ./cmd/sync-static              # One-shot static GTFS sync to the DB
+
+go test ./...                         # Run all Go tests
+go test ./drainer/... -v              # Pure-logic drainer replay tests
+go test ./collector/... -v            # Emitter chain + poller tests
+go test ./routes/... -v               # /ingest handler tests
+
+go build -o bin/server ./cmd/server   # Build the server binary
+go build -o bin/collector ./cmd/collector
+```
+
+Collector Worker scripts run from `apps/collector/`:
+
+```bash
+npx wrangler dev                      # Local Worker dev server (incl. R2 emulation)
+npx wrangler types                    # Regenerate Env types after wrangler.jsonc edits
+npx vitest run                        # Worker tests (runs on the real workerd runtime)
 ```
 
+#### Local end-to-end smoke
+
+In two terminals (no Cloudflare deps required):
+
+```bash
+# 1. Server
+cd apps/api && INGEST_SECRET=dev go run ./cmd/server
+
+# 2. Collector → server (per-tick snapshots over HTTP)
+cd apps/api && INGEST_URL=http://localhost:8080 INGEST_SECRET=dev go run ./cmd/collector
+```
+
+Then connect a WebSocket client to `ws://localhost:8080/ws/realtime` and send `{"action":"subscribe","providers":["amtrak"]}` — you should see `realtime_update` messages within ~30 s.
+
+To exercise the R2 backlog path, kill the server while the collector is running with `BACKLOG_URL` set, then restart the server with the four `R2_*` env vars to watch the drainer replay backlogged snapshots.
+
 ### Code Quality
 
 - **TypeScript** in strict mode
@@ -227,7 +282,24 @@ apps/mobile/
 └── constants/              # Theme and configuration
 
 apps/api/
-└── cmd/api/                # Entry point (main.go)
+├── cmd/server/             # On-prem ingest + WebSocket server
+├── cmd/collector/          # Edge collector (runs in Cloudflare container)
+├── cmd/sync-static/        # CLI for static GTFS sync + tile generation
+├── collector/              # Snapshot, Emitter chain (HTTP + Storage + Fallback)
+├── drainer/                # R2 backlog replay (pure Replay logic + S3 glue)
+├── realtime/               # Processor (single ingest sink → ws.Hub, TODO timescale)
+├── routes/                 # HTTP handlers incl. POST /ingest
+├── providers/              # Per-operator GTFS-RT parsers
+├── gtfs/                   # Shared GTFS static + realtime parsing
+├── ws/                     # Hub + WebSocket client handlers
+└── spec/                   # TrainPosition, TrainStopTime, etc.
+
+apps/collector/
+├── src/
+│   ├── index.ts            # Worker entry (just /health)
+│   └── collector-container.ts  # CollectorContainer DO + outbound() R2 proxy
+├── container/Dockerfile    # Multi-stage Go build for the collector
+└── wrangler.jsonc          # Container + R2 + DO bindings
 
 apps/web/
 ├── app/

apps/api/.dockerignore

@@ -0,0 +1,31 @@
+# Build artifacts
+bin/
+server
+sync-static
+sync-realtime
+
+# Local databases (tracky.db is ~800 MB!)
+*.db
+*.db-shm
+*.db-wal
+
+# Generated tile output
+*.pmtiles
+*.geojson
+
+# Debug dumps
+debug_*
+cmd/server/debug_*.json
+
+# Local env (also caught by gitignore but be explicit for build context)
+.env
+.env.*
+!.env.example
+
+# IDE / OS
+.DS_Store
+.vscode/
+
+# Dockerfiles themselves don't need to be in the image
+Dockerfile*
+.dockerignore

apps/api/Dockerfile.server

@@ -0,0 +1,30 @@
+# syntax=docker/dockerfile:1.7
+
+# ── Build stage ──────────────────────────────────────────────────────────────
+FROM golang:1.26-bookworm AS build
+WORKDIR /src
+
+# Cache module downloads as a separate layer.
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+
+COPY . .
+
+# CGO disabled because modernc.org/sqlite is pure Go.
+# Strip symbols and DWARF for a smaller binary.
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    CGO_ENABLED=0 GOOS=linux \
+    go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
+
+# ── Runtime stage ────────────────────────────────────────────────────────────
+FROM gcr.io/distroless/static-debian12:nonroot
+
+WORKDIR /app
+COPY --from=build /out/server /app/server
+
+# HTTP + WebSocket. The server reads PORT from env (defaults to 8080).
+EXPOSE 8080
+
+ENTRYPOINT ["/app/server"]

apps/api/Dockerfile.sync-static

@@ -0,0 +1,27 @@
+# ── Build tippecanoe from source ─────────────────────────────────────────────
+FROM debian:bookworm-slim AS tippecanoe-build
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git build-essential libsqlite3-dev zlib1g-dev ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+RUN git clone --depth 1 https://github.com/felt/tippecanoe.git /tippecanoe \
+    && cd /tippecanoe && make -j$(nproc)
+
+# ── Build Go binary ─────────────────────────────────────────────────────────
+FROM golang:1.26-bookworm AS go-build
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=1 go build -o /sync-static ./cmd/sync-static
+
+# ── Runtime ──────────────────────────────────────────────────────────────────
+FROM debian:bookworm-slim
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates libsqlite3-0 zlib1g \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=tippecanoe-build /tippecanoe/tippecanoe /usr/local/bin/tippecanoe
+COPY --from=go-build /sync-static /usr/local/bin/sync-static
+
+WORKDIR /data
+ENTRYPOINT ["sync-static", "--upload"]

apps/api/cmd/collector/.env.example

@@ -0,0 +1,19 @@
+# Local-dev .env for the collector. The container in production reads these
+# from its `containers` config in apps/realtime/wrangler.jsonc instead.
+#
+# Leave both URLs unset to log emits to stderr (mock mode).
+
+# Primary path: on-prem ingest endpoint. e.g. https://api.tracky.com or
+# http://localhost:8080 for local dev.
+INGEST_URL=
+INGEST_SECRET=
+
+# Fallback path: R2-shaped PUT target. In prod this is the fake hostname the
+# parent Worker's outbound() handler intercepts and proxies to the
+# BACKLOG_BUCKET R2 binding (no creds needed). For tests, point at any HTTP
+# server that accepts PUT /backlog/...
+BACKLOG_URL=http://r2-backlog.tracky.internal
+
+# Provider API keys (only needed for the providers that require them).
+METRA_API_KEY=
+CTA_API_KEY=