Stable & Syncing — a graph of Dark instances that reconcile

backend/migrations/incremental/README.md

@@ -1,18 +1,17 @@
-# Incremental migrations
+# Incremental migrations (legacy escape hatch — empty by default)
 
-This directory holds per-file additive migrations, run in lexical
-order on top of `../schema.sql` and name-dedup'd via
-`system_migrations_v0`. Empty by default.
+**The schema lives in one place: `../schema.sql`.** It is the single, complete source of truth — every
+table, column, and index is defined there in its final form. `schema.sql` is content-hashed and
+re-applied on change (durable-canon: regenerable projections are dropped + re-folded from the op log;
+the canonical op log / blobs / branch state come through intact). Edit `schema.sql` directly.
 
-When to add a file here vs editing `schema.sql`:
+**A versioned change that must coordinate across instances goes through the Release migrator**
+(`backend/src/LibDB/Releases.fs`): one `Release` step can carry forward SQL (copy-and-swap), an optional
+op-format re-serialize, and a projection re-fold — gated by the single Release coordinate that also gates
+cross-instance sync. That's the mechanism for future schema/format evolution.
 
-- **Edit `schema.sql`**: structural redesigns, adding a new table or
-  column where rebuilding from source is fine. The file is hashed +
-  kill-and-fill'd; data in the affected tables is lost.
-- **Add a file here**: data backfills, transforms, additive
-  alterations on populated dev/test DBs you don't want to nuke.
-
-File naming: `YYYYMMDD_HHMMSS_<short-tag>.sql` so lexical sort gives
-chronological order. First line may be the literal `--#[no_tx]` to
-skip the wrapping transaction (rare; for DDL SQLite refuses inside
-one).
+This `incremental/` directory is a rarely-needed escape hatch for an additive backfill on a populated dev
+DB you don't want to rebuild. It is **empty by default** and you almost never want a file here — prefer
+`schema.sql` (+ a Release step when the change must travel). Files run in lexical order, name-dedup'd via
+`system_migrations_v0`; naming is `YYYYMMDD_HHMMSS_<short-tag>.sql`, and a first line of the literal
+`--#[no_tx]` skips the wrapping transaction (rare; for DDL SQLite refuses inside one).

backend/migrations/schema.sql

@@ -5,9 +5,9 @@
 -- no "build vN then DROP and rebuild as vN+1" — kill-and-fill means the
 -- final shape is what runs against an empty DB.
 --
--- system_migrations_v0 (the legacy per-named-migration table) is the one
--- exception, since pre-cutover DBs are adopted via that table; created
--- here AND by Migrations.fs's adoptLegacyDB path.
+-- The migrator's OWN bookkeeping tables live here too (the schema-hash stamp, the Release coordinate,
+-- the legacy per-named-migration log) — schema.sql is the single home for every CREATE TABLE; the
+-- migrator code only reads/writes rows.
 --
 -- Order: bookkeeping → branches → commits → ops → package projections →
 -- locations → traces → user-data, toplevels, scripts. FK targets come
@@ -24,6 +24,20 @@ CREATE TABLE IF NOT EXISTS system_migrations_v0 (
   sql TEXT NOT NULL
 );
 
+-- The schema-hash stamp: the hash of THIS file when last applied, so a change is detected and triggers
+-- a durable-canon preserve-and-refold. Rows written by `LocalExec/Migrations.fs` after applying schema.sql.
+CREATE TABLE IF NOT EXISTS schema_state_v0 (
+  id INTEGER PRIMARY KEY,
+  hash TEXT NOT NULL
+);
+
+-- The store's Release coordinate (the op-format/language/hash version) — the same integer that gates
+-- cross-instance sync. Rows written by the Release migrator (`LibDB/Releases.fs`).
+CREATE TABLE IF NOT EXISTS release_state_v0 (
+  id INTEGER PRIMARY KEY,
+  release INTEGER NOT NULL
+);
+
 
 CREATE TABLE IF NOT EXISTS accounts_v0 (
   id TEXT PRIMARY KEY,
@@ -89,13 +103,22 @@ CREATE INDEX IF NOT EXISTS idx_commits_branch
 
 -- The source of truth for all package changes (branch-scoped).
 CREATE TABLE IF NOT EXISTS package_ops (
-  id TEXT PRIMARY KEY,
+  -- (id, branch_id) is the PK: the SAME content-addressed op id can exist on different branches, so an
+  -- op is identified by id WITHIN a branch (a branch is a self-contained op stream).
+  id TEXT NOT NULL,
   op_blob BLOB NOT NULL,
   branch_id TEXT NOT NULL REFERENCES branches(id),
   commit_hash TEXT REFERENCES commits(hash),  -- NULL = WIP
   applied INTEGER NOT NULL DEFAULT 0,
   propagation_id TEXT NULL,                   -- direct lookup for PropagateUpdate ops
-  created_at TIMESTAMP NOT NULL DEFAULT (datetime('now'))
+  created_at TIMESTAMP NOT NULL DEFAULT (datetime('now')),
+  -- Authoring timestamp, PORTABLE across sync. A
+  -- locally-authored op self-stamps here at insert; a SYNCED op preserves its origin (the sync
+  -- receiver writes the peer's value), so every instance agrees on a given op's origin_ts and
+  -- max(origin_ts) picks the same divergence winner → no swap. Distinct from `created_at` (which
+  -- is local-insert time and differs per instance for the same op).
+  origin_ts TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+  PRIMARY KEY (id, branch_id)
 );
 CREATE INDEX IF NOT EXISTS idx_package_ops_wip
   ON package_ops(branch_id) WHERE commit_hash IS NULL;
@@ -127,6 +150,7 @@ CREATE TABLE IF NOT EXISTS package_types (
   hash TEXT PRIMARY KEY,
   pt_def BLOB NOT NULL,
   rt_def BLOB NOT NULL,
+  description TEXT NOT NULL DEFAULT '',  -- plain-text doc comment (not hashed)
   created_at TEXT NOT NULL DEFAULT (datetime('now'))
 );
 
@@ -135,6 +159,7 @@ CREATE TABLE IF NOT EXISTS package_values (
   pt_def BLOB NOT NULL,
   rt_dval BLOB,                  -- NULL until evaluated
   value_type BLOB,               -- for finding values of a given ValueType
+  description TEXT NOT NULL DEFAULT '',  -- plain-text doc comment (not hashed)
   created_at TEXT NOT NULL DEFAULT (datetime('now'))
 );
 CREATE INDEX IF NOT EXISTS idx_package_values_type ON package_values(value_type);
@@ -143,6 +168,7 @@ CREATE TABLE IF NOT EXISTS package_functions (
   hash TEXT PRIMARY KEY,
   pt_def BLOB NOT NULL,
   rt_instrs BLOB NOT NULL,
+  description TEXT NOT NULL DEFAULT '',  -- plain-text doc comment (not hashed)
   created_at TEXT NOT NULL DEFAULT (datetime('now'))
 );
 
@@ -169,7 +195,12 @@ CREATE TABLE IF NOT EXISTS locations (
   branch_id TEXT NOT NULL REFERENCES branches(id),
   commit_hash TEXT REFERENCES commits(hash),  -- NULL = WIP
   created_at TIMESTAMP NOT NULL DEFAULT (datetime('now')),
-  unlisted_at TIMESTAMP NULL              -- set when a later row supersedes this one
+  unlisted_at TIMESTAMP NULL,             -- set when a later row supersedes this one
+  -- The origin_ts of the op that set THIS binding — the name→authoring-time mapping that lets
+  -- playback order by CREATION, not arrival (timestamp-LWW). A SetName whose op was created
+  -- EARLIER than the current binding (an old op arriving late via sync) is stale: playback skips
+  -- the rebind, so the latest-by-creation name wins on every instance regardless of sync order.
+  origin_ts TEXT NULL
 );
 CREATE INDEX IF NOT EXISTS idx_locations_branch_lookup
   ON locations(branch_id, owner, modules, name, item_type)
@@ -351,3 +382,78 @@ CREATE TABLE IF NOT EXISTS scripts_v0 (
   name TEXT NOT NULL UNIQUE,
   text TEXT NOT NULL
 );
+
+
+--------------------
+-- Sync (local-only setup; NOT synced) — see LibDB/{Remotes,SyncCursors,Conflicts}.fs
+--------------------
+
+-- Registered sync peers (managed via `dark remote ...`). Each row is a (name, url) the
+-- tailnet sync daemon polls.
+CREATE TABLE IF NOT EXISTS sync_remotes (
+  name TEXT PRIMARY KEY,
+  url TEXT NOT NULL
+);
+
+-- Per-remote poll resume state: how far this instance has folded each peer's op stream.
+-- The cursor is a `package_ops` rowid (SQLite's monotonic insertion order).
+CREATE TABLE IF NOT EXISTS sync_cursors (
+  remote TEXT PRIMARY KEY,
+  folded_through_rowid INTEGER NOT NULL DEFAULT 0,
+  -- how far we've applied this remote's RESOLUTIONS stream (a separate `resolutions` rowid cursor,
+  -- since resolutions sync on their own channel alongside the op log)
+  resolutions_through_rowid INTEGER NOT NULL DEFAULT 0
+);
+
+-- Resolutions — synced decisions that OVERRIDE the op-fold for a contested name. A conflict (e.g. a
+-- name diverged across instances) auto-resolves by policy (last-writer-wins), but a human (or a
+-- keep-local policy) can decide differently. That decision is NOT a new op — the op log is authored
+-- content/structure; a resolution is a thin overlay picking among existing candidates. The effective
+-- binding is: fold(ops) [LWW] → then apply resolutions per location [last-resolver-wins by `at`]. A
+-- resolution carries its own fresh `at` stamp, so it wins the same timestamp-LWW that orders bindings —
+-- which is what makes a "keep mine" decision propagate where re-emitting the original SetName (same
+-- content hash → same op id) could not. Synced (its own rowid cursors the wire); the implicit rowid
+-- orders the sync. This REPLACES the old `OverrideName` op (an op invented only to dodge that hash
+-- collision). `at` is the resolver time; `id` is a uuid carried over the wire for INSERT-OR-IGNORE dedup.
+CREATE TABLE IF NOT EXISTS resolutions (
+  id TEXT PRIMARY KEY,                      -- uuid, carried over the wire (idempotent apply)
+  owner TEXT NOT NULL,
+  modules TEXT NOT NULL,
+  name TEXT NOT NULL,
+  item_type TEXT NOT NULL,                  -- 'fn' | 'type' | 'value' (kind of the chosen content)
+  chosen_hash TEXT NOT NULL,               -- the content hash this resolution binds the name to
+  resolved_by TEXT NOT NULL,               -- 'human' | 'auto:keep-local' | …
+  branch_id TEXT NOT NULL,
+  at TEXT NOT NULL,                         -- resolver timestamp — the LWW stamp this binding competes on
+  created_at TIMESTAMP NOT NULL DEFAULT (datetime('now'))
+);
+
+-- The recorded, reviewable log of auto-resolved sync conflicts (`dark conflicts`). Recorded at pull
+-- time; auto-resolved by policy (default last-writer-wins) but never silently lost. Local-only, never
+-- synced, re-derivable by replaying the op log. Stores the STRUCTURED conflict (`conflict_blob` = a
+-- serialized PT.SyncConflict — the candidates) plus its resolution flattened into columns:
+-- `chosen_hash` (which content won) and `resolved_by` ('auto:<policy>' e.g. 'auto:last-writer-wins', or
+-- 'human').
+CREATE TABLE IF NOT EXISTS sync_conflicts (
+  id TEXT PRIMARY KEY,
+  kind TEXT NOT NULL,                              -- SyncConflict discriminator, e.g. 'divergence'
+  location TEXT NOT NULL,
+  conflict_blob BLOB NOT NULL,                     -- serialized PT.SyncConflict (the candidates)
+  chosen_hash TEXT NOT NULL,                       -- the resolution's chosen content hash
+  resolved_by TEXT NOT NULL,                       -- 'auto:<policy>' or 'human'
+  remote TEXT NOT NULL,
+  detected_at TEXT NOT NULL DEFAULT (datetime('now')),
+  status TEXT NOT NULL DEFAULT 'auto-resolved'     -- 'auto-resolved' | 'acknowledged' | 'overridden'
+);
+
+-- Structured telemetry from the autosync daemon: one row per poll cycle, so `sync events` (and a
+-- future dashboard view) can show activity as DATA rather than scraping the text log. Local-only,
+-- never synced; trimmed to the most recent rows so it stays bounded.
+CREATE TABLE IF NOT EXISTS sync_daemon_events (
+  id INTEGER PRIMARY KEY,
+  at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+  peers_polled INTEGER NOT NULL,
+  changed INTEGER NOT NULL,
+  conflicts INTEGER NOT NULL,
+  skews INTEGER NOT NULL
+);

backend/src/Builtins/Builtins.Http.Client/Libs/HttpClient.fs

@@ -761,6 +761,41 @@ let fns (config : Configuration) : List<BuiltInFn> =
       deprecated = NotDeprecated }
 
 
+    { name = fn "httpClientGetUnsafe" 0
+      typeParams = []
+      parameters =
+        [ Param.make
+            "uri"
+            TString
+            "URL to GET with SSRF guards OFF (loopback/private/tailnet allowed)" ]
+      returnType = TypeReference.result TString TString
+      description =
+        "GET <param uri> with NO SSRF guards — loopback, RFC-1918, and Tailscale (100.64/10)
+         addresses are reachable, unlike the guarded `httpClient.request`. TRUSTED-CLI use only
+         (the caller IS the code author): used by `dark sync pull <url>` to reach a tailnet peer's
+         sync server. Returns the response body as a String (Ok) or an error message (Error)."
+      fn =
+        let looseClient = BaseClient.create looseConfig
+        let resultOk = Dval.resultOk KTString KTString
+        let resultError = Dval.resultError KTString KTString
+        (function
+        | _, _, _, [ DString uri ] ->
+          uply {
+            let request : Request =
+              { url = uri; method = HttpMethod "GET"; headers = []; body = [||] }
+            let! response = makeRequest looseConfig looseClient request
+            match response with
+            | Ok r ->
+              return resultOk (DString(System.Text.Encoding.UTF8.GetString r.body))
+            | Error _ -> return resultError (DString "sync fetch failed")
+          }
+        | _ -> incorrectArgs ())
+      sqlSpec = NotQueryable
+      previewable = Impure
+      capabilities = LibExecution.Capabilities.Needs.http
+      deprecated = NotDeprecated }
+
+
     // ——————————————————————————————————————————————————————————
     // Streaming HTTP.
     //

backend/src/Builtins/Builtins.Matter/Builtin.fs

@@ -25,6 +25,8 @@ let builtins (pm : PT.PackageManager) : Builtins =
       Libs.PM.Dependencies.builtins ()
       Libs.PM.Seed.builtins
       Libs.PM.Caps.builtins
+      Libs.PM.Sync.builtins ()
+      Libs.PM.Remotes.builtins
 
       // Traces (reader surface)
       Libs.Traces.builtins ()

backend/src/Builtins/Builtins.Matter/Builtins.Matter.fsproj

@@ -20,6 +20,8 @@
     <Compile Include="Libs/PM/Dependencies.fs" />
     <Compile Include="Libs/PM/Seed.fs" />
     <Compile Include="Libs/PM/Caps.fs" />
+    <Compile Include="Libs/PM/Sync.fs" />
+    <Compile Include="Libs/PM/Remotes.fs" />
 
     <!-- Tracing (reader) -->
     <Compile Include="Libs/Traces.fs" />

backend/src/Builtins/Builtins.Matter/Libs/PM/Remotes.fs

@@ -0,0 +1,78 @@
+/// Builtins behind `dark remote` — the registry of sync peers the tailnet daemon polls. Thin
+/// wrappers over `LibDB.Remotes`; the pull/apply sync surface lives in `Libs.PM.Sync`.
+module Builtins.Matter.Libs.PM.Remotes
+
+open Prelude
+open LibExecution.RuntimeTypes
+
+module Dval = LibExecution.Dval
+module Builtin = LibExecution.Builtin
+
+open Builtin.Shortcuts
+
+
+let fns : List<BuiltInFn> =
+  [ { name = fn "pmRemoteAdd" 0
+      typeParams = []
+      parameters = [ Param.make "name" TString ""; Param.make "url" TString "" ]
+      returnType = TUnit
+      description =
+        "Register (or update) a sync remote by name. `url` is the pollable target (an http(s) URL or a
+         local data.db path). Idempotent by name."
+      fn =
+        (function
+        | _, _, _, [ DString name; DString url ] ->
+          uply {
+            do! LibDB.Remotes.add name url
+            return DUnit
+          }
+        | _ -> incorrectArgs ())
+      sqlSpec = NotQueryable
+      previewable = Impure
+      capabilities = LibExecution.Capabilities.noCaps
+      deprecated = NotDeprecated }
+
+
+    { name = fn "pmRemoteList" 0
+      typeParams = []
+      parameters = [ Param.make "unit" TUnit "" ]
+      returnType = TList TString
+      description =
+        "Registered sync remotes, one `name  →  url` line each (for `dark remote list`). Empty if none."
+      fn =
+        (function
+        | _, _, _, [ DUnit ] ->
+          uply {
+            let! remotes = LibDB.Remotes.list ()
+            let lines =
+              remotes |> List.map (fun (name, url) -> DString $"{name}  →  {url}")
+            return Dval.list KTString lines
+          }
+        | _ -> incorrectArgs ())
+      sqlSpec = NotQueryable
+      previewable = Impure
+      capabilities = LibExecution.Capabilities.noCaps
+      deprecated = NotDeprecated }
+
+
+    { name = fn "pmRemoteRemove" 0
+      typeParams = []
+      parameters = [ Param.make "name" TString "" ]
+      returnType = TBool
+      description =
+        "Unregister a sync remote by name. Returns true if it existed (its sync cursor, if any, stays)."
+      fn =
+        (function
+        | _, _, _, [ DString name ] ->
+          uply {
+            let! existed = LibDB.Remotes.remove name
+            return DBool existed
+          }
+        | _ -> incorrectArgs ())
+      sqlSpec = NotQueryable
+      previewable = Impure
+      capabilities = LibExecution.Capabilities.noCaps
+      deprecated = NotDeprecated } ]
+
+
+let builtins = LibExecution.Builtin.make [] fns