From e140516bbfe114f2af95964200f4e54e15b89d88 Mon Sep 17 00:00:00 2001 From: Luiz Gustavo Abou Hatem de Liz Date: Mon, 1 Jun 2026 16:27:20 -0300 Subject: [PATCH 1/4] fix: preflight /by_uids before cascade-cancelled insert in CandidateConfirmer (COW-990) When a generator is cancelled, the cascade in CandidateConfirmer previously inserted all orphan candidates as (cancelled, null) without checking the orderbook. If the watch-tower had already posted and a solver had filled an in-flight part, the indexer would mark it cancelled despite being fulfilled (~0.17% observed rate on Phase-3 harness data). Fix: batch-query /by_uids for all orphan UIDs before the cascade insert. Use the API status + executed amounts when available; fall back to 'cancelled' for UIDs not yet on the orderbook. Timeout errors degrade gracefully to the old behavior. The onConflictDoNothing guard still protects already-terminal discrete_order rows from being overwritten. Co-Authored-By: Claude Sonnet 4.6 --- docs/architecture.md | 2 +- src/application/handlers/blockHandler.ts | 51 +++++++++++++++++------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index d11b3ed..41212ae 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -230,5 +230,5 @@ The block handlers (C1–C5) already run on both mainnet and gnosis. Adding a ne ## Known Limitations -- Cancellation detection has a small lag. For non-deterministic generators, C1 catches `SingleOrderNotAuthed` on the next poll (every block). For deterministic generators, C5 reads `singleOrders(owner, hash)` every `DETERMINISTIC_CANCEL_SWEEP_INTERVAL` blocks (default 100) — so on-chain removal is reflected with worst-case latency of ~100 blocks (~20 min mainnet, ~8 min Gnosis). There is no on-chain event for `remove()`, so shorter detection latency would require a higher-cadence sweep. Once the generator is marked `Cancelled`, C2 and C3 cascade the state to children on the next block; API-terminal statuses (`fulfilled` / `unfilled` / `expired`) still win for children that were already traded on the orderbook. +- Cancellation detection has a small lag. For non-deterministic generators, C1 catches `SingleOrderNotAuthed` on the next poll (every block). For deterministic generators, C5 reads `singleOrders(owner, hash)` every `DETERMINISTIC_CANCEL_SWEEP_INTERVAL` blocks (default 100) — so on-chain removal is reflected with worst-case latency of ~100 blocks (~20 min mainnet, ~8 min Gnosis). There is no on-chain event for `remove()`, so shorter detection latency would require a higher-cadence sweep. Once the generator is marked `Cancelled`, C2 and C3 cascade the state to children on the next block. The C2 cascade does a preflight `/by_uids` query so that candidates already on the orderbook get their actual status rather than defaulting to `cancelled`; API-terminal statuses (`fulfilled` / `unfilled` / `expired`) still win for children already promoted to `discrete_order`. - Aave adapter owner resolution is reactive — `owner_mapping` is written when the adapter appears in settlement, which may be after the conditional order is created. The generator row keeps `resolvedOwner` equal to the adapter address when no mapping existed at insert time; that column is not backfilled when the mapping is inserted later. `ownerAddressType` on the generator IS backfilled when the mapping is inserted — after which GraphQL and REST filters on `ownerAddressType = "flash_loan_helper"` reflect the correct value. `resolvedOwner` is still not backfilled (set once at insert, unchanged thereafter). diff --git a/src/application/handlers/blockHandler.ts b/src/application/handlers/blockHandler.ts index 817453c..0bb8123 100644 --- a/src/application/handlers/blockHandler.ts +++ b/src/application/handlers/blockHandler.ts @@ -27,6 +27,7 @@ import { BOOTSTRAP_OWNER_FETCH_TIMEOUT_MS, DEFAULT_MAX_GENERATORS_PER_BLOCK, DETERMINISTIC_CANCEL_SWEEP_INTERVAL, + ORDERBOOK_HTTP_TIMEOUT_MS, RECHECK_INTERVAL, TRY_NEXT_BLOCK_WARMUP_THRESHOLD, TRY_NEXT_BLOCK_COOLDOWN_THRESHOLD, @@ -354,23 +355,42 @@ ponder.on("CandidateConfirmer:block", async ({ event, context }) => { }[]; if (orphanCandidates.length > 0) { + // COW-990: preflight /by_uids before writing cancelled. A candidate could have + // been posted by the watch-tower and filled/expired between generator creation + // and the cancellation cascade (~0.17% observed rate). Use the API status when + // available; fall back to 'cancelled' for UIDs not yet on the orderbook. + type DiscreteStatus = "open" | "fulfilled" | "unfilled" | "expired" | "cancelled"; + let preflightStatuses: Awaited>; + try { + preflightStatuses = await withTimeout( + fetchOrderStatusByUids(context, chainId, orphanCandidates.map((c) => c.orderUid)), + ORDERBOOK_HTTP_TIMEOUT_MS, + "c2:cascade:preflight", + ); + } catch { + preflightStatuses = new Map(); + } + await context.db.sql .insert(discreteOrder) .values( - orphanCandidates.map((c) => ({ - orderUid: c.orderUid, - chainId, - conditionalOrderGeneratorId: c.generatorId, - status: "cancelled" as const, - sellAmount: c.sellAmount, - buyAmount: c.buyAmount, - feeAmount: c.feeAmount, - validTo: c.validTo, - creationDate: c.creationDate, - executedSellAmount: null, - executedBuyAmount: null, - promotedAt: event.block.timestamp, - })), + orphanCandidates.map((c) => { + const apiEntry = preflightStatuses.get(c.orderUid); + return { + orderUid: c.orderUid, + chainId, + conditionalOrderGeneratorId: c.generatorId, + status: (apiEntry?.status ?? "cancelled") as DiscreteStatus, + sellAmount: c.sellAmount, + buyAmount: c.buyAmount, + feeAmount: c.feeAmount, + validTo: c.validTo, + creationDate: c.creationDate, + executedSellAmount: apiEntry?.executedSellAmount ?? null, + executedBuyAmount: apiEntry?.executedBuyAmount ?? null, + promotedAt: event.block.timestamp, + }; + }), ) .onConflictDoNothing(); @@ -386,8 +406,9 @@ ponder.on("CandidateConfirmer:block", async ({ event, context }) => { ), ); + const preflightHits = preflightStatuses.size; console.log( - `[COW:C2] block=${event.block.number} chain=${chainId} parent-cancelled=${orphanCandidates.length}`, + `[COW:C2] block=${event.block.number} chain=${chainId} parent-cancelled=${orphanCandidates.length} preflight-hits=${preflightHits}`, ); } } From e2c474b58707b689ca2fa987c43b2f86b3756d20 Mon Sep 17 00:00:00 2001 From: Luiz Gustavo Abou Hatem de Liz Date: Tue, 2 Jun 2026 17:07:35 -0300 Subject: [PATCH 2/4] fix: fix preflight timeout, migrate console.log to cowLog, deduplicate DiscreteStatus type (COW-990) Co-Authored-By: Claude Sonnet 4.6 --- src/application/handlers/blockHandler.ts | 18 +++++++++++------- src/application/helpers/cowLogger.ts | 24 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 src/application/helpers/cowLogger.ts diff --git a/src/application/handlers/blockHandler.ts b/src/application/handlers/blockHandler.ts index 0bb8123..0642000 100644 --- a/src/application/handlers/blockHandler.ts +++ b/src/application/handlers/blockHandler.ts @@ -42,6 +42,9 @@ import { parsePollError, } from "../helpers/pollResultErrors"; import { computeOrderUid, type GPv2OrderData } from "../helpers/orderUid"; +import { cowLog } from "../helpers/cowLogger"; + +type DiscreteStatus = "open" | "fulfilled" | "unfilled" | "expired" | "cancelled"; const NON_DETERMINISTIC_TYPES = ["PerpetualSwap", "GoodAfterTime", "TradeAboveThreshold", "Unknown"] as const; const SINGLE_SHOT_NON_DETERMINISTIC = ["GoodAfterTime", "TradeAboveThreshold"] as const; @@ -359,12 +362,11 @@ ponder.on("CandidateConfirmer:block", async ({ event, context }) => { // been posted by the watch-tower and filled/expired between generator creation // and the cancellation cascade (~0.17% observed rate). Use the API status when // available; fall back to 'cancelled' for UIDs not yet on the orderbook. - type DiscreteStatus = "open" | "fulfilled" | "unfilled" | "expired" | "cancelled"; let preflightStatuses: Awaited>; try { preflightStatuses = await withTimeout( fetchOrderStatusByUids(context, chainId, orphanCandidates.map((c) => c.orderUid)), - ORDERBOOK_HTTP_TIMEOUT_MS, + ORDERBOOK_HTTP_TIMEOUT_MS * 2, "c2:cascade:preflight", ); } catch { @@ -406,10 +408,13 @@ ponder.on("CandidateConfirmer:block", async ({ event, context }) => { ), ); - const preflightHits = preflightStatuses.size; - console.log( - `[COW:C2] block=${event.block.number} chain=${chainId} parent-cancelled=${orphanCandidates.length} preflight-hits=${preflightHits}`, - ); + const preflightKnown = preflightStatuses.size; + cowLog("info", "c2:parent-cancelled", { + block: String(event.block.number), + chainId, + parentCancelled: orphanCandidates.length, + preflightKnown, + }); } } @@ -449,7 +454,6 @@ ponder.on("CandidateConfirmer:block", async ({ event, context }) => { const uids = unconfirmed.map((c) => c.orderUid); const statuses = await fetchOrderStatusByUids(context, chainId, uids); - type DiscreteStatus = "open" | "fulfilled" | "unfilled" | "expired" | "cancelled"; const rowsToUpsert: (typeof discreteOrder.$inferInsert)[] = []; const confirmedUids: string[] = []; diff --git a/src/application/helpers/cowLogger.ts b/src/application/helpers/cowLogger.ts new file mode 100644 index 0000000..3969837 --- /dev/null +++ b/src/application/helpers/cowLogger.ts @@ -0,0 +1,24 @@ +/** + * Structured JSON logger for handler code. Outputs one JSON line per call so + * log aggregators (Datadog, CloudWatch, etc.) can filter by chainId, handler, + * block number, or any other field without regex parsing. + * + * Ponder's own log lines are controlled by --log-format (pretty|json) on the + * CLI. These handler lines are always JSON so they remain parseable regardless + * of Ponder's format setting. + */ + +type LogLevel = "info" | "warn" | "error"; + +export function cowLog( + level: LogLevel, + msg: string, + fields: Record = {}, +): void { + const line = JSON.stringify({ time: Date.now(), level, msg, ...fields }); + if (level === "warn" || level === "error") { + console.error(line); + } else { + console.log(line); + } +} From 844e8157ff26104d17b64d4d125a4b29ce497666 Mon Sep 17 00:00:00 2001 From: Luiz Gustavo Abou Hatem de Liz Date: Thu, 4 Jun 2026 10:47:41 -0300 Subject: [PATCH 3/4] fix: add comments explaining preflight timeout fallback and onConflictDoNothing semantics Co-Authored-By: Claude Sonnet 4.6 --- src/application/handlers/blockHandler.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/application/handlers/blockHandler.ts b/src/application/handlers/blockHandler.ts index 0642000..eac2ce5 100644 --- a/src/application/handlers/blockHandler.ts +++ b/src/application/handlers/blockHandler.ts @@ -362,6 +362,8 @@ ponder.on("CandidateConfirmer:block", async ({ event, context }) => { // been posted by the watch-tower and filled/expired between generator creation // and the cancellation cascade (~0.17% observed rate). Use the API status when // available; fall back to 'cancelled' for UIDs not yet on the orderbook. + // Bounded by ORDERBOOK_HTTP_TIMEOUT_MS * 2; on timeout the empty map fallback + // keeps correctness degraded-gracefully (all orphans written as 'cancelled'). let preflightStatuses: Awaited>; try { preflightStatuses = await withTimeout( @@ -373,6 +375,9 @@ ponder.on("CandidateConfirmer:block", async ({ event, context }) => { preflightStatuses = new Map(); } + // onConflictDoNothing: if C3 already promoted this UID with a terminal status + // (e.g. 'fulfilled'), the existing row wins and this insert is a no-op. + // preflightKnown counts API hits, not rows actually written. await context.db.sql .insert(discreteOrder) .values( From 7419a073b98e246e3531879bf56509433bf6e9de Mon Sep 17 00:00:00 2001 From: Luiz Gustavo Abou Hatem de Liz Date: Thu, 4 Jun 2026 20:21:16 -0300 Subject: [PATCH 4/4] fix: derive DiscreteStatus from schema enum, remove cowLogger scope creep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace manual DiscreteStatus union with (typeof discreteOrderStatusEnum.enumValues)[number] so the type stays in sync with the schema automatically - Delete cowLogger.ts — structured logging belongs in COW-994 (PR #87), not in this fix PR; replace the one cowLog call with a plain console.log Co-Authored-By: Claude Sonnet 4.6 --- src/application/handlers/blockHandler.ts | 15 +++++---------- src/application/helpers/cowLogger.ts | 24 ------------------------ 2 files changed, 5 insertions(+), 34 deletions(-) delete mode 100644 src/application/helpers/cowLogger.ts diff --git a/src/application/handlers/blockHandler.ts b/src/application/handlers/blockHandler.ts index eac2ce5..4ea3863 100644 --- a/src/application/handlers/blockHandler.ts +++ b/src/application/handlers/blockHandler.ts @@ -15,7 +15,7 @@ */ import { ponder } from "ponder:registry"; -import { bootstrapRetryQueue, candidateDiscreteOrder, conditionalOrderGenerator, discreteOrder } from "ponder:schema"; +import { bootstrapRetryQueue, candidateDiscreteOrder, conditionalOrderGenerator, discreteOrder, discreteOrderStatusEnum } from "ponder:schema"; import { and, asc, eq, inArray, lte, or, sql } from "ponder"; import type { Hex } from "viem"; import { @@ -42,9 +42,7 @@ import { parsePollError, } from "../helpers/pollResultErrors"; import { computeOrderUid, type GPv2OrderData } from "../helpers/orderUid"; -import { cowLog } from "../helpers/cowLogger"; - -type DiscreteStatus = "open" | "fulfilled" | "unfilled" | "expired" | "cancelled"; +type DiscreteStatus = (typeof discreteOrderStatusEnum.enumValues)[number]; const NON_DETERMINISTIC_TYPES = ["PerpetualSwap", "GoodAfterTime", "TradeAboveThreshold", "Unknown"] as const; const SINGLE_SHOT_NON_DETERMINISTIC = ["GoodAfterTime", "TradeAboveThreshold"] as const; @@ -414,12 +412,9 @@ ponder.on("CandidateConfirmer:block", async ({ event, context }) => { ); const preflightKnown = preflightStatuses.size; - cowLog("info", "c2:parent-cancelled", { - block: String(event.block.number), - chainId, - parentCancelled: orphanCandidates.length, - preflightKnown, - }); + console.log( + `[COW:C2] c2:parent-cancelled block=${event.block.number} chainId=${chainId} parentCancelled=${orphanCandidates.length} preflightKnown=${preflightKnown}`, + ); } } diff --git a/src/application/helpers/cowLogger.ts b/src/application/helpers/cowLogger.ts deleted file mode 100644 index 3969837..0000000 --- a/src/application/helpers/cowLogger.ts +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Structured JSON logger for handler code. Outputs one JSON line per call so - * log aggregators (Datadog, CloudWatch, etc.) can filter by chainId, handler, - * block number, or any other field without regex parsing. - * - * Ponder's own log lines are controlled by --log-format (pretty|json) on the - * CLI. These handler lines are always JSON so they remain parseable regardless - * of Ponder's format setting. - */ - -type LogLevel = "info" | "warn" | "error"; - -export function cowLog( - level: LogLevel, - msg: string, - fields: Record = {}, -): void { - const line = JSON.stringify({ time: Date.now(), level, msg, ...fields }); - if (level === "warn" || level === "error") { - console.error(line); - } else { - console.log(line); - } -}