From 0e75e1432178533a9bec7a89f28ecf1aa35302be Mon Sep 17 00:00:00 2001 From: Goader Date: Tue, 10 Feb 2026 02:39:30 +0100 Subject: [PATCH 01/11] status added to models --- .../referrer-leaderboard/closeout.ts | 32 +++++++++++++++++++ .../referrer-leaderboard/mocks-v1.ts | 2 ++ .../ens-referrals/src/v1/api/serialize.ts | 3 ++ .../ens-referrals/src/v1/api/zod-schemas.ts | 4 +++ .../ens-referrals/src/v1/edition-metrics.ts | 16 ++++++++++ .../ens-referrals/src/v1/leaderboard-page.ts | 10 ++++++ .../ensnode-sdk/src/shared/cache/swr-cache.ts | 13 ++++++++ 7 files changed, 80 insertions(+) create mode 100644 apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/closeout.ts diff --git a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/closeout.ts b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/closeout.ts new file mode 100644 index 000000000..9a030778a --- /dev/null +++ b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/closeout.ts @@ -0,0 +1,32 @@ +import type { ReferralProgramRules } from "@namehash/ens-referrals/v1"; +import { minutesToSeconds } from "date-fns"; + +import { type Duration, durationBetween, type UnixTimestamp } from "@ensnode/ensnode-sdk"; + +/** + * Duration after which we assume a closed edition is safe from chain reorganizations. + * + * This is a heuristic value (10 minutes) chosen to provide a reasonable safety margin + * beyond typical Ethereum finality. It is not a guarantee of immutability. + */ +export const ASSUMED_CHAIN_REORG_SAFE_DURATION: Duration = minutesToSeconds(10); + +/** + * Assumes a referral program edition is immutably closed if it ended more than + * ASSUMED_CHAIN_REORG_SAFE_DURATION ago. + * + * This is a practical heuristic for determining when edition data can be cached + * indefinitely, based on the assumption that chain reorgs become extremely unlikely + * after the safety window has passed. + * + * @param rules - The referral program rules containing endTime + * @param referenceTime - The timestamp to check against (typically accurateAsOf from cached leaderboard) + * @returns true if we assume the edition is immutably closed + */ +export function assumeReferralProgramEditionImmutablyClosed( + rules: ReferralProgramRules, + referenceTime: UnixTimestamp, +): boolean { + const timeSinceClose = durationBetween(rules.endTime, referenceTime); + return timeSinceClose > ASSUMED_CHAIN_REORG_SAFE_DURATION; +} diff --git a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/mocks-v1.ts b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/mocks-v1.ts index 7ea8bf34c..ac90c8aab 100644 --- a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/mocks-v1.ts +++ b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/mocks-v1.ts @@ -1,4 +1,5 @@ import { + ReferralProgramStatuses, type ReferrerLeaderboard, ReferrerLeaderboardPageResponseCodes, type ReferrerLeaderboardPageResponseOk, @@ -1093,6 +1094,7 @@ export const referrerLeaderboardPageResponseOk: ReferrerLeaderboardPageResponseO startIndex: 0, endIndex: 28, }, + status: ReferralProgramStatuses.Active, accurateAsOf: 1735689600, }, }; diff --git a/packages/ens-referrals/src/v1/api/serialize.ts b/packages/ens-referrals/src/v1/api/serialize.ts index 371c63ba1..7ab98e429 100644 --- a/packages/ens-referrals/src/v1/api/serialize.ts +++ b/packages/ens-referrals/src/v1/api/serialize.ts @@ -118,6 +118,7 @@ function serializeReferrerLeaderboardPage( referrers: page.referrers.map(serializeAwardedReferrerMetrics), aggregatedMetrics: serializeAggregatedReferrerMetrics(page.aggregatedMetrics), pageContext: page.pageContext, + status: page.status, accurateAsOf: page.accurateAsOf, }; } @@ -133,6 +134,7 @@ function serializeReferrerEditionMetricsRanked( rules: serializeReferralProgramRules(detail.rules), referrer: serializeAwardedReferrerMetrics(detail.referrer), aggregatedMetrics: serializeAggregatedReferrerMetrics(detail.aggregatedMetrics), + status: detail.status, accurateAsOf: detail.accurateAsOf, }; } @@ -148,6 +150,7 @@ function serializeReferrerEditionMetricsUnranked( rules: serializeReferralProgramRules(detail.rules), referrer: serializeUnrankedReferrerMetrics(detail.referrer), aggregatedMetrics: serializeAggregatedReferrerMetrics(detail.aggregatedMetrics), + status: detail.status, accurateAsOf: detail.accurateAsOf, }; } diff --git a/packages/ens-referrals/src/v1/api/zod-schemas.ts b/packages/ens-referrals/src/v1/api/zod-schemas.ts index 371f3060a..d0a9d6b6a 100644 --- a/packages/ens-referrals/src/v1/api/zod-schemas.ts +++ b/packages/ens-referrals/src/v1/api/zod-schemas.ts @@ -28,6 +28,7 @@ import { ReferrerEditionMetricsTypeIds, } from "../edition-metrics"; import { REFERRERS_PER_LEADERBOARD_PAGE_MAX } from "../leaderboard-page"; +import { ReferralProgramStatuses } from "../status"; import { MAX_EDITIONS_PER_REQUEST, ReferralProgramEditionConfigSetResponseCodes, @@ -149,6 +150,7 @@ export const makeReferrerLeaderboardPageSchema = (valueLabel: string = "Referrer referrers: z.array(makeAwardedReferrerMetricsSchema(`${valueLabel}.referrers[record]`)), aggregatedMetrics: makeAggregatedReferrerMetricsSchema(`${valueLabel}.aggregatedMetrics`), pageContext: makeReferrerLeaderboardPageContextSchema(`${valueLabel}.pageContext`), + status: z.enum(ReferralProgramStatuses), accurateAsOf: makeUnixTimestampSchema(`${valueLabel}.accurateAsOf`), }); @@ -197,6 +199,7 @@ export const makeReferrerEditionMetricsRankedSchema = ( rules: makeReferralProgramRulesSchema(`${valueLabel}.rules`), referrer: makeAwardedReferrerMetricsSchema(`${valueLabel}.referrer`), aggregatedMetrics: makeAggregatedReferrerMetricsSchema(`${valueLabel}.aggregatedMetrics`), + status: z.enum(ReferralProgramStatuses), accurateAsOf: makeUnixTimestampSchema(`${valueLabel}.accurateAsOf`), }); @@ -211,6 +214,7 @@ export const makeReferrerEditionMetricsUnrankedSchema = ( rules: makeReferralProgramRulesSchema(`${valueLabel}.rules`), referrer: makeUnrankedReferrerMetricsSchema(`${valueLabel}.referrer`), aggregatedMetrics: makeAggregatedReferrerMetricsSchema(`${valueLabel}.aggregatedMetrics`), + status: z.enum(ReferralProgramStatuses), accurateAsOf: makeUnixTimestampSchema(`${valueLabel}.accurateAsOf`), }); diff --git a/packages/ens-referrals/src/v1/edition-metrics.ts b/packages/ens-referrals/src/v1/edition-metrics.ts index 1c46f9a37..f1e54e0cb 100644 --- a/packages/ens-referrals/src/v1/edition-metrics.ts +++ b/packages/ens-referrals/src/v1/edition-metrics.ts @@ -10,6 +10,7 @@ import { type UnrankedReferrerMetrics, } from "./referrer-metrics"; import type { ReferralProgramRules } from "./rules"; +import { calcReferralProgramStatus, type ReferralProgramStatusId } from "./status"; /** * The type of referrer edition metrics data. @@ -66,6 +67,12 @@ export interface ReferrerEditionMetricsRanked { */ aggregatedMetrics: AggregatedReferrerMetrics; + /** + * The status of the referral program ("Scheduled", "Active", or "Closed") + * calculated based on the program's timing relative to {@link accurateAsOf}. + */ + status: ReferralProgramStatusId; + /** * The {@link UnixTimestamp} of when the data used to build the {@link ReferrerEditionMetricsRanked} was accurate as of. */ @@ -105,6 +112,12 @@ export interface ReferrerEditionMetricsUnranked { */ aggregatedMetrics: AggregatedReferrerMetrics; + /** + * The status of the referral program ("Scheduled", "Active", or "Closed") + * calculated based on the program's timing relative to {@link accurateAsOf}. + */ + status: ReferralProgramStatusId; + /** * The {@link UnixTimestamp} of when the data used to build the {@link ReferrerEditionMetricsUnranked} was accurate as of. */ @@ -134,6 +147,7 @@ export const getReferrerEditionMetrics = ( leaderboard: ReferrerLeaderboard, ): ReferrerEditionMetrics => { const awardedReferrerMetrics = leaderboard.referrers.get(referrer); + const status = calcReferralProgramStatus(leaderboard.rules, leaderboard.accurateAsOf); // If referrer is on the leaderboard, return their ranked metrics if (awardedReferrerMetrics) { @@ -142,6 +156,7 @@ export const getReferrerEditionMetrics = ( rules: leaderboard.rules, referrer: awardedReferrerMetrics, aggregatedMetrics: leaderboard.aggregatedMetrics, + status, accurateAsOf: leaderboard.accurateAsOf, }; } @@ -152,6 +167,7 @@ export const getReferrerEditionMetrics = ( rules: leaderboard.rules, referrer: buildUnrankedReferrerMetrics(referrer), aggregatedMetrics: leaderboard.aggregatedMetrics, + status, accurateAsOf: leaderboard.accurateAsOf, }; }; diff --git a/packages/ens-referrals/src/v1/leaderboard-page.ts b/packages/ens-referrals/src/v1/leaderboard-page.ts index b789e455d..72d5a75ff 100644 --- a/packages/ens-referrals/src/v1/leaderboard-page.ts +++ b/packages/ens-referrals/src/v1/leaderboard-page.ts @@ -5,6 +5,7 @@ import type { ReferrerLeaderboard } from "./leaderboard"; import { isNonNegativeInteger, isPositiveInteger } from "./number"; import type { AwardedReferrerMetrics } from "./referrer-metrics"; import type { ReferralProgramRules } from "./rules"; +import { calcReferralProgramStatus, type ReferralProgramStatusId } from "./status"; /** * The default number of referrers per leaderboard page. @@ -287,6 +288,12 @@ export interface ReferrerLeaderboardPage { */ pageContext: ReferrerLeaderboardPageContext; + /** + * The status of the referral program ("Scheduled", "Active", or "Closed") + * calculated based on the program's timing relative to {@link accurateAsOf}. + */ + status: ReferralProgramStatusId; + /** * The {@link UnixTimestamp} of when the data used to build the {@link ReferrerLeaderboardPage} was accurate as of. */ @@ -315,11 +322,14 @@ export const getReferrerLeaderboardPage = ( referrers = []; } + const status = calcReferralProgramStatus(leaderboard.rules, leaderboard.accurateAsOf); + return { rules: leaderboard.rules, referrers, aggregatedMetrics: leaderboard.aggregatedMetrics, pageContext, accurateAsOf: leaderboard.accurateAsOf, + status, }; }; diff --git a/packages/ensnode-sdk/src/shared/cache/swr-cache.ts b/packages/ensnode-sdk/src/shared/cache/swr-cache.ts index 39022f336..84459e5e6 100644 --- a/packages/ensnode-sdk/src/shared/cache/swr-cache.ts +++ b/packages/ensnode-sdk/src/shared/cache/swr-cache.ts @@ -169,6 +169,19 @@ export class SWRCache { return this.cache.result; } + /** + * Returns true if this cache stores data indefinitely without revalidation. + * + * A cache is considered indefinitely stored when it has infinite TTL and no + * proactive revalidation interval configured. + */ + public isIndefinitelyStored(): boolean { + return ( + this.options.ttl === Number.POSITIVE_INFINITY && + this.options.proactiveRevalidationInterval === undefined + ); + } + /** * Destroys the background revalidation interval, if exists. */ From 0e76212da93be8bfbe62b346e8be81677f671c4d Mon Sep 17 00:00:00 2001 From: Goader Date: Tue, 10 Feb 2026 03:44:23 +0100 Subject: [PATCH 02/11] upgrading caches to indefinite storage --- .../referral-leaderboard-editions.cache.ts | 8 +- .../src/handlers/ensanalytics-api-v1.test.ts | 7 ++ ...-leaderboard-editions-caches.middleware.ts | 99 +++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) diff --git a/apps/ensapi/src/cache/referral-leaderboard-editions.cache.ts b/apps/ensapi/src/cache/referral-leaderboard-editions.cache.ts index 2672356a2..1bec8638b 100644 --- a/apps/ensapi/src/cache/referral-leaderboard-editions.cache.ts +++ b/apps/ensapi/src/cache/referral-leaderboard-editions.cache.ts @@ -51,7 +51,7 @@ const supportedOmnichainIndexingStatuses: OmnichainIndexingStatusId[] = [ * @param editionConfig - The edition configuration * @returns A function that builds the leaderboard for the given edition */ -function createEditionLeaderboardBuilder( +export function createEditionLeaderboardBuilder( editionConfig: ReferralProgramEditionConfig, ): () => Promise { return async (): Promise => { @@ -120,6 +120,10 @@ let cachedInstance: ReferralLeaderboardEditionsCacheMap | null = null; * ensuring that if one edition fails to refresh, other editions' previously successful * data remains available. * + * All caches are initially created with normal refresh behavior. Caches are dynamically upgraded + * to immutable storage (infinite TTL, no proactive revalidation) by the middleware after an edition + * has been closed for more than the safety window based on accurate indexing timestamps. + * * @param editionConfigSet - The referral program edition config set to initialize caches for * @returns A map from edition slug to its dedicated SWRCache */ @@ -134,6 +138,8 @@ export function initializeReferralLeaderboardEditionsCaches( const caches: ReferralLeaderboardEditionsCacheMap = new Map(); for (const [editionSlug, editionConfig] of editionConfigSet) { + // All editions start with normal refresh behavior + // They will be dynamically upgraded to immutable storage by the middleware const cache = new SWRCache({ fn: createEditionLeaderboardBuilder(editionConfig), ttl: minutesToSeconds(1), diff --git a/apps/ensapi/src/handlers/ensanalytics-api-v1.test.ts b/apps/ensapi/src/handlers/ensanalytics-api-v1.test.ts index aa72ef716..2bbe12ef0 100644 --- a/apps/ensapi/src/handlers/ensanalytics-api-v1.test.ts +++ b/apps/ensapi/src/handlers/ensanalytics-api-v1.test.ts @@ -34,6 +34,7 @@ import { deserializeReferrerMetricsEditionsResponse, ReferralProgramEditionConfigSetResponseCodes, type ReferralProgramEditionSlug, + ReferralProgramStatuses, ReferrerEditionMetricsTypeIds, type ReferrerLeaderboard, ReferrerLeaderboardPageResponseCodes, @@ -118,6 +119,7 @@ describe("/v1/ensanalytics", () => { responseCode: ReferrerLeaderboardPageResponseCodes.Ok, data: { ...populatedReferrerLeaderboard, + status: ReferralProgramStatuses.Active, pageContext: { endIndex: 9, hasNext: true, @@ -139,6 +141,7 @@ describe("/v1/ensanalytics", () => { responseCode: ReferrerLeaderboardPageResponseCodes.Ok, data: { ...populatedReferrerLeaderboard, + status: ReferralProgramStatuses.Active, pageContext: { endIndex: 19, hasNext: true, @@ -159,6 +162,7 @@ describe("/v1/ensanalytics", () => { responseCode: ReferrerLeaderboardPageResponseCodes.Ok, data: { ...populatedReferrerLeaderboard, + status: ReferralProgramStatuses.Active, pageContext: { endIndex: 28, hasNext: false, @@ -223,6 +227,7 @@ describe("/v1/ensanalytics", () => { responseCode: ReferrerLeaderboardPageResponseCodes.Ok, data: { ...emptyReferralLeaderboard, + status: ReferralProgramStatuses.Active, pageContext: { hasNext: false, hasPrev: false, @@ -364,6 +369,7 @@ describe("/v1/ensanalytics", () => { referrer: expectedMetrics, aggregatedMetrics: populatedReferrerLeaderboard.aggregatedMetrics, accurateAsOf: expectedAccurateAsOf, + status: ReferralProgramStatuses.Active, }, "2026-03": { type: ReferrerEditionMetricsTypeIds.Ranked, @@ -371,6 +377,7 @@ describe("/v1/ensanalytics", () => { referrer: expectedMetrics, aggregatedMetrics: populatedReferrerLeaderboard.aggregatedMetrics, accurateAsOf: expectedAccurateAsOf, + status: ReferralProgramStatuses.Active, }, }, } satisfies ReferrerMetricsEditionsResponseOk; diff --git a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts index 1546b76e0..e6bdda2fa 100644 --- a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts +++ b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts @@ -1,10 +1,21 @@ +import type { ReferralProgramEditionConfigSet } from "@namehash/ens-referrals/v1"; +import { minutesToSeconds } from "date-fns"; + +import { getLatestIndexedBlockRef, SWRCache } from "@ensnode/ensnode-sdk"; + +import { indexingStatusCache } from "@/cache/indexing-status.cache"; import { + createEditionLeaderboardBuilder, initializeReferralLeaderboardEditionsCaches, type ReferralLeaderboardEditionsCacheMap, } from "@/cache/referral-leaderboard-editions.cache"; +import { assumeReferralProgramEditionImmutablyClosed } from "@/lib/ensanalytics/referrer-leaderboard/closeout"; import { factory } from "@/lib/hono-factory"; +import { makeLogger } from "@/lib/logger"; import type { referralProgramEditionConfigSetMiddleware } from "@/middleware/referral-program-edition-set.middleware"; +const logger = makeLogger("referral-leaderboard-editions-caches-middleware"); + /** * Type definition for the referral leaderboard editions caches middleware context passed to downstream middleware and handlers. */ @@ -29,6 +40,84 @@ export type ReferralLeaderboardEditionsCachesMiddlewareVariables = { referralLeaderboardEditionsCaches: ReferralLeaderboardEditionsCacheMap | Error; }; +/** + * Checks all caches and upgrades any that have become immutable to store them indefinitely. + * + * This function is called non-blocking on each request to opportunistically upgrade caches + * when editions close. Once a cache is upgraded to immutable storage (infinite TTL, no proactive + * revalidation), the quick check ensures minimal overhead on all future requests. + * + * @param caches - The map of edition caches to check and potentially upgrade + * @param editionConfigSet - The edition config set containing rules for each edition + */ +async function checkAndUpgradeImmutableCaches( + caches: ReferralLeaderboardEditionsCacheMap, + editionConfigSet: ReferralProgramEditionConfigSet, +): Promise { + for (const [editionSlug, cache] of caches) { + if (cache.isIndefinitelyStored()) { + continue; + } + + const editionConfig = editionConfigSet.get(editionSlug); + if (!editionConfig) { + logger.warn({ editionSlug }, "Edition config not found during immutability check"); + continue; + } + + // Get current indexing status to determine accurate timestamp + const indexingStatus = await indexingStatusCache.read(); + if (indexingStatus instanceof Error) { + logger.debug( + { error: indexingStatus, editionSlug }, + "Failed to read indexing status during immutability check", + ); + continue; + } + + // Get latest indexed block ref for this edition's chain + const latestIndexedBlockRef = getLatestIndexedBlockRef( + indexingStatus, + editionConfig.rules.subregistryId.chainId, + ); + + if (latestIndexedBlockRef === null) { + logger.debug( + { editionSlug, chainId: editionConfig.rules.subregistryId.chainId }, + "No indexed block ref during immutability check", + ); + continue; + } + + // Check if edition is immutably closed based on accurate timestamp + const isImmutable = assumeReferralProgramEditionImmutablyClosed( + editionConfig.rules, + latestIndexedBlockRef.timestamp, + ); + + if (!isImmutable) { + continue; + } + + // Edition is now immutable! Upgrade the cache + logger.info({ editionSlug }, "Upgrading cache to immutable storage"); + + cache.destroy(); + + const immutableCache = new SWRCache({ + fn: createEditionLeaderboardBuilder(editionConfig), + ttl: Number.POSITIVE_INFINITY, + proactiveRevalidationInterval: undefined, + errorTtl: minutesToSeconds(1), + proactivelyInitialize: true, + }); + + caches.set(editionSlug, immutableCache); + + logger.info({ editionSlug }, "Successfully upgraded cache to immutable storage"); + } +} + /** * Middleware that provides {@link ReferralLeaderboardEditionsCachesMiddlewareVariables} * to downstream middleware and handlers. @@ -36,6 +125,10 @@ export type ReferralLeaderboardEditionsCachesMiddlewareVariables = { * This middleware depends on {@link referralProgramEditionConfigSetMiddleware} to provide * the edition config set. If the edition config set failed to load, this middleware propagates the error. * Otherwise, it initializes caches for each edition in the config set. + * + * On each request, this middleware non-blocking checks if any caches should be upgraded to immutable + * storage based on accurate indexing timestamps. This allows caches to dynamically transition from + * refreshing to indefinite storage as editions close. */ export const referralLeaderboardEditionsCachesMiddleware = factory.createMiddleware( async (c, next) => { @@ -58,6 +151,12 @@ export const referralLeaderboardEditionsCachesMiddleware = factory.createMiddlew // Initialize caches for the edition config set const caches = initializeReferralLeaderboardEditionsCaches(editionConfigSet); c.set("referralLeaderboardEditionsCaches", caches); + + // Non-blocking: Check and upgrade any caches that have become immutable + checkAndUpgradeImmutableCaches(caches, editionConfigSet).catch((error) => { + logger.error({ error }, "Failed to check and upgrade immutable caches"); + }); + await next(); }, ); From f6f6ae281bfb44a8bf66b86d6120f1c15cc17738 Mon Sep 17 00:00:00 2001 From: Goader Date: Tue, 10 Feb 2026 16:10:31 +0100 Subject: [PATCH 03/11] fixed, upgraded cache upgrading :) --- .../referrer-leaderboard/closeout.ts | 6 +- ...-leaderboard-editions-caches.middleware.ts | 147 ++++++++++++++---- .../ens-referrals/src/v1/api/zod-schemas.ts | 18 ++- .../ensnode-sdk/src/shared/cache/swr-cache.ts | 3 +- 4 files changed, 140 insertions(+), 34 deletions(-) diff --git a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/closeout.ts b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/closeout.ts index 9a030778a..e1f0d7333 100644 --- a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/closeout.ts +++ b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/closeout.ts @@ -1,7 +1,7 @@ import type { ReferralProgramRules } from "@namehash/ens-referrals/v1"; import { minutesToSeconds } from "date-fns"; -import { type Duration, durationBetween, type UnixTimestamp } from "@ensnode/ensnode-sdk"; +import { addDuration, type Duration, type UnixTimestamp } from "@ensnode/ensnode-sdk"; /** * Duration after which we assume a closed edition is safe from chain reorganizations. @@ -27,6 +27,6 @@ export function assumeReferralProgramEditionImmutablyClosed( rules: ReferralProgramRules, referenceTime: UnixTimestamp, ): boolean { - const timeSinceClose = durationBetween(rules.endTime, referenceTime); - return timeSinceClose > ASSUMED_CHAIN_REORG_SAFE_DURATION; + const immutabilityThreshold = addDuration(rules.endTime, ASSUMED_CHAIN_REORG_SAFE_DURATION); + return referenceTime > immutabilityThreshold; } diff --git a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts index e6bdda2fa..f03595702 100644 --- a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts +++ b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts @@ -1,4 +1,8 @@ -import type { ReferralProgramEditionConfigSet } from "@namehash/ens-referrals/v1"; +import type { + ReferralProgramEditionConfig, + ReferralProgramEditionConfigSet, + ReferrerLeaderboard, +} from "@namehash/ens-referrals/v1"; import { minutesToSeconds } from "date-fns"; import { getLatestIndexedBlockRef, SWRCache } from "@ensnode/ensnode-sdk"; @@ -16,6 +20,12 @@ import type { referralProgramEditionConfigSetMiddleware } from "@/middleware/ref const logger = makeLogger("referral-leaderboard-editions-caches-middleware"); +/** + * Tracks in-progress cache upgrades to prevent concurrent upgrades of the same edition. + * Maps edition slug to the upgrade promise. + */ +const inProgressUpgrades = new Map>(); + /** * Type definition for the referral leaderboard editions caches middleware context passed to downstream middleware and handlers. */ @@ -40,12 +50,88 @@ export type ReferralLeaderboardEditionsCachesMiddlewareVariables = { referralLeaderboardEditionsCaches: ReferralLeaderboardEditionsCacheMap | Error; }; +/** + * Upgrades a single edition's cache from regular SWR to immutable storage. + * + * This function: + * 1. Creates a new cache with infinite TTL and proactive initialization + * 2. Waits for the new cache to successfully load data + * 3. Verifies the loaded data is immutably closed (fresh enough) + * 4. Only then destroys the old cache and swaps in the new one + * + * If initialization fails or data is not fresh enough, keeps the old cache + * and the upgrade will be retried on a future request. + * + * @param editionSlug - The edition slug being upgraded + * @param oldCache - The existing cache to be replaced + * @param editionConfig - The edition configuration + * @param caches - The map of all edition caches (for swapping) + */ +async function upgradeEditionCache( + editionSlug: string, + oldCache: SWRCache, + editionConfig: ReferralProgramEditionConfig, + caches: ReferralLeaderboardEditionsCacheMap, +): Promise { + logger.info({ editionSlug }, "Starting cache upgrade to immutable storage"); + + // Create new cache with proactive initialization (starts loading immediately) + const newCache = new SWRCache({ + fn: createEditionLeaderboardBuilder(editionConfig), + ttl: Number.POSITIVE_INFINITY, + proactiveRevalidationInterval: undefined, + errorTtl: minutesToSeconds(1), + proactivelyInitialize: true, + }); + + // Wait for the new cache to successfully initialize + const result = await newCache.read(); + + if (result instanceof Error) { + logger.warn( + { editionSlug, error: result }, + "Failed to initialize new cache, keeping old cache", + ); + newCache.destroy(); + return; + } + + // Verify the data is fresh enough (immutably closed based on its own accurateAsOf) + const isImmutable = assumeReferralProgramEditionImmutablyClosed( + result.rules, + result.accurateAsOf, + ); + + if (!isImmutable) { + logger.warn( + { editionSlug, accurateAsOf: result.accurateAsOf }, + "New cache data is not fresh enough to be considered immutably closed, keeping old cache", + ); + newCache.destroy(); + return; + } + + // Success! Swap the caches + logger.info({ editionSlug }, "New cache successfully initialized and verified, swapping caches"); + + oldCache.destroy(); + caches.set(editionSlug, newCache); + + logger.info({ editionSlug }, "Cache upgrade to immutable storage complete"); +} + /** * Checks all caches and upgrades any that have become immutable to store them indefinitely. * * This function is called non-blocking on each request to opportunistically upgrade caches - * when editions close. Once a cache is upgraded to immutable storage (infinite TTL, no proactive - * revalidation), the quick check ensures minimal overhead on all future requests. + * when editions close. Each edition's upgrade runs independently in the background, ensuring: + * - No data loss: old cache continues serving while new cache initializes + * - Graceful failure: if new cache fails to initialize, old cache remains + * - No race conditions: atomic check-and-set prevents concurrent upgrades of same edition + * - Parallel upgrades: multiple editions can upgrade simultaneously + * + * Once a cache is upgraded to immutable storage (infinite TTL, no proactive revalidation), + * the quick check ensures minimal overhead on all future requests. * * @param caches - The map of edition caches to check and potentially upgrade * @param editionConfigSet - The edition config set containing rules for each edition @@ -54,7 +140,18 @@ async function checkAndUpgradeImmutableCaches( caches: ReferralLeaderboardEditionsCacheMap, editionConfigSet: ReferralProgramEditionConfigSet, ): Promise { + // Read indexing status once before the loop and reuse for all editions + const indexingStatus = await indexingStatusCache.read(); + if (indexingStatus instanceof Error) { + logger.debug( + { error: indexingStatus }, + "Failed to read indexing status during immutability check", + ); + return; + } + for (const [editionSlug, cache] of caches) { + // Quick exit: already upgraded to immutable storage if (cache.isIndefinitelyStored()) { continue; } @@ -65,16 +162,6 @@ async function checkAndUpgradeImmutableCaches( continue; } - // Get current indexing status to determine accurate timestamp - const indexingStatus = await indexingStatusCache.read(); - if (indexingStatus instanceof Error) { - logger.debug( - { error: indexingStatus, editionSlug }, - "Failed to read indexing status during immutability check", - ); - continue; - } - // Get latest indexed block ref for this edition's chain const latestIndexedBlockRef = getLatestIndexedBlockRef( indexingStatus, @@ -89,7 +176,7 @@ async function checkAndUpgradeImmutableCaches( continue; } - // Check if edition is immutably closed based on accurate timestamp + // Check if edition is immutably closed based on current indexing timestamp const isImmutable = assumeReferralProgramEditionImmutablyClosed( editionConfig.rules, latestIndexedBlockRef.timestamp, @@ -99,22 +186,30 @@ async function checkAndUpgradeImmutableCaches( continue; } - // Edition is now immutable! Upgrade the cache - logger.info({ editionSlug }, "Upgrading cache to immutable storage"); + // Atomic check-and-set: prevent concurrent upgrades of the same edition + const upgradePromise = (() => { + // Check if upgrade already in progress + if (inProgressUpgrades.has(editionSlug)) { + return null; + } - cache.destroy(); + // Start upgrade and register promise immediately (no await in between) + const promise = upgradeEditionCache(editionSlug, cache, editionConfig, caches).finally(() => { + // Always clean up the in-progress tracking + inProgressUpgrades.delete(editionSlug); + }); - const immutableCache = new SWRCache({ - fn: createEditionLeaderboardBuilder(editionConfig), - ttl: Number.POSITIVE_INFINITY, - proactiveRevalidationInterval: undefined, - errorTtl: minutesToSeconds(1), - proactivelyInitialize: true, - }); + inProgressUpgrades.set(editionSlug, promise); + return promise; + })(); - caches.set(editionSlug, immutableCache); + if (!upgradePromise) { + // Another request is already upgrading this edition + logger.debug({ editionSlug }, "Upgrade already in progress, skipping"); + } - logger.info({ editionSlug }, "Successfully upgraded cache to immutable storage"); + // Don't await - let upgrade run in background + // Errors are logged inside upgradeEditionCache } } diff --git a/packages/ens-referrals/src/v1/api/zod-schemas.ts b/packages/ens-referrals/src/v1/api/zod-schemas.ts index d0a9d6b6a..c0a6769bd 100644 --- a/packages/ens-referrals/src/v1/api/zod-schemas.ts +++ b/packages/ens-referrals/src/v1/api/zod-schemas.ts @@ -150,7 +150,11 @@ export const makeReferrerLeaderboardPageSchema = (valueLabel: string = "Referrer referrers: z.array(makeAwardedReferrerMetricsSchema(`${valueLabel}.referrers[record]`)), aggregatedMetrics: makeAggregatedReferrerMetricsSchema(`${valueLabel}.aggregatedMetrics`), pageContext: makeReferrerLeaderboardPageContextSchema(`${valueLabel}.pageContext`), - status: z.enum(ReferralProgramStatuses), + status: z.enum([ + ReferralProgramStatuses.Scheduled, + ReferralProgramStatuses.Active, + ReferralProgramStatuses.Closed, + ]), accurateAsOf: makeUnixTimestampSchema(`${valueLabel}.accurateAsOf`), }); @@ -199,7 +203,11 @@ export const makeReferrerEditionMetricsRankedSchema = ( rules: makeReferralProgramRulesSchema(`${valueLabel}.rules`), referrer: makeAwardedReferrerMetricsSchema(`${valueLabel}.referrer`), aggregatedMetrics: makeAggregatedReferrerMetricsSchema(`${valueLabel}.aggregatedMetrics`), - status: z.enum(ReferralProgramStatuses), + status: z.enum([ + ReferralProgramStatuses.Scheduled, + ReferralProgramStatuses.Active, + ReferralProgramStatuses.Closed, + ]), accurateAsOf: makeUnixTimestampSchema(`${valueLabel}.accurateAsOf`), }); @@ -214,7 +222,11 @@ export const makeReferrerEditionMetricsUnrankedSchema = ( rules: makeReferralProgramRulesSchema(`${valueLabel}.rules`), referrer: makeUnrankedReferrerMetricsSchema(`${valueLabel}.referrer`), aggregatedMetrics: makeAggregatedReferrerMetricsSchema(`${valueLabel}.aggregatedMetrics`), - status: z.enum(ReferralProgramStatuses), + status: z.enum([ + ReferralProgramStatuses.Scheduled, + ReferralProgramStatuses.Active, + ReferralProgramStatuses.Closed, + ]), accurateAsOf: makeUnixTimestampSchema(`${valueLabel}.accurateAsOf`), }); diff --git a/packages/ensnode-sdk/src/shared/cache/swr-cache.ts b/packages/ensnode-sdk/src/shared/cache/swr-cache.ts index 84459e5e6..72017500e 100644 --- a/packages/ensnode-sdk/src/shared/cache/swr-cache.ts +++ b/packages/ensnode-sdk/src/shared/cache/swr-cache.ts @@ -177,8 +177,7 @@ export class SWRCache { */ public isIndefinitelyStored(): boolean { return ( - this.options.ttl === Number.POSITIVE_INFINITY && - this.options.proactiveRevalidationInterval === undefined + this.options.ttl === Number.POSITIVE_INFINITY && !this.options.proactiveRevalidationInterval ); } From 915eef273c7dfc568d6ce7012505bd1937348487 Mon Sep 17 00:00:00 2001 From: Goader Date: Tue, 10 Feb 2026 16:10:44 +0100 Subject: [PATCH 04/11] docs(changeset): --- .changeset/lemon-moose-count.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .changeset/lemon-moose-count.md diff --git a/.changeset/lemon-moose-count.md b/.changeset/lemon-moose-count.md new file mode 100644 index 000000000..a845151cc --- /dev/null +++ b/.changeset/lemon-moose-count.md @@ -0,0 +1,2 @@ +--- +--- From fb7779aec6b2956314cb1f2c4fbd96e8990fe1dd Mon Sep 17 00:00:00 2001 From: Goader Date: Tue, 10 Feb 2026 16:37:54 +0100 Subject: [PATCH 05/11] review applied --- .changeset/brave-waves-flow.md | 6 +++++ .changeset/clever-frogs-detect.md | 5 ++++ .changeset/lemon-moose-count.md | 3 +++ ...-leaderboard-editions-caches.middleware.ts | 16 +++++------ .../ens-referrals/src/v1/api/zod-schemas.ts | 27 +++++++++---------- 5 files changed, 34 insertions(+), 23 deletions(-) create mode 100644 .changeset/brave-waves-flow.md create mode 100644 .changeset/clever-frogs-detect.md diff --git a/.changeset/brave-waves-flow.md b/.changeset/brave-waves-flow.md new file mode 100644 index 000000000..2649eb7e7 --- /dev/null +++ b/.changeset/brave-waves-flow.md @@ -0,0 +1,6 @@ +--- +"@namehash/ens-referrals": minor +"ensapi": minor +--- + +Added `status` field to referral program API responses (`ReferrerLeaderboardPage`, `ReferrerEditionMetricsRanked`, `ReferrerEditionMetricsUnranked`) indicating whether a program is "Scheduled", "Active", or "Closed" based on the program's timing relative to `accurateAsOf`. diff --git a/.changeset/clever-frogs-detect.md b/.changeset/clever-frogs-detect.md new file mode 100644 index 000000000..ff5d08288 --- /dev/null +++ b/.changeset/clever-frogs-detect.md @@ -0,0 +1,5 @@ +--- +"@ensnode/ensnode-sdk": patch +--- + +Added `isIndefinitelyStored()` method to SWRCache for detecting caches configured with infinite TTL and no proactive revalidation. diff --git a/.changeset/lemon-moose-count.md b/.changeset/lemon-moose-count.md index a845151cc..f70a52b04 100644 --- a/.changeset/lemon-moose-count.md +++ b/.changeset/lemon-moose-count.md @@ -1,2 +1,5 @@ --- +"ensapi": minor --- + +Implemented automatic cache upgrade to indefinite storage for immutably closed referral program editions. Caches safely transition from regular SWR to immutable storage with zero-data-loss initialization, atomic race prevention, and graceful failure handling. diff --git a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts index f03595702..e8087b6e2 100644 --- a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts +++ b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts @@ -140,7 +140,6 @@ async function checkAndUpgradeImmutableCaches( caches: ReferralLeaderboardEditionsCacheMap, editionConfigSet: ReferralProgramEditionConfigSet, ): Promise { - // Read indexing status once before the loop and reuse for all editions const indexingStatus = await indexingStatusCache.read(); if (indexingStatus instanceof Error) { logger.debug( @@ -151,7 +150,6 @@ async function checkAndUpgradeImmutableCaches( } for (const [editionSlug, cache] of caches) { - // Quick exit: already upgraded to immutable storage if (cache.isIndefinitelyStored()) { continue; } @@ -162,7 +160,6 @@ async function checkAndUpgradeImmutableCaches( continue; } - // Get latest indexed block ref for this edition's chain const latestIndexedBlockRef = getLatestIndexedBlockRef( indexingStatus, editionConfig.rules.subregistryId.chainId, @@ -176,7 +173,6 @@ async function checkAndUpgradeImmutableCaches( continue; } - // Check if edition is immutably closed based on current indexing timestamp const isImmutable = assumeReferralProgramEditionImmutablyClosed( editionConfig.rules, latestIndexedBlockRef.timestamp, @@ -194,10 +190,14 @@ async function checkAndUpgradeImmutableCaches( } // Start upgrade and register promise immediately (no await in between) - const promise = upgradeEditionCache(editionSlug, cache, editionConfig, caches).finally(() => { - // Always clean up the in-progress tracking - inProgressUpgrades.delete(editionSlug); - }); + const promise = upgradeEditionCache(editionSlug, cache, editionConfig, caches) + .catch((error) => { + logger.error({ editionSlug, error }, "Unexpected error during cache upgrade"); + }) + .finally(() => { + // Always clean up the in-progress tracking + inProgressUpgrades.delete(editionSlug); + }); inProgressUpgrades.set(editionSlug, promise); return promise; diff --git a/packages/ens-referrals/src/v1/api/zod-schemas.ts b/packages/ens-referrals/src/v1/api/zod-schemas.ts index c0a6769bd..2ebe1516e 100644 --- a/packages/ens-referrals/src/v1/api/zod-schemas.ts +++ b/packages/ens-referrals/src/v1/api/zod-schemas.ts @@ -141,6 +141,15 @@ export const makeReferrerLeaderboardPageContextSchema = ( endIndex: z.optional(makeNonNegativeIntegerSchema(`${valueLabel}.endIndex`)), }); +/** + * Schema for referral program status field. + * Validates that the status is one of: "Scheduled", "Active", or "Closed". + */ +export const makeReferralProgramStatusSchema = (valueLabel: string = "status") => + z.enum(ReferralProgramStatuses, { + message: `${valueLabel} must be "Scheduled", "Active", or "Closed"`, + }); + /** * Schema for ReferrerLeaderboardPage */ @@ -150,11 +159,7 @@ export const makeReferrerLeaderboardPageSchema = (valueLabel: string = "Referrer referrers: z.array(makeAwardedReferrerMetricsSchema(`${valueLabel}.referrers[record]`)), aggregatedMetrics: makeAggregatedReferrerMetricsSchema(`${valueLabel}.aggregatedMetrics`), pageContext: makeReferrerLeaderboardPageContextSchema(`${valueLabel}.pageContext`), - status: z.enum([ - ReferralProgramStatuses.Scheduled, - ReferralProgramStatuses.Active, - ReferralProgramStatuses.Closed, - ]), + status: makeReferralProgramStatusSchema(`${valueLabel}.status`), accurateAsOf: makeUnixTimestampSchema(`${valueLabel}.accurateAsOf`), }); @@ -203,11 +208,7 @@ export const makeReferrerEditionMetricsRankedSchema = ( rules: makeReferralProgramRulesSchema(`${valueLabel}.rules`), referrer: makeAwardedReferrerMetricsSchema(`${valueLabel}.referrer`), aggregatedMetrics: makeAggregatedReferrerMetricsSchema(`${valueLabel}.aggregatedMetrics`), - status: z.enum([ - ReferralProgramStatuses.Scheduled, - ReferralProgramStatuses.Active, - ReferralProgramStatuses.Closed, - ]), + status: makeReferralProgramStatusSchema(`${valueLabel}.status`), accurateAsOf: makeUnixTimestampSchema(`${valueLabel}.accurateAsOf`), }); @@ -222,11 +223,7 @@ export const makeReferrerEditionMetricsUnrankedSchema = ( rules: makeReferralProgramRulesSchema(`${valueLabel}.rules`), referrer: makeUnrankedReferrerMetricsSchema(`${valueLabel}.referrer`), aggregatedMetrics: makeAggregatedReferrerMetricsSchema(`${valueLabel}.aggregatedMetrics`), - status: z.enum([ - ReferralProgramStatuses.Scheduled, - ReferralProgramStatuses.Active, - ReferralProgramStatuses.Closed, - ]), + status: makeReferralProgramStatusSchema(`${valueLabel}.status`), accurateAsOf: makeUnixTimestampSchema(`${valueLabel}.accurateAsOf`), }); From f590bdcedfbf542e08e56c95cbfa6dbb35d46b8a Mon Sep 17 00:00:00 2001 From: Goader Date: Tue, 10 Feb 2026 18:54:43 +0100 Subject: [PATCH 06/11] moved cache upgrading and added tests --- .../cache-upgrade.test.ts | 256 ++++++++++++++++++ .../referrer-leaderboard/cache-upgrade.ts | 185 +++++++++++++ ...-leaderboard-editions-caches.middleware.ts | 194 +------------ 3 files changed, 452 insertions(+), 183 deletions(-) create mode 100644 apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts create mode 100644 apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts diff --git a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts new file mode 100644 index 000000000..41afe998b --- /dev/null +++ b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts @@ -0,0 +1,256 @@ +import type { ReferralProgramEditionConfig } from "@namehash/ens-referrals/v1"; +import { minutesToSeconds } from "date-fns"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { addDuration, parseEth, parseUsdc, SWRCache } from "@ensnode/ensnode-sdk"; + +import { ASSUMED_CHAIN_REORG_SAFE_DURATION } from "@/lib/ensanalytics/referrer-leaderboard/closeout"; + +// Mock the cache module to avoid config loading +vi.mock("@/cache/referral-leaderboard-editions.cache", () => ({ + createEditionLeaderboardBuilder: vi.fn(), +})); + +import { createEditionLeaderboardBuilder } from "@/cache/referral-leaderboard-editions.cache"; + +import { upgradeEditionCache } from "./cache-upgrade"; + +describe("Cache upgrade to immutable storage", () => { + const now = Math.floor(Date.now() / 1000); + + // Mock leaderboard builder function + const mockBuilder = async () => ({ + rules: { + totalAwardPoolValue: parseUsdc("10000"), + maxQualifiedReferrers: 100, + startTime: now - 3600, + endTime: now - 1200, + subregistryId: { chainId: 1, address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const }, + rulesUrl: new URL("https://example.com/rules"), + }, + aggregatedMetrics: { + grandTotalReferrals: 0, + grandTotalIncrementalDuration: 0, + grandTotalRevenueContribution: parseEth("0"), + grandTotalQualifiedReferrersFinalScore: 0, + minFinalScoreToQualify: 0, + }, + referrers: new Map(), + accurateAsOf: now, + }); + + it("should detect regular cache as NOT indefinitely stored", () => { + const cache = new SWRCache({ + fn: mockBuilder, + ttl: minutesToSeconds(5), + proactiveRevalidationInterval: minutesToSeconds(1), + errorTtl: minutesToSeconds(1), + proactivelyInitialize: false, + }); + + expect(cache.isIndefinitelyStored()).toBe(false); + }); + + it("should detect immutable cache as indefinitely stored", () => { + const cache = new SWRCache({ + fn: mockBuilder, + ttl: Number.POSITIVE_INFINITY, + proactiveRevalidationInterval: undefined, + errorTtl: minutesToSeconds(1), + proactivelyInitialize: false, + }); + + expect(cache.isIndefinitelyStored()).toBe(true); + }); + + it("should NOT detect cache with infinite TTL but proactive revalidation as indefinitely stored", () => { + const cache = new SWRCache({ + fn: mockBuilder, + ttl: Number.POSITIVE_INFINITY, + proactiveRevalidationInterval: minutesToSeconds(1), + errorTtl: minutesToSeconds(1), + proactivelyInitialize: false, + }); + + expect(cache.isIndefinitelyStored()).toBe(false); + }); + + describe("upgradeEditionCache", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should successfully upgrade cache when edition is immutably closed", async () => { + const editionSlug = "test-edition"; + const closedEndTime = now - 1200; // 20 minutes ago + const immutableTimestamp = addDuration(closedEndTime, ASSUMED_CHAIN_REORG_SAFE_DURATION) + 1; + + const builderWithImmutableData = async () => ({ + rules: { + totalAwardPoolValue: parseUsdc("10000"), + maxQualifiedReferrers: 100, + startTime: closedEndTime - 3600, + endTime: closedEndTime, + subregistryId: { + chainId: 1, + address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, + }, + rulesUrl: new URL("https://example.com/rules"), + }, + aggregatedMetrics: { + grandTotalReferrals: 0, + grandTotalIncrementalDuration: 0, + grandTotalRevenueContribution: parseEth("0"), + grandTotalQualifiedReferrersFinalScore: 0, + minFinalScoreToQualify: 0, + }, + referrers: new Map(), + accurateAsOf: immutableTimestamp, + }); + + // Mock the builder to return our test data + vi.mocked(createEditionLeaderboardBuilder).mockReturnValue(builderWithImmutableData); + + const oldCache = new SWRCache({ + fn: builderWithImmutableData, + ttl: minutesToSeconds(5), + proactiveRevalidationInterval: minutesToSeconds(1), + errorTtl: minutesToSeconds(1), + proactivelyInitialize: false, + }); + + const caches = new Map(); + caches.set(editionSlug, oldCache); + + const editionConfig: ReferralProgramEditionConfig = { + slug: editionSlug, + displayName: "Test Edition", + rules: { + totalAwardPoolValue: parseUsdc("10000"), + maxQualifiedReferrers: 100, + startTime: closedEndTime - 3600, + endTime: closedEndTime, + subregistryId: { + chainId: 1, + address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, + }, + rulesUrl: new URL("https://example.com/rules"), + }, + }; + + await upgradeEditionCache(editionSlug, oldCache, editionConfig, caches); + + const upgradedCache = caches.get(editionSlug); + expect(upgradedCache).not.toBe(oldCache); + expect(upgradedCache?.isIndefinitelyStored()).toBe(true); + }); + + it("should keep old cache if new cache fails to initialize", async () => { + const editionSlug = "test-edition"; + const closedEndTime = now - 1200; + + const failingBuilder = async () => { + throw new Error("Failed to build leaderboard"); + }; + + // Mock the builder to return a failing builder + vi.mocked(createEditionLeaderboardBuilder).mockReturnValue(failingBuilder); + + const oldCache = new SWRCache({ + fn: failingBuilder, + ttl: minutesToSeconds(5), + proactiveRevalidationInterval: minutesToSeconds(1), + errorTtl: minutesToSeconds(1), + proactivelyInitialize: false, + }); + + const caches = new Map(); + caches.set(editionSlug, oldCache); + + const editionConfig: ReferralProgramEditionConfig = { + slug: editionSlug, + displayName: "Test Edition", + rules: { + totalAwardPoolValue: parseUsdc("10000"), + maxQualifiedReferrers: 100, + startTime: closedEndTime - 3600, + endTime: closedEndTime, + subregistryId: { + chainId: 1, + address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, + }, + rulesUrl: new URL("https://example.com/rules"), + }, + }; + + await upgradeEditionCache(editionSlug, oldCache, editionConfig, caches); + + // Old cache should still be there + expect(caches.get(editionSlug)).toBe(oldCache); + }); + + it("should keep old cache if new data is not fresh enough", async () => { + const editionSlug = "test-edition"; + const closedEndTime = now - 1200; + const notFreshEnoughTimestamp = closedEndTime + 30; // Not past safety window + + const builderWithStaleData = async () => ({ + rules: { + totalAwardPoolValue: parseUsdc("10000"), + maxQualifiedReferrers: 100, + startTime: closedEndTime - 3600, + endTime: closedEndTime, + subregistryId: { + chainId: 1, + address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, + }, + rulesUrl: new URL("https://example.com/rules"), + }, + aggregatedMetrics: { + grandTotalReferrals: 0, + grandTotalIncrementalDuration: 0, + grandTotalRevenueContribution: parseEth("0"), + grandTotalQualifiedReferrersFinalScore: 0, + minFinalScoreToQualify: 0, + }, + referrers: new Map(), + accurateAsOf: notFreshEnoughTimestamp, + }); + + // Mock the builder to return stale data + vi.mocked(createEditionLeaderboardBuilder).mockReturnValue(builderWithStaleData); + + const oldCache = new SWRCache({ + fn: builderWithStaleData, + ttl: minutesToSeconds(5), + proactiveRevalidationInterval: minutesToSeconds(1), + errorTtl: minutesToSeconds(1), + proactivelyInitialize: false, + }); + + const caches = new Map(); + caches.set(editionSlug, oldCache); + + const editionConfig: ReferralProgramEditionConfig = { + slug: editionSlug, + displayName: "Test Edition", + rules: { + totalAwardPoolValue: parseUsdc("10000"), + maxQualifiedReferrers: 100, + startTime: closedEndTime - 3600, + endTime: closedEndTime, + subregistryId: { + chainId: 1, + address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, + }, + rulesUrl: new URL("https://example.com/rules"), + }, + }; + + await upgradeEditionCache(editionSlug, oldCache, editionConfig, caches); + + // Old cache should still be there + expect(caches.get(editionSlug)).toBe(oldCache); + }); + }); +}); diff --git a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts new file mode 100644 index 000000000..fd6f5b7c3 --- /dev/null +++ b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts @@ -0,0 +1,185 @@ +import type { + ReferralProgramEditionConfig, + ReferralProgramEditionConfigSet, + ReferrerLeaderboard, +} from "@namehash/ens-referrals/v1"; +import { minutesToSeconds } from "date-fns"; + +import { + type CrossChainIndexingStatusSnapshot, + getLatestIndexedBlockRef, + SWRCache, +} from "@ensnode/ensnode-sdk"; + +import type { ReferralLeaderboardEditionsCacheMap } from "@/cache/referral-leaderboard-editions.cache"; +import { createEditionLeaderboardBuilder } from "@/cache/referral-leaderboard-editions.cache"; +import { makeLogger } from "@/lib/logger"; + +import { assumeReferralProgramEditionImmutablyClosed } from "./closeout"; + +const logger = makeLogger("referral-leaderboard-cache-upgrade"); + +/** + * Tracks in-progress cache upgrades to prevent concurrent upgrades of the same edition. + * Maps edition slug to the upgrade promise. + */ +const inProgressUpgrades = new Map>(); + +/** + * Upgrades a single edition's cache from regular SWR to immutable storage. + * + * This function: + * 1. Creates a new cache with infinite TTL and proactive initialization + * 2. Waits for the new cache to successfully load data + * 3. Verifies the loaded data is immutably closed (fresh enough) + * 4. Only then destroys the old cache and swaps in the new one + * + * If initialization fails or data is not fresh enough, keeps the old cache + * and the upgrade will be retried on a future request. + * + * @param editionSlug - The edition slug being upgraded + * @param oldCache - The existing cache to be replaced + * @param editionConfig - The edition configuration + * @param caches - The map of all edition caches (for swapping) + */ +export async function upgradeEditionCache( + editionSlug: string, + oldCache: SWRCache, + editionConfig: ReferralProgramEditionConfig, + caches: ReferralLeaderboardEditionsCacheMap, +): Promise { + logger.info({ editionSlug }, "Starting cache upgrade to immutable storage"); + + // Create new cache with proactive initialization (starts loading immediately) + const newCache = new SWRCache({ + fn: createEditionLeaderboardBuilder(editionConfig), + ttl: Number.POSITIVE_INFINITY, + proactiveRevalidationInterval: undefined, + errorTtl: minutesToSeconds(1), + proactivelyInitialize: true, + }); + + // Wait for the new cache to successfully initialize + const result = await newCache.read(); + + if (result instanceof Error) { + logger.warn( + { editionSlug, error: result }, + "Failed to initialize new cache, keeping old cache", + ); + newCache.destroy(); + return; + } + + // Verify the data is fresh enough (immutably closed based on its own accurateAsOf) + const isImmutable = assumeReferralProgramEditionImmutablyClosed( + result.rules, + result.accurateAsOf, + ); + + if (!isImmutable) { + logger.warn( + { editionSlug, accurateAsOf: result.accurateAsOf }, + "New cache data is not fresh enough to be considered immutably closed, keeping old cache", + ); + newCache.destroy(); + return; + } + + // Success! Swap the caches + logger.info({ editionSlug }, "New cache successfully initialized and verified, swapping caches"); + + oldCache.destroy(); + caches.set(editionSlug, newCache); + + logger.info({ editionSlug }, "Cache upgrade to immutable storage complete"); +} + +/** + * Checks all caches and upgrades any that have become immutable to store them indefinitely. + * + * This function is called non-blocking on each request to opportunistically upgrade caches + * when editions close. Each edition's upgrade runs independently in the background, ensuring: + * - No data loss: old cache continues serving while new cache initializes + * - Graceful failure: if new cache fails to initialize, old cache remains + * - No race conditions: atomic check-and-set prevents concurrent upgrades of same edition + * - Parallel upgrades: multiple editions can upgrade simultaneously + * + * Once a cache is upgraded to immutable storage (infinite TTL, no proactive revalidation), + * the quick check ensures minimal overhead on all future requests. + * + * @param caches - The map of edition caches to check and potentially upgrade + * @param editionConfigSet - The edition config set containing rules for each edition + * @param indexingStatus - The current indexing status snapshot + */ +export async function checkAndUpgradeImmutableCaches( + caches: ReferralLeaderboardEditionsCacheMap, + editionConfigSet: ReferralProgramEditionConfigSet, + indexingStatus: CrossChainIndexingStatusSnapshot, +): Promise { + for (const [editionSlug, cache] of caches) { + // Quick exit: already upgraded to immutable storage + if (cache.isIndefinitelyStored()) { + continue; + } + + const editionConfig = editionConfigSet.get(editionSlug); + if (!editionConfig) { + logger.warn({ editionSlug }, "Edition config not found during immutability check"); + continue; + } + + // Get latest indexed block ref for this edition's chain + const latestIndexedBlockRef = getLatestIndexedBlockRef( + indexingStatus, + editionConfig.rules.subregistryId.chainId, + ); + + if (latestIndexedBlockRef === null) { + logger.debug( + { editionSlug, chainId: editionConfig.rules.subregistryId.chainId }, + "No indexed block ref during immutability check", + ); + continue; + } + + // Check if edition is immutably closed based on current indexing timestamp + const isImmutable = assumeReferralProgramEditionImmutablyClosed( + editionConfig.rules, + latestIndexedBlockRef.timestamp, + ); + + if (!isImmutable) { + continue; + } + + // Atomic check-and-set: prevent concurrent upgrades of the same edition + const upgradePromise = (() => { + // Check if upgrade already in progress + if (inProgressUpgrades.has(editionSlug)) { + return null; + } + + // Start upgrade and register promise immediately (no await in between) + const promise = upgradeEditionCache(editionSlug, cache, editionConfig, caches) + .catch((error) => { + logger.error({ editionSlug, error }, "Unexpected error during cache upgrade"); + }) + .finally(() => { + // Always clean up the in-progress tracking + inProgressUpgrades.delete(editionSlug); + }); + + inProgressUpgrades.set(editionSlug, promise); + return promise; + })(); + + if (!upgradePromise) { + // Another request is already upgrading this edition + logger.debug({ editionSlug }, "Upgrade already in progress, skipping"); + } + + // Don't await - let upgrade run in background + // Errors are logged inside upgradeEditionCache + } +} diff --git a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts index e8087b6e2..038c63c56 100644 --- a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts +++ b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts @@ -1,31 +1,15 @@ -import type { - ReferralProgramEditionConfig, - ReferralProgramEditionConfigSet, - ReferrerLeaderboard, -} from "@namehash/ens-referrals/v1"; -import { minutesToSeconds } from "date-fns"; - -import { getLatestIndexedBlockRef, SWRCache } from "@ensnode/ensnode-sdk"; - import { indexingStatusCache } from "@/cache/indexing-status.cache"; import { - createEditionLeaderboardBuilder, initializeReferralLeaderboardEditionsCaches, type ReferralLeaderboardEditionsCacheMap, } from "@/cache/referral-leaderboard-editions.cache"; -import { assumeReferralProgramEditionImmutablyClosed } from "@/lib/ensanalytics/referrer-leaderboard/closeout"; +import { checkAndUpgradeImmutableCaches } from "@/lib/ensanalytics/referrer-leaderboard/cache-upgrade"; import { factory } from "@/lib/hono-factory"; import { makeLogger } from "@/lib/logger"; import type { referralProgramEditionConfigSetMiddleware } from "@/middleware/referral-program-edition-set.middleware"; const logger = makeLogger("referral-leaderboard-editions-caches-middleware"); -/** - * Tracks in-progress cache upgrades to prevent concurrent upgrades of the same edition. - * Maps edition slug to the upgrade promise. - */ -const inProgressUpgrades = new Map>(); - /** * Type definition for the referral leaderboard editions caches middleware context passed to downstream middleware and handlers. */ @@ -50,169 +34,6 @@ export type ReferralLeaderboardEditionsCachesMiddlewareVariables = { referralLeaderboardEditionsCaches: ReferralLeaderboardEditionsCacheMap | Error; }; -/** - * Upgrades a single edition's cache from regular SWR to immutable storage. - * - * This function: - * 1. Creates a new cache with infinite TTL and proactive initialization - * 2. Waits for the new cache to successfully load data - * 3. Verifies the loaded data is immutably closed (fresh enough) - * 4. Only then destroys the old cache and swaps in the new one - * - * If initialization fails or data is not fresh enough, keeps the old cache - * and the upgrade will be retried on a future request. - * - * @param editionSlug - The edition slug being upgraded - * @param oldCache - The existing cache to be replaced - * @param editionConfig - The edition configuration - * @param caches - The map of all edition caches (for swapping) - */ -async function upgradeEditionCache( - editionSlug: string, - oldCache: SWRCache, - editionConfig: ReferralProgramEditionConfig, - caches: ReferralLeaderboardEditionsCacheMap, -): Promise { - logger.info({ editionSlug }, "Starting cache upgrade to immutable storage"); - - // Create new cache with proactive initialization (starts loading immediately) - const newCache = new SWRCache({ - fn: createEditionLeaderboardBuilder(editionConfig), - ttl: Number.POSITIVE_INFINITY, - proactiveRevalidationInterval: undefined, - errorTtl: minutesToSeconds(1), - proactivelyInitialize: true, - }); - - // Wait for the new cache to successfully initialize - const result = await newCache.read(); - - if (result instanceof Error) { - logger.warn( - { editionSlug, error: result }, - "Failed to initialize new cache, keeping old cache", - ); - newCache.destroy(); - return; - } - - // Verify the data is fresh enough (immutably closed based on its own accurateAsOf) - const isImmutable = assumeReferralProgramEditionImmutablyClosed( - result.rules, - result.accurateAsOf, - ); - - if (!isImmutable) { - logger.warn( - { editionSlug, accurateAsOf: result.accurateAsOf }, - "New cache data is not fresh enough to be considered immutably closed, keeping old cache", - ); - newCache.destroy(); - return; - } - - // Success! Swap the caches - logger.info({ editionSlug }, "New cache successfully initialized and verified, swapping caches"); - - oldCache.destroy(); - caches.set(editionSlug, newCache); - - logger.info({ editionSlug }, "Cache upgrade to immutable storage complete"); -} - -/** - * Checks all caches and upgrades any that have become immutable to store them indefinitely. - * - * This function is called non-blocking on each request to opportunistically upgrade caches - * when editions close. Each edition's upgrade runs independently in the background, ensuring: - * - No data loss: old cache continues serving while new cache initializes - * - Graceful failure: if new cache fails to initialize, old cache remains - * - No race conditions: atomic check-and-set prevents concurrent upgrades of same edition - * - Parallel upgrades: multiple editions can upgrade simultaneously - * - * Once a cache is upgraded to immutable storage (infinite TTL, no proactive revalidation), - * the quick check ensures minimal overhead on all future requests. - * - * @param caches - The map of edition caches to check and potentially upgrade - * @param editionConfigSet - The edition config set containing rules for each edition - */ -async function checkAndUpgradeImmutableCaches( - caches: ReferralLeaderboardEditionsCacheMap, - editionConfigSet: ReferralProgramEditionConfigSet, -): Promise { - const indexingStatus = await indexingStatusCache.read(); - if (indexingStatus instanceof Error) { - logger.debug( - { error: indexingStatus }, - "Failed to read indexing status during immutability check", - ); - return; - } - - for (const [editionSlug, cache] of caches) { - if (cache.isIndefinitelyStored()) { - continue; - } - - const editionConfig = editionConfigSet.get(editionSlug); - if (!editionConfig) { - logger.warn({ editionSlug }, "Edition config not found during immutability check"); - continue; - } - - const latestIndexedBlockRef = getLatestIndexedBlockRef( - indexingStatus, - editionConfig.rules.subregistryId.chainId, - ); - - if (latestIndexedBlockRef === null) { - logger.debug( - { editionSlug, chainId: editionConfig.rules.subregistryId.chainId }, - "No indexed block ref during immutability check", - ); - continue; - } - - const isImmutable = assumeReferralProgramEditionImmutablyClosed( - editionConfig.rules, - latestIndexedBlockRef.timestamp, - ); - - if (!isImmutable) { - continue; - } - - // Atomic check-and-set: prevent concurrent upgrades of the same edition - const upgradePromise = (() => { - // Check if upgrade already in progress - if (inProgressUpgrades.has(editionSlug)) { - return null; - } - - // Start upgrade and register promise immediately (no await in between) - const promise = upgradeEditionCache(editionSlug, cache, editionConfig, caches) - .catch((error) => { - logger.error({ editionSlug, error }, "Unexpected error during cache upgrade"); - }) - .finally(() => { - // Always clean up the in-progress tracking - inProgressUpgrades.delete(editionSlug); - }); - - inProgressUpgrades.set(editionSlug, promise); - return promise; - })(); - - if (!upgradePromise) { - // Another request is already upgrading this edition - logger.debug({ editionSlug }, "Upgrade already in progress, skipping"); - } - - // Don't await - let upgrade run in background - // Errors are logged inside upgradeEditionCache - } -} - /** * Middleware that provides {@link ReferralLeaderboardEditionsCachesMiddlewareVariables} * to downstream middleware and handlers. @@ -248,9 +69,16 @@ export const referralLeaderboardEditionsCachesMiddleware = factory.createMiddlew c.set("referralLeaderboardEditionsCaches", caches); // Non-blocking: Check and upgrade any caches that have become immutable - checkAndUpgradeImmutableCaches(caches, editionConfigSet).catch((error) => { - logger.error({ error }, "Failed to check and upgrade immutable caches"); - }); + indexingStatusCache + .read() + .then((indexingStatus) => { + if (!(indexingStatus instanceof Error)) { + return checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); + } + }) + .catch((error) => { + logger.error({ error }, "Failed to check and upgrade immutable caches"); + }); await next(); }, From d281d2c67b3b076183a707a24c3f70095c922a57 Mon Sep 17 00:00:00 2001 From: Goader Date: Tue, 10 Feb 2026 19:55:46 +0100 Subject: [PATCH 07/11] more test coverage for cache upgrading --- .../cache-upgrade.test.ts | 400 ++++++++++++------ .../referrer-leaderboard/cache-upgrade.ts | 6 +- .../src/shared/cache/swr-cache.test.ts | 35 ++ 3 files changed, 313 insertions(+), 128 deletions(-) diff --git a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts index 41afe998b..d38ca195c 100644 --- a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts +++ b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts @@ -1,8 +1,17 @@ -import type { ReferralProgramEditionConfig } from "@namehash/ens-referrals/v1"; +import type { + ReferralProgramEditionConfig, + ReferralProgramEditionConfigSet, +} from "@namehash/ens-referrals/v1"; import { minutesToSeconds } from "date-fns"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { addDuration, parseEth, parseUsdc, SWRCache } from "@ensnode/ensnode-sdk"; +import { + addDuration, + type CrossChainIndexingStatusSnapshot, + parseEth, + parseUsdc, + SWRCache, +} from "@ensnode/ensnode-sdk"; import { ASSUMED_CHAIN_REORG_SAFE_DURATION } from "@/lib/ensanalytics/referrer-leaderboard/closeout"; @@ -13,67 +22,40 @@ vi.mock("@/cache/referral-leaderboard-editions.cache", () => ({ import { createEditionLeaderboardBuilder } from "@/cache/referral-leaderboard-editions.cache"; -import { upgradeEditionCache } from "./cache-upgrade"; +import { checkAndUpgradeImmutableCaches, upgradeEditionCache } from "./cache-upgrade"; describe("Cache upgrade to immutable storage", () => { const now = Math.floor(Date.now() / 1000); - // Mock leaderboard builder function - const mockBuilder = async () => ({ - rules: { - totalAwardPoolValue: parseUsdc("10000"), - maxQualifiedReferrers: 100, - startTime: now - 3600, - endTime: now - 1200, - subregistryId: { chainId: 1, address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const }, - rulesUrl: new URL("https://example.com/rules"), - }, - aggregatedMetrics: { - grandTotalReferrals: 0, - grandTotalIncrementalDuration: 0, - grandTotalRevenueContribution: parseEth("0"), - grandTotalQualifiedReferrersFinalScore: 0, - minFinalScoreToQualify: 0, - }, - referrers: new Map(), - accurateAsOf: now, + // Shared test fixtures + const baseSubregistryId = { + chainId: 1, + address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, + }; + + const baseAggregatedMetrics = { + grandTotalReferrals: 0, + grandTotalIncrementalDuration: 0, + grandTotalRevenueContribution: parseEth("0"), + grandTotalQualifiedReferrersFinalScore: 0, + minFinalScoreToQualify: 0, + }; + + const createBaseRules = (startTime: number, endTime: number) => ({ + totalAwardPoolValue: parseUsdc("10000"), + maxQualifiedReferrers: 100, + startTime, + endTime, + subregistryId: baseSubregistryId, + rulesUrl: new URL("https://example.com/rules"), }); - it("should detect regular cache as NOT indefinitely stored", () => { - const cache = new SWRCache({ - fn: mockBuilder, - ttl: minutesToSeconds(5), - proactiveRevalidationInterval: minutesToSeconds(1), - errorTtl: minutesToSeconds(1), - proactivelyInitialize: false, - }); - - expect(cache.isIndefinitelyStored()).toBe(false); - }); - - it("should detect immutable cache as indefinitely stored", () => { - const cache = new SWRCache({ - fn: mockBuilder, - ttl: Number.POSITIVE_INFINITY, - proactiveRevalidationInterval: undefined, - errorTtl: minutesToSeconds(1), - proactivelyInitialize: false, - }); - - expect(cache.isIndefinitelyStored()).toBe(true); - }); - - it("should NOT detect cache with infinite TTL but proactive revalidation as indefinitely stored", () => { - const cache = new SWRCache({ - fn: mockBuilder, - ttl: Number.POSITIVE_INFINITY, - proactiveRevalidationInterval: minutesToSeconds(1), - errorTtl: minutesToSeconds(1), - proactivelyInitialize: false, - }); - - expect(cache.isIndefinitelyStored()).toBe(false); - }); + const createMockIndexingStatus = (timestamp: number): CrossChainIndexingStatusSnapshot => + ({ + omnichainSnapshot: { + chains: new Map([[1, { latestIndexedBlock: { timestamp } }]]), + }, + }) as unknown as CrossChainIndexingStatusSnapshot; describe("upgradeEditionCache", () => { beforeEach(() => { @@ -86,24 +68,8 @@ describe("Cache upgrade to immutable storage", () => { const immutableTimestamp = addDuration(closedEndTime, ASSUMED_CHAIN_REORG_SAFE_DURATION) + 1; const builderWithImmutableData = async () => ({ - rules: { - totalAwardPoolValue: parseUsdc("10000"), - maxQualifiedReferrers: 100, - startTime: closedEndTime - 3600, - endTime: closedEndTime, - subregistryId: { - chainId: 1, - address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, - }, - rulesUrl: new URL("https://example.com/rules"), - }, - aggregatedMetrics: { - grandTotalReferrals: 0, - grandTotalIncrementalDuration: 0, - grandTotalRevenueContribution: parseEth("0"), - grandTotalQualifiedReferrersFinalScore: 0, - minFinalScoreToQualify: 0, - }, + rules: createBaseRules(closedEndTime - 3600, closedEndTime), + aggregatedMetrics: baseAggregatedMetrics, referrers: new Map(), accurateAsOf: immutableTimestamp, }); @@ -125,17 +91,7 @@ describe("Cache upgrade to immutable storage", () => { const editionConfig: ReferralProgramEditionConfig = { slug: editionSlug, displayName: "Test Edition", - rules: { - totalAwardPoolValue: parseUsdc("10000"), - maxQualifiedReferrers: 100, - startTime: closedEndTime - 3600, - endTime: closedEndTime, - subregistryId: { - chainId: 1, - address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, - }, - rulesUrl: new URL("https://example.com/rules"), - }, + rules: createBaseRules(closedEndTime - 3600, closedEndTime), }; await upgradeEditionCache(editionSlug, oldCache, editionConfig, caches); @@ -170,17 +126,7 @@ describe("Cache upgrade to immutable storage", () => { const editionConfig: ReferralProgramEditionConfig = { slug: editionSlug, displayName: "Test Edition", - rules: { - totalAwardPoolValue: parseUsdc("10000"), - maxQualifiedReferrers: 100, - startTime: closedEndTime - 3600, - endTime: closedEndTime, - subregistryId: { - chainId: 1, - address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, - }, - rulesUrl: new URL("https://example.com/rules"), - }, + rules: createBaseRules(closedEndTime - 3600, closedEndTime), }; await upgradeEditionCache(editionSlug, oldCache, editionConfig, caches); @@ -195,24 +141,8 @@ describe("Cache upgrade to immutable storage", () => { const notFreshEnoughTimestamp = closedEndTime + 30; // Not past safety window const builderWithStaleData = async () => ({ - rules: { - totalAwardPoolValue: parseUsdc("10000"), - maxQualifiedReferrers: 100, - startTime: closedEndTime - 3600, - endTime: closedEndTime, - subregistryId: { - chainId: 1, - address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, - }, - rulesUrl: new URL("https://example.com/rules"), - }, - aggregatedMetrics: { - grandTotalReferrals: 0, - grandTotalIncrementalDuration: 0, - grandTotalRevenueContribution: parseEth("0"), - grandTotalQualifiedReferrersFinalScore: 0, - minFinalScoreToQualify: 0, - }, + rules: createBaseRules(closedEndTime - 3600, closedEndTime), + aggregatedMetrics: baseAggregatedMetrics, referrers: new Map(), accurateAsOf: notFreshEnoughTimestamp, }); @@ -234,17 +164,7 @@ describe("Cache upgrade to immutable storage", () => { const editionConfig: ReferralProgramEditionConfig = { slug: editionSlug, displayName: "Test Edition", - rules: { - totalAwardPoolValue: parseUsdc("10000"), - maxQualifiedReferrers: 100, - startTime: closedEndTime - 3600, - endTime: closedEndTime, - subregistryId: { - chainId: 1, - address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, - }, - rulesUrl: new URL("https://example.com/rules"), - }, + rules: createBaseRules(closedEndTime - 3600, closedEndTime), }; await upgradeEditionCache(editionSlug, oldCache, editionConfig, caches); @@ -253,4 +173,232 @@ describe("Cache upgrade to immutable storage", () => { expect(caches.get(editionSlug)).toBe(oldCache); }); }); + + describe("checkAndUpgradeImmutableCaches", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should prevent concurrent upgrades of the same edition", async () => { + const editionSlug = "test-edition"; + const closedEndTime = now - 1200; // 20 minutes ago + const immutableTimestamp = addDuration(closedEndTime, ASSUMED_CHAIN_REORG_SAFE_DURATION) + 1; + + // Create a controllable promise to keep the first upgrade in progress + let resolveFirstUpgrade: () => void; + const firstUpgradePromise = new Promise((resolve) => { + resolveFirstUpgrade = resolve; + }); + + let builderCallCount = 0; + const controllableBuilder = async () => { + builderCallCount++; + await firstUpgradePromise; // Block until we resolve it + return { + rules: createBaseRules(closedEndTime - 3600, closedEndTime), + aggregatedMetrics: baseAggregatedMetrics, + referrers: new Map(), + accurateAsOf: immutableTimestamp, + }; + }; + + // Mock the builder to return our controllable builder + vi.mocked(createEditionLeaderboardBuilder).mockReturnValue(controllableBuilder); + + // Create a regular cache that needs upgrading + const oldCache = new SWRCache({ + fn: async () => ({ + rules: createBaseRules(closedEndTime - 3600, closedEndTime), + aggregatedMetrics: baseAggregatedMetrics, + referrers: new Map(), + accurateAsOf: now, + }), + ttl: minutesToSeconds(5), + proactiveRevalidationInterval: minutesToSeconds(1), + errorTtl: minutesToSeconds(1), + proactivelyInitialize: false, + }); + + const caches = new Map(); + caches.set(editionSlug, oldCache); + + const editionConfigSet = new Map() as ReferralProgramEditionConfigSet; + editionConfigSet.set(editionSlug, { + slug: editionSlug, + displayName: "Test Edition", + rules: createBaseRules(closedEndTime - 3600, closedEndTime), + }); + + const indexingStatus = createMockIndexingStatus(immutableTimestamp); + + // Call checkAndUpgradeImmutableCaches twice concurrently + const firstCall = checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); + const secondCall = checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); + + // Both calls should return immediately (non-blocking) + await Promise.all([firstCall, secondCall]); + + // At this point, the first upgrade should be in progress, second should have been skipped + // The builder should have been called only once (for the first upgrade attempt) + expect(builderCallCount).toBe(1); + + // The old cache should still be in place (upgrade not complete yet) + expect(caches.get(editionSlug)).toBe(oldCache); + + // Now resolve the first upgrade + resolveFirstUpgrade!(); + + // Wait for the upgrade to complete + await vi.waitFor( + () => { + const currentCache = caches.get(editionSlug); + expect(currentCache).not.toBe(oldCache); + expect(currentCache?.isIndefinitelyStored()).toBe(true); + }, + { timeout: 1000 }, + ); + + // Verify the builder was still only called once (no duplicate upgrade) + expect(builderCallCount).toBe(1); + }); + + it("should skip caches that are already indefinitely stored", async () => { + const editionSlug = "test-edition"; + const closedEndTime = now - 1200; + const immutableTimestamp = addDuration(closedEndTime, ASSUMED_CHAIN_REORG_SAFE_DURATION) + 1; + + // Create an already-upgraded cache (infinite TTL, no revalidation) + const alreadyUpgradedCache = new SWRCache({ + fn: async () => ({ + rules: createBaseRules(closedEndTime - 3600, closedEndTime), + aggregatedMetrics: baseAggregatedMetrics, + referrers: new Map(), + accurateAsOf: immutableTimestamp, + }), + ttl: Number.POSITIVE_INFINITY, + proactiveRevalidationInterval: undefined, + errorTtl: minutesToSeconds(1), + proactivelyInitialize: false, + }); + + const caches = new Map(); + caches.set(editionSlug, alreadyUpgradedCache); + + const editionConfigSet = new Map() as ReferralProgramEditionConfigSet; + editionConfigSet.set(editionSlug, { + slug: editionSlug, + displayName: "Test Edition", + rules: createBaseRules(closedEndTime - 3600, closedEndTime), + }); + + const indexingStatus = createMockIndexingStatus(immutableTimestamp); + + // Mock should never be called since cache is already upgraded + vi.mocked(createEditionLeaderboardBuilder).mockImplementation(() => { + throw new Error("Builder should not be called for already upgraded cache"); + }); + + await checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); + + // Cache should remain the same + expect(caches.get(editionSlug)).toBe(alreadyUpgradedCache); + expect(alreadyUpgradedCache.isIndefinitelyStored()).toBe(true); + expect(createEditionLeaderboardBuilder).not.toHaveBeenCalled(); + }); + + it("should skip caches that are not yet immutably closed", async () => { + const editionSlug = "test-edition"; + const recentEndTime = now - 60; // Only 1 minute ago, not past safety window + const notImmutableTimestamp = now; + + // Create a regular cache for an edition that hasn't closed long enough + const notReadyCache = new SWRCache({ + fn: async () => ({ + rules: createBaseRules(recentEndTime - 3600, recentEndTime), + aggregatedMetrics: baseAggregatedMetrics, + referrers: new Map(), + accurateAsOf: now, + }), + ttl: minutesToSeconds(5), + proactiveRevalidationInterval: minutesToSeconds(1), + errorTtl: minutesToSeconds(1), + proactivelyInitialize: false, + }); + + const caches = new Map(); + caches.set(editionSlug, notReadyCache); + + const editionConfigSet = new Map() as ReferralProgramEditionConfigSet; + editionConfigSet.set(editionSlug, { + slug: editionSlug, + displayName: "Test Edition", + rules: createBaseRules(recentEndTime - 3600, recentEndTime), + }); + + const indexingStatus = createMockIndexingStatus(notImmutableTimestamp); + + // Mock should never be called since edition is not immutably closed yet + vi.mocked(createEditionLeaderboardBuilder).mockImplementation(() => { + throw new Error("Builder should not be called for not-yet-closed edition"); + }); + + await checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); + + // Cache should remain the same (not upgraded) + expect(caches.get(editionSlug)).toBe(notReadyCache); + expect(notReadyCache.isIndefinitelyStored()).toBe(false); + expect(createEditionLeaderboardBuilder).not.toHaveBeenCalled(); + }); + + it("should keep old cache if upgrade initialization fails", async () => { + const editionSlug = "test-edition"; + const closedEndTime = now - 1200; + const immutableTimestamp = addDuration(closedEndTime, ASSUMED_CHAIN_REORG_SAFE_DURATION) + 1; + + // Create a regular cache that would normally be upgraded + const oldCache = new SWRCache({ + fn: async () => ({ + rules: createBaseRules(closedEndTime - 3600, closedEndTime), + aggregatedMetrics: baseAggregatedMetrics, + referrers: new Map(), + accurateAsOf: now, + }), + ttl: minutesToSeconds(5), + proactiveRevalidationInterval: minutesToSeconds(1), + errorTtl: minutesToSeconds(1), + proactivelyInitialize: false, + }); + + const caches = new Map(); + caches.set(editionSlug, oldCache); + + const editionConfigSet = new Map() as ReferralProgramEditionConfigSet; + editionConfigSet.set(editionSlug, { + slug: editionSlug, + displayName: "Test Edition", + rules: createBaseRules(closedEndTime - 3600, closedEndTime), + }); + + const indexingStatus = createMockIndexingStatus(immutableTimestamp); + + // Mock builder to fail + const failingBuilder = async () => { + throw new Error("Simulated initialization failure"); + }; + vi.mocked(createEditionLeaderboardBuilder).mockReturnValue(failingBuilder); + + await checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); + + // Wait for the upgrade attempt to complete and verify it failed gracefully + await vi.waitFor( + () => { + expect(createEditionLeaderboardBuilder).toHaveBeenCalledOnce(); + // Old cache should still be in place (upgrade failed) + expect(caches.get(editionSlug)).toBe(oldCache); + expect(oldCache.isIndefinitelyStored()).toBe(false); + }, + { timeout: 1000 }, + ); + }); + }); }); diff --git a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts index fd6f5b7c3..0178fb750 100644 --- a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts +++ b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts @@ -89,8 +89,8 @@ export async function upgradeEditionCache( // Success! Swap the caches logger.info({ editionSlug }, "New cache successfully initialized and verified, swapping caches"); - oldCache.destroy(); caches.set(editionSlug, newCache); + oldCache.destroy(); logger.info({ editionSlug }, "Cache upgrade to immutable storage complete"); } @@ -112,7 +112,7 @@ export async function upgradeEditionCache( * @param editionConfigSet - The edition config set containing rules for each edition * @param indexingStatus - The current indexing status snapshot */ -export async function checkAndUpgradeImmutableCaches( +export function checkAndUpgradeImmutableCaches( caches: ReferralLeaderboardEditionsCacheMap, editionConfigSet: ReferralProgramEditionConfigSet, indexingStatus: CrossChainIndexingStatusSnapshot, @@ -182,4 +182,6 @@ export async function checkAndUpgradeImmutableCaches( // Don't await - let upgrade run in background // Errors are logged inside upgradeEditionCache } + + return Promise.resolve(); } diff --git a/packages/ensnode-sdk/src/shared/cache/swr-cache.test.ts b/packages/ensnode-sdk/src/shared/cache/swr-cache.test.ts index 0e28a1b9e..08222f980 100644 --- a/packages/ensnode-sdk/src/shared/cache/swr-cache.test.ts +++ b/packages/ensnode-sdk/src/shared/cache/swr-cache.test.ts @@ -740,4 +740,39 @@ describe("SWRCache", () => { expect(result2).toBe("success"); }); }); + + describe("isIndefinitelyStored", () => { + it("returns true when ttl is infinity and no proactive revalidation interval", () => { + const fn = vi.fn(async () => "value"); + const cache = new SWRCache({ + fn, + ttl: Number.POSITIVE_INFINITY, + proactiveRevalidationInterval: undefined, + }); + + expect(cache.isIndefinitelyStored()).toBe(true); + }); + + it("returns false when ttl is infinity but has proactive revalidation interval", () => { + const fn = vi.fn(async () => "value"); + const cache = new SWRCache({ + fn, + ttl: Number.POSITIVE_INFINITY, + proactiveRevalidationInterval: 60, // 60 seconds + }); + + expect(cache.isIndefinitelyStored()).toBe(false); + }); + + it("returns false when ttl is finite", () => { + const fn = vi.fn(async () => "value"); + const cache = new SWRCache({ + fn, + ttl: 300, // 5 minutes + proactiveRevalidationInterval: undefined, + }); + + expect(cache.isIndefinitelyStored()).toBe(false); + }); + }); }); From d2beb23e9f59539c2d16c8afd1e486bbce8a12cc Mon Sep 17 00:00:00 2001 From: Goader Date: Tue, 10 Feb 2026 20:25:30 +0100 Subject: [PATCH 08/11] review applied, improved tests --- .../cache-upgrade.test.ts | 54 ++++++++++++++----- .../referrer-leaderboard/cache-upgrade.ts | 21 ++++++-- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts index d38ca195c..1db9bcb94 100644 --- a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts +++ b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts @@ -3,7 +3,7 @@ import type { ReferralProgramEditionConfigSet, } from "@namehash/ens-referrals/v1"; import { minutesToSeconds } from "date-fns"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { addDuration, @@ -22,7 +22,12 @@ vi.mock("@/cache/referral-leaderboard-editions.cache", () => ({ import { createEditionLeaderboardBuilder } from "@/cache/referral-leaderboard-editions.cache"; -import { checkAndUpgradeImmutableCaches, upgradeEditionCache } from "./cache-upgrade"; +import { + checkAndUpgradeImmutableCaches, + getUpgradePromise, + resetInProgressUpgrades, + upgradeEditionCache, +} from "./cache-upgrade"; describe("Cache upgrade to immutable storage", () => { const now = Math.floor(Date.now() / 1000); @@ -85,6 +90,9 @@ describe("Cache upgrade to immutable storage", () => { proactivelyInitialize: false, }); + // Spy on the destroy method to verify it's called during swap + const destroySpy = vi.spyOn(oldCache, "destroy"); + const caches = new Map(); caches.set(editionSlug, oldCache); @@ -99,6 +107,7 @@ describe("Cache upgrade to immutable storage", () => { const upgradedCache = caches.get(editionSlug); expect(upgradedCache).not.toBe(oldCache); expect(upgradedCache?.isIndefinitelyStored()).toBe(true); + expect(destroySpy).toHaveBeenCalledOnce(); }); it("should keep old cache if new cache fails to initialize", async () => { @@ -179,6 +188,10 @@ describe("Cache upgrade to immutable storage", () => { vi.clearAllMocks(); }); + afterEach(() => { + resetInProgressUpgrades(); + }); + it("should prevent concurrent upgrades of the same edition", async () => { const editionSlug = "test-edition"; const closedEndTime = now - 1200; // 20 minutes ago @@ -308,7 +321,7 @@ describe("Cache upgrade to immutable storage", () => { it("should skip caches that are not yet immutably closed", async () => { const editionSlug = "test-edition"; - const recentEndTime = now - 60; // Only 1 minute ago, not past safety window + const recentEndTime = now - Math.floor(ASSUMED_CHAIN_REORG_SAFE_DURATION / 2); // Still within safety window const notImmutableTimestamp = now; // Create a regular cache for an edition that hasn't closed long enough @@ -389,16 +402,31 @@ describe("Cache upgrade to immutable storage", () => { await checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); - // Wait for the upgrade attempt to complete and verify it failed gracefully - await vi.waitFor( - () => { - expect(createEditionLeaderboardBuilder).toHaveBeenCalledOnce(); - // Old cache should still be in place (upgrade failed) - expect(caches.get(editionSlug)).toBe(oldCache); - expect(oldCache.isIndefinitelyStored()).toBe(false); - }, - { timeout: 1000 }, - ); + // Get the upgrade promise so we can wait for it to complete + const firstUpgradePromise = getUpgradePromise(editionSlug); + expect(firstUpgradePromise).toBeDefined(); + + // Wait for the first upgrade to complete (including .finally() cleanup) + await firstUpgradePromise; + + // Verify the upgrade failed gracefully + expect(createEditionLeaderboardBuilder).toHaveBeenCalledOnce(); + expect(caches.get(editionSlug)).toBe(oldCache); + expect(oldCache.isIndefinitelyStored()).toBe(false); + + // Call checkAndUpgradeImmutableCaches again to verify the inProgressUpgrades entry was cleaned up + await checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); + + // Get the second upgrade promise and wait for it to complete + const secondUpgradePromise = getUpgradePromise(editionSlug); + expect(secondUpgradePromise).toBeDefined(); + await secondUpgradePromise; + + // Builder should have been called a second time (proving cleanup happened) + expect(createEditionLeaderboardBuilder).toHaveBeenCalledTimes(2); + // Old cache should still be in place (second upgrade also failed) + expect(caches.get(editionSlug)).toBe(oldCache); + expect(oldCache.isIndefinitelyStored()).toBe(false); }); }); }); diff --git a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts index 0178fb750..1f03235ba 100644 --- a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts +++ b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts @@ -25,6 +25,23 @@ const logger = makeLogger("referral-leaderboard-cache-upgrade"); */ const inProgressUpgrades = new Map>(); +/** + * Test-only function to reset the in-progress upgrades map. + * Call this in test afterEach hooks to ensure test isolation. + */ +export function resetInProgressUpgrades(): void { + inProgressUpgrades.clear(); +} + +/** + * Test-only function to get the in-progress upgrade promise for an edition. + * @param editionSlug - The edition slug to check + * @returns The upgrade promise if one is in progress, undefined otherwise + */ +export function getUpgradePromise(editionSlug: string): Promise | undefined { + return inProgressUpgrades.get(editionSlug); +} + /** * Upgrades a single edition's cache from regular SWR to immutable storage. * @@ -112,7 +129,7 @@ export async function upgradeEditionCache( * @param editionConfigSet - The edition config set containing rules for each edition * @param indexingStatus - The current indexing status snapshot */ -export function checkAndUpgradeImmutableCaches( +export async function checkAndUpgradeImmutableCaches( caches: ReferralLeaderboardEditionsCacheMap, editionConfigSet: ReferralProgramEditionConfigSet, indexingStatus: CrossChainIndexingStatusSnapshot, @@ -182,6 +199,4 @@ export function checkAndUpgradeImmutableCaches( // Don't await - let upgrade run in background // Errors are logged inside upgradeEditionCache } - - return Promise.resolve(); } From 5e85c3da7d8836abc02bb73998135bebbe0c888c Mon Sep 17 00:00:00 2001 From: Goader Date: Wed, 11 Feb 2026 16:47:42 +0100 Subject: [PATCH 09/11] changed swrcache upgrading to handling previously cached values --- .../referral-leaderboard-editions.cache.ts | 33 +- .../cache-upgrade.test.ts | 432 ------------------ .../referrer-leaderboard/cache-upgrade.ts | 202 -------- ...-leaderboard-editions-caches.middleware.ts | 22 +- .../src/shared/cache/swr-cache.test.ts | 172 ++++++- .../ensnode-sdk/src/shared/cache/swr-cache.ts | 22 +- 6 files changed, 183 insertions(+), 700 deletions(-) delete mode 100644 apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts delete mode 100644 apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts diff --git a/apps/ensapi/src/cache/referral-leaderboard-editions.cache.ts b/apps/ensapi/src/cache/referral-leaderboard-editions.cache.ts index 1bec8638b..2a839bfb4 100644 --- a/apps/ensapi/src/cache/referral-leaderboard-editions.cache.ts +++ b/apps/ensapi/src/cache/referral-leaderboard-editions.cache.ts @@ -8,12 +8,14 @@ import { import { minutesToSeconds } from "date-fns"; import { + type CachedResult, getLatestIndexedBlockRef, type OmnichainIndexingStatusId, OmnichainIndexingStatusIds, SWRCache, } from "@ensnode/ensnode-sdk"; +import { assumeReferralProgramEditionImmutablyClosed } from "@/lib/ensanalytics/referrer-leaderboard/closeout"; import { getReferrerLeaderboard } from "@/lib/ensanalytics/referrer-leaderboard/get-referrer-leaderboard-v1"; import { makeLogger } from "@/lib/logger"; @@ -48,15 +50,34 @@ const supportedOmnichainIndexingStatuses: OmnichainIndexingStatusId[] = [ /** * Creates a cache builder function for a specific edition. * + * The builder function checks if cached data exists and represents an immutably closed edition. + * If so, it returns the cached data without re-fetching. Otherwise, it fetches fresh data. + * * @param editionConfig - The edition configuration * @returns A function that builds the leaderboard for the given edition */ -export function createEditionLeaderboardBuilder( +function createEditionLeaderboardBuilder( editionConfig: ReferralProgramEditionConfig, -): () => Promise { - return async (): Promise => { +): (cachedResult?: CachedResult) => Promise { + return async (cachedResult?: CachedResult): Promise => { const editionSlug = editionConfig.slug; + // Check if cached data is immutable and can be returned as-is + if (cachedResult && !(cachedResult.result instanceof Error)) { + const isImmutable = assumeReferralProgramEditionImmutablyClosed( + cachedResult.result.rules, + cachedResult.result.accurateAsOf, + ); + + if (isImmutable) { + logger.debug( + { editionSlug }, + `Edition is immutably closed, returning cached data without re-fetching`, + ); + return cachedResult.result; + } + } + const indexingStatus = await indexingStatusCache.read(); if (indexingStatus instanceof Error) { logger.error( @@ -120,10 +141,6 @@ let cachedInstance: ReferralLeaderboardEditionsCacheMap | null = null; * ensuring that if one edition fails to refresh, other editions' previously successful * data remains available. * - * All caches are initially created with normal refresh behavior. Caches are dynamically upgraded - * to immutable storage (infinite TTL, no proactive revalidation) by the middleware after an edition - * has been closed for more than the safety window based on accurate indexing timestamps. - * * @param editionConfigSet - The referral program edition config set to initialize caches for * @returns A map from edition slug to its dedicated SWRCache */ @@ -138,8 +155,6 @@ export function initializeReferralLeaderboardEditionsCaches( const caches: ReferralLeaderboardEditionsCacheMap = new Map(); for (const [editionSlug, editionConfig] of editionConfigSet) { - // All editions start with normal refresh behavior - // They will be dynamically upgraded to immutable storage by the middleware const cache = new SWRCache({ fn: createEditionLeaderboardBuilder(editionConfig), ttl: minutesToSeconds(1), diff --git a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts deleted file mode 100644 index 1db9bcb94..000000000 --- a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.test.ts +++ /dev/null @@ -1,432 +0,0 @@ -import type { - ReferralProgramEditionConfig, - ReferralProgramEditionConfigSet, -} from "@namehash/ens-referrals/v1"; -import { minutesToSeconds } from "date-fns"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -import { - addDuration, - type CrossChainIndexingStatusSnapshot, - parseEth, - parseUsdc, - SWRCache, -} from "@ensnode/ensnode-sdk"; - -import { ASSUMED_CHAIN_REORG_SAFE_DURATION } from "@/lib/ensanalytics/referrer-leaderboard/closeout"; - -// Mock the cache module to avoid config loading -vi.mock("@/cache/referral-leaderboard-editions.cache", () => ({ - createEditionLeaderboardBuilder: vi.fn(), -})); - -import { createEditionLeaderboardBuilder } from "@/cache/referral-leaderboard-editions.cache"; - -import { - checkAndUpgradeImmutableCaches, - getUpgradePromise, - resetInProgressUpgrades, - upgradeEditionCache, -} from "./cache-upgrade"; - -describe("Cache upgrade to immutable storage", () => { - const now = Math.floor(Date.now() / 1000); - - // Shared test fixtures - const baseSubregistryId = { - chainId: 1, - address: "0x57f1887a8BF19b14fC0dF6Fd9B2acc9Af147eA85" as const, - }; - - const baseAggregatedMetrics = { - grandTotalReferrals: 0, - grandTotalIncrementalDuration: 0, - grandTotalRevenueContribution: parseEth("0"), - grandTotalQualifiedReferrersFinalScore: 0, - minFinalScoreToQualify: 0, - }; - - const createBaseRules = (startTime: number, endTime: number) => ({ - totalAwardPoolValue: parseUsdc("10000"), - maxQualifiedReferrers: 100, - startTime, - endTime, - subregistryId: baseSubregistryId, - rulesUrl: new URL("https://example.com/rules"), - }); - - const createMockIndexingStatus = (timestamp: number): CrossChainIndexingStatusSnapshot => - ({ - omnichainSnapshot: { - chains: new Map([[1, { latestIndexedBlock: { timestamp } }]]), - }, - }) as unknown as CrossChainIndexingStatusSnapshot; - - describe("upgradeEditionCache", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("should successfully upgrade cache when edition is immutably closed", async () => { - const editionSlug = "test-edition"; - const closedEndTime = now - 1200; // 20 minutes ago - const immutableTimestamp = addDuration(closedEndTime, ASSUMED_CHAIN_REORG_SAFE_DURATION) + 1; - - const builderWithImmutableData = async () => ({ - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - aggregatedMetrics: baseAggregatedMetrics, - referrers: new Map(), - accurateAsOf: immutableTimestamp, - }); - - // Mock the builder to return our test data - vi.mocked(createEditionLeaderboardBuilder).mockReturnValue(builderWithImmutableData); - - const oldCache = new SWRCache({ - fn: builderWithImmutableData, - ttl: minutesToSeconds(5), - proactiveRevalidationInterval: minutesToSeconds(1), - errorTtl: minutesToSeconds(1), - proactivelyInitialize: false, - }); - - // Spy on the destroy method to verify it's called during swap - const destroySpy = vi.spyOn(oldCache, "destroy"); - - const caches = new Map(); - caches.set(editionSlug, oldCache); - - const editionConfig: ReferralProgramEditionConfig = { - slug: editionSlug, - displayName: "Test Edition", - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - }; - - await upgradeEditionCache(editionSlug, oldCache, editionConfig, caches); - - const upgradedCache = caches.get(editionSlug); - expect(upgradedCache).not.toBe(oldCache); - expect(upgradedCache?.isIndefinitelyStored()).toBe(true); - expect(destroySpy).toHaveBeenCalledOnce(); - }); - - it("should keep old cache if new cache fails to initialize", async () => { - const editionSlug = "test-edition"; - const closedEndTime = now - 1200; - - const failingBuilder = async () => { - throw new Error("Failed to build leaderboard"); - }; - - // Mock the builder to return a failing builder - vi.mocked(createEditionLeaderboardBuilder).mockReturnValue(failingBuilder); - - const oldCache = new SWRCache({ - fn: failingBuilder, - ttl: minutesToSeconds(5), - proactiveRevalidationInterval: minutesToSeconds(1), - errorTtl: minutesToSeconds(1), - proactivelyInitialize: false, - }); - - const caches = new Map(); - caches.set(editionSlug, oldCache); - - const editionConfig: ReferralProgramEditionConfig = { - slug: editionSlug, - displayName: "Test Edition", - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - }; - - await upgradeEditionCache(editionSlug, oldCache, editionConfig, caches); - - // Old cache should still be there - expect(caches.get(editionSlug)).toBe(oldCache); - }); - - it("should keep old cache if new data is not fresh enough", async () => { - const editionSlug = "test-edition"; - const closedEndTime = now - 1200; - const notFreshEnoughTimestamp = closedEndTime + 30; // Not past safety window - - const builderWithStaleData = async () => ({ - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - aggregatedMetrics: baseAggregatedMetrics, - referrers: new Map(), - accurateAsOf: notFreshEnoughTimestamp, - }); - - // Mock the builder to return stale data - vi.mocked(createEditionLeaderboardBuilder).mockReturnValue(builderWithStaleData); - - const oldCache = new SWRCache({ - fn: builderWithStaleData, - ttl: minutesToSeconds(5), - proactiveRevalidationInterval: minutesToSeconds(1), - errorTtl: minutesToSeconds(1), - proactivelyInitialize: false, - }); - - const caches = new Map(); - caches.set(editionSlug, oldCache); - - const editionConfig: ReferralProgramEditionConfig = { - slug: editionSlug, - displayName: "Test Edition", - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - }; - - await upgradeEditionCache(editionSlug, oldCache, editionConfig, caches); - - // Old cache should still be there - expect(caches.get(editionSlug)).toBe(oldCache); - }); - }); - - describe("checkAndUpgradeImmutableCaches", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - afterEach(() => { - resetInProgressUpgrades(); - }); - - it("should prevent concurrent upgrades of the same edition", async () => { - const editionSlug = "test-edition"; - const closedEndTime = now - 1200; // 20 minutes ago - const immutableTimestamp = addDuration(closedEndTime, ASSUMED_CHAIN_REORG_SAFE_DURATION) + 1; - - // Create a controllable promise to keep the first upgrade in progress - let resolveFirstUpgrade: () => void; - const firstUpgradePromise = new Promise((resolve) => { - resolveFirstUpgrade = resolve; - }); - - let builderCallCount = 0; - const controllableBuilder = async () => { - builderCallCount++; - await firstUpgradePromise; // Block until we resolve it - return { - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - aggregatedMetrics: baseAggregatedMetrics, - referrers: new Map(), - accurateAsOf: immutableTimestamp, - }; - }; - - // Mock the builder to return our controllable builder - vi.mocked(createEditionLeaderboardBuilder).mockReturnValue(controllableBuilder); - - // Create a regular cache that needs upgrading - const oldCache = new SWRCache({ - fn: async () => ({ - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - aggregatedMetrics: baseAggregatedMetrics, - referrers: new Map(), - accurateAsOf: now, - }), - ttl: minutesToSeconds(5), - proactiveRevalidationInterval: minutesToSeconds(1), - errorTtl: minutesToSeconds(1), - proactivelyInitialize: false, - }); - - const caches = new Map(); - caches.set(editionSlug, oldCache); - - const editionConfigSet = new Map() as ReferralProgramEditionConfigSet; - editionConfigSet.set(editionSlug, { - slug: editionSlug, - displayName: "Test Edition", - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - }); - - const indexingStatus = createMockIndexingStatus(immutableTimestamp); - - // Call checkAndUpgradeImmutableCaches twice concurrently - const firstCall = checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); - const secondCall = checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); - - // Both calls should return immediately (non-blocking) - await Promise.all([firstCall, secondCall]); - - // At this point, the first upgrade should be in progress, second should have been skipped - // The builder should have been called only once (for the first upgrade attempt) - expect(builderCallCount).toBe(1); - - // The old cache should still be in place (upgrade not complete yet) - expect(caches.get(editionSlug)).toBe(oldCache); - - // Now resolve the first upgrade - resolveFirstUpgrade!(); - - // Wait for the upgrade to complete - await vi.waitFor( - () => { - const currentCache = caches.get(editionSlug); - expect(currentCache).not.toBe(oldCache); - expect(currentCache?.isIndefinitelyStored()).toBe(true); - }, - { timeout: 1000 }, - ); - - // Verify the builder was still only called once (no duplicate upgrade) - expect(builderCallCount).toBe(1); - }); - - it("should skip caches that are already indefinitely stored", async () => { - const editionSlug = "test-edition"; - const closedEndTime = now - 1200; - const immutableTimestamp = addDuration(closedEndTime, ASSUMED_CHAIN_REORG_SAFE_DURATION) + 1; - - // Create an already-upgraded cache (infinite TTL, no revalidation) - const alreadyUpgradedCache = new SWRCache({ - fn: async () => ({ - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - aggregatedMetrics: baseAggregatedMetrics, - referrers: new Map(), - accurateAsOf: immutableTimestamp, - }), - ttl: Number.POSITIVE_INFINITY, - proactiveRevalidationInterval: undefined, - errorTtl: minutesToSeconds(1), - proactivelyInitialize: false, - }); - - const caches = new Map(); - caches.set(editionSlug, alreadyUpgradedCache); - - const editionConfigSet = new Map() as ReferralProgramEditionConfigSet; - editionConfigSet.set(editionSlug, { - slug: editionSlug, - displayName: "Test Edition", - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - }); - - const indexingStatus = createMockIndexingStatus(immutableTimestamp); - - // Mock should never be called since cache is already upgraded - vi.mocked(createEditionLeaderboardBuilder).mockImplementation(() => { - throw new Error("Builder should not be called for already upgraded cache"); - }); - - await checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); - - // Cache should remain the same - expect(caches.get(editionSlug)).toBe(alreadyUpgradedCache); - expect(alreadyUpgradedCache.isIndefinitelyStored()).toBe(true); - expect(createEditionLeaderboardBuilder).not.toHaveBeenCalled(); - }); - - it("should skip caches that are not yet immutably closed", async () => { - const editionSlug = "test-edition"; - const recentEndTime = now - Math.floor(ASSUMED_CHAIN_REORG_SAFE_DURATION / 2); // Still within safety window - const notImmutableTimestamp = now; - - // Create a regular cache for an edition that hasn't closed long enough - const notReadyCache = new SWRCache({ - fn: async () => ({ - rules: createBaseRules(recentEndTime - 3600, recentEndTime), - aggregatedMetrics: baseAggregatedMetrics, - referrers: new Map(), - accurateAsOf: now, - }), - ttl: minutesToSeconds(5), - proactiveRevalidationInterval: minutesToSeconds(1), - errorTtl: minutesToSeconds(1), - proactivelyInitialize: false, - }); - - const caches = new Map(); - caches.set(editionSlug, notReadyCache); - - const editionConfigSet = new Map() as ReferralProgramEditionConfigSet; - editionConfigSet.set(editionSlug, { - slug: editionSlug, - displayName: "Test Edition", - rules: createBaseRules(recentEndTime - 3600, recentEndTime), - }); - - const indexingStatus = createMockIndexingStatus(notImmutableTimestamp); - - // Mock should never be called since edition is not immutably closed yet - vi.mocked(createEditionLeaderboardBuilder).mockImplementation(() => { - throw new Error("Builder should not be called for not-yet-closed edition"); - }); - - await checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); - - // Cache should remain the same (not upgraded) - expect(caches.get(editionSlug)).toBe(notReadyCache); - expect(notReadyCache.isIndefinitelyStored()).toBe(false); - expect(createEditionLeaderboardBuilder).not.toHaveBeenCalled(); - }); - - it("should keep old cache if upgrade initialization fails", async () => { - const editionSlug = "test-edition"; - const closedEndTime = now - 1200; - const immutableTimestamp = addDuration(closedEndTime, ASSUMED_CHAIN_REORG_SAFE_DURATION) + 1; - - // Create a regular cache that would normally be upgraded - const oldCache = new SWRCache({ - fn: async () => ({ - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - aggregatedMetrics: baseAggregatedMetrics, - referrers: new Map(), - accurateAsOf: now, - }), - ttl: minutesToSeconds(5), - proactiveRevalidationInterval: minutesToSeconds(1), - errorTtl: minutesToSeconds(1), - proactivelyInitialize: false, - }); - - const caches = new Map(); - caches.set(editionSlug, oldCache); - - const editionConfigSet = new Map() as ReferralProgramEditionConfigSet; - editionConfigSet.set(editionSlug, { - slug: editionSlug, - displayName: "Test Edition", - rules: createBaseRules(closedEndTime - 3600, closedEndTime), - }); - - const indexingStatus = createMockIndexingStatus(immutableTimestamp); - - // Mock builder to fail - const failingBuilder = async () => { - throw new Error("Simulated initialization failure"); - }; - vi.mocked(createEditionLeaderboardBuilder).mockReturnValue(failingBuilder); - - await checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); - - // Get the upgrade promise so we can wait for it to complete - const firstUpgradePromise = getUpgradePromise(editionSlug); - expect(firstUpgradePromise).toBeDefined(); - - // Wait for the first upgrade to complete (including .finally() cleanup) - await firstUpgradePromise; - - // Verify the upgrade failed gracefully - expect(createEditionLeaderboardBuilder).toHaveBeenCalledOnce(); - expect(caches.get(editionSlug)).toBe(oldCache); - expect(oldCache.isIndefinitelyStored()).toBe(false); - - // Call checkAndUpgradeImmutableCaches again to verify the inProgressUpgrades entry was cleaned up - await checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); - - // Get the second upgrade promise and wait for it to complete - const secondUpgradePromise = getUpgradePromise(editionSlug); - expect(secondUpgradePromise).toBeDefined(); - await secondUpgradePromise; - - // Builder should have been called a second time (proving cleanup happened) - expect(createEditionLeaderboardBuilder).toHaveBeenCalledTimes(2); - // Old cache should still be in place (second upgrade also failed) - expect(caches.get(editionSlug)).toBe(oldCache); - expect(oldCache.isIndefinitelyStored()).toBe(false); - }); - }); -}); diff --git a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts b/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts deleted file mode 100644 index 1f03235ba..000000000 --- a/apps/ensapi/src/lib/ensanalytics/referrer-leaderboard/cache-upgrade.ts +++ /dev/null @@ -1,202 +0,0 @@ -import type { - ReferralProgramEditionConfig, - ReferralProgramEditionConfigSet, - ReferrerLeaderboard, -} from "@namehash/ens-referrals/v1"; -import { minutesToSeconds } from "date-fns"; - -import { - type CrossChainIndexingStatusSnapshot, - getLatestIndexedBlockRef, - SWRCache, -} from "@ensnode/ensnode-sdk"; - -import type { ReferralLeaderboardEditionsCacheMap } from "@/cache/referral-leaderboard-editions.cache"; -import { createEditionLeaderboardBuilder } from "@/cache/referral-leaderboard-editions.cache"; -import { makeLogger } from "@/lib/logger"; - -import { assumeReferralProgramEditionImmutablyClosed } from "./closeout"; - -const logger = makeLogger("referral-leaderboard-cache-upgrade"); - -/** - * Tracks in-progress cache upgrades to prevent concurrent upgrades of the same edition. - * Maps edition slug to the upgrade promise. - */ -const inProgressUpgrades = new Map>(); - -/** - * Test-only function to reset the in-progress upgrades map. - * Call this in test afterEach hooks to ensure test isolation. - */ -export function resetInProgressUpgrades(): void { - inProgressUpgrades.clear(); -} - -/** - * Test-only function to get the in-progress upgrade promise for an edition. - * @param editionSlug - The edition slug to check - * @returns The upgrade promise if one is in progress, undefined otherwise - */ -export function getUpgradePromise(editionSlug: string): Promise | undefined { - return inProgressUpgrades.get(editionSlug); -} - -/** - * Upgrades a single edition's cache from regular SWR to immutable storage. - * - * This function: - * 1. Creates a new cache with infinite TTL and proactive initialization - * 2. Waits for the new cache to successfully load data - * 3. Verifies the loaded data is immutably closed (fresh enough) - * 4. Only then destroys the old cache and swaps in the new one - * - * If initialization fails or data is not fresh enough, keeps the old cache - * and the upgrade will be retried on a future request. - * - * @param editionSlug - The edition slug being upgraded - * @param oldCache - The existing cache to be replaced - * @param editionConfig - The edition configuration - * @param caches - The map of all edition caches (for swapping) - */ -export async function upgradeEditionCache( - editionSlug: string, - oldCache: SWRCache, - editionConfig: ReferralProgramEditionConfig, - caches: ReferralLeaderboardEditionsCacheMap, -): Promise { - logger.info({ editionSlug }, "Starting cache upgrade to immutable storage"); - - // Create new cache with proactive initialization (starts loading immediately) - const newCache = new SWRCache({ - fn: createEditionLeaderboardBuilder(editionConfig), - ttl: Number.POSITIVE_INFINITY, - proactiveRevalidationInterval: undefined, - errorTtl: minutesToSeconds(1), - proactivelyInitialize: true, - }); - - // Wait for the new cache to successfully initialize - const result = await newCache.read(); - - if (result instanceof Error) { - logger.warn( - { editionSlug, error: result }, - "Failed to initialize new cache, keeping old cache", - ); - newCache.destroy(); - return; - } - - // Verify the data is fresh enough (immutably closed based on its own accurateAsOf) - const isImmutable = assumeReferralProgramEditionImmutablyClosed( - result.rules, - result.accurateAsOf, - ); - - if (!isImmutable) { - logger.warn( - { editionSlug, accurateAsOf: result.accurateAsOf }, - "New cache data is not fresh enough to be considered immutably closed, keeping old cache", - ); - newCache.destroy(); - return; - } - - // Success! Swap the caches - logger.info({ editionSlug }, "New cache successfully initialized and verified, swapping caches"); - - caches.set(editionSlug, newCache); - oldCache.destroy(); - - logger.info({ editionSlug }, "Cache upgrade to immutable storage complete"); -} - -/** - * Checks all caches and upgrades any that have become immutable to store them indefinitely. - * - * This function is called non-blocking on each request to opportunistically upgrade caches - * when editions close. Each edition's upgrade runs independently in the background, ensuring: - * - No data loss: old cache continues serving while new cache initializes - * - Graceful failure: if new cache fails to initialize, old cache remains - * - No race conditions: atomic check-and-set prevents concurrent upgrades of same edition - * - Parallel upgrades: multiple editions can upgrade simultaneously - * - * Once a cache is upgraded to immutable storage (infinite TTL, no proactive revalidation), - * the quick check ensures minimal overhead on all future requests. - * - * @param caches - The map of edition caches to check and potentially upgrade - * @param editionConfigSet - The edition config set containing rules for each edition - * @param indexingStatus - The current indexing status snapshot - */ -export async function checkAndUpgradeImmutableCaches( - caches: ReferralLeaderboardEditionsCacheMap, - editionConfigSet: ReferralProgramEditionConfigSet, - indexingStatus: CrossChainIndexingStatusSnapshot, -): Promise { - for (const [editionSlug, cache] of caches) { - // Quick exit: already upgraded to immutable storage - if (cache.isIndefinitelyStored()) { - continue; - } - - const editionConfig = editionConfigSet.get(editionSlug); - if (!editionConfig) { - logger.warn({ editionSlug }, "Edition config not found during immutability check"); - continue; - } - - // Get latest indexed block ref for this edition's chain - const latestIndexedBlockRef = getLatestIndexedBlockRef( - indexingStatus, - editionConfig.rules.subregistryId.chainId, - ); - - if (latestIndexedBlockRef === null) { - logger.debug( - { editionSlug, chainId: editionConfig.rules.subregistryId.chainId }, - "No indexed block ref during immutability check", - ); - continue; - } - - // Check if edition is immutably closed based on current indexing timestamp - const isImmutable = assumeReferralProgramEditionImmutablyClosed( - editionConfig.rules, - latestIndexedBlockRef.timestamp, - ); - - if (!isImmutable) { - continue; - } - - // Atomic check-and-set: prevent concurrent upgrades of the same edition - const upgradePromise = (() => { - // Check if upgrade already in progress - if (inProgressUpgrades.has(editionSlug)) { - return null; - } - - // Start upgrade and register promise immediately (no await in between) - const promise = upgradeEditionCache(editionSlug, cache, editionConfig, caches) - .catch((error) => { - logger.error({ editionSlug, error }, "Unexpected error during cache upgrade"); - }) - .finally(() => { - // Always clean up the in-progress tracking - inProgressUpgrades.delete(editionSlug); - }); - - inProgressUpgrades.set(editionSlug, promise); - return promise; - })(); - - if (!upgradePromise) { - // Another request is already upgrading this edition - logger.debug({ editionSlug }, "Upgrade already in progress, skipping"); - } - - // Don't await - let upgrade run in background - // Errors are logged inside upgradeEditionCache - } -} diff --git a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts index 038c63c56..4e062dae0 100644 --- a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts +++ b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts @@ -1,15 +1,10 @@ -import { indexingStatusCache } from "@/cache/indexing-status.cache"; import { initializeReferralLeaderboardEditionsCaches, type ReferralLeaderboardEditionsCacheMap, } from "@/cache/referral-leaderboard-editions.cache"; -import { checkAndUpgradeImmutableCaches } from "@/lib/ensanalytics/referrer-leaderboard/cache-upgrade"; import { factory } from "@/lib/hono-factory"; -import { makeLogger } from "@/lib/logger"; import type { referralProgramEditionConfigSetMiddleware } from "@/middleware/referral-program-edition-set.middleware"; -const logger = makeLogger("referral-leaderboard-editions-caches-middleware"); - /** * Type definition for the referral leaderboard editions caches middleware context passed to downstream middleware and handlers. */ @@ -42,9 +37,8 @@ export type ReferralLeaderboardEditionsCachesMiddlewareVariables = { * the edition config set. If the edition config set failed to load, this middleware propagates the error. * Otherwise, it initializes caches for each edition in the config set. * - * On each request, this middleware non-blocking checks if any caches should be upgraded to immutable - * storage based on accurate indexing timestamps. This allows caches to dynamically transition from - * refreshing to indefinite storage as editions close. + * Each cache's builder function handles immutability internally - when an edition becomes immutably + * closed (past the safety window), the builder returns cached data without re-fetching. */ export const referralLeaderboardEditionsCachesMiddleware = factory.createMiddleware( async (c, next) => { @@ -68,18 +62,6 @@ export const referralLeaderboardEditionsCachesMiddleware = factory.createMiddlew const caches = initializeReferralLeaderboardEditionsCaches(editionConfigSet); c.set("referralLeaderboardEditionsCaches", caches); - // Non-blocking: Check and upgrade any caches that have become immutable - indexingStatusCache - .read() - .then((indexingStatus) => { - if (!(indexingStatus instanceof Error)) { - return checkAndUpgradeImmutableCaches(caches, editionConfigSet, indexingStatus); - } - }) - .catch((error) => { - logger.error({ error }, "Failed to check and upgrade immutable caches"); - }); - await next(); }, ); diff --git a/packages/ensnode-sdk/src/shared/cache/swr-cache.test.ts b/packages/ensnode-sdk/src/shared/cache/swr-cache.test.ts index 08222f980..3545ee8ee 100644 --- a/packages/ensnode-sdk/src/shared/cache/swr-cache.test.ts +++ b/packages/ensnode-sdk/src/shared/cache/swr-cache.test.ts @@ -741,38 +741,166 @@ describe("SWRCache", () => { }); }); - describe("isIndefinitelyStored", () => { - it("returns true when ttl is infinity and no proactive revalidation interval", () => { - const fn = vi.fn(async () => "value"); - const cache = new SWRCache({ - fn, - ttl: Number.POSITIVE_INFINITY, - proactiveRevalidationInterval: undefined, + describe("immutability support via cached result parameter", () => { + it("should pass undefined on first call when no cache exists", async () => { + const fn = vi.fn(async (cachedResult) => { + return cachedResult ? "has-cache" : "no-cache"; }); - expect(cache.isIndefinitelyStored()).toBe(true); + const cache = new SWRCache({ fn, ttl: 60 }); + + const result = await cache.read(); + expect(result).toBe("no-cache"); + expect(fn).toHaveBeenCalledWith(undefined); + cache.destroy(); }); - it("returns false when ttl is infinity but has proactive revalidation interval", () => { - const fn = vi.fn(async () => "value"); - const cache = new SWRCache({ - fn, - ttl: Number.POSITIVE_INFINITY, - proactiveRevalidationInterval: 60, // 60 seconds + it("should pass cached result on subsequent revalidations", async () => { + let callCount = 0; + const fn = vi.fn(async (cachedResult) => { + callCount++; + if (cachedResult && !(cachedResult.result instanceof Error)) { + return `call-${callCount}-saw-${cachedResult.result}`; + } + return `call-${callCount}-fresh`; }); - expect(cache.isIndefinitelyStored()).toBe(false); + const cache = new SWRCache({ fn, ttl: 1, proactivelyInitialize: true }); + + // Wait for initial load + await vi.runAllTimersAsync(); + const result1 = await cache.read(); + expect(result1).toBe("call-1-fresh"); + + // Advance past TTL to trigger revalidation + vi.advanceTimersByTime(2000); + await cache.read(); + await vi.runAllTimersAsync(); + + const result2 = await cache.read(); + expect(result2).toBe("call-2-saw-call-1-fresh"); + expect(fn).toHaveBeenCalledTimes(2); + + cache.destroy(); }); - it("returns false when ttl is finite", () => { - const fn = vi.fn(async () => "value"); - const cache = new SWRCache({ - fn, - ttl: 300, // 5 minutes - proactiveRevalidationInterval: undefined, + it("should allow fn to return cached data when immutable", async () => { + let callCount = 0; + + // Simulates an edition leaderboard that becomes immutable + const fn = vi.fn(async (cachedResult) => { + callCount++; + + // If cached data is marked immutable, return it without re-fetching + if ( + cachedResult && + !(cachedResult.result instanceof Error) && + cachedResult.result.isImmutable + ) { + return cachedResult.result; + } + + // Otherwise fetch fresh data + // Becomes immutable on second call + return { + value: `data-${callCount}`, + isImmutable: callCount >= 2, + }; + }); + + const cache = new SWRCache({ fn, ttl: 1, proactivelyInitialize: true }); + + // Initial load + await vi.runAllTimersAsync(); + const result1 = await cache.read(); + expect(result1).toEqual({ value: "data-1", isImmutable: false }); + expect(callCount).toBe(1); + + // Trigger revalidation by advancing past TTL + vi.advanceTimersByTime(2000); + await cache.read(); + await vi.runAllTimersAsync(); + + const result2 = await cache.read(); + expect(result2).toEqual({ value: "data-2", isImmutable: true }); + expect(callCount).toBe(2); + + // Trigger another revalidation + vi.advanceTimersByTime(2000); + await cache.read(); + await vi.runAllTimersAsync(); + + const result3 = await cache.read(); + // Should still be data-2 because fn returned cached immutable data + expect(result3).toEqual({ value: "data-2", isImmutable: true }); + expect(callCount).toBe(3); // fn was called, but returned cached data + + cache.destroy(); + }); + + it("should support backward compatibility with fn that ignores cached result", async () => { + let callCount = 0; + + // Old-style function that doesn't use the cachedResult parameter + const fn = vi.fn(async () => { + callCount++; + return `result-${callCount}`; + }); + + const cache = new SWRCache({ fn, ttl: 1, proactivelyInitialize: true }); + + // Initial load + await vi.runAllTimersAsync(); + const result1 = await cache.read(); + expect(result1).toBe("result-1"); + + // Trigger revalidation + vi.advanceTimersByTime(2000); + await cache.read(); + await vi.runAllTimersAsync(); + + const result2 = await cache.read(); + expect(result2).toBe("result-2"); + expect(callCount).toBe(2); + + cache.destroy(); + }); + + it("should pass cached error result to fn and eventually return success", async () => { + let shouldError = true; + const fn = vi.fn(async (cachedResult) => { + if (cachedResult && cachedResult.result instanceof Error) { + return "recovered-from-error"; + } + if (shouldError) throw new Error("test error"); + return "success"; }); - expect(cache.isIndefinitelyStored()).toBe(false); + const cache = new SWRCache({ fn, ttl: 1, errorTtl: 1, proactivelyInitialize: true }); + + // Initial load (error) + await vi.runAllTimersAsync(); + const result1 = await cache.read(); + expect(result1).toBeInstanceOf(Error); + + // Trigger revalidation after errorTtl + shouldError = false; + vi.advanceTimersByTime(2000); + await cache.read(); + await vi.runAllTimersAsync(); + + const result2 = await cache.read(); + expect(result2).toBe("recovered-from-error"); + + // Trigger another revalidation - now cached result is not an error + vi.advanceTimersByTime(2000); + await cache.read(); + await vi.runAllTimersAsync(); + + const result3 = await cache.read(); + expect(result3).toBe("success"); + + cache.destroy(); }); }); }); diff --git a/packages/ensnode-sdk/src/shared/cache/swr-cache.ts b/packages/ensnode-sdk/src/shared/cache/swr-cache.ts index 72017500e..30f1d94d1 100644 --- a/packages/ensnode-sdk/src/shared/cache/swr-cache.ts +++ b/packages/ensnode-sdk/src/shared/cache/swr-cache.ts @@ -7,7 +7,7 @@ import type { Duration, UnixTimestamp } from "../types"; /** * Data structure for a single cached result. */ -interface CachedResult { +export interface CachedResult { /** * The cached result of the fn, either its ValueType or Error. */ @@ -23,8 +23,12 @@ export interface SWRCacheOptions { /** * The async function generating a value of `ValueType` to wrap with SWR caching. It may throw an * Error type. + * + * The function optionally receives the currently cached result (or undefined if no value is + * cached yet). This allows the function to implement custom caching strategies, such as + * returning the same data for immutable values without re-fetching. */ - fn: () => Promise; + fn: (cachedResult?: CachedResult) => Promise; /** * Time-to-live duration of a cached result in seconds. After this duration: @@ -114,7 +118,7 @@ export class SWRCache { // ensure that there is exactly one in progress revalidation promise if (!this.inProgressRevalidate) { this.inProgressRevalidate = this.options - .fn() + .fn(this.cache ?? undefined) .then((result) => { // on success, always update the cache with the latest revalidation this.cache = { @@ -169,18 +173,6 @@ export class SWRCache { return this.cache.result; } - /** - * Returns true if this cache stores data indefinitely without revalidation. - * - * A cache is considered indefinitely stored when it has infinite TTL and no - * proactive revalidation interval configured. - */ - public isIndefinitelyStored(): boolean { - return ( - this.options.ttl === Number.POSITIVE_INFINITY && !this.options.proactiveRevalidationInterval - ); - } - /** * Destroys the background revalidation interval, if exists. */ From 41b805683146b728f0925ba9538bd6b2a0f7ce48 Mon Sep 17 00:00:00 2001 From: Goader Date: Wed, 11 Feb 2026 17:00:39 +0100 Subject: [PATCH 10/11] removed extra empty line --- .../referral-leaderboard-editions-caches.middleware.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts index 4e062dae0..34f713543 100644 --- a/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts +++ b/apps/ensapi/src/middleware/referral-leaderboard-editions-caches.middleware.ts @@ -61,7 +61,6 @@ export const referralLeaderboardEditionsCachesMiddleware = factory.createMiddlew // Initialize caches for the edition config set const caches = initializeReferralLeaderboardEditionsCaches(editionConfigSet); c.set("referralLeaderboardEditionsCaches", caches); - await next(); }, ); From 013be95c69962dd53c6fb05e2f473207e906222c Mon Sep 17 00:00:00 2001 From: Goader Date: Wed, 11 Feb 2026 17:13:51 +0100 Subject: [PATCH 11/11] updated changesets --- .changeset/clever-frogs-detect.md | 4 ++-- .changeset/lemon-moose-count.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/clever-frogs-detect.md b/.changeset/clever-frogs-detect.md index ff5d08288..09532c05f 100644 --- a/.changeset/clever-frogs-detect.md +++ b/.changeset/clever-frogs-detect.md @@ -1,5 +1,5 @@ --- -"@ensnode/ensnode-sdk": patch +"@ensnode/ensnode-sdk": minor --- -Added `isIndefinitelyStored()` method to SWRCache for detecting caches configured with infinite TTL and no proactive revalidation. +SWRCache `fn` now optionally receives the currently cached result as a parameter, allowing implementations to inspect cached data before deciding whether to return it or fetch fresh data. Fully backward compatible. diff --git a/.changeset/lemon-moose-count.md b/.changeset/lemon-moose-count.md index f70a52b04..7b432a31a 100644 --- a/.changeset/lemon-moose-count.md +++ b/.changeset/lemon-moose-count.md @@ -2,4 +2,4 @@ "ensapi": minor --- -Implemented automatic cache upgrade to indefinite storage for immutably closed referral program editions. Caches safely transition from regular SWR to immutable storage with zero-data-loss initialization, atomic race prevention, and graceful failure handling. +Referral program edition leaderboard caches now check for immutability within the cache builder function. Closed editions past the safety window return cached data without re-fetching.