feat: add more scan chains plus CEX-only pull with reliable multi-chain browser priming

package.json

@@ -12,6 +12,9 @@
     "lint": "eslint . --fix --cache",
     "build": "bun run tsc",
     "pull": "bun run scripts/fetch-all.ts",
+    "pull:cex": "bun run scripts/fetch-cex.ts",
+    "pull:cex:all": "bun run scripts/fetch-cex.ts --no-10k-limit",
+    "smoke:labelcloud": "bun run scripts/smoke-labelcloud.ts",
     "prepare": "husky || true",
     "tsc": "tsc -p tsconfig.json",
     "postversion": "git push --follow-tags",

scripts/Chain/AvalancheChain.ts

@@ -0,0 +1,14 @@
+import type { ApiParser } from "../ApiParser/ApiParser";
+import { EtherscanApiParser } from "../ApiParser/EtherscanApiParser";
+import { OptimismHtmlParser } from "../HtmlParser/OptimismHtmlParser";
+import { Chain } from "./Chain";
+
+export class AvalancheChain extends Chain<ApiParser, OptimismHtmlParser> {
+  public constructor() {
+    const website = "https://snowscan.xyz";
+    const chainName = "avalanche";
+    const htmlPuller = new OptimismHtmlParser();
+    const apiPuller = new EtherscanApiParser(website);
+    super(website, chainName, apiPuller, htmlPuller);
+  }
+}

scripts/Chain/BerachainChain.ts

@@ -0,0 +1,14 @@
+import type { ApiParser } from "../ApiParser/ApiParser";
+import { EtherscanApiParser } from "../ApiParser/EtherscanApiParser";
+import { OptimismHtmlParser } from "../HtmlParser/OptimismHtmlParser";
+import { Chain } from "./Chain";
+
+export class BerachainChain extends Chain<ApiParser, OptimismHtmlParser> {
+  public constructor() {
+    const website = "https://berascan.com";
+    const chainName = "berachain";
+    const htmlPuller = new OptimismHtmlParser();
+    const apiPuller = new EtherscanApiParser(website);
+    super(website, chainName, apiPuller, htmlPuller);
+  }
+}

scripts/Chain/BlastChain.ts

@@ -0,0 +1,14 @@
+import type { ApiParser } from "../ApiParser/ApiParser";
+import { EtherscanApiParser } from "../ApiParser/EtherscanApiParser";
+import { OptimismHtmlParser } from "../HtmlParser/OptimismHtmlParser";
+import { Chain } from "./Chain";
+
+export class BlastChain extends Chain<ApiParser, OptimismHtmlParser> {
+  public constructor() {
+    const website = "https://blastscan.io";
+    const chainName = "blast";
+    const htmlPuller = new OptimismHtmlParser();
+    const apiPuller = new EtherscanApiParser(website);
+    super(website, chainName, apiPuller, htmlPuller);
+  }
+}

scripts/Chain/Chain.ts

@@ -17,6 +17,14 @@ export class Chain<T extends ApiParser, T2 extends HtmlParser> {
     celo: 42220,
     bscscan: 56,
     gnosis: 100,
+    avalanche: 43114,
+    polygon: 137,
+    mantle: 5000,
+    scroll: 534352,
+    linea: 59144,
+    blast: 81457,
+    berachain: 80094,
+    worldchain: 480,
   };
 
   public constructor(

scripts/Chain/LineaChain.ts

@@ -0,0 +1,14 @@
+import type { ApiParser } from "../ApiParser/ApiParser";
+import { EtherscanApiParser } from "../ApiParser/EtherscanApiParser";
+import { OptimismHtmlParser } from "../HtmlParser/OptimismHtmlParser";
+import { Chain } from "./Chain";
+
+export class LineaChain extends Chain<ApiParser, OptimismHtmlParser> {
+  public constructor() {
+    const website = "https://lineascan.build";
+    const chainName = "linea";
+    const htmlPuller = new OptimismHtmlParser();
+    const apiPuller = new EtherscanApiParser(website);
+    super(website, chainName, apiPuller, htmlPuller);
+  }
+}

scripts/Chain/MantleChain.ts

@@ -0,0 +1,14 @@
+import type { ApiParser } from "../ApiParser/ApiParser";
+import { EtherscanApiParser } from "../ApiParser/EtherscanApiParser";
+import { OptimismHtmlParser } from "../HtmlParser/OptimismHtmlParser";
+import { Chain } from "./Chain";
+
+export class MantleChain extends Chain<ApiParser, OptimismHtmlParser> {
+  public constructor() {
+    const website = "https://mantlescan.xyz";
+    const chainName = "mantle";
+    const htmlPuller = new OptimismHtmlParser();
+    const apiPuller = new EtherscanApiParser(website);
+    super(website, chainName, apiPuller, htmlPuller);
+  }
+}

scripts/Chain/PolygonChain.ts

@@ -0,0 +1,14 @@
+import type { ApiParser } from "../ApiParser/ApiParser";
+import { EtherscanApiParser } from "../ApiParser/EtherscanApiParser";
+import { OptimismHtmlParser } from "../HtmlParser/OptimismHtmlParser";
+import { Chain } from "./Chain";
+
+export class PolygonChain extends Chain<ApiParser, OptimismHtmlParser> {
+  public constructor() {
+    const website = "https://polygonscan.com";
+    const chainName = "polygon";
+    const htmlPuller = new OptimismHtmlParser();
+    const apiPuller = new EtherscanApiParser(website);
+    super(website, chainName, apiPuller, htmlPuller);
+  }
+}

scripts/Chain/ScrollChain.ts

@@ -0,0 +1,14 @@
+import type { ApiParser } from "../ApiParser/ApiParser";
+import { EtherscanApiParser } from "../ApiParser/EtherscanApiParser";
+import { OptimismHtmlParser } from "../HtmlParser/OptimismHtmlParser";
+import { Chain } from "./Chain";
+
+export class ScrollChain extends Chain<ApiParser, OptimismHtmlParser> {
+  public constructor() {
+    const website = "https://scrollscan.com";
+    const chainName = "scroll";
+    const htmlPuller = new OptimismHtmlParser();
+    const apiPuller = new EtherscanApiParser(website);
+    super(website, chainName, apiPuller, htmlPuller);
+  }
+}

scripts/Chain/WorldChain.ts

@@ -0,0 +1,14 @@
+import type { ApiParser } from "../ApiParser/ApiParser";
+import { EtherscanApiParser } from "../ApiParser/EtherscanApiParser";
+import { OptimismHtmlParser } from "../HtmlParser/OptimismHtmlParser";
+import { Chain } from "./Chain";
+
+export class WorldChain extends Chain<ApiParser, OptimismHtmlParser> {
+  public constructor() {
+    const website = "https://worldscan.org";
+    const chainName = "worldchain";
+    const htmlPuller = new OptimismHtmlParser();
+    const apiPuller = new EtherscanApiParser(website);
+    super(website, chainName, apiPuller, htmlPuller);
+  }
+}

scripts/browser-fetch.ts

@@ -16,6 +16,7 @@ export class BrowserFetcher {
   #browser: Browser | null = null;
   #page: Page | null = null;
   #ready = false;
+  #activeOrigin: string | null = null;
 
   /**
    * Connect to a running Chrome instance and prepare for fetching.
@@ -41,23 +42,7 @@ export class BrowserFetcher {
           });
           this.#page = await this.#browser.newPage();
 
-          // Navigate to etherscan to establish Cloudflare clearance
-          console.log("  🔐 Establishing Cloudflare clearance...");
-          await this.#page.goto("https://etherscan.io/labelcloud", {
-            waitUntil: "networkidle2",
-            timeout: 60000,
-          });
-
-          // Check if we got through Cloudflare
-          const title = await this.#page.title();
-          if (title.includes("Just a moment")) {
-            // Cloudflare challenge - wait for it to resolve
-            console.log("  ⏳ Solving Cloudflare challenge...");
-            await this.#page.waitForFunction(
-              () => !document.title.includes("Just a moment"),
-              { timeout: 30000 },
-            );
-          }
+          await this.setActiveOrigin("https://etherscan.io");
 
           this.#ready = true;
           console.log("  ✅ Browser fetch ready\n");
@@ -70,39 +55,66 @@ export class BrowserFetcher {
 
     throw new Error(
       "No Chrome instance found. Start Clawdbot or launch Chrome with:\n" +
-        "  /Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --remote-debugging-port=9222",
+        "  /Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-devtools-eth-labels",
     );
   }
 
+  /**
+   * Prime Cloudflare/session state for a target origin.
+   * Must be called when switching chains (different explorer domains).
+   */
+  public async setActiveOrigin(originOrUrl: string): Promise<void> {
+    if (!this.#page) throw new Error("BrowserFetcher not initialized");
+
+    const origin = new URL(originOrUrl).origin;
+    if (this.#activeOrigin === origin) return;
+
+    console.log(`  🔐 Establishing Cloudflare clearance for ${origin}...`);
+    await this.#page.goto(`${origin}/labelcloud`, {
+      waitUntil: "networkidle2",
+      timeout: 60000,
+    });
+
+    const title = await this.#page.title();
+    if (title.includes("Just a moment")) {
+      console.log("  ⏳ Solving Cloudflare challenge...");
+      await this.#page.waitForFunction(
+        () => !document.title.includes("Just a moment"),
+        { timeout: 30000 },
+      );
+      // Give the site a moment to set cookies/session state after challenge.
+      await new Promise((r) => setTimeout(r, 1500));
+    }
+
+    this.#activeOrigin = origin;
+  }
+
   /**
    * Fetch HTML content through the browser.
    */
   public async fetchHtml(url: string): Promise<string> {
     if (!this.#page || !this.#ready)
       throw new Error("BrowserFetcher not initialized");
 
-    const result = await this.#page.evaluate(async (fetchUrl: string) => {
-      const res = await fetch(fetchUrl);
-      return { status: res.status, text: await res.text() };
-    }, url);
+    const origin = new URL(url).origin;
+    if (this.#activeOrigin !== origin) {
+      await this.setActiveOrigin(origin);
+    }
 
-    if (result.status !== 200 || result.text.includes("Just a moment...")) {
-      // Navigate directly to the page to pass Cloudflare challenge
-      await this.#page.goto(url, { waitUntil: "networkidle2", timeout: 60000 });
-
-      // Wait for Cloudflare if needed
-      const title = await this.#page.title();
-      if (title.includes("Just a moment")) {
-        await this.#page.waitForFunction(
-          () => !document.title.includes("Just a moment"),
-          { timeout: 30000 },
-        );
-      }
+    // Navigation-based fetch is slower than page.evaluate(fetch), but far more
+    // reliable across explorers and Cloudflare policies.
+    await this.#page.goto(url, { waitUntil: "networkidle2", timeout: 60000 });
 
-      return await this.#page.content();
+    const title = await this.#page.title();
+    if (title.includes("Just a moment")) {
+      await this.#page.waitForFunction(
+        () => !document.title.includes("Just a moment"),
+        { timeout: 30000 },
+      );
+      await new Promise((r) => setTimeout(r, 1500));
     }
 
-    return result.text;
+    return await this.#page.content();
   }
 
   /**
@@ -112,21 +124,34 @@ export class BrowserFetcher {
     if (!this.#page || !this.#ready)
       throw new Error("BrowserFetcher not initialized");
 
-    const result = await this.#page.evaluate(
-      async (fetchUrl: string, fetchBody: string) => {
-        const res = await fetch(fetchUrl, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            "X-Requested-With": "XMLHttpRequest",
-          },
-          body: fetchBody,
-        });
-        return { status: res.status, text: await res.text() };
-      },
-      url,
-      body,
-    );
+    const origin = new URL(url).origin;
+    if (this.#activeOrigin !== origin) {
+      await this.setActiveOrigin(origin);
+    }
+
+    const attemptPost = async () =>
+      this.#page!.evaluate(
+        async (fetchUrl: string, fetchBody: string) => {
+          const res = await fetch(fetchUrl, {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+              "X-Requested-With": "XMLHttpRequest",
+            },
+            body: fetchBody,
+          });
+          return { status: res.status, text: await res.text() };
+        },
+        url,
+        body,
+      );
+
+    let result = await attemptPost();
+    if (result.status !== 200 || result.text.includes("Just a moment...")) {
+      // Re-prime once and retry the POST.
+      await this.setActiveOrigin(origin);
+      result = await attemptPost();
+    }
 
     if (result.status !== 200 || result.text.includes("Just a moment...")) {
       throw new Error(
@@ -174,5 +199,6 @@ export class BrowserFetcher {
       this.#browser = null;
     }
     this.#ready = false;
+    this.#activeOrigin = null;
   }
 }

scripts/fetch-all.ts

@@ -15,6 +15,7 @@ void (async () => {
 
     // Process chains sequentially to avoid overwhelming the browser tab
     for (const chain of chainsToPull) {
+      await browserFetcher.setActiveOrigin(chain.website);
       const chainPuller = await ChainPuller.init(chain, browserFetcher);
       await chainPuller.pullAndWriteAllLabels();
     }