feat(webfetch): route through fetch-use when BROWSER_USE_API_KEY is set

alex · alex · commit d87f175298d0 · 2026-05-08T23:55:47.000-07:00
Adds @browser-use/bcode-browser/fetch-use Effect service that POSTs to https://fetch.browser-use.com/fetch with the X-Browser-Use-API-Key header. The webfetch tool consults fetchUse.enabled and swaps the HTTP call when the key is present, preserving the existing native HttpClient + cloudflare-retry path otherwise. Opt-out: BCODE_NO_FETCH_USE=1. Decisions ref: decisions.md S3.3 / ROADMAP B1+B2. No webfetch schema changes (session_id / output_format / proxy_country deferred until evals show demand). B3 retired by Phase H.
diff --git a/packages/bcode-browser/src/fetch-use.ts b/packages/bcode-browser/src/fetch-use.ts
@@ -0,0 +1,92 @@
+// fetch-use — Effect service that proxies HTTP requests through Browser Use's
+// fetch-use cloud (Chrome JA4 fingerprint, HTTP/2 header order, session-based
+// cookie persistence). See `memory/browsercode/fetch_use_reference.md` for
+// the API shape; decisions.md §3.3 + ROADMAP B1 for the rationale.
+//
+// The layer is always constructible. `enabled` reflects whether
+// BROWSER_USE_API_KEY is set and the user hasn't opted out via
+// BCODE_NO_FETCH_USE=1. Consumers (webfetch.ts) check `enabled` and fall back
+// to native HttpClient when false.
+
+import { Context, Effect, Layer } from "effect"
+import { HttpClient, HttpClientRequest } from "effect/unstable/http"
+
+const ENDPOINT = "https://fetch.browser-use.com/fetch"
+const DEFAULT_TIMEOUT_MS = 30_000
+
+// Mirrors the Go FetchResponse type at
+// github.com/browser-use/fetch-use/internal/types/types.go. `headers` is
+// http.Header — `map[string][]string` over the wire — so each value is an
+// array of strings, not a single string.
+interface FetchUseRaw {
+  status_code: number
+  headers?: Record<string, string[]>
+  body?: string
+  body_base64?: string
+  is_binary?: boolean
+  error?: string
+}
+
+export interface FetchOptions {
+  readonly timeoutMs?: number
+}
+
+export interface FetchResult {
+  readonly body: ArrayBuffer
+  readonly contentType: string
+  readonly statusCode: number
+}
+
+export interface Interface {
+  readonly enabled: boolean
+  readonly fetch: (url: string, opts?: FetchOptions) => Effect.Effect<FetchResult, Error>
+}
+
+export class Service extends Context.Service<Service, Interface>()("@browser-use/FetchUse") {}
+
+const headerValue = (h: Record<string, string[]> | undefined, key: string): string => {
+  if (!h) return ""
+  for (const [k, v] of Object.entries(h)) {
+    if (k.toLowerCase() === key.toLowerCase()) return v[0] ?? ""
+  }
+  return ""
+}
+
+export const layer = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    const http = yield* HttpClient.HttpClient
+    const apiKey = process.env.BROWSER_USE_API_KEY ?? ""
+    const enabled = apiKey.length > 0 && process.env.BCODE_NO_FETCH_USE !== "1"
+
+    const fetch = (url: string, opts?: FetchOptions) =>
+      Effect.gen(function* () {
+        const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS
+        const request = yield* HttpClientRequest.post(ENDPOINT).pipe(
+          HttpClientRequest.setHeaders({
+            "Content-Type": "application/json",
+            "X-Browser-Use-API-Key": apiKey,
+          }),
+          HttpClientRequest.bodyJson({ url, timeout_ms: timeoutMs }),
+        )
+        const response = yield* HttpClient.filterStatusOk(http).execute(request)
+        const data = (yield* response.json) as unknown as FetchUseRaw
+        if (data.error) return yield* Effect.fail(new Error(`fetch-use: ${data.error}`))
+
+        const body =
+          data.is_binary && data.body_base64
+            ? new Uint8Array(Buffer.from(data.body_base64, "base64")).buffer
+            : new TextEncoder().encode(data.body ?? "").buffer
+
+        return {
+          body: body as ArrayBuffer,
+          contentType: headerValue(data.headers, "Content-Type"),
+          statusCode: data.status_code,
+        }
+      }).pipe(Effect.mapError((e) => (e instanceof Error ? e : new Error(String(e)))))
+
+    return Service.of({ enabled, fetch })
+  }),
+)
+
+export * as FetchUse from "./fetch-use"
diff --git a/packages/bcode-browser/test/fetch-use.test.ts b/packages/bcode-browser/test/fetch-use.test.ts
@@ -0,0 +1,40 @@
+// FetchUse smoke tests.
+//
+// Unit: layer is constructible, `enabled` reflects env vars correctly.
+// Live: when BROWSER_USE_API_KEY is set, end-to-end POST to fetch.browser-use.com
+//       returns body bytes + content-type. Skipped without the key.
+
+import { expect, test } from "bun:test"
+import { Effect, Layer } from "effect"
+import { FetchHttpClient } from "effect/unstable/http"
+import { FetchUse } from "../src/fetch-use"
+
+const haveKey = !!process.env.BROWSER_USE_API_KEY
+
+test("layer constructs and exposes `enabled`", async () => {
+  const enabled = await Effect.gen(function* () {
+    const svc = yield* FetchUse.Service
+    return svc.enabled
+  }).pipe(
+    Effect.provide(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer))),
+    Effect.runPromise,
+  )
+  expect(typeof enabled).toBe("boolean")
+  expect(enabled).toBe(haveKey && process.env.BCODE_NO_FETCH_USE !== "1")
+})
+
+test.skipIf(!haveKey)("live: fetches httpbin and returns body + content-type", async () => {
+  const result = await Effect.gen(function* () {
+    const svc = yield* FetchUse.Service
+    return yield* svc.fetch("https://httpbin.org/get")
+  }).pipe(
+    Effect.provide(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer))),
+    Effect.runPromise,
+  )
+
+  expect(result.statusCode).toBe(200)
+  expect(result.contentType).toContain("application/json")
+  const text = new TextDecoder().decode(result.body)
+  const data = JSON.parse(text)
+  expect(data.url).toBe("https://httpbin.org/get")
+})
diff --git a/packages/opencode/src/tool/registry.ts b/packages/opencode/src/tool/registry.ts
@@ -33,6 +33,7 @@ import path from "path"
 import { pathToFileURL } from "url"
 import { Effect, Layer, Context } from "effect"
 import { FetchHttpClient, HttpClient } from "effect/unstable/http"
+import { FetchUse } from "@browser-use/bcode-browser/fetch-use"
 import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"
 import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
 import { Ripgrep } from "../file/ripgrep"
@@ -85,6 +86,7 @@ export const layer: Layer.Layer<
   | AppFileSystem.Service
   | Bus.Service
   | HttpClient.HttpClient
+  | FetchUse.Service
   | ChildProcessSpawner
   | Ripgrep.Service
   | Format.Service
@@ -349,6 +351,7 @@ export const defaultLayer = Layer.suspend(() =>
     Layer.provide(Instruction.defaultLayer),
     Layer.provide(AppFileSystem.defaultLayer),
     Layer.provide(Bus.layer),
+    Layer.provide(FetchUse.layer),
     Layer.provide(FetchHttpClient.layer),
     Layer.provide(Format.defaultLayer),
     Layer.provide(CrossSpawnSpawner.defaultLayer),
diff --git a/packages/opencode/src/tool/webfetch.ts b/packages/opencode/src/tool/webfetch.ts
@@ -1,5 +1,6 @@
 import { Effect, Schema } from "effect"
 import { HttpClient, HttpClientRequest } from "effect/unstable/http"
+import { FetchUse } from "@browser-use/bcode-browser/fetch-use"
 import * as Tool from "./tool"
 import TurndownService from "turndown"
 import DESCRIPTION from "./webfetch.txt"
@@ -24,6 +25,7 @@ export const WebFetchTool = Tool.define(
   Effect.gen(function* () {
     const http = yield* HttpClient.HttpClient
     const httpOk = HttpClient.filterStatusOk(http)
+    const fetchUse = yield* FetchUse.Service
 
     return {
       description: DESCRIPTION,
@@ -47,61 +49,77 @@ export const WebFetchTool = Tool.define(
 
           const timeout = Math.min((params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT)
 
-          // Build Accept header based on requested format with q parameters for fallbacks
-          let acceptHeader = "*/*"
-          switch (params.format) {
-            case "markdown":
-              acceptHeader = "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1"
-              break
-            case "text":
-              acceptHeader = "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1"
-              break
-            case "html":
-              acceptHeader =
-                "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1"
-              break
-            default:
-              acceptHeader =
-                "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"
-          }
-          const headers = {
-            "User-Agent":
-              "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
-            Accept: acceptHeader,
-            "Accept-Language": "en-US,en;q=0.9",
-          }
+          // BrowserCode: when fetch-use is enabled (BROWSER_USE_API_KEY set,
+          // BCODE_NO_FETCH_USE != "1"), proxy through it for Chrome JA4
+          // fingerprinting + HTTP/2 header order. Falls back to native
+          // HttpClient with cloudflare-retry when disabled.
+          const { arrayBuffer, contentType } = yield* (fetchUse.enabled
+            ? fetchUse
+                .fetch(params.url, { timeoutMs: timeout })
+                .pipe(Effect.map((r) => ({ arrayBuffer: r.body, contentType: r.contentType })))
+            : Effect.gen(function* () {
+                // Build Accept header based on requested format with q parameters for fallbacks
+                let acceptHeader = "*/*"
+                switch (params.format) {
+                  case "markdown":
+                    acceptHeader =
+                      "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1"
+                    break
+                  case "text":
+                    acceptHeader = "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1"
+                    break
+                  case "html":
+                    acceptHeader =
+                      "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1"
+                    break
+                  default:
+                    acceptHeader =
+                      "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"
+                }
+                const headers = {
+                  "User-Agent":
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
+                  Accept: acceptHeader,
+                  "Accept-Language": "en-US,en;q=0.9",
+                }
 
-          const request = HttpClientRequest.get(params.url).pipe(HttpClientRequest.setHeaders(headers))
-
-          // Retry with honest UA if blocked by Cloudflare bot detection (TLS fingerprint mismatch)
-          const response = yield* httpOk.execute(request).pipe(
-            Effect.catchIf(
-              (err) =>
-                err.reason._tag === "StatusCodeError" &&
-                err.reason.response.status === 403 &&
-                err.reason.response.headers["cf-mitigated"] === "challenge",
-              () =>
-                httpOk.execute(
-                  HttpClientRequest.get(params.url).pipe(
-                    HttpClientRequest.setHeaders({ ...headers, "User-Agent": "browsercode" }),
+                const request = HttpClientRequest.get(params.url).pipe(HttpClientRequest.setHeaders(headers))
+
+                // Retry with honest UA if blocked by Cloudflare bot detection (TLS fingerprint mismatch)
+                const response = yield* httpOk.execute(request).pipe(
+                  Effect.catchIf(
+                    (err) =>
+                      err.reason._tag === "StatusCodeError" &&
+                      err.reason.response.status === 403 &&
+                      err.reason.response.headers["cf-mitigated"] === "challenge",
+                    () =>
+                      httpOk.execute(
+                        HttpClientRequest.get(params.url).pipe(
+                          HttpClientRequest.setHeaders({ ...headers, "User-Agent": "browsercode" }),
+                        ),
+                      ),
                   ),
-                ),
-            ),
-            Effect.timeoutOrElse({ duration: timeout, orElse: () => Effect.die(new Error("Request timed out")) }),
-          )
-
-          // Check content length
-          const contentLength = response.headers["content-length"]
-          if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
-            throw new Error("Response too large (exceeds 5MB limit)")
-          }
+                  Effect.timeoutOrElse({ duration: timeout, orElse: () => Effect.die(new Error("Request timed out")) }),
+                )
+
+                // Check content length
+                const contentLength = response.headers["content-length"]
+                if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
+                  throw new Error("Response too large (exceeds 5MB limit)")
+                }
+
+                const arrayBuffer = yield* response.arrayBuffer
+                if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) {
+                  throw new Error("Response too large (exceeds 5MB limit)")
+                }
+
+                return { arrayBuffer, contentType: response.headers["content-type"] || "" }
+              }))
 
-          const arrayBuffer = yield* response.arrayBuffer
           if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) {
             throw new Error("Response too large (exceeds 5MB limit)")
           }
 
-          const contentType = response.headers["content-type"] || ""
           const mime = contentType.split(";")[0]?.trim().toLowerCase() || ""
           const title = `${params.url} (${contentType})`
 
diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts
@@ -39,6 +39,7 @@ import { Shell } from "../../src/shell/shell"
 import { Snapshot } from "../../src/snapshot"
 import { ToolRegistry } from "@/tool/registry"
 import { Truncate } from "@/tool/truncate"
+import { FetchUse } from "@browser-use/bcode-browser/fetch-use"
 import * as Log from "@opencode-ai/core/util/log"
 import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
 import * as Database from "../../src/storage/db"
@@ -176,6 +177,7 @@ function makeHttp() {
   const todo = Todo.layer.pipe(Layer.provideMerge(deps))
   const registry = ToolRegistry.layer.pipe(
     Layer.provide(Skill.defaultLayer),
+    Layer.provide(FetchUse.layer),
     Layer.provide(FetchHttpClient.layer),
     Layer.provide(CrossSpawnSpawner.defaultLayer),
     Layer.provide(Ripgrep.defaultLayer),
diff --git a/packages/opencode/test/session/snapshot-tool-race.test.ts b/packages/opencode/test/session/snapshot-tool-race.test.ts
@@ -50,6 +50,7 @@ import { SessionRunState } from "../../src/session/run-state"
 import { SessionStatus } from "../../src/session/status"
 import { Snapshot } from "../../src/snapshot"
 import { ToolRegistry } from "@/tool/registry"
+import { FetchUse } from "@browser-use/bcode-browser/fetch-use"
 import { Truncate } from "@/tool/truncate"
 import { AppFileSystem } from "@opencode-ai/core/filesystem"
 import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
@@ -126,6 +127,7 @@ function makeHttp() {
   const todo = Todo.layer.pipe(Layer.provideMerge(deps))
   const registry = ToolRegistry.layer.pipe(
     Layer.provide(Skill.defaultLayer),
+    Layer.provide(FetchUse.layer),
     Layer.provide(FetchHttpClient.layer),
     Layer.provide(CrossSpawnSpawner.defaultLayer),
     Layer.provide(Ripgrep.defaultLayer),
diff --git a/packages/opencode/test/tool/registry.test.ts b/packages/opencode/test/tool/registry.test.ts
@@ -19,6 +19,7 @@ import { LSP } from "@/lsp/lsp"
 import { Instruction } from "@/session/instruction"
 import { Bus } from "@/bus"
 import { FetchHttpClient } from "effect/unstable/http"
+import { FetchUse } from "@browser-use/bcode-browser/fetch-use"
 import { Format } from "@/format"
 import { Ripgrep } from "@/file/ripgrep"
 import * as Truncate from "@/tool/truncate"
@@ -42,6 +43,7 @@ const registryLayer = ToolRegistry.layer.pipe(
   Layer.provide(Instruction.defaultLayer),
   Layer.provide(AppFileSystem.defaultLayer),
   Layer.provide(Bus.layer),
+  Layer.provide(FetchUse.layer),
   Layer.provide(FetchHttpClient.layer),
   Layer.provide(Format.defaultLayer),
   Layer.provide(node),
diff --git a/packages/opencode/test/tool/webfetch.test.ts b/packages/opencode/test/tool/webfetch.test.ts
@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"
 import path from "path"
 import { Effect, Layer } from "effect"
 import { FetchHttpClient } from "effect/unstable/http"
+import { FetchUse } from "@browser-use/bcode-browser/fetch-use"
 import { Agent } from "../../src/agent/agent"
 import { Truncate } from "@/tool/truncate"
 import { Instance } from "../../src/project/instance"
@@ -31,7 +32,9 @@ function exec(args: { url: string; format: "text" | "markdown" | "html" }) {
   return WebFetchTool.pipe(
     Effect.flatMap((info) => info.init()),
     Effect.flatMap((tool) => tool.execute(args, ctx)),
-    Effect.provide(Layer.mergeAll(FetchHttpClient.layer, Truncate.defaultLayer, Agent.defaultLayer)),
+    Effect.provide(
+      Layer.mergeAll(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer)), FetchHttpClient.layer, Truncate.defaultLayer, Agent.defaultLayer),
+    ),
     Effect.runPromise,
   )
 }
