diff --git a/CLAUDE.md b/CLAUDE.md index 08c747e..461d7a5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ Follow these rules when editing code in this project. `sibyl` is a CLI web search/crawl tool for AI Agents (`bin: sibyl` → `dist/cli.js`) with a filesystem-based plugin system. Key modules: -- `src/cli.ts` — entry point. Ensures dirs + config exist, loads plugins, builds a `PluginContext` (`buildPluginContext`), and dispatches commands (`search`, `fetch`, `help`/`--help`/`-h`, `version`/`--version`). Only `search` and `fetch` are wired up via the async `handleSearch`/`handleFetch` helpers (awaited by `main`), each passing the context as the last arg to the selected plugin's `fn`. The `fetch` command prints the fetch plugin's output directly — the CLI doesn't dispatch a separate `parse` step, but a fetch plugin may itself run the configured parse plugin via `context.configuredPlugins.parse` (`builtin-brightdata-fetch`, `builtin-crawl4ai-fetch`, and `builtin-alterlab-fetch` do; `builtin-firecrawl-fetch` does only in its raw-HTML mode; `builtin-exa-fetch` returns content as-is). `ask` is part of the contract but not dispatched by any command. `main` is exported and only auto-runs when the file is the actual CLI entry (`import.meta.url` vs `process.argv[1]` guard), so tests can import it without side effects. +- `src/cli.ts` — entry point. Ensures dirs + config exist, loads plugins, builds a `PluginContext` (`buildPluginContext`), and dispatches commands (`search`, `fetch`, `ask`, `help`/`--help`/`-h`, `version`/`--version`). `search`, `fetch`, and `ask` are wired up via the async `handleSearch`/`handleFetch`/`handleAsk` helpers (awaited by `main`), each passing the context as the last arg to the selected plugin's `fn`. The `fetch` command prints the fetch plugin's output directly — the CLI doesn't dispatch a separate `parse` step, but a fetch plugin may itself run the configured parse plugin via `context.configuredPlugins.parse` (`builtin-brightdata-fetch`, `builtin-crawl4ai-fetch`, and `builtin-alterlab-fetch` do; `builtin-firecrawl-fetch` does only in its raw-HTML mode; `builtin-exa-fetch` returns content as-is). The `ask` command (`sibyl ask `) passes the URL as the ask plugin's first arg; analogously, an ask plugin may itself fetch that URL via `context.configuredPlugins.fetch` before answering (`builtin-ai-ask` does). `main` is exported and only auto-runs when the file is the actual CLI entry (`import.meta.url` vs `process.argv[1]` guard), so tests can import it without side effects. - `src/setup.ts` — ensures `~/.sibyl` and `~/.sibyl/plugins` exist, and loads/creates/validates `~/.sibyl/config.json` (all on every invocation). - `src/plugin-loader.ts` — assembles the active plugin set: builtin plugins + external (on-disk) plugins; validates the external ones. - `src/plugins/config.ts` — `getBuiltinPlugins()`, the in-repo builtin plugin registry. @@ -48,7 +48,7 @@ Plugins live in `~/.sibyl/plugins//main.js` (note: `.js`, loaded at runtim 3. `fn` — the function implementing the plugin's logic. Every `fn` receives a `PluginContext` as its **last** argument; its signature otherwise depends on `type` (`src/@types/plugin.ts`): - `search`: `(query, context) => Promise` - `fetch`: `(url, context) => Promise` - - `ask`: `(parsedContent, query, context) => Promise` + - `ask`: `(src, query, context) => Promise` (the dispatched `ask` command passes a URL as `src`) - `parse`: `(html, context) => Promise` `PluginContext` (`src/@types/plugin.ts`) lets a plugin reach the rest of the plugin system: `{ configuredPlugins: Partial>, allPlugins: PluginTypeDeclaration[], getPlugin(name): PluginTypeDeclaration | null }`. `configuredPlugins` is keyed by type (the per-type selection from config), `allPlugins` is everything loaded, and `getPlugin` looks up by name. It's built once in `cli.ts` and threaded to every `fn`; plugins consume it only if needed (a 1-arg `fn` still satisfies the contract via structural typing). This is how a fetch plugin runs the configured parser: `context.configuredPlugins.parse?.fn(html, context)`. @@ -65,15 +65,16 @@ When changing the plugin shape, update all three together: `src/@types/plugin.ts `loadPlugins()` (`plugin-loader.ts`) returns `[...getBuiltinPlugins(), ...externalPlugins]`. -- Builtins are **compiled into the binary, not loaded from disk**. `src/plugins/config.ts` statically imports each builtin's `SilbylPlugin` (e.g. from `src/plugins/builtin-exa-search/main.ts`) and returns them as `PluginTypeDeclaration` objects — they bypass `validatePlugin` and the `main.js` discovery path. Each builtin `main.ts` types its `SilbylPlugin` with the matching interface (`SearchPlugin` / `FetchPlugin` / `ParsePlugin`) so `type` stays a literal. +- Builtins are **compiled into the binary, not loaded from disk**. `src/plugins/config.ts` statically imports each builtin's `SilbylPlugin` (e.g. from `src/plugins/builtin-exa-search/main.ts`) and returns them as `PluginTypeDeclaration` objects — they bypass `validatePlugin` and the `main.js` discovery path. Each builtin `main.ts` types its `SilbylPlugin` with the matching interface (`SearchPlugin` / `FetchPlugin` / `AskPlugin` / `ParsePlugin`) so `type` stays a literal. - Builtin names are prefixed `builtin-` by convention. External plugin folders starting with `builtin-` are rejected during discovery (reserved namespace), so user plugins cannot shadow a builtin. - To add a builtin: create `src/plugins/builtin-/main.ts` exporting a typed `SilbylPlugin` (with `fn`), then register it in `getBuiltinPlugins()`. +- `builtin-ai-ask` (the `ask` builtin) reads a URL via the configured fetch plugin, then answers a question over that content using the **Vercel AI SDK**, with the provider selectable via `SIBYL_AI_PROVIDER` (`openai` / `anthropic` / `ollama` / `openrouter`), `SIBYL_MODEL_NAME`, and a per-provider key (`OPENAI_API_KEY` etc.; Ollama uses `OLLAMA_BASE_URL`, no key). It loads `ai` and each provider package (`@ai-sdk/*`, `ollama-ai-provider-v2`, `@openrouter/ai-sdk-provider`) via **dynamic `import()`** inside the `fn` — never at module top level — because `getBuiltinPlugins()` imports every builtin module on every CLI run, so top-level SDK imports would slow `search`/`fetch` too. ### Config (`~/.sibyl/config.json`) Shape: `SibylConfig` (`src/@types/sibyl-config.ts`) — `{ plugins: Partial>, variables: { name, value }[] }`. `plugins` maps `type` → plugin name (e.g. `{ "search": "builtin-exa-search" }`); keying by type structurally enforces at most one plugin per type. `variables` is a list of `{ name, value }` pairs injected into `process.env`. -- `loadOrCreateConfigFile()` (`setup.ts`) writes a default config (`writeDefaultSibylConfig`) when the file is missing or empty, then parses, validates, and injects variables. The default selects `builtin-searxng-search` / `builtin-crawl4ai-fetch` / `builtin-parse-htmlToMd` with no `variables` (fully local, no-API-key backends). +- `loadOrCreateConfigFile()` (`setup.ts`) writes a default config (`writeDefaultSibylConfig`) when the file is missing or empty, then parses, validates, and injects variables. The default selects `builtin-searxng-search` / `builtin-crawl4ai-fetch` / `builtin-parse-htmlToMd` / `builtin-ai-ask` plus one variable (`SIBYL_SHOW_SEARCH_DESCRIPTION=true`); the search/fetch/parse backends are fully local and need no API key, while the `ask` command additionally needs `SIBYL_AI_PROVIDER` / `SIBYL_MODEL_NAME` (and a provider API key unless using Ollama) to be set. - `injectConfigVariables()` (`setup.ts`) sets `process.env[name] = value` for each config variable. **Config wins over the environment** — a variable named in config overrides any existing env var; names absent from config fall back to their existing env value. (Plugins like `builtin-exa-search` read `process.env.EXA_API_KEY` at call time, so they pick up either source.) - `validateConfig()` checks each entry's name is a non-empty string; on failure it `console.error`s and `process.exit(1)` (hard exit, not a skip-with-warning like plugin loading). - Plugin selection: `loadPlugins()` loads _all_ available plugins (builtins + disk), then `cli.ts` picks the one to run **by name from config** — e.g. the `search` command looks up `config.plugins.search` and finds the loaded plugin whose `type === "search"` and `name` matches. Missing config entry or no matching loaded plugin → `console.error` + non-zero exit. diff --git a/README.md b/README.md index f0a00c7..ebee671 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ [![sibyl License Page](https://img.shields.io/badge/license-Apache_2.0-brightgreen)](https://raw.githubusercontent.com/postapsis/sibyl/refs/heads/main/LICENSE) [![sibyl CI Status](https://github.com/postapsis/sibyl/actions/workflows/ci.yaml/badge.svg)](https://github.com/postapsis/sibyl/actions/workflows/ci.yaml) [![codecov](https://codecov.io/gh/postapsis/sibyl/branch/main/graph/badge.svg?token=NOTP4DPWO4)](https://codecov.io/gh/postapsis/sibyl) -
--- -**_Sibyl_** gives your AI Agent the web, without the bloat — extensible and lightweight by design 🕷️ +Local-first web search and exploration for your AI agents, without the bloat.\ +Extensible and lightweight by design 🕷️ --- @@ -76,13 +76,13 @@ Get a working setup in a few steps: ## Commands -| Command | Description | -| -------------- | ----------------------------------------------------------------------------------------------------------------------------- | -| `search` | Searches the web
`sibyl search "react vite boostrap"` | -| `fetch` | Gets the content of a site in token-efficient markdown
`sibyl fetch https://vite.dev/guide` | -| `ask` | Asks a query using LLM from a site's content
`sibyl ask https://vite.dev/guide "how to start a react project with vite"` | -| `--help`, `-h` | Show help. | -| `--version` | Show version. | +| Command | Description | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| search | Searches the web
`sibyl search "react vite"` | +| fetch | Prints the content of a site in token-efficient markdown
`sibyl fetch https://vite.dev/guide` | +| ask | Asks a query using LLM from a site's content
`sibyl ask https://vite.dev/guide "how to start a react project with vite"` | +| `--help`, `-h` | Shows help. | +| `--version` | Shows version. | ## Configuration diff --git a/codecov.yml b/codecov.yml index 941fff6..c1e1870 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,7 +6,7 @@ coverage: threshold: 2% patch: default: - target: 90% + target: 85% threshold: 2% comment: diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 522b038..efa8e09 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -1,58 +1,82 @@ -## Configuration +# Configuration -### Configuration file +## Configuration file -Sibyl reads its config from `~/.sibyl/config.json`, created with sensible defaults on first run. It has two sections: +Sibyl reads its configuration from `~/.sibyl/config.json`, created with sensible defaults on first run. It has two sections: ```json { "plugins": { "search": "builtin-searxng-search", "fetch": "builtin-crawl4ai-fetch", - "parse": "builtin-parse-htmlToMd" + "parse": "builtin-parse-htmlToMd", + "ask": "builtin-ai-ask" }, - "variables": [] + "variables": [ + { + "name": "SIBYL_SHOW_SEARCH_DESCRIPTION", + "value": "true" + } + ] } ``` -#### `plugins` section +### The `plugins` section Maps each plugin type (`search` / `fetch` / `ask` / `parse`) to the **name** of the plugin to use for it. Exactly one -plugin per type. The value must match a plugin's `name` (a builtin like `builtin-exa-search`, or one of your custom-written plugins). - -#### `variables` section +plugin per type. The value must match a plugin's `name` (a built-in like `builtin-exa-search`, or one of your custom-written plugins). + +
+The built-in plugins are: + +1. SearXNG Search - `builtin-searxng-search` +2. Crawl4AI Fetch - `builtin-crawl4ai-fetch` +3. Exa Search - `builtin-exa-search` +4. Exa Fetch - `builtin-exa-fetch` +5. Firecrawl Search - `builtin-firecrawl-search` +6. Firecrawl Fetch - `builtin-firecrawl-fetch` +7. AlterLab Search - `builtin-alterlab-search` +8. AlterLab Fetch - `builtin-alterlab-fetch` +9. Bright Data Search - `builtin-brightdata-search` +10. Bright Data Fetch - `builtin-brightdata-fetch` +11. AI Ask (Supports OpenAI, Anthropic, Ollama, OpenRouter) - `builtin-ai-ask` +12. HTML to Markdown Parse - `builtin-parse-htmlToMd` + +### The `variables` section A list of `{ name, value }` pairs injected into the process environment at startup. Use this to provide secrets and settings (e.g., API keys) that plugins read via `process.env`. -Precedence: **config wins over the environment.** A variable defined here overrides any existing environment variable of +Precedence: **Variable in the configuration file wins over the environment.** A variable defined here overrides any existing environment variable of the same name; anything not listed here falls back to the real environment. For example, a plugin reading -`process.env.EXA_API_KEY` gets the config value if present, otherwise whatever was exported in your shell. +`process.env.EXA_API_KEY` gets the configuration value if present, otherwise whatever was exported in your shell. -### Plugin environment variables +## Plugin environment variables -Each builtin plugin reads the variables below (set them via `variables` or the real environment, per the precedence rule -above). A **required** variable causes the plugin to error if it is unset. +Each built-in plugin reads the variables below (set them via `variables` or the real environment, per the precedence rule +above). A **required** variable causes the plugin to error if it is not set. -All `search` plugins also honor the following environment variables +All `search` plugins also honor the following environment variables: -1. **`SIBYL_SEARCH_RESULTS_LIMIT`** (default `10`): Sibyl passes it to the search - provider's API when the provider supports a result-count parameter, and always slices the returned results down to this +1. **`SIBYL_SEARCH_RESULTS_LIMIT`** (default `10`): Sibyl passes it to the search provider if it supports a result-count parameter, and always slices the returned results down to this limit. 2. **`SIBYL_SHOW_SEARCH_DESCRIPTION`** (default `true`): When `"true"`, includes result snippet/description in the output. -#### `builtin-searxng-search` — `search` +
+Here is a list of all the environment variables for the built-in plugins: + +### SearXNG Search -| Variable | Required | Default | Description | -| ------------------------------- | -------- | ----------------------- | ---------------------------------------------------------------------------------------------------------------- | -| `SIBYL_SEARXNG_URL` | No | `http://localhost:8080` | Base URL of a running SearXNG instance. Sibyl uses `/search` endpoint with `format=json`. | -| `SIBYL_SEARXNG_ENGINES` | No | _(none)_ | Comma-separated SearXNG engines to query (e.g. `google`); omitted when unset. | -| `SIBYL_SHOW_SEARCH_DESCRIPTION` | No | `true` | When `"true"`, includes result content in the output. | -| `SIBYL_SEARCH_RESULTS_LIMIT` | No | `10` | Maximum number of search results to return; passed to the provider when supported and always applied by slicing. | +| Variable | Required | Default | Description | +| ------------------------------- | -------- | ----------------------- | ------------------------------------------------------------------------------------------------------- | +| `SIBYL_SEARXNG_URL` | No | `http://localhost:8080` | Base URL of a running SearXNG instance. Sibyl uses `/search` endpoint with `format=json`. | +| `SIBYL_SEARXNG_ENGINES` | No | _(none)_ | Comma-separated SearXNG engines to query (e.g. `google`); omitted when unset. | +| `SIBYL_SHOW_SEARCH_DESCRIPTION` | No | `true` | When `"true"`, includes result content in the output. | +| `SIBYL_SEARCH_RESULTS_LIMIT` | No | `10` | Maximum number of search results to return; passed to the when supported and always applied by slicing. | Requires a SearXNG instance with the **JSON output format enabled**. See more at [https://github.com/searxng/searxng/discussions/3542](https://github.com/searxng/searxng/discussions/3542) -#### `builtin-crawl4ai-fetch` — `fetch` +### Crawl4AI Fetch | Variable | Required | Default | Description | | -------------------- | -------- | ------------------------ | ----------------------------------------------------------------------------------------- | @@ -60,67 +84,80 @@ Requires a SearXNG instance with the **JSON output format enabled**. See more at Requires a Crawl4AI server, e.g., via Docker. See more at [https://hub.docker.com/r/unclecode/crawl4ai](https://hub.docker.com/r/unclecode/crawl4ai) -#### `builtin-exa-search` — `search` +### Exa Search -| Variable | Required | Default | Description | -| ------------------------------- | -------- | ------- | ---------------------------------------------------------------------------------------------------------------- | -| `EXA_API_KEY` | Yes | — | Exa API key. | -| `SIBYL_SHOW_SEARCH_DESCRIPTION` | No | `true` | When `"true"`, includes result highlights in the output. | -| `SIBYL_SEARCH_RESULTS_LIMIT` | No | `10` | Maximum number of search results to return; passed to the provider when supported and always applied by slicing. | +| Variable | Required | Default | Description | +| ------------------------------- | -------- | ------- | ------------------------------------------------------------------------------------------------------- | +| `EXA_API_KEY` | Yes | — | Exa API key. | +| `SIBYL_SHOW_SEARCH_DESCRIPTION` | No | `true` | When `"true"`, includes result highlights in the output. | +| `SIBYL_SEARCH_RESULTS_LIMIT` | No | `10` | Maximum number of search results to return; passed to the when supported and always applied by slicing. | -#### `builtin-exa-fetch` — `fetch` +### Exa Fetch | Variable | Required | Default | Description | | ------------- | -------- | ------- | ------------ | | `EXA_API_KEY` | Yes | — | Exa API key. | -#### `builtin-firecrawl-search` — `search` +### Firecrawl Search -| Variable | Required | Default | Description | -| ------------------------------- | -------- | ------- | ---------------------------------------------------------------------------------------------------------------- | -| `FIRECRAWL_API_KEY` | Yes | — | Firecrawl API key (includes the `fc-` prefix). | -| `SIBYL_SHOW_SEARCH_DESCRIPTION` | No | `true` | When `"true"`, includes result descriptions in the output. | -| `SIBYL_SEARCH_RESULTS_LIMIT` | No | `10` | Maximum number of search results to return; passed to the provider when supported and always applied by slicing. | +| Variable | Required | Default | Description | +| ------------------------------- | -------- | ------- | ------------------------------------------------------------------------------------------------------- | +| `FIRECRAWL_API_KEY` | Yes | — | Firecrawl API key (includes the `fc-` prefix). | +| `SIBYL_SHOW_SEARCH_DESCRIPTION` | No | `true` | When `"true"`, includes result descriptions in the output. | +| `SIBYL_SEARCH_RESULTS_LIMIT` | No | `10` | Maximum number of search results to return; passed to the when supported and always applied by slicing. | -#### `builtin-firecrawl-fetch` — `fetch` +### Firecrawl Fetch | Variable | Required | Default | Description | | -------------------------------- | -------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `FIRECRAWL_API_KEY` | Yes | — | Firecrawl API key (includes the `fc-` prefix). | | `SIBYL_FIRECRAWL_FETCH_USE_HTML` | No | `false` | When `"true"`, fetches the raw HTML and runs it through the configured `parse` plugin; otherwise returns the markdown from Firecrawl with extra blank lines collapsed. | -#### `builtin-alterlab-search` — `search` +### AlterLab Search -| Variable | Required | Default | Description | -| ------------------------------- | -------- | ------- | ---------------------------------------------------------------------------------------------------------------- | -| `ALTERLAB_API_KEY` | Yes | — | AlterLab API key. | -| `SIBYL_SHOW_SEARCH_DESCRIPTION` | No | `true` | When `"true"`, includes result snippets in the output. | -| `SIBYL_SEARCH_RESULTS_LIMIT` | No | `10` | Maximum number of search results to return; passed to the provider when supported and always applied by slicing. | +| Variable | Required | Default | Description | +| ------------------------------- | -------- | ------- | ------------------------------------------------------------------------------------------------------- | +| `ALTERLAB_API_KEY` | Yes | — | AlterLab API key. | +| `SIBYL_SHOW_SEARCH_DESCRIPTION` | No | `true` | When `"true"`, includes result snippets in the output. | +| `SIBYL_SEARCH_RESULTS_LIMIT` | No | `10` | Maximum number of search results to return; passed to the when supported and always applied by slicing. | -#### `builtin-alterlab-fetch` — `fetch` +### AlterLab Fetch | Variable | Required | Default | Description | | ------------------ | -------- | ------- | ----------------- | | `ALTERLAB_API_KEY` | Yes | — | AlterLab API key. | -#### `builtin-brightdata-search` — `search` +### Bright Data Search -| Variable | Required | Default | Description | -| ------------------------------- | -------- | -------- | ---------------------------------------------------------------------------------------------------------------- | -| `BRIGHTDATA_API_KEY` | Yes | — | Bright Data API key. | -| `BRIGHTDATA_SERP_API_ZONE` | Yes | — | Bright Data SERP API zone. | -| `SIBYL_SHOW_SEARCH_DESCRIPTION` | No | `true` | When `"true"`, includes result descriptions in the output. | -| `BRIGHTDATA_SERP_API_LANGUAGE` | No | `en` | Search language (Google `hl`). | -| `BRIGHTDATA_SERP_API_COUNTRY` | No | _(none)_ | Search country (Google `gl`); omitted when unset. | -| `SIBYL_SEARCH_RESULTS_LIMIT` | No | `10` | Maximum number of search results to return; passed to the provider when supported and always applied by slicing. | +| Variable | Required | Default | Description | +| ------------------------------- | -------- | -------- | ------------------------------------------------------------------------------------------------------- | +| `BRIGHTDATA_API_KEY` | Yes | — | Bright Data API key. | +| `BRIGHTDATA_SERP_API_ZONE` | Yes | — | Bright Data SERP API zone. | +| `SIBYL_SHOW_SEARCH_DESCRIPTION` | No | `true` | When `"true"`, includes result descriptions in the output. | +| `BRIGHTDATA_SERP_API_LANGUAGE` | No | `en` | Search language (Google `hl`). | +| `BRIGHTDATA_SERP_API_COUNTRY` | No | _(none)_ | Search country (Google `gl`); omitted when unset. | +| `SIBYL_SEARCH_RESULTS_LIMIT` | No | `10` | Maximum number of search results to return; passed to the when supported and always applied by slicing. | -#### `builtin-brightdata-fetch` — `fetch` +### Bright Data Fetch | Variable | Required | Default | Description | | ---------------------------------- | -------- | ------- | ---------------------------------- | | `BRIGHTDATA_API_KEY` | Yes | — | Bright Data API key. | | `BRIGHTDATA_WEB_UNLOCKER_API_ZONE` | Yes | — | Bright Data Web Unlocker API zone. | -#### `builtin-parse-htmlToMd` — `parse` +### AI Ask + +Reads a URL through the configured `fetch` plugin, then asks an LLM the question against that content. A working `fetch` plugin must also be configured (`plugins.fetch`). + +| Variable | Required | Default | Description | +| -------------------- | ----------- | ---------------------------- | -------------------------------------------------------------------------------------------- | +| `SIBYL_AI_PROVIDER` | Yes | — | LLM: one of `openai`, `anthropic`, `ollama`, `openrouter`. | +| `SIBYL_MODEL_NAME` | Yes | — | Model id passed to the (e.g. `gpt-5.4-mini`, `claude-sonnet-4-6`, `llama3.1`). | +| `OPENAI_API_KEY` | Conditional | — | Required when `SIBYL_AI_PROVIDER=openai`. | +| `ANTHROPIC_API_KEY` | Conditional | — | Required when `SIBYL_AI_PROVIDER=anthropic`. | +| `OPENROUTER_API_KEY` | Conditional | — | Required when `SIBYL_AI_PROVIDER=openrouter`. | +| `OLLAMA_BASE_URL` | No | `http://localhost:11434/api` | Base URL of a running Ollama server; used only when `SIBYL_AI_PROVIDER=ollama` (no API key). | + +### HTML to Markdown Parse No environment variables. diff --git a/docs/CREATING-PLUGINS.md b/docs/CREATING-PLUGINS.md index 62bce2a..3475e25 100644 --- a/docs/CREATING-PLUGINS.md +++ b/docs/CREATING-PLUGINS.md @@ -37,7 +37,7 @@ plugin system: | Field | Description | | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | | `configuredPlugins` | The plugin selected for each type in your config, keyed by type — e.g. `context.configuredPlugins.parse`. Only configured types are present. | -| `allPlugins` | An array of every loaded plugin (builtins + your custom ones). | +| `allPlugins` | An array of every loaded plugin (built-ins + your custom ones). | | `getPlugin(name)` | Returns the loaded plugin whose `name` matches, or `null` if none does. | Each entry is a `{ name, type, fn }` object, so one plugin can invoke another — e.g. a `fetch` plugin can run the diff --git a/media-kit/banner.png b/media-kit/banner.png old mode 100755 new mode 100644 index aee6867..a3a3e11 Binary files a/media-kit/banner.png and b/media-kit/banner.png differ diff --git a/package.json b/package.json index 9fc2494..75cae2c 100644 --- a/package.json +++ b/package.json @@ -61,9 +61,14 @@ "vitest": "^4.1.8" }, "dependencies": { + "@ai-sdk/anthropic": "^3.0.85", + "@ai-sdk/openai": "^3.0.72", + "@openrouter/ai-sdk-provider": "^2.9.1", + "ai": "^6.0.207", "cheerio": "^1.2.0", "defuddle": "^0.18.1", "linkedom": "^0.18.12", + "ollama-ai-provider-v2": "^3.6.0", "turndown": "^7.2.4" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9f15cd5..66b384e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,18 @@ importers: .: dependencies: + '@ai-sdk/anthropic': + specifier: ^3.0.85 + version: 3.0.85(zod@4.4.3) + '@ai-sdk/openai': + specifier: ^3.0.72 + version: 3.0.72(zod@4.4.3) + '@openrouter/ai-sdk-provider': + specifier: ^2.9.1 + version: 2.9.1(ai@6.0.207(zod@4.4.3))(zod@4.4.3) + ai: + specifier: ^6.0.207 + version: 6.0.207(zod@4.4.3) cheerio: specifier: ^1.2.0 version: 1.2.0 @@ -17,6 +29,9 @@ importers: linkedom: specifier: ^0.18.12 version: 0.18.12 + ollama-ai-provider-v2: + specifier: ^3.6.0 + version: 3.6.0(ai@6.0.207(zod@4.4.3))(zod@4.4.3) turndown: specifier: ^7.2.4 version: 7.2.4 @@ -59,10 +74,38 @@ importers: version: 8.60.1(eslint@10.4.1)(typescript@6.0.3) vitest: specifier: ^4.1.8 - version: 4.1.8(@types/node@25.9.1)(@vitest/coverage-v8@4.1.8)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.0)(tsx@4.22.4)(yaml@2.9.0)) + version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.8)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.0)(tsx@4.22.4)(yaml@2.9.0)) packages: + '@ai-sdk/anthropic@3.0.85': + resolution: {integrity: sha512-fNeDB644l5wbRNQU0FnI+F7UTtOenMnPtACfMPUJaS2zJfuBlseEa1TMg+otHkETZgaJB+6Na51NQEv0+m7czw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + '@ai-sdk/gateway@3.0.133': + resolution: {integrity: sha512-Ebs+7iS9zUgJu5B0RlxM2JmDWzq79Cpd6YdiqcCzB5qFdpfQJPUDiXutqlQP89F2XGjOdDeidulBTXUdXWzOxw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + '@ai-sdk/openai@3.0.72': + resolution: {integrity: sha512-ZF545m6pCXLUTERFfRCyfM7WsG4Nu/A+jlXQjSv7w22dGov02ssB0e1kviI3NLIERfRF/U+n2rKbFuUjzYY7CQ==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + '@ai-sdk/provider-utils@4.0.30': + resolution: {integrity: sha512-VO7I+vPffqI5sMnPoUq5DCSqKIgQIk/naJWRdQVpz2ma2zoprC/lqiJiUEl2s6DfvTD76TbhD3q39ROjlA6rGw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + '@ai-sdk/provider@3.0.10': + resolution: {integrity: sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==} + engines: {node: '>=18'} + '@babel/helper-string-parser@7.29.7': resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} engines: {node: '>=6.9.0'} @@ -327,6 +370,17 @@ packages: '@emnapi/core': ^1.7.1 '@emnapi/runtime': ^1.7.1 + '@openrouter/ai-sdk-provider@2.9.1': + resolution: {integrity: sha512-okgq07Vdkro4CB5INbfhwa0e6VR1HS7sidNcfHN/MeXLJvX1JmQCff/vem6tcxwT9r1avyFrXSlfv9B28D/Pag==} + engines: {node: '>=18'} + peerDependencies: + ai: ^6.0.0 + zod: ^3.25.0 || ^4.0.0 + + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} + engines: {node: '>=8.0.0'} + '@oxc-project/types@0.133.0': resolution: {integrity: sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==} @@ -514,6 +568,10 @@ packages: resolution: {integrity: sha512-EbGRQg4FhrmwLodl+t3JNAnXHWVr9Vp+Zl1QBZVPY4ByfkzIT8cX3K6QWODHtkIZqqJVEWvhHSx3v5PDHsaQag==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@vercel/oidc@3.2.0': + resolution: {integrity: sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==} + engines: {node: '>= 20'} + '@vitest/coverage-v8@4.1.8': resolution: {integrity: sha512-lt3kovsyHwYe00wq4D1ti0Z974fWj4NLp6siqiyEufUpyFwK9Yhi7rBhac9JL5aA0zoMrJqc4vYPZRUnI7l7nw==} peerDependencies: @@ -566,6 +624,12 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + ai@6.0.207: + resolution: {integrity: sha512-9rAHnqU+AvxyqO6WgiWj7hQENX6AprHXZWZEdBWwgnA854D2Mje/PiTmbcFqO+2Cck1lII0NLRQJY9lmdSorMw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + ajv@6.15.0: resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==} @@ -754,6 +818,10 @@ packages: eventemitter3@5.0.4: resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + eventsource-parser@3.1.0: + resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==} + engines: {node: '>=18.0.0'} + expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} @@ -878,6 +946,9 @@ packages: json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} @@ -1027,6 +1098,13 @@ packages: resolution: {integrity: sha512-AWGB9WFcRXOQs48Z/udjI5ZcZMHXwX8XPByNpOydgcGsDLIzjGizhoMWJyKAWze7AVW/2W1i+/gPX4YtKe5cyg==} engines: {node: '>=12.20.0'} + ollama-ai-provider-v2@3.6.0: + resolution: {integrity: sha512-1Om3FVJYhBwkAr5kQ+BX1s/tdVdtVdoFQWrX4PBQHDHPISyGt24CjhtggEjUYpy5ait0YeVfZwEpIYjgD8Ih7Q==} + engines: {node: '>=20'} + peerDependencies: + ai: ^5.0.0 || ^6.0.0 + zod: ^4.0.16 + onetime@7.0.0: resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} engines: {node: '>=18'} @@ -1350,8 +1428,41 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + zod@4.4.3: + resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} + snapshots: + '@ai-sdk/anthropic@3.0.85(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 3.0.10 + '@ai-sdk/provider-utils': 4.0.30(zod@4.4.3) + zod: 4.4.3 + + '@ai-sdk/gateway@3.0.133(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 3.0.10 + '@ai-sdk/provider-utils': 4.0.30(zod@4.4.3) + '@vercel/oidc': 3.2.0 + zod: 4.4.3 + + '@ai-sdk/openai@3.0.72(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 3.0.10 + '@ai-sdk/provider-utils': 4.0.30(zod@4.4.3) + zod: 4.4.3 + + '@ai-sdk/provider-utils@4.0.30(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 3.0.10 + '@standard-schema/spec': 1.1.0 + eventsource-parser: 3.1.0 + zod: 4.4.3 + + '@ai-sdk/provider@3.0.10': + dependencies: + json-schema: 0.4.0 + '@babel/helper-string-parser@7.29.7': {} '@babel/helper-validator-identifier@7.29.7': {} @@ -1529,6 +1640,13 @@ snapshots: '@tybys/wasm-util': 0.10.2 optional: true + '@openrouter/ai-sdk-provider@2.9.1(ai@6.0.207(zod@4.4.3))(zod@4.4.3)': + dependencies: + ai: 6.0.207(zod@4.4.3) + zod: 4.4.3 + + '@opentelemetry/api@1.9.1': {} + '@oxc-project/types@0.133.0': {} '@rolldown/binding-android-arm64@1.0.3': @@ -1699,6 +1817,8 @@ snapshots: '@typescript-eslint/types': 8.60.1 eslint-visitor-keys: 5.0.1 + '@vercel/oidc@3.2.0': {} + '@vitest/coverage-v8@4.1.8(vitest@4.1.8)': dependencies: '@bcoe/v8-coverage': 1.0.2 @@ -1711,7 +1831,7 @@ snapshots: obug: 2.1.2 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.8(@types/node@25.9.1)(@vitest/coverage-v8@4.1.8)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.0)(tsx@4.22.4)(yaml@2.9.0)) + vitest: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.8)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.0)(tsx@4.22.4)(yaml@2.9.0)) '@vitest/expect@4.1.8': dependencies: @@ -1763,6 +1883,14 @@ snapshots: acorn@8.16.0: {} + ai@6.0.207(zod@4.4.3): + dependencies: + '@ai-sdk/gateway': 3.0.133(zod@4.4.3) + '@ai-sdk/provider': 3.0.10 + '@ai-sdk/provider-utils': 4.0.30(zod@4.4.3) + '@opentelemetry/api': 1.9.1 + zod: 4.4.3 + ajv@6.15.0: dependencies: fast-deep-equal: 3.1.3 @@ -2005,6 +2133,8 @@ snapshots: eventemitter3@5.0.4: {} + eventsource-parser@3.1.0: {} + expect-type@1.3.0: {} fast-deep-equal@3.1.3: {} @@ -2100,6 +2230,8 @@ snapshots: json-schema-traverse@0.4.1: {} + json-schema@0.4.0: {} + json-stable-stringify-without-jsonify@1.0.1: {} keyv@4.5.4: @@ -2234,6 +2366,13 @@ snapshots: obug@2.1.2: {} + ollama-ai-provider-v2@3.6.0(ai@6.0.207(zod@4.4.3))(zod@4.4.3): + dependencies: + '@ai-sdk/provider': 3.0.10 + '@ai-sdk/provider-utils': 4.0.30(zod@4.4.3) + ai: 6.0.207(zod@4.4.3) + zod: 4.4.3 + onetime@7.0.0: dependencies: mimic-function: 5.0.1 @@ -2441,7 +2580,7 @@ snapshots: tsx: 4.22.4 yaml: 2.9.0 - vitest@4.1.8(@types/node@25.9.1)(@vitest/coverage-v8@4.1.8)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.0)(tsx@4.22.4)(yaml@2.9.0)): + vitest@4.1.8(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.8)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.0)(tsx@4.22.4)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.8 '@vitest/mocker': 4.1.8(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.0)(tsx@4.22.4)(yaml@2.9.0)) @@ -2464,6 +2603,7 @@ snapshots: vite: 8.0.16(@types/node@25.9.1)(esbuild@0.28.0)(tsx@4.22.4)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: + '@opentelemetry/api': 1.9.1 '@types/node': 25.9.1 '@vitest/coverage-v8': 4.1.8(vitest@4.1.8) transitivePeerDependencies: @@ -2502,3 +2642,5 @@ snapshots: optional: true yocto-queue@0.1.0: {} + + zod@4.4.3: {} diff --git a/src/cli.test.ts b/src/cli.test.ts index 8c1da08..e89ef49 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -24,6 +24,7 @@ import { loadOrCreateConfigFile } from "./setup.ts"; import { loadPlugins } from "./plugin-loader.ts"; import { exit } from "./exit.ts"; import type { + AskPlugin, FetchPlugin, PluginContext, PluginTypeDeclaration, @@ -39,6 +40,7 @@ const contextMatcher = expect.objectContaining({ let searchFn: Mock; let fetchFn: Mock; +let askFn: Mock; let plugins: PluginTypeDeclaration[]; let config: SibylConfig; @@ -48,13 +50,15 @@ beforeEach(() => { searchFn = vi.fn(async () => "search result"); fetchFn = vi.fn(async () => "fetch result"); + askFn = vi.fn(async () => "ask result"); const searchPlugin: SearchPlugin = { name: "test-search", type: "search", fn: searchFn }; const fetchPlugin: FetchPlugin = { name: "test-fetch", type: "fetch", fn: fetchFn }; - plugins = [searchPlugin, fetchPlugin]; + const askPlugin: AskPlugin = { name: "test-ask", type: "ask", fn: askFn }; + plugins = [searchPlugin, fetchPlugin, askPlugin]; config = { - plugins: { search: "test-search", fetch: "test-fetch" }, + plugins: { search: "test-search", fetch: "test-fetch", ask: "test-ask" }, variables: [], }; @@ -123,6 +127,32 @@ describe("dispatch & argument validation", () => { expect(console.log).toHaveBeenCalledWith("fetch result"); }); + it.each([ + { argv: ["ask"], label: "no url" }, + { argv: ["ask", "https://vite.dev"], label: "url but no question" }, + { argv: ["ask", "https://vite.dev", " "], label: "whitespace-only question" }, + ])("errors when ask is missing url or question ($label)", async ({ argv }) => { + await expect(main(argv)).rejects.toThrow("process.exit"); + + expect(console.error).toHaveBeenCalledWith('Usage: sibyl ask ""'); + expect(exit).toHaveBeenCalledWith(1); + }); + + it("errors when ask url is invalid", async () => { + await expect(main(["ask", "not-a-url", "some question"])).rejects.toThrow("process.exit"); + + expect(console.error).toHaveBeenCalledWith("Invalid URL: not-a-url"); + expect(exit).toHaveBeenCalledWith(1); + expect(askFn).not.toHaveBeenCalled(); + }); + + it("joins and trims the question before passing it to the ask plugin", async () => { + await main(["ask", "https://vite.dev", "how", "to", "start"]); + + expect(askFn).toHaveBeenCalledWith("https://vite.dev", "how to start", contextMatcher); + expect(console.log).toHaveBeenCalledWith("ask result"); + }); + it("errors and prints help on an unknown command", async () => { await expect(main(["bogus"])).rejects.toThrow("process.exit"); @@ -173,9 +203,14 @@ describe("handleSearch", () => { expect(context).not.toBeNull(); expect(context.allPlugins).toBe(plugins); - expect(context.configuredPlugins).toEqual({ search: plugins[0], fetch: plugins[1] }); + expect(context.configuredPlugins).toEqual({ + search: plugins[0], + fetch: plugins[1], + ask: plugins[2], + }); expect(context.getPlugin("test-search")).toBe(plugins[0]); expect(context.getPlugin("test-fetch")).toBe(plugins[1]); + expect(context.getPlugin("test-ask")).toBe(plugins[2]); expect(context.getPlugin("nope")).toBeNull(); }); @@ -236,3 +271,47 @@ describe("handleFetch", () => { expect(exit).toHaveBeenCalledWith(1); }); }); + +describe("handleAsk", () => { + it("errors when no ask plugin is configured", async () => { + vi.mocked(loadOrCreateConfigFile).mockReturnValue({ plugins: {}, variables: [] }); + + await expect(main(["ask", "https://vite.dev", "q"])).rejects.toThrow("process.exit"); + + expect(console.error).toHaveBeenCalledWith( + "No ask plugin configured in `~/.sibyl/config.json`", + ); + expect(exit).toHaveBeenCalledWith(1); + }); + + it("errors when the configured ask plugin is not loaded", async () => { + vi.mocked(loadOrCreateConfigFile).mockReturnValue({ + plugins: { ask: "missing-plugin" }, + variables: [], + }); + + await expect(main(["ask", "https://vite.dev", "q"])).rejects.toThrow("process.exit"); + + expect(console.error).toHaveBeenCalledWith("Configured ask plugin `missing-plugin` not found"); + expect(exit).toHaveBeenCalledWith(1); + }); + + it("logs the plugin result on success", async () => { + await main(["ask", "https://vite.dev", "q"]); + + expect(askFn).toHaveBeenCalledWith("https://vite.dev", "q", contextMatcher); + expect(console.log).toHaveBeenCalledWith("ask result"); + expect(exit).not.toHaveBeenCalled(); + }); + + it("errors when the plugin rejects", async () => { + askFn.mockRejectedValue(new Error("boom")); + + await expect(main(["ask", "https://vite.dev", "q"])).rejects.toThrow("process.exit"); + + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("Error asking using test-ask:"), + ); + expect(exit).toHaveBeenCalledWith(1); + }); +}); diff --git a/src/cli.ts b/src/cli.ts index 6510a69..41b1cd2 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -6,6 +6,7 @@ */ import { loadOrCreateConfigDir, loadOrCreateConfigFile, loadOrCreatePluginsDir } from "./setup.ts"; import type { + AskPlugin, FetchPlugin, PluginContext, PluginType, @@ -64,6 +65,23 @@ export async function main(argv: string[]): Promise { await handleFetch(plugins, config, url, context); break; } + case "ask": { + const url = rest[0]?.trim(); + const query = rest.slice(1).join(" ").trim(); + + if (!url || !query) { + console.error('Usage: sibyl ask ""'); + exit(1); + } + + if (!isValidHttpUrl(url)) { + console.error(`Invalid URL: ${url}`); + exit(1); + } + + await handleAsk(plugins, config, url, query, context); + break; + } case "--version": case "version": console.log("sibyl 0.1.0"); @@ -152,6 +170,38 @@ async function handleFetch( } } +async function handleAsk( + plugins: PluginTypeDeclaration[], + config: SibylConfig, + url: string, + query: string, + context: PluginContext, +): Promise { + const askPluginName = config.plugins.ask; + + if (!askPluginName) { + console.error("No ask plugin configured in `~/.sibyl/config.json`"); + exit(1); + } + + const askPlugin = plugins.find( + (plugin) => plugin.type === "ask" && plugin.name === askPluginName, + ) as AskPlugin; + + if (!askPlugin) { + console.error(`Configured ask plugin \`${askPluginName}\` not found`); + exit(1); + } + + try { + const result = await askPlugin.fn(url, query, context); + console.log(result); + } catch (error) { + console.error(`Error asking using ${askPlugin.name}: ${error}`); + exit(1); + } +} + function printHelp(): void { console.log(`sibyl - CLI tool @@ -159,14 +209,16 @@ Usage: sibyl [options] Commands: - search Search the web - fetch Fetch the content of a URL - help Show this help - version Show version + search Search the web + fetch Fetch the content of a URL + ask Answer a question using a URL's content via an LLM + help Show this help + version Show version Examples: - sibyl search "react vite bootstrap" + sibyl search "react vite" sibyl fetch https://vite.dev/guide + sibyl ask https://vite.dev/guide "how do I start a react project with vite?" `); } diff --git a/src/plugins/builtin-ai-ask/main.test.ts b/src/plugins/builtin-ai-ask/main.test.ts new file mode 100644 index 0000000..8e169f2 --- /dev/null +++ b/src/plugins/builtin-ai-ask/main.test.ts @@ -0,0 +1,127 @@ +/* + * Author: Jamius Siam + * Since: 18/06/2026 + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { generateText } from "ai"; +import { SilbylPlugin } from "./main.ts"; +import type { FetchPlugin, PluginContext } from "../../@types/plugin.ts"; + +vi.mock("ai", () => ({ generateText: vi.fn() })); +vi.mock("@ai-sdk/openai", () => ({ + createOpenAI: vi.fn(() => (modelName: string) => ({ provider: "openai", modelName })), +})); + +const askFn = SilbylPlugin.fn; +const mockedGenerateText = vi.mocked(generateText); + +function resolveAnswer(text: string): void { + // @ts-expect-error Complicated type in mocked return, ignoring for now. + mockedGenerateText.mockResolvedValue({ text }); +} + +function makeContext(fetchFn?: (url: string) => Promise): PluginContext { + const fetchPlugin: FetchPlugin | undefined = fetchFn + ? { name: "mock-fetch", type: "fetch", fn: (url) => fetchFn(url) } + : undefined; + + return { + configuredPlugins: fetchPlugin ? { fetch: fetchPlugin } : {}, + allPlugins: fetchPlugin ? [fetchPlugin] : [], + getPlugin: () => null, + }; +} + +const okContext = makeContext(async () => "page body content"); + +let envSnapshot: NodeJS.ProcessEnv; + +beforeEach(() => { + envSnapshot = { ...process.env }; + process.env.SIBYL_AI_PROVIDER = "openai"; + process.env.SIBYL_MODEL_NAME = "gpt-test"; + process.env.OPENAI_API_KEY = "test-key"; + delete process.env.OLLAMA_BASE_URL; + + mockedGenerateText.mockReset(); + resolveAnswer("the answer"); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + for (const key of Object.keys(process.env)) { + if (!(key in envSnapshot)) delete process.env[key]; + } + Object.assign(process.env, envSnapshot); +}); + +describe("builtin-ai-ask", () => { + it("throws when no `fetch` plugin is configured", async () => { + await expect(askFn("https://a.com", "what?", makeContext())).rejects.toThrow( + "No `fetch` plugin configured.", + ); + }); + + it("throws when `SIBYL_AI_PROVIDER` is missing or invalid", async () => { + process.env.SIBYL_AI_PROVIDER = "gemini"; + + await expect(askFn("https://a.com", "what?", okContext)).rejects.toThrow( + "Missing or invalid `SIBYL_AI_PROVIDER` environment variable.", + ); + }); + + it("throws when `SIBYL_MODEL_NAME` is missing", async () => { + delete process.env.SIBYL_MODEL_NAME; + + await expect(askFn("https://a.com", "what?", okContext)).rejects.toThrow( + "Missing `SIBYL_MODEL_NAME` environment variable.", + ); + }); + + it("throws when the provider API key is missing", async () => { + delete process.env.OPENAI_API_KEY; + + await expect(askFn("https://a.com", "what?", okContext)).rejects.toThrow( + "Missing `OPENAI_API_KEY` environment variable.", + ); + }); + + it("throws when the fetch service is unreachable", async () => { + const context = makeContext(() => Promise.reject(new Error("ECONNREFUSED"))); + + await expect(askFn("https://a.com", "what?", context)).rejects.toThrow( + /Failed to fetch `https:\/\/a\.com` using the configured fetch plugin `mock-fetch`/, + ); + expect(mockedGenerateText).not.toHaveBeenCalled(); + }); + + it("throws when the fetched content is empty", async () => { + const context = makeContext(async () => " "); + + await expect(askFn("https://a.com", "what?", context)).rejects.toThrow( + "No content fetched from: https://a.com", + ); + }); + + it("throws when the AI provider is unreachable", async () => { + mockedGenerateText.mockRejectedValue(new Error("fetch failed")); + + await expect(askFn("https://a.com", "what?", okContext)).rejects.toThrow( + /AI provider `openai` failed to answer/, + ); + }); + + it("sends the system prompt, content and question, with an abort signal", async () => { + const answer = await askFn("https://a.com", "what is it about?", okContext); + + expect(answer).toBe("the answer"); + expect(mockedGenerateText).toHaveBeenCalledWith( + expect.objectContaining({ + system: expect.stringContaining("Reply with just the answer"), + prompt: expect.stringContaining("page body content"), + abortSignal: expect.any(AbortSignal), + }), + ); + expect(mockedGenerateText.mock.calls[0]?.[0]?.prompt).toContain("what is it about?"); + }); +}); diff --git a/src/plugins/builtin-ai-ask/main.ts b/src/plugins/builtin-ai-ask/main.ts new file mode 100644 index 0000000..9864dca --- /dev/null +++ b/src/plugins/builtin-ai-ask/main.ts @@ -0,0 +1,117 @@ +/* + * Author: Jamius Siam + * Since: 18/06/2026 + */ +import type { AskPlugin, FetchPlugin, PluginContext } from "../../@types/plugin.ts"; + +const REQUEST_TIMEOUT_MS = 30_000; + +const SYSTEM_PROMPT = + "You answer the user's question using only the provided web page content. " + + "Reply with just the answer, with no preamble and without " + + "restating the question. If the content does not contain the answer, say so plainly."; + +type AiProvider = "openai" | "anthropic" | "ollama" | "openrouter"; + +function isAiProvider(value: string): value is AiProvider { + return ( + value === "openai" || value === "anthropic" || value === "ollama" || value === "openrouter" + ); +} + +function requireEnv(name: string): string { + const value = process.env[name]; + if (!value) { + throw new Error(`Missing \`${name}\` environment variable.`); + } + return value; +} + +async function buildModel(provider: AiProvider, modelName: string) { + switch (provider) { + case "openai": { + const apiKey = requireEnv("OPENAI_API_KEY"); + const { createOpenAI } = await import("@ai-sdk/openai"); + return createOpenAI({ apiKey })(modelName); + } + case "anthropic": { + const apiKey = requireEnv("ANTHROPIC_API_KEY"); + const { createAnthropic } = await import("@ai-sdk/anthropic"); + return createAnthropic({ apiKey })(modelName); + } + case "openrouter": { + const apiKey = requireEnv("OPENROUTER_API_KEY"); + const { createOpenRouter } = await import("@openrouter/ai-sdk-provider"); + return createOpenRouter({ apiKey })(modelName); + } + case "ollama": { + const baseURL = process.env.OLLAMA_BASE_URL ?? "http://localhost:11434/api"; + const { createOllama } = await import("ollama-ai-provider-v2"); + return createOllama({ baseURL })(modelName); + } + } +} + +async function askFn(src: string, query: string, context: PluginContext): Promise { + const fetchPlugin = context.configuredPlugins.fetch as FetchPlugin | undefined; + if (!fetchPlugin) { + throw new Error( + "No `fetch` plugin configured. The `ask` plugin reads the URL through the configured " + + "fetch plugin — set `plugins.fetch` in `~/.sibyl/config.json`.", + ); + } + + const provider = (process.env.SIBYL_AI_PROVIDER ?? "").trim().toLowerCase(); + if (!isAiProvider(provider)) { + throw new Error( + "Missing or invalid `SIBYL_AI_PROVIDER` environment variable. " + + "Expected one of: openai, anthropic, ollama, openrouter.", + ); + } + + const modelName = process.env.SIBYL_MODEL_NAME?.trim(); + if (!modelName) { + throw new Error("Missing `SIBYL_MODEL_NAME` environment variable."); + } + + const model = await buildModel(provider, modelName); + + let content: string; + try { + content = (await fetchPlugin.fn(src, context)).trim(); + } catch (err) { + throw new Error( + `Failed to fetch \`${src}\` using the configured fetch plugin \`${fetchPlugin.name}\`: ${err}`, + { cause: err }, + ); + } + + if (!content) { + throw new Error(`No content fetched from: ${src}`); + } + + const { generateText } = await import("ai"); + + try { + const { text } = await generateText({ + model, + system: SYSTEM_PROMPT, + prompt: `Web page content:\n\n${content}\n\nQuestion: ${query}`, + abortSignal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + + return text.trim(); + } catch (err) { + throw new Error( + `AI provider \`${provider}\` failed to answer ` + + `(is it reachable and is \`${modelName}\` a valid model?): ${err}`, + { cause: err }, + ); + } +} + +export const SilbylPlugin: AskPlugin = { + name: "builtin-ai-ask", + type: "ask", + fn: askFn, +}; diff --git a/src/plugins/config.test.ts b/src/plugins/config.test.ts index 7d4228f..628d117 100644 --- a/src/plugins/config.test.ts +++ b/src/plugins/config.test.ts @@ -21,6 +21,7 @@ describe("getBuiltinPlugins", () => { ["builtin-alterlab-search", "search"], ["builtin-firecrawl-search", "search"], ["builtin-firecrawl-fetch", "fetch"], + ["builtin-ai-ask", "ask"], ]); for (const plugin of plugins) { diff --git a/src/plugins/config.ts b/src/plugins/config.ts index babd108..606e86d 100644 --- a/src/plugins/config.ts +++ b/src/plugins/config.ts @@ -14,6 +14,7 @@ import { SilbylPlugin as firecrawlSearch } from "./builtin-firecrawl-search/main import { SilbylPlugin as firecrawlFetch } from "./builtin-firecrawl-fetch/main.ts"; import { SilbylPlugin as parseHtmlToMd } from "./builtin-parse-htmlToMd/main.ts"; import { SilbylPlugin as searxngSearch } from "./builtin-searxng-search/main.ts"; +import { SilbylPlugin as aiAsk } from "./builtin-ai-ask/main.ts"; export function getBuiltinPlugins(): PluginTypeDeclaration[] { return [ @@ -28,5 +29,6 @@ export function getBuiltinPlugins(): PluginTypeDeclaration[] { alterlabSearch, firecrawlSearch, firecrawlFetch, + aiAsk, ]; } diff --git a/src/setup.test.ts b/src/setup.test.ts index e77557c..1bcc301 100644 --- a/src/setup.test.ts +++ b/src/setup.test.ts @@ -26,8 +26,14 @@ const DEFAULT_CONFIG: SibylConfig = { search: "builtin-searxng-search", fetch: "builtin-crawl4ai-fetch", parse: "builtin-parse-htmlToMd", + ask: "builtin-ai-ask", }, - variables: [], + variables: [ + { + name: "SIBYL_SHOW_SEARCH_DESCRIPTION", + value: "true", + }, + ], }; let home: string; diff --git a/src/setup.ts b/src/setup.ts index 942fc20..712fad0 100644 --- a/src/setup.ts +++ b/src/setup.ts @@ -57,8 +57,14 @@ export function writeDefaultSibylConfig(): void { search: "builtin-searxng-search", fetch: "builtin-crawl4ai-fetch", parse: "builtin-parse-htmlToMd", + ask: "builtin-ai-ask", }, - variables: [], + variables: [ + { + name: "SIBYL_SHOW_SEARCH_DESCRIPTION", + value: "true", + }, + ], }; fs.writeFileSync(configFile, JSON.stringify(sibylConfig, null, 2));