diff --git a/.github/workflows/manual-strategy-switch.yml b/.github/workflows/manual-strategy-switch.yml index aba04d2..be6f8be 100644 --- a/.github/workflows/manual-strategy-switch.yml +++ b/.github/workflows/manual-strategy-switch.yml @@ -126,6 +126,7 @@ jobs: switch: name: Build and apply runtime switch runs-on: ubuntu-latest + timeout-minutes: 15 environment: runtime-strategy-switch permissions: contents: read diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 6f0f91a..c52246c 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -4,9 +4,13 @@ on: push: pull_request: +permissions: + contents: read + jobs: validate: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v6 with: @@ -30,6 +34,8 @@ jobs: run: python3 scripts/runtime_settings.py validate - name: Run unit tests run: python3 -m unittest discover -s tests -v + - name: Report internal dependency matrix + run: python3 scripts/check_internal_dependency_matrix.py --projects-root .. --json - name: Validate strategy switch web assets run: | set -euo pipefail diff --git a/.node-version b/.node-version new file mode 100644 index 0000000..2bd5a0a --- /dev/null +++ b/.node-version @@ -0,0 +1 @@ +22 diff --git a/internal_dependency_matrix.json b/internal_dependency_matrix.json new file mode 100644 index 0000000..3694b9f --- /dev/null +++ b/internal_dependency_matrix.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "dependencies": [ + { + "consumer_repo": "BinancePlatform", + "path": "requirements.txt", + "package": "quant-platform-kit", + "source_repo": "QuantPlatformKit", + "ref": "v0.7.35" + }, + { + "consumer_repo": "BinancePlatform", + "path": "requirements.txt", + "package": "crypto-strategies", + "source_repo": "CryptoStrategies", + "ref": "v0.4.8" + }, + { + "consumer_repo": "BinancePlatform", + "path": "requirements-lock.txt", + "package": "quant-platform-kit", + "source_repo": "QuantPlatformKit", + "ref": "v0.7.35" + }, + { + "consumer_repo": "BinancePlatform", + "path": "requirements-lock.txt", + "package": "crypto-strategies", + "source_repo": "CryptoStrategies", + "ref": "v0.4.8" + }, + { + "consumer_repo": "CharlesSchwabPlatform", + "path": "requirements.txt", + "package": "quant-platform-kit", + "source_repo": "QuantPlatformKit", + "ref": "3b6a0a9bedde72773e188041e0dc48516b38aadc" + }, + { + "consumer_repo": "CharlesSchwabPlatform", + "path": "requirements.txt", + "package": "us-equity-strategies", + "source_repo": "UsEquityStrategies", + "ref": "8278048366f1cd83e29e0c921e4048e7e25ae227" + }, + { + "consumer_repo": "CryptoStrategies", + "path": "pyproject.toml", + "package": "quant-platform-kit", + "source_repo": "QuantPlatformKit", + "ref": "v0.7.35" + }, + { + "consumer_repo": "FirstradePlatform", + "path": "pyproject.toml", + "package": "quant-platform-kit", + "source_repo": "QuantPlatformKit", + "ref": "3b6a0a9bedde72773e188041e0dc48516b38aadc" + }, + { + "consumer_repo": "FirstradePlatform", + "path": "pyproject.toml", + "package": "us-equity-strategies", + "source_repo": "UsEquityStrategies", + "ref": "8278048366f1cd83e29e0c921e4048e7e25ae227" + }, + { + "consumer_repo": "FirstradePlatform", + "path": "requirements.txt", + "package": "quant-platform-kit", + "source_repo": "QuantPlatformKit", + "ref": "3b6a0a9bedde72773e188041e0dc48516b38aadc" + }, + { + "consumer_repo": "FirstradePlatform", + "path": "requirements.txt", + "package": "us-equity-strategies", + "source_repo": "UsEquityStrategies", + "ref": "8278048366f1cd83e29e0c921e4048e7e25ae227" + }, + { + "consumer_repo": "HkEquityStrategies", + "path": "pyproject.toml", + "package": "quant-platform-kit", + "source_repo": "QuantPlatformKit", + "ref": "023641c88506c732624a7329e48b51b9dbbe3c2a" + }, + { + "consumer_repo": "InteractiveBrokersPlatform", + "path": "requirements.txt", + "package": "quant-platform-kit", + "source_repo": "QuantPlatformKit", + "ref": "3b6a0a9bedde72773e188041e0dc48516b38aadc" + }, + { + "consumer_repo": "InteractiveBrokersPlatform", + "path": "requirements.txt", + "package": "us-equity-strategies", + "source_repo": "UsEquityStrategies", + "ref": "8278048366f1cd83e29e0c921e4048e7e25ae227" + }, + { + "consumer_repo": "InteractiveBrokersPlatform", + "path": "requirements.txt", + "package": "hk-equity-strategies", + "source_repo": "HkEquityStrategies", + "ref": "b690fcfd1e26648840723a5ab8b12c873f038b9b" + }, + { + "consumer_repo": "LongBridgePlatform", + "path": "requirements.txt", + "package": "quant-platform-kit", + "source_repo": "QuantPlatformKit", + "ref": "023641c88506c732624a7329e48b51b9dbbe3c2a" + }, + { + "consumer_repo": "LongBridgePlatform", + "path": "requirements.txt", + "package": "us-equity-strategies", + "source_repo": "UsEquityStrategies", + "ref": "7d35772d1125b534d0bcca557cb6dbaf28914719" + }, + { + "consumer_repo": "LongBridgePlatform", + "path": "requirements.txt", + "package": "hk-equity-strategies", + "source_repo": "HkEquityStrategies", + "ref": "2e0075004239e7ede7ba256763a3441d4ec4ca73" + }, + { + "consumer_repo": "UsEquitySnapshotPipelines", + "path": "pyproject.toml", + "package": "quant-strategy-plugins", + "source_repo": "QuantStrategyPlugins", + "ref": "v0.1.4" + }, + { + "consumer_repo": "UsEquitySnapshotPipelines", + "path": "pyproject.toml", + "package": "us-equity-strategies", + "source_repo": "UsEquityStrategies", + "ref": "7d35772d1125b534d0bcca557cb6dbaf28914719" + }, + { + "consumer_repo": "UsEquityStrategies", + "path": "pyproject.toml", + "package": "quant-platform-kit", + "source_repo": "QuantPlatformKit", + "ref": "023641c88506c732624a7329e48b51b9dbbe3c2a" + } + ] +} diff --git a/scripts/check_internal_dependency_matrix.py b/scripts/check_internal_dependency_matrix.py new file mode 100644 index 0000000..286c100 --- /dev/null +++ b/scripts/check_internal_dependency_matrix.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Report QuantStrategyLab internal git dependency pin drift.""" + +from __future__ import annotations + +import argparse +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_MATRIX_PATH = ROOT / "internal_dependency_matrix.json" +DEFAULT_PROJECTS_ROOT = ROOT.parent +DEPENDENCY_PATTERN = re.compile( + r"(?P[A-Za-z0-9_.-]+)\s*@\s*" + r"git\+https://github\.com/QuantStrategyLab/" + r"(?P[A-Za-z0-9_.-]+)\.git@(?P[A-Za-z0-9_.-]+)" +) + + +@dataclass(frozen=True) +class DependencyPin: + consumer_repo: str + path: str + package: str + source_repo: str + ref: str + + @property + def key(self) -> tuple[str, str, str, str]: + return (self.consumer_repo, self.path, self.package, self.source_repo) + + def label(self) -> str: + return f"{self.consumer_repo}/{self.path}:{self.package}->{self.source_repo}" + + +@dataclass(frozen=True) +class MatrixReport: + checked_files: int + missing_files: list[str] + issues: list[str] + + @property + def ok(self) -> bool: + return not self.issues + + +def load_matrix(path: Path) -> list[DependencyPin]: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("schema_version") != 1: + raise ValueError("internal dependency matrix schema_version must be 1") + dependencies = payload.get("dependencies") + if not isinstance(dependencies, list): + raise ValueError("internal dependency matrix dependencies must be a list") + pins: list[DependencyPin] = [] + for index, item in enumerate(dependencies): + if not isinstance(item, dict): + raise ValueError(f"dependencies[{index}] must be an object") + pins.append( + DependencyPin( + consumer_repo=_required_string(item, "consumer_repo", index), + path=_required_string(item, "path", index), + package=_required_string(item, "package", index), + source_repo=_required_string(item, "source_repo", index), + ref=_required_string(item, "ref", index), + ) + ) + return pins + + +def _required_string(item: dict[str, Any], key: str, index: int) -> str: + value = item.get(key) + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"dependencies[{index}].{key} must be a non-empty string") + return value.strip() + + +def parse_dependency_pins(consumer_repo: str, path: str, text: str) -> list[DependencyPin]: + return [ + DependencyPin( + consumer_repo=consumer_repo, + path=path, + package=match.group("package"), + source_repo=match.group("source_repo"), + ref=match.group("ref"), + ) + for match in DEPENDENCY_PATTERN.finditer(text) + ] + + +def check_matrix(*, matrix_pins: list[DependencyPin], projects_root: Path) -> MatrixReport: + expected_by_file: dict[tuple[str, str], list[DependencyPin]] = {} + for pin in matrix_pins: + expected_by_file.setdefault((pin.consumer_repo, pin.path), []).append(pin) + + issues: list[str] = [] + missing_files: list[str] = [] + checked_files = 0 + for (consumer_repo, relative_path), expected_pins in sorted(expected_by_file.items()): + path = projects_root / consumer_repo / relative_path + if not path.exists(): + missing_files.append(f"{consumer_repo}/{relative_path}") + continue + checked_files += 1 + actual_pins = parse_dependency_pins(consumer_repo, relative_path, path.read_text(encoding="utf-8")) + actual_by_key = {pin.key: pin for pin in actual_pins} + expected_by_key = {pin.key: pin for pin in expected_pins} + + for key, expected in sorted(expected_by_key.items()): + actual = actual_by_key.get(key) + if actual is None: + issues.append(f"missing {expected.label()} expected @{expected.ref}") + elif actual.ref != expected.ref: + issues.append(f"ref mismatch {expected.label()}: expected @{expected.ref}, found @{actual.ref}") + + for key, actual in sorted(actual_by_key.items()): + if key not in expected_by_key: + issues.append(f"untracked internal dependency {actual.label()} @{actual.ref}") + + return MatrixReport(checked_files=checked_files, missing_files=missing_files, issues=issues) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Report QuantStrategyLab internal dependency pin drift.") + parser.add_argument("--matrix", type=Path, default=DEFAULT_MATRIX_PATH) + parser.add_argument("--projects-root", type=Path, default=DEFAULT_PROJECTS_ROOT) + parser.add_argument("--json", action="store_true", help="Print machine-readable report.") + parser.add_argument("--strict", action="store_true", help="Exit non-zero when drift is detected.") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + report = check_matrix(matrix_pins=load_matrix(args.matrix), projects_root=args.projects_root) + if args.json: + print( + json.dumps( + { + "checked_files": report.checked_files, + "missing_files": report.missing_files, + "issues": report.issues, + "ok": report.ok, + }, + ensure_ascii=False, + indent=2, + ) + ) + else: + print(f"checked_files={report.checked_files}") + if report.missing_files: + print("missing_files:") + for item in report.missing_files: + print(f"- {item}") + if report.issues: + print("issues:") + for issue in report.issues: + print(f"- {issue}") + if report.ok: + print("internal dependency matrix is current") + return 1 if args.strict and not report.ok else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/strategy_switch_worker_validation.mjs b/tests/strategy_switch_worker_validation.mjs index 447cced..80ed468 100644 --- a/tests/strategy_switch_worker_validation.mjs +++ b/tests/strategy_switch_worker_validation.mjs @@ -111,6 +111,32 @@ assert.equal( await __test.withTimeout(new Promise(() => {}), 1, "fallback"), "fallback", ); +const timeoutFetchResponse = await __test.fetchWithTimeout( + "https://api.github.test/user", + { headers: { Accept: "application/json" } }, + 100, + async (_resource, init) => { + assert.ok(init.signal instanceof AbortSignal); + assert.equal(init.headers.Accept, "application/json"); + return new Response('{"ok":true}', { status: 200 }); + }, +); +assert.equal(timeoutFetchResponse.status, 200); +await assert.rejects( + () => __test.fetchWithTimeout( + "https://api.github.test/slow", + {}, + 1, + (_resource, init) => new Promise((_resolve, reject) => { + init.signal.addEventListener("abort", () => { + const error = new Error("aborted"); + error.name = "AbortError"; + reject(error); + }); + }), + ), + /GitHub request timed out/, +); function captureError(fn) { try { diff --git a/tests/test_internal_dependency_matrix.py b/tests/test_internal_dependency_matrix.py new file mode 100644 index 0000000..99f73e5 --- /dev/null +++ b/tests/test_internal_dependency_matrix.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "scripts" / "check_internal_dependency_matrix.py" +SPEC = importlib.util.spec_from_file_location("check_internal_dependency_matrix", MODULE_PATH) +check_internal_dependency_matrix = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = check_internal_dependency_matrix +SPEC.loader.exec_module(check_internal_dependency_matrix) + + +class InternalDependencyMatrixTest(unittest.TestCase): + def test_parse_dependency_pins_from_requirements_and_pyproject_text(self): + text = """ +quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@v0.7.35 + "us-equity-strategies @ git+https://github.com/QuantStrategyLab/UsEquityStrategies.git@abc123", +""" + + pins = check_internal_dependency_matrix.parse_dependency_pins("ExamplePlatform", "requirements.txt", text) + + self.assertEqual([pin.package for pin in pins], ["quant-platform-kit", "us-equity-strategies"]) + self.assertEqual([pin.source_repo for pin in pins], ["QuantPlatformKit", "UsEquityStrategies"]) + self.assertEqual([pin.ref for pin in pins], ["v0.7.35", "abc123"]) + + def test_check_matrix_reports_ref_drift_and_untracked_dependency(self): + projects_root = self._make_projects_root( + { + "ExamplePlatform/requirements.txt": "\n".join( + [ + "quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@v0.7.36", + "extra-package @ git+https://github.com/QuantStrategyLab/ExtraPackage.git@v1.0.0", + ] + ) + } + ) + expected = [ + check_internal_dependency_matrix.DependencyPin( + consumer_repo="ExamplePlatform", + path="requirements.txt", + package="quant-platform-kit", + source_repo="QuantPlatformKit", + ref="v0.7.35", + ) + ] + + report = check_internal_dependency_matrix.check_matrix(matrix_pins=expected, projects_root=projects_root) + + self.assertEqual(report.checked_files, 1) + self.assertEqual(report.missing_files, []) + self.assertIn( + "ref mismatch ExamplePlatform/requirements.txt:quant-platform-kit->QuantPlatformKit: " + "expected @v0.7.35, found @v0.7.36", + report.issues, + ) + self.assertIn( + "untracked internal dependency ExamplePlatform/requirements.txt:extra-package->ExtraPackage @v1.0.0", + report.issues, + ) + + def test_current_matrix_matches_local_workspace(self): + matrix_pins = check_internal_dependency_matrix.load_matrix( + ROOT / "internal_dependency_matrix.json" + ) + + report = check_internal_dependency_matrix.check_matrix( + matrix_pins=matrix_pins, + projects_root=ROOT.parent, + ) + + expected_paths = sorted({f"{pin.consumer_repo}/{pin.path}" for pin in matrix_pins}) + if report.missing_files: + self.assertEqual(sorted(report.missing_files), expected_paths) + self.assertEqual(report.checked_files, 0) + self.assertEqual(report.issues, []) + return + + self.assertEqual(report.missing_files, []) + self.assertEqual(report.issues, []) + + def _make_projects_root(self, files: dict[str, str]) -> Path: + import tempfile + + root = Path(tempfile.mkdtemp()) + for relative_path, text in files.items(): + path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return root + + +if __name__ == "__main__": + unittest.main() diff --git a/web/strategy-switch-console/worker.js b/web/strategy-switch-console/worker.js index 143b441..bc8e2d8 100644 --- a/web/strategy-switch-console/worker.js +++ b/web/strategy-switch-console/worker.js @@ -12,6 +12,7 @@ const STRATEGY_PROFILES_KEY = "strategy_profiles"; const AUDIT_LOG_KEY = "audit_log"; const AUDIT_LOG_LIMIT = 50; const CURRENT_STRATEGIES_TIMEOUT_MS = 3500; +const GITHUB_API_TIMEOUT_MS = 8000; const SUPPORTED_PLATFORMS = ["longbridge", "ibkr", "schwab", "firstrade"]; const SUPPORTED_STRATEGY_DOMAINS = ["us_equity", "hk_equity"]; @@ -125,7 +126,7 @@ async function finishLogin(request, env) { return html(renderMessage("登录失败", "OAuth state 校验失败，请重新登录。"), 400, clearOAuthCookie()); } - const tokenResponse = await fetch("https://github.com/login/oauth/access_token", { + const tokenResponse = await fetchWithTimeout("https://github.com/login/oauth/access_token", { method: "POST", headers: { Accept: "application/json", @@ -143,7 +144,7 @@ async function finishLogin(request, env) { return html(renderMessage("登录失败", "GitHub token exchange 失败。"), 502, clearOAuthCookie()); } - const userResponse = await fetch("https://api.github.com/user", { + const userResponse = await fetchWithTimeout("https://api.github.com/user", { headers: githubHeaders(tokenPayload.access_token), }); const user = await userResponse.json(); @@ -555,6 +556,19 @@ function withTimeout(promise, timeoutMs, fallback) { return Promise.race([promise, timeout]).finally(() => clearTimeout(timeoutId)); } +async function fetchWithTimeout(resource, init = {}, timeoutMs = GITHUB_API_TIMEOUT_MS, fetchImpl = fetch) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetchImpl(resource, { ...init, signal: controller.signal }); + } catch (error) { + if (error?.name === "AbortError") throw new Error("GitHub request timed out"); + throw error; + } finally { + clearTimeout(timeoutId); + } +} + async function resolveCurrentStrategyForAccount({ platform, option, optionsCount, repository, readVariable }) { const serviceTargetsValue = await readVariable(repository, "repository", "", "CLOUD_RUN_SERVICE_TARGETS_JSON"); const serviceTarget = runtimeTargetFromServiceTargets(serviceTargetsValue, platform, option); @@ -654,7 +668,7 @@ async function dispatchSwitch(request, env) { const repository = env.RUNTIME_SETTINGS_REPO || DEFAULT_REPOSITORY; const workflow = env.RUNTIME_SETTINGS_WORKFLOW || DEFAULT_WORKFLOW; const apiUrl = `https://api.github.com/repos/${repository}/actions/workflows/${workflow}/dispatches`; - const response = await fetch(apiUrl, { + const response = await fetchWithTimeout(apiUrl, { method: "POST", headers: githubHeaders(env.RUNTIME_SETTINGS_DISPATCH_TOKEN), body: JSON.stringify({ @@ -1150,7 +1164,7 @@ async function fetchGithubVariable(token, repository, scope, githubEnvironment, const apiUrl = githubVariableUrl(repository, scope, githubEnvironment, name); if (!apiUrl) return ""; try { - const response = await fetch(apiUrl, { + const response = await fetchWithTimeout(apiUrl, { headers: githubHeaders(token), }); if (response.status === 404 || response.status === 403) return ""; @@ -1409,7 +1423,7 @@ function githubHeaders(token) { async function fetchGithubOrgLogins(token) { const orgs = []; for (let page = 1; page <= 5; page += 1) { - const response = await fetch(`https://api.github.com/user/orgs?per_page=100&page=${page}`, { + const response = await fetchWithTimeout(`https://api.github.com/user/orgs?per_page=100&page=${page}`, { headers: githubHeaders(token), }); if (!response.ok) return orgs; @@ -1800,6 +1814,7 @@ export const __test = { platformRepositories, requireSameOrigin, responseHeaders, + fetchWithTimeout, syncDefaultStrategyForAccount, supportedDomainsForAccount, updateAccountOptionsDefaultStrategy,