From e17fdd18d3c69833ebeb75accca6183cb73cfa7d Mon Sep 17 00:00:00 2001 From: Kai Haase Date: Sat, 4 Jul 2026 09:18:45 +0200 Subject: [PATCH] =?UTF-8?q?11.27.4:=20stabilize=20test=20tooling=20under?= =?UTF-8?q?=20parallel=20load=20=E2=80=94=20scope=20the=20idle=20watchdog?= =?UTF-8?q?=20to=20test=20steps,=20label=20signal-killed=20steps,=20and=20?= =?UTF-8?q?add=20an=20opt-in=20CHECK=5FLOW=5FRESOURCE=20e2e=20throttle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- FRAMEWORK-API.md | 2 +- migration-guides/11.27.3-to-11.27.4.md | 236 +++++++++++++++++++++++++ package.json | 2 +- scripts/check.mjs | 84 +++++++-- spectaql.yml | 2 +- vitest-e2e.config.ts | 28 ++- 6 files changed, 334 insertions(+), 20 deletions(-) create mode 100644 migration-guides/11.27.3-to-11.27.4.md diff --git a/FRAMEWORK-API.md b/FRAMEWORK-API.md index 4123407..94f30ba 100644 --- a/FRAMEWORK-API.md +++ b/FRAMEWORK-API.md @@ -1,6 +1,6 @@ # @lenne.tech/nest-server — Framework API Reference -> Auto-generated from source code on 2026-07-03 (v11.27.3) +> Auto-generated from source code on 2026-07-04 (v11.27.4) > File: `FRAMEWORK-API.md` — compact, machine-readable API surface for Claude Code ## CoreModule.forRoot() diff --git a/migration-guides/11.27.3-to-11.27.4.md b/migration-guides/11.27.3-to-11.27.4.md new file mode 100644 index 0000000..331634c --- /dev/null +++ b/migration-guides/11.27.3-to-11.27.4.md @@ -0,0 +1,236 @@ +# Migration Guide: 11.27.3 → 11.27.4 + +## Overview + +| Category | Details | +|----------|---------| +| **Breaking Changes** | None | +| **New Features** | Opt-in `CHECK_LOW_RESOURCE` e2e mode (keeps parallel test runs stable under load); `check.mjs` now labels signal-killed steps (`SIGTERM`/`SIGKILL`) instead of showing a bare exit code. | +| **Bugfixes** | Idle watchdog no longer false-kills output-buffering steps (build / typecheck / audit); a stray `SIGKILL` after PID reuse is prevented; an invalid `--idle-timeout` value now falls back to the default instead of silently disabling the watchdog. | +| **Migration Effort** | 0 minutes for npm consumers (`pnpm update` is enough). Optional: projects that copied `scripts/check.mjs` / `vitest-e2e.config.ts` from the starter can adopt the updated files. | + +This is a **repo-internal tooling & test-infrastructure release**. It touches +**no framework source** (`src/` is unchanged) — only `scripts/check.mjs` and +`vitest-e2e.config.ts`. The npm package (`dist/`) is functionally identical to +11.27.3; consuming projects need no code or config changes. + +--- + +## Quick Migration + +No code changes required. + +```bash +# Update package +pnpm add @lenne.tech/nest-server@11.27.4 + +# Verify build +pnpm run build + +# Run tests +pnpm test +``` + +> These files ship in the git repository / starter, **not** in the npm package. +> An npm-mode consumer is unaffected. A project that copied `scripts/check.mjs` +> and/or `vitest-e2e.config.ts` from the starter (11.27.1+) can copy the updated +> versions to get the improvements below — see **Compatibility Notes**. + +--- + +## What's New in 11.27.4 + +### 1. Opt-in low-resource e2e mode (`CHECK_LOW_RESOURCE`) + +Running several full e2e suites at the same time on one machine (e.g. multiple +parallel `lt dev` / `lt ticket` environments, or a second terminal) can saturate +CPU and the shared MongoDB. Under that load individual requests exceed the 30s +`testTimeout` and auth queries fail — surfacing as intermittent `401`s or +per-file timeouts. Data isolation was never the cause (every run already gets a +unique database); the cause is **physical resource contention**. + +`vitest-e2e.config.ts` now supports an **opt-in** throttle. It changes nothing +by default — solo and moderately parallel runs stay at full speed: + +```bash +# Full speed (default) — nothing changes +pnpm test + +# Opt in when you deliberately run MANY suites in parallel +CHECK_LOW_RESOURCE=1 pnpm test + +# Pin the fork cap explicitly (otherwise ~1/3 of CPU cores) +CHECK_LOW_RESOURCE=1 CHECK_LOW_RESOURCE_FORKS=2 pnpm test +``` + +| `CHECK_LOW_RESOURCE` | Effect | +|----------------------|--------| +| unset / `0` / `false` (default) | No cap, no timeout change — full speed | +| `1` / `true` | `poolOptions.forks.maxForks` capped (`~cores/3`, min 2), `testTimeout` 30s → 60s, `hookTimeout` 120s → 240s | +| any value + `CHECK_LOW_RESOURCE_FORKS=` | Fork cap pinned to `` | + +When active it prints a one-line notice at startup: +`[e2e] low-resource mode active: maxForks=…, timeouts raised`. + +### 2. Signal-exit labelling in `scripts/check.mjs` + +When a step's process is killed by a signal, the package manager surfaces it as +a numeric `Command failed with exit code 143` (SIGTERM) / `137` (SIGKILL) line, +while the outer shell reports only a generic exit `1`. The `check` report now +detects that and appends a clear reason so it isn't mistaken for a test-assertion +failure: + +``` +[check] step ended via SIGTERM (exit 143) — the process was killed, not an +assertion failure. Usual cause: resource pressure (parallel checks/builds +swapping) or an external kill. Re-run this project's check alone to confirm. +``` + +The hint is suppressed when the step was killed by the check's own idle +watchdog (that path already carries its own `[watchdog]` note), and it only +matches the package manager's own failure line — never an `exit code 143` a test +happens to log — so real assertion failures are never mislabeled. + +--- + +## What's Fixed in 11.27.4 + +All fixes are in `scripts/check.mjs`, refining the idle watchdog introduced in +11.27.x. + +### Watchdog now applies to test steps only + +The idle watchdog kills a step whose child produces no output for the timeout +window (default 300s). It previously watched **every** step. But `build`, +`typecheck` and `audit` legitimately buffer all their output to the end (and go +silent under a non-TTY pipe) — watching them risked false-killing a slow but +progressing run. The watchdog is now armed **only for test steps**, whose +runners stream progress continuously, so prolonged silence genuinely means +deadlocked workers. The run header reflects this: `watchdog: 5m 0s (tests)`. + +### No stray `SIGKILL` after PID reuse + +When the watchdog fires it sends `SIGTERM`, then escalates to `SIGKILL` after a +5s grace window. That escalation timer is now tracked and cancelled once the +child exits within the grace window — previously the stray timer could fire +later and `SIGKILL` an **unrelated** process that had reused the freed PID. + +### Invalid `--idle-timeout` falls back to the default + +An unparseable, negative, or unit-suffixed value (e.g. `--idle-timeout=30s`) +now keeps the watchdog at its default (300s) and prints a notice, instead of +silently evaluating to `0` and disabling the protection. Only an explicit `0` +disables the watchdog. + +```bash +--idle-timeout=120 # 120s +--idle-timeout=0 # explicitly disabled +--idle-timeout=30s # invalid → default 300s + "[check] ignoring invalid idle-timeout" notice +CHECK_IDLE_TIMEOUT=90 # env equivalent +``` + +### Clearer watchdog reason + +The `[watchdog]` note now suggests re-running the **actual** failing command +(`Re-run the step directly to debug: `) instead of a hard-coded +`pnpm run vitest`. + +--- + +## Breaking Changes + +None. + +--- + +## Compatibility Notes + +- **npm-mode consumers:** unaffected. `scripts/check.mjs` and + `vitest-e2e.config.ts` are not part of the npm package (`dist/`); no runtime + or API behavior changed. `pnpm update` is sufficient. +- **`IServerOptions` / `CoreModule.forRoot()` / all Core classes:** unchanged. + No source, export, or method signature changed in this release. +- **Starter-derived projects** (copied `scripts/check.mjs` / `vitest-e2e.config.ts` + from nest-server-starter 11.27.1+): copy the updated files to adopt the + test-only watchdog, the PID-reuse fix, the signal-exit hint, and the opt-in + `CHECK_LOW_RESOURCE` mode. All changes are backward compatible — the default + behavior (no env vars set) is unchanged, so an updated `check.mjs` / + `vitest-e2e.config.ts` is a drop-in replacement. +- **Vendor-mode consumers:** not applicable — `scripts/` and + `vitest-e2e.config.ts` are repo/starter tooling, not part of the vendored + `src/core/` file set. Nothing to sync. +- **`lt dev` / `lt ticket` parallel workflows:** the throttle is **manual** + (`CHECK_LOW_RESOURCE=1`). To have it apply automatically inside isolated + environments, the lt CLI can export it into the test env (the same way it + exports `LT_DEV_TEST_SHARDS` for the Playwright config). That is a CLI change, + not a framework change. +- **Nuxt / Playwright side:** no equivalent needed. The Playwright config + already uses `workers: 1` plus a `LT_DEV_TEST_SHARDS`-driven relaxed mode for + load, so the same class of contention is already handled there. + +--- + +## Troubleshooting + +### My `build` / `typecheck` / `audit` step is no longer killed after 300s + +Intended. Only **test** steps are watched now (they stream progress). Buffering +steps that legitimately go silent are no longer at risk of a false watchdog +kill. If such a step genuinely hangs, it still fails via its own tool timeout or +the overall run. + +### The check aborted with "step ended via SIGTERM (exit 143)" + +That step's process was killed by a signal, not by a failed assertion. The usual +cause is resource pressure (several heavy checks/builds/test suites running at +once and swapping the machine) or an external kill. Re-run that project's check +alone to confirm — it will typically pass. + +### e2e tests flake (401 / timeouts) only when I run many suites in parallel + +This is physical CPU/MongoDB contention, not a data-isolation bug (each run has +its own database). Opt into the throttle for those sessions: +`CHECK_LOW_RESOURCE=1 pnpm test`. It caps parallel forks and raises timeouts so +the suites share the machine without starving each other. + +### `CHECK_LOW_RESOURCE` seems to have no effect + +Confirm the value is truthy (`1` / `true`) and not `0` / `false` / empty. When +active, the run prints `[e2e] low-resource mode active: maxForks=…` at startup. + +--- + +## Repo-Internal Tooling Changes (no consumer action) + +Summary of everything in this release, for projects that track the tooling +(all listed above in detail): + +- `scripts/check.mjs`: idle watchdog restricted to test steps; stray-`SIGKILL` + (PID-reuse) fix; invalid `--idle-timeout` falls back to default instead of + disabling; watchdog reason shows the actual command; new `signalExitHint` + labelling `SIGTERM`/`SIGKILL` step exits. +- `vitest-e2e.config.ts`: opt-in `CHECK_LOW_RESOURCE` mode (fork cap + raised + timeouts) for stable parallel e2e runs; default behavior unchanged. + +These carry forward the tooling introduced in +[11.27.1 → 11.27.2](./11.27.1-to-11.27.2.md) (chain-faithful audit) and +[11.27.2 → 11.27.3](./11.27.2-to-11.27.3.md) (idle watchdog + per-run test +databases). + +--- + +## Module Documentation + +No core module changed in this release. Relevant testing documentation: + +- **Testing rules:** [.claude/rules/testing.md](../.claude/rules/testing.md) — + test framework, per-run database lifecycle, parallel execution +- **Reference implementation:** [nest-server-starter](https://github.com/lenneTech/nest-server-starter) + — carries the same `scripts/check.mjs` + `vitest-e2e.config.ts` + +--- + +## References + +- [Migration Guide 11.27.2 → 11.27.3](./11.27.2-to-11.27.3.md) — Previous release (verification-email crash fix + idle watchdog + per-run test DBs) +- [nest-server-starter](https://github.com/lenneTech/nest-server-starter) — reference implementation diff --git a/package.json b/package.json index 7baf910..4a4a5f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lenne.tech/nest-server", - "version": "11.27.3", + "version": "11.27.4", "description": "Modern, fast, powerful Node.js web framework in TypeScript based on Nest with a GraphQL API and a connection to MongoDB (or other databases).", "keywords": [ "node", diff --git a/scripts/check.mjs b/scripts/check.mjs index 4a0aa01..536153a 100644 --- a/scripts/check.mjs +++ b/scripts/check.mjs @@ -37,16 +37,28 @@ const NO_FIX = process.argv.includes("--no-fix"); const PROJECT_FILTERS = process.argv .filter((a) => a.startsWith("--project=")) .map((a) => a.slice("--project=".length)); -// Watchdog: kill a step whose child produces NO output for this long. A wedged -// test run (workers deadlocked at 0% CPU) otherwise spins the live view +// Watchdog: kill a TEST step whose child produces NO output for this long. A +// wedged test run (workers deadlocked at 0% CPU) otherwise spins the live view // forever — the spinner only proves the child process exists, not that it -// progresses. Override with --idle-timeout= or CHECK_IDLE_TIMEOUT -// (seconds); 0 disables the watchdog. +// progresses. Only test steps are watched: build / typecheck / audit legitimately +// buffer all their output to the end (and go silent under a non-TTY pipe), so +// watching them would false-kill a slow-but-progressing run. Override with +// --idle-timeout= or CHECK_IDLE_TIMEOUT (seconds); 0 disables it. const IDLE_TIMEOUT_MS = (() => { const flag = process.argv.find((a) => a.startsWith("--idle-timeout=")); const raw = flag ? flag.slice("--idle-timeout=".length) : process.env.CHECK_IDLE_TIMEOUT; - const seconds = raw === undefined || raw === "" ? 300 : Number(raw); - return Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : 0; + const DEFAULT_MS = 300 * 1000; + if (raw === undefined || raw === "") return DEFAULT_MS; + const seconds = Number(raw); + if (seconds === 0) return 0; // explicit opt-out + // Invalid value (typo, unit suffix, negative) → keep the protection at its + // default rather than silently disabling it — a fat-fingered value must not + // turn the watchdog off unnoticed. + if (!Number.isFinite(seconds) || seconds < 0) { + process.stderr.write(`[check] ignoring invalid idle-timeout "${raw}", using ${DEFAULT_MS / 1000}s\n`); + return DEFAULT_MS; + } + return seconds * 1000; })(); // Verbose streams raw output, so the in-place live view is disabled there. const TTY = Boolean(process.stdout.isTTY) && !VERBOSE; @@ -184,24 +196,32 @@ function killTree(child, signal = "SIGTERM") { } } -function capture(cmd, cwd) { +// idleTimeoutMs > 0 arms the no-output watchdog for this child; 0 (the default) +// runs it unwatched. Only callers that KNOW the child streams progress (test +// steps) should pass a timeout — see runGroup. +function capture(cmd, cwd, idleTimeoutMs = 0) { return new Promise((resolve) => { const child = spawn(cmd, { cwd, shell: true }); RUNNING.add(child); let out = ""; let idleTimer = null; + let killTimer = null; let watchdogHit = false; // Any output resets the watchdog — only complete silence for the full // window counts as wedged. Escalate to SIGKILL for processes that ignore // SIGTERM (deadlocked event loops usually still honor TERM, but be sure). const armWatchdog = () => { - if (!IDLE_TIMEOUT_MS) return; + if (!idleTimeoutMs) return; clearTimeout(idleTimer); idleTimer = setTimeout(() => { watchdogHit = true; killTree(child); - setTimeout(() => killTree(child, "SIGKILL"), 5000).unref(); - }, IDLE_TIMEOUT_MS); + // Track the SIGKILL escalation so a child that honors SIGTERM and exits + // within the grace window cancels it in done() — otherwise the stray + // timer could SIGKILL an unrelated process that reused the freed PID. + killTimer = setTimeout(() => killTree(child, "SIGKILL"), 5000); + killTimer.unref(); + }, idleTimeoutMs); }; const onData = (d) => { out += d; @@ -213,12 +233,13 @@ function capture(cmd, cwd) { child.stderr.on("data", onData); const done = (code, extra) => { clearTimeout(idleTimer); + clearTimeout(killTimer); RUNNING.delete(child); if (watchdogHit) { const note = - `[watchdog] step produced no output for ${Math.round(IDLE_TIMEOUT_MS / 1000)}s — ` + + `[watchdog] step produced no output for ${Math.round(idleTimeoutMs / 1000)}s — ` + "process tree killed as deadlocked. This is a hang (workers idle at 0% CPU), " + - "not a slow run. Re-run the step directly to debug, e.g. `pnpm run vitest`."; + `not a slow run. Re-run the step directly to debug: \`${cmd}\``; return resolve({ code: 1, out: `${out}\n${note}` }); } resolve({ code, out: extra ? `${out}\n${extra}` : out }); @@ -372,6 +393,31 @@ function buildGroups(projects) { return { auditCmd, groups }; } +// A child killed by a signal surfaces through the package manager as a +// "Command failed with exit code 143/137" line (SIGTERM/SIGKILL), NOT as a test +// assertion failure — and the outer shell then reports its own generic exit 1, +// so `code` alone never reveals it. Surface the signal so the reason isn't +// mistaken for a real failure: the usual cause is resource pressure (parallel +// checks/builds swapping the machine) or an external kill. +function signalExitHint(out) { + const clean = stripAnsi(out); + // The watchdog also kills via SIGTERM, so pnpm's "exit code 143" ends up in + // the output — but that path already carries its own [watchdog] note with the + // correct (deadlock) diagnosis. Don't stack a contradictory "external kill" + // hint on top of it. + if (/\[watchdog\]/.test(clean)) return null; + // Match the package manager's OWN failure line, not an arbitrary "exit code + // 143" a test happens to log, so a real assertion failure isn't mislabeled. + const m = clean.match(/Command failed with exit code (137|143)\b/); + if (!m) return null; + const sig = m[1] === "143" ? "SIGTERM" : "SIGKILL"; + return ( + `[check] step ended via ${sig} (exit ${m[1]}) — the process was killed, not an assertion failure. ` + + "Usual cause: resource pressure (parallel checks/builds swapping) or an external kill. " + + "Re-run this project's check alone to confirm." + ); +} + // ── per-project runner ─────────────────────────────────────────────────────── // Runs a group's steps in order, recording results + live state. Stops early // when another project already failed (abort.hit). @@ -384,7 +430,10 @@ async function runGroup(group, states, results, abort) { st.current = step.label; st.stepStart = Date.now(); if (!TTY) process.stdout.write(` ${C.dim("→")} ${shortRel(rel)} · ${step.label}\n`); - const { code, out } = await capture(step.cmd, step.cwd); + // Watchdog only on test steps (see IDLE_TIMEOUT_MS): a test runner streams + // output continuously, so prolonged silence == deadlocked workers. Other + // steps buffer their output and must run unwatched. + const { code, out } = await capture(step.cmd, step.cwd, step.kind === "test" ? IDLE_TIMEOUT_MS : 0); const dur = Date.now() - st.stepStart; const r = { dur, kind: step.kind, label: step.label, project: rel }; if (step.kind === "test") r.tests = parseVitest(out); @@ -394,7 +443,12 @@ async function runGroup(group, states, results, abort) { st.failed = step.label; if (!abort.hit) { abort.hit = true; - abort.failure = { out, project: rel, step: `${shortRel(rel)} · ${step.label}` }; + const hint = signalExitHint(out); + abort.failure = { + out: hint ? `${out}\n${hint}` : out, + project: rel, + step: `${shortRel(rel)} · ${step.label}`, + }; killAll(); } return; @@ -426,7 +480,7 @@ async function main() { C.dim( `${projects.length} project(s) · ${stepCount} steps · ${mode} · audit: ${auditCmd ?? "none"}` + `${NO_FIX ? "" : " · auto-fix format+lint"}` + - ` · watchdog: ${IDLE_TIMEOUT_MS ? fmtDuration(IDLE_TIMEOUT_MS) : "off"}` + + ` · watchdog: ${IDLE_TIMEOUT_MS ? `${fmtDuration(IDLE_TIMEOUT_MS)} (tests)` : "off"}` + `${VERBOSE ? " · verbose" : ""}\n`, ), ); diff --git a/spectaql.yml b/spectaql.yml index 0bf44a5..f3421a5 100644 --- a/spectaql.yml +++ b/spectaql.yml @@ -19,7 +19,7 @@ servers: info: title: lT Nest Server description: Modern, fast, powerful Node.js web framework in TypeScript based on Nest with a GraphQL API and a connection to MongoDB (or other databases). - version: 11.27.3 + version: 11.27.4 contact: name: lenne.Tech GmbH url: https://lenne.tech diff --git a/vitest-e2e.config.ts b/vitest-e2e.config.ts index 4cd8028..36cc609 100644 --- a/vitest-e2e.config.ts +++ b/vitest-e2e.config.ts @@ -1,6 +1,27 @@ +import os from 'node:os'; + import swc from 'unplugin-swc'; import { defineConfig } from 'vitest/config'; +// Opt-in low-resource mode — for running MANY e2e suites at once on one machine +// (e.g. several parallel `lt dev` / `lt ticket` environments). Set +// CHECK_LOW_RESOURCE=1 to cap parallel forks and raise timeouts so the suites +// share CPU/mongod without starving each other into request timeouts (the +// failure mode observed when 2+ full e2e runs overlap — tests hang past the +// 30s testTimeout and auth queries fail as 401s). Unset (default) = full speed, +// no cap. Optionally pin the fork cap with CHECK_LOW_RESOURCE_FORKS=. +const LOW_RESOURCE_RAW = process.env.CHECK_LOW_RESOURCE; +const LOW_RESOURCE = !!LOW_RESOURCE_RAW && LOW_RESOURCE_RAW !== '0' && LOW_RESOURCE_RAW !== 'false'; +const LOW_RESOURCE_FORKS = (() => { + if (!LOW_RESOURCE) return undefined; + const explicit = Number(process.env.CHECK_LOW_RESOURCE_FORKS); + if (Number.isInteger(explicit) && explicit > 0) return explicit; + return Math.max(2, Math.floor((os.cpus()?.length || 4) / 3)); +})(); +if (LOW_RESOURCE) { + process.stderr.write(`[e2e] low-resource mode active: maxForks=${LOW_RESOURCE_FORKS}, timeouts raised\n`); +} + export default defineConfig({ // Vite 8 switched the default TS/JS transformer from esbuild to Oxc. unplugin-swc // disables esbuild internally — without `oxc: false`, Oxc would still run in parallel. @@ -26,7 +47,7 @@ export default defineConfig({ // Hooks are NOT covered by `retry` (tests only) — a beforeAll that boots a // full Nest app can exceed 60s under load (parallel transform/import), // which failed whole files without any retry. Be generous here. - hookTimeout: 120000, + hookTimeout: LOW_RESOURCE ? 240000 : 120000, include: ['tests/**/*.ts'], // Runs in every test worker before test files are imported (filters expected log/warn noise) setupFiles: ['tests/setup.ts'], @@ -39,6 +60,9 @@ export default defineConfig({ maxConcurrency: 3, // Use forks instead of threads for better NestJS performance pool: 'forks', + // Cap parallel fork workers ONLY in low-resource mode (see CHECK_LOW_RESOURCE + // above); default = vitest's own (~CPU count) for full-speed solo runs. + ...(LOW_RESOURCE ? { poolOptions: { forks: { maxForks: LOW_RESOURCE_FORKS, minForks: 1 } } } : {}), // db-lifecycle: drops this run's unique DB on success (+ collects stale // run DBs), keeps it for debugging on failure — see tests/db-lifecycle.reporter.ts reporters: ['default', './tests/db-lifecycle.reporter.ts'], @@ -47,7 +71,7 @@ export default defineConfig({ retry: 5, root: './', teardownTimeout: 30000, - testTimeout: 30000, + testTimeout: LOW_RESOURCE ? 60000 : 30000, // Optimize file watching (not needed in CI) watch: false, },