diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 969c3b9526..1fa19339af 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -76,6 +76,10 @@ jobs: CNPMCORE_DATABASE_NAME=cnpmcore_unittest bash ./prepare-database-mysql.sh CNPMCORE_DATABASE_NAME=cnpmcore bash ./prepare-database-mysql.sh + # Generate startup manifest for faster cold start + EGG_TS_ENABLE=false npx egg-bin manifest generate --env=unittest + EGG_TS_ENABLE=false npx egg-bin manifest validate --env=unittest + EGG_TS_ENABLE=false npx eggctl start --port=7001 --env=unittest --daemon HEALTH_URL="http://127.0.0.1:7001/" START_TIME=$(date +%s) @@ -85,7 +89,6 @@ jobs: echo "Waiting for application to become healthy at ${HEALTH_URL} (timeout: ${TIMEOUT}s)..." while true; do - # Capture response body and status code STATUS=$(curl -s -o /tmp/cnpmcore-health-response -w "%{http_code}" "${HEALTH_URL}" || echo "000") echo "Health check attempt at $(date): status=${STATUS}" @@ -115,6 +118,10 @@ jobs: cat logs/cnpmcore/common-error.log 2>/dev/null || true exit 1 fi + + # Verify manifest is loaded and used (zero extra resolve I/O) + cp ../scripts/verify-manifest.mjs . + EGG_TS_ENABLE=false EGG_SERVER_ENV=unittest node verify-manifest.mjs - name: examples node-version: 24 command: | diff --git a/ecosystem-ci/scripts/verify-manifest.mjs b/ecosystem-ci/scripts/verify-manifest.mjs new file mode 100644 index 0000000000..2ff3433e73 --- /dev/null +++ b/ecosystem-ci/scripts/verify-manifest.mjs @@ -0,0 +1,42 @@ +/** + * Verify that the startup manifest is loaded and used during application boot. + * Asserts that resolveModule reads from cache with zero extra file I/O. + * + * Usage: EGG_SERVER_ENV= node ecosystem-ci/scripts/verify-manifest.mjs + * Must be run from the application's baseDir (e.g., ecosystem-ci/cnpmcore/). + */ +async function main() { + const { startEgg } = await import('egg'); + + const app = await startEgg({ + baseDir: process.cwd(), + mode: 'single', + metadataOnly: true, + }); + + if (!app.loader.manifest) { + console.error('FAIL: manifest not loaded'); + process.exit(1); + } + + const { resolveCache, fileDiscovery, tegg } = app.loader.manifest.data; + console.log('Manifest loaded successfully'); + console.log(' resolveCache entries:', Object.keys(resolveCache).length); + console.log(' fileDiscovery entries:', Object.keys(fileDiscovery).length); + console.log(' tegg moduleRefs:', tegg?.moduleReferences?.length ?? 0); + + // After ready(), the loader has run all loading phases with metadataOnly. + // With a valid manifest, resolveModule should have used the cache + // and produced zero new collector entries. + const newEntries = Object.keys(app.loader.resolveCacheCollector).length; + if (newEntries > 0) { + console.error('FAIL: resolve produced %d new I/O entries instead of using cache', newEntries); + process.exit(1); + } + console.log(' resolve cache: all hits, zero extra I/O'); + + await app.close(); + process.exit(0); +} + +void main(); diff --git a/packages/core/src/egg.ts b/packages/core/src/egg.ts index c0b61a6b39..2596923430 100644 --- a/packages/core/src/egg.ts +++ b/packages/core/src/egg.ts @@ -31,6 +31,8 @@ export interface EggCoreOptions { plugins?: any; serverScope?: string; env?: string; + /** Skip lifecycle hooks, only trigger loadMetadata for manifest generation */ + metadataOnly?: boolean; } export type EggCoreInitOptions = Partial; @@ -218,6 +220,7 @@ export class EggCore extends KoaApplication { serverScope: options.serverScope, env: options.env ?? '', EggCoreClass: EggCore, + metadataOnly: options.metadataOnly, }); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 4392e3826d..3f6716628f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -9,6 +9,7 @@ export * from './singleton.ts'; export * from './loader/egg_loader.ts'; export * from './loader/file_loader.ts'; export * from './loader/context_loader.ts'; +export * from './loader/manifest.ts'; export * from './utils/sequencify.ts'; export * from './utils/timing.ts'; export type * from './types.ts'; diff --git a/packages/core/src/lifecycle.ts b/packages/core/src/lifecycle.ts index 3efc02caaa..dd33680f2a 100644 --- a/packages/core/src/lifecycle.ts +++ b/packages/core/src/lifecycle.ts @@ -54,6 +54,13 @@ export interface ILifecycleBoot { * Do some thing before app close */ beforeClose?(): Promise; + + /** + * Collect metadata for manifest generation (metadataOnly mode). + * Called instead of configWillLoad/configDidLoad/didLoad/willReady + * when the application is started with metadataOnly: true. + */ + loadMetadata?(): Promise | void; } export type BootImplClass = new (...args: any[]) => T; @@ -72,6 +79,7 @@ export class Lifecycle extends EventEmitter { #bootHooks: (BootImplClass | ILifecycleBoot)[]; #boots: ILifecycleBoot[]; #isClosed: boolean; + #metadataOnly: boolean; #closeFunctionSet: Set; loadReady: Ready; bootReady: Ready; @@ -87,6 +95,7 @@ export class Lifecycle extends EventEmitter { this.#boots = []; this.#closeFunctionSet = new Set(); this.#isClosed = false; + this.#metadataOnly = false; this.#init = false; this.timing.start(`${this.options.app.type} Start`); @@ -110,7 +119,9 @@ export class Lifecycle extends EventEmitter { }); this.ready((err) => { - this.triggerDidReady(err); + if (!this.#metadataOnly) { + void this.triggerDidReady(err); + } debug('app ready'); this.timing.end(`${this.options.app.type} Start`); }); @@ -331,6 +342,24 @@ export class Lifecycle extends EventEmitter { })(); } + async triggerLoadMetadata(): Promise { + this.#metadataOnly = true; + debug('trigger loadMetadata start'); + for (const boot of this.#boots) { + if (typeof boot.loadMetadata === 'function') { + debug('trigger loadMetadata at %o', boot.fullPath); + try { + await boot.loadMetadata(); + } catch (err) { + debug('trigger loadMetadata error at %o, error: %s', boot.fullPath, err); + this.emit('error', err); + } + } + } + debug('trigger loadMetadata end'); + this.ready(true); + } + #initReady(): void { debug('loadReady init'); this.loadReady = new Ready({ timeout: this.readyTimeout, lazyStart: true }); diff --git a/packages/core/src/loader/egg_loader.ts b/packages/core/src/loader/egg_loader.ts index 8e1785647b..e047552c0f 100644 --- a/packages/core/src/loader/egg_loader.ts +++ b/packages/core/src/loader/egg_loader.ts @@ -23,6 +23,7 @@ import { sequencify } from '../utils/sequencify.ts'; import { Timing } from '../utils/timing.ts'; import { type ContextLoaderOptions, ContextLoader } from './context_loader.ts'; import { type FileLoaderOptions, CaseStyle, FULLPATH, FileLoader } from './file_loader.ts'; +import { ManifestStore, type ManifestTegg, type StartupManifest } from './manifest.ts'; const debug = debuglog('egg/core/loader/egg_loader'); @@ -47,6 +48,8 @@ export interface EggLoaderOptions { serverScope?: string; /** custom plugins */ plugins?: Record; + /** Skip lifecycle hooks, only trigger loadMetadata for manifest generation */ + metadataOnly?: boolean; } export type EggDirInfoType = 'app' | 'plugin' | 'framework'; @@ -67,6 +70,14 @@ export class EggLoader { readonly appInfo: EggAppInfo; readonly outDir?: string; dirs?: EggDirInfo[]; + /** Pre-computed startup manifest for skipping file I/O */ + readonly manifest: ManifestStore | null; + /** Collected resolveModule results for manifest generation */ + readonly resolveCacheCollector: Record = {}; + /** Collected file discovery results for manifest generation */ + readonly fileDiscoveryCollector: Record = {}; + /** Collected tegg manifest data (populated by tegg plugin) */ + teggManifestCollector?: ManifestTegg; /** * @class @@ -154,6 +165,12 @@ export class EggLoader { * @since 1.0.0 */ this.appInfo = this.getAppInfo(); + + // Load pre-computed startup manifest if available + this.manifest = ManifestStore.load(this.options.baseDir, this.serverEnv, this.serverScope); + if (this.manifest) { + debug('startup manifest loaded, will skip redundant file I/O'); + } } get app(): EggCore { @@ -1233,7 +1250,11 @@ export class EggLoader { */ async loadCustomApp(): Promise { await this.#loadBootHook('app'); - this.lifecycle.triggerConfigWillLoad(); + if (this.options.metadataOnly) { + await this.lifecycle.triggerLoadMetadata(); + } else { + this.lifecycle.triggerConfigWillLoad(); + } } /** @@ -1241,7 +1262,11 @@ export class EggLoader { */ async loadCustomAgent(): Promise { await this.#loadBootHook('agent'); - this.lifecycle.triggerConfigWillLoad(); + if (this.options.metadataOnly) { + await this.lifecycle.triggerLoadMetadata(); + } else { + this.lifecycle.triggerConfigWillLoad(); + } } // FIXME: no logger used after egg removed @@ -1627,6 +1652,8 @@ export class EggLoader { directory: options?.directory ?? directory, target, inject: this.app, + manifest: this.manifest, + fileDiscoveryCollector: this.fileDiscoveryCollector, }; const timingKey = `Load "${String(property)}" to Application`; @@ -1652,6 +1679,8 @@ export class EggLoader { directory: options?.directory || directory, property, inject: this.app, + manifest: this.manifest, + fileDiscoveryCollector: this.fileDiscoveryCollector, }; const timingKey = `Load "${String(property)}" to Context`; @@ -1688,6 +1717,15 @@ export class EggLoader { } resolveModule(filepath: string): string | undefined { + // Check manifest cache first + if (this.manifest) { + const cached = this.manifest.getResolveCache(filepath); + if (cached !== undefined) { + debug('[resolveModule:manifest] %o => %o', filepath, cached); + return cached ?? undefined; + } + } + let fullPath: string | undefined; try { fullPath = utils.resolvePath(filepath); @@ -1697,6 +1735,10 @@ export class EggLoader { if (!fullPath) { fullPath = this.#resolveFromOutDir(filepath); } + + // Collect for manifest generation + this.resolveCacheCollector[filepath] = fullPath ?? null; + return fullPath; } @@ -1734,6 +1776,22 @@ export class EggLoader { } } } + + /** + * Generate startup manifest from collected data. + * Should be called after all loading phases complete. + */ + generateManifest(tegg?: ManifestTegg): StartupManifest { + return ManifestStore.generate({ + baseDir: this.options.baseDir, + serverEnv: this.serverEnv, + serverScope: this.serverScope, + typescriptEnabled: isSupportTypeScript(), + tegg, + resolveCache: this.resolveCacheCollector, + fileDiscovery: this.fileDiscoveryCollector, + }); + } } // convert dep to dependencies for compatibility diff --git a/packages/core/src/loader/file_loader.ts b/packages/core/src/loader/file_loader.ts index f852f578be..2b8bfd53be 100644 --- a/packages/core/src/loader/file_loader.ts +++ b/packages/core/src/loader/file_loader.ts @@ -8,6 +8,7 @@ import globby from 'globby'; import { isClass, isGeneratorFunction, isAsyncFunction, isPrimitive } from 'is-type-of'; import utils, { type Fun } from '../utils/index.ts'; +import type { ManifestStore } from './manifest.ts'; const debug = debuglog('egg/core/file_loader'); @@ -49,6 +50,10 @@ export interface FileLoaderOptions { /** set property's case when converting a filepath to property list. */ caseStyle?: CaseStyle | CaseStyleFunction; lowercaseFirst?: boolean; + /** Pre-computed startup manifest for skipping globby scans */ + manifest?: ManifestStore | null; + /** Collector for file discovery results during manifest generation */ + fileDiscoveryCollector?: Record; } export interface FileLoaderParseItem { @@ -193,8 +198,17 @@ export class FileLoader { const items: FileLoaderParseItem[] = []; debug('[parse] parsing directories: %j', directories); for (const directory of directories) { - const filepaths = globby.sync(files, { cwd: directory }); - debug('[parse] globby files: %o, cwd: %o => %o', files, directory, filepaths); + const cachedFiles = this.options.manifest?.getFileDiscovery(directory); + const filepaths = cachedFiles ?? globby.sync(files, { cwd: directory }); + if (cachedFiles) { + debug('[parse:manifest] using cached files for %o, count: %d', directory, cachedFiles.length); + } else { + debug('[parse] globby files: %o, cwd: %o => %o', files, directory, filepaths); + // Collect for manifest generation + if (this.options.fileDiscoveryCollector) { + this.options.fileDiscoveryCollector[directory] = filepaths; + } + } for (const filepath of filepaths) { const fullpath = path.join(directory, filepath); if (!fs.statSync(fullpath).isFile()) continue; diff --git a/packages/core/src/loader/manifest.ts b/packages/core/src/loader/manifest.ts new file mode 100644 index 0000000000..fa4eb634a6 --- /dev/null +++ b/packages/core/src/loader/manifest.ts @@ -0,0 +1,258 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import path from 'node:path'; +import { debuglog } from 'node:util'; + +const debug = debuglog('egg/core/loader/manifest'); + +const MANIFEST_VERSION = 1; + +const LOCKFILE_NAMES = ['pnpm-lock.yaml', 'package-lock.json', 'yarn.lock'] as const; + +export interface ManifestModuleReference { + name: string; + path: string; + optional?: boolean; +} + +export interface ManifestModuleDescriptor { + name: string; + unitPath: string; + optional?: boolean; + /** Files containing decorated classes, relative to unitPath */ + decoratedFiles: string[]; +} + +export interface ManifestTegg { + moduleReferences: ManifestModuleReference[]; + moduleDescriptors: ManifestModuleDescriptor[]; +} + +export interface ManifestInvalidation { + lockfileFingerprint: string; + configFingerprint: string; + serverEnv: string; + serverScope: string; + baseDir: string; + typescriptEnabled: boolean; +} + +export interface StartupManifest { + version: number; + generatedAt: string; + invalidation: ManifestInvalidation; + tegg: ManifestTegg; + /** resolveModule cache: filepath -> resolved path | null */ + resolveCache: Record; + /** directory path -> file relative paths (filtered) */ + fileDiscovery: Record; +} + +export class ManifestStore { + readonly data: StartupManifest; + + private constructor(data: StartupManifest) { + this.data = data; + } + + /** + * Load and validate manifest from `.egg/manifest.json`. + * Returns null if manifest doesn't exist or is invalid. + */ + static load(baseDir: string, serverEnv: string, serverScope: string): ManifestStore | null { + if (serverEnv === 'local' && process.env.EGG_MANIFEST !== 'true') { + debug('skip manifest in local env (set EGG_MANIFEST=true to enable)'); + return null; + } + + const manifestPath = path.join(baseDir, '.egg', 'manifest.json'); + let raw: string; + try { + raw = fs.readFileSync(manifestPath, 'utf-8'); + } catch { + debug('manifest not found at %s', manifestPath); + return null; + } + + let data: StartupManifest; + try { + data = JSON.parse(raw); + } catch (e) { + debug('failed to parse manifest: %s', e); + return null; + } + + if (!ManifestStore.#validate(data, baseDir, serverEnv, serverScope)) { + return null; + } + + debug('manifest loaded successfully'); + return new ManifestStore(data); + } + + static #validate(data: StartupManifest, baseDir: string, serverEnv: string, serverScope: string): boolean { + if (data.version !== MANIFEST_VERSION) { + debug('manifest version mismatch: expected %d, got %d', MANIFEST_VERSION, data.version); + return false; + } + + const inv = data.invalidation; + if (!inv) { + debug('manifest missing invalidation data'); + return false; + } + + // Note: baseDir is NOT validated — build env and runtime env may have different paths + + if (inv.serverEnv !== serverEnv) { + debug('manifest serverEnv mismatch: expected %s, got %s', serverEnv, inv.serverEnv); + return false; + } + + if (inv.serverScope !== serverScope) { + debug('manifest serverScope mismatch: expected %s, got %s', serverScope, inv.serverScope); + return false; + } + + // Use stat-based fingerprint (mtime+size) for cheap validation + const currentLockfileFingerprint = ManifestStore.#lockfileFingerprint(baseDir); + if (inv.lockfileFingerprint !== currentLockfileFingerprint) { + debug('manifest lockfileFingerprint mismatch'); + return false; + } + + const currentConfigFingerprint = ManifestStore.#directoryFingerprint(path.join(baseDir, 'config')); + if (inv.configFingerprint !== currentConfigFingerprint) { + debug('manifest configFingerprint mismatch'); + return false; + } + + return true; + } + + // --- Query APIs --- + + getResolveCache(filepath: string): string | null | undefined { + const cache = this.data.resolveCache; + if (!cache || !(filepath in cache)) return undefined; + return cache[filepath]; + } + + getFileDiscovery(directory: string): string[] | undefined { + return this.data.fileDiscovery?.[directory]; + } + + get tegg(): ManifestTegg | undefined { + return this.data.tegg; + } + + // --- Generation APIs --- + + static generate(options: ManifestGenerateOptions): StartupManifest { + return { + version: MANIFEST_VERSION, + generatedAt: new Date().toISOString(), + invalidation: { + lockfileFingerprint: ManifestStore.#lockfileFingerprint(options.baseDir), + configFingerprint: ManifestStore.#directoryFingerprint(path.join(options.baseDir, 'config')), + serverEnv: options.serverEnv, + serverScope: options.serverScope, + baseDir: options.baseDir, + typescriptEnabled: options.typescriptEnabled, + }, + tegg: options.tegg ?? { moduleReferences: [], moduleDescriptors: [] }, + resolveCache: options.resolveCache ?? {}, + fileDiscovery: options.fileDiscovery ?? {}, + }; + } + + static async write(baseDir: string, manifest: StartupManifest): Promise { + const dir = path.join(baseDir, '.egg'); + await fsp.mkdir(dir, { recursive: true }); + const manifestPath = path.join(dir, 'manifest.json'); + await fsp.writeFile(manifestPath, JSON.stringify(manifest, null, 2)); + debug('manifest written to %s', manifestPath); + } + + static clean(baseDir: string): void { + const manifestPath = path.join(baseDir, '.egg', 'manifest.json'); + try { + fs.unlinkSync(manifestPath); + debug('manifest removed: %s', manifestPath); + } catch { + // file doesn't exist, nothing to do + } + } + + // --- Fingerprint Utilities (stat-based, no content reads) --- + + /** Fingerprint a file by mtime+size — avoids reading file content. */ + static #statFingerprint(filepath: string): string | null { + try { + const stat = fs.statSync(filepath); + return `${stat.mtimeMs}:${stat.size}`; + } catch { + return null; + } + } + + /** Find and fingerprint the project's lockfile. */ + static #lockfileFingerprint(baseDir: string): string { + for (const name of LOCKFILE_NAMES) { + const fp = ManifestStore.#statFingerprint(path.join(baseDir, name)); + if (fp) return `${name}:${fp}`; + } + return ''; + } + + /** Fingerprint a directory tree by file names, mtimes, and sizes. */ + static #directoryFingerprint(dirpath: string): string { + const hash = createHash('md5'); + const visited = new Set(); + ManifestStore.#fingerprintRecursive(dirpath, hash, visited); + return hash.digest('hex'); + } + + static #fingerprintRecursive(dirpath: string, hash: ReturnType, visited: Set): void { + let realPath: string; + try { + realPath = fs.realpathSync(dirpath); + } catch { + return; + } + // Prevent symlink cycles + if (visited.has(realPath)) return; + visited.add(realPath); + + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dirpath, { withFileTypes: true }); + } catch { + return; + } + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + if (entry.isSymbolicLink()) continue; + const fullPath = path.join(dirpath, entry.name); + if (entry.isDirectory()) { + hash.update(`dir:${entry.name}\n`); + ManifestStore.#fingerprintRecursive(fullPath, hash, visited); + } else if (entry.isFile()) { + // Use stat metadata instead of reading file contents + const fp = ManifestStore.#statFingerprint(fullPath); + hash.update(`file:${entry.name}:${fp ?? 'missing'}\n`); + } + } + } +} + +export interface ManifestGenerateOptions { + baseDir: string; + serverEnv: string; + serverScope: string; + typescriptEnabled: boolean; + tegg?: ManifestTegg; + resolveCache?: Record; + fileDiscovery?: Record; +} diff --git a/packages/core/test/__snapshots__/index.test.ts.snap b/packages/core/test/__snapshots__/index.test.ts.snap index 27071368f5..9594df5a13 100644 --- a/packages/core/test/__snapshots__/index.test.ts.snap +++ b/packages/core/test/__snapshots__/index.test.ts.snap @@ -18,6 +18,7 @@ exports[`should expose properties 1`] = ` "KoaRequest", "KoaResponse", "Lifecycle", + "ManifestStore", "Request", "Response", "Router", diff --git a/packages/core/test/loader/manifest.test.ts b/packages/core/test/loader/manifest.test.ts new file mode 100644 index 0000000000..bcc9b03f9c --- /dev/null +++ b/packages/core/test/loader/manifest.test.ts @@ -0,0 +1,363 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +import mm from 'mm'; +import { describe, it, beforeEach, afterEach } from 'vitest'; + +import { ManifestStore } from '../../src/loader/manifest.ts'; +import { createTmpDir, setupBaseDir, generateAndWrite } from './manifest_helper.ts'; + +let tmpDir: string; + +describe('ManifestStore', () => { + beforeEach(() => { + tmpDir = createTmpDir(); + }); + + afterEach(() => { + mm.restore(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + describe('generate()', () => { + it('should generate manifest with correct structure', () => { + const baseDir = setupBaseDir(); + try { + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'prod', + serverScope: '', + typescriptEnabled: true, + }); + + assert.equal(manifest.version, 1); + assert.ok(manifest.generatedAt); + assert.ok(new Date(manifest.generatedAt).getTime() > 0); + assert.equal(manifest.invalidation.serverEnv, 'prod'); + assert.equal(manifest.invalidation.serverScope, ''); + assert.equal(manifest.invalidation.baseDir, baseDir); + assert.equal(manifest.invalidation.typescriptEnabled, true); + assert.ok(manifest.invalidation.lockfileFingerprint); + assert.ok(manifest.invalidation.configFingerprint); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should default optional fields to empty', () => { + const baseDir = setupBaseDir(); + try { + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'prod', + serverScope: '', + typescriptEnabled: false, + }); + + assert.deepStrictEqual(manifest.tegg, { moduleReferences: [], moduleDescriptors: [] }); + assert.deepStrictEqual(manifest.resolveCache, {}); + assert.deepStrictEqual(manifest.fileDiscovery, {}); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should preserve provided tegg, resolveCache, fileDiscovery', () => { + const baseDir = setupBaseDir(); + try { + const tegg = { + moduleReferences: [{ name: 'mod', path: '/tmp/mod' }], + moduleDescriptors: [{ name: 'mod', unitPath: '/tmp/mod', decoratedFiles: ['a.ts'] }], + }; + const resolveCache = { '/foo/bar': '/resolved/bar', '/foo/missing': null }; + const fileDiscovery = { '/app/service': ['user.ts', 'post.ts'] }; + + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'prod', + serverScope: '', + typescriptEnabled: true, + tegg, + resolveCache, + fileDiscovery, + }); + + assert.deepStrictEqual(manifest.tegg, tegg); + assert.deepStrictEqual(manifest.resolveCache, resolveCache); + assert.deepStrictEqual(manifest.fileDiscovery, fileDiscovery); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should detect pnpm lockfile', () => { + const baseDir = setupBaseDir({ lockfile: 'pnpm' }); + try { + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'prod', + serverScope: '', + typescriptEnabled: true, + }); + assert.ok(manifest.invalidation.lockfileFingerprint.startsWith('pnpm-lock.yaml:')); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should detect npm lockfile', () => { + const baseDir = setupBaseDir({ lockfile: 'npm' }); + try { + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'prod', + serverScope: '', + typescriptEnabled: true, + }); + assert.ok(manifest.invalidation.lockfileFingerprint.startsWith('package-lock.json:')); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should detect yarn lockfile', () => { + const baseDir = setupBaseDir({ lockfile: 'yarn' }); + try { + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'prod', + serverScope: '', + typescriptEnabled: true, + }); + assert.ok(manifest.invalidation.lockfileFingerprint.startsWith('yarn.lock:')); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should return empty lockfile fingerprint when no lockfile', () => { + const baseDir = setupBaseDir({ lockfile: 'none' }); + try { + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'prod', + serverScope: '', + typescriptEnabled: true, + }); + assert.equal(manifest.invalidation.lockfileFingerprint, ''); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should produce deterministic config fingerprint', () => { + const baseDir = setupBaseDir({ configFiles: { 'a.ts': 'const a = 1;' } }); + try { + const m1 = ManifestStore.generate({ baseDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + const m2 = ManifestStore.generate({ baseDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + assert.equal(m1.invalidation.configFingerprint, m2.invalidation.configFingerprint); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + }); + + describe('write() and clean()', () => { + it('should create .egg/ directory and write manifest', async () => { + const baseDir = setupBaseDir(); + try { + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'prod', + serverScope: '', + typescriptEnabled: true, + }); + await ManifestStore.write(baseDir, manifest); + + const manifestPath = path.join(baseDir, '.egg', 'manifest.json'); + assert.ok(fs.existsSync(manifestPath)); + + const written = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')); + assert.equal(written.version, 1); + assert.equal(written.invalidation.baseDir, baseDir); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should overwrite existing manifest', async () => { + const baseDir = setupBaseDir(); + try { + const m1 = ManifestStore.generate({ baseDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + await ManifestStore.write(baseDir, m1); + + const m2 = ManifestStore.generate({ baseDir, serverEnv: 'test', serverScope: '', typescriptEnabled: true }); + await ManifestStore.write(baseDir, m2); + + const written = JSON.parse(fs.readFileSync(path.join(baseDir, '.egg', 'manifest.json'), 'utf-8')); + assert.equal(written.invalidation.serverEnv, 'test'); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should clean manifest file', async () => { + const baseDir = setupBaseDir(); + try { + await generateAndWrite(baseDir); + assert.ok(fs.existsSync(path.join(baseDir, '.egg', 'manifest.json'))); + + ManifestStore.clean(baseDir); + assert.ok(!fs.existsSync(path.join(baseDir, '.egg', 'manifest.json'))); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should not throw when cleaning non-existent manifest', () => { + assert.doesNotThrow(() => { + ManifestStore.clean(tmpDir); + }); + }); + }); + + describe('load()', () => { + it('should load a valid manifest', async () => { + const baseDir = setupBaseDir(); + try { + await generateAndWrite(baseDir); + const store = ManifestStore.load(baseDir, 'prod', ''); + assert.ok(store); + assert.equal(store.data.version, 1); + assert.equal(store.data.invalidation.baseDir, baseDir); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should return null when manifest file does not exist', () => { + const store = ManifestStore.load(tmpDir, 'prod', ''); + assert.equal(store, null); + }); + + it('should return null for invalid JSON', () => { + const eggDir = path.join(tmpDir, '.egg'); + fs.mkdirSync(eggDir, { recursive: true }); + fs.writeFileSync(path.join(eggDir, 'manifest.json'), 'not json{{{'); + const store = ManifestStore.load(tmpDir, 'prod', ''); + assert.equal(store, null); + }); + + it('should return null when version mismatches', async () => { + const baseDir = setupBaseDir(); + try { + await generateAndWrite(baseDir, { version: 999 }); + assert.equal(ManifestStore.load(baseDir, 'prod', ''), null); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should load manifest even when stored baseDir differs from actual path', async () => { + const baseDir = setupBaseDir(); + try { + await generateAndWrite(baseDir); + // Tamper the stored baseDir to simulate build→deploy path change + const manifestPath = path.join(baseDir, '.egg', 'manifest.json'); + const data = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')); + data.invalidation.baseDir = '/some/build/path'; + fs.writeFileSync(manifestPath, JSON.stringify(data)); + // Should still load — baseDir is not validated + const store = ManifestStore.load(baseDir, 'prod', ''); + assert.ok(store); + assert.equal(store.data.invalidation.baseDir, '/some/build/path'); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should return null when serverEnv mismatches', async () => { + const baseDir = setupBaseDir(); + try { + await generateAndWrite(baseDir); + assert.equal(ManifestStore.load(baseDir, 'test', ''), null); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should return null when serverScope mismatches', async () => { + const baseDir = setupBaseDir(); + try { + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'prod', + serverScope: 'scopeA', + typescriptEnabled: true, + }); + await ManifestStore.write(baseDir, manifest); + assert.equal(ManifestStore.load(baseDir, 'prod', 'scopeB'), null); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should return null when lockfile changes', async () => { + const baseDir = setupBaseDir({ lockfile: 'pnpm' }); + try { + await generateAndWrite(baseDir); + fs.writeFileSync(path.join(baseDir, 'pnpm-lock.yaml'), 'lockfileVersion: 9\nmodified: true\n'); + assert.equal(ManifestStore.load(baseDir, 'prod', ''), null); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should return null when config directory changes', async () => { + const baseDir = setupBaseDir({ configFiles: { 'config.default.ts': 'export default {};' } }); + try { + await generateAndWrite(baseDir); + fs.writeFileSync(path.join(baseDir, 'config', 'config.prod.ts'), 'export default { port: 8080 };'); + assert.equal(ManifestStore.load(baseDir, 'prod', ''), null); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should return null in local env by default', async () => { + const baseDir = setupBaseDir(); + try { + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'local', + serverScope: '', + typescriptEnabled: true, + }); + await ManifestStore.write(baseDir, manifest); + assert.equal(ManifestStore.load(baseDir, 'local', ''), null); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should load in local env when EGG_MANIFEST=true', async () => { + const baseDir = setupBaseDir(); + try { + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'local', + serverScope: '', + typescriptEnabled: true, + }); + await ManifestStore.write(baseDir, manifest); + mm(process.env, 'EGG_MANIFEST', 'true'); + const store = ManifestStore.load(baseDir, 'local', ''); + assert.ok(store); + assert.equal(store.data.invalidation.serverEnv, 'local'); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + }); +}); diff --git a/packages/core/test/loader/manifest_fingerprint.test.ts b/packages/core/test/loader/manifest_fingerprint.test.ts new file mode 100644 index 0000000000..029758b618 --- /dev/null +++ b/packages/core/test/loader/manifest_fingerprint.test.ts @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { describe, it, beforeEach, afterEach } from 'vitest'; + +import { ManifestStore } from '../../src/loader/manifest.ts'; +import { createTmpDir, setupBaseDir } from './manifest_helper.ts'; + +let tmpDir: string; + +describe('ManifestStore fingerprint stability', () => { + beforeEach(() => { + tmpDir = createTmpDir(); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('should produce same fingerprint for unchanged file', () => { + fs.writeFileSync(path.join(tmpDir, 'test.txt'), 'hello'); + const m1 = ManifestStore.generate({ baseDir: tmpDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + const m2 = ManifestStore.generate({ baseDir: tmpDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + assert.equal(m1.invalidation.configFingerprint, m2.invalidation.configFingerprint); + }); + + it('should change config fingerprint when file is added', async () => { + const baseDir = setupBaseDir({ configFiles: { 'a.ts': 'const a = 1;' } }); + try { + const m1 = ManifestStore.generate({ baseDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + await new Promise((resolve) => setTimeout(resolve, 50)); + fs.writeFileSync(path.join(baseDir, 'config', 'b.ts'), 'const b = 2;'); + const m2 = ManifestStore.generate({ baseDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + assert.notEqual(m1.invalidation.configFingerprint, m2.invalidation.configFingerprint); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should change config fingerprint when file is deleted', () => { + const baseDir = setupBaseDir({ configFiles: { 'a.ts': '1', 'b.ts': '2' } }); + try { + const m1 = ManifestStore.generate({ baseDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + fs.unlinkSync(path.join(baseDir, 'config', 'b.ts')); + const m2 = ManifestStore.generate({ baseDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + assert.notEqual(m1.invalidation.configFingerprint, m2.invalidation.configFingerprint); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should produce deterministic fingerprint for non-existent config directory', () => { + const m1 = ManifestStore.generate({ baseDir: tmpDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + const m2 = ManifestStore.generate({ baseDir: tmpDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + assert.equal(m1.invalidation.configFingerprint, m2.invalidation.configFingerprint); + }); + + it('should handle symlink cycles without infinite recursion', () => { + const baseDir = setupBaseDir({ configFiles: { 'a.ts': '1' } }); + try { + const configDir = path.join(baseDir, 'config'); + try { + fs.symlinkSync(configDir, path.join(configDir, 'loop')); + } catch { + return; // Symlinks may not be supported + } + const manifest = ManifestStore.generate({ baseDir, serverEnv: 'prod', serverScope: '', typescriptEnabled: true }); + assert.ok(manifest.invalidation.configFingerprint); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/core/test/loader/manifest_helper.ts b/packages/core/test/loader/manifest_helper.ts new file mode 100644 index 0000000000..8cca4c8915 --- /dev/null +++ b/packages/core/test/loader/manifest_helper.ts @@ -0,0 +1,52 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { ManifestStore } from '../../src/loader/manifest.ts'; +import type { StartupManifest } from '../../src/loader/manifest.ts'; + +export function createTmpDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), 'egg-manifest-test-')); +} + +export function setupBaseDir(options?: { + lockfile?: 'pnpm' | 'npm' | 'yarn' | 'none'; + configFiles?: Record; +}): string { + const baseDir = createTmpDir(); + const lockfile = options?.lockfile ?? 'pnpm'; + if (lockfile === 'pnpm') { + fs.writeFileSync(path.join(baseDir, 'pnpm-lock.yaml'), 'lockfileVersion: 9\n'); + } else if (lockfile === 'npm') { + fs.writeFileSync(path.join(baseDir, 'package-lock.json'), '{"lockfileVersion": 3}'); + } else if (lockfile === 'yarn') { + fs.writeFileSync(path.join(baseDir, 'yarn.lock'), '# yarn lockfile v1\n'); + } + + const configDir = path.join(baseDir, 'config'); + fs.mkdirSync(configDir, { recursive: true }); + if (options?.configFiles) { + for (const [name, content] of Object.entries(options.configFiles)) { + fs.writeFileSync(path.join(configDir, name), content); + } + } else { + fs.writeFileSync(path.join(configDir, 'config.default.ts'), 'export default {};\n'); + } + + return baseDir; +} + +export async function generateAndWrite( + baseDir: string, + overrides?: Partial, +): Promise { + const manifest = ManifestStore.generate({ + baseDir, + serverEnv: 'prod', + serverScope: '', + typescriptEnabled: true, + }); + Object.assign(manifest, overrides); + await ManifestStore.write(baseDir, manifest); + return manifest; +} diff --git a/packages/core/test/loader/manifest_query.test.ts b/packages/core/test/loader/manifest_query.test.ts new file mode 100644 index 0000000000..0eed7a59ab --- /dev/null +++ b/packages/core/test/loader/manifest_query.test.ts @@ -0,0 +1,101 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; + +import { describe, it, beforeEach, afterEach } from 'vitest'; + +import { ManifestStore } from '../../src/loader/manifest.ts'; +import { createTmpDir, setupBaseDir, generateAndWrite } from './manifest_helper.ts'; + +let tmpDir: string; + +describe('ManifestStore query APIs', () => { + beforeEach(() => { + tmpDir = createTmpDir(); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + describe('getResolveCache()', () => { + it('should return cached path', async () => { + const baseDir = setupBaseDir(); + try { + await generateAndWrite(baseDir, { + resolveCache: { '/app/config/plugin': '/resolved/plugin.ts' }, + }); + const store = ManifestStore.load(baseDir, 'prod', '')!; + assert.equal(store.getResolveCache('/app/config/plugin'), '/resolved/plugin.ts'); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should return null for null-cached entry', async () => { + const baseDir = setupBaseDir(); + try { + await generateAndWrite(baseDir, { + resolveCache: { '/app/missing': null }, + }); + const store = ManifestStore.load(baseDir, 'prod', '')!; + assert.equal(store.getResolveCache('/app/missing'), null); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should return undefined for uncached entry', async () => { + const baseDir = setupBaseDir(); + try { + await generateAndWrite(baseDir, { resolveCache: {} }); + const store = ManifestStore.load(baseDir, 'prod', '')!; + assert.equal(store.getResolveCache('/not/in/cache'), undefined); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + }); + + describe('getFileDiscovery()', () => { + it('should return cached file list', async () => { + const baseDir = setupBaseDir(); + try { + await generateAndWrite(baseDir, { + fileDiscovery: { '/app/service': ['user.ts', 'post.ts'] }, + }); + const store = ManifestStore.load(baseDir, 'prod', '')!; + assert.deepStrictEqual(store.getFileDiscovery('/app/service'), ['user.ts', 'post.ts']); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should return undefined for uncached directory', async () => { + const baseDir = setupBaseDir(); + try { + await generateAndWrite(baseDir, { fileDiscovery: {} }); + const store = ManifestStore.load(baseDir, 'prod', '')!; + assert.equal(store.getFileDiscovery('/not/cached'), undefined); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + }); + + describe('tegg getter', () => { + it('should return tegg data', async () => { + const baseDir = setupBaseDir(); + const tegg = { + moduleReferences: [{ name: 'myModule', path: '/tmp/myModule' }], + moduleDescriptors: [], + }; + try { + await generateAndWrite(baseDir, { tegg }); + const store = ManifestStore.load(baseDir, 'prod', '')!; + assert.deepStrictEqual(store.tegg, tegg); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + }); +}); diff --git a/packages/core/test/loader/manifest_roundtrip.test.ts b/packages/core/test/loader/manifest_roundtrip.test.ts new file mode 100644 index 0000000000..d2fed20534 --- /dev/null +++ b/packages/core/test/loader/manifest_roundtrip.test.ts @@ -0,0 +1,92 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { describe, it, beforeEach, afterEach } from 'vitest'; + +import { ManifestStore } from '../../src/loader/manifest.ts'; +import { createTmpDir, setupBaseDir, generateAndWrite } from './manifest_helper.ts'; + +let tmpDir: string; + +describe('ManifestStore roundtrip: generate → write → load', () => { + beforeEach(() => { + tmpDir = createTmpDir(); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('should round-trip manifest data correctly', async () => { + const baseDir = setupBaseDir(); + try { + const resolveCache = { '/some/path': '/resolved/path', '/missing': null }; + const fileDiscovery = { '/app/controller': ['home.ts', 'user.ts'] }; + const tegg = { + moduleReferences: [{ name: 'foo', path: '/tmp/foo', optional: true }], + moduleDescriptors: [{ name: 'foo', unitPath: '/tmp/foo', decoratedFiles: ['service.ts'] }], + }; + + const original = ManifestStore.generate({ + baseDir, + serverEnv: 'prod', + serverScope: '', + typescriptEnabled: true, + resolveCache, + fileDiscovery, + tegg, + }); + await ManifestStore.write(baseDir, original); + + const store = ManifestStore.load(baseDir, 'prod', '')!; + assert.ok(store); + assert.deepStrictEqual(store.data.resolveCache, resolveCache); + assert.deepStrictEqual(store.data.fileDiscovery, fileDiscovery); + assert.deepStrictEqual(store.data.tegg, tegg); + assert.equal(store.data.version, original.version); + assert.equal(store.data.generatedAt, original.generatedAt); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should invalidate after lockfile modification', async () => { + const baseDir = setupBaseDir({ lockfile: 'pnpm' }); + try { + await generateAndWrite(baseDir); + assert.ok(ManifestStore.load(baseDir, 'prod', '')); + + fs.appendFileSync(path.join(baseDir, 'pnpm-lock.yaml'), '\n# modified'); + assert.equal(ManifestStore.load(baseDir, 'prod', ''), null); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should invalidate after config file modification', async () => { + const baseDir = setupBaseDir({ configFiles: { 'config.default.ts': 'export default {};' } }); + try { + await generateAndWrite(baseDir); + assert.ok(ManifestStore.load(baseDir, 'prod', '')); + + await new Promise((resolve) => setTimeout(resolve, 50)); + fs.writeFileSync(path.join(baseDir, 'config', 'config.prod.ts'), 'export default { port: 3000 };'); + assert.equal(ManifestStore.load(baseDir, 'prod', ''), null); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); + + it('should remain valid when nothing changes', async () => { + const baseDir = setupBaseDir(); + try { + await generateAndWrite(baseDir); + assert.ok(ManifestStore.load(baseDir, 'prod', '')); + assert.ok(ManifestStore.load(baseDir, 'prod', '')); + assert.ok(ManifestStore.load(baseDir, 'prod', '')); + } finally { + fs.rmSync(baseDir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/egg/src/lib/egg.ts b/packages/egg/src/lib/egg.ts index 35de909f23..153e819939 100644 --- a/packages/egg/src/lib/egg.ts +++ b/packages/egg/src/lib/egg.ts @@ -7,7 +7,7 @@ import path from 'node:path'; import { performance } from 'node:perf_hooks'; import { Cookies as ContextCookies } from '@eggjs/cookies'; -import { EggCore, Router } from '@eggjs/core'; +import { EggCore, Router, ManifestStore } from '@eggjs/core'; import type { EggCoreOptions, Next, MiddlewareFunc as EggCoreMiddlewareFunc, ILifecycleBoot } from '@eggjs/core'; import { utils as eggUtils } from '@eggjs/core'; import { extend } from '@eggjs/extend2'; @@ -190,6 +190,7 @@ export class EggApplicationCore extends EggCore { const dumpStartTime = Date.now(); this.dumpConfig(); this.dumpTiming(); + this.dumpManifest(); this.coreLogger.info('[egg] dump config after ready, %sms', Date.now() - dumpStartTime); }), ); @@ -214,7 +215,7 @@ export class EggApplicationCore extends EggCore { // single process mode will close agent before app close if (this.type === 'application' && this.options.mode === 'single') { - await this.agent!.close(); + await this.agent?.close(); } for (const logger of this.loggers.values()) { @@ -533,6 +534,28 @@ export class EggApplicationCore extends EggCore { } } + /** + * Generate and save startup manifest for faster subsequent startups. + * Only generates when no valid manifest exists (avoids overwriting during manifest-accelerated starts). + * Tegg data is collected by the tegg plugin via `loader.teggManifestCollector`. + * @private + */ + dumpManifest(): void { + try { + // Skip if we already loaded from a valid manifest + if (this.loader.manifest) { + return; + } + + const manifest = this.loader.generateManifest(this.loader.teggManifestCollector); + ManifestStore.write(this.baseDir, manifest).catch((err: Error) => { + this.coreLogger.warn(`[egg] dumpManifest write error: ${err.message}`); + }); + } catch (err: any) { + this.coreLogger.warn(`[egg] dumpManifest error: ${err.message}`); + } + } + protected override customEggPaths(): string[] { return [path.dirname(import.meta.dirname), ...super.customEggPaths()]; } diff --git a/packages/egg/src/lib/loader/AppWorkerLoader.ts b/packages/egg/src/lib/loader/AppWorkerLoader.ts index 88c4f26eb5..d0aee96fed 100644 --- a/packages/egg/src/lib/loader/AppWorkerLoader.ts +++ b/packages/egg/src/lib/loader/AppWorkerLoader.ts @@ -1,3 +1,5 @@ +import path from 'node:path'; + import { EggApplicationLoader } from './EggApplicationLoader.ts'; /** @@ -37,6 +39,11 @@ export class AppWorkerLoader extends EggApplicationLoader { // app await this.loadController(); // app - await this.loadRouter(); // Depend on controllers + if (this.options.metadataOnly) { + // Resolve router path to collect metadata, but don't execute it + this.resolveModule(path.join(this.options.baseDir, 'app/router')); + } else { + await this.loadRouter(); // Depend on controllers + } } } diff --git a/packages/egg/src/lib/start.ts b/packages/egg/src/lib/start.ts index e2b5b445f8..b04b3c88ea 100644 --- a/packages/egg/src/lib/start.ts +++ b/packages/egg/src/lib/start.ts @@ -17,6 +17,8 @@ export interface StartEggOptions { mode?: 'single'; env?: string; plugins?: EggPlugin; + /** Skip lifecycle hooks, only trigger loadMetadata for manifest generation */ + metadataOnly?: boolean; } export interface SingleModeApplication extends Application { @@ -53,18 +55,27 @@ export async function startEgg(options: StartEggOptions = {}): Promise Loader; +export interface ManifestModuleDescriptor { + name: string; + unitPath: string; + optional?: boolean; + /** Files containing decorated classes, relative to unitPath */ + decoratedFiles: string[]; +} + +export interface LoadAppManifest { + moduleDescriptors: ManifestModuleDescriptor[]; +} + export class LoaderFactory { private static loaderCreatorMap: Map = new Map(); @@ -25,14 +37,37 @@ export class LoaderFactory { this.loaderCreatorMap.set(type, creator); } - static async loadApp(moduleReferences: readonly ModuleReference[]): Promise { + static async loadApp( + moduleReferences: readonly ModuleReference[], + manifest?: LoadAppManifest, + ): Promise { const result: ModuleDescriptor[] = []; const multiInstanceClazzList: EggProtoImplClass[] = []; + + const manifestMap = new Map(); + if (manifest?.moduleDescriptors) { + for (const desc of manifest.moduleDescriptors) { + manifestMap.set(desc.unitPath, desc); + } + } + + // Lazy-load ModuleLoader once if manifest is used (avoid repeated dynamic imports) + let ModuleLoaderClass: (typeof import('./impl/ModuleLoader.ts'))['ModuleLoader'] | undefined; + if (manifestMap.size > 0) { + ModuleLoaderClass = (await import('./impl/ModuleLoader.ts')).ModuleLoader; + } + for (const moduleReference of moduleReferences) { - const loader = LoaderFactory.createLoader( - moduleReference.path, - moduleReference.loaderType || EggLoadUnitType.MODULE, - ); + const manifestDesc = manifestMap.get(moduleReference.path); + const loaderType = moduleReference.loaderType || EggLoadUnitType.MODULE; + + let loader: Loader; + if (manifestDesc && ModuleLoaderClass && loaderType === EggLoadUnitType.MODULE) { + loader = new ModuleLoaderClass(moduleReference.path, manifestDesc.decoratedFiles); + } else { + loader = LoaderFactory.createLoader(moduleReference.path, loaderType); + } + const res: ModuleDescriptor = { name: moduleReference.name, unitPath: moduleReference.path, diff --git a/tegg/core/loader/src/impl/ModuleLoader.ts b/tegg/core/loader/src/impl/ModuleLoader.ts index aaf654968c..aa070a4a0e 100644 --- a/tegg/core/loader/src/impl/ModuleLoader.ts +++ b/tegg/core/loader/src/impl/ModuleLoader.ts @@ -12,9 +12,12 @@ const debug = debuglog('egg/tegg/loader/impl/ModuleLoader'); export class ModuleLoader implements Loader { private readonly moduleDir: string; private protoClazzList: EggProtoImplClass[]; + /** Pre-computed file list from manifest (only decorated files) */ + private readonly precomputedFiles?: string[]; - constructor(moduleDir: string) { + constructor(moduleDir: string, precomputedFiles?: string[]) { this.moduleDir = moduleDir; + this.precomputedFiles = precomputedFiles; } async load(): Promise { @@ -23,10 +26,17 @@ export class ModuleLoader implements Loader { return this.protoClazzList; } const protoClassList: EggProtoImplClass[] = []; - const filePattern = LoaderUtil.filePattern(); - const files = await globby(filePattern, { cwd: this.moduleDir }); - debug('load files: %o, filePattern: %o, moduleDir: %o', files, filePattern, this.moduleDir); + let files: string[]; + if (this.precomputedFiles) { + files = this.precomputedFiles; + debug('load from manifest, files: %o, moduleDir: %o', files, this.moduleDir); + } else { + const filePattern = LoaderUtil.filePattern(); + files = await globby(filePattern, { cwd: this.moduleDir }); + debug('load files: %o, filePattern: %o, moduleDir: %o', files, filePattern, this.moduleDir); + } + for (const file of files) { const realPath = path.join(this.moduleDir, file); const fileClazzList = await LoaderUtil.loadFile(realPath); diff --git a/tegg/core/metadata/src/model/ModuleDescriptor.ts b/tegg/core/metadata/src/model/ModuleDescriptor.ts index 346125f98f..6270366949 100644 --- a/tegg/core/metadata/src/model/ModuleDescriptor.ts +++ b/tegg/core/metadata/src/model/ModuleDescriptor.ts @@ -66,4 +66,19 @@ export class ModuleDescriptorDumper { await fs.mkdir(path.dirname(dumpPath), { recursive: true }); await fs.writeFile(dumpPath, ModuleDescriptorDumper.stringifyDescriptor(desc)); } + + /** + * Extract decorated file paths (relative to unitPath) from a ModuleDescriptor. + * Used for manifest generation to record which files contain egg prototypes. + */ + static getDecoratedFiles(desc: ModuleDescriptor): string[] { + const fileSet = new Set(); + for (const clazz of [...desc.clazzList, ...desc.multiInstanceClazzList]) { + const filePath = PrototypeUtil.getFilePath(clazz); + if (filePath) { + fileSet.add(path.relative(desc.unitPath, filePath)); + } + } + return Array.from(fileSet); + } } diff --git a/tegg/plugin/config/src/app.ts b/tegg/plugin/config/src/app.ts index ebd455f670..761956e317 100644 --- a/tegg/plugin/config/src/app.ts +++ b/tegg/plugin/config/src/app.ts @@ -20,30 +20,50 @@ export default class App implements ILifecycleBoot { } configWillLoad(): void { + this.#scanModuleReferences(); + this.#loadModuleConfigs(); + } + + async loadMetadata(): Promise { + this.#scanModuleReferences(); + this.#loadModuleConfigs(); + } + + #scanModuleReferences(): void { const { readModuleOptions } = this.app.config.tegg; - // Auto-exclude outDir (e.g. dist/) from module scanning to avoid - // duplicate modules when both source and compiled output exist - const outDir = this.app.loader.outDir; - if (outDir) { - const extraFilePattern = readModuleOptions.extraFilePattern || []; - const excludePattern = `!**/${outDir}`; - if (!extraFilePattern.includes(excludePattern)) { - readModuleOptions.extraFilePattern = [...extraFilePattern, excludePattern]; + + // Try to use manifest for module references (skip expensive globby scan) + const manifest = this.app.loader.manifest; + const manifestTegg = manifest?.tegg; + + let moduleReferences: readonly ModuleReference[]; + if (manifestTegg?.moduleReferences?.length) { + moduleReferences = manifestTegg.moduleReferences; + debug('load moduleReferences from manifest: %o', moduleReferences); + } else { + // Auto-exclude outDir (e.g. dist/) from module scanning to avoid + // duplicate modules when both source and compiled output exist + const outDir = this.app.loader.outDir; + if (outDir) { + const extraFilePattern = readModuleOptions.extraFilePattern || []; + const excludePattern = `!**/${outDir}`; + if (!extraFilePattern.includes(excludePattern)) { + readModuleOptions.extraFilePattern = [...extraFilePattern, excludePattern]; + } + } + const moduleScanner = new ModuleScanner(this.app.baseDir, readModuleOptions); + moduleReferences = moduleScanner.loadModuleReferences(); + + if (outDir) { + moduleReferences = this.#rewriteModulePaths(moduleReferences, outDir); } - } - const moduleScanner = new ModuleScanner(this.app.baseDir, readModuleOptions); - let moduleReferences = moduleScanner.loadModuleReferences(); - - // When outDir is configured and compiled output exists, rewrite module paths - // from source (e.g. app/port/) to compiled output (e.g. dist/app/port/) - // so that LoaderUtil can find .js files in production mode - if (outDir) { - moduleReferences = this.#rewriteModulePaths(moduleReferences, outDir); } this.app.moduleReferences = moduleReferences; debug('load moduleReferences: %o', this.app.moduleReferences); + } + #loadModuleConfigs(): void { this.app.moduleConfigs = {}; for (const reference of this.app.moduleReferences) { const absoluteRef: ModuleReference = { diff --git a/tegg/plugin/tegg/src/app.ts b/tegg/plugin/tegg/src/app.ts index 3795960a8a..c865be6b8a 100644 --- a/tegg/plugin/tegg/src/app.ts +++ b/tegg/plugin/tegg/src/app.ts @@ -3,12 +3,14 @@ import './lib/AppLoadUnit.ts'; import './lib/AppLoadUnitInstance.ts'; import './lib/EggCompatibleObject.ts'; import { LoadUnitMultiInstanceProtoHook } from '@eggjs/metadata'; +import { LoaderFactory } from '@eggjs/tegg-loader'; import type { Application, ILifecycleBoot } from 'egg'; import { CompatibleUtil } from './lib/CompatibleUtil.ts'; import { ConfigSourceLoadUnitHook } from './lib/ConfigSourceLoadUnitHook.ts'; import { EggContextCompatibleHook } from './lib/EggContextCompatibleHook.ts'; import { EggContextHandler } from './lib/EggContextHandler.ts'; +import { EggModuleLoader } from './lib/EggModuleLoader.ts'; import { EggQualifierProtoHook } from './lib/EggQualifierProtoHook.ts'; import { ModuleHandler } from './lib/ModuleHandler.ts'; import { hijackRunInBackground } from './lib/run_in_background.ts'; @@ -54,6 +56,15 @@ export default class TeggAppBoot implements ILifecycleBoot { this.app.eggContextLifecycleUtil.registerLifecycle(this.compatibleHook); } + async loadMetadata(): Promise { + if (!this.app.moduleReferences) return; + const moduleDescriptors = await LoaderFactory.loadApp(this.app.moduleReferences); + this.app.loader.teggManifestCollector = EggModuleLoader.buildTeggManifestData( + this.app.moduleReferences, + moduleDescriptors, + ); + } + async beforeClose(): Promise { CompatibleUtil.clean(); await this.app.moduleHandler.destroy(); diff --git a/tegg/plugin/tegg/src/lib/EggModuleLoader.ts b/tegg/plugin/tegg/src/lib/EggModuleLoader.ts index 0c200264e7..0a2d28420a 100644 --- a/tegg/plugin/tegg/src/lib/EggModuleLoader.ts +++ b/tegg/plugin/tegg/src/lib/EggModuleLoader.ts @@ -1,6 +1,8 @@ import { EggLoadUnitType, LoadUnitFactory, GlobalGraph, ModuleDescriptorDumper } from '@eggjs/metadata'; -import type { GlobalGraphBuildHook } from '@eggjs/metadata'; +import type { GlobalGraphBuildHook, ModuleDescriptor } from '@eggjs/metadata'; import { LoaderFactory } from '@eggjs/tegg-loader'; +import type { LoadAppManifest } from '@eggjs/tegg-loader'; +import type { ModuleReference } from '@eggjs/tegg-types'; import type { Application } from 'egg'; import { EggAppLoader } from './EggAppLoader.ts'; @@ -18,13 +20,13 @@ export class EggModuleLoader { this.pendingBuildHooks.push(hook); } - private async loadApp() { + private async loadApp(): Promise { const loader = new EggAppLoader(this.app); const loadUnit = await LoadUnitFactory.createLoadUnit(this.app.baseDir, EggLoadUnitType.APP, loader); this.app.moduleHandler.loadUnits.push(loadUnit); } - private async buildAppGraph() { + private async buildAppGraph(): Promise { for (const plugin of Object.values(this.app.plugins)) { if (!plugin.enable) continue; const modulePlugin = this.app.moduleReferences.find((t) => t.path === plugin.path); @@ -32,11 +34,26 @@ export class EggModuleLoader { modulePlugin.optional = false; } } - const moduleDescriptors = await LoaderFactory.loadApp(this.app.moduleReferences); + + // Pass manifest data to LoaderFactory if available + const manifest = this.app.loader.manifest; + const manifestTegg = manifest?.tegg; + let loadAppManifest: LoadAppManifest | undefined; + if (manifestTegg?.moduleDescriptors?.length) { + loadAppManifest = { + moduleDescriptors: manifestTegg.moduleDescriptors, + }; + } + + const moduleDescriptors = await LoaderFactory.loadApp(this.app.moduleReferences, loadAppManifest); + + // Dump module descriptors and collect manifest data + this.#collectTeggManifest(moduleDescriptors); + for (const moduleDescriptor of moduleDescriptors) { ModuleDescriptorDumper.dump(moduleDescriptor, { dumpDir: this.app.baseDir, - }).catch((e) => { + }).catch((e: Error) => { e.message = 'dump module descriptor failed: ' + e.message; this.app.logger.warn(e); }); @@ -45,7 +62,41 @@ export class EggModuleLoader { return graph; } - private async loadModule() { + /** + * Build tegg manifest data from module references and descriptors. + * Shared by both normal startup (#collectTeggManifest) and metadataOnly mode (loadMetadata hook). + */ + static buildTeggManifestData( + moduleReferences: readonly ModuleReference[], + moduleDescriptors: readonly ModuleDescriptor[], + ): { + moduleReferences: Array<{ name: string; path: string; optional?: boolean }>; + moduleDescriptors: Array<{ name: string; unitPath: string; optional?: boolean; decoratedFiles: string[] }>; + } { + return { + moduleReferences: moduleReferences.map((ref) => ({ + name: ref.name, + path: ref.path, + optional: ref.optional, + })), + moduleDescriptors: moduleDescriptors.map((desc) => ({ + name: desc.name, + unitPath: desc.unitPath, + optional: desc.optional, + decoratedFiles: ModuleDescriptorDumper.getDecoratedFiles(desc), + })), + }; + } + + #collectTeggManifest(moduleDescriptors: ModuleDescriptor[]): void { + if (this.app.loader.manifest) return; + this.app.loader.teggManifestCollector = EggModuleLoader.buildTeggManifestData( + this.app.moduleReferences, + moduleDescriptors, + ); + } + + private async loadModule(): Promise { this.globalGraph.build(); this.globalGraph.sort(); const moduleConfigList = this.globalGraph.moduleConfigList; diff --git a/tools/egg-bin/package.json b/tools/egg-bin/package.json index 84b47d792f..5c14d05bf8 100644 --- a/tools/egg-bin/package.json +++ b/tools/egg-bin/package.json @@ -30,6 +30,7 @@ "./baseCommand": "./src/baseCommand.ts", "./commands/cov": "./src/commands/cov.ts", "./commands/dev": "./src/commands/dev.ts", + "./commands/manifest": "./src/commands/manifest.ts", "./commands/test": "./src/commands/test.ts", "./types": "./src/types.ts", "./utils": "./src/utils.ts", @@ -42,6 +43,7 @@ "./baseCommand": "./dist/baseCommand.js", "./commands/cov": "./dist/commands/cov.js", "./commands/dev": "./dist/commands/dev.js", + "./commands/manifest": "./dist/commands/manifest.js", "./commands/test": "./dist/commands/test.js", "./types": "./dist/types.js", "./utils": "./dist/utils.js", @@ -56,6 +58,7 @@ "ci": "npm run cov" }, "dependencies": { + "@eggjs/core": "workspace:*", "@eggjs/tegg-vitest": "workspace:*", "@eggjs/utils": "workspace:*", "@oclif/core": "catalog:", diff --git a/tools/egg-bin/scripts/manifest-generate.mjs b/tools/egg-bin/scripts/manifest-generate.mjs new file mode 100644 index 0000000000..e2a4d291d0 --- /dev/null +++ b/tools/egg-bin/scripts/manifest-generate.mjs @@ -0,0 +1,43 @@ +import { debuglog } from 'node:util'; + +import { importModule } from '@eggjs/utils'; + +const debug = debuglog('egg/bin/scripts/manifest-generate'); + +async function main() { + debug('argv: %o', process.argv); + const options = JSON.parse(process.argv[2]); + debug('manifest generate options: %o', options); + + process.env.EGG_SERVER_ENV = options.env ?? 'prod'; + + const egg = await importModule(options.framework || 'egg'); + const { ManifestStore } = await importModule('@eggjs/core'); + + // startEgg with metadataOnly: skips lifecycle hooks, only triggers loadMetadata + const app = await egg.startEgg({ + baseDir: options.baseDir, + mode: 'single', + metadataOnly: true, + }); + + const manifest = app.loader.generateManifest(app.loader.teggManifestCollector); + await ManifestStore.write(options.baseDir, manifest); + + debug( + 'manifest generated, resolveCache: %d, fileDiscovery: %d, tegg: %o', + Object.keys(manifest.resolveCache).length, + Object.keys(manifest.fileDiscovery).length, + manifest.tegg + ? { + moduleReferences: manifest.tegg.moduleReferences.length, + moduleDescriptors: manifest.tegg.moduleDescriptors.length, + } + : 'none', + ); + + await app.close(); + process.exit(0); +} + +void main(); diff --git a/tools/egg-bin/src/baseCommand.ts b/tools/egg-bin/src/baseCommand.ts index 447a115e88..601fe50e90 100644 --- a/tools/egg-bin/src/baseCommand.ts +++ b/tools/egg-bin/src/baseCommand.ts @@ -315,6 +315,19 @@ export abstract class BaseCommand extends Command { return `--require "${modulePath}"`; } + protected async buildRequireExecArgv(): Promise { + const requires = await this.formatRequires(); + const execArgv: string[] = []; + for (const r of requires) { + const module = this.formatImportModule(r); + const splitIndex = module.indexOf(' '); + if (splitIndex !== -1) { + execArgv.push(module.slice(0, splitIndex), module.slice(splitIndex + 2, -1)); + } + } + return execArgv; + } + protected addNodeOptions(options: string) { if (this.env.NODE_OPTIONS) { if (!this.env.NODE_OPTIONS.includes(options)) { diff --git a/tools/egg-bin/src/commands/dev.ts b/tools/egg-bin/src/commands/dev.ts index 3a2a1b9e53..0cf8ce9757 100644 --- a/tools/egg-bin/src/commands/dev.ts +++ b/tools/egg-bin/src/commands/dev.ts @@ -41,19 +41,7 @@ export default class Dev extends BaseCommand { const serverBin = getSourceFilename(`../scripts/start-cluster.${ext}`); const eggStartOptions = await this.formatEggStartOptions(); const args = [JSON.stringify(eggStartOptions)]; - const requires = await this.formatRequires(); - const execArgv: string[] = []; - for (const r of requires) { - const module = this.formatImportModule(r); - - // Remove the quotes from the path - // --require "module path" -> ['--require', 'module path'] - // --import "module path" -> ['--import', 'module path'] - const splitIndex = module.indexOf(' '); - if (splitIndex !== -1) { - execArgv.push(module.slice(0, splitIndex), module.slice(splitIndex + 2, -1)); - } - } + const execArgv = await this.buildRequireExecArgv(); await this.forkNode(serverBin, args, { execArgv }); } diff --git a/tools/egg-bin/src/commands/manifest.ts b/tools/egg-bin/src/commands/manifest.ts new file mode 100644 index 0000000000..2667287a35 --- /dev/null +++ b/tools/egg-bin/src/commands/manifest.ts @@ -0,0 +1,106 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { debuglog } from 'node:util'; + +import { ManifestStore } from '@eggjs/core'; +import { Args, Flags } from '@oclif/core'; + +import { BaseCommand } from '../baseCommand.ts'; +import { getSourceFilename } from '../utils.ts'; + +const debug = debuglog('egg/bin/commands/manifest'); + +export default class Manifest extends BaseCommand { + static override description = 'Generate, validate, or clean the startup manifest for faster cold starts'; + + static override examples = [ + '<%= config.bin %> <%= command.id %> generate', + '<%= config.bin %> <%= command.id %> validate', + '<%= config.bin %> <%= command.id %> clean', + ]; + + static override flags = { + framework: Flags.string({ + description: 'specify framework, default is "egg"', + }), + env: Flags.string({ + description: 'server environment, default is "prod"', + default: 'prod', + }), + }; + + static override args = { + action: Args.string({ + description: 'action to perform: generate, validate, or clean', + required: true, + options: ['generate', 'validate', 'clean'], + }), + }; + + public async run(): Promise { + const { args, flags } = await this.parse(Manifest); + const baseDir = path.resolve(flags.base ?? process.cwd()); + const action = args.action as 'generate' | 'validate' | 'clean'; + + debug('manifest %s, baseDir: %s', action, baseDir); + + switch (action) { + case 'generate': + await this.generate(baseDir, flags); + break; + case 'validate': + this.validate(baseDir, flags); + break; + case 'clean': + ManifestStore.clean(baseDir); + this.log('Manifest cleaned at: %s', baseDir); + break; + } + } + + private async generate(baseDir: string, flags: Record): Promise { + this.log('Generating startup manifest...'); + this.log(' baseDir: %s', baseDir); + + const ext = this.isESM ? 'mjs' : 'cjs'; + const scriptFile = getSourceFilename(`../scripts/manifest-generate.${ext}`); + const args = [ + JSON.stringify({ + baseDir, + framework: flags.framework, + env: flags.env ?? 'prod', + }), + ]; + + const execArgv = await this.buildRequireExecArgv(); + await this.forkNode(scriptFile, args, { execArgv }); + this.log('Manifest generated at: %s', path.join(baseDir, '.egg', 'manifest.json')); + } + + private validate(baseDir: string, flags: Record): void { + const manifestPath = path.join(baseDir, '.egg', 'manifest.json'); + + if (!fs.existsSync(manifestPath)) { + this.log('No manifest found at: %s', manifestPath); + this.exit(1); + } + + const env = flags.env ?? 'prod'; + const store = ManifestStore.load(baseDir, env, ''); + if (store) { + this.log('Manifest is valid'); + this.log(' version: %d', store.data.version); + this.log(' generated: %s', store.data.generatedAt); + this.log(' env: %s', store.data.invalidation.serverEnv); + this.log(' resolveCache entries: %d', Object.keys(store.data.resolveCache).length); + this.log(' fileDiscovery entries: %d', Object.keys(store.data.fileDiscovery).length); + if (store.data.tegg) { + this.log(' tegg moduleReferences: %d', store.data.tegg.moduleReferences.length); + this.log(' tegg moduleDescriptors: %d', store.data.tegg.moduleDescriptors.length); + } + } else { + this.log('Manifest is invalid or outdated'); + this.exit(1); + } + } +} diff --git a/tools/egg-bin/tsdown.config.ts b/tools/egg-bin/tsdown.config.ts index 0f51f29776..67ebe782ff 100644 --- a/tools/egg-bin/tsdown.config.ts +++ b/tools/egg-bin/tsdown.config.ts @@ -7,7 +7,8 @@ export default defineConfig({ unused: { level: 'error', // @vitest/coverage-v8 is loaded by vitest at runtime as a coverage provider, not directly imported - ignore: ['utility', '@vitest/coverage-v8'], + // @eggjs/core is used by manifest command and scripts/manifest-generate.mjs + ignore: ['utility', '@vitest/coverage-v8', '@eggjs/core'], }, copy: [ {