diff --git a/step-core/src/main/java/step/automation/packages/AutomationPackageResourceUploader.java b/step-core/src/main/java/step/automation/packages/AutomationPackageResourceUploader.java index 0638532e96..dcd04ef795 100644 --- a/step-core/src/main/java/step/automation/packages/AutomationPackageResourceUploader.java +++ b/step-core/src/main/java/step/automation/packages/AutomationPackageResourceUploader.java @@ -30,11 +30,21 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; public class AutomationPackageResourceUploader { private static final Logger logger = LoggerFactory.getLogger(AutomationPackageResourceUploader.class); + private final Map uniqueResourceReferences = new ConcurrentHashMap<>(); + + public String applyUniqueResourceReference(String resourceReference, + String resourceType, + StagingAutomationPackageContext context) { + return uniqueResourceReferences.computeIfAbsent(resourceReference, key -> applyResourceReference(resourceReference, resourceType, context)); + }; + public String applyResourceReference(String resourceReference, String resourceType, StagingAutomationPackageContext context) { diff --git a/step-core/src/main/java/step/automation/packages/StagingAutomationPackageContext.java b/step-core/src/main/java/step/automation/packages/StagingAutomationPackageContext.java index 9a453223ba..02c560f75d 100644 --- a/step-core/src/main/java/step/automation/packages/StagingAutomationPackageContext.java +++ b/step-core/src/main/java/step/automation/packages/StagingAutomationPackageContext.java @@ -8,6 +8,7 @@ public class StagingAutomationPackageContext extends AutomationPackageContext { private final AutomationPackageArchive automationPackageArchive; + private final AutomationPackageResourceUploader resourceUploader = new AutomationPackageResourceUploader(); public StagingAutomationPackageContext(AutomationPackage automationPackage, AutomationPackageOperationMode operationMode, ResourceManager resourceManager, AutomationPackageArchive automationPackageArchive, @@ -19,4 +20,8 @@ public StagingAutomationPackageContext(AutomationPackage automationPackage, Auto public AutomationPackageArchive getAutomationPackageArchive() { return automationPackageArchive; } + + public AutomationPackageResourceUploader getResourceUploader() { + return resourceUploader; + } } diff --git a/step-functions-plugins/step-functions-plugins-node/step-functions-plugins-node-def/src/main/java/step/plugins/node/automation/YamlNodeFunction.java b/step-functions-plugins/step-functions-plugins-node/step-functions-plugins-node-def/src/main/java/step/plugins/node/automation/YamlNodeFunction.java index 68306926dc..c22585a902 100644 --- a/step-functions-plugins/step-functions-plugins-node/step-functions-plugins-node-def/src/main/java/step/plugins/node/automation/YamlNodeFunction.java +++ b/step-functions-plugins/step-functions-plugins-node/step-functions-plugins-node-def/src/main/java/step/plugins/node/automation/YamlNodeFunction.java @@ -44,10 +44,10 @@ public void setJsfile(DynamicValue jsfile) { @Override protected void fillDeclaredFields(NodeFunction function, StagingAutomationPackageContext context) { super.fillDeclaredFields(function, context); - AutomationPackageResourceUploader resourceUploader = new AutomationPackageResourceUploader(); + AutomationPackageResourceUploader resourceUploader = context.getResourceUploader(); String filePath = jsfile.get(); - String fileRef = resourceUploader.applyResourceReference(filePath, ResourceManager.RESOURCE_TYPE_FUNCTIONS, context); + String fileRef = resourceUploader.applyUniqueResourceReference(filePath, ResourceManager.RESOURCE_TYPE_FUNCTIONS, context); if (fileRef != null) { function.setJsFile(new DynamicValue<>(fileRef)); } diff --git a/step-node/step-node-agent/.eslintrc.js b/step-node/step-node-agent/.eslintrc.js deleted file mode 100644 index 2f16b25ff8..0000000000 --- a/step-node/step-node-agent/.eslintrc.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - 'extends': 'standard' -} diff --git a/step-node/step-node-agent/.gitignore b/step-node/step-node-agent/.gitignore index 94f083ab06..45c1d7df58 100644 --- a/step-node/step-node-agent/.gitignore +++ b/step-node/step-node-agent/.gitignore @@ -1,4 +1,6 @@ node_modules/ .npm !/bin -filemanager/work/ \ No newline at end of file +filemanager/work/ +/coverage/ +/npm-project-workspaces/ diff --git a/step-node/step-node-agent/api/controllers/agent-fork.js b/step-node/step-node-agent/api/controllers/agent-fork.js new file mode 100644 index 0000000000..4a21c0439a --- /dev/null +++ b/step-node/step-node-agent/api/controllers/agent-fork.js @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2026, exense GmbH + * + * This file is part of Step + * + * Step is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Step is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Step. If not, see . + */ + +const { OutputBuilder } = require("./output"); +const Session = require("./session"); +const fs = require("fs"); +const path = require('path') +const session = new Session(); + +process.on('message', async ({ type, projectPath, functionName, input, properties, keywordDirectory }) => { + if (type === 'KEYWORD') { + console.log("[Agent fork] Calling keyword " + functionName) + const outputBuilder = new OutputBuilder(); + try { + if (!keywordDirectoryExists(projectPath, keywordDirectory)) { + outputBuilder.fail("The keyword directory '" + keywordDirectory + "' doesn't exist in " + path.basename(projectPath) + ". Possible cause: If using TypeScript, the keywords may not have been compiled. Fix: Ensure your project is built before deploying to Step or during 'npm install'.") + } else { + const kwModules = await importAllKeywords(projectPath, keywordDirectory); + let keywordSearchResult = searchKeyword(kwModules, functionName); + if (!keywordSearchResult) { + console.log('[Agent fork] Unable to find Keyword ' + functionName + "'"); + outputBuilder.fail("Unable to find Keyword '" + functionName + "'"); + } else { + const module = keywordSearchResult.keywordModule; + const keyword = keywordSearchResult.keywordFunction; + + try { + const beforeKeyword = module['beforeKeyword']; + if(beforeKeyword) { + await beforeKeyword(functionName); + } + await keyword(input, outputBuilder, session, properties); + } catch (e) { + const onError = module['onError']; + if (onError) { + if (await onError(e, input, outputBuilder, session, properties)) { + console.log('[Agent fork] Keyword execution failed and onError hook returned \'true\'') + outputBuilder.fail(e) + } else { + console.log('[Agent fork] Keyword execution failed and onError hook returned \'false\'') + } + } else { + console.log('[Agent fork] Keyword execution failed. No onError hook defined') + outputBuilder.fail(e) + } + } finally { + let afterKeyword = module['afterKeyword']; + if (afterKeyword) { + await afterKeyword(functionName); + } + } + } + } + } finally { + console.log("[Agent fork] Returning output") + process.send(outputBuilder.build()); + } + } else if (type === 'KILL') { + console.log("[Agent fork] Exiting...") + await session.asyncDispose(); + process.exit(1) + } + + function keywordDirectoryExists(projectPath, keywordDirectory) { + return fs.existsSync(path.resolve(projectPath, keywordDirectory)) + } + + async function importAllKeywords(projectPath, keywordDirectory) { + const kwModules = []; + const kwDir = path.resolve(projectPath, keywordDirectory); + console.log("[Agent fork] Searching keywords in: " + kwDir) + const kwFiles = fs.readdirSync(kwDir); + for (const kwFile of kwFiles) { + if (kwFile.endsWith('.js')) { + let kwModule = "file://" + path.resolve(kwDir, kwFile); + console.log("[Agent fork] Importing keywords from module: " + kwModule) + let module = await import(kwModule); + kwModules.push(module); + } + } + return kwModules; + } + + function searchKeyword(kwModules, keywordName) { + const kwModule = kwModules.find(m => m[keywordName]); + return kwModule ? {keywordFunction: kwModule[keywordName], keywordModule: kwModule} : undefined; + } +}); + +process.on('unhandledRejection', error => { + console.log('[Agent fork] Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported', error) +}) + +process.on('uncaughtException', error => { + console.log('[Agent fork] Critical: an unhandled error (uncaught exception) occurred and might not have been reported', error) +}) + +process.on('SIGTERM', () => { + console.log("[Agent fork] Received SIGTERM. Exiting...") + process.exit(1); +}); diff --git a/step-node/step-node-agent/api/controllers/agent.js b/step-node/step-node-agent/api/controllers/agent.js new file mode 100644 index 0000000000..c2ef070e99 --- /dev/null +++ b/step-node/step-node-agent/api/controllers/agent.js @@ -0,0 +1,440 @@ +const fs = require("fs"); +const path = require("path"); +const {fork, spawn} = require("child_process"); +const Session = require('./session'); +const { OutputBuilder } = require('./output'); +const logger = require('../logger').child({ component: 'Agent' }); + +const npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm'; + +process.on('unhandledRejection', error => { + logger.error('Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported:', error) +}) + +process.on('uncaughtException', error => { + logger.error('Critical: an unhandled error (uncaught exception) occurred and might not have been reported:', error) +}) + +class Agent { + constructor(agentContext, fileManager, mode) { + this.agentContext = agentContext; + this.filemanager = fileManager; + this.mode = mode; + this.redirectIO = mode !== 'agent'; + this.npmProjectWorkspaces = new Map(); // cacheKey -> { path, inUse, lastFreeAt } + this.npmProjectWorkspaceCleanupIdleTimeMs = agentContext.npmProjectWorkspaceCleanupIdleTimeMs ?? 3600000; + + if(mode === 'agent' && this.npmProjectWorkspaceCleanupIdleTimeMs > 0) { + logger.info(`Scheduling npm project workspace cleanup every ${this.npmProjectWorkspaceCleanupIdleTimeMs}ms`); + setInterval(() => this.cleanupUnusedWorkspaces(this.npmProjectWorkspaceCleanupIdleTimeMs), this.npmProjectWorkspaceCleanupIdleTimeMs); + } + } + + isRunning(req, res) { + res.status(200).json('Agent is running') + } + + reserveToken(req, res) { + this.reserveToken_(req.params.tokenId) + res.json({}) + } + + reserveToken_(tokenId) { + logger.info('Reserving token: ' + tokenId) + } + + releaseToken(req, res) { + this.releaseToken_(req.params.tokenId) + res.json({}) + } + + async releaseToken_(tokenId) { + logger.info('Releasing token: ' + tokenId) + + const session = this.agentContext.tokenSessions[tokenId] + if (session) { + // Close the session and all objects it contains + await session.asyncDispose(); + this.agentContext.tokenSessions[tokenId] = null; + } else { + logger.warn('No session found for token: ' + tokenId) + } + } + + interruptExecution(req, _res) { + const tokenId = req.params.tokenId + logger.warn('Interrupting token: ' + tokenId + ' : not implemented') + } + + async process(req, res) { + const tokenId = req.params.tokenId + let input = req.body.payload; + const keywordName = input.function + let offset = 1000; + const callTimeoutMs = Math.max(offset, input.functionCallTimeout ? input.functionCallTimeout : 180000 - offset); + const argument = input.payload + const properties = input.properties + + let output; + let token = this.agentContext.tokens.find(value => value.id === tokenId); + if(token) { + logger.info('Using token: ' + tokenId + ' to execute ' + keywordName) + + // add the agent properties + const agentProperties = this.agentContext.properties + if (agentProperties) { + Object.entries(agentProperties).forEach(([key, value]) => { properties[key] = value }) + } + + // add the properties of the tokenGroup + const additionalProperties = this.agentContext.tokenProperties[tokenId] + if (additionalProperties) { + Object.entries(additionalProperties).forEach(([key, value]) => { properties[key] = value }) + } + + output = await this.process_(tokenId, keywordName, argument, properties, callTimeoutMs) + } else { + const outputBuilder = new OutputBuilder(); + outputBuilder.fail("The token '" + tokenId + " doesn't exist on this agent. This usually means that the agent crashed and restarted."); + output = outputBuilder.build(); + } + res.json(output) + } + + async process_(tokenId, keywordName, argument, properties, callTimeoutMs) { + const outputBuilder = new OutputBuilder(); + try { + let fileId = properties['$node.js.file.id']; + let fileVersionId = properties['$node.js.file.version']; + const file = await this.filemanager.loadOrGetKeywordFile( + this.agentContext.controllerUrl + '/grid/file/', + fileId, + fileVersionId, + keywordName + ) + + let npmProjectPath; + let fileBasename = path.basename(file); + let wrapperDirectory = path.resolve(file, fileBasename.substring(0, fileBasename.length - 4)); + + if (file.toUpperCase().endsWith('.ZIP') && fs.existsSync(wrapperDirectory)) { + // If the ZIP contains a top-level wrapper folder + npmProjectPath = path.resolve(file, wrapperDirectory); + } else { + npmProjectPath = path.resolve(file) + } + + let workspacePath; + if (this.mode === 'agent') { + // Create a copy of the npm project for each token + workspacePath = await this.getOrCreateNpmProjectWorkspace(tokenId, {fileId, fileVersionId, file: npmProjectPath}); + this.markWorkspaceInUse(workspacePath); + } else { + // When running keywords locally we're working directly in npm project passed by the runner + workspacePath = npmProjectPath; + } + try { + await this.executeKeyword(keywordName, workspacePath, tokenId, argument, properties, outputBuilder, callTimeoutMs) + } finally { + if (this.mode === 'agent') { + this.markWorkspaceFree(workspacePath); + if (this.npmProjectWorkspaceCleanupIdleTimeMs === 0) { + await this.cleanupUnusedWorkspaces(0); + } + } + } + } catch (e) { + logger.error('Unexpected error while executing keyword ' + keywordName + ':', e) + outputBuilder.fail('Unexpected error while executing keyword', e) + } + return outputBuilder.build(); + } + + async executeKeyword(keywordName, npmProjectPath, tokenId, argument, properties, outputBuilder, callTimeoutMs) { + let isDebugEnabled = properties['debug'] === 'true'; + let npmAttachment = null; + let forkedAgentProcessOutputAttachment = null; + try { + logger.info('Executing keyword in project ' + npmProjectPath + ' for token ' + tokenId) + + let session = this.getOrCreateSession(tokenId); + + const npmProjectPathInSession = session.get('npmProjectPath'); + if (npmProjectPathInSession && npmProjectPathInSession !== npmProjectPath) { + throw new Error("Multiple npm projects are not supported within the same session"); + } else { + session.set('npmProjectPath', npmProjectPath); + } + + let forkedAgent = session.get('forkedAgent'); + if (!forkedAgent) { + logger.info('Starting agent fork in ' + npmProjectPath + ' for token ' + tokenId) + forkedAgent = createForkedAgent(npmProjectPath); + session.set('forkedAgent', forkedAgent); + + logger.info('Running npm install in ' + npmProjectPath + ' for token ' + tokenId) + const npmInstallResult = await this.executeNpmInstall(npmProjectPath); + const npmInstallFailed = npmInstallResult.status !== 0 || npmInstallResult.error != null; + if (npmInstallFailed || isDebugEnabled) { + npmAttachment = npmInstallResult.processOutputAttachment; + } + + if (npmInstallFailed) { + throw npmInstallResult.error || new Error('npm install exited with code ' + npmInstallResult.status); + } + + session.set('keywordDirectory', await readStepKeywordDirectory(npmProjectPath)); + } + + const keywordDirectory = session.get('keywordDirectory'); + logger.info('Executing keyword \'' + keywordName + '\' in ' + npmProjectPath + ' for token ' + tokenId) + const { result, processOutputAttachment } = await forkedAgent.runKeywordTask(npmProjectPath, keywordName, argument, properties, callTimeoutMs, this.redirectIO, keywordDirectory); + outputBuilder.merge(result.payload) + if (result.error || isDebugEnabled) { + forkedAgentProcessOutputAttachment = processOutputAttachment; + } + } catch (e) { + if (e instanceof CategorizedError) { + logger.error('Error occurred while executing keyword:' + e.message) + outputBuilder.fail(e.message) + forkedAgentProcessOutputAttachment = e.processOutputAttachment; + } else { + logger.error('Unexpected error occurred while executing keyword:', e) + outputBuilder.fail('Unexpected error: ' + e.message, e) + } + } finally { + if (npmAttachment) { + outputBuilder.attach(npmAttachment); + } + if (forkedAgentProcessOutputAttachment) { + outputBuilder.attach(forkedAgentProcessOutputAttachment); + } + } + } + + async getOrCreateNpmProjectWorkspace(tokenId, keywordPackage) { + const cacheKey = `${keywordPackage.fileId}_${keywordPackage.fileVersionId}_${tokenId}`; + const workspace = this.npmProjectWorkspaces.get(cacheKey); + if (workspace) { + return workspace.path; + } + const baseDir = path.resolve((this.agentContext.workingDir ?? '.'), 'npm-project-workspaces'); + const workspacePath = path.join(baseDir, cacheKey); + if (!fs.existsSync(workspacePath)) { + logger.info(`Creating npm project workspace at ${workspacePath}`); + await fs.promises.cp(keywordPackage.file, workspacePath, { recursive: true }); + } + this.npmProjectWorkspaces.set(cacheKey, { path: workspacePath, inUse: false, lastFreeAt: Date.now() }); + return workspacePath; + } + + markWorkspaceInUse(workspacePath) { + for (const workspace of this.npmProjectWorkspaces.values()) { + if (workspace.path === workspacePath) { + workspace.inUse = true; + return; + } + } + } + + markWorkspaceFree(workspacePath) { + for (const workspace of this.npmProjectWorkspaces.values()) { + if (workspace.path === workspacePath) { + workspace.inUse = false; + workspace.lastFreeAt = Date.now(); + return; + } + } + } + + async cleanupUnusedWorkspaces(idleTimeMs) { + const now = Date.now(); + for (const [cacheKey, workspace] of this.npmProjectWorkspaces) { + if (!workspace.inUse && (now - workspace.lastFreeAt) >= idleTimeMs) { + logger.info(`Deleting npm project workspace unused for ${idleTimeMs}ms: ${workspace.path}`); + try { + await fs.promises.rm(workspace.path, { recursive: true, force: true }); + } catch (e) { + logger.error(`Failed to delete npm project workspace ${workspace.path}:`, e); + continue; + } + this.npmProjectWorkspaces.delete(cacheKey); + } + } + } + + getOrCreateSession(tokenId) { + let session = this.agentContext.tokenSessions[tokenId] + if (!session) { + session = new Session(); + this.agentContext.tokenSessions[tokenId] = session; + } + return session; + } + + async executeNpmInstall(npmProjectPath) { + return await new Promise((resolve) => { + const child = spawn(npmCommand, ['install'], {cwd: npmProjectPath, shell: true}); + const stdChunks = []; + + child.stdout.on('data', (data) => { + stdChunks.push(data); + if (this.redirectIO) { + process.stdout.write(data) + } + }); + + child.stderr.on('data', (data) => { + stdChunks.push(data); + if (this.redirectIO) { + process.stderr.write(data) + } + }); + + child.on('error', (error) => { + resolve({status: null, error, processOutputAttachment: getNpmInstallProcessOutputAttachment()}); + }); + + child.on('close', (code) => { + resolve({status: code, error: null, processOutputAttachment: getNpmInstallProcessOutputAttachment()}); + }); + + function getNpmInstallProcessOutputAttachment() { + const npmInstallOutput = Buffer.concat(stdChunks); + return { + name: 'npm-install.log', + isDirectory: false, + description: 'npm install output', + hexContent: npmInstallOutput.toString('base64') + }; + } + }); + } +} + +async function readStepKeywordDirectory(npmProjectPath) { + try { + const content = await fs.promises.readFile(path.join(npmProjectPath, 'package.json'), 'utf8'); + return JSON.parse(content)?.step?.keywords ?? './keywords'; + } catch { + return './keywords'; + } +} + +function createForkedAgent(keywordProjectPath) { + return new ForkedAgent(keywordProjectPath); +} + +class ForkedAgent { + + constructor(keywordProjectPath) { + const agentForkerLibPath = path.join(keywordProjectPath, 'agent-fork-libs'); + fs.mkdirSync(agentForkerLibPath, { recursive: true }); + fs.copyFileSync(path.resolve(__dirname, 'agent-fork.js'), path.join(agentForkerLibPath, 'agent-fork.js')); + fs.copyFileSync(path.join(__dirname, 'output.js'), path.join(agentForkerLibPath, 'output.js')); + fs.copyFileSync(path.join(__dirname, 'session.js'), path.join(agentForkerLibPath, 'session.js')); + this.agentForkerLibPath = agentForkerLibPath; + this.forkProcess = fork(path.join(agentForkerLibPath, 'agent-fork.js'), [], {cwd: keywordProjectPath, silent: true}); + } + + runKeywordTask(keywordProjectPath, functionName, input, properties, timeoutMs, redirectIO, keywordDirectory) { + return new Promise((resolve, reject) => { + try { + const stdChunks = []; + + const stdoutListener = (data) => { + stdChunks.push(data); + if(redirectIO) { + process.stdout.write(data); + } + }; + const stderrListener = (data) => { + stdChunks.push(data); + if(redirectIO) { + process.stderr.write(data); + } + }; + + if (this.forkProcess.stdout) { + this.forkProcess.stdout.on('data', stdoutListener); + } + + if (this.forkProcess.stderr) { + this.forkProcess.stderr.on('data', stderrListener); + } + + const timeoutHandle = timeoutMs != null ? setTimeout(() => { + cleanup(); + let processOutputAttachment = buildProcessOutputAttachment(stdChunks); + reject(new CategorizedError(`Keyword execution timed out after ${timeoutMs}ms`, processOutputAttachment)); + }, timeoutMs) : null; + + const cleanup = () => { + clearTimeout(timeoutHandle); + if (this.forkProcess.stdout) { + this.forkProcess.stdout.removeListener('data', stdoutListener); + } + if (this.forkProcess.stderr) { + this.forkProcess.stderr.removeListener('data', stderrListener); + } + }; + + this.forkProcess.removeAllListeners('message'); + this.forkProcess.on('message', (result) => { + logger.info(`Keyword '${functionName}' execution completed in forked agent.`) + cleanup(); + + let processOutputAttachment = buildProcessOutputAttachment(stdChunks); + resolve({ result, processOutputAttachment}); + }); + + this.forkProcess.removeAllListeners('error'); + this.forkProcess.on('error', (err) => { + logger.error('Error while calling forked agent:', err) + }); + + this.forkProcess.send({ type: "KEYWORD", projectPath: keywordProjectPath, functionName, input, properties, keywordDirectory }); + } catch (e) { + logger.error('Unexpected error while calling forked agent:', e) + } + }); + + function buildProcessOutputAttachment(stdChunks) { + const outputBuffer = Buffer.concat(stdChunks); + return { + name: 'keyword-process.log', + isDirectory: false, + description: 'Output of the forked keyword process', + hexContent: outputBuffer.toString('base64'), + }; + } + } + + async close() { + const exitPromise = new Promise(resolve => { + if (this.forkProcess.exitCode !== null) { + resolve(); + } else { + this.forkProcess.once('exit', resolve); + } + }); + try { + this.forkProcess.send({ type: "KILL" }); + } catch { + this.forkProcess.kill(); + } + await exitPromise; + fs.rmSync(this.agentForkerLibPath, {recursive: true, force: true}); + } +} + +class CategorizedError extends Error { + processOutputAttachment; + constructor(message, processOutputAttachment) { + super(message); // (1) + this.name = "CategorizedError"; + this.processOutputAttachment = processOutputAttachment; + } +} + +module.exports = Agent; diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js deleted file mode 100644 index 316d554834..0000000000 --- a/step-node/step-node-agent/api/controllers/controller.js +++ /dev/null @@ -1,174 +0,0 @@ -module.exports = function Controller (agentContext, fileManager) { - process.on('unhandledRejection', error => { - console.log('[Controller] Critical: an unhandled error (unhandled promise rejection) occured and might not have been reported', error) - }) - - process.on('uncaughtException', error => { - console.log('[Controller] Critical: an unhandled error (uncaught exception) occured and might not have been reported', error) - }) - - let exports = {} - - const fs = require('fs') - const path = require('path') - const OutputBuilder = require('./output') - - exports.filemanager = fileManager - - exports.isRunning = function (req, res) { - res.status(200).json('Agent is running') - } - - exports.reserveToken = function (req, res) { - exports.reserveToken_(req.params.tokenId) - res.json({}) - } - - exports.reserveToken_ = function (tokenId) { - console.log('[Controller] Reserving token: ' + tokenId) - } - - exports.releaseToken = function (req, res) { - exports.releaseToken_(req.params.tokenId) - res.json({}) - } - - exports.releaseToken_ = function (tokenId) { - console.log('[Controller] Releasing token: ' + tokenId) - - let session = agentContext.tokenSessions[tokenId] - if (session) { - // call close() for each closeable object in the session: - Object.entries(session).forEach(function (element) { - if (typeof element[1]['close'] === 'function') { - console.log('[Controller] Closing closeable object \'' + element[0] + '\' for token: ' + tokenId) - element[1].close() - } - }) - agentContext.tokenSessions[tokenId] = {} - } else { - console.log('[Controller] No session founds for token: ' + tokenId) - } - } - - exports.interruptExecution = function (req, res) { - const tokenId = req.params.tokenId - console.warn('[Controller] Interrupting token: ' + tokenId + ' : not implemented') - } - - exports.process = function (req, res) { - const tokenId = req.params.tokenId - const keywordName = req.body.payload.function - const argument = req.body.payload.payload - const properties = req.body.payload.properties - - console.log('[Controller] Using token: ' + tokenId + ' to execute ' + keywordName) - - // add the agent properties - let agentProperties = agentContext.properties - Object.entries(agentProperties).forEach(function (element) { - properties[element[0]] = element[1] - }) - - // add the properties of the tokenGroup - let additionalProperties = agentContext.tokenProperties[tokenId] - Object.entries(additionalProperties).forEach(function (element) { - properties[element[0]] = element[1] - }) - - exports.process_(tokenId, keywordName, argument, properties, function (payload) { - res.json(payload) - }) - } - - exports.process_ = function (tokenId, keywordName, argument, properties, callback) { - const outputBuilder = new OutputBuilder(function (output) { - console.log(`[Controller] Keyword ${keywordName} successfully executed on token ${tokenId}`) - callback(output) - }) - - try { - const filepathPromise = exports.filemanager.loadOrGetKeywordFile(agentContext.controllerUrl + '/grid/file/', properties['$node.js.file.id'], properties['$node.js.file.version'], keywordName) - - filepathPromise.then(function (keywordPackageFile) { - console.log('[Controller] Executing keyword ' + keywordName + ' using filepath ' + keywordPackageFile) - exports.executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder, agentContext) - }, function (err) { - console.log('[Controller] Error while attempting to run keyword ' + keywordName + ' :' + err) - outputBuilder.fail('Error while attempting to run keyword', err) - }) - } catch (e) { - outputBuilder.fail(e) - } - } - - exports.executeKeyword = async function (keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder, agentContext) { - try { - var kwDir - - if (keywordPackageFile.toUpperCase().endsWith('ZIP')) { - if (exports.filemanager.isFirstLevelKeywordFolder(keywordPackageFile)) { - kwDir = path.resolve(keywordPackageFile + '/keywords') - } else { - kwDir = path.resolve(keywordPackageFile + '/' + exports.filemanager.getFolderName(keywordPackageFile) + '/keywords') - } - } else { - // Local execution with KeywordRunner - kwDir = path.resolve(keywordPackageFile + '/keywords') - } - - console.log('[Controller] Search keyword file in ' + kwDir + ' for token ' + tokenId) - - var keywordFunction = searchAndRequireKeyword(kwDir, keywordName) - - if (keywordFunction) { - console.log('[Controller] Found keyword for token ' + tokenId) - let session = agentContext.tokenSessions[tokenId] - - if (!session) session = {} - - console.log('[Controller] Executing keyword ' + keywordName + ' on token ' + tokenId) - - try { - await keywordFunction(argument, outputBuilder, session, properties) - } catch (e) { - var onError = searchAndRequireKeyword(kwDir, 'onError') - if (onError) { - if (await onError(e, argument, outputBuilder, session, properties)) { - console.log('[Controller] Keyword execution marked as failed: onError function returned \'true\' on token ' + tokenId) - outputBuilder.fail(e) - } else { - console.log('[Controller] Keyword execution marked as successful: execution failed but the onError function returned \'false\' on token ' + tokenId) - outputBuilder.send() - } - } else { - console.log('[Controller] Keyword execution marked as failed: Keyword execution failed and no onError function found on token ' + tokenId) - outputBuilder.fail(e) - } - } - } else { - outputBuilder.fail('Unable to find keyword ' + keywordName) - } - } catch (e) { - outputBuilder.fail('An error occured while attempting to execute the keyword ' + keywordName, e) - } - } - - function searchAndRequireKeyword (kwDir, keywordName) { - var keywordFunction - var kwFiles = fs.readdirSync(kwDir) - kwFiles.every(function (kwFile) { - if (kwFile.endsWith('.js')) { - const kwMod = require(kwDir + '/' + kwFile) - if (kwMod[keywordName]) { - keywordFunction = kwMod[keywordName] - return false - } - } - return true - }) - return keywordFunction - } - - return exports -} diff --git a/step-node/step-node-agent/api/controllers/output.js b/step-node/step-node-agent/api/controllers/output.js index 4bf1f99433..1c3fb649b7 100644 --- a/step-node/step-node-agent/api/controllers/output.js +++ b/step-node/step-node-agent/api/controllers/output.js @@ -1,61 +1,191 @@ -module.exports = function OutputBuilder (callback) { - let exports = {} +const MeasureStatus = Object.freeze({ + PASSED: 'PASSED', + FAILED: 'FAILED', + TECHNICAL_ERROR: 'TECHNICAL_ERROR', +}); - exports.builder = { payload: { attachments: [], payload: {} }, attachments: [] } +const VALID_STATUSES = new Set(Object.values(MeasureStatus)); - exports.send = function (payload) { - exports.builder.payload.payload = payload - if (callback) { - callback(exports.builder) - } +function assertValidStatus(status) { + if (status !== undefined && !VALID_STATUSES.has(status)) { + throw new TypeError(`Invalid measure status: "${status}". Must be one of: ${[...VALID_STATUSES].join(', ')}`); } +} - function buildDefaultTechnicalError (message) { - return { msg: message, type: 'TECHNICAL', root: true, code: 0 } - } +/** + * Represents a file attachment added to a keyword output. + * Mirrors the Java class `step.grid.io.Attachment`. + * + * @typedef {object} Attachment + * @property {string} name - Display name of the attachment (e.g. "screenshot.png"). + * @property {string} [mimeType] - MIME type of the content (e.g. "image/png", "text/plain"). + * @property {string} [description] - Human-readable description shown in the Step UI. + * @property {string} [hexContent] - File content encoded as a **base64** string despite the field + * name. Use `Buffer.from(data).toString('base64')` to produce it. + * @property {boolean} [isDirectory] - Set to `true` when the attachment represents a directory + * rather than a single file. + */ - function buildDefaultBusinessError (message) { - return { msg: message, type: 'BUSINESS', root: true, code: 0 } +class OutputBuilder { + constructor() { + this.builder = { payload: { attachments: [], payload: {} }, attachments: [] } + this._currentMeasure = null; } - function attachException (e) { - exports.attach( - { - 'name': 'exception.log', - 'isDirectory': false, - 'description': 'exception stacktrace from keyword', - 'hexContent': Buffer.from(e.stack).toString('base64') - }) + /** + * Adds an output attribute to the payload. Can be called multiple times to build the payload + * incrementally. The accumulated payload is returned automatically — no need to call send(). + * @param {string} name + * @param {*} value + * @returns {OutputBuilder} this, for chaining + */ + add(name, value) { + this.builder.payload.payload[name] = value; + return this; } - exports.fail = function (arg1, arg2) { - exports.setError(arg1, arg2) - if (callback) { - callback(exports.builder) + /** + * @deprecated Calling send() is no longer required. The output accumulated via add(), + * setError(), attach(), etc. is returned automatically when the keyword finishes. + * This method is kept for backward compatibility and has no negative effect if called: + * - send() with no arguments is a no-op. + * - send(payload) still replaces the entire payload object, for code that relied on that behaviour. + * @param {object} [payload] + */ + send(payload) { + if (payload !== undefined) { + this.builder.payload.payload = payload; } } - exports.setError = function (arg1, arg2) { + merge(output) { + this.builder.payload = output; + } + + fail(arg1, arg2) { + this.setError(arg1, arg2) + } + + build() { + return this.builder; + } + + /** + * Sets a technical error, replacing any existing error. + * Accepts a string message, an Error object, or a raw error object. + * When a string + Error are passed the exception stack is attached. + * @returns {OutputBuilder} this, for chaining + */ + setError(arg1, arg2) { if (typeof arg1 === 'string' || arg1 instanceof String) { - exports.builder.payload.error = buildDefaultTechnicalError(arg1) + this.builder.payload.error = buildDefaultTechnicalError(String(arg1)) if (arg2 && arg2 instanceof Error) { - attachException(arg2) + this.#attachException(arg2) } } else if (arg1 instanceof Error) { - exports.builder.payload.error = buildDefaultTechnicalError(arg1.message) - attachException(arg1) + this.builder.payload.error = buildDefaultTechnicalError(arg1.message) + this.#attachException(arg1) } else if (typeof arg1 === 'object') { - exports.builder.payload.error = arg1 + this.builder.payload.error = arg1 + } + return this; + } + + /** + * Appends a message to the existing technical error, or creates one if none exists. + * @param {string} message + * @returns {OutputBuilder} this, for chaining + */ + appendError(message) { + if (this.builder.payload.error) { + this.builder.payload.error.msg = (this.builder.payload.error.msg || '') + message; + } else { + this.builder.payload.error = buildDefaultTechnicalError(message); } + return this; } - exports.setBusinessError = function (errorMessage) { - exports.builder.payload.error = buildDefaultBusinessError(errorMessage) + hasError() { + return this.builder.payload.error; } - exports.attach = function (attachment) { - exports.builder.payload.attachments.push(attachment) + /** + * Sets a business error (results in FAILED status rather than ERROR in Step). + * @param {string} errorMessage + * @returns {OutputBuilder} this, for chaining + */ + setBusinessError(errorMessage) { + this.builder.payload.error = buildDefaultBusinessError(errorMessage) + return this; } - return exports + /** + * Adds an attachment to the output. + * @param {Attachment} attachment + * @returns {OutputBuilder} this, for chaining + */ + attach(attachment) { + this.builder.payload.attachments.push(attachment) + } + + // --- Measurement methods --- + + /** + * Starts a measurement with the given name. Must be followed by stopMeasure(). + * @param {string} id - the measurement name + * @param {number} [begin=Date.now()] - explicit start timestamp in ms + */ + startMeasure(id, begin = Date.now()) { + this._currentMeasure = { id, begin }; + } + + /** + * Stops the current measurement started by startMeasure() and records it. + * @param {{ status?: MeasureStatus, data?: object }} [options] + */ + stopMeasure({ status = MeasureStatus.PASSED, data } = {}) { + assertValidStatus(status); + if (!this._currentMeasure) return; + const duration = Date.now() - this._currentMeasure.begin; + this.addMeasure(this._currentMeasure.id, duration, { begin: this._currentMeasure.begin, status, data }); + this._currentMeasure = null; + } + + /** + * Adds a pre-timed measurement directly. + * @param {string} name + * @param {number} durationMillis + * @param {{ begin?: number, data?: object, status?: MeasureStatus }} [options] + */ + addMeasure(name, durationMillis, { begin, data, status = MeasureStatus.PASSED } = {}) { + assertValidStatus(status); + if (!this.builder.payload.measures) { + this.builder.payload.measures = []; + } + const measure = { name, duration: durationMillis, status }; + if (begin !== undefined) measure.begin = begin; + if (data !== undefined) measure.data = data; + this.builder.payload.measures.push(measure); + } + + // --- Private helpers --- + + #attachException(e) { + this.attach({ + 'name': 'exception.log', + 'isDirectory': false, + 'description': 'exception stacktrace from keyword', + 'hexContent': Buffer.from(e.stack).toString('base64') + }) + } +} + +function buildDefaultTechnicalError(message) { + return { msg: message, type: 'TECHNICAL', root: true, code: 0 } } + +function buildDefaultBusinessError(message) { + return { msg: message, type: 'BUSINESS', root: true, code: 0 } +} + +module.exports = { OutputBuilder, MeasureStatus }; diff --git a/step-node/step-node-agent/api/controllers/session.js b/step-node/step-node-agent/api/controllers/session.js new file mode 100644 index 0000000000..58b1c3c3d3 --- /dev/null +++ b/step-node/step-node-agent/api/controllers/session.js @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2026, exense GmbH + * + * This file is part of Step + * + * Step is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Step is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Step. If not, see . + */ +let logger; +try { + logger = require('../logger').child({ component: 'Session' }); +} catch { + logger = { info: console.log.bind(console), warn: console.warn.bind(console), error: console.error.bind(console) }; +} + +class Session extends Map { + + async asyncDispose() { + logger.info(`Async-disposing Session: Cleaning up ${this.size} resources...`); + const promises = []; + + for (const [key, resource] of this) { + try { + if (resource && typeof resource[Symbol.asyncDispose] === 'function') { + promises.push(resource[Symbol.asyncDispose]()); + } else if (resource && typeof resource[Symbol.dispose] === 'function') { + resource[Symbol.dispose](); + } else if (resource && typeof resource.kill === 'function') { + resource.kill(); + } else if (resource && typeof resource.close === 'function') { + const result = resource.close(); + if (result && typeof result.then === 'function') promises.push(result); + } + logger.debug(`Successfully closed resource: ${key}`); + } catch (err) { + logger.error(`Failed to close resource ${key}:`, err); + } + } + + // Clean up Object properties (Added via .dot notation) + for (const key of Object.keys(this)) { + const resource = this[key]; + if (resource && typeof resource.close === 'function') { + try { + const result = resource.close(); + if (result && typeof result.then === 'function') promises.push(result); + } catch (err) { + logger.error(`Failed to close dot-notation resource ${key}:`, err); + } + } + } + + await Promise.allSettled(promises); + this.clear(); + } +} +module.exports = Session; diff --git a/step-node/step-node-agent/api/filemanager/filemanager.js b/step-node/step-node-agent/api/filemanager/filemanager.js index de67d10295..eab0cba3cb 100644 --- a/step-node/step-node-agent/api/filemanager/filemanager.js +++ b/step-node/step-node-agent/api/filemanager/filemanager.js @@ -1,49 +1,38 @@ -module.exports = function FileManager (agentContext) { - const fs = require('fs') - const shell = require('shelljs') - const http = require('http') - const https = require('https') - const url = require('url') - const unzip = require('unzip-stream') - const jwtUtils = require('../../utils/jwtUtils') - - let exports = {} - const filemanagerPath = agentContext.properties['filemanagerPath'] ? agentContext.properties['filemanagerPath'] : 'filemanager' - const workingDir = filemanagerPath + '/work/' - console.log('[FileManager] Starting file manager using working directory: ' + workingDir) - - console.log('[FileManager] Clearing working dir: ' + workingDir) - shell.rm('-rf', workingDir) - shell.mkdir('-p', workingDir) - - let filemanagerMap = {} - - exports.isFirstLevelKeywordFolder = function (path) { - if (fs.existsSync(path + '/keywords')) { - return true - } - return false +const fs = require('fs') +const http = require('http') +const https = require('https') +const url = require('url') +const unzip = require('unzip-stream') +const jwtUtils = require('../../utils/jwtUtils') +const logger = require('../logger').child({ component: 'FileManager' }) + +class FileManager { + constructor(agentContext) { + this.agentContext = agentContext; + const filemanagerPath = agentContext.properties['filemanagerPath'] || 'filemanager' + this.workingDir = filemanagerPath + '/work/' + logger.info('Starting file manager using working directory: ' + this.workingDir) + + logger.info('Clearing working dir: ' + this.workingDir) + fs.rmSync(this.workingDir, { recursive: true, force: true }) + fs.mkdirSync(this.workingDir, { recursive: true }) + + this.filemanagerMap = {} } - exports.getFolderName = function (keywordPackageFile) { - try { - let splitNodes = keywordPackageFile.split('/') - let lastNode = splitNodes[splitNodes.length - 1] - let splitExt = lastNode.split('.') - return splitExt[0] - } catch (e) { - throw new Error('A problem occured while attempting to retrieve subfolder name from zipped project:' + keywordPackageFile) - } + isFirstLevelKeywordFolder(filePath) { + return fs.existsSync(filePath + '/keywords') } - exports.loadOrGetKeywordFile = function (controllerUrl, fileId, fileVersionId) { - return new Promise(function (resolve, reject) { - const filePath = workingDir + fileId + '/' + fileVersionId - const cacheEntry = getCacheEntry(fileId, fileVersionId) + async loadOrGetKeywordFile(controllerUrl, fileId, fileVersionId) { + return new Promise((resolve, reject) => { + const filePath = this.workingDir + fileId + '/' + fileVersionId + + const cacheEntry = this.#getCacheEntry(fileId, fileVersionId) if (cacheEntry) { if (!cacheEntry.loading) { - console.log('[FileManager] Entry found for fileId ' + fileId + ': ' + cacheEntry.name) + logger.info('Entry found for fileId ' + fileId + ': ' + cacheEntry.name) const fileName = cacheEntry.name if (fs.existsSync(filePath + '/' + fileName)) { @@ -52,56 +41,54 @@ module.exports = function FileManager (agentContext) { reject(new Error('Entry exists but no file found: ' + filePath + '/' + fileName)) } } else { - console.log('[FileManager] Waiting for cache entry to be loaded for fileId ' + fileId) + logger.info('Waiting for cache entry to be loaded for fileId ' + fileId) cacheEntry.promises.push((result) => { - console.log('[FileManager] Cache entry loaded for fileId ' + fileId) + logger.info('Cache entry loaded for fileId ' + fileId) resolve(result) }) } } else { - putCacheEntry(fileId, fileVersionId, {'loading': true, 'promises': []}) - - console.log('[FileManager] No entry found for fileId ' + fileId + '. Loading...') - shell.mkdir('-p', filePath) - console.log('[FileManager] Created file path: ' + filePath + ' for fileId ' + fileId) + this.#putCacheEntry(fileId, fileVersionId, { loading: true, promises: [] }) - var fileVersionUrl = controllerUrl + fileId + '/' + fileVersionId - console.log('[FileManager] Requesting file from: ' + fileVersionUrl) - const filenamePromise = getKeywordFile(fileVersionUrl, filePath) + logger.info('No entry found for fileId ' + fileId + '. Loading...') + fs.mkdirSync(filePath, { recursive: true }) + logger.info('Created file path: ' + filePath + ' for fileId ' + fileId) - filenamePromise.then(function (result) { - console.log('[FileManager] Transfered file ' + result + ' from ' + fileVersionUrl) + const fileVersionUrl = controllerUrl + fileId + '/' + fileVersionId + logger.info('Requesting file from: ' + fileVersionUrl) + this.#getKeywordFile(fileVersionUrl, filePath).then((result) => { + logger.info('Transferred file ' + result + ' from ' + fileVersionUrl) - let cacheEntry = getCacheEntry(fileId, fileVersionId) + const cacheEntry = this.#getCacheEntry(fileId, fileVersionId) cacheEntry.name = result cacheEntry.loading = false - putCacheEntry(fileId, fileVersionId, cacheEntry) + this.#putCacheEntry(fileId, fileVersionId, cacheEntry) if (cacheEntry.promises) { - cacheEntry.promises.forEach(callback => callback(filePath + '/' + result)) // eslint-disable-line + cacheEntry.promises.forEach(callback => callback(filePath + '/' + result)) } delete cacheEntry.promises resolve(filePath + '/' + result) - }, function (err) { - console.log('Error :' + err) + }, (err) => { + logger.error('Error downloading file:', err) reject(err) }) } }) } - function getCacheEntry (fileId, fileVersion) { - return filemanagerMap[fileId + fileVersion] + #getCacheEntry(fileId, fileVersion) { + return this.filemanagerMap[fileId + fileVersion] } - function putCacheEntry (fileId, fileVersion, entry) { - filemanagerMap[fileId + fileVersion] = entry + #putCacheEntry(fileId, fileVersion, entry) { + this.filemanagerMap[fileId + fileVersion] = entry } - function getKeywordFile (controllerFileUrl, targetDir) { - return new Promise(function (resolve, reject) { + #getKeywordFile(controllerFileUrl, targetDir) { + return new Promise((resolve, reject) => { const parsedUrl = url.parse(controllerFileUrl) const httpModule = parsedUrl.protocol === 'https:' ? https : http @@ -113,24 +100,22 @@ module.exports = function FileManager (agentContext) { } // Add bearer token if gridSecurity is configured - const token = jwtUtils.generateJwtToken(agentContext.gridSecurity, 300); // 5 minutes expiration + const token = jwtUtils.generateJwtToken(this.agentContext.gridSecurity, 300); // 5 minutes expiration if (token) { - requestOptions.headers = { - 'Authorization': 'Bearer ' + token - }; + requestOptions.headers = { 'Authorization': 'Bearer ' + token }; } const req = httpModule.request(requestOptions, (resp) => { - const filename = parseName(resp.headers) + const filename = this.#parseName(resp.headers) const filepath = targetDir + '/' + filename - if (isDir(resp.headers) || filename.toUpperCase().endsWith('ZIP')) { + if (this.#isDir(resp.headers) || filename.toUpperCase().endsWith('ZIP')) { resp.pipe(unzip.Extract({path: filepath})).on('close', () => resolve(filename)) } else { const myFile = fs.createWriteStream(filepath) resp.pipe(myFile).on('finish', () => resolve(filename)) } }).on('error', (err) => { - console.log('Error: ' + err.message) + logger.error('HTTP request error:', err) reject(err) }) @@ -138,14 +123,17 @@ module.exports = function FileManager (agentContext) { }) } - function parseName (headers) { - const contentDisposition = JSON.stringify(headers['content-disposition']) - return contentDisposition.split('filename = ')[1].split(';')[0] + #parseName(headers) { + const contentDisposition = headers['content-disposition'] || '' + const match = contentDisposition.match(/filename\s*=\s*([^;]+)/) + return match ? match[1].trim() : '' } - function isDir (headers) { - const contentDisposition = JSON.stringify(headers['content-disposition']) - return contentDisposition.split('type = ')[1].split(';')[0].startsWith('dir') + #isDir(headers) { + const contentDisposition = headers['content-disposition'] || '' + const match = contentDisposition.match(/type\s*=\s*([^;]+)/) + return match ? match[1].trim().startsWith('dir') : false } - return exports } + +module.exports = FileManager; diff --git a/step-node/step-node-agent/api/logger.js b/step-node/step-node-agent/api/logger.js new file mode 100644 index 0000000000..8ccbbfa103 --- /dev/null +++ b/step-node/step-node-agent/api/logger.js @@ -0,0 +1,17 @@ +const { createLogger, format, transports } = require('winston'); + +const logger = createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: format.combine( + format.errors({ stack: true }), + format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), + format.printf((info) => { + const prefix = info.component ? `[${info.component}] ` : ''; + const logMessage = info.stack ? `${info.message} - ${info.stack}` : info.message; + return `${info.timestamp} [${info.level.toUpperCase()}] ${prefix}${logMessage}` + }) + ), + transports: [new transports.Console()], +}); + +module.exports = logger; diff --git a/step-node/step-node-agent/api/routes/routes.js b/step-node/step-node-agent/api/routes/routes.js index ef7aeb76b0..8ede4b79c2 100644 --- a/step-node/step-node-agent/api/routes/routes.js +++ b/step-node/step-node-agent/api/routes/routes.js @@ -1,12 +1,12 @@ 'use strict' module.exports = function (app, agentContext) { - const Controller = require('../controllers/controller') + const Controller = require('../controllers/agent') const FileManager = require('../filemanager/filemanager') - const controller = new Controller(agentContext, new FileManager(agentContext)) + const controller = new Controller(agentContext, new FileManager(agentContext), 'agent') - app.route('/running').get(controller.isRunning) - app.route('/token/:tokenId/reserve').get(controller.reserveToken) - app.route('/token/:tokenId/release').get(controller.releaseToken) - app.route('/token/:tokenId/process').post(controller.process) - app.route('/token/:tokenId/interrupt-execution').post(controller.interruptExecution) + app.route('/running').get(controller.isRunning.bind(controller)) + app.route('/token/:tokenId/reserve').get(controller.reserveToken.bind(controller)) + app.route('/token/:tokenId/release').get(controller.releaseToken.bind(controller)) + app.route('/token/:tokenId/process').post(controller.process.bind(controller)) + app.route('/token/:tokenId/interrupt-execution').post(controller.interruptExecution.bind(controller)) } diff --git a/step-node/step-node-agent/api/runner/runner.js b/step-node/step-node-agent/api/runner/runner.js index ca52c22062..b58b31184a 100644 --- a/step-node/step-node-agent/api/runner/runner.js +++ b/step-node/step-node-agent/api/runner/runner.js @@ -1,26 +1,41 @@ +const Session = require('../controllers/session'); +const logger = require('../logger').child({ component: 'Runner' }) module.exports = function (properties = {}) { - const tokenId = 'local' + const tokenId = 'local'; + let throwExceptionOnError = true; const agentContext = {tokens: [], tokenSessions: {}, properties: properties} - agentContext.tokenSessions[tokenId] = {} + const tokenSession = new Session(); + agentContext.tokenSessions[tokenId] = tokenSession - var fileManager = { - loadOrGetKeywordFile: function (url, fileId, fileVersion, keywordName) { - return new Promise(function (resolve, reject) { - resolve('.') - }) - } + const fileManager = { + loadOrGetKeywordFile: () => Promise.resolve('.') } - const Controller = require('../controllers/controller') - const controller = new Controller(agentContext, fileManager) + const Controller = require('../controllers/agent') + const controller = new Controller(agentContext, fileManager, 'runner') const api = {} - api.run = function (keywordName, input) { - return new Promise(resolve => { - controller.process_(tokenId, keywordName, input, properties, function (output) { resolve(output.payload) }) - }) + api.setThrowExceptionOnError = function(isThrowExceptionOnError) { + throwExceptionOnError = isThrowExceptionOnError; + } + + api.run = async function (keywordName, input) { + const output = await controller.process_(tokenId, keywordName, input, properties) + const payload = output.payload; + if (payload.error) { + if(throwExceptionOnError) { + throw new Error('The keyword execution returned an error: ' + JSON.stringify(payload.error)) + } else { + logger.warn('The keyword execution returned an error: ' + JSON.stringify(payload.error)) + } + } + return output.payload + } + + api.close = async function () { + return await tokenSession.asyncDispose(); } return api diff --git a/step-node/step-node-agent/eslint.config.mjs b/step-node/step-node-agent/eslint.config.mjs new file mode 100644 index 0000000000..b530909961 --- /dev/null +++ b/step-node/step-node-agent/eslint.config.mjs @@ -0,0 +1,34 @@ +import js from "@eslint/js"; +import globals from "globals"; +import { defineConfig } from "eslint/config"; + +export default defineConfig([ + { + ignores: [ + "node_modules/**", + "filemanager/work/**", + "npm-project-workspaces/**", + ], + }, + { + files: ["**/*.{js,mjs,cjs}"], + plugins: { js }, + extends: ["js/recommended"], + languageOptions: { globals: globals.node }, + rules: { + // Allow intentionally-unused params/vars when prefixed with _ + "no-unused-vars": ["error", { vars: "all", args: "after-used", argsIgnorePattern: "^_", ignoreRestSiblings: true }], + }, + }, + { files: ["**/*.js"], languageOptions: { sourceType: "commonjs" } }, + // Test files: provide Jest globals + { + files: ["test/**/*.test.js", "test/**/*.spec.js"], + languageOptions: { globals: globals.jest }, + }, + // Keyword stubs intentionally declare the full API signature without using every param + { + files: ["test/keywords/**/*.js"], + rules: { "no-unused-vars": ["error", { vars: "all", args: "none" }] }, + }, +]); \ No newline at end of file diff --git a/step-node/step-node-agent/keywords/keywords.js b/step-node/step-node-agent/keywords/keywords.js deleted file mode 100644 index aaa52c450c..0000000000 --- a/step-node/step-node-agent/keywords/keywords.js +++ /dev/null @@ -1,50 +0,0 @@ -exports.Echo = async (input, output, session, properties) => { - input['properties'] = properties - output.send(input) -} - -exports.ErrorTestKW = async (input, output, session, properties) => { - throw new Error(input['ErrorMsg']) -} - -exports.SetErrorTestKW = async (input, output, session, properties) => { - output.setError(input['ErrorMsg']) - output.send() -} - -exports.SetErrorWithExceptionKW = async (input, output, session, properties) => { - output.setError(new Error(input['ErrorMsg'])) - output.send() -} - -exports.SetErrorWithMessageAndExceptionKW = async (input, output, session, properties) => { - output.setError(input['ErrorMsg'], new Error(input['ErrorMsg'])) - output.send() -} - -exports.FailKW = async (input, output, session, properties) => { - output.fail(input['ErrorMsg']) -} - -exports.BusinessErrorTestKW = async (input, output, session, properties) => { - output.setBusinessError(input['ErrorMsg']) - output.send() -} - -exports.ErrorRejectedPromiseTestKW = async (input, output, session, properties) => { - Promise.reject(new Error('test')) - output.send() -} - -exports.ErrorUncaughtExceptionTestKW = async (input, output, session, properties) => { - process.nextTick(function () { - throw new Error() - }) - output.send() -} - -exports.onError = async (exception, input, output, session, properties) => { - console.log('[onError] Exception is: \'' + exception + '\'') - global.isOnErrorCalled = true - return input['rethrow_error'] -} diff --git a/step-node/step-node-agent/package.json b/step-node/step-node-agent/package.json index 1bd9ec2534..53ee93f831 100644 --- a/step-node/step-node-agent/package.json +++ b/step-node/step-node-agent/package.json @@ -1,62 +1,74 @@ { "name": "step-node-agent", "version": "0.0.0", - "description": "The official STEP Agent implementation for Node.js", + "description": "The official Step Agent implementation for Node.js", "main": "index.js", "scripts": { "debug": "node --inspect-brk server.js", "eslint": "node_modules/.bin/eslint .", "start": "node server.js", - "test": "node test/test.js", + "test": "jest", "build": "", "deploy": "npm publish" }, - "author": "Jerome Comte", + "jest": { + "testMatch": [ + "**/test/**/*.test.js" + ], + "testTimeout": 30000 + }, + "step": { + "keywords": "test/keywords" + }, + "engines": { + "node": ">=20.0.0" + }, + "author": "exense GmbH", "license": "AGPL-3.0", "repository": { "type": "git", "url": "git@github.com:exense/step.git" }, "devDependencies": { - "eslint": "^4.19.1", - "eslint-config-standard": "^11.0.0", - "eslint-plugin-import": "^2.14.0", - "eslint-plugin-node": "^6.0.1", - "eslint-plugin-promise": "^3.8.0", - "eslint-plugin-standard": "^3.1.0", + "@eslint/js": "^10.0.1", + "eslint": "^10.0.3", + "globals": "^15.15.0", "husky": "^0.14.3", - "nodemon": "^1.18.7" + "jest": "^29.7.0", + "nodemon": "^3.1.14" }, "dependencies": { - "body-parser": "^1.19.1", "express": "^4.17.1", "get-fqdn": "^1.0.0", - "http": "0.0.0", - "jsonwebtoken": "^9.0.2", + "jsonwebtoken": "9.0.3", "minimist": "^1.2.5", - "npm": "^6.14.15", - "request": "^2.88.0", "shelljs": "^0.8.4", - "underscore": "^1.13.1", "unzip-stream": "^0.3.1", "uuid": "^3.4.0", + "winston": "^3.19.0", "yaml": "^2.4.2" }, "bundledDependencies": [ - "body-parser", "express", - "http", + "get-fqdn", + "jsonwebtoken", "minimist", - "npm", - "request", "shelljs", - "underscore", "unzip-stream", "uuid", - "yaml", - "get-fqdn" + "yaml" ], "bin": { "step-node-agent": "./bin/step-node-agent" - } + }, + "bundleDependencies": [ + "express", + "get-fqdn", + "jsonwebtoken", + "minimist", + "shelljs", + "unzip-stream", + "uuid", + "yaml" + ] } diff --git a/step-node/step-node-agent/server.js b/step-node/step-node-agent/server.js index 1f0ba2cf31..9acf768128 100644 --- a/step-node/step-node-agent/server.js +++ b/step-node/step-node-agent/server.js @@ -1,36 +1,39 @@ const minimist = require('minimist') const path = require('path') const YAML = require('yaml') +const logger = require('./api/logger').child({ component: 'Agent' }) let args = minimist(process.argv.slice(2), { default: { f: path.join(__dirname, 'AgentConf.yaml') } }) -console.log('[Agent] Using arguments ' + JSON.stringify(args)) +logger.info('Using arguments ' + JSON.stringify(args)) const agentConfFile = args.f -console.log('[Agent] Reading agent configuration ' + agentConfFile) +logger.info('Reading agent configuration ' + agentConfFile) const fs = require('fs') const content = fs.readFileSync(agentConfFile, 'utf8') const agentConfFileExt = path.extname(agentConfFile) -var agentConf -if(agentConfFileExt === '.yaml') { - agentConf = YAML.parse(content) -} else if(agentConfFileExt === '.json') { - agentConf = JSON.parse(content) -} else { - throw new Error('Unsupported extension ' + agentConfFileExt + " for agent configuration " + content); +const agentConf = parseAgentConf(); + +function parseAgentConf() { + if (agentConfFileExt === '.yaml') { + return YAML.parse(content) + } else if (agentConfFileExt === '.json') { + return JSON.parse(content) + } else { + throw new Error('Unsupported extension ' + agentConfFileExt + " for agent configuration " + content); + } } -console.log('[Agent] Creating agent context and tokens') +logger.info('Creating agent context and tokens') const uuid = require('uuid/v4') -const _ = require('underscore') const jwtUtils = require('./utils/jwtUtils') const agentType = 'node' const agent = {id: uuid()} -let agentContext = { tokens: [], tokenSessions: [], tokenProperties: [], properties: agentConf.properties, controllerUrl: agentConf.gridHost, gridSecurity: agentConf.gridSecurity } -_.each(agentConf.tokenGroups, function (tokenGroup) { +const agentContext = { tokens: [], tokenSessions: [], tokenProperties: [], properties: agentConf.properties, controllerUrl: agentConf.gridHost, gridSecurity: agentConf.gridSecurity, workingDir: agentConf.workingDir, npmProjectWorkspaceCleanupIdleTimeMs: agentConf.npmProjectWorkspaceCleanupIdleTimeMs } +agentConf.tokenGroups.forEach(function (tokenGroup) { const tokenConf = tokenGroup.tokenConf let attributes = tokenConf.attributes // Transform the selectionPatterns map to @@ -46,20 +49,18 @@ _.each(agentConf.tokenGroups, function (tokenGroup) { for (let i = 0; i < tokenGroup.capacity; i++) { const token = { id: uuid(), agentid: agent.id, attributes: attributes, selectionPatterns: tokenSelectionPatterns } agentContext.tokens.push(token) - agentContext.tokenSessions[token.id] = {} + agentContext.tokenSessions[token.id] = null agentContext.tokenProperties[token.id] = additionalProperties } }) -console.log('[Agent] Starting agent services') +logger.info('Starting agent services') const express = require('express') const app = express() const port = agentConf.agentPort || 3000 const timeout = agentConf.agentServerTimeout || 600000 -const bodyParser = require('body-parser') - -app.use(bodyParser.urlencoded({extended: true})) -app.use(bodyParser.json()) +app.use(express.urlencoded({extended: true})) +app.use(express.json()) // Apply JWT authentication middleware const createJwtAuthMiddleware = require('./middleware/jwtAuth') @@ -69,41 +70,39 @@ app.use(jwtAuthMiddleware) const routes = require('./api/routes/routes') routes(app, agentContext) -var server = app.listen(port) +const server = app.listen(port) server.setTimeout(timeout) -startWithAgentUrl = function(agentUrl) { - console.log('[Agent] Registering agent as ' + agentUrl + ' to grid ' + agentConf.gridHost) - console.log('[Agent] Creating registration timer') +const startWithAgentUrl = async function(agentUrl) { + logger.info('Registering agent as ' + agentUrl + ' to grid ' + agentConf.gridHost) + logger.info('Creating registration timer') const registrationPeriod = agentConf.registrationPeriod || 5000 - const request = require('request') - setInterval(function () { - const requestOptions = { - uri: agentConf.gridHost + '/grid/register', - method: 'POST', - json: true, - body: { agentRef: { agentId: agent.id, agentUrl: agentUrl, agentType: agentType }, tokens: agentContext.tokens } - }; + setInterval(async () => { + const body = { agentRef: { agentId: agent.id, agentUrl: agentUrl, agentType: agentType }, tokens: agentContext.tokens } + const headers = { 'Content-Type': 'application/json' } // Add bearer token if gridSecurity is configured const token = jwtUtils.generateJwtToken(agentConf.gridSecurity, 3600); // 1 hour expiration if (token) { - requestOptions.headers = { - 'Authorization': 'Bearer ' + token - }; + headers['Authorization'] = 'Bearer ' + token; } - request(requestOptions, function (err, res, body) { - if (err) { - console.log("[Agent] Error while registering agent to grid") - console.log(err) - } else if (res.statusCode !== 204) { - console.log("[Agent] Failed to register agent: grid responded with status " + res.statusCode + (body != null ? ". Response body: " + JSON.stringify(body) : "")) + try { + const res = await fetch(agentConf.gridHost + '/grid/register', { + method: 'POST', + headers, + body: JSON.stringify(body) + }) + if (res.status !== 204) { + const responseBody = await res.text().catch(() => null) + logger.warn('Failed to register agent: grid responded with status ' + res.status + (responseBody != null ? '. Response body: ' + responseBody : '')) } - }) + } catch (err) { + logger.error('Error while registering agent to grid:', err) + } }, registrationPeriod) - console.log('[Agent] Successfully started on: ' + port) + logger.info('Successfully started on port ' + port) } if(agentConf.agentUrl) { @@ -113,6 +112,14 @@ if(agentConf.agentUrl) { getFQDN().then(FQDN => { startWithAgentUrl('http://' + FQDN + ':' + port) }).catch(e => { - console.log('[Agent] Error while getting FQDN ' + e) + logger.error('Error while getting FQDN:', e) }) } + +const v8 = require('v8'); + +process.on('SIGUSR2', () => { + const fileName = `/tmp/heap-${process.pid}-${Date.now()}.heapsnapshot`; + v8.writeHeapSnapshot(fileName); + logger.info('Heap dump written to ' + fileName) +}); diff --git a/step-node/step-node-agent/test/keywords/keywords.js b/step-node/step-node-agent/test/keywords/keywords.js new file mode 100644 index 0000000000..1fd5dc4167 --- /dev/null +++ b/step-node/step-node-agent/test/keywords/keywords.js @@ -0,0 +1,152 @@ +exports.Echo = async (input, output, session, properties) => { + input['properties'] = properties + Object.entries(input).forEach(([k, v]) => output.add(k, v)) +} + +exports.ErrorTestKW = async (input, output, session, properties) => { + throw new Error(input['ErrorMsg']) +} + +exports.SetErrorTestKW = async (input, output, session, properties) => { + output.setError(input['ErrorMsg']) +} + +exports.SetErrorWithExceptionKW = async (input, output, session, properties) => { + output.setError(new Error(input['ErrorMsg'])) +} + +exports.SetErrorWithMessageAndExceptionKW = async (input, output, session, properties) => { + output.setError(input['ErrorMsg'], new Error(input['ErrorMsg'])) +} + +exports.FailKW = async (input, output, session, properties) => { + output.fail(input['ErrorMsg']) +} + +exports.BusinessErrorTestKW = async (input, output, session, properties) => { + output.setBusinessError(input['ErrorMsg']) +} + +exports.ErrorRejectedPromiseTestKW = async (input, output, session, properties) => { + Promise.reject(new Error('test')) +} + +exports.ErrorUncaughtExceptionTestKW = async (input, output, session, properties) => { + process.nextTick(function () { + throw new Error() + }) +} + +exports.onError = async (exception, input, output, session, properties) => { + console.log('[onError] Exception is: \'' + exception + '\'') + output.builder.payload.payload.onErrorCalled = true + return input['rethrow_error'] +} + +// --- output.add --- + +exports.AddKW = async (input, output, session, properties) => { + output.add('name', 'Alice').add('score', 42).add('active', true) +} + +// --- output.appendError --- + +exports.AppendErrorToExistingKW = async (input, output, session, properties) => { + output.setError('base error').appendError(' + extra detail') +} + +exports.AppendErrorToNoneKW = async (input, output, session, properties) => { + output.appendError('fresh error') +} + +// --- output.attach --- + +exports.AttachKW = async (input, output, session, properties) => { + output.attach({ name: 'report.txt', isDirectory: false, description: 'test attachment', hexContent: Buffer.from('hello').toString('base64') }) +} + +// --- measurement methods --- + +exports.StartStopMeasureKW = async (input, output, session, properties) => { + output.startMeasure('step1') + await new Promise(r => setTimeout(r, 10)) + output.stopMeasure() +} + +exports.StartStopMeasureWithStatusKW = async (input, output, session, properties) => { + output.startMeasure('failing-step') + output.stopMeasure({ status: 'FAILED', data: { reason: 'assertion failed' } }) +} + +exports.AddMeasureKW = async (input, output, session, properties) => { + output.addMeasure('pre-timed', 150, { status: 'TECHNICAL_ERROR', begin: Date.now() - 150, data: { info: 'test' } }) +} + +exports.MultipleMeasuresKW = async (input, output, session, properties) => { + output.startMeasure('first') + output.stopMeasure() + output.startMeasure('second') + output.stopMeasure({ status: 'FAILED' }) + output.addMeasure('third', 50) +} + +// --- session --- + +exports.SessionSetKW = async (input, output, session) => { + session.set('sharedKey', input['value']) +} + +exports.SessionGetKW = async (input, output, session) => { + output.add('value', session.get('sharedKey')) +} + +exports.SessionSetDotKW = async (input, output, session) => { + session.dotKey = input['value'] +} + +exports.SessionGetDotKW = async (input, output, session) => { + output.add('value', session.dotKey) +} + +// --- beforeKeyword / afterKeyword hook tracking --- + +let _hookCalls = [] + +exports.beforeKeyword = async (functionName) => { + _hookCalls.push(`before:${functionName}`) +} + +exports.afterKeyword = async (functionName) => { + _hookCalls.push(`after:${functionName}`) +} + +exports.GetHookCallsKW = async (input, output) => { + const calls = [..._hookCalls] + _hookCalls = [] + output.add('calls', calls) +} + +// --- properties --- + +exports.GetPropertyKW = async (input, output, session, properties) => { + output.add('value', properties[input['key']]) +} + +// --- session auto-disposal --- + +exports.StoreCloseableKW = async (input, output, session) => { + session.set('resource', { + close() { require('fs').writeFileSync(input['closePath'], 'closed') } + }) +} + +// --- backward compat: output.send() is no longer required but must still work --- + +exports.SendNoArgCompatKW = async (input, output) => { + output.add('result', 'ok') + output.send() +} + +exports.SendWithPayloadCompatKW = async (input, output) => { + output.send({ result: 'ok' }) +} diff --git a/step-node/step-node-agent/test/runner.test.js b/step-node/step-node-agent/test/runner.test.js new file mode 100644 index 0000000000..c21c97f80a --- /dev/null +++ b/step-node/step-node-agent/test/runner.test.js @@ -0,0 +1,333 @@ +const { OutputBuilder, MeasureStatus } = require('../api/controllers/output') + +describe('runner', () => { + let runner + + beforeAll(() => { + runner = require('../api/runner/runner')({ Property1: 'Prop1' }) + runner.setThrowExceptionOnError(false) + }) + + afterAll(async () => { + await runner.close() + }) + + // --------------------------------------------------------------------------- + // Happy path + // --------------------------------------------------------------------------- + + test('Echo KW returns input param and agent property', async () => { + const output = await runner.run('Echo', { Param1: 'Val1' }) + expect(output.payload.Param1).toBe('Val1') + expect(output.payload.properties.Property1).toBe('Prop1') + }) + + // --------------------------------------------------------------------------- + // Error-setting methods + // --------------------------------------------------------------------------- + + describe('output.setError', () => { + test('sets error message and TECHNICAL type', async () => { + const output = await runner.run('SetErrorTestKW', { ErrorMsg: 'MyError', rethrow_error: true }) + expect(output.error.msg).toBe('MyError') + expect(output.error.type).toBe('TECHNICAL') + }) + + test('accepts an exception as argument', async () => { + const output = await runner.run('SetErrorWithExceptionKW', { ErrorMsg: 'MyError2', rethrow_error: true }) + expect(output.error.msg).toBe('MyError2') + }) + + test('accepts a message and exception, attaches stack trace', async () => { + const output = await runner.run('SetErrorWithMessageAndExceptionKW', { ErrorMsg: 'MyError3', rethrow_error: true }) + expect(output.error.msg).toBe('MyError3') + expect(output.attachments.find(a => a.name === 'exception.log')).toBeDefined() + }) + }) + + test('output.fail sets error message', async () => { + const output = await runner.run('FailKW', { ErrorMsg: 'MyError4', rethrow_error: true }) + expect(output.error.msg).toBe('MyError4') + }) + + test('output.setBusinessError sets error message', async () => { + const output = await runner.run('BusinessErrorTestKW', { ErrorMsg: 'MyBusinessError', rethrow_error: true }) + expect(output.error.msg).toBe('MyBusinessError') + }) + + // --------------------------------------------------------------------------- + // onError hook + // --------------------------------------------------------------------------- + + describe('onError hook', () => { + test('is called and error is propagated when rethrow_error=true', async () => { + const output = await runner.run('ErrorTestKW', { ErrorMsg: 'Error - rethrow', rethrow_error: true }) + expect(output.error.msg).toBe('Error - rethrow') + expect(output.payload.onErrorCalled).toBe(true) + }) + + test('is called and error is suppressed when rethrow_error=false', async () => { + const output = await runner.run('ErrorTestKW', { ErrorMsg: 'Error - do not rethrow', rethrow_error: false }) + expect(output.error).toBeUndefined() + expect(output.payload.onErrorCalled).toBe(true) + }) + }) + + // --------------------------------------------------------------------------- + // Unhandled async errors + // --------------------------------------------------------------------------- + + test('rejected promises do not surface as output error', async () => { + const output = await runner.run('ErrorRejectedPromiseTestKW', { Param1: 'Val1' }) + expect(output.error).toBeUndefined() + }) + + test('uncaught exceptions do not surface as output error', async () => { + const output = await runner.run('ErrorUncaughtExceptionTestKW', { Param1: 'Val1' }) + expect(output.error).toBeUndefined() + }) + + // --------------------------------------------------------------------------- + // Unknown keyword + // --------------------------------------------------------------------------- + + test('returns error for non-existing keyword', async () => { + const output = await runner.run('Not existing Keyword', {}) + expect(output.error.msg).toBe("Unable to find Keyword 'Not existing Keyword'") + }) + + // --------------------------------------------------------------------------- + // output.add + // --------------------------------------------------------------------------- + + test('output.add builds payload incrementally', async () => { + const output = await runner.run('AddKW', {}) + expect(output.payload.name).toBe('Alice') + expect(output.payload.score).toBe(42) + expect(output.payload.active).toBe(true) + }) + + // --------------------------------------------------------------------------- + // output.appendError + // --------------------------------------------------------------------------- + + describe('output.appendError', () => { + test('appends to an existing error', async () => { + const output = await runner.run('AppendErrorToExistingKW', {}) + expect(output.error.msg).toBe('base error + extra detail') + expect(output.error.type).toBe('TECHNICAL') + }) + + test('creates a new error when none exists', async () => { + const output = await runner.run('AppendErrorToNoneKW', {}) + expect(output.error.msg).toBe('fresh error') + expect(output.error.type).toBe('TECHNICAL') + }) + }) + + // --------------------------------------------------------------------------- + // output.attach + // --------------------------------------------------------------------------- + + test('output.attach adds an attachment', async () => { + const output = await runner.run('AttachKW', {}) + expect(output.attachments).toHaveLength(1) + expect(output.attachments[0].name).toBe('report.txt') + expect(output.attachments[0].isDirectory).toBe(false) + }) + + // --------------------------------------------------------------------------- + // Measurement methods + // --------------------------------------------------------------------------- + + describe('measurements', () => { + test('startMeasure / stopMeasure produces a PASSED measure', async () => { + const output = await runner.run('StartStopMeasureKW', {}) + expect(output.measures).toHaveLength(1) + expect(output.measures[0].name).toBe('step1') + expect(output.measures[0].status).toBe(MeasureStatus.PASSED) + expect(output.measures[0].duration).toBeGreaterThanOrEqual(10) + expect(typeof output.measures[0].begin).toBe('number') + }) + + test('stopMeasure accepts an explicit FAILED status and custom data', async () => { + const output = await runner.run('StartStopMeasureWithStatusKW', {}) + expect(output.measures[0].name).toBe('failing-step') + expect(output.measures[0].status).toBe(MeasureStatus.FAILED) + expect(output.measures[0].data.reason).toBe('assertion failed') + }) + + test('addMeasure accepts a pre-set duration and TECHNICAL_ERROR status', async () => { + const output = await runner.run('AddMeasureKW', {}) + expect(output.measures[0].name).toBe('pre-timed') + expect(output.measures[0].duration).toBe(150) + expect(output.measures[0].status).toBe(MeasureStatus.TECHNICAL_ERROR) + expect(output.measures[0].data.info).toBe('test') + expect(typeof output.measures[0].begin).toBe('number') + }) + + test('multiple measures accumulate in a single keyword call', async () => { + const output = await runner.run('MultipleMeasuresKW', {}) + expect(output.measures).toHaveLength(3) + expect(output.measures[0].name).toBe('first') + expect(output.measures[0].status).toBe(MeasureStatus.PASSED) + expect(output.measures[1].name).toBe('second') + expect(output.measures[1].status).toBe(MeasureStatus.FAILED) + expect(output.measures[2].name).toBe('third') + expect(output.measures[2].duration).toBe(50) + }) + }) + + // --------------------------------------------------------------------------- + // session + // --------------------------------------------------------------------------- + + describe('session', () => { + test('values stored via session.set() are accessible in a later keyword via session.get()', async () => { + await runner.run('SessionSetKW', { value: 'hello' }) + const output = await runner.run('SessionGetKW') + expect(output.payload.value).toBe('hello') + }) + + test('values stored via dot notation are accessible in a later keyword', async () => { + await runner.run('SessionSetDotKW', { value: 'world' }) + const output = await runner.run('SessionGetDotKW') + expect(output.payload.value).toBe('world') + }) + + test('session value is updated when set again', async () => { + await runner.run('SessionSetKW', { value: 'first' }) + await runner.run('SessionSetKW', { value: 'second' }) + const output = await runner.run('SessionGetKW') + expect(output.payload.value).toBe('second') + }) + }) + + // --------------------------------------------------------------------------- + // session auto-disposal on runner.close() + // + // Keywords run in a forked subprocess whose own session holds user-stored + // resources. When runner.close() is called, the fork receives KILL and its + // session disposes those resources. The close() side-effect (writing a temp + // file) is the observable signal crossing the process boundary. + // --------------------------------------------------------------------------- + + describe('session auto-disposal on runner.close()', () => { + const os = require('os') + const path = require('path') + const fs = require('fs') + + test('close() is called on a resource stored in the session when runner.close() is invoked', async () => { + const closePath = path.join(os.tmpdir(), `step-session-close-${Date.now()}.txt`) + const r = require('../api/runner/runner')() + r.setThrowExceptionOnError(false) + try { + await r.run('StoreCloseableKW', { closePath }) + expect(fs.existsSync(closePath)).toBe(false) // resource not yet closed + + await r.close() + + expect(fs.existsSync(closePath)).toBe(true) // resource closed synchronously with the fork + } finally { + if (fs.existsSync(closePath)) fs.unlinkSync(closePath) + } + }) + }) + + // --------------------------------------------------------------------------- + // properties + // --------------------------------------------------------------------------- + + describe('properties', () => { + test('property passed to runner constructor is accessible in a keyword', async () => { + const output = await runner.run('GetPropertyKW', { key: 'Property1' }) + expect(output.payload.value).toBe('Prop1') + }) + + test('all properties passed to runner constructor are accessible', async () => { + const r = require('../api/runner/runner')({ KeyA: 'ValA', KeyB: 'ValB' }) + r.setThrowExceptionOnError(false) + try { + const outputA = await r.run('GetPropertyKW', { key: 'KeyA' }) + const outputB = await r.run('GetPropertyKW', { key: 'KeyB' }) + expect(outputA.payload.value).toBe('ValA') + expect(outputB.payload.value).toBe('ValB') + } finally { + await r.close() + } + }) + + test('runner without properties gives keywords an empty properties object', async () => { + const r = require('../api/runner/runner')() + r.setThrowExceptionOnError(false) + try { + const output = await r.run('GetPropertyKW', { key: 'anyKey' }) + expect(output.payload.value).toBeUndefined() + expect(output.error).toBeUndefined() + } finally { + await r.close() + } + }) + }) + + // --------------------------------------------------------------------------- + // beforeKeyword and afterKeyword hooks + // --------------------------------------------------------------------------- + + describe('beforeKeyword and afterKeyword hooks', () => { + beforeEach(async () => { + await runner.run('GetHookCallsKW') + }) + + test('beforeKeyword is called with the keyword name before execution', async () => { + await runner.run('Echo', {}) + const { payload: { calls } } = await runner.run('GetHookCallsKW') + expect(calls).toContain('before:Echo') + }) + + test('afterKeyword is called with the keyword name after successful execution', async () => { + await runner.run('Echo', {}) + const { payload: { calls } } = await runner.run('GetHookCallsKW') + expect(calls).toContain('after:Echo') + }) + + test('beforeKeyword is called before afterKeyword', async () => { + await runner.run('Echo', {}) + const { payload: { calls } } = await runner.run('GetHookCallsKW') + expect(calls.indexOf('before:Echo')).toBeLessThan(calls.indexOf('after:Echo')) + }) + + test('afterKeyword is called even when the keyword throws', async () => { + await runner.run('ErrorTestKW', { ErrorMsg: 'test error', rethrow_error: false }) + const { payload: { calls } } = await runner.run('GetHookCallsKW') + expect(calls).toContain('after:ErrorTestKW') + }) + }) +}) + +// --------------------------------------------------------------------------- +// MeasureStatus validation (no runner needed) +// --------------------------------------------------------------------------- + +describe('MeasureStatus validation', () => { + test('stopMeasure throws TypeError for unknown status', () => { + const ob = new OutputBuilder(null) + ob.startMeasure('test') + expect(() => ob.stopMeasure({ status: 'INVALID_STATUS' })).toThrow(TypeError) + expect(() => ob.stopMeasure({ status: 'INVALID_STATUS' })).toThrow('INVALID_STATUS') + }) + + test('addMeasure throws TypeError for unknown status', () => { + const ob = new OutputBuilder(null) + expect(() => ob.addMeasure('test', 100, { status: 'WRONG' })).toThrow(TypeError) + expect(() => ob.addMeasure('test', 100, { status: 'WRONG' })).toThrow('WRONG') + }) + + test('all valid MeasureStatus values are accepted without throwing', () => { + const ob = new OutputBuilder(null) + for (const status of Object.values(MeasureStatus)) { + ob.startMeasure('check') + expect(() => ob.stopMeasure({ status })).not.toThrow() + } + }) +}) diff --git a/step-node/step-node-agent/test/session.test.js b/step-node/step-node-agent/test/session.test.js new file mode 100644 index 0000000000..0d755f1f43 --- /dev/null +++ b/step-node/step-node-agent/test/session.test.js @@ -0,0 +1,108 @@ +const Session = require("../api/controllers/session"); + +describe('Session auto-disposal', () => { + + test('[Symbol.dispose]() is called on resources stored via session.set()', async () => { + const session = new Session() + const disposed = jest.fn() + session.set('res', {[Symbol.dispose]: disposed}) + await session.asyncDispose() + expect(disposed).toHaveBeenCalledTimes(1) + }) + + test('.close() is called when the resource has no [Symbol.dispose]', async () => { + const session = new Session() + const closed = jest.fn() + session.set('res', {close: closed}) + await session.asyncDispose() + expect(closed).toHaveBeenCalledTimes(1) + }) + + test('.kill() is called when the resource has no [Symbol.dispose] or .close()', async () => { + const session = new Session() + const killed = jest.fn() + session.set('res', {kill: killed}) + await session.asyncDispose() + expect(killed).toHaveBeenCalledTimes(1) + }) + + test('[Symbol.asyncDispose]() is awaited before asyncDispose() resolves', async () => { + const session = new Session() + let resolved = false + session.set('res', { + [Symbol.asyncDispose]: async () => { + await new Promise(r => setTimeout(r, 10)) + resolved = true + } + }) + await session.asyncDispose() + expect(resolved).toBe(true) + }) + + test('[Symbol.asyncDispose]() takes precedence over .close() and .kill()', async () => { + const session = new Session() + const asyncDisposed = jest.fn().mockResolvedValue(undefined) + const closed = jest.fn() + const killed = jest.fn() + session.set('res', {[Symbol.asyncDispose]: asyncDisposed, close: closed, kill: killed}) + await session.asyncDispose() + expect(asyncDisposed).toHaveBeenCalledTimes(1) + expect(closed).not.toHaveBeenCalled() + expect(killed).not.toHaveBeenCalled() + }) + + test('[Symbol.dispose]() takes precedence over .close() and .kill()', async () => { + const session = new Session() + const disposed = jest.fn() + const closed = jest.fn() + const killed = jest.fn() + session.set('res', {[Symbol.dispose]: disposed, close: closed, kill: killed}) + await session.asyncDispose() + expect(disposed).toHaveBeenCalledTimes(1) + expect(closed).not.toHaveBeenCalled() + expect(killed).not.toHaveBeenCalled() + }) + + test('.close() is called on resources stored via dot notation', async () => { + const session = new Session() + const closed = jest.fn() + session.dotResource = {close: closed} + await session.asyncDispose() + expect(closed).toHaveBeenCalledTimes(1) + }) + + test('resources with no disposal method are silently skipped', () => { + const session = new Session() + session.set('plain', { value: 42 }) + expect(async () => await session.asyncDispose()).not.toThrow() + }) + + test('all entries are cleared after disposal', async () => { + const session = new Session() + session.set('a', {close: jest.fn()}) + session.set('b', {close: jest.fn()}) + await session.asyncDispose() + expect(session.size).toBe(0) + }) + + test('disposal continues for remaining resources when one throws', async () => { + const session = new Session() + session.set('bad', { + [Symbol.dispose]: () => { + throw new Error('oops') + } + }) + const goodClosed = jest.fn() + session.set('good', {close: goodClosed}) + await session.asyncDispose() + expect(goodClosed).toHaveBeenCalledTimes(1) + }) + + test('multiple resources stored via session.set() are all disposed', async () => { + const session = new Session() + const fns = [jest.fn(), jest.fn(), jest.fn()] + fns.forEach((fn, i) => session.set(`res${i}`, {[Symbol.dispose]: fn})) + await session.asyncDispose() + fns.forEach(fn => expect(fn).toHaveBeenCalledTimes(1)) + }) +}) diff --git a/step-node/step-node-agent/test/test.js b/step-node/step-node-agent/test/test.js deleted file mode 100644 index 0e47be0bc0..0000000000 --- a/step-node/step-node-agent/test/test.js +++ /dev/null @@ -1,60 +0,0 @@ -const runner = require('../api/runner/runner')({'Property1': 'Prop1'}) -const assert = require('assert') - -;(async () => { - // Test the happy path - var output = await runner.run('Echo', {Param1: 'Val1'}) - assert.equal(output.payload.Param1, 'Val1') - assert.equal(output.payload.properties.Property1, 'Prop1') - - // Test the method output.setError - var errorMsg = 'MyError' - output = await runner.run('SetErrorTestKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - assert.equal(output.error.type, 'TECHNICAL') - - // Test the method output.setError with an exception as argument - errorMsg = 'MyError2' - output = await runner.run('SetErrorWithExceptionKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - - // Test the method output.setError with an error message and an exception as argument - errorMsg = 'MyError3' - output = await runner.run('SetErrorWithMessageAndExceptionKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - assert.equal(output.attachments.length, 1) - - // Test the method output.fail - errorMsg = 'MyError4' - output = await runner.run('FailKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - - // Test the method output.setBusinessError - errorMsg = 'MyBusinessError' - output = await runner.run('BusinessErrorTestKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - - // Test onError hook - global.isOnErrorCalled = false - const errorMsg1 = 'Error - rethrow' - const output1 = await runner.run('ErrorTestKW', {ErrorMsg: errorMsg1, rethrow_error: true}) - assert.equal(output1.error.msg, errorMsg1) - assert.equal(global.isOnErrorCalled, true) - - // Test onError hook with no rethrow - global.isOnErrorCalled = false - const errorMsg2 = 'Error - do not rethrow' - const output2 = await runner.run('ErrorTestKW', {ErrorMsg: errorMsg2, rethrow_error: false}) - assert.equal(output2.error, undefined) - assert.equal(global.isOnErrorCalled, true) - - // Test rejected promises - output = await runner.run('ErrorRejectedPromiseTestKW', {Param1: 'Val1'}) - assert.equal(output.error, undefined) - - // Test uncaught exceptions - output = await runner.run('ErrorUncaughtExceptionTestKW', {Param1: 'Val1'}) - assert.equal(output.error, undefined) - - console.log('PASSED') -})()