From 3c9ead07ba729c612ad60bd3bb509e95364e0f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 6 Mar 2026 14:05:59 +0100 Subject: [PATCH 01/21] SED-4581 Implementing agent forking in Node.js agent and support for on-the-fly dependency installation --- .../api/controllers/controller.js | 128 +++++++++--------- .../step-node-agent/api/controllers/output.js | 7 + .../api/controllers/session.js | 56 ++++++++ .../step-node-agent/api/runner/runner.js | 8 +- .../step-node-agent/keywords/keywords.js | 2 +- step-node/step-node-agent/package.json | 13 +- step-node/step-node-agent/server.js | 11 +- step-node/step-node-agent/test/test.js | 116 ++++++++-------- step-node/step-node-agent/worker-wrapper.js | 95 +++++++++++++ 9 files changed, 299 insertions(+), 137 deletions(-) create mode 100644 step-node/step-node-agent/api/controllers/session.js create mode 100644 step-node/step-node-agent/worker-wrapper.js diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index 316d554834..e5319a84e8 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -1,16 +1,18 @@ +const fs = require("fs"); +const path = require("path"); +const {fork, execSync} = require("child_process"); +const Session = require('./session'); module.exports = function Controller (agentContext, fileManager) { process.on('unhandledRejection', error => { - console.log('[Controller] Critical: an unhandled error (unhandled promise rejection) occured and might not have been reported', error) + console.log('[Controller] Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported', error) }) process.on('uncaughtException', error => { - console.log('[Controller] Critical: an unhandled error (uncaught exception) occured and might not have been reported', error) + console.log('[Controller] Critical: an unhandled error (uncaught exception) occurred and might not have been reported', error) }) let exports = {} - const fs = require('fs') - const path = require('path') const OutputBuilder = require('./output') exports.filemanager = fileManager @@ -38,14 +40,9 @@ module.exports = function Controller (agentContext, fileManager) { let session = agentContext.tokenSessions[tokenId] if (session) { - // call close() for each closeable object in the session: - Object.entries(session).forEach(function (element) { - if (typeof element[1]['close'] === 'function') { - console.log('[Controller] Closing closeable object \'' + element[0] + '\' for token: ' + tokenId) - element[1].close() - } - }) - agentContext.tokenSessions[tokenId] = {} + // Close the session and all objects it contains + session[Symbol.dispose](); + agentContext.tokenSessions[tokenId] = null; } else { console.log('[Controller] No session founds for token: ' + tokenId) } @@ -104,70 +101,75 @@ module.exports = function Controller (agentContext, fileManager) { exports.executeKeyword = async function (keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder, agentContext) { try { - var kwDir - + let npmProjectPath; if (keywordPackageFile.toUpperCase().endsWith('ZIP')) { if (exports.filemanager.isFirstLevelKeywordFolder(keywordPackageFile)) { - kwDir = path.resolve(keywordPackageFile + '/keywords') + npmProjectPath = path.resolve(keywordPackageFile); } else { - kwDir = path.resolve(keywordPackageFile + '/' + exports.filemanager.getFolderName(keywordPackageFile) + '/keywords') + npmProjectPath = path.resolve(keywordPackageFile, exports.filemanager.getFolderName(keywordPackageFile)) } } else { // Local execution with KeywordRunner - kwDir = path.resolve(keywordPackageFile + '/keywords') + npmProjectPath = path.resolve(keywordPackageFile) } - console.log('[Controller] Search keyword file in ' + kwDir + ' for token ' + tokenId) - - var keywordFunction = searchAndRequireKeyword(kwDir, keywordName) - - if (keywordFunction) { - console.log('[Controller] Found keyword for token ' + tokenId) - let session = agentContext.tokenSessions[tokenId] - - if (!session) session = {} - - console.log('[Controller] Executing keyword ' + keywordName + ' on token ' + tokenId) - - try { - await keywordFunction(argument, outputBuilder, session, properties) - } catch (e) { - var onError = searchAndRequireKeyword(kwDir, 'onError') - if (onError) { - if (await onError(e, argument, outputBuilder, session, properties)) { - console.log('[Controller] Keyword execution marked as failed: onError function returned \'true\' on token ' + tokenId) - outputBuilder.fail(e) - } else { - console.log('[Controller] Keyword execution marked as successful: execution failed but the onError function returned \'false\' on token ' + tokenId) - outputBuilder.send() - } - } else { - console.log('[Controller] Keyword execution marked as failed: Keyword execution failed and no onError function found on token ' + tokenId) - outputBuilder.fail(e) - } - } - } else { - outputBuilder.fail('Unable to find keyword ' + keywordName) + console.log('[Controller] Executing keyword in project ' + npmProjectPath + ' for token ' + tokenId) + + let session = agentContext.tokenSessions[tokenId] + if (!session) { + session = new Session(); + agentContext.tokenSessions[tokenId] = session; + } + + let forkedAgent = session.get('forkedAgent'); + + let project = session.get('npmProjectPath'); + if(!project && forkedAgent) { + throw new Error("Multiple projects not supported within the same session"); + } + + if(!forkedAgent) { + console.log('[Controller] Starting agent fork in ' + npmProjectPath + ' for token ' + tokenId) + forkedAgent = createForkedAgent(npmProjectPath); + console.log('[Controller] Running npm install in ' + npmProjectPath + ' for token ' + tokenId) + execSync('npm install', { cwd: npmProjectPath, stdio: 'inherit' }); + session.set('forkedAgent', forkedAgent); + session.set('npmProjectPath', npmProjectPath); } + + const output = await runKeywordTask(forkedAgent, npmProjectPath, keywordName, argument, properties); + outputBuilder.merge(output.payload) } catch (e) { - outputBuilder.fail('An error occured while attempting to execute the keyword ' + keywordName, e) + console.log("[Controller] Unexpected error occurred while executing keyword ", e) + outputBuilder.fail('An error occurred while attempting to execute the keyword ' + keywordName, e) } } - function searchAndRequireKeyword (kwDir, keywordName) { - var keywordFunction - var kwFiles = fs.readdirSync(kwDir) - kwFiles.every(function (kwFile) { - if (kwFile.endsWith('.js')) { - const kwMod = require(kwDir + '/' + kwFile) - if (kwMod[keywordName]) { - keywordFunction = kwMod[keywordName] - return false - } - } - return true - }) - return keywordFunction + function createForkedAgent(keywordProjectPath) { + const parentCwd = process.cwd(); + const fileName = './worker-wrapper.js'; + + fs.copyFileSync(path.join(parentCwd, fileName), path.join(keywordProjectPath, fileName)); + fs.copyFileSync(path.join(parentCwd, './api/controllers/output.js'), path.join(keywordProjectPath, './output.js')); + fs.copyFileSync(path.join(parentCwd, './api/controllers/session.js'), path.join(keywordProjectPath, './session.js')); + + return fork('./worker-wrapper.js', [], {cwd: keywordProjectPath}); + } + + function runKeywordTask(forkedAgent, keywordProjectPath, functionName, input, properties) { + return new Promise((resolve, reject) => { + forkedAgent.send({ projectPath: keywordProjectPath, functionName: functionName, input: input, properties: properties }); + + forkedAgent.removeAllListeners('message'); + forkedAgent.on('message', (result) => { + resolve(result); + }); + + forkedAgent.removeAllListeners('error'); + forkedAgent.on('error', (err) => { + console.error('Error while calling forked agent:', err); + }); + }); } return exports diff --git a/step-node/step-node-agent/api/controllers/output.js b/step-node/step-node-agent/api/controllers/output.js index 4bf1f99433..6eb256330f 100644 --- a/step-node/step-node-agent/api/controllers/output.js +++ b/step-node/step-node-agent/api/controllers/output.js @@ -10,6 +10,13 @@ module.exports = function OutputBuilder (callback) { } } + exports.merge = function (output) { + exports.builder.payload = output; + if (callback) { + callback(exports.builder) + } + } + function buildDefaultTechnicalError (message) { return { msg: message, type: 'TECHNICAL', root: true, code: 0 } } diff --git a/step-node/step-node-agent/api/controllers/session.js b/step-node/step-node-agent/api/controllers/session.js new file mode 100644 index 0000000000..94a0590961 --- /dev/null +++ b/step-node/step-node-agent/api/controllers/session.js @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2026, exense GmbH + * + * This file is part of Step + * + * Step is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Step is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Step. If not, see . + */ +class Session extends Map { + [Symbol.dispose]() { + console.log(`Disposing Session: Cleaning up ${this.size} resources...`); + + for (const [key, resource] of this) { + try { + // 1. Try modern [Symbol.dispose] + if (resource && typeof resource[Symbol.dispose] === 'function') { + resource[Symbol.dispose](); + } + // 2. Fallback to Node.js child_process .kill() + else if (resource && typeof resource.kill === 'function') { + resource.kill(); + } + // 3. Fallback to generic .close() + else if (resource && typeof resource.close === 'function') { + resource.close(); + } + + console.log(`Successfully closed resource: ${key}`); + } catch (err) { + console.error(`Failed to close resource ${key}:`, err); + } + } + + // Clean up Object properties (Added via .dot notation) to support keywords written for older versions of the agent exposing the session as simple object + for (const key of Object.keys(this)) { + const resource = this[key]; + if (resource && typeof resource.close === 'function') { + resource.close(); + } + } + + // Clear the map after disposal so resources aren't held in memory + this.clear(); + } +} +module.exports = Session; diff --git a/step-node/step-node-agent/api/runner/runner.js b/step-node/step-node-agent/api/runner/runner.js index ca52c22062..a8be0a4834 100644 --- a/step-node/step-node-agent/api/runner/runner.js +++ b/step-node/step-node-agent/api/runner/runner.js @@ -1,8 +1,10 @@ +const Session = require('../controllers/session'); module.exports = function (properties = {}) { const tokenId = 'local' const agentContext = {tokens: [], tokenSessions: {}, properties: properties} - agentContext.tokenSessions[tokenId] = {} + let tokenSession = new Session(); + agentContext.tokenSessions[tokenId] = tokenSession var fileManager = { loadOrGetKeywordFile: function (url, fileId, fileVersion, keywordName) { @@ -23,5 +25,9 @@ module.exports = function (properties = {}) { }) } + api.close = function () { + tokenSession[Symbol.dispose](); + } + return api } diff --git a/step-node/step-node-agent/keywords/keywords.js b/step-node/step-node-agent/keywords/keywords.js index aaa52c450c..d6fbb5a1c1 100644 --- a/step-node/step-node-agent/keywords/keywords.js +++ b/step-node/step-node-agent/keywords/keywords.js @@ -45,6 +45,6 @@ exports.ErrorUncaughtExceptionTestKW = async (input, output, session, properties exports.onError = async (exception, input, output, session, properties) => { console.log('[onError] Exception is: \'' + exception + '\'') - global.isOnErrorCalled = true + output.builder.payload.payload.onErrorCalled = true return input['rethrow_error'] } diff --git a/step-node/step-node-agent/package.json b/step-node/step-node-agent/package.json index 1bd9ec2534..87dbd0d022 100644 --- a/step-node/step-node-agent/package.json +++ b/step-node/step-node-agent/package.json @@ -28,33 +28,26 @@ "nodemon": "^1.18.7" }, "dependencies": { - "body-parser": "^1.19.1", "express": "^4.17.1", "get-fqdn": "^1.0.0", - "http": "0.0.0", "jsonwebtoken": "^9.0.2", "minimist": "^1.2.5", - "npm": "^6.14.15", "request": "^2.88.0", "shelljs": "^0.8.4", - "underscore": "^1.13.1", "unzip-stream": "^0.3.1", "uuid": "^3.4.0", "yaml": "^2.4.2" }, "bundledDependencies": [ - "body-parser", "express", - "http", + "get-fqdn", + "jsonwebtoken", "minimist", - "npm", "request", "shelljs", - "underscore", "unzip-stream", "uuid", - "yaml", - "get-fqdn" + "yaml" ], "bin": { "step-node-agent": "./bin/step-node-agent" diff --git a/step-node/step-node-agent/server.js b/step-node/step-node-agent/server.js index 1f0ba2cf31..d09e6634a2 100644 --- a/step-node/step-node-agent/server.js +++ b/step-node/step-node-agent/server.js @@ -25,12 +25,11 @@ if(agentConfFileExt === '.yaml') { console.log('[Agent] Creating agent context and tokens') const uuid = require('uuid/v4') -const _ = require('underscore') const jwtUtils = require('./utils/jwtUtils') const agentType = 'node' const agent = {id: uuid()} let agentContext = { tokens: [], tokenSessions: [], tokenProperties: [], properties: agentConf.properties, controllerUrl: agentConf.gridHost, gridSecurity: agentConf.gridSecurity } -_.each(agentConf.tokenGroups, function (tokenGroup) { +agentConf.tokenGroups.forEach(function (tokenGroup) { const tokenConf = tokenGroup.tokenConf let attributes = tokenConf.attributes // Transform the selectionPatterns map to @@ -46,7 +45,7 @@ _.each(agentConf.tokenGroups, function (tokenGroup) { for (let i = 0; i < tokenGroup.capacity; i++) { const token = { id: uuid(), agentid: agent.id, attributes: attributes, selectionPatterns: tokenSelectionPatterns } agentContext.tokens.push(token) - agentContext.tokenSessions[token.id] = {} + agentContext.tokenSessions[token.id] = null agentContext.tokenProperties[token.id] = additionalProperties } }) @@ -56,10 +55,8 @@ const express = require('express') const app = express() const port = agentConf.agentPort || 3000 const timeout = agentConf.agentServerTimeout || 600000 -const bodyParser = require('body-parser') - -app.use(bodyParser.urlencoded({extended: true})) -app.use(bodyParser.json()) +app.use(express.urlencoded({extended: true})) +app.use(express.json()) // Apply JWT authentication middleware const createJwtAuthMiddleware = require('./middleware/jwtAuth') diff --git a/step-node/step-node-agent/test/test.js b/step-node/step-node-agent/test/test.js index 0e47be0bc0..6948b396f8 100644 --- a/step-node/step-node-agent/test/test.js +++ b/step-node/step-node-agent/test/test.js @@ -2,59 +2,65 @@ const runner = require('../api/runner/runner')({'Property1': 'Prop1'}) const assert = require('assert') ;(async () => { - // Test the happy path - var output = await runner.run('Echo', {Param1: 'Val1'}) - assert.equal(output.payload.Param1, 'Val1') - assert.equal(output.payload.properties.Property1, 'Prop1') - - // Test the method output.setError - var errorMsg = 'MyError' - output = await runner.run('SetErrorTestKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - assert.equal(output.error.type, 'TECHNICAL') - - // Test the method output.setError with an exception as argument - errorMsg = 'MyError2' - output = await runner.run('SetErrorWithExceptionKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - - // Test the method output.setError with an error message and an exception as argument - errorMsg = 'MyError3' - output = await runner.run('SetErrorWithMessageAndExceptionKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - assert.equal(output.attachments.length, 1) - - // Test the method output.fail - errorMsg = 'MyError4' - output = await runner.run('FailKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - - // Test the method output.setBusinessError - errorMsg = 'MyBusinessError' - output = await runner.run('BusinessErrorTestKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - - // Test onError hook - global.isOnErrorCalled = false - const errorMsg1 = 'Error - rethrow' - const output1 = await runner.run('ErrorTestKW', {ErrorMsg: errorMsg1, rethrow_error: true}) - assert.equal(output1.error.msg, errorMsg1) - assert.equal(global.isOnErrorCalled, true) - - // Test onError hook with no rethrow - global.isOnErrorCalled = false - const errorMsg2 = 'Error - do not rethrow' - const output2 = await runner.run('ErrorTestKW', {ErrorMsg: errorMsg2, rethrow_error: false}) - assert.equal(output2.error, undefined) - assert.equal(global.isOnErrorCalled, true) - - // Test rejected promises - output = await runner.run('ErrorRejectedPromiseTestKW', {Param1: 'Val1'}) - assert.equal(output.error, undefined) - - // Test uncaught exceptions - output = await runner.run('ErrorUncaughtExceptionTestKW', {Param1: 'Val1'}) - assert.equal(output.error, undefined) - - console.log('PASSED') + try { + // Test the happy path + var output = await runner.run('Echo', {Param1: 'Val1'}) + assert.equal(output.payload.Param1, 'Val1') + assert.equal(output.payload.properties.Property1, 'Prop1') + + // Test the method output.setError + var errorMsg = 'MyError' + output = await runner.run('SetErrorTestKW', {ErrorMsg: errorMsg, rethrow_error: true}) + assert.equal(output.error.msg, errorMsg) + assert.equal(output.error.type, 'TECHNICAL') + + // Test the method output.setError with an exception as argument + errorMsg = 'MyError2' + output = await runner.run('SetErrorWithExceptionKW', {ErrorMsg: errorMsg, rethrow_error: true}) + assert.equal(output.error.msg, errorMsg) + + // Test the method output.setError with an error message and an exception as argument + errorMsg = 'MyError3' + output = await runner.run('SetErrorWithMessageAndExceptionKW', {ErrorMsg: errorMsg, rethrow_error: true}) + assert.equal(output.error.msg, errorMsg) + assert.equal(output.attachments.length, 1) + + // Test the method output.fail + errorMsg = 'MyError4' + output = await runner.run('FailKW', {ErrorMsg: errorMsg, rethrow_error: true}) + assert.equal(output.error.msg, errorMsg) + + // Test the method output.setBusinessError + errorMsg = 'MyBusinessError' + output = await runner.run('BusinessErrorTestKW', {ErrorMsg: errorMsg, rethrow_error: true}) + assert.equal(output.error.msg, errorMsg) + + // Test onError hook + const errorMsg1 = 'Error - rethrow' + const output1 = await runner.run('ErrorTestKW', {ErrorMsg: errorMsg1, rethrow_error: true}) + assert.equal(output1.error.msg, errorMsg1) + assert.equal(output1.payload.onErrorCalled, true) + + // Test onError hook with no rethrow + const errorMsg2 = 'Error - do not rethrow' + const output2 = await runner.run('ErrorTestKW', {ErrorMsg: errorMsg2, rethrow_error: false}) + assert.equal(output2.error, undefined) + assert.equal(output1.payload.onErrorCalled, true) + + // Test rejected promises + output = await runner.run('ErrorRejectedPromiseTestKW', {Param1: 'Val1'}) + assert.equal(output.error, undefined) + + // Test uncaught exceptions + output = await runner.run('ErrorUncaughtExceptionTestKW', {Param1: 'Val1'}) + assert.equal(output.error, undefined) + + // Test not existing keyword + output = await runner.run('Not existing Keyword', {}) + assert.equal(output.error.msg, "Unable to find Keyword 'Not existing Keyword'") + + console.log('PASSED') + } finally { + runner.close(); + } })() diff --git a/step-node/step-node-agent/worker-wrapper.js b/step-node/step-node-agent/worker-wrapper.js new file mode 100644 index 0000000000..8b4428b781 --- /dev/null +++ b/step-node/step-node-agent/worker-wrapper.js @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2026, exense GmbH + * + * This file is part of Step + * + * Step is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Step is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Step. If not, see . + */ + +const OutputBuilder = require("./output"); +const Session = require("./session"); +const fs = require("fs"); +const path = require('path') +const session = new Session(); + +process.on('message', async ({ projectPath, functionName, input, properties }) => { + console.log("[Agent fork] Calling keyword " + functionName) + const kwModules = await importAllKeywords(projectPath); + + const outputBuilder = new OutputBuilder(function (output) { + console.log(`[Agent fork] Keyword ${functionName} successfully executed`) + process.send(output); + }) + + let keyword = searchKeyword(kwModules, functionName); + if(!keyword) { + outputBuilder.fail("Unable to find Keyword '" + functionName + "'"); + } else { + try { + await keyword(input, outputBuilder, session, properties); + } catch (e) { + let onError = searchKeyword(kwModules, 'onError'); + if (onError) { + if (await onError(e, input, outputBuilder, session, properties)) { + console.log('[Agent fork] Keyword execution marked as failed: onError function returned \'true\'') + outputBuilder.fail(e) + } else { + console.log('[Agent fork] Keyword execution marked as successful: execution failed but the onError function returned \'false\'') + outputBuilder.send() + } + } else { + console.log('[Agent fork] Keyword execution marked as failed: Keyword execution failed and no onError function found') + outputBuilder.fail(e) + } + } + } + + async function importAllKeywords(projectPath) { + const kwModules = []; + const kwDir = path.resolve(projectPath, "./keywords"); + console.log("[Agent fork] Search for keywords in: " + kwDir) + const kwFiles = fs.readdirSync(kwDir); + for (const kwFile of kwFiles) { + if (kwFile.endsWith('.js')) { + let kwModule = "file://" + path.resolve(kwDir, kwFile); + console.log("[Agent fork] Importing keywords from: " + kwModule) + let module = await import(kwModule); + kwModules.push(module); + } + } + return kwModules; + } + + function searchKeyword (kwModules, keywordName) { + let keywordFunction; + kwModules.forEach(function (kwModule) { + if (kwModule[keywordName]) { + keywordFunction = kwModule[keywordName] + } + }) + return keywordFunction + } +}); + +process.on('exit', () => { + session[Symbol.dispose](); +}) + +process.on('unhandledRejection', error => { + console.log('[Agent fork] Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported', error) +}) + +process.on('uncaughtException', error => { + console.log('[Agent fork] Critical: an unhandled error (uncaught exception) occurred and might not have been reported', error) +}) From 1a76f3add4efa8c063b42632b86c87e632478da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 6 Mar 2026 16:52:59 +0100 Subject: [PATCH 02/21] SED-4581 Fixing copying of agent forker files --- .../step-node-agent/api/controllers/controller.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index e5319a84e8..5e6608ed96 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -146,13 +146,9 @@ module.exports = function Controller (agentContext, fileManager) { } function createForkedAgent(keywordProjectPath) { - const parentCwd = process.cwd(); - const fileName = './worker-wrapper.js'; - - fs.copyFileSync(path.join(parentCwd, fileName), path.join(keywordProjectPath, fileName)); - fs.copyFileSync(path.join(parentCwd, './api/controllers/output.js'), path.join(keywordProjectPath, './output.js')); - fs.copyFileSync(path.join(parentCwd, './api/controllers/session.js'), path.join(keywordProjectPath, './session.js')); - + fs.copyFileSync(path.resolve(__dirname, '../../worker-wrapper.js'), path.join(keywordProjectPath, './worker-wrapper.js')); + fs.copyFileSync(path.join(__dirname, 'output.js'), path.join(keywordProjectPath, './output.js')); + fs.copyFileSync(path.join(__dirname, 'session.js'), path.join(keywordProjectPath, './session.js')); return fork('./worker-wrapper.js', [], {cwd: keywordProjectPath}); } From 38f8fb95b16f7a71085a6d45a671002b94c8b041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Sat, 7 Mar 2026 10:47:33 +0100 Subject: [PATCH 03/21] SED-4581 Refactoring Node.js agent code --- .../api/controllers/controller.js | 173 +++++++++--------- .../step-node-agent/api/controllers/output.js | 90 ++++----- .../api/filemanager/filemanager.js | 120 ++++++------ .../step-node-agent/api/routes/routes.js | 10 +- .../step-node-agent/api/runner/runner.js | 17 +- step-node/step-node-agent/server.js | 66 ++++--- step-node/step-node-agent/test/test.js | 4 +- step-node/step-node-agent/worker-wrapper.js | 11 +- 8 files changed, 248 insertions(+), 243 deletions(-) diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index 5e6608ed96..d9527b031f 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -2,111 +2,121 @@ const fs = require("fs"); const path = require("path"); const {fork, execSync} = require("child_process"); const Session = require('./session'); -module.exports = function Controller (agentContext, fileManager) { - process.on('unhandledRejection', error => { - console.log('[Controller] Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported', error) - }) +const OutputBuilder = require('./output'); - process.on('uncaughtException', error => { - console.log('[Controller] Critical: an unhandled error (uncaught exception) occurred and might not have been reported', error) - }) +process.on('unhandledRejection', error => { + console.log('[Controller] Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported', error) +}) - let exports = {} +process.on('uncaughtException', error => { + console.log('[Controller] Critical: an unhandled error (uncaught exception) occurred and might not have been reported', error) +}) - const OutputBuilder = require('./output') - - exports.filemanager = fileManager +class Controller { + constructor(agentContext, fileManager) { + this.agentContext = agentContext; + this.filemanager = fileManager; + } - exports.isRunning = function (req, res) { + isRunning(req, res) { res.status(200).json('Agent is running') } - exports.reserveToken = function (req, res) { - exports.reserveToken_(req.params.tokenId) + reserveToken(req, res) { + this.reserveToken_(req.params.tokenId) res.json({}) } - exports.reserveToken_ = function (tokenId) { + reserveToken_(tokenId) { console.log('[Controller] Reserving token: ' + tokenId) } - exports.releaseToken = function (req, res) { - exports.releaseToken_(req.params.tokenId) + releaseToken(req, res) { + this.releaseToken_(req.params.tokenId) res.json({}) } - exports.releaseToken_ = function (tokenId) { + releaseToken_(tokenId) { console.log('[Controller] Releasing token: ' + tokenId) - let session = agentContext.tokenSessions[tokenId] + const session = this.agentContext.tokenSessions[tokenId] if (session) { // Close the session and all objects it contains session[Symbol.dispose](); - agentContext.tokenSessions[tokenId] = null; + this.agentContext.tokenSessions[tokenId] = null; } else { console.log('[Controller] No session founds for token: ' + tokenId) } } - exports.interruptExecution = function (req, res) { + interruptExecution(req, res) { const tokenId = req.params.tokenId console.warn('[Controller] Interrupting token: ' + tokenId + ' : not implemented') } - exports.process = function (req, res) { + async process(req, res) { const tokenId = req.params.tokenId const keywordName = req.body.payload.function const argument = req.body.payload.payload const properties = req.body.payload.properties - console.log('[Controller] Using token: ' + tokenId + ' to execute ' + keywordName) + let token = this.agentContext.tokens.find(value => value.id == tokenId); + if(token) { + console.log('[Controller] Using token: ' + tokenId + ' to execute ' + keywordName) - // add the agent properties - let agentProperties = agentContext.properties - Object.entries(agentProperties).forEach(function (element) { - properties[element[0]] = element[1] - }) + // add the agent properties + const agentProperties = this.agentContext.properties + Object.entries(agentProperties).forEach(([key, value]) => { properties[key] = value }) - // add the properties of the tokenGroup - let additionalProperties = agentContext.tokenProperties[tokenId] - Object.entries(additionalProperties).forEach(function (element) { - properties[element[0]] = element[1] - }) + // add the properties of the tokenGroup + const additionalProperties = this.agentContext.tokenProperties[tokenId] + Object.entries(additionalProperties).forEach(([key, value]) => { properties[key] = value }) - exports.process_(tokenId, keywordName, argument, properties, function (payload) { + const payload = await this.process_(tokenId, keywordName, argument, properties) res.json(payload) - }) - } + } else { + const outputBuilder = new OutputBuilder(payload => res.json(payload)); + outputBuilder.fail("The token '" + tokenId + " doesn't exist on this agent. This usually means that the agent crashed and restarted."); + } - exports.process_ = function (tokenId, keywordName, argument, properties, callback) { - const outputBuilder = new OutputBuilder(function (output) { - console.log(`[Controller] Keyword ${keywordName} successfully executed on token ${tokenId}`) - callback(output) - }) + } - try { - const filepathPromise = exports.filemanager.loadOrGetKeywordFile(agentContext.controllerUrl + '/grid/file/', properties['$node.js.file.id'], properties['$node.js.file.version'], keywordName) - - filepathPromise.then(function (keywordPackageFile) { - console.log('[Controller] Executing keyword ' + keywordName + ' using filepath ' + keywordPackageFile) - exports.executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder, agentContext) - }, function (err) { - console.log('[Controller] Error while attempting to run keyword ' + keywordName + ' :' + err) - outputBuilder.fail('Error while attempting to run keyword', err) + process_(tokenId, keywordName, argument, properties) { + return new Promise((resolve) => { + const outputBuilder = new OutputBuilder((output) => { + console.log(`[Controller] Keyword ${keywordName} successfully executed on token ${tokenId}`) + resolve(output) }) - } catch (e) { - outputBuilder.fail(e) - } + + try { + const filepathPromise = this.filemanager.loadOrGetKeywordFile( + this.agentContext.controllerUrl + '/grid/file/', + properties['$node.js.file.id'], + properties['$node.js.file.version'], + keywordName + ) + + filepathPromise.then((keywordPackageFile) => { + console.log('[Controller] Executing keyword ' + keywordName + ' using filepath ' + keywordPackageFile) + this.executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder) + }, (err) => { + console.log('[Controller] Error while attempting to run keyword ' + keywordName + ' :' + err) + outputBuilder.fail('Error while attempting to run keyword', err) + }) + } catch (e) { + outputBuilder.fail(e) + } + }) } - exports.executeKeyword = async function (keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder, agentContext) { + async executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder) { try { let npmProjectPath; if (keywordPackageFile.toUpperCase().endsWith('ZIP')) { - if (exports.filemanager.isFirstLevelKeywordFolder(keywordPackageFile)) { + if (this.filemanager.isFirstLevelKeywordFolder(keywordPackageFile)) { npmProjectPath = path.resolve(keywordPackageFile); } else { - npmProjectPath = path.resolve(keywordPackageFile, exports.filemanager.getFolderName(keywordPackageFile)) + npmProjectPath = path.resolve(keywordPackageFile, this.filemanager.getFolderName(keywordPackageFile)) } } else { // Local execution with KeywordRunner @@ -115,20 +125,19 @@ module.exports = function Controller (agentContext, fileManager) { console.log('[Controller] Executing keyword in project ' + npmProjectPath + ' for token ' + tokenId) - let session = agentContext.tokenSessions[tokenId] + let session = this.agentContext.tokenSessions[tokenId] if (!session) { session = new Session(); - agentContext.tokenSessions[tokenId] = session; + this.agentContext.tokenSessions[tokenId] = session; } let forkedAgent = session.get('forkedAgent'); - - let project = session.get('npmProjectPath'); - if(!project && forkedAgent) { + const project = session.get('npmProjectPath'); + if (!project && forkedAgent) { throw new Error("Multiple projects not supported within the same session"); } - if(!forkedAgent) { + if (!forkedAgent) { console.log('[Controller] Starting agent fork in ' + npmProjectPath + ' for token ' + tokenId) forkedAgent = createForkedAgent(npmProjectPath); console.log('[Controller] Running npm install in ' + npmProjectPath + ' for token ' + tokenId) @@ -144,29 +153,29 @@ module.exports = function Controller (agentContext, fileManager) { outputBuilder.fail('An error occurred while attempting to execute the keyword ' + keywordName, e) } } +} - function createForkedAgent(keywordProjectPath) { - fs.copyFileSync(path.resolve(__dirname, '../../worker-wrapper.js'), path.join(keywordProjectPath, './worker-wrapper.js')); - fs.copyFileSync(path.join(__dirname, 'output.js'), path.join(keywordProjectPath, './output.js')); - fs.copyFileSync(path.join(__dirname, 'session.js'), path.join(keywordProjectPath, './session.js')); - return fork('./worker-wrapper.js', [], {cwd: keywordProjectPath}); - } - - function runKeywordTask(forkedAgent, keywordProjectPath, functionName, input, properties) { - return new Promise((resolve, reject) => { - forkedAgent.send({ projectPath: keywordProjectPath, functionName: functionName, input: input, properties: properties }); +function createForkedAgent(keywordProjectPath) { + fs.copyFileSync(path.resolve(__dirname, '../../worker-wrapper.js'), path.join(keywordProjectPath, './worker-wrapper.js')); + fs.copyFileSync(path.join(__dirname, 'output.js'), path.join(keywordProjectPath, './output.js')); + fs.copyFileSync(path.join(__dirname, 'session.js'), path.join(keywordProjectPath, './session.js')); + return fork('./worker-wrapper.js', [], {cwd: keywordProjectPath}); +} - forkedAgent.removeAllListeners('message'); - forkedAgent.on('message', (result) => { - resolve(result); - }); +function runKeywordTask(forkedAgent, keywordProjectPath, functionName, input, properties) { + return new Promise((resolve) => { + forkedAgent.send({ projectPath: keywordProjectPath, functionName, input, properties }); - forkedAgent.removeAllListeners('error'); - forkedAgent.on('error', (err) => { - console.error('Error while calling forked agent:', err); - }); + forkedAgent.removeAllListeners('message'); + forkedAgent.on('message', (result) => { + resolve(result); }); - } - return exports + forkedAgent.removeAllListeners('error'); + forkedAgent.on('error', (err) => { + console.error('Error while calling forked agent:', err); + }); + }); } + +module.exports = Controller; diff --git a/step-node/step-node-agent/api/controllers/output.js b/step-node/step-node-agent/api/controllers/output.js index 6eb256330f..a3dd99e7ef 100644 --- a/step-node/step-node-agent/api/controllers/output.js +++ b/step-node/step-node-agent/api/controllers/output.js @@ -1,68 +1,68 @@ -module.exports = function OutputBuilder (callback) { - let exports = {} - - exports.builder = { payload: { attachments: [], payload: {} }, attachments: [] } - - exports.send = function (payload) { - exports.builder.payload.payload = payload - if (callback) { - callback(exports.builder) - } +class OutputBuilder { + constructor(callback) { + this.callback = callback; + this.builder = { payload: { attachments: [], payload: {} }, attachments: [] } } - exports.merge = function (output) { - exports.builder.payload = output; - if (callback) { - callback(exports.builder) + send(payload) { + this.builder.payload.payload = payload + if (this.callback) { + this.callback(this.builder) } } - function buildDefaultTechnicalError (message) { - return { msg: message, type: 'TECHNICAL', root: true, code: 0 } - } - - function buildDefaultBusinessError (message) { - return { msg: message, type: 'BUSINESS', root: true, code: 0 } - } - - function attachException (e) { - exports.attach( - { - 'name': 'exception.log', - 'isDirectory': false, - 'description': 'exception stacktrace from keyword', - 'hexContent': Buffer.from(e.stack).toString('base64') - }) + merge(output) { + this.builder.payload = output; + if (this.callback) { + this.callback(this.builder) + } } - exports.fail = function (arg1, arg2) { - exports.setError(arg1, arg2) - if (callback) { - callback(exports.builder) + fail(arg1, arg2) { + this.setError(arg1, arg2) + if (this.callback) { + this.callback(this.builder) } } - exports.setError = function (arg1, arg2) { + setError(arg1, arg2) { if (typeof arg1 === 'string' || arg1 instanceof String) { - exports.builder.payload.error = buildDefaultTechnicalError(arg1) + this.builder.payload.error = buildDefaultTechnicalError(arg1) if (arg2 && arg2 instanceof Error) { - attachException(arg2) + this.#attachException(arg2) } } else if (arg1 instanceof Error) { - exports.builder.payload.error = buildDefaultTechnicalError(arg1.message) - attachException(arg1) + this.builder.payload.error = buildDefaultTechnicalError(arg1.message) + this.#attachException(arg1) } else if (typeof arg1 === 'object') { - exports.builder.payload.error = arg1 + this.builder.payload.error = arg1 } } - exports.setBusinessError = function (errorMessage) { - exports.builder.payload.error = buildDefaultBusinessError(errorMessage) + setBusinessError(errorMessage) { + this.builder.payload.error = buildDefaultBusinessError(errorMessage) + } + + attach(attachment) { + this.builder.payload.attachments.push(attachment) } - exports.attach = function (attachment) { - exports.builder.payload.attachments.push(attachment) + #attachException(e) { + this.attach({ + 'name': 'exception.log', + 'isDirectory': false, + 'description': 'exception stacktrace from keyword', + 'hexContent': Buffer.from(e.stack).toString('base64') + }) } +} - return exports +function buildDefaultTechnicalError(message) { + return { msg: message, type: 'TECHNICAL', root: true, code: 0 } } + +function buildDefaultBusinessError(message) { + return { msg: message, type: 'BUSINESS', root: true, code: 0 } +} + +module.exports = OutputBuilder; diff --git a/step-node/step-node-agent/api/filemanager/filemanager.js b/step-node/step-node-agent/api/filemanager/filemanager.js index de67d10295..e23156b1fc 100644 --- a/step-node/step-node-agent/api/filemanager/filemanager.js +++ b/step-node/step-node-agent/api/filemanager/filemanager.js @@ -1,46 +1,43 @@ -module.exports = function FileManager (agentContext) { - const fs = require('fs') - const shell = require('shelljs') - const http = require('http') - const https = require('https') - const url = require('url') - const unzip = require('unzip-stream') - const jwtUtils = require('../../utils/jwtUtils') - - let exports = {} - const filemanagerPath = agentContext.properties['filemanagerPath'] ? agentContext.properties['filemanagerPath'] : 'filemanager' - const workingDir = filemanagerPath + '/work/' - console.log('[FileManager] Starting file manager using working directory: ' + workingDir) - - console.log('[FileManager] Clearing working dir: ' + workingDir) - shell.rm('-rf', workingDir) - shell.mkdir('-p', workingDir) - - let filemanagerMap = {} - - exports.isFirstLevelKeywordFolder = function (path) { - if (fs.existsSync(path + '/keywords')) { - return true - } - return false +const fs = require('fs') +const http = require('http') +const https = require('https') +const url = require('url') +const unzip = require('unzip-stream') +const jwtUtils = require('../../utils/jwtUtils') + +class FileManager { + constructor(agentContext) { + this.agentContext = agentContext; + const filemanagerPath = agentContext.properties['filemanagerPath'] || 'filemanager' + this.workingDir = filemanagerPath + '/work/' + console.log('[FileManager] Starting file manager using working directory: ' + this.workingDir) + + console.log('[FileManager] Clearing working dir: ' + this.workingDir) + fs.rmSync(this.workingDir, { recursive: true, force: true }) + fs.mkdirSync(this.workingDir, { recursive: true }) + + this.filemanagerMap = {} + } + + isFirstLevelKeywordFolder(filePath) { + return fs.existsSync(filePath + '/keywords') } - exports.getFolderName = function (keywordPackageFile) { + getFolderName(keywordPackageFile) { try { - let splitNodes = keywordPackageFile.split('/') - let lastNode = splitNodes[splitNodes.length - 1] - let splitExt = lastNode.split('.') - return splitExt[0] + const splitNodes = keywordPackageFile.split('/') + const lastNode = splitNodes[splitNodes.length - 1] + return lastNode.split('.')[0] } catch (e) { throw new Error('A problem occured while attempting to retrieve subfolder name from zipped project:' + keywordPackageFile) } } - exports.loadOrGetKeywordFile = function (controllerUrl, fileId, fileVersionId) { - return new Promise(function (resolve, reject) { - const filePath = workingDir + fileId + '/' + fileVersionId + loadOrGetKeywordFile(controllerUrl, fileId, fileVersionId) { + return new Promise((resolve, reject) => { + const filePath = this.workingDir + fileId + '/' + fileVersionId - const cacheEntry = getCacheEntry(fileId, fileVersionId) + const cacheEntry = this.#getCacheEntry(fileId, fileVersionId) if (cacheEntry) { if (!cacheEntry.loading) { console.log('[FileManager] Entry found for fileId ' + fileId + ': ' + cacheEntry.name) @@ -59,24 +56,22 @@ module.exports = function FileManager (agentContext) { }) } } else { - putCacheEntry(fileId, fileVersionId, {'loading': true, 'promises': []}) + this.#putCacheEntry(fileId, fileVersionId, { loading: true, promises: [] }) console.log('[FileManager] No entry found for fileId ' + fileId + '. Loading...') - shell.mkdir('-p', filePath) + fs.mkdirSync(filePath, { recursive: true }) console.log('[FileManager] Created file path: ' + filePath + ' for fileId ' + fileId) - var fileVersionUrl = controllerUrl + fileId + '/' + fileVersionId + const fileVersionUrl = controllerUrl + fileId + '/' + fileVersionId console.log('[FileManager] Requesting file from: ' + fileVersionUrl) - const filenamePromise = getKeywordFile(fileVersionUrl, filePath) - - filenamePromise.then(function (result) { + this.#getKeywordFile(fileVersionUrl, filePath).then((result) => { console.log('[FileManager] Transfered file ' + result + ' from ' + fileVersionUrl) - let cacheEntry = getCacheEntry(fileId, fileVersionId) + const cacheEntry = this.#getCacheEntry(fileId, fileVersionId) cacheEntry.name = result cacheEntry.loading = false - putCacheEntry(fileId, fileVersionId, cacheEntry) + this.#putCacheEntry(fileId, fileVersionId, cacheEntry) if (cacheEntry.promises) { cacheEntry.promises.forEach(callback => callback(filePath + '/' + result)) // eslint-disable-line @@ -84,7 +79,7 @@ module.exports = function FileManager (agentContext) { delete cacheEntry.promises resolve(filePath + '/' + result) - }, function (err) { + }, (err) => { console.log('Error :' + err) reject(err) }) @@ -92,16 +87,16 @@ module.exports = function FileManager (agentContext) { }) } - function getCacheEntry (fileId, fileVersion) { - return filemanagerMap[fileId + fileVersion] + #getCacheEntry(fileId, fileVersion) { + return this.filemanagerMap[fileId + fileVersion] } - function putCacheEntry (fileId, fileVersion, entry) { - filemanagerMap[fileId + fileVersion] = entry + #putCacheEntry(fileId, fileVersion, entry) { + this.filemanagerMap[fileId + fileVersion] = entry } - function getKeywordFile (controllerFileUrl, targetDir) { - return new Promise(function (resolve, reject) { + #getKeywordFile(controllerFileUrl, targetDir) { + return new Promise((resolve, reject) => { const parsedUrl = url.parse(controllerFileUrl) const httpModule = parsedUrl.protocol === 'https:' ? https : http @@ -113,17 +108,15 @@ module.exports = function FileManager (agentContext) { } // Add bearer token if gridSecurity is configured - const token = jwtUtils.generateJwtToken(agentContext.gridSecurity, 300); // 5 minutes expiration + const token = jwtUtils.generateJwtToken(this.agentContext.gridSecurity, 300); // 5 minutes expiration if (token) { - requestOptions.headers = { - 'Authorization': 'Bearer ' + token - }; + requestOptions.headers = { 'Authorization': 'Bearer ' + token }; } const req = httpModule.request(requestOptions, (resp) => { - const filename = parseName(resp.headers) + const filename = this.#parseName(resp.headers) const filepath = targetDir + '/' + filename - if (isDir(resp.headers) || filename.toUpperCase().endsWith('ZIP')) { + if (this.#isDir(resp.headers) || filename.toUpperCase().endsWith('ZIP')) { resp.pipe(unzip.Extract({path: filepath})).on('close', () => resolve(filename)) } else { const myFile = fs.createWriteStream(filepath) @@ -138,14 +131,17 @@ module.exports = function FileManager (agentContext) { }) } - function parseName (headers) { - const contentDisposition = JSON.stringify(headers['content-disposition']) - return contentDisposition.split('filename = ')[1].split(';')[0] + #parseName(headers) { + const contentDisposition = headers['content-disposition'] || '' + const match = contentDisposition.match(/filename\s*=\s*([^;]+)/) + return match ? match[1].trim() : '' } - function isDir (headers) { - const contentDisposition = JSON.stringify(headers['content-disposition']) - return contentDisposition.split('type = ')[1].split(';')[0].startsWith('dir') + #isDir(headers) { + const contentDisposition = headers['content-disposition'] || '' + const match = contentDisposition.match(/type\s*=\s*([^;]+)/) + return match ? match[1].trim().startsWith('dir') : false } - return exports } + +module.exports = FileManager; diff --git a/step-node/step-node-agent/api/routes/routes.js b/step-node/step-node-agent/api/routes/routes.js index ef7aeb76b0..3676d00e3c 100644 --- a/step-node/step-node-agent/api/routes/routes.js +++ b/step-node/step-node-agent/api/routes/routes.js @@ -4,9 +4,9 @@ module.exports = function (app, agentContext) { const FileManager = require('../filemanager/filemanager') const controller = new Controller(agentContext, new FileManager(agentContext)) - app.route('/running').get(controller.isRunning) - app.route('/token/:tokenId/reserve').get(controller.reserveToken) - app.route('/token/:tokenId/release').get(controller.releaseToken) - app.route('/token/:tokenId/process').post(controller.process) - app.route('/token/:tokenId/interrupt-execution').post(controller.interruptExecution) + app.route('/running').get(controller.isRunning.bind(controller)) + app.route('/token/:tokenId/reserve').get(controller.reserveToken.bind(controller)) + app.route('/token/:tokenId/release').get(controller.releaseToken.bind(controller)) + app.route('/token/:tokenId/process').post(controller.process.bind(controller)) + app.route('/token/:tokenId/interrupt-execution').post(controller.interruptExecution.bind(controller)) } diff --git a/step-node/step-node-agent/api/runner/runner.js b/step-node/step-node-agent/api/runner/runner.js index a8be0a4834..f686cf1c3b 100644 --- a/step-node/step-node-agent/api/runner/runner.js +++ b/step-node/step-node-agent/api/runner/runner.js @@ -3,15 +3,11 @@ module.exports = function (properties = {}) { const tokenId = 'local' const agentContext = {tokens: [], tokenSessions: {}, properties: properties} - let tokenSession = new Session(); + const tokenSession = new Session(); agentContext.tokenSessions[tokenId] = tokenSession - var fileManager = { - loadOrGetKeywordFile: function (url, fileId, fileVersion, keywordName) { - return new Promise(function (resolve, reject) { - resolve('.') - }) - } + const fileManager = { + loadOrGetKeywordFile: () => Promise.resolve('.') } const Controller = require('../controllers/controller') @@ -19,10 +15,9 @@ module.exports = function (properties = {}) { const api = {} - api.run = function (keywordName, input) { - return new Promise(resolve => { - controller.process_(tokenId, keywordName, input, properties, function (output) { resolve(output.payload) }) - }) + api.run = async function (keywordName, input) { + const output = await controller.process_(tokenId, keywordName, input, properties) + return output.payload } api.close = function () { diff --git a/step-node/step-node-agent/server.js b/step-node/step-node-agent/server.js index d09e6634a2..392d7fa75e 100644 --- a/step-node/step-node-agent/server.js +++ b/step-node/step-node-agent/server.js @@ -14,13 +14,16 @@ console.log('[Agent] Reading agent configuration ' + agentConfFile) const fs = require('fs') const content = fs.readFileSync(agentConfFile, 'utf8') const agentConfFileExt = path.extname(agentConfFile) -var agentConf -if(agentConfFileExt === '.yaml') { - agentConf = YAML.parse(content) -} else if(agentConfFileExt === '.json') { - agentConf = JSON.parse(content) -} else { - throw new Error('Unsupported extension ' + agentConfFileExt + " for agent configuration " + content); +const agentConf = parseAgentConf(); + +function parseAgentConf() { + if (agentConfFileExt === '.yaml') { + return YAML.parse(content) + } else if (agentConfFileExt === '.json') { + return JSON.parse(content) + } else { + throw new Error('Unsupported extension ' + agentConfFileExt + " for agent configuration " + content); + } } console.log('[Agent] Creating agent context and tokens') @@ -28,7 +31,7 @@ const uuid = require('uuid/v4') const jwtUtils = require('./utils/jwtUtils') const agentType = 'node' const agent = {id: uuid()} -let agentContext = { tokens: [], tokenSessions: [], tokenProperties: [], properties: agentConf.properties, controllerUrl: agentConf.gridHost, gridSecurity: agentConf.gridSecurity } +const agentContext = { tokens: [], tokenSessions: [], tokenProperties: [], properties: agentConf.properties, controllerUrl: agentConf.gridHost, gridSecurity: agentConf.gridSecurity } agentConf.tokenGroups.forEach(function (tokenGroup) { const tokenConf = tokenGroup.tokenConf let attributes = tokenConf.attributes @@ -66,38 +69,37 @@ app.use(jwtAuthMiddleware) const routes = require('./api/routes/routes') routes(app, agentContext) -var server = app.listen(port) +const server = app.listen(port) server.setTimeout(timeout) -startWithAgentUrl = function(agentUrl) { +const startWithAgentUrl = async function(agentUrl) { console.log('[Agent] Registering agent as ' + agentUrl + ' to grid ' + agentConf.gridHost) console.log('[Agent] Creating registration timer') const registrationPeriod = agentConf.registrationPeriod || 5000 - const request = require('request') - setInterval(function () { - const requestOptions = { - uri: agentConf.gridHost + '/grid/register', - method: 'POST', - json: true, - body: { agentRef: { agentId: agent.id, agentUrl: agentUrl, agentType: agentType }, tokens: agentContext.tokens } - }; + setInterval(async () => { + const body = { agentRef: { agentId: agent.id, agentUrl: agentUrl, agentType: agentType }, tokens: agentContext.tokens } + const headers = { 'Content-Type': 'application/json' } // Add bearer token if gridSecurity is configured const token = jwtUtils.generateJwtToken(agentConf.gridSecurity, 3600); // 1 hour expiration if (token) { - requestOptions.headers = { - 'Authorization': 'Bearer ' + token - }; + headers['Authorization'] = 'Bearer ' + token; } - request(requestOptions, function (err, res, body) { - if (err) { - console.log("[Agent] Error while registering agent to grid") - console.log(err) - } else if (res.statusCode !== 204) { - console.log("[Agent] Failed to register agent: grid responded with status " + res.statusCode + (body != null ? ". Response body: " + JSON.stringify(body) : "")) + try { + const res = await fetch(agentConf.gridHost + '/grid/register', { + method: 'POST', + headers, + body: JSON.stringify(body) + }) + if (res.status !== 204) { + const responseBody = await res.text().catch(() => null) + console.log("[Agent] Failed to register agent: grid responded with status " + res.status + (responseBody != null ? ". Response body: " + responseBody : "")) } - }) + } catch (err) { + console.log("[Agent] Error while registering agent to grid") + console.log(err) + } }, registrationPeriod) console.log('[Agent] Successfully started on: ' + port) @@ -113,3 +115,11 @@ if(agentConf.agentUrl) { console.log('[Agent] Error while getting FQDN ' + e) }) } + +const v8 = require('v8'); + +process.on('SIGUSR2', () => { + const fileName = `/tmp/heap-${process.pid}-${Date.now()}.heapsnapshot`; + v8.writeHeapSnapshot(fileName); + console.log(`Heap dump written to ${fileName}`); +}); diff --git a/step-node/step-node-agent/test/test.js b/step-node/step-node-agent/test/test.js index 6948b396f8..4e2bae1bd3 100644 --- a/step-node/step-node-agent/test/test.js +++ b/step-node/step-node-agent/test/test.js @@ -4,12 +4,12 @@ const assert = require('assert') ;(async () => { try { // Test the happy path - var output = await runner.run('Echo', {Param1: 'Val1'}) + let output = await runner.run('Echo', {Param1: 'Val1'}) assert.equal(output.payload.Param1, 'Val1') assert.equal(output.payload.properties.Property1, 'Prop1') // Test the method output.setError - var errorMsg = 'MyError' + let errorMsg = 'MyError' output = await runner.run('SetErrorTestKW', {ErrorMsg: errorMsg, rethrow_error: true}) assert.equal(output.error.msg, errorMsg) assert.equal(output.error.type, 'TECHNICAL') diff --git a/step-node/step-node-agent/worker-wrapper.js b/step-node/step-node-agent/worker-wrapper.js index 8b4428b781..0cd89ce7ca 100644 --- a/step-node/step-node-agent/worker-wrapper.js +++ b/step-node/step-node-agent/worker-wrapper.js @@ -71,14 +71,9 @@ process.on('message', async ({ projectPath, functionName, input, properties }) = return kwModules; } - function searchKeyword (kwModules, keywordName) { - let keywordFunction; - kwModules.forEach(function (kwModule) { - if (kwModule[keywordName]) { - keywordFunction = kwModule[keywordName] - } - }) - return keywordFunction + function searchKeyword(kwModules, keywordName) { + const kwModule = kwModules.find(m => m[keywordName]); + return kwModule ? kwModule[keywordName] : undefined; } }); From d563c0f298bffb16050ddec69eafdf480af6fd9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Sat, 7 Mar 2026 13:13:45 +0100 Subject: [PATCH 04/21] SED-4581 Aligning Node.js KW API with Java --- .../api/controllers/controller.js | 2 +- .../step-node-agent/api/controllers/output.js | 113 +++++++++++++++++- .../step-node-agent/keywords/keywords.js | 52 ++++++++ step-node/step-node-agent/test/test.js | 85 +++++++++++++ step-node/step-node-agent/worker-wrapper.js | 2 +- 5 files changed, 249 insertions(+), 5 deletions(-) diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index d9527b031f..f46ff387fc 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -2,7 +2,7 @@ const fs = require("fs"); const path = require("path"); const {fork, execSync} = require("child_process"); const Session = require('./session'); -const OutputBuilder = require('./output'); +const { OutputBuilder } = require('./output'); process.on('unhandledRejection', error => { console.log('[Controller] Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported', error) diff --git a/step-node/step-node-agent/api/controllers/output.js b/step-node/step-node-agent/api/controllers/output.js index a3dd99e7ef..a4629be0bf 100644 --- a/step-node/step-node-agent/api/controllers/output.js +++ b/step-node/step-node-agent/api/controllers/output.js @@ -1,11 +1,45 @@ +const MeasureStatus = Object.freeze({ + PASSED: 'PASSED', + FAILED: 'FAILED', + TECHNICAL_ERROR: 'TECHNICAL_ERROR', +}); + +const VALID_STATUSES = new Set(Object.values(MeasureStatus)); + +function assertValidStatus(status) { + if (status !== undefined && !VALID_STATUSES.has(status)) { + throw new TypeError(`Invalid measure status: "${status}". Must be one of: ${[...VALID_STATUSES].join(', ')}`); + } +} + class OutputBuilder { constructor(callback) { this.callback = callback; this.builder = { payload: { attachments: [], payload: {} }, attachments: [] } + this._currentMeasure = null; } + /** + * Adds an output attribute to the payload. Can be called multiple times to build the payload + * incrementally. Call send() with no arguments afterwards to flush. + * @param {string} name + * @param {*} value + * @returns {OutputBuilder} this, for chaining + */ + add(name, value) { + this.builder.payload.payload[name] = value; + return this; + } + + /** + * Sends the output. If a payload object is provided it replaces the current payload entirely. + * If called with no arguments, the payload built up via add() is used. + * @param {object} [payload] + */ send(payload) { - this.builder.payload.payload = payload + if (payload !== undefined) { + this.builder.payload.payload = payload; + } if (this.callback) { this.callback(this.builder) } @@ -25,9 +59,15 @@ class OutputBuilder { } } + /** + * Sets a technical error, replacing any existing error. + * Accepts a string message, an Error object, or a raw error object. + * When a string + Error are passed the exception stack is attached. + * @returns {OutputBuilder} this, for chaining + */ setError(arg1, arg2) { if (typeof arg1 === 'string' || arg1 instanceof String) { - this.builder.payload.error = buildDefaultTechnicalError(arg1) + this.builder.payload.error = buildDefaultTechnicalError(String(arg1)) if (arg2 && arg2 instanceof Error) { this.#attachException(arg2) } @@ -37,16 +77,83 @@ class OutputBuilder { } else if (typeof arg1 === 'object') { this.builder.payload.error = arg1 } + return this; + } + + /** + * Appends a message to the existing technical error, or creates one if none exists. + * @param {string} message + * @returns {OutputBuilder} this, for chaining + */ + appendError(message) { + if (this.builder.payload.error) { + this.builder.payload.error.msg = (this.builder.payload.error.msg || '') + message; + } else { + this.builder.payload.error = buildDefaultTechnicalError(message); + } + return this; } + /** + * Sets a business error (results in FAILED status rather than ERROR in Step). + * @param {string} errorMessage + * @returns {OutputBuilder} this, for chaining + */ setBusinessError(errorMessage) { this.builder.payload.error = buildDefaultBusinessError(errorMessage) + return this; } + /** + * Adds an attachment to the output. + * @param {object} attachment + */ attach(attachment) { this.builder.payload.attachments.push(attachment) } + // --- Measurement methods --- + + /** + * Starts a measurement with the given name. Must be followed by stopMeasure(). + * @param {string} id - the measurement name + * @param {number} [begin=Date.now()] - explicit start timestamp in ms + */ + startMeasure(id, begin = Date.now()) { + this._currentMeasure = { id, begin }; + } + + /** + * Stops the current measurement started by startMeasure() and records it. + * @param {{ status?: MeasureStatus, data?: object }} [options] + */ + stopMeasure({ status = MeasureStatus.PASSED, data } = {}) { + assertValidStatus(status); + if (!this._currentMeasure) return; + const duration = Date.now() - this._currentMeasure.begin; + this.addMeasure(this._currentMeasure.id, duration, { begin: this._currentMeasure.begin, status, data }); + this._currentMeasure = null; + } + + /** + * Adds a pre-timed measurement directly. + * @param {string} name + * @param {number} durationMillis + * @param {{ begin?: number, data?: object, status?: MeasureStatus }} [options] + */ + addMeasure(name, durationMillis, { begin, data, status = MeasureStatus.PASSED } = {}) { + assertValidStatus(status); + if (!this.builder.payload.measures) { + this.builder.payload.measures = []; + } + const measure = { name, duration: durationMillis, status }; + if (begin !== undefined) measure.begin = begin; + if (data !== undefined) measure.data = data; + this.builder.payload.measures.push(measure); + } + + // --- Private helpers --- + #attachException(e) { this.attach({ 'name': 'exception.log', @@ -65,4 +172,4 @@ function buildDefaultBusinessError(message) { return { msg: message, type: 'BUSINESS', root: true, code: 0 } } -module.exports = OutputBuilder; +module.exports = { OutputBuilder, MeasureStatus }; diff --git a/step-node/step-node-agent/keywords/keywords.js b/step-node/step-node-agent/keywords/keywords.js index d6fbb5a1c1..d14b153988 100644 --- a/step-node/step-node-agent/keywords/keywords.js +++ b/step-node/step-node-agent/keywords/keywords.js @@ -48,3 +48,55 @@ exports.onError = async (exception, input, output, session, properties) => { output.builder.payload.payload.onErrorCalled = true return input['rethrow_error'] } + +// --- output.add --- + +exports.AddKW = async (input, output, session, properties) => { + output.add('name', 'Alice').add('score', 42).add('active', true).send() +} + +// --- output.appendError --- + +exports.AppendErrorToExistingKW = async (input, output, session, properties) => { + output.setError('base error').appendError(' + extra detail').send() +} + +exports.AppendErrorToNoneKW = async (input, output, session, properties) => { + output.appendError('fresh error').send() +} + +// --- output.attach --- + +exports.AttachKW = async (input, output, session, properties) => { + output.attach({ name: 'report.txt', isDirectory: false, description: 'test attachment', hexContent: Buffer.from('hello').toString('base64') }) + output.send() +} + +// --- measurement methods --- + +exports.StartStopMeasureKW = async (input, output, session, properties) => { + output.startMeasure('step1') + await new Promise(r => setTimeout(r, 10)) + output.stopMeasure() + output.send() +} + +exports.StartStopMeasureWithStatusKW = async (input, output, session, properties) => { + output.startMeasure('failing-step') + output.stopMeasure({ status: 'FAILED', data: { reason: 'assertion failed' } }) + output.send() +} + +exports.AddMeasureKW = async (input, output, session, properties) => { + output.addMeasure('pre-timed', 150, { status: 'TECHNICAL_ERROR', begin: Date.now() - 150, data: { info: 'test' } }) + output.send() +} + +exports.MultipleMeasuresKW = async (input, output, session, properties) => { + output.startMeasure('first') + output.stopMeasure() + output.startMeasure('second') + output.stopMeasure({ status: 'FAILED' }) + output.addMeasure('third', 50) + output.send() +} diff --git a/step-node/step-node-agent/test/test.js b/step-node/step-node-agent/test/test.js index 4e2bae1bd3..f7924cc24a 100644 --- a/step-node/step-node-agent/test/test.js +++ b/step-node/step-node-agent/test/test.js @@ -1,5 +1,6 @@ const runner = require('../api/runner/runner')({'Property1': 'Prop1'}) const assert = require('assert') +const { OutputBuilder, MeasureStatus } = require('../api/controllers/output') ;(async () => { try { @@ -59,6 +60,90 @@ const assert = require('assert') output = await runner.run('Not existing Keyword', {}) assert.equal(output.error.msg, "Unable to find Keyword 'Not existing Keyword'") + // --- output.add --- + + // Test building payload incrementally with add() + output = await runner.run('AddKW', {}) + assert.equal(output.payload.name, 'Alice') + assert.equal(output.payload.score, 42) + assert.equal(output.payload.active, true) + + // --- output.appendError --- + + // Test appending to an existing error + output = await runner.run('AppendErrorToExistingKW', {}) + assert.equal(output.error.msg, 'base error + extra detail') + assert.equal(output.error.type, 'TECHNICAL') + + // Test appendError creating a new error when none exists + output = await runner.run('AppendErrorToNoneKW', {}) + assert.equal(output.error.msg, 'fresh error') + assert.equal(output.error.type, 'TECHNICAL') + + // --- output.attach --- + + output = await runner.run('AttachKW', {}) + assert.equal(output.attachments.length, 1) + assert.equal(output.attachments[0].name, 'report.txt') + assert.equal(output.attachments[0].isDirectory, false) + + // --- measurement methods --- + + // Test startMeasure / stopMeasure with default PASSED status + output = await runner.run('StartStopMeasureKW', {}) + assert.equal(output.measures.length, 1) + assert.equal(output.measures[0].name, 'step1') + assert.equal(output.measures[0].status, MeasureStatus.PASSED) + assert.ok(output.measures[0].duration >= 10, 'duration should be at least the sleep time') + assert.ok(typeof output.measures[0].begin === 'number') + + // Test stopMeasure with explicit FAILED status and custom data + output = await runner.run('StartStopMeasureWithStatusKW', {}) + assert.equal(output.measures[0].name, 'failing-step') + assert.equal(output.measures[0].status, MeasureStatus.FAILED) + assert.equal(output.measures[0].data.reason, 'assertion failed') + + // Test addMeasure with pre-set duration and TECHNICAL_ERROR status + output = await runner.run('AddMeasureKW', {}) + assert.equal(output.measures[0].name, 'pre-timed') + assert.equal(output.measures[0].duration, 150) + assert.equal(output.measures[0].status, MeasureStatus.TECHNICAL_ERROR) + assert.equal(output.measures[0].data.info, 'test') + assert.ok(typeof output.measures[0].begin === 'number') + + // Test multiple measures accumulated in one keyword call + output = await runner.run('MultipleMeasuresKW', {}) + assert.equal(output.measures.length, 3) + assert.equal(output.measures[0].name, 'first') + assert.equal(output.measures[0].status, MeasureStatus.PASSED) + assert.equal(output.measures[1].name, 'second') + assert.equal(output.measures[1].status, MeasureStatus.FAILED) + assert.equal(output.measures[2].name, 'third') + assert.equal(output.measures[2].duration, 50) + + // --- MeasureStatus validation (inline, no fork) --- + + // stopMeasure rejects unknown status strings + const ob = new OutputBuilder(null) + ob.startMeasure('test') + assert.throws( + () => ob.stopMeasure({ status: 'INVALID_STATUS' }), + (err) => err instanceof TypeError && err.message.includes('INVALID_STATUS') + ) + + // addMeasure rejects unknown status strings + assert.throws( + () => ob.addMeasure('test', 100, { status: 'WRONG' }), + (err) => err instanceof TypeError && err.message.includes('WRONG') + ) + + // All valid MeasureStatus values are accepted without throwing + const ob2 = new OutputBuilder(null) + for (const status of Object.values(MeasureStatus)) { + ob2.startMeasure('check') + assert.doesNotThrow(() => ob2.stopMeasure({ status })) + } + console.log('PASSED') } finally { runner.close(); diff --git a/step-node/step-node-agent/worker-wrapper.js b/step-node/step-node-agent/worker-wrapper.js index 0cd89ce7ca..084a0cf03c 100644 --- a/step-node/step-node-agent/worker-wrapper.js +++ b/step-node/step-node-agent/worker-wrapper.js @@ -17,7 +17,7 @@ * along with Step. If not, see . */ -const OutputBuilder = require("./output"); +const { OutputBuilder } = require("./output"); const Session = require("./session"); const fs = require("fs"); const path = require('path') From a12119c5fe3061f57b750baa19210aa801212fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Sat, 7 Mar 2026 14:10:13 +0100 Subject: [PATCH 05/21] SED-4581 Adding debug logs --- step-node/step-node-agent/worker-wrapper.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/step-node/step-node-agent/worker-wrapper.js b/step-node/step-node-agent/worker-wrapper.js index 084a0cf03c..f76055b8fb 100644 --- a/step-node/step-node-agent/worker-wrapper.js +++ b/step-node/step-node-agent/worker-wrapper.js @@ -28,7 +28,7 @@ process.on('message', async ({ projectPath, functionName, input, properties }) = const kwModules = await importAllKeywords(projectPath); const outputBuilder = new OutputBuilder(function (output) { - console.log(`[Agent fork] Keyword ${functionName} successfully executed`) + console.log(`[Agent fork] Keyword ${functionName} output sent`, output) process.send(output); }) @@ -39,6 +39,7 @@ process.on('message', async ({ projectPath, functionName, input, properties }) = try { await keyword(input, outputBuilder, session, properties); } catch (e) { + console.log("[Agent fork] Keyword execution failed", e) let onError = searchKeyword(kwModules, 'onError'); if (onError) { if (await onError(e, input, outputBuilder, session, properties)) { From 0afb11695d2b3e0b99d61a659fd4acc46f7c97da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Sat, 7 Mar 2026 20:50:34 +0100 Subject: [PATCH 06/21] SED-4581 Replacing callback in OutputBuilder by async KW functions --- .../api/controllers/controller.js | 66 +++++++++---------- .../step-node-agent/api/controllers/output.js | 16 ++--- .../api/controllers/session.js | 6 +- .../api/filemanager/filemanager.js | 2 +- step-node/step-node-agent/worker-wrapper.js | 48 +++++++------- 5 files changed, 63 insertions(+), 75 deletions(-) diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index f46ff387fc..12407ba575 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -75,38 +75,28 @@ class Controller { const payload = await this.process_(tokenId, keywordName, argument, properties) res.json(payload) } else { - const outputBuilder = new OutputBuilder(payload => res.json(payload)); + const outputBuilder = new OutputBuilder(); outputBuilder.fail("The token '" + tokenId + " doesn't exist on this agent. This usually means that the agent crashed and restarted."); } } - process_(tokenId, keywordName, argument, properties) { - return new Promise((resolve) => { - const outputBuilder = new OutputBuilder((output) => { - console.log(`[Controller] Keyword ${keywordName} successfully executed on token ${tokenId}`) - resolve(output) - }) - - try { - const filepathPromise = this.filemanager.loadOrGetKeywordFile( - this.agentContext.controllerUrl + '/grid/file/', - properties['$node.js.file.id'], - properties['$node.js.file.version'], - keywordName - ) - - filepathPromise.then((keywordPackageFile) => { - console.log('[Controller] Executing keyword ' + keywordName + ' using filepath ' + keywordPackageFile) - this.executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder) - }, (err) => { - console.log('[Controller] Error while attempting to run keyword ' + keywordName + ' :' + err) - outputBuilder.fail('Error while attempting to run keyword', err) - }) - } catch (e) { - outputBuilder.fail(e) - } - }) + async process_(tokenId, keywordName, argument, properties) { + const outputBuilder = new OutputBuilder(); + try { + const keywordPackageFile = await this.filemanager.loadOrGetKeywordFile( + this.agentContext.controllerUrl + '/grid/file/', + properties['$node.js.file.id'], + properties['$node.js.file.version'], + keywordName + ) + + await this.executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder) + } catch (e) { + console.log('[Controller] Unexpected error while executing keyword ' + keywordName + ' :' + err) + outputBuilder.fail('Unexpected error while executing keyword', err) + } + return outputBuilder.build(); } async executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder) { @@ -164,17 +154,21 @@ function createForkedAgent(keywordProjectPath) { function runKeywordTask(forkedAgent, keywordProjectPath, functionName, input, properties) { return new Promise((resolve) => { - forkedAgent.send({ projectPath: keywordProjectPath, functionName, input, properties }); + try { + forkedAgent.send({ projectPath: keywordProjectPath, functionName, input, properties }); - forkedAgent.removeAllListeners('message'); - forkedAgent.on('message', (result) => { - resolve(result); - }); + forkedAgent.removeAllListeners('message'); + forkedAgent.on('message', (result) => { + resolve(result); + }); - forkedAgent.removeAllListeners('error'); - forkedAgent.on('error', (err) => { - console.error('Error while calling forked agent:', err); - }); + forkedAgent.removeAllListeners('error'); + forkedAgent.on('error', (err) => { + console.error('Error while calling forked agent:', err); + }); + } catch (e) { + console.log('[Controller] Unexpected error while calling forked agent', e); + } }); } diff --git a/step-node/step-node-agent/api/controllers/output.js b/step-node/step-node-agent/api/controllers/output.js index a4629be0bf..74e9cc6013 100644 --- a/step-node/step-node-agent/api/controllers/output.js +++ b/step-node/step-node-agent/api/controllers/output.js @@ -13,8 +13,7 @@ function assertValidStatus(status) { } class OutputBuilder { - constructor(callback) { - this.callback = callback; + constructor() { this.builder = { payload: { attachments: [], payload: {} }, attachments: [] } this._currentMeasure = null; } @@ -40,23 +39,18 @@ class OutputBuilder { if (payload !== undefined) { this.builder.payload.payload = payload; } - if (this.callback) { - this.callback(this.builder) - } } merge(output) { this.builder.payload = output; - if (this.callback) { - this.callback(this.builder) - } } fail(arg1, arg2) { this.setError(arg1, arg2) - if (this.callback) { - this.callback(this.builder) - } + } + + build() { + return this.builder; } /** diff --git a/step-node/step-node-agent/api/controllers/session.js b/step-node/step-node-agent/api/controllers/session.js index 94a0590961..924e2fc8b4 100644 --- a/step-node/step-node-agent/api/controllers/session.js +++ b/step-node/step-node-agent/api/controllers/session.js @@ -18,7 +18,7 @@ */ class Session extends Map { [Symbol.dispose]() { - console.log(`Disposing Session: Cleaning up ${this.size} resources...`); + console.log(`[Session] Disposing Session: Cleaning up ${this.size} resources...`); for (const [key, resource] of this) { try { @@ -35,9 +35,9 @@ class Session extends Map { resource.close(); } - console.log(`Successfully closed resource: ${key}`); + console.log(`[Session] Successfully closed resource: ${key}`); } catch (err) { - console.error(`Failed to close resource ${key}:`, err); + console.error(`[Session] Failed to close resource ${key}:`, err); } } diff --git a/step-node/step-node-agent/api/filemanager/filemanager.js b/step-node/step-node-agent/api/filemanager/filemanager.js index e23156b1fc..45d722ed91 100644 --- a/step-node/step-node-agent/api/filemanager/filemanager.js +++ b/step-node/step-node-agent/api/filemanager/filemanager.js @@ -33,7 +33,7 @@ class FileManager { } } - loadOrGetKeywordFile(controllerUrl, fileId, fileVersionId) { + async loadOrGetKeywordFile(controllerUrl, fileId, fileVersionId) { return new Promise((resolve, reject) => { const filePath = this.workingDir + fileId + '/' + fileVersionId diff --git a/step-node/step-node-agent/worker-wrapper.js b/step-node/step-node-agent/worker-wrapper.js index f76055b8fb..63b4f48f2b 100644 --- a/step-node/step-node-agent/worker-wrapper.js +++ b/step-node/step-node-agent/worker-wrapper.js @@ -27,44 +27,44 @@ process.on('message', async ({ projectPath, functionName, input, properties }) = console.log("[Agent fork] Calling keyword " + functionName) const kwModules = await importAllKeywords(projectPath); - const outputBuilder = new OutputBuilder(function (output) { - console.log(`[Agent fork] Keyword ${functionName} output sent`, output) - process.send(output); - }) + const outputBuilder = new OutputBuilder(); - let keyword = searchKeyword(kwModules, functionName); - if(!keyword) { - outputBuilder.fail("Unable to find Keyword '" + functionName + "'"); - } else { - try { - await keyword(input, outputBuilder, session, properties); - } catch (e) { - console.log("[Agent fork] Keyword execution failed", e) - let onError = searchKeyword(kwModules, 'onError'); - if (onError) { - if (await onError(e, input, outputBuilder, session, properties)) { - console.log('[Agent fork] Keyword execution marked as failed: onError function returned \'true\'') - outputBuilder.fail(e) + try { + let keyword = searchKeyword(kwModules, functionName); + if (!keyword) { + outputBuilder.fail("Unable to find Keyword '" + functionName + "'"); + } else { + try { + await keyword(input, outputBuilder, session, properties); + } catch (e) { + let onError = searchKeyword(kwModules, 'onError'); + if (onError) { + if (await onError(e, input, outputBuilder, session, properties)) { + console.log('[Agent fork] Keyword execution failed and onError hook returned \'true\'') + outputBuilder.fail(e) + } else { + console.log('[Agent fork] Keyword execution failed and onError hook returned \'false\'') + } } else { - console.log('[Agent fork] Keyword execution marked as successful: execution failed but the onError function returned \'false\'') - outputBuilder.send() + console.log('[Agent fork] Keyword execution failed. No onError hook defined') + outputBuilder.fail(e) } - } else { - console.log('[Agent fork] Keyword execution marked as failed: Keyword execution failed and no onError function found') - outputBuilder.fail(e) } } + } finally { + console.log("[Agent fork] Returning output") + process.send(outputBuilder.build()); } async function importAllKeywords(projectPath) { const kwModules = []; const kwDir = path.resolve(projectPath, "./keywords"); - console.log("[Agent fork] Search for keywords in: " + kwDir) + console.log("[Agent fork] Searching keywords in: " + kwDir) const kwFiles = fs.readdirSync(kwDir); for (const kwFile of kwFiles) { if (kwFile.endsWith('.js')) { let kwModule = "file://" + path.resolve(kwDir, kwFile); - console.log("[Agent fork] Importing keywords from: " + kwModule) + console.log("[Agent fork] Importing keywords from module: " + kwModule) let module = await import(kwModule); kwModules.push(module); } From cd4efbb31a227955f201c3f597693b37d8d8632d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Sat, 7 Mar 2026 22:24:46 +0100 Subject: [PATCH 07/21] SED-4581 Fixing forked process leak in linux --- .../api/controllers/controller.js | 49 +++++++++----- .../step-node-agent/api/runner/runner.js | 4 ++ step-node/step-node-agent/worker-wrapper.js | 64 +++++++++++-------- 3 files changed, 72 insertions(+), 45 deletions(-) diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index 12407ba575..d9e9cc7ae5 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -136,7 +136,7 @@ class Controller { session.set('npmProjectPath', npmProjectPath); } - const output = await runKeywordTask(forkedAgent, npmProjectPath, keywordName, argument, properties); + const output = await forkedAgent.runKeywordTask(forkedAgent, npmProjectPath, keywordName, argument, properties); outputBuilder.merge(output.payload) } catch (e) { console.log("[Controller] Unexpected error occurred while executing keyword ", e) @@ -149,27 +149,42 @@ function createForkedAgent(keywordProjectPath) { fs.copyFileSync(path.resolve(__dirname, '../../worker-wrapper.js'), path.join(keywordProjectPath, './worker-wrapper.js')); fs.copyFileSync(path.join(__dirname, 'output.js'), path.join(keywordProjectPath, './output.js')); fs.copyFileSync(path.join(__dirname, 'session.js'), path.join(keywordProjectPath, './session.js')); - return fork('./worker-wrapper.js', [], {cwd: keywordProjectPath}); + return new ForkedAgent(fork('./worker-wrapper.js', [], {cwd: keywordProjectPath})); } -function runKeywordTask(forkedAgent, keywordProjectPath, functionName, input, properties) { - return new Promise((resolve) => { - try { - forkedAgent.send({ projectPath: keywordProjectPath, functionName, input, properties }); +class ForkedAgent { + + constructor(process) { + this.process = process; + } - forkedAgent.removeAllListeners('message'); - forkedAgent.on('message', (result) => { - resolve(result); - }); + runKeywordTask(forkedAgent, keywordProjectPath, functionName, input, properties) { + return new Promise((resolve) => { + try { + this.process.send({ type: "KEYWORD", projectPath: keywordProjectPath, functionName, input, properties }); + + this.process.removeAllListeners('message'); + this.process.on('message', (result) => { + resolve(result); + }); + + this.process.removeAllListeners('error'); + this.process.on('error', (err) => { + console.error('Error while calling forked agent:', err); + }); + } catch (e) { + console.log('[Controller] Unexpected error while calling forked agent', e); + } + }); + } - forkedAgent.removeAllListeners('error'); - forkedAgent.on('error', (err) => { - console.error('Error while calling forked agent:', err); - }); + close() { + try { + this.process.send({ type: "KILL" }); } catch (e) { - console.log('[Controller] Unexpected error while calling forked agent', e); + this.process.kill(); } - }); -} + } +} module.exports = Controller; diff --git a/step-node/step-node-agent/api/runner/runner.js b/step-node/step-node-agent/api/runner/runner.js index f686cf1c3b..4f802ac912 100644 --- a/step-node/step-node-agent/api/runner/runner.js +++ b/step-node/step-node-agent/api/runner/runner.js @@ -17,6 +17,10 @@ module.exports = function (properties = {}) { api.run = async function (keywordName, input) { const output = await controller.process_(tokenId, keywordName, input, properties) + const payload = output.payload; + if (payload.error) { + console.log("[Runner] The keyword execution returned an error", payload.error); + } return output.payload } diff --git a/step-node/step-node-agent/worker-wrapper.js b/step-node/step-node-agent/worker-wrapper.js index 63b4f48f2b..155493fe58 100644 --- a/step-node/step-node-agent/worker-wrapper.js +++ b/step-node/step-node-agent/worker-wrapper.js @@ -23,37 +23,44 @@ const fs = require("fs"); const path = require('path') const session = new Session(); -process.on('message', async ({ projectPath, functionName, input, properties }) => { - console.log("[Agent fork] Calling keyword " + functionName) - const kwModules = await importAllKeywords(projectPath); +process.on('message', async ({ type, projectPath, functionName, input, properties }) => { + if (type === 'KEYWORD') { + console.log("[Agent fork] Calling keyword " + functionName) + const kwModules = await importAllKeywords(projectPath); - const outputBuilder = new OutputBuilder(); + const outputBuilder = new OutputBuilder(); - try { - let keyword = searchKeyword(kwModules, functionName); - if (!keyword) { - outputBuilder.fail("Unable to find Keyword '" + functionName + "'"); - } else { - try { - await keyword(input, outputBuilder, session, properties); - } catch (e) { - let onError = searchKeyword(kwModules, 'onError'); - if (onError) { - if (await onError(e, input, outputBuilder, session, properties)) { - console.log('[Agent fork] Keyword execution failed and onError hook returned \'true\'') - outputBuilder.fail(e) + try { + let keyword = searchKeyword(kwModules, functionName); + if (!keyword) { + console.log('[Agent fork] Unable to find Keyword ' + functionName + "'"); + outputBuilder.fail("Unable to find Keyword '" + functionName + "'"); + } else { + try { + await keyword(input, outputBuilder, session, properties); + } catch (e) { + let onError = searchKeyword(kwModules, 'onError'); + if (onError) { + if (await onError(e, input, outputBuilder, session, properties)) { + console.log('[Agent fork] Keyword execution failed and onError hook returned \'true\'') + outputBuilder.fail(e) + } else { + console.log('[Agent fork] Keyword execution failed and onError hook returned \'false\'') + } } else { - console.log('[Agent fork] Keyword execution failed and onError hook returned \'false\'') + console.log('[Agent fork] Keyword execution failed. No onError hook defined') + outputBuilder.fail(e) } - } else { - console.log('[Agent fork] Keyword execution failed. No onError hook defined') - outputBuilder.fail(e) } } + } finally { + console.log("[Agent fork] Returning output") + process.send(outputBuilder.build()); } - } finally { - console.log("[Agent fork] Returning output") - process.send(outputBuilder.build()); + } else if (type === 'KILL') { + console.log("[Agent fork] Exiting...") + session[Symbol.dispose](); + process.exit(1) } async function importAllKeywords(projectPath) { @@ -78,10 +85,6 @@ process.on('message', async ({ projectPath, functionName, input, properties }) = } }); -process.on('exit', () => { - session[Symbol.dispose](); -}) - process.on('unhandledRejection', error => { console.log('[Agent fork] Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported', error) }) @@ -89,3 +92,8 @@ process.on('unhandledRejection', error => { process.on('uncaughtException', error => { console.log('[Agent fork] Critical: an unhandled error (uncaught exception) occurred and might not have been reported', error) }) + +process.on('SIGTERM', () => { + console.log("[Agent fork] Received SIGTERM. Exiting...") + process.exit(1); +}); From 4ed4fdc261628a318e1a5be731622e0fce689da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Mon, 9 Mar 2026 09:38:20 +0100 Subject: [PATCH 08/21] SED-4581 Migrating tests to jest --- .../{worker-wrapper.js => agent-fork.js} | 0 .../api/controllers/controller.js | 4 +- step-node/step-node-agent/package.json | 22 +- step-node/step-node-agent/test/runner.test.js | 206 ++++++++++++++++++ step-node/step-node-agent/test/test.js | 151 ------------- 5 files changed, 228 insertions(+), 155 deletions(-) rename step-node/step-node-agent/{worker-wrapper.js => agent-fork.js} (100%) create mode 100644 step-node/step-node-agent/test/runner.test.js delete mode 100644 step-node/step-node-agent/test/test.js diff --git a/step-node/step-node-agent/worker-wrapper.js b/step-node/step-node-agent/agent-fork.js similarity index 100% rename from step-node/step-node-agent/worker-wrapper.js rename to step-node/step-node-agent/agent-fork.js diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index d9e9cc7ae5..ae7b6faf84 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -146,10 +146,10 @@ class Controller { } function createForkedAgent(keywordProjectPath) { - fs.copyFileSync(path.resolve(__dirname, '../../worker-wrapper.js'), path.join(keywordProjectPath, './worker-wrapper.js')); + fs.copyFileSync(path.resolve(__dirname, '../../agent-fork.js'), path.join(keywordProjectPath, './agent-fork.js')); fs.copyFileSync(path.join(__dirname, 'output.js'), path.join(keywordProjectPath, './output.js')); fs.copyFileSync(path.join(__dirname, 'session.js'), path.join(keywordProjectPath, './session.js')); - return new ForkedAgent(fork('./worker-wrapper.js', [], {cwd: keywordProjectPath})); + return new ForkedAgent(fork('./agent-fork.js', [], {cwd: keywordProjectPath})); } class ForkedAgent { diff --git a/step-node/step-node-agent/package.json b/step-node/step-node-agent/package.json index 87dbd0d022..a08ecbc3f7 100644 --- a/step-node/step-node-agent/package.json +++ b/step-node/step-node-agent/package.json @@ -7,10 +7,16 @@ "debug": "node --inspect-brk server.js", "eslint": "node_modules/.bin/eslint .", "start": "node server.js", - "test": "node test/test.js", + "test": "jest", "build": "", "deploy": "npm publish" }, + "jest": { + "testMatch": [ + "**/test/**/*.test.js" + ], + "testTimeout": 30000 + }, "author": "Jerome Comte", "license": "AGPL-3.0", "repository": { @@ -25,6 +31,7 @@ "eslint-plugin-promise": "^3.8.0", "eslint-plugin-standard": "^3.1.0", "husky": "^0.14.3", + "jest": "^29.7.0", "nodemon": "^1.18.7" }, "dependencies": { @@ -51,5 +58,16 @@ ], "bin": { "step-node-agent": "./bin/step-node-agent" - } + }, + "bundleDependencies": [ + "express", + "get-fqdn", + "jsonwebtoken", + "minimist", + "request", + "shelljs", + "unzip-stream", + "uuid", + "yaml" + ] } diff --git a/step-node/step-node-agent/test/runner.test.js b/step-node/step-node-agent/test/runner.test.js new file mode 100644 index 0000000000..7618303887 --- /dev/null +++ b/step-node/step-node-agent/test/runner.test.js @@ -0,0 +1,206 @@ +const { OutputBuilder, MeasureStatus } = require('../api/controllers/output') + +describe('runner', () => { + let runner + + beforeAll(() => { + runner = require('../api/runner/runner')({ Property1: 'Prop1' }) + }) + + afterAll(() => { + runner.close() + }) + + // --------------------------------------------------------------------------- + // Happy path + // --------------------------------------------------------------------------- + + test('Echo KW returns input param and agent property', async () => { + const output = await runner.run('Echo', { Param1: 'Val1' }) + expect(output.payload.Param1).toBe('Val1') + expect(output.payload.properties.Property1).toBe('Prop1') + }) + + // --------------------------------------------------------------------------- + // Error-setting methods + // --------------------------------------------------------------------------- + + describe('output.setError', () => { + test('sets error message and TECHNICAL type', async () => { + const output = await runner.run('SetErrorTestKW', { ErrorMsg: 'MyError', rethrow_error: true }) + expect(output.error.msg).toBe('MyError') + expect(output.error.type).toBe('TECHNICAL') + }) + + test('accepts an exception as argument', async () => { + const output = await runner.run('SetErrorWithExceptionKW', { ErrorMsg: 'MyError2', rethrow_error: true }) + expect(output.error.msg).toBe('MyError2') + }) + + test('accepts a message and exception, attaches stack trace', async () => { + const output = await runner.run('SetErrorWithMessageAndExceptionKW', { ErrorMsg: 'MyError3', rethrow_error: true }) + expect(output.error.msg).toBe('MyError3') + expect(output.attachments).toHaveLength(1) + }) + }) + + test('output.fail sets error message', async () => { + const output = await runner.run('FailKW', { ErrorMsg: 'MyError4', rethrow_error: true }) + expect(output.error.msg).toBe('MyError4') + }) + + test('output.setBusinessError sets error message', async () => { + const output = await runner.run('BusinessErrorTestKW', { ErrorMsg: 'MyBusinessError', rethrow_error: true }) + expect(output.error.msg).toBe('MyBusinessError') + }) + + // --------------------------------------------------------------------------- + // onError hook + // --------------------------------------------------------------------------- + + describe('onError hook', () => { + test('is called and error is propagated when rethrow_error=true', async () => { + const output = await runner.run('ErrorTestKW', { ErrorMsg: 'Error - rethrow', rethrow_error: true }) + expect(output.error.msg).toBe('Error - rethrow') + expect(output.payload.onErrorCalled).toBe(true) + }) + + test('is called and error is suppressed when rethrow_error=false', async () => { + const output = await runner.run('ErrorTestKW', { ErrorMsg: 'Error - do not rethrow', rethrow_error: false }) + expect(output.error).toBeUndefined() + expect(output.payload.onErrorCalled).toBe(true) + }) + }) + + // --------------------------------------------------------------------------- + // Unhandled async errors + // --------------------------------------------------------------------------- + + test('rejected promises do not surface as output error', async () => { + const output = await runner.run('ErrorRejectedPromiseTestKW', { Param1: 'Val1' }) + expect(output.error).toBeUndefined() + }) + + test('uncaught exceptions do not surface as output error', async () => { + const output = await runner.run('ErrorUncaughtExceptionTestKW', { Param1: 'Val1' }) + expect(output.error).toBeUndefined() + }) + + // --------------------------------------------------------------------------- + // Unknown keyword + // --------------------------------------------------------------------------- + + test('returns error for non-existing keyword', async () => { + const output = await runner.run('Not existing Keyword', {}) + expect(output.error.msg).toBe("Unable to find Keyword 'Not existing Keyword'") + }) + + // --------------------------------------------------------------------------- + // output.add + // --------------------------------------------------------------------------- + + test('output.add builds payload incrementally', async () => { + const output = await runner.run('AddKW', {}) + expect(output.payload.name).toBe('Alice') + expect(output.payload.score).toBe(42) + expect(output.payload.active).toBe(true) + }) + + // --------------------------------------------------------------------------- + // output.appendError + // --------------------------------------------------------------------------- + + describe('output.appendError', () => { + test('appends to an existing error', async () => { + const output = await runner.run('AppendErrorToExistingKW', {}) + expect(output.error.msg).toBe('base error + extra detail') + expect(output.error.type).toBe('TECHNICAL') + }) + + test('creates a new error when none exists', async () => { + const output = await runner.run('AppendErrorToNoneKW', {}) + expect(output.error.msg).toBe('fresh error') + expect(output.error.type).toBe('TECHNICAL') + }) + }) + + // --------------------------------------------------------------------------- + // output.attach + // --------------------------------------------------------------------------- + + test('output.attach adds an attachment', async () => { + const output = await runner.run('AttachKW', {}) + expect(output.attachments).toHaveLength(1) + expect(output.attachments[0].name).toBe('report.txt') + expect(output.attachments[0].isDirectory).toBe(false) + }) + + // --------------------------------------------------------------------------- + // Measurement methods + // --------------------------------------------------------------------------- + + describe('measurements', () => { + test('startMeasure / stopMeasure produces a PASSED measure', async () => { + const output = await runner.run('StartStopMeasureKW', {}) + expect(output.measures).toHaveLength(1) + expect(output.measures[0].name).toBe('step1') + expect(output.measures[0].status).toBe(MeasureStatus.PASSED) + expect(output.measures[0].duration).toBeGreaterThanOrEqual(10) + expect(typeof output.measures[0].begin).toBe('number') + }) + + test('stopMeasure accepts an explicit FAILED status and custom data', async () => { + const output = await runner.run('StartStopMeasureWithStatusKW', {}) + expect(output.measures[0].name).toBe('failing-step') + expect(output.measures[0].status).toBe(MeasureStatus.FAILED) + expect(output.measures[0].data.reason).toBe('assertion failed') + }) + + test('addMeasure accepts a pre-set duration and TECHNICAL_ERROR status', async () => { + const output = await runner.run('AddMeasureKW', {}) + expect(output.measures[0].name).toBe('pre-timed') + expect(output.measures[0].duration).toBe(150) + expect(output.measures[0].status).toBe(MeasureStatus.TECHNICAL_ERROR) + expect(output.measures[0].data.info).toBe('test') + expect(typeof output.measures[0].begin).toBe('number') + }) + + test('multiple measures accumulate in a single keyword call', async () => { + const output = await runner.run('MultipleMeasuresKW', {}) + expect(output.measures).toHaveLength(3) + expect(output.measures[0].name).toBe('first') + expect(output.measures[0].status).toBe(MeasureStatus.PASSED) + expect(output.measures[1].name).toBe('second') + expect(output.measures[1].status).toBe(MeasureStatus.FAILED) + expect(output.measures[2].name).toBe('third') + expect(output.measures[2].duration).toBe(50) + }) + }) +}) + +// --------------------------------------------------------------------------- +// MeasureStatus validation (no runner needed) +// --------------------------------------------------------------------------- + +describe('MeasureStatus validation', () => { + test('stopMeasure throws TypeError for unknown status', () => { + const ob = new OutputBuilder(null) + ob.startMeasure('test') + expect(() => ob.stopMeasure({ status: 'INVALID_STATUS' })).toThrow(TypeError) + expect(() => ob.stopMeasure({ status: 'INVALID_STATUS' })).toThrow('INVALID_STATUS') + }) + + test('addMeasure throws TypeError for unknown status', () => { + const ob = new OutputBuilder(null) + expect(() => ob.addMeasure('test', 100, { status: 'WRONG' })).toThrow(TypeError) + expect(() => ob.addMeasure('test', 100, { status: 'WRONG' })).toThrow('WRONG') + }) + + test('all valid MeasureStatus values are accepted without throwing', () => { + const ob = new OutputBuilder(null) + for (const status of Object.values(MeasureStatus)) { + ob.startMeasure('check') + expect(() => ob.stopMeasure({ status })).not.toThrow() + } + }) +}) diff --git a/step-node/step-node-agent/test/test.js b/step-node/step-node-agent/test/test.js deleted file mode 100644 index f7924cc24a..0000000000 --- a/step-node/step-node-agent/test/test.js +++ /dev/null @@ -1,151 +0,0 @@ -const runner = require('../api/runner/runner')({'Property1': 'Prop1'}) -const assert = require('assert') -const { OutputBuilder, MeasureStatus } = require('../api/controllers/output') - -;(async () => { - try { - // Test the happy path - let output = await runner.run('Echo', {Param1: 'Val1'}) - assert.equal(output.payload.Param1, 'Val1') - assert.equal(output.payload.properties.Property1, 'Prop1') - - // Test the method output.setError - let errorMsg = 'MyError' - output = await runner.run('SetErrorTestKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - assert.equal(output.error.type, 'TECHNICAL') - - // Test the method output.setError with an exception as argument - errorMsg = 'MyError2' - output = await runner.run('SetErrorWithExceptionKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - - // Test the method output.setError with an error message and an exception as argument - errorMsg = 'MyError3' - output = await runner.run('SetErrorWithMessageAndExceptionKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - assert.equal(output.attachments.length, 1) - - // Test the method output.fail - errorMsg = 'MyError4' - output = await runner.run('FailKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - - // Test the method output.setBusinessError - errorMsg = 'MyBusinessError' - output = await runner.run('BusinessErrorTestKW', {ErrorMsg: errorMsg, rethrow_error: true}) - assert.equal(output.error.msg, errorMsg) - - // Test onError hook - const errorMsg1 = 'Error - rethrow' - const output1 = await runner.run('ErrorTestKW', {ErrorMsg: errorMsg1, rethrow_error: true}) - assert.equal(output1.error.msg, errorMsg1) - assert.equal(output1.payload.onErrorCalled, true) - - // Test onError hook with no rethrow - const errorMsg2 = 'Error - do not rethrow' - const output2 = await runner.run('ErrorTestKW', {ErrorMsg: errorMsg2, rethrow_error: false}) - assert.equal(output2.error, undefined) - assert.equal(output1.payload.onErrorCalled, true) - - // Test rejected promises - output = await runner.run('ErrorRejectedPromiseTestKW', {Param1: 'Val1'}) - assert.equal(output.error, undefined) - - // Test uncaught exceptions - output = await runner.run('ErrorUncaughtExceptionTestKW', {Param1: 'Val1'}) - assert.equal(output.error, undefined) - - // Test not existing keyword - output = await runner.run('Not existing Keyword', {}) - assert.equal(output.error.msg, "Unable to find Keyword 'Not existing Keyword'") - - // --- output.add --- - - // Test building payload incrementally with add() - output = await runner.run('AddKW', {}) - assert.equal(output.payload.name, 'Alice') - assert.equal(output.payload.score, 42) - assert.equal(output.payload.active, true) - - // --- output.appendError --- - - // Test appending to an existing error - output = await runner.run('AppendErrorToExistingKW', {}) - assert.equal(output.error.msg, 'base error + extra detail') - assert.equal(output.error.type, 'TECHNICAL') - - // Test appendError creating a new error when none exists - output = await runner.run('AppendErrorToNoneKW', {}) - assert.equal(output.error.msg, 'fresh error') - assert.equal(output.error.type, 'TECHNICAL') - - // --- output.attach --- - - output = await runner.run('AttachKW', {}) - assert.equal(output.attachments.length, 1) - assert.equal(output.attachments[0].name, 'report.txt') - assert.equal(output.attachments[0].isDirectory, false) - - // --- measurement methods --- - - // Test startMeasure / stopMeasure with default PASSED status - output = await runner.run('StartStopMeasureKW', {}) - assert.equal(output.measures.length, 1) - assert.equal(output.measures[0].name, 'step1') - assert.equal(output.measures[0].status, MeasureStatus.PASSED) - assert.ok(output.measures[0].duration >= 10, 'duration should be at least the sleep time') - assert.ok(typeof output.measures[0].begin === 'number') - - // Test stopMeasure with explicit FAILED status and custom data - output = await runner.run('StartStopMeasureWithStatusKW', {}) - assert.equal(output.measures[0].name, 'failing-step') - assert.equal(output.measures[0].status, MeasureStatus.FAILED) - assert.equal(output.measures[0].data.reason, 'assertion failed') - - // Test addMeasure with pre-set duration and TECHNICAL_ERROR status - output = await runner.run('AddMeasureKW', {}) - assert.equal(output.measures[0].name, 'pre-timed') - assert.equal(output.measures[0].duration, 150) - assert.equal(output.measures[0].status, MeasureStatus.TECHNICAL_ERROR) - assert.equal(output.measures[0].data.info, 'test') - assert.ok(typeof output.measures[0].begin === 'number') - - // Test multiple measures accumulated in one keyword call - output = await runner.run('MultipleMeasuresKW', {}) - assert.equal(output.measures.length, 3) - assert.equal(output.measures[0].name, 'first') - assert.equal(output.measures[0].status, MeasureStatus.PASSED) - assert.equal(output.measures[1].name, 'second') - assert.equal(output.measures[1].status, MeasureStatus.FAILED) - assert.equal(output.measures[2].name, 'third') - assert.equal(output.measures[2].duration, 50) - - // --- MeasureStatus validation (inline, no fork) --- - - // stopMeasure rejects unknown status strings - const ob = new OutputBuilder(null) - ob.startMeasure('test') - assert.throws( - () => ob.stopMeasure({ status: 'INVALID_STATUS' }), - (err) => err instanceof TypeError && err.message.includes('INVALID_STATUS') - ) - - // addMeasure rejects unknown status strings - assert.throws( - () => ob.addMeasure('test', 100, { status: 'WRONG' }), - (err) => err instanceof TypeError && err.message.includes('WRONG') - ) - - // All valid MeasureStatus values are accepted without throwing - const ob2 = new OutputBuilder(null) - for (const status of Object.values(MeasureStatus)) { - ob2.startMeasure('check') - assert.doesNotThrow(() => ob2.stopMeasure({ status })) - } - - console.log('PASSED') - } finally { - runner.close(); - } -})() From eda0ee6a2122fb7db1b92d5f88f0b87c4c5e60b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Mon, 9 Mar 2026 09:39:30 +0100 Subject: [PATCH 09/21] SED-4581 Migrating tests to jest --- step-node/step-node-agent/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/step-node/step-node-agent/.gitignore b/step-node/step-node-agent/.gitignore index 94f083ab06..57b766a6d0 100644 --- a/step-node/step-node-agent/.gitignore +++ b/step-node/step-node-agent/.gitignore @@ -1,4 +1,5 @@ node_modules/ .npm !/bin -filemanager/work/ \ No newline at end of file +filemanager/work/ +/coverage/ From a5bdc4a0a44c27b78a712a160ecfa4ee4b5769d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Wed, 11 Mar 2026 21:45:48 +0100 Subject: [PATCH 10/21] SED-4581 Implementing attachment of npm install and agent forker logs --- .../api/controllers/controller.js | 112 ++++++++++++++++-- 1 file changed, 103 insertions(+), 9 deletions(-) diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index ae7b6faf84..b3ecd1ead1 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -1,9 +1,11 @@ const fs = require("fs"); const path = require("path"); -const {fork, execSync} = require("child_process"); +const {fork, spawn, spawnSync} = require("child_process"); const Session = require('./session'); const { OutputBuilder } = require('./output'); +const npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm'; + process.on('unhandledRejection', error => { console.log('[Controller] Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported', error) }) @@ -93,8 +95,8 @@ class Controller { await this.executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder) } catch (e) { - console.log('[Controller] Unexpected error while executing keyword ' + keywordName + ' :' + err) - outputBuilder.fail('Unexpected error while executing keyword', err) + console.log('[Controller] Unexpected error while executing keyword ' + keywordName + ' :', e) + outputBuilder.fail('Unexpected error while executing keyword', e) } return outputBuilder.build(); } @@ -127,29 +129,89 @@ class Controller { throw new Error("Multiple projects not supported within the same session"); } + let npmAttachment = null; + let isDebugEnabled = properties['debug'] === 'true'; + if (!forkedAgent) { console.log('[Controller] Starting agent fork in ' + npmProjectPath + ' for token ' + tokenId) forkedAgent = createForkedAgent(npmProjectPath); + console.log('[Controller] Running npm install in ' + npmProjectPath + ' for token ' + tokenId) - execSync('npm install', { cwd: npmProjectPath, stdio: 'inherit' }); + const npmInstallResult = await this.executeNpmInstall(npmProjectPath); + const npmInstallOutput = Buffer.concat([npmInstallResult.stdout || Buffer.alloc(0), npmInstallResult.stderr || Buffer.alloc(0)]); + const npmInstallFailed = npmInstallResult.status !== 0 || npmInstallResult.error != null; + if (npmInstallFailed || isDebugEnabled) { + npmAttachment = { + name: 'npm-install.log', + isDirectory: false, + description: 'npm install output', + hexContent: npmInstallOutput.toString('base64') + }; + } + session.set('forkedAgent', forkedAgent); session.set('npmProjectPath', npmProjectPath); + + if (npmInstallFailed) { + throw npmInstallResult.error || new Error('npm install exited with code ' + npmInstallResult.status); + } + } + + console.log('[Controller] Executing keyword \'' + keywordName + ' in ' + npmProjectPath + ' for token ' + tokenId) + const { result, outputBuffer } = await forkedAgent.runKeywordTask(forkedAgent, npmProjectPath, keywordName, argument, properties); + outputBuilder.merge(result.payload) + + if (npmAttachment) { + outputBuilder.attach(npmAttachment); } - const output = await forkedAgent.runKeywordTask(forkedAgent, npmProjectPath, keywordName, argument, properties); - outputBuilder.merge(output.payload) + if (outputBuffer && (result.payload.error || isDebugEnabled)) { + const forkAttachment = { + name: 'keyword-process.log', + isDirectory: false, + description: 'Output of the forked keyword process', + hexContent: outputBuffer.toString('base64'), + }; + outputBuilder.attach(forkAttachment); + } } catch (e) { console.log("[Controller] Unexpected error occurred while executing keyword ", e) outputBuilder.fail('An error occurred while attempting to execute the keyword ' + keywordName, e) } } + + async executeNpmInstall(npmProjectPath) { + return await new Promise((resolve) => { + const child = spawn(npmCommand, ['install'], {cwd: npmProjectPath, shell: false}); + const stdoutChunks = []; + const stderrChunks = []; + + child.stdout.on('data', (data) => { + stdoutChunks.push(data); + process.stdout.write(data); + }); + + child.stderr.on('data', (data) => { + stderrChunks.push(data); + process.stderr.write(data); + }); + + child.on('error', (error) => { + resolve({status: null, error, stdout: Buffer.concat(stdoutChunks), stderr: Buffer.concat(stderrChunks)}); + }); + + child.on('close', (code) => { + resolve({status: code, error: null, stdout: Buffer.concat(stdoutChunks), stderr: Buffer.concat(stderrChunks)}); + }); + }); + } } function createForkedAgent(keywordProjectPath) { fs.copyFileSync(path.resolve(__dirname, '../../agent-fork.js'), path.join(keywordProjectPath, './agent-fork.js')); fs.copyFileSync(path.join(__dirname, 'output.js'), path.join(keywordProjectPath, './output.js')); fs.copyFileSync(path.join(__dirname, 'session.js'), path.join(keywordProjectPath, './session.js')); - return new ForkedAgent(fork('./agent-fork.js', [], {cwd: keywordProjectPath})); + return new ForkedAgent(fork('./agent-fork.js', [], {cwd: keywordProjectPath, silent: true})); } class ForkedAgent { @@ -161,17 +223,49 @@ class ForkedAgent { runKeywordTask(forkedAgent, keywordProjectPath, functionName, input, properties) { return new Promise((resolve) => { try { - this.process.send({ type: "KEYWORD", projectPath: keywordProjectPath, functionName, input, properties }); + const stdoutChunks = []; + const stderrChunks = []; + + const stdoutListener = (data) => { + stdoutChunks.push(data); + process.stdout.write(data); + }; + const stderrListener = (data) => { + stderrChunks.push(data); + process.stderr.write(data); + }; + + if (this.process.stdout) { + this.process.stdout.on('data', stdoutListener); + } + + if (this.process.stderr) { + this.process.stderr.on('data', stderrListener); + } this.process.removeAllListeners('message'); this.process.on('message', (result) => { - resolve(result); + if (this.process.stdout) { + this.process.stdout.removeListener('data', stdoutListener); + } + if (this.process.stderr) { + this.process.stderr.removeListener('data', stderrListener); + } + + const outputBuffer = Buffer.concat([ + ...stdoutChunks, + ...stderrChunks, + ]); + + resolve({ result, outputBuffer }); }); this.process.removeAllListeners('error'); this.process.on('error', (err) => { console.error('Error while calling forked agent:', err); }); + + this.process.send({ type: "KEYWORD", projectPath: keywordProjectPath, functionName, input, properties }); } catch (e) { console.log('[Controller] Unexpected error while calling forked agent', e); } From ed1404c5d4b4b8f6c68647ce5ea88da5eb69389e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Thu, 12 Mar 2026 09:18:20 +0100 Subject: [PATCH 11/21] SED-4581 Refactoring logging --- .../api/controllers/controller.js | 32 ++++++++++--------- .../api/controllers/session.js | 13 ++++++-- .../api/filemanager/filemanager.js | 23 ++++++------- step-node/step-node-agent/api/logger.js | 15 +++++++++ .../step-node-agent/api/runner/runner.js | 3 +- step-node/step-node-agent/package.json | 1 + step-node/step-node-agent/server.js | 24 +++++++------- step-node/step-node-agent/test/runner.test.js | 2 +- 8 files changed, 70 insertions(+), 43 deletions(-) create mode 100644 step-node/step-node-agent/api/logger.js diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index b3ecd1ead1..4e96a43380 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -3,15 +3,16 @@ const path = require("path"); const {fork, spawn, spawnSync} = require("child_process"); const Session = require('./session'); const { OutputBuilder } = require('./output'); +const logger = require('../logger').child({ component: 'Controller' }); const npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm'; process.on('unhandledRejection', error => { - console.log('[Controller] Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported', error) + logger.error('Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported: ' + error) }) process.on('uncaughtException', error => { - console.log('[Controller] Critical: an unhandled error (uncaught exception) occurred and might not have been reported', error) + logger.error('Critical: an unhandled error (uncaught exception) occurred and might not have been reported: ' + error) }) class Controller { @@ -30,7 +31,7 @@ class Controller { } reserveToken_(tokenId) { - console.log('[Controller] Reserving token: ' + tokenId) + logger.info('Reserving token: ' + tokenId) } releaseToken(req, res) { @@ -39,7 +40,7 @@ class Controller { } releaseToken_(tokenId) { - console.log('[Controller] Releasing token: ' + tokenId) + logger.info('Releasing token: ' + tokenId) const session = this.agentContext.tokenSessions[tokenId] if (session) { @@ -47,13 +48,13 @@ class Controller { session[Symbol.dispose](); this.agentContext.tokenSessions[tokenId] = null; } else { - console.log('[Controller] No session founds for token: ' + tokenId) + logger.warn('No session found for token: ' + tokenId) } } interruptExecution(req, res) { const tokenId = req.params.tokenId - console.warn('[Controller] Interrupting token: ' + tokenId + ' : not implemented') + logger.warn('Interrupting token: ' + tokenId + ' : not implemented') } async process(req, res) { @@ -64,7 +65,7 @@ class Controller { let token = this.agentContext.tokens.find(value => value.id == tokenId); if(token) { - console.log('[Controller] Using token: ' + tokenId + ' to execute ' + keywordName) + logger.info('Using token: ' + tokenId + ' to execute ' + keywordName) // add the agent properties const agentProperties = this.agentContext.properties @@ -95,7 +96,7 @@ class Controller { await this.executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder) } catch (e) { - console.log('[Controller] Unexpected error while executing keyword ' + keywordName + ' :', e) + logger.error('Unexpected error while executing keyword ' + keywordName + ': ' + e) outputBuilder.fail('Unexpected error while executing keyword', e) } return outputBuilder.build(); @@ -115,7 +116,7 @@ class Controller { npmProjectPath = path.resolve(keywordPackageFile) } - console.log('[Controller] Executing keyword in project ' + npmProjectPath + ' for token ' + tokenId) + logger.info('Executing keyword in project ' + npmProjectPath + ' for token ' + tokenId) let session = this.agentContext.tokenSessions[tokenId] if (!session) { @@ -133,10 +134,10 @@ class Controller { let isDebugEnabled = properties['debug'] === 'true'; if (!forkedAgent) { - console.log('[Controller] Starting agent fork in ' + npmProjectPath + ' for token ' + tokenId) + logger.info('Starting agent fork in ' + npmProjectPath + ' for token ' + tokenId) forkedAgent = createForkedAgent(npmProjectPath); - console.log('[Controller] Running npm install in ' + npmProjectPath + ' for token ' + tokenId) + logger.info('Running npm install in ' + npmProjectPath + ' for token ' + tokenId) const npmInstallResult = await this.executeNpmInstall(npmProjectPath); const npmInstallOutput = Buffer.concat([npmInstallResult.stdout || Buffer.alloc(0), npmInstallResult.stderr || Buffer.alloc(0)]); const npmInstallFailed = npmInstallResult.status !== 0 || npmInstallResult.error != null; @@ -153,11 +154,12 @@ class Controller { session.set('npmProjectPath', npmProjectPath); if (npmInstallFailed) { + outputBuilder.attach(npmAttachment); throw npmInstallResult.error || new Error('npm install exited with code ' + npmInstallResult.status); } } - console.log('[Controller] Executing keyword \'' + keywordName + ' in ' + npmProjectPath + ' for token ' + tokenId) + logger.info('Executing keyword \'' + keywordName + '\' in ' + npmProjectPath + ' for token ' + tokenId) const { result, outputBuffer } = await forkedAgent.runKeywordTask(forkedAgent, npmProjectPath, keywordName, argument, properties); outputBuilder.merge(result.payload) @@ -175,7 +177,7 @@ class Controller { outputBuilder.attach(forkAttachment); } } catch (e) { - console.log("[Controller] Unexpected error occurred while executing keyword ", e) + logger.error('Unexpected error occurred while executing keyword: ' + e) outputBuilder.fail('An error occurred while attempting to execute the keyword ' + keywordName, e) } } @@ -262,12 +264,12 @@ class ForkedAgent { this.process.removeAllListeners('error'); this.process.on('error', (err) => { - console.error('Error while calling forked agent:', err); + logger.error('Error while calling forked agent: ' + err) }); this.process.send({ type: "KEYWORD", projectPath: keywordProjectPath, functionName, input, properties }); } catch (e) { - console.log('[Controller] Unexpected error while calling forked agent', e); + logger.error('Unexpected error while calling forked agent: ' + e) } }); } diff --git a/step-node/step-node-agent/api/controllers/session.js b/step-node/step-node-agent/api/controllers/session.js index 924e2fc8b4..0492a23040 100644 --- a/step-node/step-node-agent/api/controllers/session.js +++ b/step-node/step-node-agent/api/controllers/session.js @@ -16,9 +16,16 @@ * You should have received a copy of the GNU Affero General Public License * along with Step. If not, see . */ +let logger; +try { + logger = require('../logger').child({ component: 'Session' }); +} catch { + logger = { info: console.log.bind(console), warn: console.warn.bind(console), error: console.error.bind(console) }; +} + class Session extends Map { [Symbol.dispose]() { - console.log(`[Session] Disposing Session: Cleaning up ${this.size} resources...`); + logger.info(`Disposing Session: Cleaning up ${this.size} resources...`); for (const [key, resource] of this) { try { @@ -35,9 +42,9 @@ class Session extends Map { resource.close(); } - console.log(`[Session] Successfully closed resource: ${key}`); + logger.info(`Successfully closed resource: ${key}`); } catch (err) { - console.error(`[Session] Failed to close resource ${key}:`, err); + logger.error(`Failed to close resource ${key}: ${err}`); } } diff --git a/step-node/step-node-agent/api/filemanager/filemanager.js b/step-node/step-node-agent/api/filemanager/filemanager.js index 45d722ed91..f31a578c77 100644 --- a/step-node/step-node-agent/api/filemanager/filemanager.js +++ b/step-node/step-node-agent/api/filemanager/filemanager.js @@ -4,15 +4,16 @@ const https = require('https') const url = require('url') const unzip = require('unzip-stream') const jwtUtils = require('../../utils/jwtUtils') +const logger = require('../logger').child({ component: 'FileManager' }) class FileManager { constructor(agentContext) { this.agentContext = agentContext; const filemanagerPath = agentContext.properties['filemanagerPath'] || 'filemanager' this.workingDir = filemanagerPath + '/work/' - console.log('[FileManager] Starting file manager using working directory: ' + this.workingDir) + logger.info('Starting file manager using working directory: ' + this.workingDir) - console.log('[FileManager] Clearing working dir: ' + this.workingDir) + logger.info('Clearing working dir: ' + this.workingDir) fs.rmSync(this.workingDir, { recursive: true, force: true }) fs.mkdirSync(this.workingDir, { recursive: true }) @@ -40,7 +41,7 @@ class FileManager { const cacheEntry = this.#getCacheEntry(fileId, fileVersionId) if (cacheEntry) { if (!cacheEntry.loading) { - console.log('[FileManager] Entry found for fileId ' + fileId + ': ' + cacheEntry.name) + logger.info('Entry found for fileId ' + fileId + ': ' + cacheEntry.name) const fileName = cacheEntry.name if (fs.existsSync(filePath + '/' + fileName)) { @@ -49,23 +50,23 @@ class FileManager { reject(new Error('Entry exists but no file found: ' + filePath + '/' + fileName)) } } else { - console.log('[FileManager] Waiting for cache entry to be loaded for fileId ' + fileId) + logger.info('Waiting for cache entry to be loaded for fileId ' + fileId) cacheEntry.promises.push((result) => { - console.log('[FileManager] Cache entry loaded for fileId ' + fileId) + logger.info('Cache entry loaded for fileId ' + fileId) resolve(result) }) } } else { this.#putCacheEntry(fileId, fileVersionId, { loading: true, promises: [] }) - console.log('[FileManager] No entry found for fileId ' + fileId + '. Loading...') + logger.info('No entry found for fileId ' + fileId + '. Loading...') fs.mkdirSync(filePath, { recursive: true }) - console.log('[FileManager] Created file path: ' + filePath + ' for fileId ' + fileId) + logger.info('Created file path: ' + filePath + ' for fileId ' + fileId) const fileVersionUrl = controllerUrl + fileId + '/' + fileVersionId - console.log('[FileManager] Requesting file from: ' + fileVersionUrl) + logger.info('Requesting file from: ' + fileVersionUrl) this.#getKeywordFile(fileVersionUrl, filePath).then((result) => { - console.log('[FileManager] Transfered file ' + result + ' from ' + fileVersionUrl) + logger.info('Transferred file ' + result + ' from ' + fileVersionUrl) const cacheEntry = this.#getCacheEntry(fileId, fileVersionId) cacheEntry.name = result @@ -80,7 +81,7 @@ class FileManager { resolve(filePath + '/' + result) }, (err) => { - console.log('Error :' + err) + logger.error('Error downloading file: ' + err) reject(err) }) } @@ -123,7 +124,7 @@ class FileManager { resp.pipe(myFile).on('finish', () => resolve(filename)) } }).on('error', (err) => { - console.log('Error: ' + err.message) + logger.error('HTTP request error: ' + err.message) reject(err) }) diff --git a/step-node/step-node-agent/api/logger.js b/step-node/step-node-agent/api/logger.js new file mode 100644 index 0000000000..aadb26c2bd --- /dev/null +++ b/step-node/step-node-agent/api/logger.js @@ -0,0 +1,15 @@ +const { createLogger, format, transports } = require('winston'); + +const logger = createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: format.combine( + format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), + format.printf(({ timestamp, level, message, component }) => { + const prefix = component ? `[${component}] ` : '' + return `${timestamp} [${level.toUpperCase()}] ${prefix}${message}` + }) + ), + transports: [new transports.Console()], +}); + +module.exports = logger; diff --git a/step-node/step-node-agent/api/runner/runner.js b/step-node/step-node-agent/api/runner/runner.js index 4f802ac912..07a9990a73 100644 --- a/step-node/step-node-agent/api/runner/runner.js +++ b/step-node/step-node-agent/api/runner/runner.js @@ -1,4 +1,5 @@ const Session = require('../controllers/session'); +const logger = require('../logger').child({ component: 'Runner' }) module.exports = function (properties = {}) { const tokenId = 'local' @@ -19,7 +20,7 @@ module.exports = function (properties = {}) { const output = await controller.process_(tokenId, keywordName, input, properties) const payload = output.payload; if (payload.error) { - console.log("[Runner] The keyword execution returned an error", payload.error); + logger.warn('The keyword execution returned an error: ' + JSON.stringify(payload.error)) } return output.payload } diff --git a/step-node/step-node-agent/package.json b/step-node/step-node-agent/package.json index a08ecbc3f7..97e08807d4 100644 --- a/step-node/step-node-agent/package.json +++ b/step-node/step-node-agent/package.json @@ -43,6 +43,7 @@ "shelljs": "^0.8.4", "unzip-stream": "^0.3.1", "uuid": "^3.4.0", + "winston": "^3.19.0", "yaml": "^2.4.2" }, "bundledDependencies": [ diff --git a/step-node/step-node-agent/server.js b/step-node/step-node-agent/server.js index 392d7fa75e..81b0aa077e 100644 --- a/step-node/step-node-agent/server.js +++ b/step-node/step-node-agent/server.js @@ -1,16 +1,17 @@ const minimist = require('minimist') const path = require('path') const YAML = require('yaml') +const logger = require('./api/logger').child({ component: 'Agent' }) let args = minimist(process.argv.slice(2), { default: { f: path.join(__dirname, 'AgentConf.yaml') } }) -console.log('[Agent] Using arguments ' + JSON.stringify(args)) +logger.info('Using arguments ' + JSON.stringify(args)) const agentConfFile = args.f -console.log('[Agent] Reading agent configuration ' + agentConfFile) +logger.info('Reading agent configuration ' + agentConfFile) const fs = require('fs') const content = fs.readFileSync(agentConfFile, 'utf8') const agentConfFileExt = path.extname(agentConfFile) @@ -26,7 +27,7 @@ function parseAgentConf() { } } -console.log('[Agent] Creating agent context and tokens') +logger.info('Creating agent context and tokens') const uuid = require('uuid/v4') const jwtUtils = require('./utils/jwtUtils') const agentType = 'node' @@ -53,7 +54,7 @@ agentConf.tokenGroups.forEach(function (tokenGroup) { } }) -console.log('[Agent] Starting agent services') +logger.info('Starting agent services') const express = require('express') const app = express() const port = agentConf.agentPort || 3000 @@ -73,8 +74,8 @@ const server = app.listen(port) server.setTimeout(timeout) const startWithAgentUrl = async function(agentUrl) { - console.log('[Agent] Registering agent as ' + agentUrl + ' to grid ' + agentConf.gridHost) - console.log('[Agent] Creating registration timer') + logger.info('Registering agent as ' + agentUrl + ' to grid ' + agentConf.gridHost) + logger.info('Creating registration timer') const registrationPeriod = agentConf.registrationPeriod || 5000 setInterval(async () => { const body = { agentRef: { agentId: agent.id, agentUrl: agentUrl, agentType: agentType }, tokens: agentContext.tokens } @@ -94,15 +95,14 @@ const startWithAgentUrl = async function(agentUrl) { }) if (res.status !== 204) { const responseBody = await res.text().catch(() => null) - console.log("[Agent] Failed to register agent: grid responded with status " + res.status + (responseBody != null ? ". Response body: " + responseBody : "")) + logger.warn('Failed to register agent: grid responded with status ' + res.status + (responseBody != null ? '. Response body: ' + responseBody : '')) } } catch (err) { - console.log("[Agent] Error while registering agent to grid") - console.log(err) + logger.error('Error while registering agent to grid: ' + err) } }, registrationPeriod) - console.log('[Agent] Successfully started on: ' + port) + logger.info('Successfully started on port ' + port) } if(agentConf.agentUrl) { @@ -112,7 +112,7 @@ if(agentConf.agentUrl) { getFQDN().then(FQDN => { startWithAgentUrl('http://' + FQDN + ':' + port) }).catch(e => { - console.log('[Agent] Error while getting FQDN ' + e) + logger.error('Error while getting FQDN: ' + e) }) } @@ -121,5 +121,5 @@ const v8 = require('v8'); process.on('SIGUSR2', () => { const fileName = `/tmp/heap-${process.pid}-${Date.now()}.heapsnapshot`; v8.writeHeapSnapshot(fileName); - console.log(`Heap dump written to ${fileName}`); + logger.info('Heap dump written to ' + fileName) }); diff --git a/step-node/step-node-agent/test/runner.test.js b/step-node/step-node-agent/test/runner.test.js index 7618303887..19b7900ab3 100644 --- a/step-node/step-node-agent/test/runner.test.js +++ b/step-node/step-node-agent/test/runner.test.js @@ -40,7 +40,7 @@ describe('runner', () => { test('accepts a message and exception, attaches stack trace', async () => { const output = await runner.run('SetErrorWithMessageAndExceptionKW', { ErrorMsg: 'MyError3', rethrow_error: true }) expect(output.error.msg).toBe('MyError3') - expect(output.attachments).toHaveLength(1) + expect(output.attachments.find(a => a.name === 'exception.log')).toBeDefined() }) }) From 23815bf75dd37a8cebd8d3bbcfb588d33bac9cd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 13 Mar 2026 11:10:37 +0100 Subject: [PATCH 12/21] SED-4581 Implementing KW timeout handling, improving attachment of process outputs and error reporting --- .../AutomationPackageResourceUploader.java | 10 + .../StagingAutomationPackageContext.java | 5 + .../node/automation/YamlNodeFunction.java | 4 +- .../api/controllers/controller.js | 213 +++++++++++------- .../step-node-agent/api/controllers/output.js | 4 + .../step-node-agent/api/routes/routes.js | 2 +- .../step-node-agent/api/runner/runner.js | 15 +- step-node/step-node-agent/test/runner.test.js | 1 + 8 files changed, 165 insertions(+), 89 deletions(-) diff --git a/step-core/src/main/java/step/automation/packages/AutomationPackageResourceUploader.java b/step-core/src/main/java/step/automation/packages/AutomationPackageResourceUploader.java index 0638532e96..dcd04ef795 100644 --- a/step-core/src/main/java/step/automation/packages/AutomationPackageResourceUploader.java +++ b/step-core/src/main/java/step/automation/packages/AutomationPackageResourceUploader.java @@ -30,11 +30,21 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; public class AutomationPackageResourceUploader { private static final Logger logger = LoggerFactory.getLogger(AutomationPackageResourceUploader.class); + private final Map uniqueResourceReferences = new ConcurrentHashMap<>(); + + public String applyUniqueResourceReference(String resourceReference, + String resourceType, + StagingAutomationPackageContext context) { + return uniqueResourceReferences.computeIfAbsent(resourceReference, key -> applyResourceReference(resourceReference, resourceType, context)); + }; + public String applyResourceReference(String resourceReference, String resourceType, StagingAutomationPackageContext context) { diff --git a/step-core/src/main/java/step/automation/packages/StagingAutomationPackageContext.java b/step-core/src/main/java/step/automation/packages/StagingAutomationPackageContext.java index 9a453223ba..02c560f75d 100644 --- a/step-core/src/main/java/step/automation/packages/StagingAutomationPackageContext.java +++ b/step-core/src/main/java/step/automation/packages/StagingAutomationPackageContext.java @@ -8,6 +8,7 @@ public class StagingAutomationPackageContext extends AutomationPackageContext { private final AutomationPackageArchive automationPackageArchive; + private final AutomationPackageResourceUploader resourceUploader = new AutomationPackageResourceUploader(); public StagingAutomationPackageContext(AutomationPackage automationPackage, AutomationPackageOperationMode operationMode, ResourceManager resourceManager, AutomationPackageArchive automationPackageArchive, @@ -19,4 +20,8 @@ public StagingAutomationPackageContext(AutomationPackage automationPackage, Auto public AutomationPackageArchive getAutomationPackageArchive() { return automationPackageArchive; } + + public AutomationPackageResourceUploader getResourceUploader() { + return resourceUploader; + } } diff --git a/step-functions-plugins/step-functions-plugins-node/step-functions-plugins-node-def/src/main/java/step/plugins/node/automation/YamlNodeFunction.java b/step-functions-plugins/step-functions-plugins-node/step-functions-plugins-node-def/src/main/java/step/plugins/node/automation/YamlNodeFunction.java index 68306926dc..c22585a902 100644 --- a/step-functions-plugins/step-functions-plugins-node/step-functions-plugins-node-def/src/main/java/step/plugins/node/automation/YamlNodeFunction.java +++ b/step-functions-plugins/step-functions-plugins-node/step-functions-plugins-node-def/src/main/java/step/plugins/node/automation/YamlNodeFunction.java @@ -44,10 +44,10 @@ public void setJsfile(DynamicValue jsfile) { @Override protected void fillDeclaredFields(NodeFunction function, StagingAutomationPackageContext context) { super.fillDeclaredFields(function, context); - AutomationPackageResourceUploader resourceUploader = new AutomationPackageResourceUploader(); + AutomationPackageResourceUploader resourceUploader = context.getResourceUploader(); String filePath = jsfile.get(); - String fileRef = resourceUploader.applyResourceReference(filePath, ResourceManager.RESOURCE_TYPE_FUNCTIONS, context); + String fileRef = resourceUploader.applyUniqueResourceReference(filePath, ResourceManager.RESOURCE_TYPE_FUNCTIONS, context); if (fileRef != null) { function.setJsFile(new DynamicValue<>(fileRef)); } diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index 4e96a43380..252ab11fa0 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -16,9 +16,10 @@ process.on('uncaughtException', error => { }) class Controller { - constructor(agentContext, fileManager) { + constructor(agentContext, fileManager, mode) { this.agentContext = agentContext; this.filemanager = fileManager; + this.redirectIO = mode !== 'agent'; } isRunning(req, res) { @@ -59,9 +60,12 @@ class Controller { async process(req, res) { const tokenId = req.params.tokenId - const keywordName = req.body.payload.function - const argument = req.body.payload.payload - const properties = req.body.payload.properties + let input = req.body.payload; + const keywordName = input.function + let offset = 1000; + const callTimeoutMs = Math.max(offset, input.functionCallTimeout ? input.functionCallTimeout : 180000 - offset); + const argument = input.payload + const properties = input.properties let token = this.agentContext.tokens.find(value => value.id == tokenId); if(token) { @@ -75,16 +79,15 @@ class Controller { const additionalProperties = this.agentContext.tokenProperties[tokenId] Object.entries(additionalProperties).forEach(([key, value]) => { properties[key] = value }) - const payload = await this.process_(tokenId, keywordName, argument, properties) + const payload = await this.process_(tokenId, keywordName, argument, properties, callTimeoutMs) res.json(payload) } else { const outputBuilder = new OutputBuilder(); outputBuilder.fail("The token '" + tokenId + " doesn't exist on this agent. This usually means that the agent crashed and restarted."); } - } - async process_(tokenId, keywordName, argument, properties) { + async process_(tokenId, keywordName, argument, properties, callTimeoutMs) { const outputBuilder = new OutputBuilder(); try { const keywordPackageFile = await this.filemanager.loadOrGetKeywordFile( @@ -94,7 +97,7 @@ class Controller { keywordName ) - await this.executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder) + await this.executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder, callTimeoutMs) } catch (e) { logger.error('Unexpected error while executing keyword ' + keywordName + ': ' + e) outputBuilder.fail('Unexpected error while executing keyword', e) @@ -102,7 +105,10 @@ class Controller { return outputBuilder.build(); } - async executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder) { + async executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder, callTimeoutMs) { + let isDebugEnabled = properties['debug'] === 'true'; + let npmAttachment = null; + let forkedAgentProcessOutputAttachment = null; try { let npmProjectPath; if (keywordPackageFile.toUpperCase().endsWith('ZIP')) { @@ -118,93 +124,103 @@ class Controller { logger.info('Executing keyword in project ' + npmProjectPath + ' for token ' + tokenId) - let session = this.agentContext.tokenSessions[tokenId] - if (!session) { - session = new Session(); - this.agentContext.tokenSessions[tokenId] = session; - } + let session = this.getOrCreateSession(tokenId); - let forkedAgent = session.get('forkedAgent'); - const project = session.get('npmProjectPath'); - if (!project && forkedAgent) { - throw new Error("Multiple projects not supported within the same session"); + const npmProjectPathInSession = session.get('npmProjectPath'); + if (npmProjectPathInSession && npmProjectPathInSession !== npmProjectPath) { + throw new Error("Multiple npm projects are not supported within the same session"); + } else { + session.set('npmProjectPath', npmProjectPath); } - let npmAttachment = null; - let isDebugEnabled = properties['debug'] === 'true'; - + let forkedAgent = session.get('forkedAgent'); if (!forkedAgent) { logger.info('Starting agent fork in ' + npmProjectPath + ' for token ' + tokenId) forkedAgent = createForkedAgent(npmProjectPath); + session.set('forkedAgent', forkedAgent); logger.info('Running npm install in ' + npmProjectPath + ' for token ' + tokenId) const npmInstallResult = await this.executeNpmInstall(npmProjectPath); - const npmInstallOutput = Buffer.concat([npmInstallResult.stdout || Buffer.alloc(0), npmInstallResult.stderr || Buffer.alloc(0)]); const npmInstallFailed = npmInstallResult.status !== 0 || npmInstallResult.error != null; if (npmInstallFailed || isDebugEnabled) { - npmAttachment = { - name: 'npm-install.log', - isDirectory: false, - description: 'npm install output', - hexContent: npmInstallOutput.toString('base64') - }; + npmAttachment = npmInstallResult.processOutputAttachment; } - session.set('forkedAgent', forkedAgent); - session.set('npmProjectPath', npmProjectPath); - if (npmInstallFailed) { - outputBuilder.attach(npmAttachment); throw npmInstallResult.error || new Error('npm install exited with code ' + npmInstallResult.status); } } logger.info('Executing keyword \'' + keywordName + '\' in ' + npmProjectPath + ' for token ' + tokenId) - const { result, outputBuffer } = await forkedAgent.runKeywordTask(forkedAgent, npmProjectPath, keywordName, argument, properties); + const { result, processOutputAttachment } = await forkedAgent.runKeywordTask(forkedAgent, npmProjectPath, keywordName, argument, properties, callTimeoutMs, this.redirectIO); outputBuilder.merge(result.payload) - + if (result.error || isDebugEnabled) { + forkedAgentProcessOutputAttachment = processOutputAttachment; + } + } catch (e) { + if (e instanceof CategorizedError) { + logger.error('Error occurred while executing keyword: ' + e.message) + outputBuilder.fail(e.message) + forkedAgentProcessOutputAttachment = e.processOutputAttachment; + } else { + logger.error('Unexpected error occurred while executing keyword: ' + e) + outputBuilder.fail('Unexpected error: ' + e.message, e) + } + } finally { if (npmAttachment) { outputBuilder.attach(npmAttachment); } - - if (outputBuffer && (result.payload.error || isDebugEnabled)) { - const forkAttachment = { - name: 'keyword-process.log', - isDirectory: false, - description: 'Output of the forked keyword process', - hexContent: outputBuffer.toString('base64'), - }; - outputBuilder.attach(forkAttachment); + if (forkedAgentProcessOutputAttachment) { + outputBuilder.attach(forkedAgentProcessOutputAttachment); } - } catch (e) { - logger.error('Unexpected error occurred while executing keyword: ' + e) - outputBuilder.fail('An error occurred while attempting to execute the keyword ' + keywordName, e) } } + getOrCreateSession(tokenId) { + let session = this.agentContext.tokenSessions[tokenId] + if (!session) { + session = new Session(); + this.agentContext.tokenSessions[tokenId] = session; + } + return session; + } + async executeNpmInstall(npmProjectPath) { return await new Promise((resolve) => { const child = spawn(npmCommand, ['install'], {cwd: npmProjectPath, shell: false}); - const stdoutChunks = []; - const stderrChunks = []; + const stdChunks = []; child.stdout.on('data', (data) => { - stdoutChunks.push(data); - process.stdout.write(data); + stdChunks.push(data); + if (this.redirectIO) { + process.stdout.write(data) + } }); child.stderr.on('data', (data) => { - stderrChunks.push(data); - process.stderr.write(data); + stdChunks.push(data); + if (this.redirectIO) { + process.stderr.write(data) + } }); child.on('error', (error) => { - resolve({status: null, error, stdout: Buffer.concat(stdoutChunks), stderr: Buffer.concat(stderrChunks)}); + resolve({status: null, error, processOutputAttachment: getNpmInstallProcessOutputAttachment()}); }); child.on('close', (code) => { - resolve({status: code, error: null, stdout: Buffer.concat(stdoutChunks), stderr: Buffer.concat(stderrChunks)}); + resolve({status: code, error: null, processOutputAttachment: getNpmInstallProcessOutputAttachment()}); }); + + function getNpmInstallProcessOutputAttachment() { + const npmInstallOutput = Buffer.concat(stdChunks); + return { + name: 'npm-install.log', + isDirectory: false, + description: 'npm install output', + hexContent: npmInstallOutput.toString('base64') + }; + } }); } } @@ -219,68 +235,99 @@ function createForkedAgent(keywordProjectPath) { class ForkedAgent { constructor(process) { - this.process = process; + this.forkProcess = process; } - runKeywordTask(forkedAgent, keywordProjectPath, functionName, input, properties) { - return new Promise((resolve) => { + runKeywordTask(forkedAgent, keywordProjectPath, functionName, input, properties, timeoutMs, redirectIO) { + return new Promise((resolve, reject) => { try { - const stdoutChunks = []; - const stderrChunks = []; + const stdChunks = []; const stdoutListener = (data) => { - stdoutChunks.push(data); - process.stdout.write(data); + stdChunks.push(data); + if(redirectIO) { + process.stdout.write(data); + } }; const stderrListener = (data) => { - stderrChunks.push(data); - process.stderr.write(data); + stdChunks.push(data); + if(redirectIO) { + process.stderr.write(data); + } }; - if (this.process.stdout) { - this.process.stdout.on('data', stdoutListener); + if (this.forkProcess.stdout) { + this.forkProcess.stdout.on('data', stdoutListener); } - if (this.process.stderr) { - this.process.stderr.on('data', stderrListener); + if (this.forkProcess.stderr) { + this.forkProcess.stderr.on('data', stderrListener); } - this.process.removeAllListeners('message'); - this.process.on('message', (result) => { - if (this.process.stdout) { - this.process.stdout.removeListener('data', stdoutListener); + const timeoutHandle = timeoutMs != null ? setTimeout(() => { + cleanup(); + let processOutputAttachment = buildProcessOutputAttachment(stdChunks); + reject(new CategorizedError(`Keyword execution timed out after ${timeoutMs}ms`, processOutputAttachment)); + }, timeoutMs) : null; + + const cleanup = () => { + clearTimeout(timeoutHandle); + if (this.forkProcess.stdout) { + this.forkProcess.stdout.removeListener('data', stdoutListener); } - if (this.process.stderr) { - this.process.stderr.removeListener('data', stderrListener); + if (this.forkProcess.stderr) { + this.forkProcess.stderr.removeListener('data', stderrListener); } + }; - const outputBuffer = Buffer.concat([ - ...stdoutChunks, - ...stderrChunks, - ]); + this.forkProcess.removeAllListeners('message'); + this.forkProcess.on('message', (result) => { + logger.info(`Keyword '${functionName}' execution completed in forked agent.`) + cleanup(); - resolve({ result, outputBuffer }); + let processOutputAttachment = buildProcessOutputAttachment(stdChunks); + resolve({ result, processOutputAttachment}); }); - this.process.removeAllListeners('error'); - this.process.on('error', (err) => { + this.forkProcess.removeAllListeners('error'); + this.forkProcess.on('error', (err) => { logger.error('Error while calling forked agent: ' + err) }); - this.process.send({ type: "KEYWORD", projectPath: keywordProjectPath, functionName, input, properties }); + this.forkProcess.send({ type: "KEYWORD", projectPath: keywordProjectPath, functionName, input, properties }); } catch (e) { logger.error('Unexpected error while calling forked agent: ' + e) } }); + + function buildProcessOutputAttachment(stdChunks) { + const outputBuffer = Buffer.concat(stdChunks); + return { + name: 'keyword-process.log', + isDirectory: false, + description: 'Output of the forked keyword process', + hexContent: outputBuffer.toString('base64'), + }; + } } close() { try { - this.process.send({ type: "KILL" }); + this.forkProcess.send({ type: "KILL" }); } catch (e) { - this.process.kill(); + this.forkProcess.kill(); } } } + +class CategorizedError extends Error { + processOutputAttachment; + constructor(message, processOutputAttachment) { + super(message); // (1) + this.name = "CategorizedError"; + this.processOutputAttachment = processOutputAttachment; + } +} + module.exports = Controller; diff --git a/step-node/step-node-agent/api/controllers/output.js b/step-node/step-node-agent/api/controllers/output.js index 74e9cc6013..4498c96d2e 100644 --- a/step-node/step-node-agent/api/controllers/output.js +++ b/step-node/step-node-agent/api/controllers/output.js @@ -88,6 +88,10 @@ class OutputBuilder { return this; } + hasError() { + return this.builder.payload.error; + } + /** * Sets a business error (results in FAILED status rather than ERROR in Step). * @param {string} errorMessage diff --git a/step-node/step-node-agent/api/routes/routes.js b/step-node/step-node-agent/api/routes/routes.js index 3676d00e3c..8f4950e75b 100644 --- a/step-node/step-node-agent/api/routes/routes.js +++ b/step-node/step-node-agent/api/routes/routes.js @@ -2,7 +2,7 @@ module.exports = function (app, agentContext) { const Controller = require('../controllers/controller') const FileManager = require('../filemanager/filemanager') - const controller = new Controller(agentContext, new FileManager(agentContext)) + const controller = new Controller(agentContext, new FileManager(agentContext), 'agent') app.route('/running').get(controller.isRunning.bind(controller)) app.route('/token/:tokenId/reserve').get(controller.reserveToken.bind(controller)) diff --git a/step-node/step-node-agent/api/runner/runner.js b/step-node/step-node-agent/api/runner/runner.js index 07a9990a73..7565274fe7 100644 --- a/step-node/step-node-agent/api/runner/runner.js +++ b/step-node/step-node-agent/api/runner/runner.js @@ -1,7 +1,8 @@ const Session = require('../controllers/session'); const logger = require('../logger').child({ component: 'Runner' }) module.exports = function (properties = {}) { - const tokenId = 'local' + const tokenId = 'local'; + let throwExceptionOnError = true; const agentContext = {tokens: [], tokenSessions: {}, properties: properties} const tokenSession = new Session(); @@ -12,15 +13,23 @@ module.exports = function (properties = {}) { } const Controller = require('../controllers/controller') - const controller = new Controller(agentContext, fileManager) + const controller = new Controller(agentContext, fileManager, 'runner') const api = {} + api.setThrowExceptionOnError = function(isThrowExceptionOnError) { + throwExceptionOnError = isThrowExceptionOnError; + } + api.run = async function (keywordName, input) { const output = await controller.process_(tokenId, keywordName, input, properties) const payload = output.payload; if (payload.error) { - logger.warn('The keyword execution returned an error: ' + JSON.stringify(payload.error)) + if(throwExceptionOnError) { + throw new Error('The keyword execution returned an error: ' + JSON.stringify(payload.error)) + } else { + logger.warn('The keyword execution returned an error: ' + JSON.stringify(payload.error)) + } } return output.payload } diff --git a/step-node/step-node-agent/test/runner.test.js b/step-node/step-node-agent/test/runner.test.js index 19b7900ab3..ef8ceedc66 100644 --- a/step-node/step-node-agent/test/runner.test.js +++ b/step-node/step-node-agent/test/runner.test.js @@ -5,6 +5,7 @@ describe('runner', () => { beforeAll(() => { runner = require('../api/runner/runner')({ Property1: 'Prop1' }) + runner.setThrowExceptionOnError(false) }) afterAll(() => { From 8a6a87ee97ad5eef18140090eb669733b290bc8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 13 Mar 2026 12:59:54 +0100 Subject: [PATCH 13/21] SED-4581 Improving error handling when keyword dir is missing --- step-node/step-node-agent/agent-fork.js | 44 +++++++++++-------- .../api/controllers/controller.js | 10 +++-- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/step-node/step-node-agent/agent-fork.js b/step-node/step-node-agent/agent-fork.js index 155493fe58..707b2a585f 100644 --- a/step-node/step-node-agent/agent-fork.js +++ b/step-node/step-node-agent/agent-fork.js @@ -26,30 +26,32 @@ const session = new Session(); process.on('message', async ({ type, projectPath, functionName, input, properties }) => { if (type === 'KEYWORD') { console.log("[Agent fork] Calling keyword " + functionName) - const kwModules = await importAllKeywords(projectPath); - const outputBuilder = new OutputBuilder(); - try { - let keyword = searchKeyword(kwModules, functionName); - if (!keyword) { - console.log('[Agent fork] Unable to find Keyword ' + functionName + "'"); - outputBuilder.fail("Unable to find Keyword '" + functionName + "'"); + if (!keywordDirectoryExists(projectPath)) { + outputBuilder.fail("Could not find the 'keywords' directory in " + path.basename(projectPath) + ". Possible cause: If using TypeScript, the keywords may not have been compiled. Fix: Ensure your project is built before deploying to Step or during 'npm install'.") } else { - try { - await keyword(input, outputBuilder, session, properties); - } catch (e) { - let onError = searchKeyword(kwModules, 'onError'); - if (onError) { - if (await onError(e, input, outputBuilder, session, properties)) { - console.log('[Agent fork] Keyword execution failed and onError hook returned \'true\'') - outputBuilder.fail(e) + const kwModules = await importAllKeywords(projectPath); + let keyword = searchKeyword(kwModules, functionName); + if (!keyword) { + console.log('[Agent fork] Unable to find Keyword ' + functionName + "'"); + outputBuilder.fail("Unable to find Keyword '" + functionName + "'"); + } else { + try { + await keyword(input, outputBuilder, session, properties); + } catch (e) { + let onError = searchKeyword(kwModules, 'onError'); + if (onError) { + if (await onError(e, input, outputBuilder, session, properties)) { + console.log('[Agent fork] Keyword execution failed and onError hook returned \'true\'') + outputBuilder.fail(e) + } else { + console.log('[Agent fork] Keyword execution failed and onError hook returned \'false\'') + } } else { - console.log('[Agent fork] Keyword execution failed and onError hook returned \'false\'') + console.log('[Agent fork] Keyword execution failed. No onError hook defined') + outputBuilder.fail(e) } - } else { - console.log('[Agent fork] Keyword execution failed. No onError hook defined') - outputBuilder.fail(e) } } } @@ -63,6 +65,10 @@ process.on('message', async ({ type, projectPath, functionName, input, propertie process.exit(1) } + function keywordDirectoryExists(projectPath) { + return fs.existsSync(path.resolve(projectPath, "./keywords")) + } + async function importAllKeywords(projectPath) { const kwModules = []; const kwDir = path.resolve(projectPath, "./keywords"); diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index 252ab11fa0..4dae81e999 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -112,10 +112,12 @@ class Controller { try { let npmProjectPath; if (keywordPackageFile.toUpperCase().endsWith('ZIP')) { - if (this.filemanager.isFirstLevelKeywordFolder(keywordPackageFile)) { - npmProjectPath = path.resolve(keywordPackageFile); - } else { + if (fs.existsSync(path.resolve(keywordPackageFile, this.filemanager.getFolderName(keywordPackageFile)))) { + // If the ZIP contains a top-level wrapper folder npmProjectPath = path.resolve(keywordPackageFile, this.filemanager.getFolderName(keywordPackageFile)) + } else { + // Normal path with automation packages + npmProjectPath = path.resolve(keywordPackageFile); } } else { // Local execution with KeywordRunner @@ -134,7 +136,7 @@ class Controller { } let forkedAgent = session.get('forkedAgent'); - if (!forkedAgent) { + if (!forkedAgent) { logger.info('Starting agent fork in ' + npmProjectPath + ' for token ' + tokenId) forkedAgent = createForkedAgent(npmProjectPath); session.set('forkedAgent', forkedAgent); From 8b68411371ad12cc59c4dae6d395748c6ab86ec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 13 Mar 2026 22:20:05 +0100 Subject: [PATCH 14/21] SED-4581 Fixing error when agent properties not set and improving logging of errors --- .../api/controllers/controller.js | 22 +++++++++++-------- .../api/controllers/session.js | 4 ++-- .../api/filemanager/filemanager.js | 4 ++-- step-node/step-node-agent/api/logger.js | 8 ++++--- step-node/step-node-agent/server.js | 4 ++-- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index 4dae81e999..5280fc1b4f 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -8,11 +8,11 @@ const logger = require('../logger').child({ component: 'Controller' }); const npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm'; process.on('unhandledRejection', error => { - logger.error('Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported: ' + error) + logger.error('Critical: an unhandled error (unhandled promise rejection) occurred and might not have been reported:', error) }) process.on('uncaughtException', error => { - logger.error('Critical: an unhandled error (uncaught exception) occurred and might not have been reported: ' + error) + logger.error('Critical: an unhandled error (uncaught exception) occurred and might not have been reported:', error) }) class Controller { @@ -73,11 +73,15 @@ class Controller { // add the agent properties const agentProperties = this.agentContext.properties - Object.entries(agentProperties).forEach(([key, value]) => { properties[key] = value }) + if (agentProperties) { + Object.entries(agentProperties).forEach(([key, value]) => { properties[key] = value }) + } // add the properties of the tokenGroup const additionalProperties = this.agentContext.tokenProperties[tokenId] - Object.entries(additionalProperties).forEach(([key, value]) => { properties[key] = value }) + if (additionalProperties) { + Object.entries(additionalProperties).forEach(([key, value]) => { properties[key] = value }) + } const payload = await this.process_(tokenId, keywordName, argument, properties, callTimeoutMs) res.json(payload) @@ -99,7 +103,7 @@ class Controller { await this.executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder, callTimeoutMs) } catch (e) { - logger.error('Unexpected error while executing keyword ' + keywordName + ': ' + e) + logger.error('Unexpected error while executing keyword ' + keywordName + ':', e) outputBuilder.fail('Unexpected error while executing keyword', e) } return outputBuilder.build(); @@ -161,11 +165,11 @@ class Controller { } } catch (e) { if (e instanceof CategorizedError) { - logger.error('Error occurred while executing keyword: ' + e.message) + logger.error('Error occurred while executing keyword:' + e.message) outputBuilder.fail(e.message) forkedAgentProcessOutputAttachment = e.processOutputAttachment; } else { - logger.error('Unexpected error occurred while executing keyword: ' + e) + logger.error('Unexpected error occurred while executing keyword:', e) outputBuilder.fail('Unexpected error: ' + e.message, e) } } finally { @@ -293,12 +297,12 @@ class ForkedAgent { this.forkProcess.removeAllListeners('error'); this.forkProcess.on('error', (err) => { - logger.error('Error while calling forked agent: ' + err) + logger.error('Error while calling forked agent:', err) }); this.forkProcess.send({ type: "KEYWORD", projectPath: keywordProjectPath, functionName, input, properties }); } catch (e) { - logger.error('Unexpected error while calling forked agent: ' + e) + logger.error('Unexpected error while calling forked agent:', e) } }); diff --git a/step-node/step-node-agent/api/controllers/session.js b/step-node/step-node-agent/api/controllers/session.js index 0492a23040..7f4521ac09 100644 --- a/step-node/step-node-agent/api/controllers/session.js +++ b/step-node/step-node-agent/api/controllers/session.js @@ -42,9 +42,9 @@ class Session extends Map { resource.close(); } - logger.info(`Successfully closed resource: ${key}`); + logger.debug(`Successfully closed resource: ${key}`); } catch (err) { - logger.error(`Failed to close resource ${key}: ${err}`); + logger.error(`Failed to close resource ${key}:`, err); } } diff --git a/step-node/step-node-agent/api/filemanager/filemanager.js b/step-node/step-node-agent/api/filemanager/filemanager.js index f31a578c77..ba0ee0ccbb 100644 --- a/step-node/step-node-agent/api/filemanager/filemanager.js +++ b/step-node/step-node-agent/api/filemanager/filemanager.js @@ -81,7 +81,7 @@ class FileManager { resolve(filePath + '/' + result) }, (err) => { - logger.error('Error downloading file: ' + err) + logger.error('Error downloading file:', err) reject(err) }) } @@ -124,7 +124,7 @@ class FileManager { resp.pipe(myFile).on('finish', () => resolve(filename)) } }).on('error', (err) => { - logger.error('HTTP request error: ' + err.message) + logger.error('HTTP request error:', err) reject(err) }) diff --git a/step-node/step-node-agent/api/logger.js b/step-node/step-node-agent/api/logger.js index aadb26c2bd..8ccbbfa103 100644 --- a/step-node/step-node-agent/api/logger.js +++ b/step-node/step-node-agent/api/logger.js @@ -3,10 +3,12 @@ const { createLogger, format, transports } = require('winston'); const logger = createLogger({ level: process.env.LOG_LEVEL || 'info', format: format.combine( + format.errors({ stack: true }), format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), - format.printf(({ timestamp, level, message, component }) => { - const prefix = component ? `[${component}] ` : '' - return `${timestamp} [${level.toUpperCase()}] ${prefix}${message}` + format.printf((info) => { + const prefix = info.component ? `[${info.component}] ` : ''; + const logMessage = info.stack ? `${info.message} - ${info.stack}` : info.message; + return `${info.timestamp} [${info.level.toUpperCase()}] ${prefix}${logMessage}` }) ), transports: [new transports.Console()], diff --git a/step-node/step-node-agent/server.js b/step-node/step-node-agent/server.js index 81b0aa077e..3fa1f6d75b 100644 --- a/step-node/step-node-agent/server.js +++ b/step-node/step-node-agent/server.js @@ -98,7 +98,7 @@ const startWithAgentUrl = async function(agentUrl) { logger.warn('Failed to register agent: grid responded with status ' + res.status + (responseBody != null ? '. Response body: ' + responseBody : '')) } } catch (err) { - logger.error('Error while registering agent to grid: ' + err) + logger.error('Error while registering agent to grid:', err) } }, registrationPeriod) @@ -112,7 +112,7 @@ if(agentConf.agentUrl) { getFQDN().then(FQDN => { startWithAgentUrl('http://' + FQDN + ':' + port) }).catch(e => { - logger.error('Error while getting FQDN: ' + e) + logger.error('Error while getting FQDN:', e) }) } From 706dba3a5f7ac45b12fd7a1041d70096585dd414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Mon, 16 Mar 2026 17:40:13 +0100 Subject: [PATCH 15/21] SED-4581 Supporting concurrency via npm project workspaces --- .../{ => api/controllers}/agent-fork.js | 0 .../api/controllers/controller.js | 134 ++++++++++++++---- step-node/step-node-agent/server.js | 2 +- 3 files changed, 107 insertions(+), 29 deletions(-) rename step-node/step-node-agent/{ => api/controllers}/agent-fork.js (100%) diff --git a/step-node/step-node-agent/agent-fork.js b/step-node/step-node-agent/api/controllers/agent-fork.js similarity index 100% rename from step-node/step-node-agent/agent-fork.js rename to step-node/step-node-agent/api/controllers/agent-fork.js diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/controller.js index 5280fc1b4f..cb372c0e36 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/controller.js @@ -19,7 +19,15 @@ class Controller { constructor(agentContext, fileManager, mode) { this.agentContext = agentContext; this.filemanager = fileManager; + this.mode = mode; this.redirectIO = mode !== 'agent'; + this.npmProjectWorkspaces = new Map(); // cacheKey -> { path, inUse, lastFreeAt } + this.npmProjectWorkspaceCleanupIdleTimeMs = agentContext.npmProjectWorkspaceCleanupIdleTimeMs ?? 3600000; + + if(mode === 'agent' && this.npmProjectWorkspaceCleanupIdleTimeMs > 0) { + logger.info(`Scheduling npm project workspace cleanup every ${this.npmProjectWorkspaceCleanupIdleTimeMs}ms`); + setInterval(() => this.cleanupUnusedWorkspaces(this.npmProjectWorkspaceCleanupIdleTimeMs), this.npmProjectWorkspaceCleanupIdleTimeMs); + } } isRunning(req, res) { @@ -67,6 +75,7 @@ class Controller { const argument = input.payload const properties = input.properties + let output; let token = this.agentContext.tokens.find(value => value.id == tokenId); if(token) { logger.info('Using token: ' + tokenId + ' to execute ' + keywordName) @@ -83,25 +92,54 @@ class Controller { Object.entries(additionalProperties).forEach(([key, value]) => { properties[key] = value }) } - const payload = await this.process_(tokenId, keywordName, argument, properties, callTimeoutMs) - res.json(payload) + output = await this.process_(tokenId, keywordName, argument, properties, callTimeoutMs) } else { const outputBuilder = new OutputBuilder(); outputBuilder.fail("The token '" + tokenId + " doesn't exist on this agent. This usually means that the agent crashed and restarted."); + output = outputBuilder.build(); } + res.json(output) } async process_(tokenId, keywordName, argument, properties, callTimeoutMs) { const outputBuilder = new OutputBuilder(); try { - const keywordPackageFile = await this.filemanager.loadOrGetKeywordFile( + let fileId = properties['$node.js.file.id']; + let fileVersionId = properties['$node.js.file.version']; + const file = await this.filemanager.loadOrGetKeywordFile( this.agentContext.controllerUrl + '/grid/file/', - properties['$node.js.file.id'], - properties['$node.js.file.version'], + fileId, + fileVersionId, keywordName ) - await this.executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder, callTimeoutMs) + let npmProjectPath; + if (file.toUpperCase().endsWith('ZIP') && fs.existsSync(path.resolve(file, this.filemanager.getFolderName(file)))) { + // If the ZIP contains a top-level wrapper folder + npmProjectPath = path.resolve(file, this.filemanager.getFolderName(file)) + } else { + npmProjectPath = path.resolve(file) + } + + let workspacePath; + if (this.mode === 'agent') { + // Create a copy of the npm project for each token + workspacePath = this.getOrCreateNpmProjectWorkspace(tokenId, {fileId, fileVersionId, file: npmProjectPath}); + this.markWorkspaceInUse(workspacePath); + } else { + // When running keywords locally we're working directly in npm project passed by the runner + workspacePath = npmProjectPath; + } + try { + await this.executeKeyword(keywordName, workspacePath, tokenId, argument, properties, outputBuilder, callTimeoutMs) + } finally { + if (this.mode === 'agent') { + this.markWorkspaceFree(workspacePath); + if (this.npmProjectWorkspaceCleanupIdleTimeMs === 0) { + this.cleanupUnusedWorkspaces(0); + } + } + } } catch (e) { logger.error('Unexpected error while executing keyword ' + keywordName + ':', e) outputBuilder.fail('Unexpected error while executing keyword', e) @@ -109,25 +147,11 @@ class Controller { return outputBuilder.build(); } - async executeKeyword(keywordName, keywordPackageFile, tokenId, argument, properties, outputBuilder, callTimeoutMs) { + async executeKeyword(keywordName, npmProjectPath, tokenId, argument, properties, outputBuilder, callTimeoutMs) { let isDebugEnabled = properties['debug'] === 'true'; let npmAttachment = null; let forkedAgentProcessOutputAttachment = null; try { - let npmProjectPath; - if (keywordPackageFile.toUpperCase().endsWith('ZIP')) { - if (fs.existsSync(path.resolve(keywordPackageFile, this.filemanager.getFolderName(keywordPackageFile)))) { - // If the ZIP contains a top-level wrapper folder - npmProjectPath = path.resolve(keywordPackageFile, this.filemanager.getFolderName(keywordPackageFile)) - } else { - // Normal path with automation packages - npmProjectPath = path.resolve(keywordPackageFile); - } - } else { - // Local execution with KeywordRunner - npmProjectPath = path.resolve(keywordPackageFile) - } - logger.info('Executing keyword in project ' + npmProjectPath + ' for token ' + tokenId) let session = this.getOrCreateSession(tokenId); @@ -182,6 +206,57 @@ class Controller { } } + getOrCreateNpmProjectWorkspace(tokenId, keywordPackage) { + const cacheKey = `${keywordPackage.fileId}_${keywordPackage.fileVersionId}_${tokenId}`; + const workspace = this.npmProjectWorkspaces.get(cacheKey); + if (workspace) { + return workspace.path; + } + const baseDir = path.resolve((this.agentContext.workingDir ?? '.'), 'npm-project-workspaces'); + const workspacePath = path.join(baseDir, cacheKey); + if (!fs.existsSync(workspacePath)) { + logger.info(`Creating npm project workspace at ${workspacePath}`); + fs.cpSync(keywordPackage.file, workspacePath, { recursive: true }); + } + this.npmProjectWorkspaces.set(cacheKey, { path: workspacePath, inUse: false, lastFreeAt: Date.now() }); + return workspacePath; + } + + markWorkspaceInUse(workspacePath) { + for (const workspace of this.npmProjectWorkspaces.values()) { + if (workspace.path === workspacePath) { + workspace.inUse = true; + return; + } + } + } + + markWorkspaceFree(workspacePath) { + for (const workspace of this.npmProjectWorkspaces.values()) { + if (workspace.path === workspacePath) { + workspace.inUse = false; + workspace.lastFreeAt = Date.now(); + return; + } + } + } + + cleanupUnusedWorkspaces(idleTimeMs) { + const now = Date.now(); + for (const [cacheKey, workspace] of this.npmProjectWorkspaces) { + if (!workspace.inUse && (now - workspace.lastFreeAt) >= idleTimeMs) { + logger.info(`Deleting npm project workspace unused for ${idleTimeMs}ms: ${workspace.path}`); + try { + fs.rmSync(workspace.path, { recursive: true, force: true }); + } catch (e) { + logger.error(`Failed to delete npm project workspace ${workspace.path}:`, e); + continue; + } + this.npmProjectWorkspaces.delete(cacheKey); + } + } + } + getOrCreateSession(tokenId) { let session = this.agentContext.tokenSessions[tokenId] if (!session) { @@ -232,16 +307,19 @@ class Controller { } function createForkedAgent(keywordProjectPath) { - fs.copyFileSync(path.resolve(__dirname, '../../agent-fork.js'), path.join(keywordProjectPath, './agent-fork.js')); - fs.copyFileSync(path.join(__dirname, 'output.js'), path.join(keywordProjectPath, './output.js')); - fs.copyFileSync(path.join(__dirname, 'session.js'), path.join(keywordProjectPath, './session.js')); - return new ForkedAgent(fork('./agent-fork.js', [], {cwd: keywordProjectPath, silent: true})); + return new ForkedAgent(keywordProjectPath); } class ForkedAgent { - constructor(process) { - this.forkProcess = process; + constructor(keywordProjectPath) { + const agentForkerLibPath = path.join(keywordProjectPath, 'agent-fork-libs'); + fs.mkdirSync(agentForkerLibPath, { recursive: true }); + fs.copyFileSync(path.resolve(__dirname, 'agent-fork.js'), path.join(agentForkerLibPath, 'agent-fork.js')); + fs.copyFileSync(path.join(__dirname, 'output.js'), path.join(agentForkerLibPath, 'output.js')); + fs.copyFileSync(path.join(__dirname, 'session.js'), path.join(agentForkerLibPath, 'session.js')); + this.agentForkerLibPath = agentForkerLibPath; + this.forkProcess = fork(path.join(agentForkerLibPath, 'agent-fork.js'), [], {cwd: keywordProjectPath, silent: true}); } runKeywordTask(forkedAgent, keywordProjectPath, functionName, input, properties, timeoutMs, redirectIO) { @@ -323,7 +401,7 @@ class ForkedAgent { } catch (e) { this.forkProcess.kill(); } - + fs.rmSync(this.agentForkerLibPath, {recursive: true, force: true}); } } diff --git a/step-node/step-node-agent/server.js b/step-node/step-node-agent/server.js index 3fa1f6d75b..9acf768128 100644 --- a/step-node/step-node-agent/server.js +++ b/step-node/step-node-agent/server.js @@ -32,7 +32,7 @@ const uuid = require('uuid/v4') const jwtUtils = require('./utils/jwtUtils') const agentType = 'node' const agent = {id: uuid()} -const agentContext = { tokens: [], tokenSessions: [], tokenProperties: [], properties: agentConf.properties, controllerUrl: agentConf.gridHost, gridSecurity: agentConf.gridSecurity } +const agentContext = { tokens: [], tokenSessions: [], tokenProperties: [], properties: agentConf.properties, controllerUrl: agentConf.gridHost, gridSecurity: agentConf.gridSecurity, workingDir: agentConf.workingDir, npmProjectWorkspaceCleanupIdleTimeMs: agentConf.npmProjectWorkspaceCleanupIdleTimeMs } agentConf.tokenGroups.forEach(function (tokenGroup) { const tokenConf = tokenGroup.tokenConf let attributes = tokenConf.attributes From fe0038ecd80715028eb1d11ea29f542ef7345ab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Thu, 19 Mar 2026 22:24:37 +0100 Subject: [PATCH 16/21] SED-4581 PR feedbacks --- step-node/step-node-agent/.gitignore | 1 + .../api/controllers/agent-fork.js | 36 +++++++---- .../controllers/{controller.js => agent.js} | 51 ++++++++++------ .../step-node-agent/api/controllers/output.js | 17 +++++- .../api/filemanager/filemanager.js | 9 --- .../step-node-agent/api/routes/routes.js | 2 +- .../step-node-agent/api/runner/runner.js | 2 +- step-node/step-node-agent/package.json | 3 + .../{ => test}/keywords/keywords.js | 38 ++++++++++++ step-node/step-node-agent/test/runner.test.js | 59 +++++++++++++++++++ 10 files changed, 176 insertions(+), 42 deletions(-) rename step-node/step-node-agent/api/controllers/{controller.js => agent.js} (89%) rename step-node/step-node-agent/{ => test}/keywords/keywords.js (77%) diff --git a/step-node/step-node-agent/.gitignore b/step-node/step-node-agent/.gitignore index 57b766a6d0..45c1d7df58 100644 --- a/step-node/step-node-agent/.gitignore +++ b/step-node/step-node-agent/.gitignore @@ -3,3 +3,4 @@ node_modules/ !/bin filemanager/work/ /coverage/ +/npm-project-workspaces/ diff --git a/step-node/step-node-agent/api/controllers/agent-fork.js b/step-node/step-node-agent/api/controllers/agent-fork.js index 707b2a585f..ce33453717 100644 --- a/step-node/step-node-agent/api/controllers/agent-fork.js +++ b/step-node/step-node-agent/api/controllers/agent-fork.js @@ -23,24 +23,31 @@ const fs = require("fs"); const path = require('path') const session = new Session(); -process.on('message', async ({ type, projectPath, functionName, input, properties }) => { +process.on('message', async ({ type, projectPath, functionName, input, properties, keywordDirectory }) => { if (type === 'KEYWORD') { console.log("[Agent fork] Calling keyword " + functionName) const outputBuilder = new OutputBuilder(); try { - if (!keywordDirectoryExists(projectPath)) { - outputBuilder.fail("Could not find the 'keywords' directory in " + path.basename(projectPath) + ". Possible cause: If using TypeScript, the keywords may not have been compiled. Fix: Ensure your project is built before deploying to Step or during 'npm install'.") + if (!keywordDirectoryExists(projectPath, keywordDirectory)) { + outputBuilder.fail("The keyword directory '" + keywordDirectory + "' doesn't exist in " + path.basename(projectPath) + ". Possible cause: If using TypeScript, the keywords may not have been compiled. Fix: Ensure your project is built before deploying to Step or during 'npm install'.") } else { - const kwModules = await importAllKeywords(projectPath); - let keyword = searchKeyword(kwModules, functionName); - if (!keyword) { + const kwModules = await importAllKeywords(projectPath, keywordDirectory); + let keywordSearchResult = searchKeyword(kwModules, functionName); + if (!keywordSearchResult) { console.log('[Agent fork] Unable to find Keyword ' + functionName + "'"); outputBuilder.fail("Unable to find Keyword '" + functionName + "'"); } else { + const module = keywordSearchResult.keywordModule; + const keyword = keywordSearchResult.keywordFunction; + try { + const beforeKeyword = module['beforeKeyword']; + if(beforeKeyword) { + await beforeKeyword(functionName); + } await keyword(input, outputBuilder, session, properties); } catch (e) { - let onError = searchKeyword(kwModules, 'onError'); + const onError = module['onError']; if (onError) { if (await onError(e, input, outputBuilder, session, properties)) { console.log('[Agent fork] Keyword execution failed and onError hook returned \'true\'') @@ -52,6 +59,11 @@ process.on('message', async ({ type, projectPath, functionName, input, propertie console.log('[Agent fork] Keyword execution failed. No onError hook defined') outputBuilder.fail(e) } + } finally { + let afterKeyword = module['afterKeyword']; + if (afterKeyword) { + await afterKeyword(functionName); + } } } } @@ -65,13 +77,13 @@ process.on('message', async ({ type, projectPath, functionName, input, propertie process.exit(1) } - function keywordDirectoryExists(projectPath) { - return fs.existsSync(path.resolve(projectPath, "./keywords")) + function keywordDirectoryExists(projectPath, keywordDirectory) { + return fs.existsSync(path.resolve(projectPath, keywordDirectory)) } - async function importAllKeywords(projectPath) { + async function importAllKeywords(projectPath, keywordDirectory) { const kwModules = []; - const kwDir = path.resolve(projectPath, "./keywords"); + const kwDir = path.resolve(projectPath, keywordDirectory); console.log("[Agent fork] Searching keywords in: " + kwDir) const kwFiles = fs.readdirSync(kwDir); for (const kwFile of kwFiles) { @@ -87,7 +99,7 @@ process.on('message', async ({ type, projectPath, functionName, input, propertie function searchKeyword(kwModules, keywordName) { const kwModule = kwModules.find(m => m[keywordName]); - return kwModule ? kwModule[keywordName] : undefined; + return kwModule ? {keywordFunction: kwModule[keywordName], keywordModule: kwModule} : undefined; } }); diff --git a/step-node/step-node-agent/api/controllers/controller.js b/step-node/step-node-agent/api/controllers/agent.js similarity index 89% rename from step-node/step-node-agent/api/controllers/controller.js rename to step-node/step-node-agent/api/controllers/agent.js index cb372c0e36..6dc460e52a 100644 --- a/step-node/step-node-agent/api/controllers/controller.js +++ b/step-node/step-node-agent/api/controllers/agent.js @@ -1,9 +1,9 @@ const fs = require("fs"); const path = require("path"); -const {fork, spawn, spawnSync} = require("child_process"); +const {fork, spawn} = require("child_process"); const Session = require('./session'); const { OutputBuilder } = require('./output'); -const logger = require('../logger').child({ component: 'Controller' }); +const logger = require('../logger').child({ component: 'Agent' }); const npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm'; @@ -15,7 +15,7 @@ process.on('uncaughtException', error => { logger.error('Critical: an unhandled error (uncaught exception) occurred and might not have been reported:', error) }) -class Controller { +class Agent { constructor(agentContext, fileManager, mode) { this.agentContext = agentContext; this.filemanager = fileManager; @@ -76,7 +76,7 @@ class Controller { const properties = input.properties let output; - let token = this.agentContext.tokens.find(value => value.id == tokenId); + let token = this.agentContext.tokens.find(value => value.id === tokenId); if(token) { logger.info('Using token: ' + tokenId + ' to execute ' + keywordName) @@ -114,9 +114,12 @@ class Controller { ) let npmProjectPath; - if (file.toUpperCase().endsWith('ZIP') && fs.existsSync(path.resolve(file, this.filemanager.getFolderName(file)))) { + let fileBasename = path.basename(file); + let wrapperDirectory = path.resolve(file, fileBasename.substring(0, fileBasename.length - 4)); + + if (file.toUpperCase().endsWith('.ZIP') && fs.existsSync(wrapperDirectory)) { // If the ZIP contains a top-level wrapper folder - npmProjectPath = path.resolve(file, this.filemanager.getFolderName(file)) + npmProjectPath = path.resolve(file, wrapperDirectory); } else { npmProjectPath = path.resolve(file) } @@ -124,7 +127,7 @@ class Controller { let workspacePath; if (this.mode === 'agent') { // Create a copy of the npm project for each token - workspacePath = this.getOrCreateNpmProjectWorkspace(tokenId, {fileId, fileVersionId, file: npmProjectPath}); + workspacePath = await this.getOrCreateNpmProjectWorkspace(tokenId, {fileId, fileVersionId, file: npmProjectPath}); this.markWorkspaceInUse(workspacePath); } else { // When running keywords locally we're working directly in npm project passed by the runner @@ -136,7 +139,7 @@ class Controller { if (this.mode === 'agent') { this.markWorkspaceFree(workspacePath); if (this.npmProjectWorkspaceCleanupIdleTimeMs === 0) { - this.cleanupUnusedWorkspaces(0); + await this.cleanupUnusedWorkspaces(0); } } } @@ -164,7 +167,7 @@ class Controller { } let forkedAgent = session.get('forkedAgent'); - if (!forkedAgent) { + if (!forkedAgent) { logger.info('Starting agent fork in ' + npmProjectPath + ' for token ' + tokenId) forkedAgent = createForkedAgent(npmProjectPath); session.set('forkedAgent', forkedAgent); @@ -179,10 +182,13 @@ class Controller { if (npmInstallFailed) { throw npmInstallResult.error || new Error('npm install exited with code ' + npmInstallResult.status); } + + session.set('keywordDirectory', await readStepKeywordDirectory(npmProjectPath)); } + const keywordDirectory = session.get('keywordDirectory'); logger.info('Executing keyword \'' + keywordName + '\' in ' + npmProjectPath + ' for token ' + tokenId) - const { result, processOutputAttachment } = await forkedAgent.runKeywordTask(forkedAgent, npmProjectPath, keywordName, argument, properties, callTimeoutMs, this.redirectIO); + const { result, processOutputAttachment } = await forkedAgent.runKeywordTask(npmProjectPath, keywordName, argument, properties, callTimeoutMs, this.redirectIO, keywordDirectory); outputBuilder.merge(result.payload) if (result.error || isDebugEnabled) { forkedAgentProcessOutputAttachment = processOutputAttachment; @@ -206,7 +212,7 @@ class Controller { } } - getOrCreateNpmProjectWorkspace(tokenId, keywordPackage) { + async getOrCreateNpmProjectWorkspace(tokenId, keywordPackage) { const cacheKey = `${keywordPackage.fileId}_${keywordPackage.fileVersionId}_${tokenId}`; const workspace = this.npmProjectWorkspaces.get(cacheKey); if (workspace) { @@ -216,7 +222,7 @@ class Controller { const workspacePath = path.join(baseDir, cacheKey); if (!fs.existsSync(workspacePath)) { logger.info(`Creating npm project workspace at ${workspacePath}`); - fs.cpSync(keywordPackage.file, workspacePath, { recursive: true }); + await fs.promises.cp(keywordPackage.file, workspacePath, { recursive: true }); } this.npmProjectWorkspaces.set(cacheKey, { path: workspacePath, inUse: false, lastFreeAt: Date.now() }); return workspacePath; @@ -241,13 +247,13 @@ class Controller { } } - cleanupUnusedWorkspaces(idleTimeMs) { + async cleanupUnusedWorkspaces(idleTimeMs) { const now = Date.now(); for (const [cacheKey, workspace] of this.npmProjectWorkspaces) { if (!workspace.inUse && (now - workspace.lastFreeAt) >= idleTimeMs) { logger.info(`Deleting npm project workspace unused for ${idleTimeMs}ms: ${workspace.path}`); try { - fs.rmSync(workspace.path, { recursive: true, force: true }); + await fs.promises.rm(workspace.path, { recursive: true, force: true }); } catch (e) { logger.error(`Failed to delete npm project workspace ${workspace.path}:`, e); continue; @@ -268,7 +274,7 @@ class Controller { async executeNpmInstall(npmProjectPath) { return await new Promise((resolve) => { - const child = spawn(npmCommand, ['install'], {cwd: npmProjectPath, shell: false}); + const child = spawn(npmCommand, ['install'], {cwd: npmProjectPath, shell: true}); const stdChunks = []; child.stdout.on('data', (data) => { @@ -306,6 +312,15 @@ class Controller { } } +async function readStepKeywordDirectory(npmProjectPath) { + try { + const content = await fs.promises.readFile(path.join(npmProjectPath, 'package.json'), 'utf8'); + return JSON.parse(content)?.step?.keywords ?? './keywords'; + } catch { + return './keywords'; + } +} + function createForkedAgent(keywordProjectPath) { return new ForkedAgent(keywordProjectPath); } @@ -322,7 +337,7 @@ class ForkedAgent { this.forkProcess = fork(path.join(agentForkerLibPath, 'agent-fork.js'), [], {cwd: keywordProjectPath, silent: true}); } - runKeywordTask(forkedAgent, keywordProjectPath, functionName, input, properties, timeoutMs, redirectIO) { + runKeywordTask(keywordProjectPath, functionName, input, properties, timeoutMs, redirectIO, keywordDirectory) { return new Promise((resolve, reject) => { try { const stdChunks = []; @@ -378,7 +393,7 @@ class ForkedAgent { logger.error('Error while calling forked agent:', err) }); - this.forkProcess.send({ type: "KEYWORD", projectPath: keywordProjectPath, functionName, input, properties }); + this.forkProcess.send({ type: "KEYWORD", projectPath: keywordProjectPath, functionName, input, properties, keywordDirectory }); } catch (e) { logger.error('Unexpected error while calling forked agent:', e) } @@ -414,4 +429,4 @@ class CategorizedError extends Error { } } -module.exports = Controller; +module.exports = Agent; diff --git a/step-node/step-node-agent/api/controllers/output.js b/step-node/step-node-agent/api/controllers/output.js index 4498c96d2e..d115e8999c 100644 --- a/step-node/step-node-agent/api/controllers/output.js +++ b/step-node/step-node-agent/api/controllers/output.js @@ -12,6 +12,20 @@ function assertValidStatus(status) { } } +/** + * Represents a file attachment added to a keyword output. + * Mirrors the Java class `step.grid.io.Attachment`. + * + * @typedef {object} Attachment + * @property {string} name - Display name of the attachment (e.g. "screenshot.png"). + * @property {string} [mimeType] - MIME type of the content (e.g. "image/png", "text/plain"). + * @property {string} [description] - Human-readable description shown in the Step UI. + * @property {string} [hexContent] - File content encoded as a **base64** string despite the field + * name. Use `Buffer.from(data).toString('base64')` to produce it. + * @property {boolean} [isDirectory] - Set to `true` when the attachment represents a directory + * rather than a single file. + */ + class OutputBuilder { constructor() { this.builder = { payload: { attachments: [], payload: {} }, attachments: [] } @@ -104,7 +118,8 @@ class OutputBuilder { /** * Adds an attachment to the output. - * @param {object} attachment + * @param {Attachment} attachment + * @returns {OutputBuilder} this, for chaining */ attach(attachment) { this.builder.payload.attachments.push(attachment) diff --git a/step-node/step-node-agent/api/filemanager/filemanager.js b/step-node/step-node-agent/api/filemanager/filemanager.js index ba0ee0ccbb..e5e8f74a5c 100644 --- a/step-node/step-node-agent/api/filemanager/filemanager.js +++ b/step-node/step-node-agent/api/filemanager/filemanager.js @@ -24,15 +24,6 @@ class FileManager { return fs.existsSync(filePath + '/keywords') } - getFolderName(keywordPackageFile) { - try { - const splitNodes = keywordPackageFile.split('/') - const lastNode = splitNodes[splitNodes.length - 1] - return lastNode.split('.')[0] - } catch (e) { - throw new Error('A problem occured while attempting to retrieve subfolder name from zipped project:' + keywordPackageFile) - } - } async loadOrGetKeywordFile(controllerUrl, fileId, fileVersionId) { return new Promise((resolve, reject) => { diff --git a/step-node/step-node-agent/api/routes/routes.js b/step-node/step-node-agent/api/routes/routes.js index 8f4950e75b..8ede4b79c2 100644 --- a/step-node/step-node-agent/api/routes/routes.js +++ b/step-node/step-node-agent/api/routes/routes.js @@ -1,6 +1,6 @@ 'use strict' module.exports = function (app, agentContext) { - const Controller = require('../controllers/controller') + const Controller = require('../controllers/agent') const FileManager = require('../filemanager/filemanager') const controller = new Controller(agentContext, new FileManager(agentContext), 'agent') diff --git a/step-node/step-node-agent/api/runner/runner.js b/step-node/step-node-agent/api/runner/runner.js index 7565274fe7..e846661ca1 100644 --- a/step-node/step-node-agent/api/runner/runner.js +++ b/step-node/step-node-agent/api/runner/runner.js @@ -12,7 +12,7 @@ module.exports = function (properties = {}) { loadOrGetKeywordFile: () => Promise.resolve('.') } - const Controller = require('../controllers/controller') + const Controller = require('../controllers/agent') const controller = new Controller(agentContext, fileManager, 'runner') const api = {} diff --git a/step-node/step-node-agent/package.json b/step-node/step-node-agent/package.json index 97e08807d4..93e2d97149 100644 --- a/step-node/step-node-agent/package.json +++ b/step-node/step-node-agent/package.json @@ -17,6 +17,9 @@ ], "testTimeout": 30000 }, + "step": { + "keywords": "test/keywords" + }, "author": "Jerome Comte", "license": "AGPL-3.0", "repository": { diff --git a/step-node/step-node-agent/keywords/keywords.js b/step-node/step-node-agent/test/keywords/keywords.js similarity index 77% rename from step-node/step-node-agent/keywords/keywords.js rename to step-node/step-node-agent/test/keywords/keywords.js index d14b153988..8b39b78dfc 100644 --- a/step-node/step-node-agent/keywords/keywords.js +++ b/step-node/step-node-agent/test/keywords/keywords.js @@ -100,3 +100,41 @@ exports.MultipleMeasuresKW = async (input, output, session, properties) => { output.addMeasure('third', 50) output.send() } + +// --- session --- + +exports.SessionSetKW = async (input, output, session) => { + session.set('sharedKey', input['value']) + output.send() +} + +exports.SessionGetKW = async (input, output, session) => { + output.add('value', session.get('sharedKey')).send() +} + +exports.SessionSetDotKW = async (input, output, session) => { + session.dotKey = input['value'] + output.send() +} + +exports.SessionGetDotKW = async (input, output, session) => { + output.add('value', session.dotKey).send() +} + +// --- beforeKeyword / afterKeyword hook tracking --- + +let _hookCalls = [] + +exports.beforeKeyword = async (functionName) => { + _hookCalls.push(`before:${functionName}`) +} + +exports.afterKeyword = async (functionName) => { + _hookCalls.push(`after:${functionName}`) +} + +exports.GetHookCallsKW = async (input, output) => { + const calls = [..._hookCalls] + _hookCalls = [] + output.add('calls', calls).send() +} diff --git a/step-node/step-node-agent/test/runner.test.js b/step-node/step-node-agent/test/runner.test.js index ef8ceedc66..937ddb7f59 100644 --- a/step-node/step-node-agent/test/runner.test.js +++ b/step-node/step-node-agent/test/runner.test.js @@ -177,6 +177,65 @@ describe('runner', () => { expect(output.measures[2].duration).toBe(50) }) }) + + // --------------------------------------------------------------------------- + // session + // --------------------------------------------------------------------------- + + describe('session', () => { + test('values stored via session.set() are accessible in a later keyword via session.get()', async () => { + await runner.run('SessionSetKW', { value: 'hello' }) + const output = await runner.run('SessionGetKW') + expect(output.payload.value).toBe('hello') + }) + + test('values stored via dot notation are accessible in a later keyword', async () => { + await runner.run('SessionSetDotKW', { value: 'world' }) + const output = await runner.run('SessionGetDotKW') + expect(output.payload.value).toBe('world') + }) + + test('session value is updated when set again', async () => { + await runner.run('SessionSetKW', { value: 'first' }) + await runner.run('SessionSetKW', { value: 'second' }) + const output = await runner.run('SessionGetKW') + expect(output.payload.value).toBe('second') + }) + }) + + // --------------------------------------------------------------------------- + // beforeKeyword and afterKeyword hooks + // --------------------------------------------------------------------------- + + describe('beforeKeyword and afterKeyword hooks', () => { + beforeEach(async () => { + await runner.run('GetHookCallsKW') + }) + + test('beforeKeyword is called with the keyword name before execution', async () => { + await runner.run('Echo', {}) + const { payload: { calls } } = await runner.run('GetHookCallsKW') + expect(calls).toContain('before:Echo') + }) + + test('afterKeyword is called with the keyword name after successful execution', async () => { + await runner.run('Echo', {}) + const { payload: { calls } } = await runner.run('GetHookCallsKW') + expect(calls).toContain('after:Echo') + }) + + test('beforeKeyword is called before afterKeyword', async () => { + await runner.run('Echo', {}) + const { payload: { calls } } = await runner.run('GetHookCallsKW') + expect(calls.indexOf('before:Echo')).toBeLessThan(calls.indexOf('after:Echo')) + }) + + test('afterKeyword is called even when the keyword throws', async () => { + await runner.run('ErrorTestKW', { ErrorMsg: 'test error', rethrow_error: false }) + const { payload: { calls } } = await runner.run('GetHookCallsKW') + expect(calls).toContain('after:ErrorTestKW') + }) + }) }) // --------------------------------------------------------------------------- From f12d0ec1b8ccab9f8279360a97a3ba8c96bfb68b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Thu, 19 Mar 2026 22:56:16 +0100 Subject: [PATCH 17/21] SED-4581 Fixing vulnerabilities --- step-node/step-node-agent/.eslintrc.js | 3 -- step-node/step-node-agent/eslint.config.js | 59 ++++++++++++++++++++++ step-node/step-node-agent/package.json | 26 +++++----- 3 files changed, 72 insertions(+), 16 deletions(-) delete mode 100644 step-node/step-node-agent/.eslintrc.js create mode 100644 step-node/step-node-agent/eslint.config.js diff --git a/step-node/step-node-agent/.eslintrc.js b/step-node/step-node-agent/.eslintrc.js deleted file mode 100644 index 2f16b25ff8..0000000000 --- a/step-node/step-node-agent/.eslintrc.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - 'extends': 'standard' -} diff --git a/step-node/step-node-agent/eslint.config.js b/step-node/step-node-agent/eslint.config.js new file mode 100644 index 0000000000..aab21fd210 --- /dev/null +++ b/step-node/step-node-agent/eslint.config.js @@ -0,0 +1,59 @@ +const js = require('@eslint/js') +const pluginN = require('eslint-plugin-n') +const pluginPromise = require('eslint-plugin-promise') +const pluginImport = require('eslint-plugin-import-x') +const globals = require('globals') + +module.exports = [ + { + ignores: [ + 'node_modules/**', + 'filemanager/**', + 'npm-project-workspaces/**', + ], + }, + js.configs.recommended, + pluginN.configs['flat/recommended'], + pluginPromise.configs['flat/recommended'], + { + plugins: { + import: pluginImport, + }, + languageOptions: { + globals: globals.node, + ecmaVersion: 2022, + sourceType: 'commonjs', + }, + rules: { + 'import/no-unresolved': 'error', + 'import/no-extraneous-dependencies': 'warn', + }, + }, + // process.exit() is legitimate in agent/fork entry-points. + { + rules: { + 'n/no-process-exit': 'off', + }, + }, + // Test files: provide Jest globals and relax rules that don't apply. + { + files: ['test/**/*.test.js', 'test/**/*.spec.js'], + languageOptions: { + globals: globals.jest, + }, + }, + // Keyword stubs intentionally declare the full API signature without using every param. + { + files: ['test/keywords/**/*.js'], + rules: { + 'no-unused-vars': ['error', { vars: 'all', args: 'none' }], + }, + }, + // The config file itself uses devDependencies — allow unpublished requires here. + { + files: ['eslint.config.js'], + rules: { + 'n/no-unpublished-require': 'off', + }, + }, +] \ No newline at end of file diff --git a/step-node/step-node-agent/package.json b/step-node/step-node-agent/package.json index 93e2d97149..f8a4fb2b88 100644 --- a/step-node/step-node-agent/package.json +++ b/step-node/step-node-agent/package.json @@ -1,7 +1,7 @@ { "name": "step-node-agent", "version": "0.0.0", - "description": "The official STEP Agent implementation for Node.js", + "description": "The official Step Agent implementation for Node.js", "main": "index.js", "scripts": { "debug": "node --inspect-brk server.js", @@ -20,29 +20,31 @@ "step": { "keywords": "test/keywords" }, - "author": "Jerome Comte", + "engines": { + "node": ">=20.0.0" + }, + "author": "exense GmbH", "license": "AGPL-3.0", "repository": { "type": "git", "url": "git@github.com:exense/step.git" }, "devDependencies": { - "eslint": "^4.19.1", - "eslint-config-standard": "^11.0.0", - "eslint-plugin-import": "^2.14.0", - "eslint-plugin-node": "^6.0.1", - "eslint-plugin-promise": "^3.8.0", - "eslint-plugin-standard": "^3.1.0", + "@eslint/js": "^10.0.1", + "eslint": "^10.0.3", + "eslint-plugin-import-x": "^4.16.2", + "eslint-plugin-n": "^17.24.0", + "eslint-plugin-promise": "^7.2.1", + "globals": "^15.15.0", "husky": "^0.14.3", "jest": "^29.7.0", - "nodemon": "^1.18.7" + "nodemon": "^3.1.14" }, "dependencies": { "express": "^4.17.1", "get-fqdn": "^1.0.0", - "jsonwebtoken": "^9.0.2", + "jsonwebtoken": "9.0.3", "minimist": "^1.2.5", - "request": "^2.88.0", "shelljs": "^0.8.4", "unzip-stream": "^0.3.1", "uuid": "^3.4.0", @@ -54,7 +56,6 @@ "get-fqdn", "jsonwebtoken", "minimist", - "request", "shelljs", "unzip-stream", "uuid", @@ -68,7 +69,6 @@ "get-fqdn", "jsonwebtoken", "minimist", - "request", "shelljs", "unzip-stream", "uuid", From 81c404181894a8bc6462f5cd3c5936439dd7580e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 20 Mar 2026 07:28:26 +0100 Subject: [PATCH 18/21] SED-4581 Fixing eslint configuration --- .../step-node-agent/api/controllers/agent.js | 4 +- .../api/filemanager/filemanager.js | 2 +- step-node/step-node-agent/eslint.config.js | 59 ------------------- step-node/step-node-agent/eslint.config.mjs | 34 +++++++++++ step-node/step-node-agent/package.json | 3 - 5 files changed, 37 insertions(+), 65 deletions(-) delete mode 100644 step-node/step-node-agent/eslint.config.js create mode 100644 step-node/step-node-agent/eslint.config.mjs diff --git a/step-node/step-node-agent/api/controllers/agent.js b/step-node/step-node-agent/api/controllers/agent.js index 6dc460e52a..d7a65389c5 100644 --- a/step-node/step-node-agent/api/controllers/agent.js +++ b/step-node/step-node-agent/api/controllers/agent.js @@ -61,7 +61,7 @@ class Agent { } } - interruptExecution(req, res) { + interruptExecution(req, _res) { const tokenId = req.params.tokenId logger.warn('Interrupting token: ' + tokenId + ' : not implemented') } @@ -413,7 +413,7 @@ class ForkedAgent { close() { try { this.forkProcess.send({ type: "KILL" }); - } catch (e) { + } catch { this.forkProcess.kill(); } fs.rmSync(this.agentForkerLibPath, {recursive: true, force: true}); diff --git a/step-node/step-node-agent/api/filemanager/filemanager.js b/step-node/step-node-agent/api/filemanager/filemanager.js index e5e8f74a5c..eab0cba3cb 100644 --- a/step-node/step-node-agent/api/filemanager/filemanager.js +++ b/step-node/step-node-agent/api/filemanager/filemanager.js @@ -66,7 +66,7 @@ class FileManager { this.#putCacheEntry(fileId, fileVersionId, cacheEntry) if (cacheEntry.promises) { - cacheEntry.promises.forEach(callback => callback(filePath + '/' + result)) // eslint-disable-line + cacheEntry.promises.forEach(callback => callback(filePath + '/' + result)) } delete cacheEntry.promises diff --git a/step-node/step-node-agent/eslint.config.js b/step-node/step-node-agent/eslint.config.js deleted file mode 100644 index aab21fd210..0000000000 --- a/step-node/step-node-agent/eslint.config.js +++ /dev/null @@ -1,59 +0,0 @@ -const js = require('@eslint/js') -const pluginN = require('eslint-plugin-n') -const pluginPromise = require('eslint-plugin-promise') -const pluginImport = require('eslint-plugin-import-x') -const globals = require('globals') - -module.exports = [ - { - ignores: [ - 'node_modules/**', - 'filemanager/**', - 'npm-project-workspaces/**', - ], - }, - js.configs.recommended, - pluginN.configs['flat/recommended'], - pluginPromise.configs['flat/recommended'], - { - plugins: { - import: pluginImport, - }, - languageOptions: { - globals: globals.node, - ecmaVersion: 2022, - sourceType: 'commonjs', - }, - rules: { - 'import/no-unresolved': 'error', - 'import/no-extraneous-dependencies': 'warn', - }, - }, - // process.exit() is legitimate in agent/fork entry-points. - { - rules: { - 'n/no-process-exit': 'off', - }, - }, - // Test files: provide Jest globals and relax rules that don't apply. - { - files: ['test/**/*.test.js', 'test/**/*.spec.js'], - languageOptions: { - globals: globals.jest, - }, - }, - // Keyword stubs intentionally declare the full API signature without using every param. - { - files: ['test/keywords/**/*.js'], - rules: { - 'no-unused-vars': ['error', { vars: 'all', args: 'none' }], - }, - }, - // The config file itself uses devDependencies — allow unpublished requires here. - { - files: ['eslint.config.js'], - rules: { - 'n/no-unpublished-require': 'off', - }, - }, -] \ No newline at end of file diff --git a/step-node/step-node-agent/eslint.config.mjs b/step-node/step-node-agent/eslint.config.mjs new file mode 100644 index 0000000000..b530909961 --- /dev/null +++ b/step-node/step-node-agent/eslint.config.mjs @@ -0,0 +1,34 @@ +import js from "@eslint/js"; +import globals from "globals"; +import { defineConfig } from "eslint/config"; + +export default defineConfig([ + { + ignores: [ + "node_modules/**", + "filemanager/work/**", + "npm-project-workspaces/**", + ], + }, + { + files: ["**/*.{js,mjs,cjs}"], + plugins: { js }, + extends: ["js/recommended"], + languageOptions: { globals: globals.node }, + rules: { + // Allow intentionally-unused params/vars when prefixed with _ + "no-unused-vars": ["error", { vars: "all", args: "after-used", argsIgnorePattern: "^_", ignoreRestSiblings: true }], + }, + }, + { files: ["**/*.js"], languageOptions: { sourceType: "commonjs" } }, + // Test files: provide Jest globals + { + files: ["test/**/*.test.js", "test/**/*.spec.js"], + languageOptions: { globals: globals.jest }, + }, + // Keyword stubs intentionally declare the full API signature without using every param + { + files: ["test/keywords/**/*.js"], + rules: { "no-unused-vars": ["error", { vars: "all", args: "none" }] }, + }, +]); \ No newline at end of file diff --git a/step-node/step-node-agent/package.json b/step-node/step-node-agent/package.json index f8a4fb2b88..53ee93f831 100644 --- a/step-node/step-node-agent/package.json +++ b/step-node/step-node-agent/package.json @@ -32,9 +32,6 @@ "devDependencies": { "@eslint/js": "^10.0.1", "eslint": "^10.0.3", - "eslint-plugin-import-x": "^4.16.2", - "eslint-plugin-n": "^17.24.0", - "eslint-plugin-promise": "^7.2.1", "globals": "^15.15.0", "husky": "^0.14.3", "jest": "^29.7.0", From 4e607c62624757a07418ae8814ecba0283e8e748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 20 Mar 2026 10:04:15 +0100 Subject: [PATCH 19/21] SED-4581 Deprecating output.send() --- .../step-node-agent/api/controllers/output.js | 9 +++-- .../step-node-agent/test/keywords/keywords.js | 38 +++++++++---------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/step-node/step-node-agent/api/controllers/output.js b/step-node/step-node-agent/api/controllers/output.js index d115e8999c..1c3fb649b7 100644 --- a/step-node/step-node-agent/api/controllers/output.js +++ b/step-node/step-node-agent/api/controllers/output.js @@ -34,7 +34,7 @@ class OutputBuilder { /** * Adds an output attribute to the payload. Can be called multiple times to build the payload - * incrementally. Call send() with no arguments afterwards to flush. + * incrementally. The accumulated payload is returned automatically — no need to call send(). * @param {string} name * @param {*} value * @returns {OutputBuilder} this, for chaining @@ -45,8 +45,11 @@ class OutputBuilder { } /** - * Sends the output. If a payload object is provided it replaces the current payload entirely. - * If called with no arguments, the payload built up via add() is used. + * @deprecated Calling send() is no longer required. The output accumulated via add(), + * setError(), attach(), etc. is returned automatically when the keyword finishes. + * This method is kept for backward compatibility and has no negative effect if called: + * - send() with no arguments is a no-op. + * - send(payload) still replaces the entire payload object, for code that relied on that behaviour. * @param {object} [payload] */ send(payload) { diff --git a/step-node/step-node-agent/test/keywords/keywords.js b/step-node/step-node-agent/test/keywords/keywords.js index 8b39b78dfc..a8713d321a 100644 --- a/step-node/step-node-agent/test/keywords/keywords.js +++ b/step-node/step-node-agent/test/keywords/keywords.js @@ -1,6 +1,6 @@ exports.Echo = async (input, output, session, properties) => { input['properties'] = properties - output.send(input) + Object.entries(input).forEach(([k, v]) => output.add(k, v)) } exports.ErrorTestKW = async (input, output, session, properties) => { @@ -9,17 +9,14 @@ exports.ErrorTestKW = async (input, output, session, properties) => { exports.SetErrorTestKW = async (input, output, session, properties) => { output.setError(input['ErrorMsg']) - output.send() } exports.SetErrorWithExceptionKW = async (input, output, session, properties) => { output.setError(new Error(input['ErrorMsg'])) - output.send() } exports.SetErrorWithMessageAndExceptionKW = async (input, output, session, properties) => { output.setError(input['ErrorMsg'], new Error(input['ErrorMsg'])) - output.send() } exports.FailKW = async (input, output, session, properties) => { @@ -28,19 +25,16 @@ exports.FailKW = async (input, output, session, properties) => { exports.BusinessErrorTestKW = async (input, output, session, properties) => { output.setBusinessError(input['ErrorMsg']) - output.send() } exports.ErrorRejectedPromiseTestKW = async (input, output, session, properties) => { Promise.reject(new Error('test')) - output.send() } exports.ErrorUncaughtExceptionTestKW = async (input, output, session, properties) => { process.nextTick(function () { throw new Error() }) - output.send() } exports.onError = async (exception, input, output, session, properties) => { @@ -52,24 +46,23 @@ exports.onError = async (exception, input, output, session, properties) => { // --- output.add --- exports.AddKW = async (input, output, session, properties) => { - output.add('name', 'Alice').add('score', 42).add('active', true).send() + output.add('name', 'Alice').add('score', 42).add('active', true) } // --- output.appendError --- exports.AppendErrorToExistingKW = async (input, output, session, properties) => { - output.setError('base error').appendError(' + extra detail').send() + output.setError('base error').appendError(' + extra detail') } exports.AppendErrorToNoneKW = async (input, output, session, properties) => { - output.appendError('fresh error').send() + output.appendError('fresh error') } // --- output.attach --- exports.AttachKW = async (input, output, session, properties) => { output.attach({ name: 'report.txt', isDirectory: false, description: 'test attachment', hexContent: Buffer.from('hello').toString('base64') }) - output.send() } // --- measurement methods --- @@ -78,18 +71,15 @@ exports.StartStopMeasureKW = async (input, output, session, properties) => { output.startMeasure('step1') await new Promise(r => setTimeout(r, 10)) output.stopMeasure() - output.send() } exports.StartStopMeasureWithStatusKW = async (input, output, session, properties) => { output.startMeasure('failing-step') output.stopMeasure({ status: 'FAILED', data: { reason: 'assertion failed' } }) - output.send() } exports.AddMeasureKW = async (input, output, session, properties) => { output.addMeasure('pre-timed', 150, { status: 'TECHNICAL_ERROR', begin: Date.now() - 150, data: { info: 'test' } }) - output.send() } exports.MultipleMeasuresKW = async (input, output, session, properties) => { @@ -98,27 +88,24 @@ exports.MultipleMeasuresKW = async (input, output, session, properties) => { output.startMeasure('second') output.stopMeasure({ status: 'FAILED' }) output.addMeasure('third', 50) - output.send() } // --- session --- exports.SessionSetKW = async (input, output, session) => { session.set('sharedKey', input['value']) - output.send() } exports.SessionGetKW = async (input, output, session) => { - output.add('value', session.get('sharedKey')).send() + output.add('value', session.get('sharedKey')) } exports.SessionSetDotKW = async (input, output, session) => { session.dotKey = input['value'] - output.send() } exports.SessionGetDotKW = async (input, output, session) => { - output.add('value', session.dotKey).send() + output.add('value', session.dotKey) } // --- beforeKeyword / afterKeyword hook tracking --- @@ -136,5 +123,16 @@ exports.afterKeyword = async (functionName) => { exports.GetHookCallsKW = async (input, output) => { const calls = [..._hookCalls] _hookCalls = [] - output.add('calls', calls).send() + output.add('calls', calls) +} + +// --- backward compat: output.send() is no longer required but must still work --- + +exports.SendNoArgCompatKW = async (input, output) => { + output.add('result', 'ok') + output.send() +} + +exports.SendWithPayloadCompatKW = async (input, output) => { + output.send({ result: 'ok' }) } From d4e78aee82a4c37555f84f6bd47fe1f2d505a96e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 20 Mar 2026 11:07:52 +0100 Subject: [PATCH 20/21] SED-4581 Adding test coverage for properties --- .../step-node-agent/test/keywords/keywords.js | 6 ++++ step-node/step-node-agent/test/runner.test.js | 36 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/step-node/step-node-agent/test/keywords/keywords.js b/step-node/step-node-agent/test/keywords/keywords.js index a8713d321a..4b202b6bea 100644 --- a/step-node/step-node-agent/test/keywords/keywords.js +++ b/step-node/step-node-agent/test/keywords/keywords.js @@ -126,6 +126,12 @@ exports.GetHookCallsKW = async (input, output) => { output.add('calls', calls) } +// --- properties --- + +exports.GetPropertyKW = async (input, output, session, properties) => { + output.add('value', properties[input['key']]) +} + // --- backward compat: output.send() is no longer required but must still work --- exports.SendNoArgCompatKW = async (input, output) => { diff --git a/step-node/step-node-agent/test/runner.test.js b/step-node/step-node-agent/test/runner.test.js index 937ddb7f59..3ea6976ea0 100644 --- a/step-node/step-node-agent/test/runner.test.js +++ b/step-node/step-node-agent/test/runner.test.js @@ -203,6 +203,42 @@ describe('runner', () => { }) }) + // --------------------------------------------------------------------------- + // properties + // --------------------------------------------------------------------------- + + describe('properties', () => { + test('property passed to runner constructor is accessible in a keyword', async () => { + const output = await runner.run('GetPropertyKW', { key: 'Property1' }) + expect(output.payload.value).toBe('Prop1') + }) + + test('all properties passed to runner constructor are accessible', async () => { + const r = require('../api/runner/runner')({ KeyA: 'ValA', KeyB: 'ValB' }) + r.setThrowExceptionOnError(false) + try { + const outputA = await r.run('GetPropertyKW', { key: 'KeyA' }) + const outputB = await r.run('GetPropertyKW', { key: 'KeyB' }) + expect(outputA.payload.value).toBe('ValA') + expect(outputB.payload.value).toBe('ValB') + } finally { + r.close() + } + }) + + test('runner without properties gives keywords an empty properties object', async () => { + const r = require('../api/runner/runner')() + r.setThrowExceptionOnError(false) + try { + const output = await r.run('GetPropertyKW', { key: 'anyKey' }) + expect(output.payload.value).toBeUndefined() + expect(output.error).toBeUndefined() + } finally { + r.close() + } + }) + }) + // --------------------------------------------------------------------------- // beforeKeyword and afterKeyword hooks // --------------------------------------------------------------------------- From c05b6882c7147176eff7c29504c4ffb024089d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Sat, 21 Mar 2026 07:00:17 +0100 Subject: [PATCH 21/21] SED-4581 Implementing async close --- .../api/controllers/agent-fork.js | 2 +- .../step-node-agent/api/controllers/agent.js | 14 ++- .../api/controllers/session.js | 34 +++--- .../step-node-agent/api/runner/runner.js | 4 +- .../step-node-agent/test/keywords/keywords.js | 8 ++ step-node/step-node-agent/test/runner.test.js | 39 ++++++- .../step-node-agent/test/session.test.js | 108 ++++++++++++++++++ 7 files changed, 184 insertions(+), 25 deletions(-) create mode 100644 step-node/step-node-agent/test/session.test.js diff --git a/step-node/step-node-agent/api/controllers/agent-fork.js b/step-node/step-node-agent/api/controllers/agent-fork.js index ce33453717..4a21c0439a 100644 --- a/step-node/step-node-agent/api/controllers/agent-fork.js +++ b/step-node/step-node-agent/api/controllers/agent-fork.js @@ -73,7 +73,7 @@ process.on('message', async ({ type, projectPath, functionName, input, propertie } } else if (type === 'KILL') { console.log("[Agent fork] Exiting...") - session[Symbol.dispose](); + await session.asyncDispose(); process.exit(1) } diff --git a/step-node/step-node-agent/api/controllers/agent.js b/step-node/step-node-agent/api/controllers/agent.js index d7a65389c5..c2ef070e99 100644 --- a/step-node/step-node-agent/api/controllers/agent.js +++ b/step-node/step-node-agent/api/controllers/agent.js @@ -48,13 +48,13 @@ class Agent { res.json({}) } - releaseToken_(tokenId) { + async releaseToken_(tokenId) { logger.info('Releasing token: ' + tokenId) const session = this.agentContext.tokenSessions[tokenId] if (session) { // Close the session and all objects it contains - session[Symbol.dispose](); + await session.asyncDispose(); this.agentContext.tokenSessions[tokenId] = null; } else { logger.warn('No session found for token: ' + tokenId) @@ -410,12 +410,20 @@ class ForkedAgent { } } - close() { + async close() { + const exitPromise = new Promise(resolve => { + if (this.forkProcess.exitCode !== null) { + resolve(); + } else { + this.forkProcess.once('exit', resolve); + } + }); try { this.forkProcess.send({ type: "KILL" }); } catch { this.forkProcess.kill(); } + await exitPromise; fs.rmSync(this.agentForkerLibPath, {recursive: true, force: true}); } } diff --git a/step-node/step-node-agent/api/controllers/session.js b/step-node/step-node-agent/api/controllers/session.js index 7f4521ac09..58b1c3c3d3 100644 --- a/step-node/step-node-agent/api/controllers/session.js +++ b/step-node/step-node-agent/api/controllers/session.js @@ -24,39 +24,43 @@ try { } class Session extends Map { - [Symbol.dispose]() { - logger.info(`Disposing Session: Cleaning up ${this.size} resources...`); + + async asyncDispose() { + logger.info(`Async-disposing Session: Cleaning up ${this.size} resources...`); + const promises = []; for (const [key, resource] of this) { try { - // 1. Try modern [Symbol.dispose] - if (resource && typeof resource[Symbol.dispose] === 'function') { + if (resource && typeof resource[Symbol.asyncDispose] === 'function') { + promises.push(resource[Symbol.asyncDispose]()); + } else if (resource && typeof resource[Symbol.dispose] === 'function') { resource[Symbol.dispose](); - } - // 2. Fallback to Node.js child_process .kill() - else if (resource && typeof resource.kill === 'function') { + } else if (resource && typeof resource.kill === 'function') { resource.kill(); + } else if (resource && typeof resource.close === 'function') { + const result = resource.close(); + if (result && typeof result.then === 'function') promises.push(result); } - // 3. Fallback to generic .close() - else if (resource && typeof resource.close === 'function') { - resource.close(); - } - logger.debug(`Successfully closed resource: ${key}`); } catch (err) { logger.error(`Failed to close resource ${key}:`, err); } } - // Clean up Object properties (Added via .dot notation) to support keywords written for older versions of the agent exposing the session as simple object + // Clean up Object properties (Added via .dot notation) for (const key of Object.keys(this)) { const resource = this[key]; if (resource && typeof resource.close === 'function') { - resource.close(); + try { + const result = resource.close(); + if (result && typeof result.then === 'function') promises.push(result); + } catch (err) { + logger.error(`Failed to close dot-notation resource ${key}:`, err); + } } } - // Clear the map after disposal so resources aren't held in memory + await Promise.allSettled(promises); this.clear(); } } diff --git a/step-node/step-node-agent/api/runner/runner.js b/step-node/step-node-agent/api/runner/runner.js index e846661ca1..b58b31184a 100644 --- a/step-node/step-node-agent/api/runner/runner.js +++ b/step-node/step-node-agent/api/runner/runner.js @@ -34,8 +34,8 @@ module.exports = function (properties = {}) { return output.payload } - api.close = function () { - tokenSession[Symbol.dispose](); + api.close = async function () { + return await tokenSession.asyncDispose(); } return api diff --git a/step-node/step-node-agent/test/keywords/keywords.js b/step-node/step-node-agent/test/keywords/keywords.js index 4b202b6bea..1fd5dc4167 100644 --- a/step-node/step-node-agent/test/keywords/keywords.js +++ b/step-node/step-node-agent/test/keywords/keywords.js @@ -132,6 +132,14 @@ exports.GetPropertyKW = async (input, output, session, properties) => { output.add('value', properties[input['key']]) } +// --- session auto-disposal --- + +exports.StoreCloseableKW = async (input, output, session) => { + session.set('resource', { + close() { require('fs').writeFileSync(input['closePath'], 'closed') } + }) +} + // --- backward compat: output.send() is no longer required but must still work --- exports.SendNoArgCompatKW = async (input, output) => { diff --git a/step-node/step-node-agent/test/runner.test.js b/step-node/step-node-agent/test/runner.test.js index 3ea6976ea0..c21c97f80a 100644 --- a/step-node/step-node-agent/test/runner.test.js +++ b/step-node/step-node-agent/test/runner.test.js @@ -8,8 +8,8 @@ describe('runner', () => { runner.setThrowExceptionOnError(false) }) - afterAll(() => { - runner.close() + afterAll(async () => { + await runner.close() }) // --------------------------------------------------------------------------- @@ -203,6 +203,37 @@ describe('runner', () => { }) }) + // --------------------------------------------------------------------------- + // session auto-disposal on runner.close() + // + // Keywords run in a forked subprocess whose own session holds user-stored + // resources. When runner.close() is called, the fork receives KILL and its + // session disposes those resources. The close() side-effect (writing a temp + // file) is the observable signal crossing the process boundary. + // --------------------------------------------------------------------------- + + describe('session auto-disposal on runner.close()', () => { + const os = require('os') + const path = require('path') + const fs = require('fs') + + test('close() is called on a resource stored in the session when runner.close() is invoked', async () => { + const closePath = path.join(os.tmpdir(), `step-session-close-${Date.now()}.txt`) + const r = require('../api/runner/runner')() + r.setThrowExceptionOnError(false) + try { + await r.run('StoreCloseableKW', { closePath }) + expect(fs.existsSync(closePath)).toBe(false) // resource not yet closed + + await r.close() + + expect(fs.existsSync(closePath)).toBe(true) // resource closed synchronously with the fork + } finally { + if (fs.existsSync(closePath)) fs.unlinkSync(closePath) + } + }) + }) + // --------------------------------------------------------------------------- // properties // --------------------------------------------------------------------------- @@ -222,7 +253,7 @@ describe('runner', () => { expect(outputA.payload.value).toBe('ValA') expect(outputB.payload.value).toBe('ValB') } finally { - r.close() + await r.close() } }) @@ -234,7 +265,7 @@ describe('runner', () => { expect(output.payload.value).toBeUndefined() expect(output.error).toBeUndefined() } finally { - r.close() + await r.close() } }) }) diff --git a/step-node/step-node-agent/test/session.test.js b/step-node/step-node-agent/test/session.test.js new file mode 100644 index 0000000000..0d755f1f43 --- /dev/null +++ b/step-node/step-node-agent/test/session.test.js @@ -0,0 +1,108 @@ +const Session = require("../api/controllers/session"); + +describe('Session auto-disposal', () => { + + test('[Symbol.dispose]() is called on resources stored via session.set()', async () => { + const session = new Session() + const disposed = jest.fn() + session.set('res', {[Symbol.dispose]: disposed}) + await session.asyncDispose() + expect(disposed).toHaveBeenCalledTimes(1) + }) + + test('.close() is called when the resource has no [Symbol.dispose]', async () => { + const session = new Session() + const closed = jest.fn() + session.set('res', {close: closed}) + await session.asyncDispose() + expect(closed).toHaveBeenCalledTimes(1) + }) + + test('.kill() is called when the resource has no [Symbol.dispose] or .close()', async () => { + const session = new Session() + const killed = jest.fn() + session.set('res', {kill: killed}) + await session.asyncDispose() + expect(killed).toHaveBeenCalledTimes(1) + }) + + test('[Symbol.asyncDispose]() is awaited before asyncDispose() resolves', async () => { + const session = new Session() + let resolved = false + session.set('res', { + [Symbol.asyncDispose]: async () => { + await new Promise(r => setTimeout(r, 10)) + resolved = true + } + }) + await session.asyncDispose() + expect(resolved).toBe(true) + }) + + test('[Symbol.asyncDispose]() takes precedence over .close() and .kill()', async () => { + const session = new Session() + const asyncDisposed = jest.fn().mockResolvedValue(undefined) + const closed = jest.fn() + const killed = jest.fn() + session.set('res', {[Symbol.asyncDispose]: asyncDisposed, close: closed, kill: killed}) + await session.asyncDispose() + expect(asyncDisposed).toHaveBeenCalledTimes(1) + expect(closed).not.toHaveBeenCalled() + expect(killed).not.toHaveBeenCalled() + }) + + test('[Symbol.dispose]() takes precedence over .close() and .kill()', async () => { + const session = new Session() + const disposed = jest.fn() + const closed = jest.fn() + const killed = jest.fn() + session.set('res', {[Symbol.dispose]: disposed, close: closed, kill: killed}) + await session.asyncDispose() + expect(disposed).toHaveBeenCalledTimes(1) + expect(closed).not.toHaveBeenCalled() + expect(killed).not.toHaveBeenCalled() + }) + + test('.close() is called on resources stored via dot notation', async () => { + const session = new Session() + const closed = jest.fn() + session.dotResource = {close: closed} + await session.asyncDispose() + expect(closed).toHaveBeenCalledTimes(1) + }) + + test('resources with no disposal method are silently skipped', () => { + const session = new Session() + session.set('plain', { value: 42 }) + expect(async () => await session.asyncDispose()).not.toThrow() + }) + + test('all entries are cleared after disposal', async () => { + const session = new Session() + session.set('a', {close: jest.fn()}) + session.set('b', {close: jest.fn()}) + await session.asyncDispose() + expect(session.size).toBe(0) + }) + + test('disposal continues for remaining resources when one throws', async () => { + const session = new Session() + session.set('bad', { + [Symbol.dispose]: () => { + throw new Error('oops') + } + }) + const goodClosed = jest.fn() + session.set('good', {close: goodClosed}) + await session.asyncDispose() + expect(goodClosed).toHaveBeenCalledTimes(1) + }) + + test('multiple resources stored via session.set() are all disposed', async () => { + const session = new Session() + const fns = [jest.fn(), jest.fn(), jest.fn()] + fns.forEach((fn, i) => session.set(`res${i}`, {[Symbol.dispose]: fn})) + await session.asyncDispose() + fns.forEach(fn => expect(fn).toHaveBeenCalledTimes(1)) + }) +})