From 5adcdd46c5ad506fd68422254ffbc09a65c41fe5 Mon Sep 17 00:00:00 2001 From: Steve Churchill Date: Tue, 14 Apr 2026 14:21:19 +0100 Subject: [PATCH 1/3] Add Playwright flaky & metrics reporters/configs Introduce dedicated Playwright configs and reporters for flaky-test detection and tag-based metrics. Adds playwright.flaky.config.ts and playwright.metrics.config.ts, new reporters (flaky-test-reporter.ts, metrics-reporter.ts), tags-helper and report-utils helpers, and wiring to output reports to reports/. Updates e2etests package.json with new scripts (playwright:metrics, playwright:flaky) and adds @reportportal/agent-js-playwright as a dependency. Ignore reports/ in .gitignore and apply related adjustments across many e2etests specs to integrate the reporting/metrics workflow. --- .gitignore | 1 + e2etests/package-lock.json | 519 ++++++++++++++++++- e2etests/package.json | 3 + e2etests/playwright.flaky.config.ts | 83 +++ e2etests/playwright.metrics.config.ts | 42 ++ e2etests/reporting/flaky-test-reporter.ts | 258 +++++++++ e2etests/reporting/metrics-reporter.ts | 398 ++++++++++++++ e2etests/reporting/tags-helper.ts | 195 +++++++ e2etests/tests/anomalies-tests.spec.ts | 22 +- e2etests/tests/auth-api-tests.spec.ts | 18 +- e2etests/tests/cloud-accounts-tests.spec.ts | 20 +- e2etests/tests/expenses-tests.spec.ts | 32 +- e2etests/tests/homepage-tests.spec.ts | 22 +- e2etests/tests/invitation-flow-tests.spec.ts | 16 +- e2etests/tests/perspective-tests.spec.ts | 16 +- e2etests/tests/policies-tests.spec.ts | 26 +- e2etests/tests/pools-tests.spec.ts | 20 +- e2etests/tests/recommendations-tests.spec.ts | 26 +- e2etests/tests/resources-tests.spec.ts | 22 +- e2etests/tests/ri-sp-coverage-test.spec.ts | 2 +- e2etests/tests/tagging-policy-tests.spec.ts | 12 +- e2etests/utils/report-utils.ts | 27 + 22 files changed, 1652 insertions(+), 128 deletions(-) create mode 100644 e2etests/playwright.flaky.config.ts create mode 100644 e2etests/playwright.metrics.config.ts create mode 100644 e2etests/reporting/flaky-test-reporter.ts create mode 100644 e2etests/reporting/metrics-reporter.ts create mode 100644 e2etests/reporting/tags-helper.ts create mode 100644 e2etests/utils/report-utils.ts diff --git a/.gitignore b/.gitignore index da97ec52e..9db6b6147 100644 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,5 @@ optscale-deploy/.vagrant/ /e2etests/.cache/ /e2etests/.cache/* /e2etests/tests/downloads/* +/e2etests/reports/* diff --git a/e2etests/package-lock.json b/e2etests/package-lock.json index 4fd74cfb0..14dfee569 100644 --- a/e2etests/package-lock.json +++ b/e2etests/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "@reportportal/agent-js-playwright": "^5.4.0", "playwright": "^1.56.1" }, "devDependencies": { @@ -310,6 +311,52 @@ "node": ">=18" } }, + "node_modules/@reportportal/agent-js-playwright": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/@reportportal/agent-js-playwright/-/agent-js-playwright-5.4.0.tgz", + "integrity": "sha512-zJFstaje46HXdbsiXimmpiivTErnefxmApT+h/HLcW1sFyk58K8mHT1QAh0jVGONFP6mOWQ8osguIoCZpbqImQ==", + "license": "Apache-2.0", + "dependencies": { + "@reportportal/client-javascript": "~5.5.10", + "strip-ansi": "~6.0.1" + }, + "engines": { + "node": ">=14.x" + } + }, + "node_modules/@reportportal/client-javascript": { + "version": "5.5.10", + "resolved": "https://registry.npmjs.org/@reportportal/client-javascript/-/client-javascript-5.5.10.tgz", + "integrity": "sha512-yC4tEHBy4k8x5QNuFlv74JXmyejExE1ozd6swxOH4sdRQtdyoQoassMTcpMErWqnHLk0NcLnXRXeJBSnq/tT5g==", + "license": "Apache-2.0", + "dependencies": { + "axios": "^1.12.2", + "axios-retry": "^4.5.0", + "glob": "^13.0.1", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", + "ini": "^2.0.0", + "proxy-from-env": "^1.1.0", + "uniqid": "^5.4.0", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14.x" + } + }, + "node_modules/@reportportal/client-javascript/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -625,6 +672,15 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -642,6 +698,15 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -665,6 +730,44 @@ "dev": true, "license": "Python-2.0" }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz", + "integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", + "proxy-from-env": "^2.1.0" + } + }, + "node_modules/axios-retry": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-4.5.0.tgz", + "integrity": "sha512-aR99oXhpEDGo0UuAlYcn2iGRds30k366Zfa05XWScR9QaQD4JYiP3/1Qt1u7YlefUOK+cn0CcwoL1oefavQUlQ==", + "license": "Apache-2.0", + "dependencies": { + "is-retry-allowed": "^2.2.0" + }, + "peerDependencies": { + "axios": "0.x || 1.x" + } + }, + "node_modules/axios/node_modules/proxy-from-env": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", + "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -696,6 +799,19 @@ "node": ">=8" } }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -743,6 +859,18 @@ "dev": true, "license": "MIT" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -788,7 +916,6 @@ "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -809,6 +936,15 @@ "dev": true, "license": "MIT" }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/dotenv": { "version": "16.5.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz", @@ -821,6 +957,65 @@ "url": "https://dotenvx.com" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -1162,6 +1357,42 @@ "dev": true, "license": "ISC" }, + "node_modules/follow-redirects": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -1176,6 +1407,69 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -1189,6 +1483,42 @@ "node": ">=10.13.0" } }, + "node_modules/glob/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -1202,6 +1532,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/graphemer": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", @@ -1219,6 +1561,71 @@ "node": ">=8" } }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -1256,6 +1663,15 @@ "node": ">=0.8.19" } }, + "node_modules/ini": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz", + "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -1289,6 +1705,18 @@ "node": ">=0.12.0" } }, + "node_modules/is-retry-allowed": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz", + "integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -1377,6 +1805,24 @@ "dev": true, "license": "MIT" }, + "node_modules/lru-cache": { + "version": "11.3.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.5.tgz", + "integrity": "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -1401,6 +1847,27 @@ "node": ">=8.6" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -1414,11 +1881,19 @@ "node": "*" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/natural-compare": { @@ -1511,6 +1986,22 @@ "node": ">=8" } }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/pdf-poppler": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/pdf-poppler/-/pdf-poppler-0.2.2.tgz", @@ -1610,6 +2101,12 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -1722,6 +2219,18 @@ "node": ">=8" } }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -1845,6 +2354,12 @@ "dev": true, "license": "MIT" }, + "node_modules/uniqid": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/uniqid/-/uniqid-5.4.0.tgz", + "integrity": "sha512-38JRbJ4Fj94VmnC7G/J/5n5SC7Ab46OM5iNtSstB/ko3l1b5g7ALt4qzHFgGciFkyiRNtDXtLNb+VsxtMSE77A==", + "license": "MIT" + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", diff --git a/e2etests/package.json b/e2etests/package.json index 00d0afc5a..dd47d0b18 100644 --- a/e2etests/package.json +++ b/e2etests/package.json @@ -29,6 +29,8 @@ "playwright:regression:headed": "npm run playwright:regression -- --headed --workers=1", "playwright:regression:update": "npm run playwright:regression -- --update-snapshots", "playwright:devops": "npx playwright test --grep @devops --config=playwright.devops.config.ts", + "playwright:metrics": "npx playwright test --config=playwright.metrics.config.ts --grep @ui", + "playwright:flaky": "npx playwright test --config=playwright.flaky.config.ts --grep @ui", "docker:test": "./run_pw.sh", "docker:update": "./run_pw.sh -u" }, @@ -55,6 +57,7 @@ "uuid": "^11.0.5" }, "dependencies": { + "@reportportal/agent-js-playwright": "^5.4.0", "playwright": "^1.56.1" } } diff --git a/e2etests/playwright.flaky.config.ts b/e2etests/playwright.flaky.config.ts new file mode 100644 index 000000000..f0ca0e5f4 --- /dev/null +++ b/e2etests/playwright.flaky.config.ts @@ -0,0 +1,83 @@ +/** + * playwright.flaky.config.ts + * + * Playwright config for detecting flaky and consistently-failing tests across + * the optscale e2e suite. + * + * Key settings: + * - retries: 2 — each test gets up to 2 retries, giving the reporter enough + * runs per test to identify flakiness (pass on retry = flaky, + * fail on all retries = consistently failing). + * - The FlakyTestReporter writes a per-shard JSON report to + * reports/flaky/flaky-test-shard-.json + * + * Shard usage (CI): + * npx playwright test --config=playwright.flaky.config.ts --shard=1/4 + * npx playwright test --config=playwright.flaky.config.ts --shard=2/4 + * ... etc. + * + * Local usage (all tests, single shard): + * npm run playwright:flaky + * npm run playwright:flaky -- --grep @p1 # only p1 tests + * npm run playwright:flaky -- --grep @expenses # only expenses module + */ + +import { defineConfig } from '@playwright/test'; +import dotenv from 'dotenv'; +import path from 'path'; + +dotenv.config({ path: path.resolve(__dirname, '.env.local') }); + +export default defineConfig({ + globalSetup: './setup/global-setup.ts', + globalTeardown: './setup/global-teardown.ts', + testDir: './tests', + testIgnore: ['**/regression-tests/**'], + + fullyParallel: false, + forbidOnly: !!process.env.CI, + + /** + * retries: 2 is essential — a test must have at least one pass AND one failure + * within the same run for the reporter to classify it as flaky. + */ + retries: 2, + + workers: process.env.CI ? 1 : 1, + timeout: 45000, + + reporter: [ + ['list'], + ['json', { outputFile: 'results.json' }], + ['./reporting/flaky-test-reporter.ts'], + ], + + use: { + actionTimeout: 10000, + baseURL: process.env.BASE_URL, + testIdAttribute: 'data-test-id', + timezoneId: 'UTC', + headless: true, + trace: 'retain-on-failure', + video: 'retain-on-failure', + screenshot: { + mode: 'only-on-failure', + fullPage: true, + }, + contextOptions: { + reducedMotion: 'reduce', + ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === 'true', + viewport: { width: 1900, height: 1050 }, + }, + }, + + projects: [ + { name: 'setup', testMatch: /e2etests\/setup\/.*\.setup\.ts/ }, + { + name: 'chrome', + use: { channel: 'chrome' }, + dependencies: ['setup'], + }, + ], +}); + diff --git a/e2etests/playwright.metrics.config.ts b/e2etests/playwright.metrics.config.ts new file mode 100644 index 000000000..62760d079 --- /dev/null +++ b/e2etests/playwright.metrics.config.ts @@ -0,0 +1,42 @@ +/** + * playwright.metrics.config.ts + * + * Standalone Playwright config for generating tag-based metrics reports. + * No tests are executed — the MetricsReporter collects tag metadata + * from the suite and writes HTML + CSV reports to the 'reports/' folder, + * then exits the process. + * + * Usage: + * npx playwright test --config=playwright.metrics.config.ts + */ + +import { defineConfig } from '@playwright/test'; +import dotenv from 'dotenv'; +import path from 'path'; + +dotenv.config({ path: path.resolve(__dirname, '.env.local') }); + +export default defineConfig({ + // Point at the full test suite so the reporter can see all tags + testDir: './tests', + // Gather dev-ops tests too (optional — comment out if not needed) + // testDir: '.', + // testMatch: ['tests/**/*.spec.ts', 'dev-ops-tests/**/*.spec.ts'], + + fullyParallel: false, + workers: 1, + retries: 0, + timeout: 30000, + + use: { + headless: true, + baseURL: process.env.BASE_URL ?? 'http://localhost', + }, + + reporter: [ + // The metrics reporter collects tags on onBegin() and calls process.exit(0), + // so no tests will actually run. + ['./reporting/metrics-reporter.ts'], + ], +}); + diff --git a/e2etests/reporting/flaky-test-reporter.ts b/e2etests/reporting/flaky-test-reporter.ts new file mode 100644 index 000000000..2ba48f9c5 --- /dev/null +++ b/e2etests/reporting/flaky-test-reporter.ts @@ -0,0 +1,258 @@ +/** + * FlakyTestReporter - Playwright custom reporter for detecting flaky and failing tests. + * + * Adapted from the generic FlakyTestReporter for the optscale e2e test framework. + * Outputs a per-shard JSON report locally to 'reports/flaky/' — no external API dependencies. + * + * Severity is derived from the optscale priority tags: @p1, @p2, @p3. + * Any test without a recognised priority tag is classified as 'unclassified'. + * + * Shard detection order: + * 1. SHARD_NUMBER env var (explicit override) + * 2. PLAYWRIGHT_SHARD_INDEX env var (0-based → converted to 1-based) + * 3. Falls back to '1' + * + * Usage — add to any playwright config's reporter array: + * reporter: [ + * ['list'], + * ['./reporting/flaky-test-reporter.ts'], + * ] + * + * Or run directly with the dedicated config: + * npx playwright test --config=playwright.flaky.config.ts + * + * @module OptscaleFlakyTestReporter + */ + +import { Reporter, TestCase, TestResult } from '@playwright/test/reporter'; +import * as fs from 'fs'; +import * as path from 'path'; +import { buildTagAttributes } from './tags-helper'; + +// --------------------------------------------------------------------------- +// Output configuration +// --------------------------------------------------------------------------- + +const OUTPUT_FOLDER = path.join('reports', 'flaky'); + +// --------------------------------------------------------------------------- +// Interfaces +// --------------------------------------------------------------------------- + +type Severity = 'p1' | 'p2' | 'p3' | 'unclassified'; + +interface FlakyTestEntry { + testId: string; + testTitle: string; + fileName: string; + totalRuns: number; + failures: number; + failureRate: number; + lastFailure?: string; + tags: string[]; + severity: Severity; + tagAttributes: ReturnType; + detectedAt: string; +} + +interface FailedTestEntry { + testId: string; + testTitle: string; + fileName: string; + tags: string[]; + severity: Severity; + tagAttributes: ReturnType; + failedAt: string; +} + +// --------------------------------------------------------------------------- +// FlakyTestCollector +// --------------------------------------------------------------------------- + +class FlakyTestCollector { + private flakyEntries: FlakyTestEntry[] = []; + private failedEntries: FailedTestEntry[] = []; + private testResults: Map = new Map(); + + addTestResult(test: TestCase, result: TestResult): void { + const testId = `${test.location.file}::${test.title}`; + + if (!this.testResults.has(testId)) { + this.testResults.set(testId, { test, results: [] }); + } + this.testResults.get(testId)!.results.push(result); + } + + analyzeFlakyTests(shardNumber: string): void { + console.log(`🔍 [OptScale] Analyzing ${this.testResults.size} unique tests for shard ${shardNumber}...`); + + this.testResults.forEach(({ test, results }, testId) => { + const totalRuns = results.length; + + const failures = results.filter( + r => r.status === 'failed' || r.status === 'timedOut' || r.status === 'interrupted' + ).length; + + const passes = results.filter(r => r.status === 'passed').length; + const skipped = results.filter(r => r.status === 'skipped').length; + + if (totalRuns > 1 && failures > 0 && passes > 0) { + // ── Flaky: passed at least once AND failed at least once ────────── + console.log(`🔄 FLAKY: ${test.title} (${failures}/${totalRuns} failed)`); + + const lastFailure = [...results] + .reverse() + .find(r => r.status === 'failed' || r.status === 'timedOut' || r.status === 'interrupted'); + + this.flakyEntries.push({ + testId, + testTitle: test.title, + fileName: test.location.file, + totalRuns, + failures, + failureRate: failures / totalRuns, + lastFailure: lastFailure?.error?.message, + tags: test.tags ?? [], + severity: this.extractSeverity(test.tags ?? []), + tagAttributes: buildTagAttributes(test.tags ?? [], test.title), + detectedAt: new Date().toISOString(), + }); + } else if (failures > 0 && passes === 0) { + // ── Consistently failing ────────────────────────────────────────── + console.log(`❌ FAILED: ${test.title} (${failures}/${totalRuns} failed)`); + + this.failedEntries.push({ + testId, + testTitle: test.title, + fileName: test.location.file, + tags: test.tags ?? [], + severity: this.extractSeverity(test.tags ?? []), + tagAttributes: buildTagAttributes(test.tags ?? [], test.title), + failedAt: new Date().toISOString(), + }); + } else if (passes === totalRuns) { + console.log(`✅ PASSED: ${test.title} (${passes}/${totalRuns} passed)`); + } else if (skipped === totalRuns) { + console.log(`⏭️ SKIPPED: ${test.title} (${skipped}/${totalRuns} skipped)`); + } else { + console.log(`❓ UNKNOWN: ${test.title} — no clear status pattern`); + } + }); + + console.log(`📋 Shard ${shardNumber} analysis complete:`); + console.log(` 🔄 Flaky tests: ${this.flakyEntries.length}`); + console.log(` ❌ Failed tests: ${this.failedEntries.length}`); + } + + /** Extract severity from the optscale priority tags (@p1 / @p2 / @p3). */ + private extractSeverity(tags: string[]): Severity { + const clean = tags.map(t => t.replace(/^@/, '').toLowerCase()); + if (clean.includes('p1')) return 'p1'; + if (clean.includes('p2')) return 'p2'; + if (clean.includes('p3')) return 'p3'; + return 'unclassified'; + } + + getFlakyTests(): FlakyTestEntry[] { + return [...this.flakyEntries].sort((a, b) => this.severityOrder(a.severity) - this.severityOrder(b.severity)); + } + + getFailedTests(): FailedTestEntry[] { + return [...this.failedEntries].sort((a, b) => this.severityOrder(a.severity) - this.severityOrder(b.severity)); + } + + private severityOrder(s: Severity): number { + return { p1: 0, p2: 1, p3: 2, unclassified: 3 }[s]; + } +} + +// --------------------------------------------------------------------------- +// FlakyTestReporter +// --------------------------------------------------------------------------- + +class FlakyTestReporter implements Reporter { + private collector = new FlakyTestCollector(); + private readonly shardNumber: string; + + constructor() { + console.log('🔍 [OptScale] FlakyTestReporter initialising...'); + console.log(' SHARD_NUMBER:', process.env.SHARD_NUMBER ?? '(not set)'); + console.log(' PLAYWRIGHT_SHARD_INDEX:', process.env.PLAYWRIGHT_SHARD_INDEX ?? '(not set)'); + + const shardEnvKeys = Object.keys(process.env).filter(k => k.includes('SHARD')); + if (shardEnvKeys.length > 0) { + console.log( + ' Shard-related env vars:', + shardEnvKeys.map(k => `${k}=${process.env[k]}`).join(', ') + ); + } + + this.shardNumber = + process.env.SHARD_NUMBER ?? + (process.env.PLAYWRIGHT_SHARD_INDEX + ? (parseInt(process.env.PLAYWRIGHT_SHARD_INDEX, 10) + 1).toString() + : '1'); + + console.log(`✅ [OptScale] FlakyTestReporter using shard number: ${this.shardNumber}`); + } + + onTestEnd(test: TestCase, result: TestResult): void { + this.collector.addTestResult(test, result); + } + + onEnd(): void { + console.log(`\n🔍 [OptScale] FlakyTestReporter: onEnd() called for shard ${this.shardNumber}`); + + this.collector.analyzeFlakyTests(this.shardNumber); + + // Ensure output folder exists + const absoluteOutputFolder = path.resolve(OUTPUT_FOLDER); + if (!fs.existsSync(absoluteOutputFolder)) { + fs.mkdirSync(absoluteOutputFolder, { recursive: true }); + } + + this.generateJsonReport(); + } + + // ------------------------------------------------------------------------- + // JSON report + // ------------------------------------------------------------------------- + + private generateJsonReport(): void { + const flakyTests = this.collector.getFlakyTests(); + const failedTests = this.collector.getFailedTests(); + + const report = { + generatedAt: new Date().toISOString(), + shardNumber: this.shardNumber, + totalFlakyTests: flakyTests.length, + uniqueFlakyTests: flakyTests, + uniqueFailedTests: failedTests, + summary: { + flaky: { + p1: flakyTests.filter(t => t.severity === 'p1').length, + p2: flakyTests.filter(t => t.severity === 'p2').length, + p3: flakyTests.filter(t => t.severity === 'p3').length, + unclassified: flakyTests.filter(t => t.severity === 'unclassified').length, + }, + failed: { + p1: failedTests.filter(t => t.severity === 'p1').length, + p2: failedTests.filter(t => t.severity === 'p2').length, + p3: failedTests.filter(t => t.severity === 'p3').length, + unclassified: failedTests.filter(t => t.severity === 'unclassified').length, + }, + }, + }; + + const outputPath = path.resolve(OUTPUT_FOLDER, `flaky-test-shard-${this.shardNumber}.json`); + fs.writeFileSync(outputPath, JSON.stringify(report, null, 2)); + + console.log( + `✅ Shard ${this.shardNumber} JSON report written to ${outputPath}` + + ` (${flakyTests.length} flaky, ${failedTests.length} failed)` + ); + } +} + +export default FlakyTestReporter; + diff --git a/e2etests/reporting/metrics-reporter.ts b/e2etests/reporting/metrics-reporter.ts new file mode 100644 index 000000000..ce9afbe1a --- /dev/null +++ b/e2etests/reporting/metrics-reporter.ts @@ -0,0 +1,398 @@ +/** + * MetricsReporter - Playwright custom reporter for generating tag-based test metrics reports. + * + * Adapted from the generic MetricsReporter for the optscale e2e test framework. + * Outputs HTML and CSV reports locally — no external reporting API dependencies. + * + * Tag categories (based on actual tag usage across optscale e2e tests): + * - Priority tags: @p1, @p2, @p3, ... (regex: /^p\d+$/) + * - Speed tags: @slow + * - Module tags: @expenses, @homepage, @recommendations, @resources, @pools, + * @policies, @tagging-policies, @cloud-accounts, @events, + * @perspectives, @risp-coverage, @invitation-flow + * - Type tags: @ui, @devops + * - Other: anything that doesn't fit above + * + * Reports are saved in the 'reports' folder at the project root, with filenames + * including the current git branch and a timestamp. + * + * The reporter exits the process after report generation to prevent test execution. + * + * @module OptscaleMetricsReporter + */ + +import { Reporter, Suite, FullConfig } from '@playwright/test/reporter'; +import * as fs from 'fs'; +import * as path from 'path'; +import { execSync } from 'child_process'; +import { reportDebugLog, limitString } from '../utils/report-utils'; +import { logTagAttributes } from './tags-helper'; + +// --------------------------------------------------------------------------- +// Output configuration +// --------------------------------------------------------------------------- + +const OUTPUT_FOLDER = 'reports'; + +// --------------------------------------------------------------------------- +// Tag definitions — aligned with the optscale e2e test suite +// --------------------------------------------------------------------------- + +/** Priority tags — matched via regex rather than an explicit list */ +const priorityRegex = /^p\d+$/; + +/** Test-case ID tags — e.g. [231181] */ +const testCaseIdRegex = /^\[\d+]$/; + +/** + * Module tags — functional areas covered by the test suite. + * Sourced from actual @tags found in tests/** and regression-tests/**. + */ +export const TAGS_MODULE = [ + 'expenses', // Expenses page / cost analysis + 'homepage', // Home / dashboard page + 'recommendations', // Recommendations module + 'resources', // Resources management + 'pools', // Pools management + 'policies', // Policies (general) + 'tagging-policies', // Tagging policies + 'cloud-accounts', // Cloud account integration + 'events', // Events log + 'perspectives', // Perspectives / saved views + 'risp-coverage', // RISP coverage reports + 'invitation-flow', // Invitation / onboarding flow +]; + +/** Speed tags */ +export const TAGS_SPEED: string[] = ['slow']; + +/** Test type tags */ +export const TAGS_TYPE = ['ui', 'devops']; + +/** Debug / development-only tests */ +export const TAGS_DEBUG: string[] = ['debug']; + +// --------------------------------------------------------------------------- +// Timestamp & branch helpers +// --------------------------------------------------------------------------- + +function getTimestamp(): string { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + const hours = String(now.getHours()).padStart(2, '0'); + const minutes = String(now.getMinutes()).padStart(2, '0'); + const seconds = String(now.getSeconds()).padStart(2, '0'); + return `${year}${month}${day}-${hours}${minutes}${seconds}`; +} + +function getBranchName(): string { + try { + const branch = execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf-8' }).trim(); + return branch.replace(/\//g, '-').replace(/[^a-zA-Z0-9._-]/g, '_'); + } catch { + return 'unknown-branch'; + } +} + +const timestamp = getTimestamp(); +const branchName = getBranchName(); +const OUTPUT_FILE_HTML = `${OUTPUT_FOLDER}/tag-metrics-report-${branchName}-${timestamp}.html`; +const OUTPUT_FILE_CSV = `${OUTPUT_FOLDER}/tag-metrics-report-${branchName}-${timestamp}.csv`; + +// --------------------------------------------------------------------------- +// Tag sorting helpers +// --------------------------------------------------------------------------- + +/** + * Custom sort: priority first → speed → module → type → debug → other → test-case IDs last. + */ +function customSortPriorityFirstIdsLast(arr: string[]): string[] { + return [...arr].sort((a, b) => { + const rank = (tag: string): number => { + if (priorityRegex.test(tag)) return 0; + if (TAGS_SPEED.includes(tag)) return 1; + if (TAGS_MODULE.includes(tag)) return 2; + if (TAGS_TYPE.includes(tag)) return 3; + if (TAGS_DEBUG.includes(tag)) return 4; + if (testCaseIdRegex.test(tag)) return 9; + return 5; // other + }; + const diff = rank(a) - rank(b); + return diff !== 0 ? diff : a.localeCompare(b); + }); +} + +// --------------------------------------------------------------------------- +// Data model +// --------------------------------------------------------------------------- + +interface MetricsEntry { + /** Full test title (prefixed with describe block title when available). */ + title: string; + /** Cleaned tags (@ prefix stripped). */ + tags: string[]; +} + +// --------------------------------------------------------------------------- +// MetricsCollector +// --------------------------------------------------------------------------- + +class MetricsCollector { + private entries: MetricsEntry[] = []; + public maxNumberOfTags = 0; + + addEntry(entry: MetricsEntry): void { + entry.tags = entry.tags.map(tag => tag.replace(/^@/, '')); + this.entries.push(entry); + if (entry.tags.length > this.maxNumberOfTags) this.maxNumberOfTags = entry.tags.length; + } + + getAllEntries(): MetricsEntry[] { + return this.entries; + } + + extractEntriesPerTag(): Record { + const tagMap: Record = {}; + this.entries.forEach(entry => { + entry.tags.forEach(tag => { + if (!tagMap[tag]) tagMap[tag] = []; + tagMap[tag].push(entry); + }); + }); + return tagMap; + } + + extractTagsPerTitle(title: string): string[] { + return this.entries.find(e => e.title === title)?.tags ?? []; + } + + extractEntriesForTag(tag: string): MetricsEntry[] { + return this.entries.filter(entry => entry.tags.includes(tag)); + } + + getUniqueTags(): string[] { + return Object.keys(this.extractEntriesPerTag()); + } + + /** + * Returns the ordered list of category column names used in the CSV header. + */ + getCategorisedTagsSorted(): string[] { + const tags = customSortPriorityFirstIdsLast(this.getUniqueTags()); + const categories: string[] = []; + + for (const tag of tags) { + if (priorityRegex.test(tag)) { if (!categories.includes('Priority')) categories.push('Priority'); } + else if (TAGS_SPEED.includes(tag)) { if (!categories.includes('Speed')) categories.push('Speed'); } + else if (TAGS_MODULE.includes(tag)) { if (!categories.includes('Module')) categories.push('Module'); } + else if (TAGS_TYPE.includes(tag)) { if (!categories.includes('Type')) categories.push('Type'); } + else if (TAGS_DEBUG.includes(tag)) { if (!categories.includes('Debug')) categories.push('Debug'); } + } + + const hasOther = tags.some( + tag => + !priorityRegex.test(tag) && + !TAGS_SPEED.includes(tag) && + !TAGS_MODULE.includes(tag) && + !TAGS_TYPE.includes(tag) && + !TAGS_DEBUG.includes(tag) + ); + if (hasOther) categories.push('Other'); + + return categories; + } + + /** + * Returns a comma-delimited string of categorised tag values for a given test title. + * Column order: Priority, Speed, Module, Type, Debug, Other + */ + getEntriesCategorisedDelimited(entryTitle: string, delimiters: string[] = [',', ';']): string { + const entryTags = this.extractTagsPerTitle(entryTitle); + let priorityTag = ''; + let speedTag = ''; + const moduleTags: string[] = []; + let typeTag = ''; + let debugTag = ''; + const otherTags: string[] = []; + + for (const tag of entryTags) { + if (priorityRegex.test(tag)) priorityTag = tag; + else if (TAGS_SPEED.includes(tag)) speedTag = tag; + else if (TAGS_MODULE.includes(tag)) moduleTags.push(tag); + else if (TAGS_TYPE.includes(tag)) typeTag = tag; + else if (TAGS_DEBUG.includes(tag)) debugTag = tag; + else otherTags.push(tag); + } + + const del1 = delimiters[0]; + const del2 = delimiters[delimiters.length - 1]; + + return `${priorityTag}${del1}${speedTag}${del1}${moduleTags.join(del2)}${del1}${typeTag}${del1}${debugTag}${del1}${otherTags.join(del2)}`; + } +} + +// --------------------------------------------------------------------------- +// MetricsReporter +// --------------------------------------------------------------------------- + +class MetricsReporter implements Reporter { + private collector = new MetricsCollector(); + + onBegin(_config: FullConfig, suite: Suite): void { + reportDebugLog(`Ingesting ${suite.allTests().length} tests for metrics report`, 'info'); + + suite.allTests().forEach(test => { + const revisedTitle = + test.parent.type === 'describe' ? `${test.parent.title}: ${test.title}` : test.title; + reportDebugLog(`test: ${revisedTitle}, tags: ${test.tags}`, 'info'); + logTagAttributes(test.tags, revisedTitle); + this.collector.addEntry({ title: revisedTitle, tags: test.tags }); + }); + + // Ensure the output folder exists + const absoluteOutputFolder = path.resolve(OUTPUT_FOLDER); + if (!fs.existsSync(absoluteOutputFolder)) { + fs.mkdirSync(absoluteOutputFolder, { recursive: true }); + } + + // Write CSV report + const csvContent = this.generateTagsSummaryCsv(this.collector); + const csvPath = path.resolve(OUTPUT_FILE_CSV); + fs.writeFileSync(csvPath, csvContent); + reportDebugLog(`CSV report written to ${csvPath}`, 'info'); + + // Write HTML report + const htmlContent = this.generateHtmlContent(this.collector); + const htmlPath = path.resolve(OUTPUT_FILE_HTML); + fs.writeFileSync(htmlPath, htmlContent); + reportDebugLog(`HTML report written to ${htmlPath}`, 'info'); + + console.log(`\n✅ Metrics reports generated:`); + console.log(` HTML → ${htmlPath}`); + console.log(` CSV → ${csvPath}\n`); + + // Exit to prevent actual test execution + process.exit(0); + } + + // ------------------------------------------------------------------------- + // HTML generation + // ------------------------------------------------------------------------- + + private generateHtmlContent(collector: MetricsCollector): string { + const tags = Object.keys(collector.extractEntriesPerTag()); + const tagCounts: Record = {}; + const tagTests: Record = {}; + + tags.forEach(tag => { + tagCounts[tag] = collector.extractEntriesForTag(tag).length; + tagTests[tag] = collector.extractEntriesForTag(tag); + }); + + const rowEntriesTagCounts: string[] = []; + const rowEntriesTestCasesPerTag: string[] = []; + const rowEntriesTestCasesAll: string[] = []; + + tags.forEach(tag => { + // Exclude raw test-case ID tags from the counts summary + if (!testCaseIdRegex.test(tag)) { + rowEntriesTagCounts.push(`${tag}${tagCounts[tag]}`); + } + + rowEntriesTestCasesPerTag.push(`${tag}`); + for (const t of tagTests[tag]) { + rowEntriesTestCasesPerTag.push(`${t.title}${t.tags.join(', ')}`); + } + + for (const entry of collector.extractEntriesForTag(tag)) { + rowEntriesTestCasesAll.push(`${entry.title}${entry.tags.join(', ')}`); + } + }); + + return ` + + + + OptScale E2E Tag Metrics Report + + + +

OptScale E2E Tag Metrics Report — Branch: ${branchName} — Generated: ${timestamp}

+ + +
+ + + + ${rowEntriesTagCounts.join('\n ')} +
TagTest count
Total tests: ${collector.getAllEntries().length}  |  Unique tags: ${tags.length}
+
+ + +
+ + + ${rowEntriesTestCasesPerTag.join('\n ')} +
TestTags
+
+ + +
+ + + ${rowEntriesTestCasesAll.join('\n ')} +
TestTags
+
+ + + +`; + } + + // ------------------------------------------------------------------------- + // CSV generation + // ------------------------------------------------------------------------- + + private generateTagsSummaryCsv(collector: MetricsCollector): string { + const entries = collector.getAllEntries(); + const categories = collector.getCategorisedTagsSorted(); + + // Header: Title, Number of tags, + const header = `Title,Number of tags,${categories.join(',')}\n`; + + const rows = entries + .map(entry => `"${entry.title.replace(/"/g, '""')}",${entry.tags.length},${collector.getEntriesCategorisedDelimited(entry.title)}`) + .join('\n'); + + const csv = header + rows; + reportDebugLog(`CSV preview: ${limitString(csv, 255)}`, 'debug'); + return csv; + } +} + +export default MetricsReporter; + + diff --git a/e2etests/reporting/tags-helper.ts b/e2etests/reporting/tags-helper.ts new file mode 100644 index 000000000..5057d4149 --- /dev/null +++ b/e2etests/reporting/tags-helper.ts @@ -0,0 +1,195 @@ +/** + * OptScale Tags Helper + * + * Provides utilities to categorise Playwright test tags into structured attributes. + * Drop-in replacement for the ReportPortal-coupled tags helper — no external + * dependencies, works entirely locally. + * + * Tag Categories (based on actual tag usage across the optscale e2e suite): + * - Priority: p1, p2 (regex: /^p\d+$/) + * - Speed: fast, slow + * - Module: expenses, homepage, recommendations, resources, pools, + * policies, tagging-policies, cloud-accounts, events, + * perspectives, risp-coverage, invitation-flow, anomalies + * - Test Type: ui, devops, api + * - Debug: debug + * - Test Case ID: [230778] format extracted from test title or tag + * + * Tag Usage Statistics (from codebase analysis – Apr 2026): + * - @fast: majority of tests + * - @slow: perspective, invitation-flow, and selected resource/pool tests + * - @p1: highest-priority smoke tests + * - @p2: standard regression tests + * - @ui: all browser-based tests + * - @devops: devops-specific tests + * - @api: API-only tests + * - @expenses: Cost Explorer / Expenses page tests + * - @homepage: Home / Dashboard page tests + * - @recommendations: Recommendations module tests + * - @resources: Resources management tests + * - @pools: Pool management tests + * - @policies: Budget / quota policy tests + * - @tagging-policies: Tagging policy tests + * - @cloud-accounts: Cloud account integration tests + * - @events: Events log tests + * - @perspectives: Perspectives / saved-view tests + * - @risp-coverage: RI/SP coverage report tests + * - @invitation-flow: Invitation / onboarding flow tests + * - @anomalies: Anomaly detection tests + */ + +// --------------------------------------------------------------------------- +// Tag lists +// --------------------------------------------------------------------------- + +/** Module tags — functional areas of the optscale application */ +export const TAGS_MODULE = [ + 'expenses', // Cost Explorer / Expenses page + 'homepage', // Home / Dashboard page + 'recommendations', // Recommendations page + 'resources', // Resources management + 'pools', // Pool management + 'policies', // Budget / quota policies + 'tagging-policies', // Tagging policies + 'cloud-accounts', // Cloud account integrations + 'events', // Events log + 'perspectives', // Perspectives / saved views + 'risp-coverage', // RI/SP coverage reports + 'invitation-flow', // Invitation / onboarding flow + 'anomalies', // Anomaly detection +]; + +export const TAGS_SPEED: string[] = ['fast', 'slow']; +export const TAGS_TYPE: string[] = ['ui', 'devops', 'api']; +export const TAGS_DEBUG: string[] = ['debug']; + +// --------------------------------------------------------------------------- +// Regex helpers +// --------------------------------------------------------------------------- + +const priorityRegex = /^p\d+$/; +const testCaseIdRegex = /^\[(\d+)]$/; + +// --------------------------------------------------------------------------- +// Interfaces +// --------------------------------------------------------------------------- + +export interface TagAttributes { + priority?: string; + speed?: string; + module: string[]; + testType?: string; + debug?: boolean; + testCaseId?: string; + other: string[]; +} + +export interface TagAttribute { + key: string; + value: string; +} + +// --------------------------------------------------------------------------- +// Core functions +// --------------------------------------------------------------------------- + +/** + * Categorises a single tag (with or without the leading `@`) into its category. + */ +export function categorizeTag(tag: string): { category: string; value: string } { + const clean = tag.replace(/^@/, '').toLowerCase(); + + if (priorityRegex.test(clean)) return { category: 'priority', value: clean }; + if (TAGS_SPEED.includes(clean)) return { category: 'speed', value: clean }; + if (TAGS_MODULE.includes(clean)) return { category: 'module', value: clean }; + if (TAGS_TYPE.includes(clean)) return { category: 'testType', value: clean }; + if (TAGS_DEBUG.includes(clean)) return { category: 'debug', value: clean }; + + const testCaseMatch = clean.match(testCaseIdRegex); + if (testCaseMatch) return { category: 'testCaseId', value: testCaseMatch[1] }; + + return { category: 'other', value: clean }; +} + +/** + * Extracts and categorises all tags into a structured `TagAttributes` object. + */ +export function extractTagAttributes(tags: string[]): TagAttributes { + const attributes: TagAttributes = { + module: [], + other: [], + }; + + for (const tag of tags) { + const { category, value } = categorizeTag(tag); + + switch (category) { + case 'priority': attributes.priority = value; break; + case 'speed': attributes.speed = value; break; + case 'module': attributes.module.push(value); break; + case 'testType': attributes.testType = value; break; + case 'debug': attributes.debug = true; break; + case 'testCaseId': attributes.testCaseId = value; break; + default: attributes.other.push(value); break; + } + } + + return attributes; +} + +/** + * Converts `TagAttributes` into a flat key/value attribute list + * suitable for consumption by any reporter or logging utility. + */ +export function convertToAttributes(tagAttrs: TagAttributes): TagAttribute[] { + const attrs: TagAttribute[] = []; + + if (tagAttrs.priority) attrs.push({ key: 'priority', value: tagAttrs.priority }); + if (tagAttrs.speed) attrs.push({ key: 'speed', value: tagAttrs.speed }); + if (tagAttrs.testType) attrs.push({ key: 'testType', value: tagAttrs.testType }); + if (tagAttrs.testCaseId) attrs.push({ key: 'testCaseId', value: tagAttrs.testCaseId }); + if (tagAttrs.debug) attrs.push({ key: 'debug', value: 'true' }); + + for (const mod of tagAttrs.module) attrs.push({ key: 'module', value: mod }); + for (const other of tagAttrs.other) attrs.push({ key: 'tag', value: other }); + + return attrs; +} + +/** + * Convenience function: takes a raw tags array (and optionally a test title + * to extract a numeric test-case ID from), and returns the full structured + * attribute list. + * + * @example + * const attrs = buildTagAttributes(['@fast', '@p1', '@ui', '@expenses'], '[231181] Verify layout'); + * // => [{ key: 'priority', value: 'p1' }, { key: 'speed', value: 'fast' }, ...] + */ +export function buildTagAttributes(tags: string[], testTitle?: string): TagAttribute[] { + const tagsWithId = [...tags]; + + // Extract a numeric test-case ID from the title when not already present as a tag + if (testTitle) { + const match = testTitle.match(/\[(\d+)]/); + if (match && !tagsWithId.some(t => t.includes(match[1]))) { + tagsWithId.push(`[${match[1]}]`); + } + } + + return convertToAttributes(extractTagAttributes(tagsWithId)); +} + +/** + * Logs tag attributes to the console (useful for local debugging and CI log + * traces without any external service dependency). + * + * Output format: [TAG] priority=p1 | speed=fast | testType=ui | module=expenses + */ +export function logTagAttributes(tags: string[], testTitle?: string): void { + const attrs = buildTagAttributes(tags, testTitle); + if (attrs.length === 0) return; + + const formatted = attrs.map(a => `${a.key}=${a.value}`).join(' | '); + console.log(`[TAG] ${formatted}`); +} + diff --git a/e2etests/tests/anomalies-tests.spec.ts b/e2etests/tests/anomalies-tests.spec.ts index 85ebfc0d8..9e2e06194 100644 --- a/e2etests/tests/anomalies-tests.spec.ts +++ b/e2etests/tests/anomalies-tests.spec.ts @@ -30,7 +30,7 @@ test.describe('[MPT-14737] Anomalies Tests', { tag: ['@ui', '@anomalies'] }, () await anomaliesPage.navigateToURL(); }); - test('[231429] Anomalies page components', async ({ anomaliesPage }) => { + test('[231429] Anomalies page components', { tag: ['@fast', '@p2'] }, async ({ anomaliesPage }) => { await test.step('Verify page header components', async () => { await expect.soft(anomaliesPage.heading).toHaveText('Anomaly detection'); await expect.soft(anomaliesPage.addBtn).toBeVisible(); @@ -56,7 +56,7 @@ test.describe('[MPT-14737] Anomalies Tests', { tag: ['@ui', '@anomalies'] }, () }); }); - test('[231432] Verify navigation of link and show resources button', async ({ anomaliesPage, resourcesPage }) => { + test('[231432] Verify navigation of link and show resources button', { tag: ['@fast', '@p2'] }, async ({ anomaliesPage, resourcesPage }) => { await test.step('Navigate to policy details and verify values', async () => { await anomaliesPage.waitForAllProgressBarsToDisappear(); await anomaliesPage.click(anomaliesPage.defaultExpenseAnomalyLink); @@ -77,7 +77,7 @@ test.describe('[MPT-14737] Anomalies Tests', { tag: ['@ui', '@anomalies'] }, () test( '[231488] API responses matches expected structure for default anomaly detection policies', - { tag: '@p1' }, + { tag: ['@fast', '@p1'] }, async ({ anomaliesPage }) => { let anomalyData: DefaultAnomalyResponse; await test.step('Load expenses data', async () => { @@ -168,7 +168,7 @@ test.describe('[MPT-14737] Anomalies Tests', { tag: ['@ui', '@anomalies'] }, () } ); - test('[231431] Anomalies page search function', async ({ anomaliesPage }) => { + test('[231431] Anomalies page search function', { tag: ['@fast', '@p2'] }, async ({ anomaliesPage }) => { await test.step('Search by "expense" shows only expense anomaly', async () => { await anomaliesPage.searchAnomaly('expense'); await expect.soft(anomaliesPage.defaultExpenseAnomalyLink).toBeVisible(); @@ -194,7 +194,7 @@ test.describe('[MPT-14737] Anomalies Tests', { tag: ['@ui', '@anomalies'] }, () }); }); - test('[231433] Add a resource count anomaly detection policy', { tag: '@p1' }, async ({ anomaliesPage, anomaliesCreatePage }) => { + test('[231433] Add a resource count anomaly detection policy', { tag: ['@fast', '@p1'] }, async ({ anomaliesPage, anomaliesCreatePage }) => { const policyName = `E2E Test - Resource Count Anomaly - ${Date.now()}`; await test.step('Create a new resource count anomaly policy', async () => { @@ -212,7 +212,7 @@ test.describe('[MPT-14737] Anomalies Tests', { tag: ['@ui', '@anomalies'] }, () }); }); - test('[231434] Add an expenses anomaly detection policy with filter', async ({ anomaliesPage, anomaliesCreatePage }) => { + test('[231434] Add an expenses anomaly detection policy with filter', { tag: ['@fast', '@p2'] }, async ({ anomaliesPage, anomaliesCreatePage }) => { const policyName = `E2E Test - Expense Anomaly - ${Date.now()}`; await test.step('Create a new expenses anomaly policy with a filter', async () => { @@ -239,7 +239,7 @@ test.describe('[MPT-14737] Anomalies Tests', { tag: ['@ui', '@anomalies'] }, () }); }); - test('[231441] Verify delete policy functions correctly', async ({ anomaliesPage, anomaliesCreatePage }) => { + test('[231441] Verify delete policy functions correctly', { tag: ['@fast', '@p2'] }, async ({ anomaliesPage, anomaliesCreatePage }) => { const policyName = `E2E Test - Delete Anomaly Policy - ${Date.now()}`; await test.step('Create a new anomaly policy', async () => { @@ -316,7 +316,7 @@ test.describe('[MPT-14737] Mocked Anomalies Tests', { tag: ['@ui', '@anomalies'] interceptAPI: { entries: apiInterceptions, failOnInterceptionMissing: false }, }); - test('[231435] Verify Chart export for each category by comparing downloaded png', async ({ anomaliesPage }) => { + test('[231435] Verify Chart export for each category by comparing downloaded png', { tag: ['@fast', '@p2'] }, async ({ anomaliesPage }) => { test.fixme(process.env.CI === '1', 'Tests do not work in CI. It appears that the png comparison is unsupported on linux'); let actualPath = path.resolve('tests', 'downloads', 'anomaly-expenses-region-daily-chart-export.png'); let expectedPath = path.resolve('tests', 'expected', 'expected-anomaly-expenses-region-daily-chart-export.png'); @@ -382,7 +382,7 @@ test.describe('[MPT-14737] Mocked Anomalies Tests', { tag: ['@ui', '@anomalies'] }); }); - test('[231436] Verify Chart export for each expenses option by comparing downloaded png', async ({ anomaliesPage }) => { + test('[231436] Verify Chart export for each expenses option by comparing downloaded png', { tag: ['@fast', '@p2'] }, async ({ anomaliesPage }) => { test.fixme(process.env.CI === '1', 'Tests do not work in CI. It appears that the png comparison is unsupported on linux'); let actualPath: string; let expectedPath: string; @@ -440,7 +440,7 @@ test.describe('[MPT-14737] Mocked Anomalies Tests', { tag: ['@ui', '@anomalies'] }); }); - test('[231439] Verify detected anomalies are displayed in the table correctly', async ({ anomaliesPage }) => { + test('[231439] Verify detected anomalies are displayed in the table correctly', { tag: ['@fast', '@p2'] }, async ({ anomaliesPage }) => { await anomaliesPage.page.clock.setFixedTime(new Date('2025-11-11T14:11:00Z')); await anomaliesPage.navigateToURL(); @@ -460,7 +460,7 @@ test.describe('[MPT-14737] Mocked Anomalies Tests', { tag: ['@ui', '@anomalies'] expect.soft(await anomaliesPage.getAverageActualExpensesByIndex(4)).toBe('$10,737 ⟶ $43,279.5 (303%)'); }); - test('[231440] Verify detected anomalies are resource button navigate correctly', async ({ anomaliesPage, resourcesPage }) => { + test('[231440] Verify detected anomalies are resource button navigate correctly', { tag: ['@fast', '@p2'] }, async ({ anomaliesPage, resourcesPage }) => { await anomaliesPage.page.clock.setFixedTime(new Date('2025-11-11T14:11:00Z')); await anomaliesPage.navigateToURL(); diff --git a/e2etests/tests/auth-api-tests.spec.ts b/e2etests/tests/auth-api-tests.spec.ts index db3520ebe..766bc27fe 100644 --- a/e2etests/tests/auth-api-tests.spec.ts +++ b/e2etests/tests/auth-api-tests.spec.ts @@ -8,7 +8,7 @@ test.describe('Auth API tests @api_tests', { tag: '@api' }, () => { const password = process.env.DEFAULT_USER_PASSWORD; const userId = process.env.DEFAULT_AUTH_USER_ID; - test('Set verification codes', async ({ authRequest }) => { + test('Set verification codes', { tag: ['@fast', '@p2'] }, async ({ authRequest }) => { const response = await authRequest.setVerificationCode(email, '123456'); await test.step('Verify response status and payload fields', async () => { @@ -16,7 +16,7 @@ test.describe('Auth API tests @api_tests', { tag: '@api' }, () => { }); }); - test('Authorize user payload', async ({ authRequest }) => { + test('Authorize user payload', { tag: ['@fast', '@p2'] }, async ({ authRequest }) => { const response = await authRequest.authorization(email, password); const payload = JSON.parse(await response.text()) as AuthResponse; @@ -39,23 +39,23 @@ test.describe('Auth API tests @api_tests', { tag: '@api' }, () => { }); }); - test('Authorize user with invalid credentials', async ({ authRequest }) => { + test('Authorize user with invalid credentials', { tag: ['@fast', '@p2'] }, async ({ authRequest }) => { const response = await authRequest.authorization(email, 'invalidPassword'); expect(response.status()).toBe(403); }); - test('Authorize user with invalid email', async ({ authRequest }) => { + test('Authorize user with invalid email', { tag: ['@fast', '@p2'] }, async ({ authRequest }) => { const invalidEmail = generateRandomEmail(); const response = await authRequest.authorization(invalidEmail, password); expect(response.status()).toBe(403); }); - test('Authorize user with empty email', async ({ authRequest }) => { + test('Authorize user with empty email', { tag: ['@fast', '@p2'] }, async ({ authRequest }) => { const response = await authRequest.authorization('', password); expect(response.status()).toBe(400); }); - test('Get users with Cluster Secret', async ({ authRequest }) => { + test('Get users with Cluster Secret', { tag: ['@fast', '@p2'] }, async ({ authRequest }) => { const email = process.env.DEFAULT_USER_EMAIL.toLowerCase(); const response = await authRequest.getUsersWithClusterSecret(userId); const payload = JSON.parse(await response.text()) as UsersResponse; @@ -80,17 +80,17 @@ test.describe('Auth API tests @api_tests', { tag: '@api' }, () => { } }); - test('Get user with Cluster Secret but no user ID', async ({ authRequest }) => { + test('Get user with Cluster Secret but no user ID', { tag: ['@fast', '@p2'] }, async ({ authRequest }) => { const response = await authRequest.getUsersWithClusterSecret(); expect(response.status()).toBe(401); }); - test('Get user with Cluster Secret and invalid user ID', async ({ authRequest }) => { + test('Get user with Cluster Secret and invalid user ID', { tag: ['@fast', '@p2'] }, async ({ authRequest }) => { const response = await authRequest.getUsersWithClusterSecret('invalidUserID'); expect(response.status()).toBe(404); }); - test('Get users with bad cluster secret', async ({ authRequest }) => { + test('Get users with bad cluster secret', { tag: ['@fast', '@p2'] }, async ({ authRequest }) => { const response = await authRequest.getUsersWithBadClusterSecret(userId); expect(response.status()).toBe(403); }); diff --git a/e2etests/tests/cloud-accounts-tests.spec.ts b/e2etests/tests/cloud-accounts-tests.spec.ts index 4392efeab..473041849 100644 --- a/e2etests/tests/cloud-accounts-tests.spec.ts +++ b/e2etests/tests/cloud-accounts-tests.spec.ts @@ -18,7 +18,7 @@ import { } from '../mocks/cloud-accounts-page.mocks'; import { getCurrentUTCTimestamp, getTimestampWithVariance } from '../utils/date-range-utils'; -test.describe('Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] }, () => { +test.describe('[MPT-14561] Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] }, () => { test.fixme(); test.describe.configure({ mode: 'serial' }); test.use({ restoreSession: true }); @@ -27,7 +27,7 @@ test.describe('Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] }, () => // the test datasource that we can configure without external dependencies. test.fixme( '[231860] A successful billing import should have been successful within the last 24 hours', - { tag: '@p1' }, + { tag: ['@fast', '@p1'] }, async ({ page, cloudAccountsPage }) => { let dataSourceResponse: DataSourceBillingResponse; const now = Math.floor(Date.now() / 1000); @@ -57,7 +57,7 @@ test.describe('Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] }, () => } ); - test('[231861] Verify adding a new AWS Assumed role - Management', async ({ cloudAccountsPage, cloudAccountsConnectPage }) => { + test('[231861] Verify adding a new AWS Assumed role - Management', { tag: ['@fast', '@p2'] }, async ({ cloudAccountsPage, cloudAccountsConnectPage }) => { test.fixme(); //'Skipping due to these tests possibly corrupting data due to orphaned sub-pools when disconnecting accounts' await cloudAccountsPage.navigateToCloudAccountsPage(); const awsAccountName = 'Marketplace (Dev)'; @@ -80,7 +80,7 @@ test.describe('Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] }, () => test( '[231862] Verify adding a new AWS Assumed role - Member', - { tag: '@p1' }, + { tag: ['@fast', '@p1'] }, async ({ cloudAccountsPage, cloudAccountsConnectPage }) => { test.fixme(); //'Skipping due to these tests possibly corrupting data due to orphaned sub-pools when disconnecting accounts' await cloudAccountsPage.navigateToCloudAccountsPage(); @@ -103,7 +103,7 @@ test.describe('Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] }, () => } ); - test('[231863] Verify adding a new AWS Assumed role - Standalone', async ({ cloudAccountsPage, cloudAccountsConnectPage }) => { + test('[231863] Verify adding a new AWS Assumed role - Standalone', { tag: ['@fast', '@p2'] }, async ({ cloudAccountsPage, cloudAccountsConnectPage }) => { test.fixme(); //'Skipping due to these tests possibly corrupting data due to orphaned sub-pools when disconnecting accounts' await cloudAccountsPage.navigateToCloudAccountsPage(); const awsAccountName = 'Marketplace (Dev)'; @@ -124,7 +124,7 @@ test.describe('Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] }, () => }); }); - test('[232861] Verify that a message is displayed recommending the Assume role method for AWS accounts, when Access key method is selected', async ({ + test('[232861] Verify that a message is displayed recommending the Assume role method for AWS accounts, when Access key method is selected', { tag: ['@fast', '@p2'] }, async ({ cloudAccountsPage, cloudAccountsConnectPage, }) => { @@ -145,7 +145,7 @@ test.describe('Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] }, () => }); }); - test('[232862] Verify that the user can schedule a billing reimport, and see warning alert', async ({ cloudAccountsPage }) => { + test('[232862] Verify that the user can schedule a billing reimport, and see warning alert', { tag: ['@fast', '@p2'] }, async ({ cloudAccountsPage }) => { const expectedAlertMessage = 'Reimporting billing starting from the selected import date will overwrite existing billing data. This action may cause discrepancies or breaks in the current billing records and can take some time to complete. The new billing data will be imported during the next billing import report processing. Please proceed with caution, as this process cannot be undone. Ensure that this action is necessary and that you are prepared for any potential data loss and inaccuracies in billing tracking.'; @@ -176,7 +176,7 @@ test.describe('Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] }, () => }); }); -test.describe('Mocked Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] }, () => { +test.describe('[MPT-14561] Mocked Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] }, () => { test.fixme(); //'Skipping due to these tests possibly corrupting data due to orphaned sub-pools when disconnecting accounts' test.describe.configure({ mode: 'serial' }); @@ -222,7 +222,7 @@ test.describe('Mocked Cloud Accounts Tests', { tag: ['@ui', '@cloud-accounts'] } ]; test.use({ restoreSession: true, interceptAPI: { entries: apiInterceptions, failOnInterceptionMissing: true } }); - test('[232859] Verify the correct messages are displayed when updating an AWS Access Key account', async ({ cloudAccountsPage }) => { + test('[232859] Verify the correct messages are displayed when updating an AWS Access Key account', { tag: ['@fast', '@p2'] }, async ({ cloudAccountsPage }) => { const accessKeyMessage = 'Access keys are a set of permanent credentials. This authentication type is not recommended by SoftwareOne or AWS - use an assumed role where possible.More information'; const permissionsMessage = @@ -263,7 +263,7 @@ test.describe( test.fixme(); //'Skipping due to these tests possibly corrupting data due to orphaned sub-pools when disconnecting accounts' test.use({ restoreSession: true }); - test('[232954] Verify that disconnecting and creating a cloud account is recorded in the events log', async ({ + test('[232954] Verify that disconnecting and creating a cloud account is recorded in the events log', { tag: ['@fast', '@p2'] }, async ({ cloudAccountsPage, cloudAccountsConnectPage, eventsPage, diff --git a/e2etests/tests/expenses-tests.spec.ts b/e2etests/tests/expenses-tests.spec.ts index 626c152b1..658dec1dd 100644 --- a/e2etests/tests/expenses-tests.spec.ts +++ b/e2etests/tests/expenses-tests.spec.ts @@ -44,7 +44,7 @@ test.describe('[MPT-12859] Expenses Page default view Tests', { tag: ['@ui', '@e } }); - test('[231181] Verify default Expenses Page layout', async ({ expensesPage }) => { + test('[231181] Verify default Expenses Page layout', { tag: ['@fast', '@p2'] }, async ({ expensesPage }) => { await expect.soft(expensesPage.downloadButton).toBeVisible(); expect.soft(dateRangeReset).toBe(false); expect.soft(await expensesPage.evaluateActiveButton(expensesPage.dailyBtn)).toBe(true); @@ -55,7 +55,7 @@ test.describe('[MPT-12859] Expenses Page default view Tests', { tag: ['@ui', '@e await expect(expensesPage.geographyBtn).toBeVisible(); }); - test('Validate API default chart data', { tag: '@p1' }, async ({ expensesPage }) => { + test('[231182] Validate API default chart data', { tag: ['@fast', '@p1'] }, async ({ expensesPage }) => { //if it's the first day of the month, the API will return 0 expenses for the current month, so we need to get the date range for last 7 days to validate the data const { startDate, endDate } = currentDate.getDate() === 1 ? getLast7DaysUnixRange() : getThisMonthUnixDateRange(); let expensesData: ExpensesResponse; @@ -96,7 +96,7 @@ test.describe('[MPT-12859] Expenses Page default view Tests', { tag: ['@ui', '@e }); }); - test('[231212] Breakdown by Geography button navigates to Cost map page', async ({ expensesPage, expensesMapPage }) => { + test('[231212] Breakdown by Geography button navigates to Cost map page', { tag: ['@fast', '@p2'] }, async ({ expensesPage, expensesMapPage }) => { await expensesPage.geographyBtn.click(); await expect(expensesMapPage.heading).toBeVisible(); }); @@ -127,7 +127,7 @@ test.describe('[MPT-12859] Expenses page default view mocked tests', { tag: ['@u await expensesPage.clickDailyBtnIfNotSelected(); }); }); - test('[231183] Verify expenses chart download', { tag: '@p1' }, async ({ expensesPage }) => { + test('[231183] Verify expenses chart download', { tag: ['@fast', '@p1'] }, async ({ expensesPage }) => { let actualPath = path.resolve('tests', 'downloads', 'expenses-page-daily-chart.pdf'); let expectedPath = path.resolve('tests', 'expected', 'expected-expenses-page-daily-chart.pdf'); let diffPath = path.resolve('tests', 'downloads', 'expenses-page-daily-chart-diff.png'); @@ -190,7 +190,7 @@ test.describe('[MPT-12859] Expenses Page Source Breakdown Tests', { tag: ['@ui', await expensesPage.waitForCanvas(); }); - test('[231214] Verify Expenses Page Source Breakdown layout', async ({ expensesPage, datePicker }) => { + test('[231214] Verify Expenses Page Source Breakdown layout', { tag: ['@fast', '@p2'] }, async ({ expensesPage, datePicker }) => { await test.step('Verify Expenses Page Source Breakdown elements', async () => { await expect(expensesPage.downloadButton).toBeHidden(); await expect(expensesPage.seeExpensesBreakdownGrid).toBeHidden(); @@ -207,7 +207,7 @@ test.describe('[MPT-12859] Expenses Page Source Breakdown Tests', { tag: ['@ui', }); }); - test('[231215] Validate API Source Breakdown chart data', async ({ expensesPage }) => { + test('[231215] Validate API Source Breakdown chart data', { tag: ['@fast', '@p2'] }, async ({ expensesPage }) => { //if it's the first day of the month, the API will return 0 expenses for the current month, so we need to get the date range for last 7 days to validate the data const { startDate, endDate } = currentDate.getDate() === 1 ? getLast7DaysUnixRange() : getThisMonthUnixDateRange(); let expensesData: ExpensesFilterByDataSourceResponse; @@ -273,7 +273,7 @@ test.describe('[MPT-12859] Expenses Page Source Breakdown Tests', { tag: ['@ui', test( '[231216] Verify data source expenses total for(default) period matches chart and table totals', - { tag: '@p1' }, + { tag: ['@fast', '@p1'] }, async ({ expensesPage }) => { const totalForPeriod = await expensesPage.getTotalExpensesForSelectedPeriod(); debugLog(`Total expenses for selected period: ${totalForPeriod}`); @@ -289,7 +289,7 @@ test.describe('[MPT-12859] Expenses Page Source Breakdown Tests', { tag: ['@ui', } ); - test('[231217] Verify data source expenses total for (last month) period matches chart and table totals', async ({ + test('[231217] Verify data source expenses total for (last month) period matches chart and table totals', { tag: ['@fast', '@p2'] }, async ({ expensesPage, datePicker, }) => { @@ -341,7 +341,7 @@ test.describe('[MPT-12859] Expenses Page Pool Breakdown Tests', { tag: ['@ui', ' await expensesPage.waitForCanvas(); }); - test('[231218] Verify Expenses Page Pool Breakdown layout', async ({ expensesPage, datePicker }) => { + test('[231218] Verify Expenses Page Pool Breakdown layout', { tag: ['@fast', '@p2'] }, async ({ expensesPage, datePicker }) => { await test.step('Verify Expenses Page Pool Breakdown elements', async () => { await expect(expensesPage.downloadButton).toBeHidden(); await expect(expensesPage.seeExpensesBreakdownGrid).toBeHidden(); @@ -358,7 +358,7 @@ test.describe('[MPT-12859] Expenses Page Pool Breakdown Tests', { tag: ['@ui', ' }); }); - test('[231219] Validate API Pool Breakdown chart data', { tag: '@p1' }, async ({ expensesPage }) => { + test('[231219] Validate API Pool Breakdown chart data', { tag: ['@fast', '@p1'] }, async ({ expensesPage }) => { //if it's the first day of the month, the API will return 0 expenses for the current month, so we need to get the date range for last 7 days to validate the data const { startDate, endDate } = currentDate.getDate() === 1 ? getLast7DaysUnixRange() : getThisMonthUnixDateRange(); let expensesData: ExpensesFilterByPoolResponse; @@ -412,7 +412,7 @@ test.describe('[MPT-12859] Expenses Page Pool Breakdown Tests', { tag: ['@ui', ' }); }); - test('[231220] Verify pool expenses total for(default) period matches chart and table totals', async ({ expensesPage }) => { + test('[231220] Verify pool expenses total for(default) period matches chart and table totals', { tag: ['@fast', '@p2'] }, async ({ expensesPage }) => { const totalForPeriod = await expensesPage.getTotalExpensesForSelectedPeriod(); debugLog(`Total expenses for selected period: ${totalForPeriod}`); const chartTotal = await expensesPage.getExpensesPieChartValue(); @@ -426,7 +426,7 @@ test.describe('[MPT-12859] Expenses Page Pool Breakdown Tests', { tag: ['@ui', ' }); }); - test('[231221] Verify pool expenses total for (last 7 days) period matches chart and table totals', async ({ + test('[231221] Verify pool expenses total for (last 7 days) period matches chart and table totals', { tag: ['@fast', '@p2'] }, async ({ expensesPage, datePicker, }) => { @@ -477,7 +477,7 @@ test.describe('[MPT-12859] Expenses Page Owner Breakdown Tests', { tag: ['@ui', await expensesPage.waitForCanvas(); }); - test('[231222] Verify Expenses Page Owner Breakdown layout', async ({ expensesPage, datePicker }) => { + test('[231222] Verify Expenses Page Owner Breakdown layout', { tag: ['@fast', '@p2'] }, async ({ expensesPage, datePicker }) => { await test.step('Verify Expenses Page Owner Breakdown elements', async () => { await expect(expensesPage.downloadButton).toBeHidden(); await expect(expensesPage.seeExpensesBreakdownGrid).toBeHidden(); @@ -494,7 +494,7 @@ test.describe('[MPT-12859] Expenses Page Owner Breakdown Tests', { tag: ['@ui', }); }); - test('[231223] Validate API Owner Breakdown chart data', async ({ expensesPage }) => { + test('[231223] Validate API Owner Breakdown chart data', { tag: ['@fast', '@p2'] }, async ({ expensesPage }) => { //if it's the first day of the month, the API will return 0 expenses for the current month, so we need to get the date range for last 7 days to validate the data const { startDate, endDate } = currentDate.getDate() === 1 ? getLast7DaysUnixRange() : getThisMonthUnixDateRange(); let expensesData: ExpensesFilterByEmployeeResponse; @@ -547,7 +547,7 @@ test.describe('[MPT-12859] Expenses Page Owner Breakdown Tests', { tag: ['@ui', }); }); - test('[231224] Verify owner expenses total for(default) period matches chart and table totals', async ({ expensesPage }) => { + test('[231224] Verify owner expenses total for(default) period matches chart and table totals', { tag: ['@fast', '@p2'] }, async ({ expensesPage }) => { const totalForPeriod = await expensesPage.getTotalExpensesForSelectedPeriod(); debugLog(`Total expenses for selected period: ${totalForPeriod}`); const chartTotal = await expensesPage.getExpensesPieChartValue(); @@ -561,7 +561,7 @@ test.describe('[MPT-12859] Expenses Page Owner Breakdown Tests', { tag: ['@ui', }); }); - test('[231225] Verify owner expenses total for (last 30 days) period matches chart and table totals', async ({ + test('[231225] Verify owner expenses total for (last 30 days) period matches chart and table totals', { tag: ['@fast', '@p2'] }, async ({ expensesPage, datePicker, }) => { diff --git a/e2etests/tests/homepage-tests.spec.ts b/e2etests/tests/homepage-tests.spec.ts index b70b865e7..1129b878f 100644 --- a/e2etests/tests/homepage-tests.spec.ts +++ b/e2etests/tests/homepage-tests.spec.ts @@ -19,7 +19,7 @@ test.describe('[MPT-11464] Home Page Recommendations block tests', { tag: ['@ui' test( '[230550] Compare possible savings on home page with those on recommendations page', - { tag: '@p1' }, + { tag: ['@fast', '@p1'] }, async ({ homePage, recommendationsPage }) => { const homePageValue = await homePage.getRecommendationsPossibleSavingsValue(); await homePage.recommendationsBtn.click(); @@ -28,7 +28,7 @@ test.describe('[MPT-11464] Home Page Recommendations block tests', { tag: ['@ui' } ); - test('[230551] Verify Cost items displayed in the recommendations block match the sum total of items displayed on cards with savings', async ({ + test('[230551] Verify Cost items displayed in the recommendations block match the sum total of items displayed on cards with savings', { tag: ['@fast', '@p2'] }, async ({ homePage, recommendationsPage, }) => { @@ -38,7 +38,7 @@ test.describe('[MPT-11464] Home Page Recommendations block tests', { tag: ['@ui' expect.soft(await recommendationsPage.getTotalSumOfItemsFromSeeItemsButtons()).toBe(homePageValue); }); - test('[230552] Verify Security items displayed in the recommendations block match the sum total of items displayed on cards in the security category', async ({ + test('[230552] Verify Security items displayed in the recommendations block match the sum total of items displayed on cards in the security category', { tag: ['@fast', '@p2'] }, async ({ homePage, recommendationsPage, }) => { @@ -49,7 +49,7 @@ test.describe('[MPT-11464] Home Page Recommendations block tests', { tag: ['@ui' }); // Test failing due to bug MPT-11558 The home page recommendations block not returning the real Critical item count - test('[230553] Verify Critical items displayed in the recommendations block match the sum total of items displayed on cards with the critical status', async ({ + test('[230553] Verify Critical items displayed in the recommendations block match the sum total of items displayed on cards with the critical status', { tag: ['@fast', '@p2'] }, async ({ homePage, recommendationsPage, }) => { @@ -73,7 +73,7 @@ test.describe('[MPT-11958] Home Page Resource block tests', { tag: ['@ui', '@res }); }); - test('[230838] Verify Top Resource block Resource link works correctly', async ({ homePage, resourcesPage }) => { + test('[230838] Verify Top Resource block Resource link works correctly', { tag: ['@fast', '@p2'] }, async ({ homePage, resourcesPage }) => { await test.step('Click on Top Resources button', async () => { await homePage.clickTopResourcesBtn(); await expect.soft(resourcesPage.heading).toBeVisible(); @@ -82,7 +82,7 @@ test.describe('[MPT-11958] Home Page Resource block tests', { tag: ['@ui', '@res test( '[230839] Verify top Resource link navigates to the correct resource details page and last 30 days value match', - { tag: '@p1' }, + { tag: ['@fast', '@p1'] }, async ({ homePage, resourceDetailsPage, datePicker }) => { let homepageResourceTitle: string; let homePageExpenseValue: number; @@ -113,7 +113,7 @@ test.describe('[MPT-11958] Home Page Resource block tests', { tag: ['@ui', '@res } ); - test('[230842] Verify Top Resource Block displayed correctly', async ({ homePage }) => { + test('[230842] Verify Top Resource Block displayed correctly', { tag: ['@fast', '@p2'] }, async ({ homePage }) => { await test.step('Verify that the Top Resources section is displayed with 6 or fewer resources and include names for each', async () => { const count = await homePage.topResourcesAllLinks.count(); expect.soft(count).toBeLessThanOrEqual(6); @@ -131,7 +131,7 @@ test.describe('[MPT-12743] Home Page test for Pools requiring attention block', test.slow(); test.describe.configure({ mode: 'serial' }); //Tests in this describe block are state dependent, so they should not run in parallel with pools tests. - test('[230921] Verify Pools requiring attention block is displayed and link navigates to the pools page', async ({ + test('[230921] Verify Pools requiring attention block is displayed and link navigates to the pools page', { tag: '@p2' }, async ({ homePage, poolsPage, }) => { @@ -147,7 +147,7 @@ test.describe('[MPT-12743] Home Page test for Pools requiring attention block', }); }); - test('[230922] Verify that Pools Requiring attention is empty when the are no qualifying pools', async ({ + test('[230922] Verify that Pools Requiring attention is empty when the are no qualifying pools', { tag: '@p2' }, async ({ homePage, poolsPage, mainMenu, @@ -212,7 +212,7 @@ test.describe('[MPT-12743] Home Page test for Pools requiring attention block', } ); - test('[230924] Verify that Pools Requiring attention shows Pool and Sub-pools that are forecasted to overspend', async ({ + test('[230924] Verify that Pools Requiring attention shows Pool and Sub-pools that are forecasted to overspend', { tag: '@p2' }, async ({ homePage, poolsPage, mainMenu, @@ -260,7 +260,7 @@ test.describe('[MPT-18353] Home Page test for Policy Violation block', { tag: [' interceptAPI: { entries: apiInterceptions, failOnInterceptionMissing: true }, }); - test('[232876] Verify that Policy Violation block displays policy violations correctly', async ({ homePage }) => { + test('[232876] Verify that Policy Violation block displays policy violations correctly', { tag: ['@fast', '@p2'] }, async ({ homePage }) => { await test.step('Navigate to home page', async () => { await homePage.page.clock.setFixedTime(new Date('2026-02-24T11:00:00Z')); await homePage.navigateToURL(); diff --git a/e2etests/tests/invitation-flow-tests.spec.ts b/e2etests/tests/invitation-flow-tests.spec.ts index 19e008a80..9457dcd30 100644 --- a/e2etests/tests/invitation-flow-tests.spec.ts +++ b/e2etests/tests/invitation-flow-tests.spec.ts @@ -10,7 +10,7 @@ const verificationCode = '123456'; let invitationEmail: string; let inviteLink: string; -test.describe('MPT-8230 Invitation Flow Tests for new users', { tag: ['@invitation-flow', '@ui', '@slow'] }, () => { +test.describe('[MPT-8230] Invitation Flow Tests for new users', { tag: ['@invitation-flow', '@ui', '@slow'] }, () => { test.skip(process.env.USE_LIVE_DEMO === 'true', 'Live demo environment does not support invitation flow tests'); test.describe.configure({ mode: 'parallel' }); @@ -22,9 +22,10 @@ test.describe('MPT-8230 Invitation Flow Tests for new users', { tag: ['@invitati await loginPage.waitForLoadingPageImgToDisappear(); }); + // [229865] already has @p1 — the describe has @slow so no @fast test( '[229865] Invite new user to organisation, user accepts', - { tag: '@p1' }, + { tag: ['@p1'] }, async ({ header, mainMenu, usersPage, usersInvitePage, registerPage, pendingInvitationsPage, emailVerificationPage, baseRequest }) => { test.slow(); @@ -69,7 +70,7 @@ test.describe('MPT-8230 Invitation Flow Tests for new users', { tag: ['@invitati } ); - test('[229866] Invite new user to organisation, but user declines', async ({ + test('[229866] Invite new user to organisation, but user declines', { tag: '@p2' }, async ({ header, mainMenu, usersPage, @@ -117,7 +118,7 @@ test.describe('MPT-8230 Invitation Flow Tests for new users', { tag: ['@invitati }); }); - test('[229867] Invite new user to organisation, who has previously declined @slow', async ({ + test('[229867] Invite new user to organisation, who has previously declined @slow', { tag: '@p2' }, async ({ header, loginPage, mainMenu, @@ -207,7 +208,7 @@ test.describe('MPT-8230 Invitation Flow Tests for new users', { tag: ['@invitati test.describe('MPT-8229 Validate invitations in the settings', { tag: ['@invitation-flow', '@ui', '@slow'] }, () => { test.skip(process.env.USE_LIVE_DEMO === 'true', 'Live demo environment does not support invitation flow tests'); - test('[229868] Invitation is visible in Settings Tab @slow', async ({ + test('[229868] Invitation is visible in Settings Tab @slow', { tag: '@p2' }, async ({ loginPage, header, mainMenu, @@ -304,7 +305,7 @@ test.describe('MPT-8229 Validate invitations in the settings', { tag: ['@invitat test.describe('MPT-8231 Invitation Flow Tests for an existing user', { tag: ['@invitation-flow', '@ui', '@slow'] }, () => { test.skip(process.env.USE_LIVE_DEMO === 'true', 'Live demo environment does not support invitation flow tests'); - test('[229869] Invite existing user with a new role @slow', async ({ + test('[229869] Invite existing user with a new role @slow', { tag: '@p2' }, async ({ header, loginPage, mainMenu, @@ -398,7 +399,8 @@ test.describe( () => { test.use({ restoreSession: true }); - test('[232868] Invite a new user and verify the event is logged', async ({ mainMenu, usersPage, usersInvitePage, eventsPage }) => { + // [MPT-18378] has no @slow — tests here get @fast + test('[232868] Invite a new user and verify the event is logged', { tag: ['@fast', '@p2'] }, async ({ mainMenu, usersPage, usersInvitePage, eventsPage }) => { invitationEmail = generateRandomEmail(); let date: string; diff --git a/e2etests/tests/perspective-tests.spec.ts b/e2etests/tests/perspective-tests.spec.ts index c8d1e3a5d..8bf2ab2fc 100644 --- a/e2etests/tests/perspective-tests.spec.ts +++ b/e2etests/tests/perspective-tests.spec.ts @@ -9,7 +9,7 @@ test.describe('[MPT-18579] Perspective Tests', { tag: ['@ui', '@resources', '@pe test.use({ restoreSession: true }); test.slow(); - test('[232963] User can create an Expenses perspective and the chart options are saved and applied correctly', async ({ + test('[232963] User can create an Expenses perspective and the chart options are saved and applied correctly', { tag: '@p2' }, async ({ resourcesPage, perspectivesPage, }) => { @@ -79,7 +79,7 @@ test.describe('[MPT-18579] Perspective Tests', { tag: ['@ui', '@resources', '@pe }); }); - test('[232964] User can create perspective for resource count and the perspective is saved and applied correctly', async ({ + test('[232964] User can create perspective for resource count and the perspective is saved and applied correctly', { tag: '@p2' }, async ({ resourcesPage, }) => { await resourcesPage.navigateToResourcesPageAndResetFilters(); @@ -124,7 +124,7 @@ test.describe('[MPT-18579] Perspective Tests', { tag: ['@ui', '@resources', '@pe }); }); - test('[232965] User can create a perspective a Tags chart is saved and applied correctly', async ({ resourcesPage }) => { + test('[232965] User can create a perspective a Tags chart is saved and applied correctly', { tag: '@p2' }, async ({ resourcesPage }) => { await resourcesPage.navigateToResourcesPageAndResetFilters(); const filter = 'Pool'; @@ -163,7 +163,7 @@ test.describe('[MPT-18579] Perspective Tests', { tag: ['@ui', '@resources', '@pe }); }); - test('[232966] User can create a perspective for the Meta chart and the perspective is saved and applied correctly', async ({ + test('[232966] User can create a perspective for the Meta chart and the perspective is saved and applied correctly', { tag: '@p2' }, async ({ resourcesPage, }) => { await resourcesPage.navigateToResourcesPageAndResetFilters(); @@ -205,7 +205,7 @@ test.describe('[MPT-18579] Perspective Tests', { tag: ['@ui', '@resources', '@pe }); }); - test('[232967] User can create a perspective and delete it via the perspectives table', async ({ resourcesPage, perspectivesPage }) => { + test('[232967] User can create a perspective and delete it via the perspectives table', { tag: '@p2' }, async ({ resourcesPage, perspectivesPage }) => { await perspectivesPage.navigateToURL(); const initialPerspectivesCount = await perspectivesPage.getPerspectivesCount(); debugLog(`Initial perspectives count: ${initialPerspectivesCount}`); @@ -243,7 +243,7 @@ test.describe('[MPT-18579] Perspective Tests', { tag: ['@ui', '@resources', '@pe }); }); - test('[232969] User can create a perspective with multiple filters', async ({ resourcesPage, perspectivesPage }) => { + test('[232969] User can create a perspective with multiple filters', { tag: '@p2' }, async ({ resourcesPage, perspectivesPage }) => { await resourcesPage.navigateToResourcesPageAndResetFilters(); const filter1 = 'Region'; @@ -284,7 +284,7 @@ test.describe('[MPT-18579] Perspective Tests', { tag: ['@ui', '@resources', '@pe }); }); - test('[232970] Creating a perspective with an existing name shows a message stating that the original will be overwritten', async ({ + test('[232970] Creating a perspective with an existing name shows a message stating that the original will be overwritten', { tag: '@p2' }, async ({ resourcesPage, }) => { await resourcesPage.navigateToResourcesPageAndResetFilters(); @@ -329,7 +329,7 @@ test.describe('[MPT-18579] Perspective Tests', { tag: ['@ui', '@resources', '@pe }); }); - test('[232968] No perspectives message is displayed and perspectives button is hidden if there are no perspectives', async ({ + test('[232968] No perspectives message is displayed and perspectives button is hidden if there are no perspectives', { tag: '@p2' }, async ({ resourcesPage, perspectivesPage, }) => { diff --git a/e2etests/tests/policies-tests.spec.ts b/e2etests/tests/policies-tests.spec.ts index 0d35ac6ee..741864a33 100644 --- a/e2etests/tests/policies-tests.spec.ts +++ b/e2etests/tests/policies-tests.spec.ts @@ -43,7 +43,7 @@ test.describe('[MPT-16366] Policies Tests', { tag: ['@ui', '@policies'] }, () => }); }); - test('[232286] Verify that Sample data pop-up is visible when no policies exist', async ({ policiesPage }) => { + test('[232286] Verify that Sample data pop-up is visible when no policies exist', { tag: ['@fast', '@p2'] }, async ({ policiesPage }) => { await test.step('Ensure all policies are deleted', async () => { // eslint-disable-next-line playwright/no-conditional-in-test if(!await policiesPage.realDataAddBtn.isVisible()) { @@ -58,7 +58,7 @@ test.describe('[MPT-16366] Policies Tests', { tag: ['@ui', '@policies'] }, () => }); }); - test('[232287] Verify that user can add a resource quota policy', async ({ policiesPage, policiesCreatePage }) => { + test('[232287] Verify that user can add a resource quota policy', { tag: ['@fast', '@p2'] }, async ({ policiesPage, policiesCreatePage }) => { const NBSP = '\u00A0' const policyName = `Resource Policy ${Date.now()}`; const resourceCount = 10; @@ -96,7 +96,7 @@ test.describe('[MPT-16366] Policies Tests', { tag: ['@ui', '@policies'] }, () => }); }); - test('[232288] Verify that user can create a recurring budget policy', async ({ policiesPage, policiesCreatePage }) => { + test('[232288] Verify that user can create a recurring budget policy', { tag: ['@fast', '@p2'] }, async ({ policiesPage, policiesCreatePage }) => { const policyName = `Recurring Budget ${Date.now()}`; const budgetAmount = 1000; const formattedAmount = formatCurrency(budgetAmount); @@ -129,7 +129,7 @@ test.describe('[MPT-16366] Policies Tests', { tag: ['@ui', '@policies'] }, () => }); }); - test('[232289] Verify that user can create an expiring budget policy', async ({ policiesPage, policiesCreatePage }) => { + test('[232289] Verify that user can create an expiring budget policy', { tag: ['@fast', '@p2'] }, async ({ policiesPage, policiesCreatePage }) => { const policyName = `Expiring Budget ${Date.now()}`; const budgetAmount = 500; const formattedAmount = formatCurrency(budgetAmount); @@ -165,7 +165,7 @@ test.describe('[MPT-16366] Policies Tests', { tag: ['@ui', '@policies'] }, () => }); }); - test('[232290] Verify that user can delete a policy from the policy details page', async ({ policiesPage, policiesCreatePage }) => { + test('[232290] Verify that user can delete a policy from the policy details page', { tag: ['@fast', '@p2'] }, async ({ policiesPage, policiesCreatePage }) => { const policyName = `Policy To Be Deleted ${Date.now()}`; const resourceCount = 5; @@ -208,7 +208,7 @@ test.describe('[MPT-16366] Mocked Policies Tests', { tag: ['@ui', '@policies'] } { gql: 'GetOrganizationLimitHits', mock: ExpiringBudgetOverLimitHitsResponse, - variableMatch: { constraintId: 'e0f96d76-bdb2-4c7e-906b-fb6966dd4c23' }, + variableMatch: { constraintId: 'e0f96d76-bdb2-4c7e-906b-fb69666dd4c23' }, }, { gql: 'GetOrganizationConstraints', @@ -275,7 +275,7 @@ test.describe('[MPT-16366] Mocked Policies Tests', { tag: ['@ui', '@policies'] } }); }); - test('[232337] Verify that statuses are displayed correctly from each policy type when over and under limit', async ({ + test('[232337] Verify that statuses are displayed correctly from each policy type when over and under limit', { tag: ['@fast', '@p2'] }, async ({ policiesPage, }) => { expect(await policiesPage.getColorFromElement(policiesPage.resourceUnderLimitStatus)).toBe(policiesPage.successColor); @@ -286,7 +286,7 @@ test.describe('[MPT-16366] Mocked Policies Tests', { tag: ['@ui', '@policies'] } expect(await policiesPage.getColorFromElement(policiesPage.expiringBudgetOverLimitStatus)).toBe(policiesPage.errorColor); }); - test('[232338] Verify that expiring budget over limit displays violation in violation history table', async ({ policiesPage }) => { + test('[232338] Verify that expiring budget over limit displays violation in violation history table', { tag: ['@fast', '@p2'] }, async ({ policiesPage }) => { await policiesPage.expiringBudgetOverLimitLink.click(); await policiesPage.policyDetailsDiv.waitFor(); @@ -298,7 +298,7 @@ test.describe('[MPT-16366] Mocked Policies Tests', { tag: ['@ui', '@policies'] } await expect(firstBudgetActualExpensesTableCell).toHaveText('$1 ⟶ $48,646.2'); }); - test('[232339] Verify that expiring budget under limit shows no violations in violation history table', async ({ policiesPage }) => { + test('[232339] Verify that expiring budget under limit shows no violations in violation history table', { tag: ['@fast', '@p2'] }, async ({ policiesPage }) => { await policiesPage.expiringBudgetUnderLimitLink.click(); await policiesPage.policyDetailsDiv.waitFor(); @@ -306,7 +306,7 @@ test.describe('[MPT-16366] Mocked Policies Tests', { tag: ['@ui', '@policies'] } await expect(policiesPage.table).toBeHidden(); }); - test('[232340] Verify that recurring budget over limit displays violation in violation history table', async ({ policiesPage }) => { + test('[232340] Verify that recurring budget over limit displays violation in violation history table', { tag: ['@fast', '@p2'] }, async ({ policiesPage }) => { await policiesPage.recurringBudgetOverLimitLink.click(); await policiesPage.policyDetailsDiv.waitFor(); @@ -318,7 +318,7 @@ test.describe('[MPT-16366] Mocked Policies Tests', { tag: ['@ui', '@policies'] } await expect(firstBudgetActualExpensesTableCell).toHaveText('$10 ⟶ $48,646.2'); }); - test('[232341] Verify that recurring budget under limit shows no violations in violation history table', async ({ policiesPage }) => { + test('[232341] Verify that recurring budget under limit shows no violations in violation history table', { tag: ['@fast', '@p2'] }, async ({ policiesPage }) => { await policiesPage.recurringBudgetUnderLimitLink.click(); await policiesPage.policyDetailsDiv.waitFor(); @@ -326,7 +326,7 @@ test.describe('[MPT-16366] Mocked Policies Tests', { tag: ['@ui', '@policies'] } await expect(policiesPage.table).toBeHidden(); }); - test('[232342] Verify that resource quota over limit displays violation in violation history table', async ({ policiesPage }) => { + test('[232342] Verify that resource quota over limit displays violation in violation history table', { tag: ['@fast', '@p2'] }, async ({ policiesPage }) => { await policiesPage.resourceOverLimitLink.click(); await policiesPage.policyDetailsDiv.waitFor(); @@ -338,7 +338,7 @@ test.describe('[MPT-16366] Mocked Policies Tests', { tag: ['@ui', '@policies'] } await expect(firstQuotaActualResourceCountTableCell).toHaveText('1 ⟶ 3,012'); }); - test('[232343] Verify that resource quota under limit shows no violations in violation history table', async ({ policiesPage }) => { + test('[232343] Verify that resource quota under limit shows no violations in violation history table', { tag: ['@fast', '@p2'] }, async ({ policiesPage }) => { await policiesPage.resourceUnderLimitLink.click(); await policiesPage.policyDetailsDiv.waitFor(); diff --git a/e2etests/tests/pools-tests.spec.ts b/e2etests/tests/pools-tests.spec.ts index 550867a1b..3d4ba62d9 100644 --- a/e2etests/tests/pools-tests.spec.ts +++ b/e2etests/tests/pools-tests.spec.ts @@ -24,7 +24,7 @@ test.describe('[MPT-12743] Pools Tests', { tag: ['@ui', '@pools'] }, () => { await poolsPage.toggleExpandPool(); }); - test('[230911] Verify Pools page column selection', async ({ poolsPage }) => { + test('[230911] Verify Pools page column selection', { tag: ['@fast', '@p2'] }, async ({ poolsPage }) => { const defaultColumns = [ poolsPage.nameTableHeading, poolsPage.monthlyLimitTableHeading, @@ -81,7 +81,7 @@ test.describe('[MPT-12743] Pools Tests', { tag: ['@ui', '@pools'] }, () => { }); }); - test('[230912] Verify Organization limit, Pools Expenses and Forecast this month match totals in the table', async ({ poolsPage }) => { + test('[230912] Verify Organization limit, Pools Expenses and Forecast this month match totals in the table', { tag: ['@fast', '@p2'] }, async ({ poolsPage }) => { // test.fail((await poolsPage.getPoolCount()) !== 1, `Expected 1 pool, but found ${await poolsPage.getPoolCount()}`); let organizationLimitValue: number; @@ -131,7 +131,7 @@ test.describe('[MPT-12743] Pools Tests', { tag: ['@ui', '@pools'] }, () => { }); }); - test('[230913] Verify Organisation Limit functionality - limit not set', async ({ poolsPage }) => { + test('[230913] Verify Organisation Limit functionality - limit not set', { tag: ['@fast', '@p2'] }, async ({ poolsPage }) => { test.fail((await poolsPage.getPoolCount()) !== 1, `Expected 1 pool, but found ${await poolsPage.getPoolCount()}`); await test.step('Remove organisation limit if it is set.', async () => { @@ -155,7 +155,7 @@ test.describe('[MPT-12743] Pools Tests', { tag: ['@ui', '@pools'] }, () => { }); }); - test('[230914] Verify Organisation Limit functionality - expenses are less than 90% of limit', async ({ poolsPage }) => { + test('[230914] Verify Organisation Limit functionality - expenses are less than 90% of limit', { tag: ['@fast', '@p2'] }, async ({ poolsPage }) => { test.fail((await poolsPage.getPoolCount()) !== 1, `Expected 1 pool, but found ${await poolsPage.getPoolCount()}`); const expensesThisMonth = await poolsPage.getExpensesThisMonth(); @@ -184,7 +184,7 @@ test.describe('[MPT-12743] Pools Tests', { tag: ['@ui', '@pools'] }, () => { }); }); - test('[230915] Verify Organisation Limit functionality - expenses are greater than 90% of limit', async ({ poolsPage }) => { + test('[230915] Verify Organisation Limit functionality - expenses are greater than 90% of limit', { tag: ['@fast', '@p2'] }, async ({ poolsPage }) => { test.fail((await poolsPage.getPoolCount()) !== 1, `Expected 1 pool, but found ${await poolsPage.getPoolCount()}`); const expensesThisMonth = await poolsPage.getExpensesThisMonth(); @@ -215,7 +215,7 @@ test.describe('[MPT-12743] Pools Tests', { tag: ['@ui', '@pools'] }, () => { }); }); - test('[230916] Verify Organisation Limit functionality - limit set lower than expenses this month', async ({ poolsPage }) => { + test('[230916] Verify Organisation Limit functionality - limit set lower than expenses this month', { tag: ['@fast', '@p2'] }, async ({ poolsPage }) => { test.fail((await poolsPage.getPoolCount()) !== 1, `Expected 1 pool, but found ${await poolsPage.getPoolCount()}`); const expensesThisMonth = await poolsPage.getExpensesThisMonth(); @@ -245,7 +245,7 @@ test.describe('[MPT-12743] Pools Tests', { tag: ['@ui', '@pools'] }, () => { }); }); - test('[230917] Verify Organisation Limit functionality - limit set lower than forecast', async ({ poolsPage }) => { + test('[230917] Verify Organisation Limit functionality - limit set lower than forecast', { tag: ['@fast', '@p2'] }, async ({ poolsPage }) => { const expensesThisMonth = await poolsPage.getExpensesThisMonth(); const forecastThisMonth = await poolsPage.getForecastThisMonth(); test.skip(expensesThisMonth <= 1, 'Skipping test as it requires expenses to be greater than 1'); @@ -272,7 +272,7 @@ test.describe('[MPT-12743] Pools Tests', { tag: ['@ui', '@pools'] }, () => { }); }); - test('[230918] Verify sub-pool monthly limit behaviour', async ({ poolsPage }) => { + test('[230918] Verify sub-pool monthly limit behaviour', { tag: ['@slow', '@p2'] }, async ({ poolsPage }) => { test.fail((await poolsPage.getPoolCount()) !== 1, `Expected 1 pool, but found ${await poolsPage.getPoolCount()}`); test.setTimeout(75000); @@ -312,7 +312,7 @@ test.describe('[MPT-12743] Pools Tests', { tag: ['@ui', '@pools'] }, () => { }); }); - test('[230919] Verify pool exceeded count and expand requiring attention', { tag: '@p1' }, async ({ poolsPage }) => { + test('[230919] Verify pool exceeded count and expand requiring attention', { tag: ['@slow', '@p1'] }, async ({ poolsPage }) => { // test.fail((await poolsPage.getPoolCount()) !== 1, `Expected 1 pool, but found ${await poolsPage.getPoolCount()}`); test.setTimeout(75000); @@ -356,7 +356,7 @@ test.describe('[MPT-12743] Pools Tests', { tag: ['@ui', '@pools'] }, () => { }); }); - test('[232865] Verify that updating limits of pools and sub-pools is recorded in the logs', async ({ poolsPage, eventsPage }) => { + test('[232865] Verify that updating limits of pools and sub-pools is recorded in the logs', { tag: ['@slow', '@p2'] }, async ({ poolsPage, eventsPage }) => { test.setTimeout(60000); let timestamp: string; const randomNumber = Math.floor(Math.random() * 1_000); diff --git a/e2etests/tests/recommendations-tests.spec.ts b/e2etests/tests/recommendations-tests.spec.ts index a51a1e1f1..6462da256 100644 --- a/e2etests/tests/recommendations-tests.spec.ts +++ b/e2etests/tests/recommendations-tests.spec.ts @@ -18,7 +18,7 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme await recommendationsPage.selectCategory('All'); }); - test('[230511] Verify Card total savings match possible monthly savings', { tag: '@p1' }, async ({ recommendationsPage }) => { + test('[230511] Verify Card total savings match possible monthly savings', { tag: ['@fast', '@p1'] }, async ({ recommendationsPage }) => { let possibleMonthlySavings: number; let cardTotalSavings: number; @@ -37,7 +37,7 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme }); }); - test('[230597] Verify Data Source selection works correctly', async ({ recommendationsPage }) => { + test('[230597] Verify Data Source selection works correctly', { tag: ['@fast', '@p2'] }, async ({ recommendationsPage }) => { const dataSource = process.env.USE_LIVE_DEMO === 'true' ? 'Azure QA' : 'CPA (Development and Test)'; await test.step(`Select data source: ${dataSource}`, async () => { @@ -58,7 +58,7 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme }); //It appears that environments don't have the correct permissions to run S3 Duplicate checks, so marking this as FIXME for now. - test.fixme('[230513] Verify S3 Duplicate Possible monthly savings matches that on S3 Duplicate Finder page', async ({ + test.fixme('[230513] Verify S3 Duplicate Possible monthly savings matches that on S3 Duplicate Finder page', { tag: ['@fast', '@p2'] }, async ({ recommendationsPage, s3DuplicateFinder, }) => { @@ -92,7 +92,7 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme }); }); - test('[230514] Verify Search functionality works correctly', async ({ recommendationsPage }) => { + test('[230514] Verify Search functionality works correctly', { tag: ['@fast', '@p2'] }, async ({ recommendationsPage }) => { await recommendationsPage.searchByName('Public'); await recommendationsPage.allCardHeadings.last().waitFor(); @@ -105,7 +105,7 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme test( '[230598] Verify only the correct applicable services are displayed for SWO Customisation', - { tag: '@p1' }, + { tag: ['@fast', '@p1'] }, async ({ recommendationsPage }) => { await test.step('Verify applicable services combo box options shows expected items', async () => { await recommendationsPage.applicableServices.click(); @@ -227,11 +227,11 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme ]; // eslint-disable-next-line playwright/expect-expect - test('[230515] Verify all expected cards are present when All category selected', async ({ recommendationsPage }) => { + test('[230515] Verify all expected cards are present when All category selected', { tag: ['@fast', '@p2'] }, async ({ recommendationsPage }) => { await verifyCardsAndTable(recommendationsPage, 'All', allExpectedCardHeadings); }); - test(`[231467] Verify no cards are displaying errors`, async ({ recommendationsPage }) => { + test(`[231467] Verify no cards are displaying errors`, { tag: ['@fast', '@p2'] }, async ({ recommendationsPage }) => { await recommendationsPage.selectCategory('All'); await recommendationsPage.allCardHeadings.last().waitFor(); @@ -249,7 +249,7 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme expect(errorCount, 'No cards should be displaying errors').toBe(0); }); // eslint-disable-next-line playwright/expect-expect - test('[230518] Verify all expected cards are present when Savings category selected', async ({ recommendationsPage }) => { + test('[230518] Verify all expected cards are present when Savings category selected', { tag: ['@fast', '@p2'] }, async ({ recommendationsPage }) => { const expectedCardHeadings = [ 'Abandoned Amazon S3 buckets', 'Abandoned Images', @@ -276,7 +276,7 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme }); // eslint-disable-next-line playwright/expect-expect - test('[230519] Verify all expected cards are present when Security category selected', async ({ recommendationsPage }) => { + test('[230519] Verify all expected cards are present when Security category selected', { tag: ['@fast', '@p2'] }, async ({ recommendationsPage }) => { const expectedCardHeadings = [ 'IAM users with unused console access', 'Inactive IAM users', @@ -286,7 +286,7 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme await verifyCardsAndTable(recommendationsPage, 'Security', expectedCardHeadings); }); - test('[230520] Verify all cards display critical icon when Critical category selected', async ({ recommendationsPage }) => { + test('[230520] Verify all cards display critical icon when Critical category selected', { tag: ['@fast', '@p2'] }, async ({ recommendationsPage }) => { let count: number; let actualHeadings: string[]; let criticalIconCount: number; @@ -324,7 +324,7 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme }); }); - test('[230521] Verify that only cards with See Item buttons are displayed when Non-empty category selected', async ({ + test('[230521] Verify that only cards with See Item buttons are displayed when Non-empty category selected', { tag: ['@fast', '@p2'] }, async ({ recommendationsPage, }) => { let count: number; @@ -357,7 +357,7 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme }); }); - test('[230523] Verify filtering by applicable service works correctly', async ({ recommendationsPage }) => { + test('[230523] Verify filtering by applicable service works correctly', { tag: ['@fast', '@p2'] }, async ({ recommendationsPage }) => { let count: number; let actualHeadings: string[]; let rdsCount: number; @@ -426,7 +426,7 @@ test.describe('[MPT-11310] Recommendations page tests', { tag: ['@ui', '@recomme ]; for (const cardName of cardEntries) { - test(`[230524] ${cardName}: Cards displaying possible savings, should match itemised modal total and table total`, async ({ + test(`[230524] ${cardName}: Cards displaying possible savings, should match itemised modal total and table total`, { tag: ['@fast', '@p2'] }, async ({ recommendationsPage, }) => { // Find this card’s full metadata at runtime diff --git a/e2etests/tests/resources-tests.spec.ts b/e2etests/tests/resources-tests.spec.ts index 292f92975..3b02200be 100644 --- a/e2etests/tests/resources-tests.spec.ts +++ b/e2etests/tests/resources-tests.spec.ts @@ -49,7 +49,7 @@ test.describe('[MPT-11957] Resources page tests', { tag: ['@ui', '@resources'] } }); }); - test('[230778] All expected filters are displayed', async ({ resourcesPage }) => { + test('[230778] All expected filters are displayed', { tag: ['@fast', '@p2'] }, async ({ resourcesPage }) => { await test.step('Click Show more filters button', async () => { await resourcesPage.clickShowMoreFilters(); await resourcesPage.showLessFiltersBtn.waitFor(); @@ -84,7 +84,7 @@ test.describe('[MPT-11957] Resources page tests', { tag: ['@ui', '@resources'] } }); }); - test('[230779] Verify table column selection', async ({ resourcesPage }) => { + test('[230779] Verify table column selection', { tag: ['@fast', '@p2'] }, async ({ resourcesPage }) => { const defaultColumns = [ resourcesPage.resourceTableHeading, resourcesPage.expensesTableHeading, @@ -151,7 +151,7 @@ test.describe('[MPT-11957] Resources page tests', { tag: ['@ui', '@resources'] } }); }); - test('[230780] Unfiltered Total expenses matches table itemised total', { tag: '@slow' }, async ({ resourcesPage, datePicker }) => { + test('[230780] Unfiltered Total expenses matches table itemised total', { tag: ['@slow', '@p2'] }, async ({ resourcesPage, datePicker }) => { test.setTimeout(1200000); test.skip( (await resourcesPage.getResourceCountValue()) > 5000, @@ -181,7 +181,7 @@ test.describe('[MPT-11957] Resources page tests', { tag: ['@ui', '@resources'] } }); }); - test('[230788] Filtered Total expenses matches table itemised total', { tag: '@slow' }, async ({ resourcesPage }) => { + test('[230788] Filtered Total expenses matches table itemised total', { tag: ['@slow', '@p2'] }, async ({ resourcesPage }) => { test.setTimeout(120000); let initialTotalExpensesValue: number; @@ -221,7 +221,7 @@ test.describe('[MPT-11957] Resources page tests', { tag: ['@ui', '@resources'] } test( '[230781] Total expenses matches table itemised total for date range set to last 7 days', - { tag: '@slow' }, + { tag: ['@slow', '@p2'] }, async ({ resourcesPage, datePicker }) => { test.setTimeout(120000); @@ -246,7 +246,7 @@ test.describe('[MPT-11957] Resources page tests', { tag: ['@ui', '@resources'] } } ); - test('[230782] Validate API default chart/table data for 7 days', async ({ resourcesPage, datePicker }) => { + test('[230782] Validate API default chart/table data for 7 days', { tag: ['@slow', '@p2'] }, async ({ resourcesPage, datePicker }) => { test.slow(); const { startDate, endDate } = getLast7DaysUnixRange(); @@ -327,7 +327,7 @@ test.describe('[MPT-11957] Resources page tests', { tag: ['@ui', '@resources'] } test( '[230783] Validate API data for the daily expenses chart by breakdown for 7 days', - { tag: '@slow' }, + { tag: ['@slow', '@p2'] }, async ({ resourcesPage, datePicker }) => { test.setTimeout(120000); const { startDate, endDate } = getLast7DaysUnixRange(); @@ -746,7 +746,7 @@ test.describe('[MPT-11957] Resources page mocked tests', { tag: ['@ui', '@resour }); }); - test('[230784] Verify default service daily expenses chart export with and without legend', { tag: '@p1' }, async ({ resourcesPage }) => { + test('[230784] Verify default service daily expenses chart export with and without legend', { tag: ['@fast', '@p1'] }, async ({ resourcesPage }) => { test.fixme(process.env.CI === '1', 'Tests do not work in CI. It appears that the png comparison is unsupported on linux'); let actualPath = path.resolve('tests', 'downloads', 'expenses-chart-export.png'); let expectedPath = path.resolve('tests', 'expected', 'expected-expenses-chart-export.png'); @@ -777,7 +777,7 @@ test.describe('[MPT-11957] Resources page mocked tests', { tag: ['@ui', '@resour }); }); - test('[230785] Verify weekly and monthly expenses chart export', async ({ resourcesPage }) => { + test('[230785] Verify weekly and monthly expenses chart export', { tag: ['@fast', '@p2'] }, async ({ resourcesPage }) => { test.fixme(process.env.CI === '1', 'Tests do not work in CI. It appears that the png comparison is unsupported on linux'); let actualPath = path.resolve('tests', 'downloads', 'weekly-expenses-chart-export.png'); let expectedPath = path.resolve('tests', 'expected', 'expected-weekly-expenses-chart-export.png'); @@ -803,7 +803,7 @@ test.describe('[MPT-11957] Resources page mocked tests', { tag: ['@ui', '@resour }); }); - test('[230786] Verify expenses chart export with different categories', async ({ resourcesPage }) => { + test('[230786] Verify expenses chart export with different categories', { tag: ['@fast', '@p2'] }, async ({ resourcesPage }) => { test.fixme(process.env.CI === '1', 'Tests do not work in CI. It appears that the png comparison is unsupported on linux'); let actualPath = path.resolve('tests', 'downloads', 'region-expenses-chart-export.png'); let expectedPath = path.resolve('tests', 'expected', 'expected-region-expenses-chart-export.png'); @@ -895,7 +895,7 @@ test.describe('[MPT-11957] Resources page mocked tests', { tag: ['@ui', '@resour }); }); - test('[230787] Verify table grouping', async ({ resourcesPage }) => { + test('[230787] Verify table grouping', { tag: ['@fast', '@p2'] }, async ({ resourcesPage }) => { await test.step('Verify default grouping is None', async () => { await resourcesPage.table.waitFor(); await resourcesPage.table.scrollIntoViewIfNeeded(); diff --git a/e2etests/tests/ri-sp-coverage-test.spec.ts b/e2etests/tests/ri-sp-coverage-test.spec.ts index 95a3a0020..f89235764 100644 --- a/e2etests/tests/ri-sp-coverage-test.spec.ts +++ b/e2etests/tests/ri-sp-coverage-test.spec.ts @@ -81,7 +81,7 @@ test.describe('Mocked RI/SP coverage page test', { tag: ['@ui', '@risp-coverage' test.use({ restoreSession: true, interceptAPI: { entries: apiInterceptions, failOnInterceptionMissing: true } }); - test('[232683] Verify mocked table data is displayed on the RI/SP coverage page', async ({ recommendationsPage, riSpCoveragePage }) => { + test('[232683] Verify mocked table data is displayed on the RI/SP coverage page', { tag: ['@fast', '@p2'] }, async ({ recommendationsPage, riSpCoveragePage }) => { let savingsValue: number; await test.step('Navigate to the RI/SP coverage page from recommendations page', async () => { diff --git a/e2etests/tests/tagging-policy-tests.spec.ts b/e2etests/tests/tagging-policy-tests.spec.ts index 604352da7..2f7017a23 100644 --- a/e2etests/tests/tagging-policy-tests.spec.ts +++ b/e2etests/tests/tagging-policy-tests.spec.ts @@ -31,7 +31,7 @@ test.describe('[MPT-17042] Tagging Policy Tests', { tag: ['@ui', '@tagging-polic }); }); - test('[232655] Verify that Sample data pop-up is visible when no policies exist', async ({ taggingPoliciesPage }) => { + test('[232655] Verify that Sample data pop-up is visible when no policies exist', { tag: ['@fast', '@p2'] }, async ({ taggingPoliciesPage }) => { await test.step('Ensure all policies are deleted', async () => { // eslint-disable-next-line playwright/no-conditional-in-test if (!(await taggingPoliciesPage.addRealDataBtn.isVisible())) { @@ -46,7 +46,7 @@ test.describe('[MPT-17042] Tagging Policy Tests', { tag: ['@ui', '@tagging-polic }); }); - test('[232656] Verify that a user can create a required tagging policy', async ({ taggingPoliciesPage, taggingPoliciesCreatePage }) => { + test('[232656] Verify that a user can create a required tagging policy', { tag: ['@fast', '@p2'] }, async ({ taggingPoliciesPage, taggingPoliciesCreatePage }) => { const policyName = `Required Tag Policy ${Date.now()}`; const tagName = 'AccountId'; @@ -66,7 +66,7 @@ test.describe('[MPT-17042] Tagging Policy Tests', { tag: ['@ui', '@tagging-polic }); }); - test('[232657] Verify that a user can create a prohibited tagging policy', async ({ taggingPoliciesPage, taggingPoliciesCreatePage }) => { + test('[232657] Verify that a user can create a prohibited tagging policy', { tag: ['@fast', '@p2'] }, async ({ taggingPoliciesPage, taggingPoliciesCreatePage }) => { const policyName = `Prohibited Tag Policy ${Date.now()}`; const tagName = '__department'; const filter = 'Activity'; @@ -96,7 +96,7 @@ test.describe('[MPT-17042] Tagging Policy Tests', { tag: ['@ui', '@tagging-polic }); }); - test('[232658] Verify that a user can create a tags correlation tagging policy', async ({ + test('[232658] Verify that a user can create a tags correlation tagging policy', { tag: ['@fast', '@p2'] }, async ({ taggingPoliciesPage, taggingPoliciesCreatePage, }) => { @@ -122,7 +122,7 @@ test.describe('[MPT-17042] Tagging Policy Tests', { tag: ['@ui', '@tagging-polic }); }); - test('[232659] Verify that user can delete a policy from the tagging policy details page', async ({ + test('[232659] Verify that user can delete a policy from the tagging policy details page', { tag: ['@fast', '@p2'] }, async ({ taggingPoliciesPage, taggingPoliciesCreatePage, }) => { @@ -175,7 +175,7 @@ test.describe('[MPT-17042] Mocked Tagging Policies Tests', { tag: ['@ui', '@tagg }); }); - test('[232660] Verify that tagging policies are displayed with the correct status', async ({ taggingPoliciesPage }) => { + test('[232660] Verify that tagging policies are displayed with the correct status', { tag: ['@fast', '@p2'] }, async ({ taggingPoliciesPage }) => { const cancelIconXpath = '//*[@data-testid="CancelIcon"]'; const checkCheckIconXpath = '//*[@data-testid="CheckCircleIcon"]'; const correlatedTagStatus = taggingPoliciesPage.table.locator('(//a[contains(text(), "Correlated Tag")]/ancestor::tr/td[2]/div)[1]'); diff --git a/e2etests/utils/report-utils.ts b/e2etests/utils/report-utils.ts new file mode 100644 index 000000000..92462200d --- /dev/null +++ b/e2etests/utils/report-utils.ts @@ -0,0 +1,27 @@ + +export function limitString(inputStr: string, maxLength: number): string { + return inputStr.length > maxLength ? inputStr.substring(0, maxLength) + '...' : inputStr; +} + +export function reportDebugLog(message: string, messageType: string = 'debug'): void { + if (process.env.DEBUG_LOG) { + console.log(`${formatDateToYmdHms(new Date())} [${limitString(messageType.toUpperCase(), 10)}]: ${message}`); + } +} + +export function padTo2Digits(num: number): string { + return num.toString().padStart(2, '0'); +} + +export function formatDateToYmd(date: Date): string { + return [date.getFullYear(), padTo2Digits(date.getMonth() + 1), padTo2Digits(date.getDate())].join('-'); +} + +export function formatDateToHms(date: Date): string { + return [padTo2Digits(date.getHours()), padTo2Digits(date.getMinutes()), padTo2Digits(date.getSeconds())].join(':'); +} + +export function formatDateToYmdHms(date: Date): string { + return `${formatDateToYmd(date)} ${formatDateToHms(date)}`; +} + From 59c451c353e175bb278c928bb59759fb77cb10bb Mon Sep 17 00:00:00 2001 From: Steve Churchill Date: Wed, 15 Apr 2026 13:02:32 +0100 Subject: [PATCH 2/3] Use shared tag constants from tags-helper Import TAGS_MODULE, TAGS_SPEED, TAGS_TYPE and TAGS_DEBUG from tags-helper and remove the duplicated local tag definitions in metrics-reporter.ts. Updated the comment to note tags are sourced from tags-helper (single source of truth). This is a refactor to centralize tag definitions and avoid duplication; no functional changes intended. --- e2etests/reporting/metrics-reporter.ts | 31 ++------------------------ 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/e2etests/reporting/metrics-reporter.ts b/e2etests/reporting/metrics-reporter.ts index ce9afbe1a..5077c8821 100644 --- a/e2etests/reporting/metrics-reporter.ts +++ b/e2etests/reporting/metrics-reporter.ts @@ -26,7 +26,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { execSync } from 'child_process'; import { reportDebugLog, limitString } from '../utils/report-utils'; -import { logTagAttributes } from './tags-helper'; +import { logTagAttributes, TAGS_MODULE, TAGS_SPEED, TAGS_TYPE, TAGS_DEBUG } from './tags-helper'; // --------------------------------------------------------------------------- // Output configuration @@ -35,7 +35,7 @@ import { logTagAttributes } from './tags-helper'; const OUTPUT_FOLDER = 'reports'; // --------------------------------------------------------------------------- -// Tag definitions — aligned with the optscale e2e test suite +// Tag definitions — imported from tags-helper.ts (single source of truth) // --------------------------------------------------------------------------- /** Priority tags — matched via regex rather than an explicit list */ @@ -44,33 +44,6 @@ const priorityRegex = /^p\d+$/; /** Test-case ID tags — e.g. [231181] */ const testCaseIdRegex = /^\[\d+]$/; -/** - * Module tags — functional areas covered by the test suite. - * Sourced from actual @tags found in tests/** and regression-tests/**. - */ -export const TAGS_MODULE = [ - 'expenses', // Expenses page / cost analysis - 'homepage', // Home / dashboard page - 'recommendations', // Recommendations module - 'resources', // Resources management - 'pools', // Pools management - 'policies', // Policies (general) - 'tagging-policies', // Tagging policies - 'cloud-accounts', // Cloud account integration - 'events', // Events log - 'perspectives', // Perspectives / saved views - 'risp-coverage', // RISP coverage reports - 'invitation-flow', // Invitation / onboarding flow -]; - -/** Speed tags */ -export const TAGS_SPEED: string[] = ['slow']; - -/** Test type tags */ -export const TAGS_TYPE = ['ui', 'devops']; - -/** Debug / development-only tests */ -export const TAGS_DEBUG: string[] = ['debug']; // --------------------------------------------------------------------------- // Timestamp & branch helpers From 8f95fd2830d63b84803e2c3bb6e6077e3a5750b6 Mon Sep 17 00:00:00 2001 From: Steve Churchill Date: Mon, 20 Apr 2026 10:57:48 +0100 Subject: [PATCH 3/3] Log ENVIRONMENT and add large data timeout Print process.env.ENVIRONMENT in global setup for easier environment debugging. Import LARGE_DATA_TIMEOUT in expenses-tests.spec.ts and pass it to page.waitForResponse for the /pools_expenses/ request to handle large payloads and reduce flaky timeouts during heavy data loads. --- e2etests/setup/global-setup.ts | 1 + e2etests/tests/expenses-tests.spec.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/e2etests/setup/global-setup.ts b/e2etests/setup/global-setup.ts index abc76815f..8d964aabd 100644 --- a/e2etests/setup/global-setup.ts +++ b/e2etests/setup/global-setup.ts @@ -23,6 +23,7 @@ async function globalSetup(config: FullConfig) { const localHostURL = 'http://localhost:3000'; // Log key environment variables for debugging purposes + console.log(`Environment: ${process.env.ENVIRONMENT}`); console.log(`Tests running on ${process.env.BASE_URL}`); console.log(`Ignoring HTTPS errors: ${process.env.IGNORE_HTTPS_ERRORS}`); console.log(`SCREENSHOT_UPDATE_DELAY: ${process.env.SCREENSHOT_UPDATE_DELAY}`); diff --git a/e2etests/tests/expenses-tests.spec.ts b/e2etests/tests/expenses-tests.spec.ts index 658dec1dd..a99205b4f 100644 --- a/e2etests/tests/expenses-tests.spec.ts +++ b/e2etests/tests/expenses-tests.spec.ts @@ -14,6 +14,7 @@ import { ExpensesDefaultResponse } from '../mocks/expenses-page-mocks'; import { comparePdfFiles } from '../utils/pdf-comparison'; import { isWithinRoundingDrift } from '../utils/custom-assertions'; import { getEnvironmentTestOrgName } from '../utils/environment-util'; +import { LARGE_DATA_TIMEOUT } from '../playwright.config'; import path from 'path'; test.describe('[MPT-12859] Expenses Page default view Tests', { tag: ['@ui', '@expenses'] }, () => { @@ -62,7 +63,7 @@ test.describe('[MPT-12859] Expenses Page default view Tests', { tag: ['@ui', '@e await test.step('Load expenses data for the this month', async () => { const [expensesResponse] = await Promise.all([ - expensesPage.page.waitForResponse(resp => resp.url().includes('/pools_expenses/') && resp.request().method() === 'GET'), + expensesPage.page.waitForResponse(resp => resp.url().includes('/pools_expenses/') && resp.request().method() === 'GET', { timeout: LARGE_DATA_TIMEOUT }), expensesPage.page.reload(), ]);