From 55c6fa27d0cce03130cac19dc1505889b4a53300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Mon, 22 Dec 2025 16:01:04 +0100 Subject: [PATCH 1/3] input: define checkout-/build-/packageAuditFiles Using these keys, selected files can be included into the audit trail. This can be used to store any supplementary information in the audit trail as necessary. Examples are hash sums of individual source files, custom license texts or other build time meta data. By default, the files are read with the UTF-8 codec. This can be changed by setting an "encoding" attribute. Sometimes the presence of files depends on other factors. The "if" property allows to include them conditionally. --- pym/bob/audit.py | 8 +++ pym/bob/builder.py | 18 +++++- pym/bob/input.py | 127 ++++++++++++++++++++++++++++++---------- pym/bob/intermediate.py | 4 ++ 4 files changed, 125 insertions(+), 32 deletions(-) diff --git a/pym/bob/audit.py b/pym/bob/audit.py index 6cb89c9d..45f7a980 100644 --- a/pym/bob/audit.py +++ b/pym/bob/audit.py @@ -81,6 +81,7 @@ class Artifact: }, "env" : str, schema.Optional('metaEnv') : { schema.Optional(str) : str }, + schema.Optional('files') : { schema.Optional(str) : str }, "scms" : [ dict ], schema.Optional("recipes") : dict, schema.Optional("layers") : dict, @@ -205,6 +206,10 @@ def addMetaEnv(self, var, value): self.__data.setdefault("metaEnv", {})[var] = value self.__invalidateId() + def addAuditFile(self, var, value): + self.__data.setdefault("files", {})[var] = value + self.__invalidateId() + def setSandbox(self, sandboxId): self.__data["dependencies"]["sandbox"] = asHexStr(sandboxId) self.__invalidateId() @@ -383,6 +388,9 @@ def addTool(self, name, tool): def addMetaEnv(self, var, value): self.__artifact.addMetaEnv(var, value) + def addAuditFile(self, var, value): + self.__artifact.addAuditFile(var, value) + def setSandbox(self, sandbox): audit = Audit.fromFile(sandbox) self.__merge(audit) diff --git a/pym/bob/builder.py b/pym/bob/builder.py index 36627f2e..39e3cd30 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -17,8 +17,9 @@ SKIPPED, EXECUTED, INFO, WARNING, DEFAULT, \ ALWAYS, IMPORTANT, NORMAL, INFO, DEBUG, TRACE from .utils import asHexStr, hashDirectory, removePath, emptyDirectory, \ - isWindows, INVALID_CHAR_TRANS, quoteCmdExe, getPlatformTag, canSymlink + isWindows, INVALID_CHAR_TRANS, quoteCmdExe, getPlatformTag, canSymlink, isAbsPath from .share import NullShare +from base64 import b64encode from shlex import quote from textwrap import dedent import argparse @@ -710,6 +711,21 @@ def auditOf(s): for var, val in step.getPackage().getMetaEnv().items(): audit.addMetaEnv(var, val) audit.setRecipesAudit(await step.getPackage().getRecipe().getRecipeSet().getScmAudit()) + for var, (fn, encoding) in step.getAuditFileNames().items(): + if isAbsPath(fn): + raise BuildError(f"Audit: {var}: Path is not relative: {fn}") + fn = os.path.join(step.getWorkspacePath(), fn) + try: + if encoding == "base64": + with open(fn, "rb") as f: + audit.addAuditFile(var, b64encode(f.read()).decode("ascii")) + else: + with open(fn, encoding=encoding) as f: + audit.addAuditFile(var, f.read()) + except (OSError, ValueError) as e: + raise BuildError(f"Audit: cannot read '{fn}': {e}") + except LookupError: + raise BuildError(f"Audit: error reading '{fn}': encoding '{encoding}' not supported") # The following things make only sense if we just executed the step if executed: diff --git a/pym/bob/input.py b/pym/bob/input.py index b2560387..7ee3fab2 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -843,10 +843,11 @@ def getUser(self): class CoreStep(CoreItem): __slots__ = ( "corePackage", "digestEnv", "env", "args", "providedEnv", "providedTools", "providedDeps", "providedSandbox", - "variantId", "deterministic", "isValid", "toolDep", "toolDepWeak" ) + "variantId", "deterministic", "isValid", "toolDep", "toolDepWeak", + "auditFileNames" ) def __init__(self, corePackage, isValid, deterministic, digestEnv, env, args, - toolDep, toolDepWeak): + toolDep, toolDepWeak, auditFileNames): self.corePackage = corePackage self.isValid = isValid self.digestEnv = digestEnv.detach() @@ -861,6 +862,7 @@ def __init__(self, corePackage, isValid, deterministic, digestEnv, env, args, self.providedTools = {} self.providedDeps = [] self.providedSandbox = None + self.auditFileNames = auditFileNames def getPreRunCmds(self): return [] @@ -1305,6 +1307,9 @@ def toolDep(self): def toolDepWeak(self): return self._coreStep.toolDepWeak + def getAuditFileNames(self): + return self._coreStep.auditFileNames + class CoreCheckoutStep(CoreStep): __slots__ = ( "scmList", "__checkoutUpdateIf", "__checkoutUpdateDeterministic", "__checkoutAsserts" ) @@ -1312,7 +1317,7 @@ class CoreCheckoutStep(CoreStep): def __init__(self, corePackage, checkout=None, checkoutSCMs=[], fullEnv=Env(), digestEnv=Env(), env=Env(), args=[], checkoutUpdateIf=[], checkoutUpdateDeterministic=True, - toolDep=set(), toolDepWeak=set()): + toolDep=set(), toolDepWeak=set(), auditFileNames={}): if checkout: recipeSet = corePackage.recipe.getRecipeSet() overrides = recipeSet.scmOverrides() @@ -1353,7 +1358,8 @@ def __init__(self, corePackage, checkout=None, checkoutSCMs=[], self.__checkoutUpdateIf = checkoutUpdateIf self.__checkoutUpdateDeterministic = checkoutUpdateDeterministic deterministic = corePackage.recipe.checkoutDeterministic - super().__init__(corePackage, isValid, deterministic, digestEnv, env, args, toolDep, toolDepWeak) + super().__init__(corePackage, isValid, deterministic, digestEnv, env, args, toolDep, + toolDepWeak, auditFileNames) def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathsConfig) @@ -1438,10 +1444,12 @@ class CoreBuildStep(CoreStep): __slots__ = ["fingerprintMask"] def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), - env=Env(), args=[], fingerprintMask=0, toolDep=set(), toolDepWeak=set()): + env=Env(), args=[], fingerprintMask=0, toolDep=set(), toolDepWeak=set(), + auditFileNames={}): isValid = script[1] is not None self.fingerprintMask = fingerprintMask - super().__init__(corePackage, isValid, True, digestEnv, env, args, toolDep, toolDepWeak) + super().__init__(corePackage, isValid, True, digestEnv, env, args, toolDep, toolDepWeak, + auditFileNames) def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathsConfig) @@ -1475,10 +1483,11 @@ class CorePackageStep(CoreStep): __slots__ = ["fingerprintMask"] def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), env=Env(), args=[], - fingerprintMask=0, toolDep=set(), toolDepWeak=set()): + fingerprintMask=0, toolDep=set(), toolDepWeak=set(), auditFileNames={}): isValid = script[1] is not None self.fingerprintMask = fingerprintMask - super().__init__(corePackage, isValid, True, digestEnv, env, args, toolDep, toolDepWeak) + super().__init__(corePackage, isValid, True, digestEnv, env, args, toolDep, toolDepWeak, + auditFileNames) def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathsConfig) @@ -1553,27 +1562,27 @@ def refDeref(self, stack, inputTools, inputSandbox, pathsConfig): def createCoreCheckoutStep(self, checkout, checkoutSCMs, fullEnv, digestEnv, env, args, checkoutUpdateIf, checkoutUpdateDeterministic, - toolDep, toolDepWeak): + toolDep, toolDepWeak, auditFileNames): ret = self.checkoutStep = CoreCheckoutStep(self, checkout, checkoutSCMs, fullEnv, digestEnv, env, args, checkoutUpdateIf, checkoutUpdateDeterministic, - toolDep, toolDepWeak) + toolDep, toolDepWeak, auditFileNames) return ret def createInvalidCoreCheckoutStep(self): ret = self.checkoutStep = CoreCheckoutStep(self) return ret - def createCoreBuildStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak): + def createCoreBuildStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak, auditFileNames): ret = self.buildStep = CoreBuildStep(self, script, digestEnv, env, args, - fingerprintMask, toolDep, toolDepWeak) + fingerprintMask, toolDep, toolDepWeak, auditFileNames) return ret def createInvalidCoreBuildStep(self, args): ret = self.buildStep = CoreBuildStep(self, args=args) return ret - def createCorePackageStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak): - ret = self.packageStep = CorePackageStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak) + def createCorePackageStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak, auditFileNames): + ret = self.packageStep = CorePackageStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak, auditFileNames) return ret def getCorePackageStep(self): @@ -1855,12 +1864,8 @@ def validate(self, data): return LayerSpec(name, RecipeSet.LAYERS_SCM_SCHEMA.validate(_data)[0]) -class VarDefineValidator: +class KeyValDefineValidator: VAR_NAME = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$') - VAR_DEF = schema.Schema({ - 'value' : str, - schema.Optional("if"): schema.Or(str, IfExpression), - }) def __init__(self, keyword, conditional=True): self.__keyword = keyword @@ -1875,31 +1880,73 @@ def validate(self, data): for key,value in sorted(data.items()): if not isinstance(key, str): raise schema.SchemaUnexpectedTypeError( - "{}: bad variable '{}'. Environment variable names must be strings!" - .format(self.__keyword, key), + f"{self.__keyword}: bad variable '{key}'. Variable names must be strings!", None) if key.startswith("BOB_"): raise schema.SchemaWrongKeyError( - "{}: bad variable '{}'. Environment variables starting with 'BOB_' are reserved!" - .format(self.__keyword, key), + f"{self.__keyword}: bad variable '{key}'. Variables starting with 'BOB_' are reserved!", None) if self.VAR_NAME.match(key) is None: raise schema.SchemaWrongKeyError( - "{}: bad variable name '{}'.".format(self.__keyword, key), + f"{self.__keyword}: bad variable name '{key}'.", None) if isinstance(value, dict) and self.__conditional: self.VAR_DEF.validate(value, error=f"{self.__keyword}: {key}: invalid definition!") - data[key] = (value['value'], value.get('if')) + data[key] = self._convertItemDict(value) elif isinstance(value, str): if self.__conditional: - data[key] = (value, None) + data[key] = self._convertItemStr(value) else: raise schema.SchemaUnexpectedTypeError( - "{}: {}: bad variable definition type." - .format(self.__keyword, key), + f"{self.__keyword}: {key}: bad variable definition type.", None) return data +class VarDefineValidator(KeyValDefineValidator): + VAR_DEF = schema.Schema({ + 'value' : str, + schema.Optional("if"): schema.Or(str, IfExpression), + }) + + def _convertItemDict(self, value): + return (value['value'], value.get('if')) + + def _convertItemStr(self, value): + return (value, None) + +class AuditFile: + __slots__ = ('filename', 'condition', 'encoding') + + def __init__(self, filename, condition=None, encoding="utf-8"): + self.filename = filename + self.condition = condition + self.encoding = encoding + + def substitute(self, env): + return (env.substitute(self.filename, "filename"), + env.substitute(self.encoding, "encoding")) + +class AuditFilesValidator(KeyValDefineValidator): + VAR_DEF = schema.Schema({ + 'filename' : str, + schema.Optional("if"): schema.Or(str, IfExpression), + schema.Optional("encoding") : str, + }) + + def _convertItemDict(self, value): + return AuditFile(value['filename'], value.get('if'), value.get('encoding', "utf-8")) + + def _convertItemStr(self, value): + return AuditFile(value) + +def substituteAuditFiles(key, auditFiles, env): + try: + return { key : val.substitute(env) + for key, val in auditFiles.items() + if env.evaluate(val.condition, key) } + except ParseError as e: + raise ParseError(f"{key}: {e.slogan}") from e + RECIPE_NAME_SCHEMA = schema.Regex(r'^[0-9A-Za-z_.+-]+$') MULTIPACKAGE_NAME_SCHEMA = schema.Regex(r'^[0-9A-Za-z_.+-]*$') @@ -2170,14 +2217,17 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b self.__varSelf = recipe.get("environment", {}) self.__varPrivate = recipe.get("privateEnvironment", {}) self.__metaEnv = recipe.get("metaEnvironment", {}) + self.__checkoutAuditFiles = recipe.get("checkoutAuditFiles", {}) self.__checkoutDeterministic = recipe.get("checkoutDeterministic") self.__checkoutVars = set(recipe.get("checkoutVars", [])) self.__checkoutVarsWeak = set(recipe.get("checkoutVarsWeak", [])) + self.__buildAuditFiles = recipe.get("buildAuditFiles", {}) self.__buildVars = set(recipe.get("buildVars", [])) self.__buildVars |= self.__checkoutVars self.__buildVarsWeak = set(recipe.get("buildVarsWeak", [])) self.__buildVarsWeak |= self.__checkoutVarsWeak self.__packageDepends = recipe.get("packageDepends") + self.__packageAuditFiles = recipe.get("packageAuditFiles", {}) self.__packageVars = set(recipe.get("packageVars", [])) self.__packageVars |= self.__buildVars self.__packageVarsWeak = set(recipe.get("packageVarsWeak", [])) @@ -2360,6 +2410,15 @@ def coDet(r): if self.__packageNetAccess is None: self.__packageNetAccess = cls.__packageNetAccess for (n, p) in self.__properties.items(): p.inherit(cls.__properties[n]) + tmp = cls.__checkoutAuditFiles.copy() + tmp.update(self.__checkoutAuditFiles) + self.__checkoutAuditFiles = tmp + tmp = cls.__buildAuditFiles.copy() + tmp.update(self.__buildAuditFiles) + self.__buildAuditFiles = tmp + tmp = cls.__packageAuditFiles.copy() + tmp.update(self.__packageAuditFiles) + self.__packageAuditFiles = tmp # the package step must always be valid if self.__package[1] is None: @@ -2798,7 +2857,8 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, srcCoreStep = p.createCoreCheckoutStep(self.__checkout, self.__checkoutSCMs, env, checkoutDigestEnv, checkoutEnv, checkoutDeps, checkoutUpdateIf, checkoutUpdateDeterministic, - toolDepCheckout, toolDepCheckoutWeak) + toolDepCheckout, toolDepCheckoutWeak, + substituteAuditFiles("checkoutAuditFiles", self.__checkoutAuditFiles, env)) else: srcCoreStep = p.createInvalidCoreCheckoutStep() @@ -2808,7 +2868,8 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, buildEnv = ( env.prune(self.__buildVars | self.__buildVarsWeak) if self.__buildVarsWeak else buildDigestEnv ) buildCoreStep = p.createCoreBuildStep(self.__build, buildDigestEnv, buildEnv, - [CoreRef(srcCoreStep)] + results, doFingerprintBuild, toolDepBuild, toolDepBuildWeak) + [CoreRef(srcCoreStep)] + results, doFingerprintBuild, toolDepBuild, toolDepBuildWeak, + substituteAuditFiles("buildAuditFiles", self.__buildAuditFiles, env)) else: buildCoreStep = p.createInvalidCoreBuildStep([CoreRef(srcCoreStep)] + results) @@ -2820,7 +2881,8 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, if self.__packageDepends: packageDeps.extend(results) packageCoreStep = p.createCorePackageStep(self.__package, packageDigestEnv, packageEnv, - packageDeps, doFingerprint, toolDepPackage, toolDepPackageWeak) + packageDeps, doFingerprint, toolDepPackage, toolDepPackageWeak, + substituteAuditFiles("packageAuditFiles", self.__packageAuditFiles, env)) # provide environment packageCoreStep.providedEnv = env.substituteCondDict(self.__provideVars, "provideVars") @@ -4179,6 +4241,9 @@ def __createSchemas(self): schema.Use(ScriptLanguage)), schema.Optional('jobServer') : schema.Or(bool, "pipe", "fifo", "fifo-or-pipe"), schema.Optional('packageDepends') : bool, + schema.Optional('checkoutAuditFiles') : AuditFilesValidator("checkoutAuditFiles"), + schema.Optional('buildAuditFiles') : AuditFilesValidator("buildAuditFiles"), + schema.Optional('packageAuditFiles') : AuditFilesValidator("packageAuditFiles"), } for (name, prop) in self.__properties.items(): classSchemaSpec[schema.Optional(name)] = schema.Schema(prop.validate, diff --git a/pym/bob/intermediate.py b/pym/bob/intermediate.py index 19b7d4d7..c80e4747 100644 --- a/pym/bob/intermediate.py +++ b/pym/bob/intermediate.py @@ -96,6 +96,7 @@ def fromStep(cls, step, graph, partial=False): self.__data['scmDirectories'] = { d : (h.hex(), p) for (d, (h, p)) in step.getScmDirectories().items() } self.__data['toolKeysWeak'] = sorted(step._coreStep.toolDepWeak) self.__data['digestEnv'] = step._coreStep.digestEnv + self.__data['auditFileNames'] = step.getAuditFileNames() return self @@ -393,6 +394,9 @@ def getUpdateScriptDigest(self): h.update((key+val).encode('utf8')) return h.digest() + def getAuditFileNames(self): + return self.__data['auditFileNames'] + class PackageIR(AbstractIR): From 7a7f1d3824f5386ddc01df30708e71e01c789882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Mon, 22 Dec 2025 16:59:40 +0100 Subject: [PATCH 2/3] doc: document checkout-/build-/packageAuditFiles --- doc/manual/audit-trail.rst | 16 +++++++++++ doc/manual/configuration.rst | 54 ++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/doc/manual/audit-trail.rst b/doc/manual/audit-trail.rst index f6cd4280..0172d6cc 100644 --- a/doc/manual/audit-trail.rst +++ b/doc/manual/audit-trail.rst @@ -74,6 +74,9 @@ Example of a single audit record:: "c5b2a8231156f43728af34f3a2dcb731ade2f76a" ] }, + "files" : { + "hashes" : "0dd432edfab90223f22e49c02e2124f87d6f0a56 ./COPYING" + }, "meta" : { "language" : "bash", "recipe" : "root", @@ -309,3 +312,16 @@ found under the ``build`` key and contains the following fields: information. The ``os-release`` field, if present, is more reliable in this case. +Audit files +~~~~~~~~~~~ + +Additional files can be included in the audit trail by using +:ref:`configuration-recipes-auditfiles`. Essentially, they are included as is +as strings into a key/value mapping under the ``files`` key. Example:: + + { + "files" : { + "hashes" : "0dd432edfab90223f22e49c02e2124f87d6f0a56 ./COPYING" + }, + } + diff --git a/doc/manual/configuration.rst b/doc/manual/configuration.rst index 61562d46..9c320649 100644 --- a/doc/manual/configuration.rst +++ b/doc/manual/configuration.rst @@ -489,6 +489,60 @@ can be configured. Recipe and class keywords ------------------------- +.. _configuration-recipes-auditfiles: + +{checkout,build,package}AuditFiles +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Type: Dictionary (String -> String | AuditFileDefinition) + +The :ref:`audit-trail` records where and when a package was built, the state of +the recipes and the checked out sources. Additionally, selected files of a step +can be included into the audit trail too. Example:: + + # Create a checksum of all files except the ".bob" folder. + checkoutDeterministic: True + checkoutScript: | + ... + mkdir .bob + find . -path ./.bob -prune -o \( -type f -print \) | xargs sha1sum > .bob/file-hashes + + checkoutAuditFiles: + FILE_HASHES: .bob/file-hashes + +This will include the content of ``.bob/file-hashes`` into the audit trail:: + + { + "files" : { + "FILE_HASHES" : "0dd432edfab90223f22e49c02e2124f87d6f0a56 ./COPYING" + }, + } + +By default, the named file(s) must be present and are read with UTF-8 encoding. +Both properties can be changed with the long format:: + + packageAuditFiles: + COPYING: + filename: COPYING + encoding: latin1 + if: "$INCLUDE_COPYING" + +The file is only added to the audit trail when the ``if`` :ref:`condition +` is true. The file name must always be a +relative path. File names and encodings can use +:ref:`configuration-principle-subst`. There is a special encoding ``"base64"`` +which can read binary file and includes them base64 encoded into the audit +trail. See the `Python standard encodings +`_ for a list +of possible encodings. + +Note that changing any of the audit files properties does not lead to a rebuild +of affected packages. These settings do not influence the build result and +therefore also do not contribute to variant management. If two identical +packages use different audit file settings it is unspecified which setting is +applied. Therefore, keep the audit file settings static or ensure that they +are configured consistent between package variants. + .. _configuration-recipes-scripts: {checkout,build,package}Script[{Bash,Pwsh}] From 03f3c25e1ad4c0278113fb14d7c70ec5e89b16f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Mon, 22 Dec 2025 17:13:21 +0100 Subject: [PATCH 3/3] test: add audit files tests --- test/black-box/audit/config.yaml | 2 +- test/black-box/audit/extract.py | 10 ++ test/black-box/audit/recipes/root.yaml | 5 + test/black-box/audit/run.sh | 10 ++ test/unit/test_input_recipeset.py | 136 +++++++++++++++++++++++++ 5 files changed, 162 insertions(+), 1 deletion(-) create mode 100755 test/black-box/audit/extract.py diff --git a/test/black-box/audit/config.yaml b/test/black-box/audit/config.yaml index f7286367..6877518f 100644 --- a/test/black-box/audit/config.yaml +++ b/test/black-box/audit/config.yaml @@ -1 +1 @@ -bobMinimumVersion: "0.16" +bobMinimumVersion: "1.1" diff --git a/test/black-box/audit/extract.py b/test/black-box/audit/extract.py new file mode 100755 index 00000000..e430b8f6 --- /dev/null +++ b/test/black-box/audit/extract.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +import gzip +import io +import json +import sys + +with gzip.open(sys.argv[1], 'rb') as gzf: + tree = json.load(io.TextIOWrapper(gzf, encoding='utf8')) + print(tree["artifact"]["files"][sys.argv[2]]) diff --git a/test/black-box/audit/recipes/root.yaml b/test/black-box/audit/recipes/root.yaml index 1f5d43ed..a147a8fc 100644 --- a/test/black-box/audit/recipes/root.yaml +++ b/test/black-box/audit/recipes/root.yaml @@ -10,5 +10,10 @@ buildScript: | cp $1/root.txt . cp $2/foo.txt . +packageAuditFiles: + ROOT: root.txt + FOO: + filename: foo.txt + encoding: base64 packageScript: | cp $1/* . diff --git a/test/black-box/audit/run.sh b/test/black-box/audit/run.sh index 2daa9f2c..a9b2fa67 100755 --- a/test/black-box/audit/run.sh +++ b/test/black-box/audit/run.sh @@ -31,3 +31,13 @@ test -z "$(/usr/bin/find "$archiveDir" -type f -name '*.tgz')" # Rebuild forced and upload again. Now it must create artifcats in the archive. run_bob dev -v --upload -f root test -n "$(/usr/bin/find "$archiveDir" -type f -name '*.tgz')" + +# Verify additional files in audit trail +expect_equal "$(./extract.py dev/dist/root/1/audit.json.gz ROOT)" foo +expect_equal "$(./extract.py dev/dist/root/1/audit.json.gz FOO)" "$(echo foo | base64)" + +# Provoke audit errors +expect_fail run_bob dev audit-absolute +expect_fail run_bob dev audit-missing +expect_fail run_bob dev audit-encoding-error +expect_fail run_bob dev audit-invalid-encoding diff --git a/test/unit/test_input_recipeset.py b/test/unit/test_input_recipeset.py index 2d37ffd1..aa79bfdc 100644 --- a/test/unit/test_input_recipeset.py +++ b/test/unit/test_input_recipeset.py @@ -2615,3 +2615,139 @@ def testPrecedence(self): ScmOverride({ "set" : { "url" : "include_l1" }}), ScmOverride({ "set" : { "url" : "include_l2" }}) ], cfg.scmOverrides()) + +class TestAuditFiles(RecipesTmp, TestCase): + """Test packageAuditFiles """ + + def testSimple(self): + """Simple packageAuditFiles""" + self.writeRecipe("root", """\ + root: True + checkoutScript: "true" + checkoutAuditFiles: + CHECKOUT: some + buildScript: "true" + buildAuditFiles: + BUILD: files + packageAuditFiles: + FOO: foo + BAR: bar + """) + + p = self.generate().walkPackagePath("root") + + af = p.getCheckoutStep().getAuditFileNames() + self.assertEqual(set(af.keys()), {"CHECKOUT"}) + self.assertEqual(af["CHECKOUT"], ("some", "utf-8")) + + af = p.getBuildStep().getAuditFileNames() + self.assertEqual(set(af.keys()), {"BUILD"}) + self.assertEqual(af["BUILD"], ("files", "utf-8")) + + for name, (filename, encoding) in p.getPackageStep().getAuditFileNames().items(): + self.assertIn(name, {"FOO", "BAR"}) + self.assertEqual(name.lower(), filename) + self.assertEqual(encoding, "utf-8") + + def testInherit(self): + """Inherited classes are merged on a key-by-key basis""" + self.writeRecipe("root", """\ + root: True + inherit: [cls] + checkoutScript: "true" + buildScript: "true" + buildAuditFiles: + BUILD: files + packageAuditFiles: + BAR: bar + """) + self.writeClass("cls", """\ + checkoutAuditFiles: + CHECKOUT: some + buildAuditFiles: + BUILD: xxxxx + packageAuditFiles: + FOO: "$FOO" + BAR: baz + """) + + p = self.generate(env={"FOO" : "foo"}).walkPackagePath("root") + + af = p.getCheckoutStep().getAuditFileNames() + self.assertEqual(set(af.keys()), {"CHECKOUT"}) + self.assertEqual(af["CHECKOUT"], ("some", "utf-8")) + + af = p.getBuildStep().getAuditFileNames() + self.assertEqual(set(af.keys()), {"BUILD"}) + self.assertEqual(af["BUILD"], ("files", "utf-8")) + + for name, (filename, encoding) in p.getPackageStep().getAuditFileNames().items(): + self.assertIn(name, {"FOO", "BAR"}) + self.assertEqual(name.lower(), filename) + self.assertEqual(encoding, "utf-8") + + def testCustomEncoding(self): + """A custom encoding can be set per audit file""" + self.writeRecipe("root", """\ + root: True + packageAuditFiles: + FOO: + filename: foo + encoding: custom + """) + + p = self.generate().walkPackagePath("root") + filename, encoding = p.getPackageStep().getAuditFileNames()["FOO"] + self.assertEqual(filename, "foo") + self.assertEqual(encoding, "custom") + + def testSubstitution(self): + """Audit filename and encoding are string substituted""" + self.writeRecipe("root", """\ + root: True + packageAuditFiles: + FOO: + filename: "${FN:-foo}" + encoding: "${ENC:-utf8}" + """) + + p = self.generate().walkPackagePath("root") + filename, encoding = p.getPackageStep().getAuditFileNames()["FOO"] + self.assertEqual(filename, "foo") + self.assertEqual(encoding, "utf8") + + p = self.generate(env={"FN" : "bar", "ENC" : "binary"}).walkPackagePath("root") + filename, encoding = p.getPackageStep().getAuditFileNames()["FOO"] + self.assertEqual(filename, "bar") + self.assertEqual(encoding, "binary") + + def testConditional(self): + self.writeRecipe("root", """\ + root: True + packageAuditFiles: + FOO: + filename: foo + if: "${FOO}" + BAR: + filename: bar + if: !expr >- + "$BAR" == "enabled" + """) + + with self.assertRaises(ParseError): + self.generate().walkPackagePath("root") + + p = self.generate(env={"FOO" : "0"}).walkPackagePath("root") + self.assertEqual(set(p.getPackageStep().getAuditFileNames().keys()), set()) + + p = self.generate(env={"FOO" : "1"}).walkPackagePath("root") + self.assertEqual(set(p.getPackageStep().getAuditFileNames().keys()), {'FOO'}) + filename, encoding = p.getPackageStep().getAuditFileNames()["FOO"] + self.assertEqual(filename, "foo") + self.assertEqual(encoding, "utf-8") + + p = self.generate(env={"FOO" : "true", "BAR" : "enabled"}).walkPackagePath("root") + self.assertEqual(set(p.getPackageStep().getAuditFileNames().keys()), {'FOO', 'BAR'}) + filename, encoding = p.getPackageStep().getAuditFileNames()["BAR"] + self.assertEqual(filename, "bar") + self.assertEqual(encoding, "utf-8")