From dfb958389ac481a39edddb94bc51e9e50ff9ffb2 Mon Sep 17 00:00:00 2001 From: cemeceme <26171877+cemeceme@users.noreply.github.com> Date: Wed, 22 May 2024 15:39:46 +0200 Subject: [PATCH 1/9] Added dependency check after post_build step. --- src/build/build_step.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/build/build_step.go b/src/build/build_step.go index 1c274c2ab..b87b878df 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -78,6 +78,23 @@ func Build(state *core.BuildState, target *core.BuildTarget, remote bool) { } else { successfulLocalTargetBuildDuration.Observe(float64(time.Since(start).Milliseconds())) } + + // Check if the target had a post build step and make sure to handle any dependencies added within it. + if target.PostBuildFunction != nil { + // Use declared dependencies and get the targets from their labels, because addDep creates a depInfo struct with the deps field unpopulated. + for _, label := range target.DeclaredDependencies() { + depTarget := state.Graph.Target(label) + depTarget.WaitForBuild() + if depTarget.State() >= core.DependencyFailed { // Either the target failed or its dependencies failed + // Give up and set the original target as dependency failed + target.SetState(core.DependencyFailed) + state.LogBuildResult(target, core.TargetBuilt, "Dependency failed") + target.FinishBuild() + return + } + } + } + // Mark the target as having finished building. target.FinishBuild() if target.IsTest() && state.NeedTests && state.IsOriginalTarget(target) { From 89ff26dd510320f7c34d5fc93f3771f018b7daa8 Mon Sep 17 00:00:00 2001 From: cemeceme <26171877+cemeceme@users.noreply.github.com> Date: Fri, 21 Jun 2024 13:05:32 +0200 Subject: [PATCH 2/9] Refactored post build dependencies and fixed exported dependencies. --- src/build/build_step.go | 26 ++++++++++---------------- src/core/build_target.go | 18 ++++++++++++++++++ src/parse/asp/builtins.go | 1 + 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/build/build_step.go b/src/build/build_step.go index b87b878df..c74b1e0ac 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -79,22 +79,6 @@ func Build(state *core.BuildState, target *core.BuildTarget, remote bool) { successfulLocalTargetBuildDuration.Observe(float64(time.Since(start).Milliseconds())) } - // Check if the target had a post build step and make sure to handle any dependencies added within it. - if target.PostBuildFunction != nil { - // Use declared dependencies and get the targets from their labels, because addDep creates a depInfo struct with the deps field unpopulated. - for _, label := range target.DeclaredDependencies() { - depTarget := state.Graph.Target(label) - depTarget.WaitForBuild() - if depTarget.State() >= core.DependencyFailed { // Either the target failed or its dependencies failed - // Give up and set the original target as dependency failed - target.SetState(core.DependencyFailed) - state.LogBuildResult(target, core.TargetBuilt, "Dependency failed") - target.FinishBuild() - return - } - } - } - // Mark the target as having finished building. target.FinishBuild() if target.IsTest() && state.NeedTests && state.IsOriginalTarget(target) { @@ -360,6 +344,16 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b return err } + // Wait for any new dependencies added by post-build commands before continuing. + for _, dep := range target.Dependencies() { + dep.WaitForBuild() + if dep.State() >= core.DependencyFailed { // Either the target failed or its dependencies failed + // Give up and set the original target as dependency failed + target.SetState(core.DependencyFailed) + return fmt.Errorf("error in post-rule dependency for %s: %s", target.Label, dep.Label) + } + } + if runRemotely && len(outs) != len(target.Outputs()) { // postBuildFunction has changed the target - must rebuild it log.Info("Rebuilding %s after post-build function", target) diff --git a/src/core/build_target.go b/src/core/build_target.go index 166dbb515..b9f06f7da 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -677,6 +677,15 @@ func (target *BuildTarget) ExportedDependencies() []BuildLabel { defer target.mutex.RUnlock() ret := make(BuildLabels, 0, len(target.dependencies)) for _, info := range target.dependencies { + + logString := "" + for i, dep := range info.deps { + logString += dep.Label.Name + if i+1 < len(info.deps) { + logString += ", " + } + } + log.Info("Exported deps call for target %s: %s , exported: %t", target.Label, logString, info.exported) if info.exported { ret = append(ret, *info.declared) } @@ -1653,6 +1662,15 @@ func (target *BuildTarget) AddMaybeExportedDependency(dep BuildLabel, exported, } } +// RegisterDependencyTarget registers a build target to be used for a dependency label on the given target. +func (target *BuildTarget) RegisterDependencyTarget(dep BuildLabel, deptarget *BuildTarget) { + info := target.dependencyInfo(dep) + if info == nil { + log.Fatalf("Target %s doesn't contain dependency %s.\n", target.Label, dep) + } + info.deps = append(info.deps, deptarget) +} + // IsTool returns true if the given build label is a tool used by this target. func (target *BuildTarget) IsTool(tool BuildLabel) bool { for _, t := range target.Tools { diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index e3a04b53a..cc82f74b6 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -1125,6 +1125,7 @@ func addDep(s *scope, args []pyObject) pyObject { dep := s.parseLabelInPackage(string(args[1].(pyString)), s.pkg) exported := args[2].IsTruthy() target.AddMaybeExportedDependency(dep, exported, false, false) + target.RegisterDependencyTarget(dep, s.state.Graph.Target(dep)) // Queue this dependency if it'll be needed. if target.State() > core.Inactive { err := s.state.QueueTarget(dep, target.Label, false, core.ParseModeNormal) From cefd2c25a37ed61f06df5d40e9bfdcc7d37610ec Mon Sep 17 00:00:00 2001 From: cemeceme <26171877+cemeceme@users.noreply.github.com> Date: Tue, 9 Jul 2024 17:06:52 +0200 Subject: [PATCH 3/9] Removed leading newline. --- src/core/build_target.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/build_target.go b/src/core/build_target.go index b9f06f7da..98beb6f58 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -677,7 +677,6 @@ func (target *BuildTarget) ExportedDependencies() []BuildLabel { defer target.mutex.RUnlock() ret := make(BuildLabels, 0, len(target.dependencies)) for _, info := range target.dependencies { - logString := "" for i, dep := range info.deps { logString += dep.Label.Name From 9a94e313c9264f91de8da5d884467bcd5f74dbb3 Mon Sep 17 00:00:00 2001 From: cemeceme <26171877+cemeceme@users.noreply.github.com> Date: Sat, 10 Aug 2024 18:58:59 +0200 Subject: [PATCH 4/9] Linter fixes. --- src/core/build_target.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/build_target.go b/src/core/build_target.go index 98beb6f58..9c6586940 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -1666,8 +1666,9 @@ func (target *BuildTarget) RegisterDependencyTarget(dep BuildLabel, deptarget *B info := target.dependencyInfo(dep) if info == nil { log.Fatalf("Target %s doesn't contain dependency %s.\n", target.Label, dep) + } else { + info.deps = append(info.deps, deptarget) } - info.deps = append(info.deps, deptarget) } // IsTool returns true if the given build label is a tool used by this target. From 3261eec309c180f8e0db9fb6dc318718145651b5 Mon Sep 17 00:00:00 2001 From: cemeceme <26171877+cemeceme@users.noreply.github.com> Date: Tue, 13 Aug 2024 15:24:33 +0300 Subject: [PATCH 5/9] Removed debug print. --- src/build/build_step.go | 1 - src/core/build_target.go | 8 -------- 2 files changed, 9 deletions(-) diff --git a/src/build/build_step.go b/src/build/build_step.go index c74b1e0ac..d028eb67d 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -78,7 +78,6 @@ func Build(state *core.BuildState, target *core.BuildTarget, remote bool) { } else { successfulLocalTargetBuildDuration.Observe(float64(time.Since(start).Milliseconds())) } - // Mark the target as having finished building. target.FinishBuild() if target.IsTest() && state.NeedTests && state.IsOriginalTarget(target) { diff --git a/src/core/build_target.go b/src/core/build_target.go index 9c6586940..fb91a935b 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -677,14 +677,6 @@ func (target *BuildTarget) ExportedDependencies() []BuildLabel { defer target.mutex.RUnlock() ret := make(BuildLabels, 0, len(target.dependencies)) for _, info := range target.dependencies { - logString := "" - for i, dep := range info.deps { - logString += dep.Label.Name - if i+1 < len(info.deps) { - logString += ", " - } - } - log.Info("Exported deps call for target %s: %s , exported: %t", target.Label, logString, info.exported) if info.exported { ret = append(ret, *info.declared) } From dc4857e44e21deb64e59514b8bb4992735792b0c Mon Sep 17 00:00:00 2001 From: cemeceme <26171877+cemeceme@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:05:16 +0100 Subject: [PATCH 6/9] Added pre-build dependency blocking --- src/build/build_step.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/build/build_step.go b/src/build/build_step.go index d028eb67d..f975db6b7 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -185,6 +185,16 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b return err } log.Debug("Finished pre-build function for %s", target.Label) + + // Wait for any new dependencies added by pre-build commands before continuing. + for _, dep := range target.Dependencies() { + dep.WaitForBuild() + if dep.State() >= core.DependencyFailed { // Either the target failed or its dependencies failed + // Give up and set the original target as dependency failed + target.SetState(core.DependencyFailed) + return fmt.Errorf("error in pre-rule dependency for %s: %s", target.Label, dep.Label) + } + } } state.LogBuildResult(target, core.TargetBuilding, "Preparing...") From 82eb0ab89438b0142d747b75e92d1cf9b8087c07 Mon Sep 17 00:00:00 2001 From: cemeceme <26171877+cemeceme@users.noreply.github.com> Date: Sun, 29 Dec 2024 03:21:00 +0100 Subject: [PATCH 7/9] Update parameters for the new args of WaitForBuild --- src/build/build_step.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build/build_step.go b/src/build/build_step.go index 4f73978f7..4344f3995 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -185,7 +185,7 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b // Wait for any new dependencies added by pre-build commands before continuing. for _, dep := range target.Dependencies() { - dep.WaitForBuild() + dep.WaitForBuild(target.Label) if dep.State() >= core.DependencyFailed { // Either the target failed or its dependencies failed // Give up and set the original target as dependency failed target.SetState(core.DependencyFailed) @@ -352,7 +352,7 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b // Wait for any new dependencies added by post-build commands before continuing. for _, dep := range target.Dependencies() { - dep.WaitForBuild() + dep.WaitForBuild(target.Label) if dep.State() >= core.DependencyFailed { // Either the target failed or its dependencies failed // Give up and set the original target as dependency failed target.SetState(core.DependencyFailed) From 580a211f67961bf413eeb66eb75f98c822217716 Mon Sep 17 00:00:00 2001 From: cemeceme <26171877+cemeceme@users.noreply.github.com> Date: Tue, 28 Jan 2025 09:16:01 +0100 Subject: [PATCH 8/9] Added single output label support. --- src/build/build_step.go | 1 + src/core/build_input.go | 45 ++++++++++++++++++++++++++++++++++++ src/parse/asp/interpreter.go | 10 ++++++++ 3 files changed, 56 insertions(+) diff --git a/src/build/build_step.go b/src/build/build_step.go index 4344f3995..fc63a6c91 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -192,6 +192,7 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b return fmt.Errorf("error in pre-rule dependency for %s: %s", target.Label, dep.Label) } } + log.Debug("Finished waiting for dependencies for %s", target.Label) } state.LogBuildResult(target, core.TargetBuilding, "Preparing...") diff --git a/src/core/build_input.go b/src/core/build_input.go index 010e047b0..d066338fc 100644 --- a/src/core/build_input.go +++ b/src/core/build_input.go @@ -220,6 +220,51 @@ func (label SystemPathLabel) String() string { return label.Name } +// SingleOutputLabel represents a build label for a specific output of a rule. +// This can be used to target a specific output of this rule when depended on or an entry point when used in +// the context of tools. +type SingleOutputLabel struct { + BuildLabel + Output string +} + +// MarshalText implements the encoding.TextMarshaler interface, which makes AnnotatedOutputLabel +// usable as map keys in JSON. +func (label SingleOutputLabel) MarshalText() ([]byte, error) { + return []byte(label.String()), nil +} + +// Paths returns a slice of paths to the files of this input. +func (label SingleOutputLabel) Paths(graph *BuildGraph) []string { + target := graph.TargetOrDie(label.BuildLabel) + return addPathPrefix([]string{label.Output}, target.PackageDir()) +} + +// FullPaths is like Paths but includes the leading plz-out/gen directory. +func (label SingleOutputLabel) FullPaths(graph *BuildGraph) []string { + target := graph.TargetOrDie(label.BuildLabel) + return addPathPrefix([]string{label.Output}, target.OutDir()) +} + +// LocalPaths returns paths within the local package +func (label SingleOutputLabel) LocalPaths(graph *BuildGraph) []string { + return []string{label.Output} +} + +// Label returns the build rule associated with this input. For a SingleOutputLabel it's always non-nil. +func (label SingleOutputLabel) Label() (BuildLabel, bool) { + return label.BuildLabel, true +} + +func (label SingleOutputLabel) nonOutputLabel() (BuildLabel, bool) { + return BuildLabel{}, false +} + +// String returns a string representation of this input. +func (label SingleOutputLabel) String() string { + return label.BuildLabel.String() + "+" + label.Output +} + // AnnotatedOutputLabel represents a build label with an annotation e.g. //foo:bar|baz where baz constitutes the // annotation. This can be used to target a named output of this rule when depended on or an entry point when used in // the context of tools. diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index df5272982..35cf72b4c 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -310,6 +310,16 @@ type scope struct { // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided // package. It may return an AnnotatedOutputLabel or a BuildLabel depending on if the label is annotated. func (s *scope) parseAnnotatedLabelInPackage(label string, pkg *core.Package) core.BuildInput { + if(strings.Contains(label, "+")){ + parts := strings.Split(label, "+") + if len(parts) == 2 { + return core.SingleOutputLabel{ + BuildLabel: s.parseLabelInPackage(parts[0], pkg), + Output: parts[1], + } + } + } + label, annotation := core.SplitLabelAnnotation(label) if annotation != "" { return core.AnnotatedOutputLabel{ From 457ff8c7d65b5c4db708aa7bf0a3972789332be1 Mon Sep 17 00:00:00 2001 From: cemeceme <26171877+cemeceme@users.noreply.github.com> Date: Sun, 2 Feb 2025 19:39:37 +0100 Subject: [PATCH 9/9] Linter fixes. --- src/build/build_step.go | 2 +- src/parse/asp/interpreter.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/build/build_step.go b/src/build/build_step.go index fc63a6c91..82fdc1565 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -182,7 +182,7 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b return err } log.Debug("Finished pre-build function for %s", target.Label) - + // Wait for any new dependencies added by pre-build commands before continuing. for _, dep := range target.Dependencies() { dep.WaitForBuild(target.Label) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index d3360efe4..c09cb4ac4 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -310,12 +310,12 @@ type scope struct { // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided // package. It may return an AnnotatedOutputLabel or a BuildLabel depending on if the label is annotated. func (s *scope) parseAnnotatedLabelInPackage(label string, pkg *core.Package) core.BuildInput { - if(strings.Contains(label, "+")){ + if strings.Contains(label, "+") { parts := strings.Split(label, "+") if len(parts) == 2 { return core.SingleOutputLabel{ BuildLabel: s.parseLabelInPackage(parts[0], pkg), - Output: parts[1], + Output: parts[1], } } }