From 7dff39a234cf227d74abf781a259cd7fb16a79b2 Mon Sep 17 00:00:00 2001 From: ChrisRackauckas-Claude Date: Tue, 16 Jun 2026 05:51:07 -0400 Subject: [PATCH 1/2] Fix: don't develop a dep's test-only [sources] transitively (Julia <1.11) The tests.yml "Develop in-repo [sources] path deps" step walked the [sources] graph transitively on Julia <1.11. When the package-under-test sources an in-repo dependency that itself declares test-only [sources] (names in [extras]/[targets].test, not [deps]), those test-only path deps were Pkg.develop-ed into the active environment as phantom direct deps. Aqua's stale-deps and deps-compat checks then failed. This is what makes every Optimization.jl sublibrary's QA group fail on Julia lts: OptimizationBase's test-only [sources] (OptimizationLBFGSB, OptimizationManopt) were pulled into ~26 dependent sublibraries' envs. QA on Julia >=1.11 is green because the whole helper is a no-op there. Fix: extract the walk into scripts/develop_sources.jl and, when recursing into an already-developed dependency, only follow [sources] entries that are also that dependency's runtime [deps]. The package-under-test itself (the root of the walk) is exempt, so its own test-only [sources] are still developed for its own test suite. tests.yml now checks out SciML/.github (at dotgithub-ref, default v1, matching the sibling workflows) and calls the script, giving a single tested source of truth. Adds test/runtests.jl coverage asserting a dependency's test-only [sources] are excluded while the runtime transitive chain (and the root's own test-only sources) are still developed. Fixes SciML/Optimization.jl#1228 Co-Authored-By: Chris Rackauckas Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/tests.yml | 52 ++++++++------------ scripts/develop_sources.jl | 96 +++++++++++++++++++++++++++++++++++++ test/runtests.jl | 83 ++++++++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 31 deletions(-) create mode 100644 scripts/develop_sources.jl diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9ef9114..7e63549 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -106,6 +106,11 @@ on: default: false required: false type: boolean + dotgithub-ref: + description: "Ref of SciML/.github to source the develop-sources helper script from" + default: "v1" + required: false + type: string jobs: tests: @@ -139,41 +144,26 @@ jobs: with: token: "${{ secrets.GITHUB_TOKEN }}" + - name: "Checkout SciML/.github for the develop-sources helper" + if: "${{ inputs.buildpkg && inputs.project != '@.' && inputs.project != '.' }}" + uses: actions/checkout@v6 + with: + repository: SciML/.github + ref: "${{ inputs.dotgithub-ref }}" + path: .sciml-dotgithub + - name: "Develop in-repo [sources] path deps of ${{ inputs.project }}" + # On Julia < 1.11 the [sources] section is not auto-resolved during + # build/test, so develop the in-repo path deps the sub-project (e.g. a + # lib/* sublibrary) declares -- transitively, but skipping a developed + # dependency's *test-only* [sources] so they don't become phantom direct + # deps (SciML/Optimization.jl#1228). No-op on >= 1.11. See + # scripts/develop_sources.jl for the full rationale. if: "${{ inputs.buildpkg && inputs.project != '@.' && inputs.project != '.' }}" shell: julia --color=yes {0} run: | - using Pkg - proj = raw"${{ inputs.project }}" - # On Julia < 1.11 the [sources] section is not auto-resolved during - # build/test, so develop the in-repo path deps the sub-project (e.g. a - # lib/* sublibrary) declares -- transitively, since a source dep can - # itself declare further [sources]. No-op on >= 1.11 and for projects - # without a [sources] section. - if VERSION < v"1.11.0-DEV.0" - Pkg.activate(proj) - developed = Set{String}([normpath(abspath(proj))]) # never develop the active project (cyclic [sources]) - specs = Pkg.PackageSpec[] - queue = String[proj] - while !isempty(queue) - dir = popfirst!(queue) - tomlpath = joinpath(dir, "Project.toml") - isfile(tomlpath) || continue - toml = Pkg.TOML.parsefile(tomlpath) - haskey(toml, "sources") || continue - for (dep, spec) in toml["sources"] - if spec isa Dict && haskey(spec, "path") - p = normpath(abspath(joinpath(dir, spec["path"]))) - if isdir(p) && !(p in developed) - push!(developed, p) - push!(specs, Pkg.PackageSpec(path = p)) - push!(queue, p) # resolve this dep's own [sources] too - end - end - end - end - isempty(specs) || Pkg.develop(specs) - end + include(joinpath(".sciml-dotgithub", "scripts", "develop_sources.jl")) + develop_sources(raw"${{ inputs.project }}") - name: "Install system packages" if: "${{ inputs.apt-packages != '' && runner.os == 'Linux' }}" diff --git a/scripts/develop_sources.jl b/scripts/develop_sources.jl new file mode 100644 index 0000000..76f895c --- /dev/null +++ b/scripts/develop_sources.jl @@ -0,0 +1,96 @@ +#!/usr/bin/env julia +# +# Develop the in-repo [sources] path deps of a sub-project. +# +# On Julia < 1.11 the [sources] table is ignored when an environment is +# resolved/built/tested, so a monorepo sublibrary (e.g. lib/) that relies +# on [sources] to pin its in-repo siblings would otherwise resolve them as +# registered packages. This script restores the 1.11+ behavior on 1.10 (the +# SciML LTS) by Pkg.develop-ing each in-repo `path =` source by path. On Julia +# >= 1.11 it is a no-op (the table is honored natively). +# +# The walk is transitive: a developed source dep can itself declare further +# *runtime* [sources] that must also be developed for the environment to load. +# BUT when recursing into an already-developed dependency, only [sources] that +# are also that dependency's runtime [deps] are followed. A dependency's +# [sources] table may contain entries that exist purely for that dependency's +# own test suite -- their names live in [extras]/[targets].test, not [deps]. +# Developing those would inject phantom direct deps into the package-under-test's +# environment, which then trips Aqua's stale-deps / deps-compat checks +# (SciML/Optimization.jl#1228). The package-under-test itself (the root of the +# walk) is exempt from this filter: its own test-only [sources] are legitimately +# needed when its own test suite runs. +# +# Usage (from a workflow step): +# include("scripts/develop_sources.jl") +# develop_sources(project_dir) +# +# `develop_sources` activates `project_dir`, computes the source paths to +# develop via `collect_source_paths`, and Pkg.develop-s them. The pure +# path-collection logic is split out so it can be unit-tested without mutating +# any environment. + +using Pkg + +""" + collect_source_paths(proj) -> Vector{String} + +Walk the `[sources]` graph rooted at `proj` (a directory containing a +`Project.toml`) and return the ordered list of absolute `path` sources to +develop. Each `path` source is resolved relative to the `Project.toml` that +declares it. The walk recurses into a developed dependency's own `[sources]`, +but for non-root projects only follows sources that are also runtime `[deps]` +of that project (skipping the dep's test-only sources). The root project itself +is excluded from the result (it is the active project; developing it would be a +cyclic self-develop). Only `path =` sources are returned; `url`/`rev` git +sources are left to `Pkg`. Visiting each resolved path once handles cycles and +diamonds in the graph. This function is version-independent and mutates nothing. +""" +function collect_source_paths(proj::AbstractString) + projroot = normpath(abspath(proj)) + developed = Set{String}([projroot]) # never develop the active project + paths = String[] + queue = String[projroot] + while !isempty(queue) + dir = popfirst!(queue) + tomlpath = joinpath(dir, "Project.toml") + isfile(tomlpath) || continue + toml = Pkg.TOML.parsefile(tomlpath) + sources = get(toml, "sources", nothing) + sources isa AbstractDict || continue + isroot = normpath(abspath(dir)) == projroot + runtimedeps = keys(get(toml, "deps", Dict{String, Any}())) + for (dep, spec) in sources + # For dependencies (not the package-under-test), skip [sources] that + # are not runtime deps -- those are the dep's test-only sources and + # must not leak into the active environment. + isroot || dep in runtimedeps || continue + spec isa AbstractDict && haskey(spec, "path") || continue + p = normpath(abspath(joinpath(dir, spec["path"]))) + if isdir(p) && !(p in developed) + push!(developed, p) + push!(paths, p) + push!(queue, p) # resolve this dep's own runtime [sources] too + end + end + end + return paths +end + +""" + develop_sources(proj) + +Activate `proj` and, on Julia < 1.11, `Pkg.develop` its in-repo `[sources]` +path deps (see `collect_source_paths`). No-op on Julia >= 1.11. +""" +function develop_sources(proj::AbstractString) + Pkg.activate(proj) + VERSION < v"1.11.0-DEV.0" || return nothing + paths = collect_source_paths(proj) + isempty(paths) || Pkg.develop([Pkg.PackageSpec(path = p) for p in paths]) + return nothing +end + +if abspath(PROGRAM_FILE) == @__FILE__ + develop_sources(ARGS[1]) +end diff --git a/test/runtests.jl b/test/runtests.jl index ea4afdc..444566f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -346,3 +346,86 @@ end ["self-hosted", "Linux", "X64", "gpu"] end end + +# develop_sources.jl: the [sources] develop helper used by tests.yml on Julia +# <1.11. The pure path-collection (collect_source_paths) is tested here without +# mutating any environment. +include(joinpath(@__DIR__, "..", "scripts", "develop_sources.jl")) + +# Build a fixture monorepo. `Top` runtime-sources `Mid`; `Mid` runtime-sources +# `Leaf` (in its [deps]) and test-only-sources `TestOnly` (only in its +# [extras]/[targets].test, NOT [deps]). The walk from `Top` must develop the +# runtime chain {Mid, Leaf} and must NOT pull in `Mid`'s test-only `TestOnly`. +function make_sources_fixture() + root = mktempdir() + function pkg(name, toml) + d = joinpath(root, name) + mkpath(joinpath(d, "src")) + write(joinpath(d, "Project.toml"), toml) + write(joinpath(d, "src", "$name.jl"), "module $name\nend\n") + return d + end + pkg("Leaf", "name = \"Leaf\"\nuuid = \"11111111-1111-1111-1111-111111111111\"\n") + pkg("TestOnly", "name = \"TestOnly\"\nuuid = \"44444444-4444-4444-4444-444444444444\"\n") + pkg( + "Mid", """ + name = "Mid" + uuid = "22222222-2222-2222-2222-222222222222" + + [deps] + Leaf = "11111111-1111-1111-1111-111111111111" + + [sources] + Leaf = {path = "../Leaf"} + TestOnly = {path = "../TestOnly"} + + [extras] + TestOnly = "44444444-4444-4444-4444-444444444444" + + [targets] + test = ["TestOnly"] + """ + ) + pkg( + "Top", """ + name = "Top" + uuid = "33333333-3333-3333-3333-333333333333" + + [deps] + Mid = "22222222-2222-2222-2222-222222222222" + + [sources] + Mid = {path = "../Mid"} + """ + ) + return root +end + +@testset "develop_sources: runtime [sources] developed, dep test-only [sources] excluded" begin + root = make_sources_fixture() + names = sort(basename.(collect_source_paths(joinpath(root, "Top")))) + # Runtime transitive chain Top -> Mid -> Leaf is developed. + @test names == ["Leaf", "Mid"] + # Mid's test-only source (TestOnly) is NOT pulled into Top's env + # (SciML/Optimization.jl#1228). + @test "TestOnly" ∉ names + # The active project (Top) is never self-developed. + @test "Top" ∉ names +end + +@testset "develop_sources: package-under-test's OWN test-only [sources] ARE developed" begin + # When the project at the root of the walk is itself being tested, its + # test-only [sources] (e.g. OptimizationBase developing LBFGSB/Manopt for + # its own test suite) must still be developed -- only a *dependency's* + # test-only sources are filtered out. + root = make_sources_fixture() + names = sort(basename.(collect_source_paths(joinpath(root, "Mid")))) + # Testing Mid directly: its runtime source (Leaf) AND its test-only source + # (TestOnly) are both developed; only Leaf is then recursed into. + @test names == ["Leaf", "TestOnly"] +end + +@testset "develop_sources: no [sources] table -> nothing to develop" begin + root = make_sources_fixture() + @test isempty(collect_source_paths(joinpath(root, "Leaf"))) +end From 9da789d18fa340ae3562c99537336f5243ca8df5 Mon Sep 17 00:00:00 2001 From: ChrisRackauckas-Claude Date: Tue, 16 Jun 2026 07:45:59 -0400 Subject: [PATCH 2/2] CI: pin actionlint job to GitHub-hosted ubuntu-24.04 The actionlint+shellcheck job uses a docker:// action, but `runs-on: ubuntu-latest` lets it land on the self-hosted pool (which squats `ubuntu-latest` and cannot pull/run Docker images) -> intermittent "Docker pull failed with exit code 1". Self-hosted runners do not carry the `ubuntu-24.04` label, so pinning forces a GitHub-hosted runner where Docker works. The Julia script-tests job is left on ubuntu-latest (no Docker; self-hosted is fine). Co-Authored-By: Chris Rackauckas Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9e78ee..a316482 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,10 @@ concurrency: jobs: actionlint: name: "actionlint + shellcheck" - runs-on: ubuntu-latest + # Pinned to a GitHub-hosted runner: this job pulls a docker:// image, which the + # self-hosted pool (squats `ubuntu-latest`, no Docker) fails to pull. Self-hosted + # runners do not carry the `ubuntu-24.04` label, so this forces GitHub-hosted. + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 # The rhysd/actionlint image bundles shellcheck, so `run:` scripts in the