Fix: don't develop a dependency's test-only [sources] transitively on Julia <1.11

.github/workflows/ci.yml

@@ -20,7 +20,10 @@ concurrency:
 jobs:
   actionlint:
     name: "actionlint + shellcheck"
-    runs-on: ubuntu-latest
+    # Pinned to a GitHub-hosted runner: this job pulls a docker:// image, which the
+    # self-hosted pool (squats `ubuntu-latest`, no Docker) fails to pull. Self-hosted
+    # runners do not carry the `ubuntu-24.04` label, so this forces GitHub-hosted.
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v6
       # The rhysd/actionlint image bundles shellcheck, so `run:` scripts in the

.github/workflows/tests.yml

@@ -106,6 +106,11 @@ on:
         default: false
         required: false
         type: boolean
+      dotgithub-ref:
+        description: "Ref of SciML/.github to source the develop-sources helper script from"
+        default: "v1"
+        required: false
+        type: string
 
 jobs:
   tests:
@@ -139,41 +144,26 @@ jobs:
         with:
           token: "${{ secrets.GITHUB_TOKEN }}"
 
+      - name: "Checkout SciML/.github for the develop-sources helper"
+        if: "${{ inputs.buildpkg && inputs.project != '@.' && inputs.project != '.' }}"
+        uses: actions/checkout@v6
+        with:
+          repository: SciML/.github
+          ref: "${{ inputs.dotgithub-ref }}"
+          path: .sciml-dotgithub
+
       - name: "Develop in-repo [sources] path deps of ${{ inputs.project }}"
+        # On Julia < 1.11 the [sources] section is not auto-resolved during
+        # build/test, so develop the in-repo path deps the sub-project (e.g. a
+        # lib/* sublibrary) declares -- transitively, but skipping a developed
+        # dependency's *test-only* [sources] so they don't become phantom direct
+        # deps (SciML/Optimization.jl#1228). No-op on >= 1.11. See
+        # scripts/develop_sources.jl for the full rationale.
         if: "${{ inputs.buildpkg && inputs.project != '@.' && inputs.project != '.' }}"
         shell: julia --color=yes {0}
         run: |
-          using Pkg
-          proj = raw"${{ inputs.project }}"
-          # On Julia < 1.11 the [sources] section is not auto-resolved during
-          # build/test, so develop the in-repo path deps the sub-project (e.g. a
-          # lib/* sublibrary) declares -- transitively, since a source dep can
-          # itself declare further [sources]. No-op on >= 1.11 and for projects
-          # without a [sources] section.
-          if VERSION < v"1.11.0-DEV.0"
-            Pkg.activate(proj)
-            developed = Set{String}([normpath(abspath(proj))])  # never develop the active project (cyclic [sources])
-            specs = Pkg.PackageSpec[]
-            queue = String[proj]
-            while !isempty(queue)
-              dir = popfirst!(queue)
-              tomlpath = joinpath(dir, "Project.toml")
-              isfile(tomlpath) || continue
-              toml = Pkg.TOML.parsefile(tomlpath)
-              haskey(toml, "sources") || continue
-              for (dep, spec) in toml["sources"]
-                if spec isa Dict && haskey(spec, "path")
-                  p = normpath(abspath(joinpath(dir, spec["path"])))
-                  if isdir(p) && !(p in developed)
-                    push!(developed, p)
-                    push!(specs, Pkg.PackageSpec(path = p))
-                    push!(queue, p)  # resolve this dep's own [sources] too
-                  end
-                end
-              end
-            end
-            isempty(specs) || Pkg.develop(specs)
-          end
+          include(joinpath(".sciml-dotgithub", "scripts", "develop_sources.jl"))
+          develop_sources(raw"${{ inputs.project }}")
 
       - name: "Install system packages"
         if: "${{ inputs.apt-packages != '' && runner.os == 'Linux' }}"

scripts/develop_sources.jl

@@ -0,0 +1,96 @@
+#!/usr/bin/env julia
+#
+# Develop the in-repo [sources] path deps of a sub-project.
+#
+# On Julia < 1.11 the [sources] table is ignored when an environment is
+# resolved/built/tested, so a monorepo sublibrary (e.g. lib/<name>) that relies
+# on [sources] to pin its in-repo siblings would otherwise resolve them as
+# registered packages. This script restores the 1.11+ behavior on 1.10 (the
+# SciML LTS) by Pkg.develop-ing each in-repo `path =` source by path. On Julia
+# >= 1.11 it is a no-op (the table is honored natively).
+#
+# The walk is transitive: a developed source dep can itself declare further
+# *runtime* [sources] that must also be developed for the environment to load.
+# BUT when recursing into an already-developed dependency, only [sources] that
+# are also that dependency's runtime [deps] are followed. A dependency's
+# [sources] table may contain entries that exist purely for that dependency's
+# own test suite -- their names live in [extras]/[targets].test, not [deps].
+# Developing those would inject phantom direct deps into the package-under-test's
+# environment, which then trips Aqua's stale-deps / deps-compat checks
+# (SciML/Optimization.jl#1228). The package-under-test itself (the root of the
+# walk) is exempt from this filter: its own test-only [sources] are legitimately
+# needed when its own test suite runs.
+#
+# Usage (from a workflow step):
+#   include("scripts/develop_sources.jl")
+#   develop_sources(project_dir)
+#
+# `develop_sources` activates `project_dir`, computes the source paths to
+# develop via `collect_source_paths`, and Pkg.develop-s them. The pure
+# path-collection logic is split out so it can be unit-tested without mutating
+# any environment.
+
+using Pkg
+
+"""
+    collect_source_paths(proj) -> Vector{String}
+
+Walk the `[sources]` graph rooted at `proj` (a directory containing a
+`Project.toml`) and return the ordered list of absolute `path` sources to
+develop. Each `path` source is resolved relative to the `Project.toml` that
+declares it. The walk recurses into a developed dependency's own `[sources]`,
+but for non-root projects only follows sources that are also runtime `[deps]`
+of that project (skipping the dep's test-only sources). The root project itself
+is excluded from the result (it is the active project; developing it would be a
+cyclic self-develop). Only `path =` sources are returned; `url`/`rev` git
+sources are left to `Pkg`. Visiting each resolved path once handles cycles and
+diamonds in the graph. This function is version-independent and mutates nothing.
+"""
+function collect_source_paths(proj::AbstractString)
+    projroot = normpath(abspath(proj))
+    developed = Set{String}([projroot])  # never develop the active project
+    paths = String[]
+    queue = String[projroot]
+    while !isempty(queue)
+        dir = popfirst!(queue)
+        tomlpath = joinpath(dir, "Project.toml")
+        isfile(tomlpath) || continue
+        toml = Pkg.TOML.parsefile(tomlpath)
+        sources = get(toml, "sources", nothing)
+        sources isa AbstractDict || continue
+        isroot = normpath(abspath(dir)) == projroot
+        runtimedeps = keys(get(toml, "deps", Dict{String, Any}()))
+        for (dep, spec) in sources
+            # For dependencies (not the package-under-test), skip [sources] that
+            # are not runtime deps -- those are the dep's test-only sources and
+            # must not leak into the active environment.
+            isroot || dep in runtimedeps || continue
+            spec isa AbstractDict && haskey(spec, "path") || continue
+            p = normpath(abspath(joinpath(dir, spec["path"])))
+            if isdir(p) && !(p in developed)
+                push!(developed, p)
+                push!(paths, p)
+                push!(queue, p)  # resolve this dep's own runtime [sources] too
+            end
+        end
+    end
+    return paths
+end
+
+"""
+    develop_sources(proj)
+
+Activate `proj` and, on Julia < 1.11, `Pkg.develop` its in-repo `[sources]`
+path deps (see `collect_source_paths`). No-op on Julia >= 1.11.
+"""
+function develop_sources(proj::AbstractString)
+    Pkg.activate(proj)
+    VERSION < v"1.11.0-DEV.0" || return nothing
+    paths = collect_source_paths(proj)
+    isempty(paths) || Pkg.develop([Pkg.PackageSpec(path = p) for p in paths])
+    return nothing
+end
+
+if abspath(PROGRAM_FILE) == @__FILE__
+    develop_sources(ARGS[1])
+end

test/runtests.jl

@@ -346,3 +346,86 @@ end
               ["self-hosted", "Linux", "X64", "gpu"]
     end
 end
+
+# develop_sources.jl: the [sources] develop helper used by tests.yml on Julia
+# <1.11. The pure path-collection (collect_source_paths) is tested here without
+# mutating any environment.
+include(joinpath(@__DIR__, "..", "scripts", "develop_sources.jl"))
+
+# Build a fixture monorepo. `Top` runtime-sources `Mid`; `Mid` runtime-sources
+# `Leaf` (in its [deps]) and test-only-sources `TestOnly` (only in its
+# [extras]/[targets].test, NOT [deps]). The walk from `Top` must develop the
+# runtime chain {Mid, Leaf} and must NOT pull in `Mid`'s test-only `TestOnly`.
+function make_sources_fixture()
+    root = mktempdir()
+    function pkg(name, toml)
+        d = joinpath(root, name)
+        mkpath(joinpath(d, "src"))
+        write(joinpath(d, "Project.toml"), toml)
+        write(joinpath(d, "src", "$name.jl"), "module $name\nend\n")
+        return d
+    end
+    pkg("Leaf", "name = \"Leaf\"\nuuid = \"11111111-1111-1111-1111-111111111111\"\n")
+    pkg("TestOnly", "name = \"TestOnly\"\nuuid = \"44444444-4444-4444-4444-444444444444\"\n")
+    pkg(
+        "Mid", """
+        name = "Mid"
+        uuid = "22222222-2222-2222-2222-222222222222"
+
+        [deps]
+        Leaf = "11111111-1111-1111-1111-111111111111"
+
+        [sources]
+        Leaf = {path = "../Leaf"}
+        TestOnly = {path = "../TestOnly"}
+
+        [extras]
+        TestOnly = "44444444-4444-4444-4444-444444444444"
+
+        [targets]
+        test = ["TestOnly"]
+        """
+    )
+    pkg(
+        "Top", """
+        name = "Top"
+        uuid = "33333333-3333-3333-3333-333333333333"
+
+        [deps]
+        Mid = "22222222-2222-2222-2222-222222222222"
+
+        [sources]
+        Mid = {path = "../Mid"}
+        """
+    )
+    return root
+end
+
+@testset "develop_sources: runtime [sources] developed, dep test-only [sources] excluded" begin
+    root = make_sources_fixture()
+    names = sort(basename.(collect_source_paths(joinpath(root, "Top"))))
+    # Runtime transitive chain Top -> Mid -> Leaf is developed.
+    @test names == ["Leaf", "Mid"]
+    # Mid's test-only source (TestOnly) is NOT pulled into Top's env
+    # (SciML/Optimization.jl#1228).
+    @test "TestOnly" ∉ names
+    # The active project (Top) is never self-developed.
+    @test "Top" ∉ names
+end
+
+@testset "develop_sources: package-under-test's OWN test-only [sources] ARE developed" begin
+    # When the project at the root of the walk is itself being tested, its
+    # test-only [sources] (e.g. OptimizationBase developing LBFGSB/Manopt for
+    # its own test suite) must still be developed -- only a *dependency's*
+    # test-only sources are filtered out.
+    root = make_sources_fixture()
+    names = sort(basename.(collect_source_paths(joinpath(root, "Mid"))))
+    # Testing Mid directly: its runtime source (Leaf) AND its test-only source
+    # (TestOnly) are both developed; only Leaf is then recursed into.
+    @test names == ["Leaf", "TestOnly"]
+end
+
+@testset "develop_sources: no [sources] table -> nothing to develop" begin
+    root = make_sources_fixture()
+    @test isempty(collect_source_paths(joinpath(root, "Leaf")))
+end