From 03dac067fdd64ea47e7fff7433066321481f50f7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 23 Apr 2024 13:40:40 -0700 Subject: [PATCH 001/364] CommutationMatrix type replace comm_matrix helper functions with a CommutationMatrix and overloaded linalg ops --- src/StructuralEquationModels.jl | 4 + .../commutation_matrix.jl | 68 ++++++++++++ src/additional_functions/helper.jl | 103 ------------------ src/loss/ML/FIML.jl | 13 +-- 4 files changed, 77 insertions(+), 111 deletions(-) create mode 100644 src/additional_functions/commutation_matrix.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 048b7181c..113022960 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -24,6 +24,10 @@ const SEM = StructuralEquationModels # type hierarchy include("types.jl") include("objective_gradient_hessian.jl") + +# helper objects and functions +include("additional_functions/commutation_matrix.jl") + # fitted objects include("frontend/fit/SemFit.jl") # specification of models diff --git a/src/additional_functions/commutation_matrix.jl b/src/additional_functions/commutation_matrix.jl new file mode 100644 index 000000000..9a321e173 --- /dev/null +++ b/src/additional_functions/commutation_matrix.jl @@ -0,0 +1,68 @@ +""" + + transpose_linear_indices(n, [m]) + +Put each linear index of the *n×m* matrix to the position of the +corresponding element in the transposed matrix. + +## Example +` +1 4 +2 5 => 1 2 3 +3 6 4 5 6 +` +""" +transpose_linear_indices(n::Integer, m::Integer = n) = + repeat(1:n, inner = m) .+ repeat((0:(m-1)) * n, outer = n) + +""" + CommutationMatrix(n::Integer) <: AbstractMatrix{Int} + +A *commutation matrix* *C* is a n²×n² matrix of 0s and 1s. +If *vec(A)* is a vectorized form of a n×n matrix *A*, +then ``C * vec(A) = vec(Aᵀ)``. +""" +struct CommutationMatrix <: AbstractMatrix{Int} + n::Int + n²::Int + transpose_inds::Vector{Int} # maps the linear indices of n×n matrix *B* to the indices of matrix *B'* + + CommutationMatrix(n::Integer) = new(n, n^2, transpose_linear_indices(n)) +end + +Base.size(A::CommutationMatrix) = (A.n², A.n²) +Base.size(A::CommutationMatrix, dim::Integer) = + 1 <= dim <= 2 ? A.n² : throw(ArgumentError("invalid matrix dimension $dim")) +Base.length(A::CommutationMatrix) = A.n²^2 +Base.getindex(A::CommutationMatrix, i::Int, j::Int) = j == A.transpose_inds[i] ? 1 : 0 + +function Base.:(*)(A::CommutationMatrix, B::AbstractMatrix) + size(A, 2) == size(B, 1) || throw( + DimensionMismatch("A has $(size(A, 2)) columns, but B has $(size(B, 1)) rows"), + ) + return B[A.transpose_inds, :] +end + +function Base.:(*)(A::CommutationMatrix, B::SparseMatrixCSC) + size(A, 2) == size(B, 1) || throw( + DimensionMismatch("A has $(size(A, 2)) columns, but B has $(size(B, 1)) rows"), + ) + return SparseMatrixCSC( + size(B, 1), + size(B, 2), + copy(B.colptr), + A.transpose_inds[B.rowval], + copy(B.nzval), + ) +end + +function LinearAlgebra.lmul!(A::CommutationMatrix, B::SparseMatrixCSC) + size(A, 2) == size(B, 1) || throw( + DimensionMismatch("A has $(size(A, 2)) columns, but B has $(size(B, 1)) rows"), + ) + + @inbounds for (i, rowind) in enumerate(B.rowval) + B.rowval[i] = A.transpose_inds[rowind] + end + return B +end diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index abc37207c..bb3d6dd9b 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -148,106 +148,3 @@ function elimination_matrix(nobs) end return L end - -function commutation_matrix(n; tosparse = false) - M = zeros(n^2, n^2) - - for i in 1:n - for j in 1:n - M[i+n*(j-1), j+n*(i-1)] = 1.0 - end - end - - if tosparse - M = sparse(M) - end - - return M -end - -function commutation_matrix_pre_square(A) - n2 = size(A, 1) - n = Int(sqrt(n2)) - - ind = repeat(1:n, inner = n) - indadd = (0:(n-1)) * n - for i in 1:n - ind[((i-1)*n+1):i*n] .+= indadd - end - - A_post = A[ind, :] - - return A_post -end - -function commutation_matrix_pre_square_add!(B, A) # comuptes B + KₙA - n2 = size(A, 1) - n = Int(sqrt(n2)) - - ind = repeat(1:n, inner = n) - indadd = (0:(n-1)) * n - for i in 1:n - ind[((i-1)*n+1):i*n] .+= indadd - end - - @views @inbounds B .+= A[ind, :] - - return B -end - -function get_commutation_lookup(n2::Int64) - n = Int(sqrt(n2)) - ind = repeat(1:n, inner = n) - indadd = (0:(n-1)) * n - for i in 1:n - ind[((i-1)*n+1):i*n] .+= indadd - end - - lookup = Dict{Int64, Int64}() - - for i in 1:n2 - j = findall(x -> (x == i), ind)[1] - push!(lookup, i => j) - end - - return lookup -end - -function commutation_matrix_pre_square!(A::SparseMatrixCSC, lookup) # comuptes B + KₙA - for (i, rowind) in enumerate(A.rowval) - A.rowval[i] = lookup[rowind] - end -end - -function commutation_matrix_pre_square!(A::SparseMatrixCSC) # computes KₙA - lookup = get_commutation_lookup(size(A, 2)) - commutation_matrix_pre_square!(A, lookup) -end - -function commutation_matrix_pre_square(A::SparseMatrixCSC) - B = copy(A) - commutation_matrix_pre_square!(B) - return B -end - -function commutation_matrix_pre_square(A::SparseMatrixCSC, lookup) - B = copy(A) - commutation_matrix_pre_square!(B, lookup) - return B -end - -function commutation_matrix_pre_square_add_mt!(B, A) # comuptes B + KₙA # 0 allocations but slower - n2 = size(A, 1) - n = Int(sqrt(n2)) - - indadd = (0:(n-1)) * n - - Threads.@threads for i in 1:n - for j in 1:n - row = i + indadd[j] - @views @inbounds B[row, :] .+= A[row, :] - end - end - - return B -end diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 7a27e7615..1cc7c123c 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -24,7 +24,7 @@ Analytic gradients are available. ## Implementation Subtype of `SemLossFunction`. """ -mutable struct SemFIML{INV, C, L, O, M, IM, I, T, U, W} <: SemLossFunction +mutable struct SemFIML{INV, C, L, O, M, IM, I, T, W} <: SemLossFunction inverses::INV #preallocated inverses of imp_cov choleskys::C #preallocated choleskys logdets::L #logdets of implied covmats @@ -37,7 +37,7 @@ mutable struct SemFIML{INV, C, L, O, M, IM, I, T, U, W} <: SemLossFunction mult::T - commutation_indices::U + commutator::CommutationMatrix interaction::W end @@ -64,8 +64,6 @@ function SemFIML(; observed, specification, kwargs...) ∇ind = [findall(x -> !(x[1] ∈ ind || x[2] ∈ ind), ∇ind) for ind in patterns_not(observed)] - commutation_indices = get_commutation_lookup(get_n_nodes(specification)^2) - return SemFIML( inverses, choleskys, @@ -75,7 +73,7 @@ function SemFIML(; observed, specification, kwargs...) meandiff, imp_inv, mult, - commutation_indices, + CommutationMatrix(get_n_nodes(specification)), nothing, ) end @@ -163,10 +161,9 @@ function ∇F_fiml_outer(JΣ, Jμ, imply, model, semfiml) Iₙ = sparse(1.0I, size(A(imply))...) P = kron(F⨉I_A⁻¹(imply), F⨉I_A⁻¹(imply)) Q = kron(S(imply) * I_A⁻¹(imply)', Iₙ) - #commutation_matrix_pre_square_add!(Q, Q) - Q2 = commutation_matrix_pre_square(Q, semfiml.commutation_indices) + Q .+= semfiml.commutator * Q - ∇Σ = P * (∇S(imply) + (Q + Q2) * ∇A(imply)) + ∇Σ = P * (∇S(imply) + Q * ∇A(imply)) ∇μ = F⨉I_A⁻¹(imply) * ∇M(imply) + From 47a1757e4d0594a6c8fc52d9b7940a7ed0b285f4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 14 Apr 2024 13:21:02 -0700 Subject: [PATCH 002/364] simplify elimination_matrix() --- src/additional_functions/helper.jl | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index bb3d6dd9b..d5f459b3a 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -131,19 +131,17 @@ function duplication_matrix(nobs) return D end -function elimination_matrix(nobs) - nobs = Int(nobs) - n1 = Int(nobs * (nobs + 1) * 0.5) - n2 = Int(nobs^2) - L = zeros(n1, n2) - - for j in 1:nobs - for i in j:nobs - u = zeros(n1) - u[Int((j - 1) * nobs + i - 0.5 * j * (j - 1))] = 1 - T = zeros(nobs, nobs) - T[i, j] = 1 - L += u * transpose(vec(T)) +# (n(n+1)/2)×n² matrix to transform a +# vectorized form of a n×n symmetric matrix +# into vector of its lower triangular entries, +# opposite of duplication_matrix() +function elimination_matrix(n::Integer) + ntri = div(n * (n + 1), 2) + L = zeros(ntri, n^2) + for j in 1:n + for i in j:n + tri_ix = (j - 1) * n + i - div(j * (j - 1), 2) + L[tri_ix, i+n*(j-1)] = 1 end end return L From fd212c0953d2406318c70b1da10e4a4474da50c9 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 00:43:28 -0700 Subject: [PATCH 003/364] simplify duplication_matrix() --- src/additional_functions/helper.jl | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index d5f459b3a..2e85d1183 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -111,23 +111,19 @@ function cov_and_mean(rows; corrected = false) return obs_cov, vec(obs_mean) end -function duplication_matrix(nobs) - nobs = Int(nobs) - n1 = Int(nobs * (nobs + 1) * 0.5) - n2 = Int(nobs^2) - Dt = zeros(n1, n2) - - for j in 1:nobs - for i in j:nobs - u = zeros(n1) - u[Int((j - 1) * nobs + i - 0.5 * j * (j - 1))] = 1 - T = zeros(nobs, nobs) - T[j, i] = 1 - T[i, j] = 1 - Dt += u * transpose(vec(T)) +# n²×(n(n+1)/2) matrix to transform a vector of lower +# triangular entries into a vectorized form of a n×n symmetric matrix, +# opposite of elimination_matrix() +function duplication_matrix(n::Integer) + ntri = div(n * (n + 1), 2) + D = zeros(n^2, ntri) + for j in 1:n + for i in j:n + tri_ix = (j - 1) * n + i - div(j * (j - 1), 2) + D[j+n*(i-1), tri_ix] = 1 + D[i+n*(j-1), tri_ix] = 1 end end - D = transpose(Dt) return D end From 29842031eab0ef7cf0f31922c15d1f05051466ff Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 23 Apr 2024 11:10:46 +0200 Subject: [PATCH 004/364] add tests for commutation/dublication/elimination matrices --- src/additional_functions/helper.jl | 2 +- test/unit_tests/matrix_helpers.jl | 36 ++++++++++++++++++++++++++++++ test/unit_tests/unit_tests.jl | 4 ++++ 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 test/unit_tests/matrix_helpers.jl diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 2e85d1183..b96813dc3 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -41,7 +41,7 @@ function get_observed(rowind, data, semobserved; args = (), kwargs = NamedTuple( return observed_vec end -skipmissing_mean(mat::AbstractMatrix) = +skipmissing_mean(mat::AbstractMatrix) = [mean(skipmissing(coldata)) for coldata in eachcol(mat)] function F_one_person(imp_mean, meandiff, inverse, data, logdet) diff --git a/test/unit_tests/matrix_helpers.jl b/test/unit_tests/matrix_helpers.jl new file mode 100644 index 000000000..dcf339e3c --- /dev/null +++ b/test/unit_tests/matrix_helpers.jl @@ -0,0 +1,36 @@ +using StructuralEquationModels, Test, Random, SparseArrays, LinearAlgebra +using StructuralEquationModels: + CommutationMatrix, transpose_linear_indices, duplication_matrix, elimination_matrix + +Random.seed!(73721) + +n = 4 +m = 5 + +@testset "Commutation matrix" begin + # transpose linear indices + A = rand(n, m) + @test reshape(A[transpose_linear_indices(n, m)], m, n) == A' + # commutation matrix multiplication + K = CommutationMatrix(n) + B = rand(n, n) + @test K * vec(B) == vec(B') + C = sprand(n, n, 0.5) + @test K * vec(C) == vec(C') + # lmul! + D = sprand(n^2, n^2, 0.1) + E = copy(D) + lmul!(K, D) + @test D == K * E +end + +@testset "Duplication / elimination matrix" begin + A = rand(m, m) + A = A * A' + # dupication + D = duplication_matrix(m) + @test D * A[tril(trues(size(A)))] == vec(A) + # elimination + D = elimination_matrix(m) + @test D * vec(A) == A[tril(trues(size(A)))] +end diff --git a/test/unit_tests/unit_tests.jl b/test/unit_tests/unit_tests.jl index 87fdde2f1..eb58650c1 100644 --- a/test/unit_tests/unit_tests.jl +++ b/test/unit_tests/unit_tests.jl @@ -7,3 +7,7 @@ end @safetestset "SemObs" begin include("data_input_formats.jl") end + +@safetestset "Matrix algebra helper functions" begin + include("matrix_helpers.jl") +end From b6db6b1bc9e3b7493e2cf1ee4d0dc86779da43c1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 23 Apr 2024 13:41:19 -0700 Subject: [PATCH 005/364] small unit test fixes --- test/unit_tests/matrix_helpers.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/unit_tests/matrix_helpers.jl b/test/unit_tests/matrix_helpers.jl index dcf339e3c..0fff05021 100644 --- a/test/unit_tests/matrix_helpers.jl +++ b/test/unit_tests/matrix_helpers.jl @@ -14,6 +14,7 @@ m = 5 # commutation matrix multiplication K = CommutationMatrix(n) B = rand(n, n) + @test_throws DimensionMismatch K * rand(n, m) @test K * vec(B) == vec(B') C = sprand(n, n, 0.5) @test K * vec(C) == vec(C') @@ -27,10 +28,12 @@ end @testset "Duplication / elimination matrix" begin A = rand(m, m) A = A * A' + # dupication D = duplication_matrix(m) @test D * A[tril(trues(size(A)))] == vec(A) + # elimination - D = elimination_matrix(m) - @test D * vec(A) == A[tril(trues(size(A)))] + E = elimination_matrix(m) + @test E * vec(A) == A[tril(trues(size(A)))] end From 3abe92ea5000c63422b767a1c5a2723b6539dadd Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 23 Apr 2024 13:58:51 -0700 Subject: [PATCH 006/364] commutation_matrix * vec method --- src/additional_functions/commutation_matrix.jl | 7 +++++++ test/unit_tests/matrix_helpers.jl | 2 ++ 2 files changed, 9 insertions(+) diff --git a/src/additional_functions/commutation_matrix.jl b/src/additional_functions/commutation_matrix.jl index 9a321e173..345f809e0 100644 --- a/src/additional_functions/commutation_matrix.jl +++ b/src/additional_functions/commutation_matrix.jl @@ -36,6 +36,13 @@ Base.size(A::CommutationMatrix, dim::Integer) = Base.length(A::CommutationMatrix) = A.n²^2 Base.getindex(A::CommutationMatrix, i::Int, j::Int) = j == A.transpose_inds[i] ? 1 : 0 +function Base.:(*)(A::CommutationMatrix, B::AbstractVector) + size(A, 2) == size(B, 1) || throw( + DimensionMismatch("A has $(size(A, 2)) columns, but B has $(size(B, 1)) elements"), + ) + return B[A.transpose_inds] +end + function Base.:(*)(A::CommutationMatrix, B::AbstractMatrix) size(A, 2) == size(B, 1) || throw( DimensionMismatch("A has $(size(A, 2)) columns, but B has $(size(B, 1)) rows"), diff --git a/test/unit_tests/matrix_helpers.jl b/test/unit_tests/matrix_helpers.jl index 0fff05021..d5eb69f22 100644 --- a/test/unit_tests/matrix_helpers.jl +++ b/test/unit_tests/matrix_helpers.jl @@ -21,8 +21,10 @@ m = 5 # lmul! D = sprand(n^2, n^2, 0.1) E = copy(D) + F = Matrix(E) lmul!(K, D) @test D == K * E + @test Matrix(D) == K * F end @testset "Duplication / elimination matrix" begin From c1e7a6942a7f047720ce8ab8d47aacbd83c69f97 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 23 Apr 2024 14:15:16 -0700 Subject: [PATCH 007/364] more comm_matrix tests --- test/unit_tests/matrix_helpers.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/unit_tests/matrix_helpers.jl b/test/unit_tests/matrix_helpers.jl index d5eb69f22..b2f32f31a 100644 --- a/test/unit_tests/matrix_helpers.jl +++ b/test/unit_tests/matrix_helpers.jl @@ -13,6 +13,14 @@ m = 5 @test reshape(A[transpose_linear_indices(n, m)], m, n) == A' # commutation matrix multiplication K = CommutationMatrix(n) + # test K array interface methods + @test size(K) == (n^2, n^2) + @test size(K, 1) == n^2 + @test length(K) == n^4 + nn_linind = LinearIndices((n, n)) + @test K[nn_linind[3, 2], nn_linind[2, 3]] == 1 + @test K[nn_linind[3, 2], nn_linind[3, 2]] == 0 + B = rand(n, n) @test_throws DimensionMismatch K * rand(n, m) @test K * vec(B) == vec(B') From 22542eaa14262013bb70a95c3cd651533d237ca8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 28 Apr 2024 01:50:15 -0700 Subject: [PATCH 008/364] SemSpecification base type --- src/frontend/specification/ParameterTable.jl | 2 -- src/frontend/specification/RAMMatrices.jl | 2 +- src/types.jl | 7 +++++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 1910d666e..4fc9d1513 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -1,5 +1,3 @@ -abstract type AbstractParameterTable end - ############################################################################################ ### Types ############################################################################################ diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index e0fcc575c..2faf84a8f 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -6,7 +6,7 @@ AbstractArrayParamsMap = AbstractVector{<:AbstractVector{<:Integer}} ArrayParamsMap = Vector{Vector{Int}} -struct RAMMatrices +struct RAMMatrices <: SemSpecification A_ind::ArrayParamsMap S_ind::ArrayParamsMap F_ind::Vector{Int} diff --git a/src/types.jl b/src/types.jl index 803bc733a..46b2781fb 100644 --- a/src/types.jl +++ b/src/types.jl @@ -247,3 +247,10 @@ loss(model::AbstractSemSingle) = model.loss Returns the optimizer part of a model. """ optimizer(model::AbstractSemSingle) = model.optimizer + +""" +Base type for all SEM specifications. +""" +abstract type SemSpecification end + +abstract type AbstractParameterTable <: SemSpecification end From 8d4187477554e4972e713006c5b8d1aea804846d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 28 Apr 2024 01:50:32 -0700 Subject: [PATCH 009/364] SemSpecification: use in methods --- src/imply/RAM/generic.jl | 2 +- src/imply/RAM/symbolic.jl | 2 +- src/observed/covariance.jl | 2 +- src/observed/data.jl | 2 +- src/observed/missing.jl | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 00c0d0ef9..c14121bb4 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -121,7 +121,7 @@ using StructuralEquationModels ############################################################################################ function RAM(; - specification, + specification::SemSpecification, #vech = false, gradient = true, meanstructure = false, diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 5c5e52112..fae687e1d 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -88,7 +88,7 @@ end ############################################################################################ function RAMSymbolic(; - specification, + specification::SemSpecification, loss_types = nothing, vech = false, gradient = true, diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 1b5de9fc2..8d73b1a99 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -47,7 +47,7 @@ struct SemObservedCovariance{B, C} <: SemObserved end function SemObservedCovariance(; - specification, + specification::Union{SemSpecification, Nothing}, obs_cov, obs_colnames = nothing, spec_colnames = nothing, diff --git a/src/observed/data.jl b/src/observed/data.jl index 0d9ad3a04..89deefd04 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -55,7 +55,7 @@ function check_arguments_SemObservedData(kwargs...) end function SemObservedData(; - specification, + specification::Union{SemSpecification, Nothing}, data, obs_colnames = nothing, spec_colnames = nothing, diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 6cfd09391..439e3d837 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -86,7 +86,7 @@ end ############################################################################################ function SemObservedMissing(; - specification, + specification::Union{SemSpecification, Nothing}, data, obs_colnames = nothing, spec_colnames = nothing, From 239a9426314d3e2257e84f2fe36b65bfe85372aa Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 28 Apr 2024 13:46:46 -0700 Subject: [PATCH 010/364] rename identifier -> param * identifier() -> param_indices() (Dict{Symbol, Int}) * get_identifier_indices() -> param_to_indices() (Vector{Int}) * parameters -> params (Vector{Symbol}) --- src/StructuralEquationModels.jl | 4 +- src/additional_functions/identifier.jl | 48 +++++----- src/additional_functions/parameters.jl | 16 ++-- .../start_val/start_fabin3.jl | 6 +- .../start_val/start_partable.jl | 8 +- .../start_val/start_simple.jl | 8 +- src/frontend/fit/fitmeasures/n_par.jl | 6 +- .../specification/EnsembleParameterTable.jl | 4 +- src/frontend/specification/ParameterTable.jl | 20 ++-- src/frontend/specification/RAMMatrices.jl | 69 +++++++------- src/frontend/specification/StenoGraphs.jl | 20 ++-- src/frontend/specification/documentation.jl | 14 +-- src/imply/RAM/generic.jl | 21 ++--- src/imply/RAM/symbolic.jl | 12 +-- src/imply/empty.jl | 10 +- src/loss/ML/FIML.jl | 20 ++-- src/loss/regularization/ridge.jl | 2 +- src/objective_gradient_hessian.jl | 92 +++++++++---------- src/types.jl | 14 +-- test/examples/helper.jl | 26 +++--- test/examples/multigroup/build_models.jl | 2 +- test/examples/multigroup/multigroup.jl | 4 +- .../political_democracy.jl | 4 +- .../recover_parameters_twofact.jl | 2 +- test/unit_tests/specification.jl | 14 +-- 25 files changed, 222 insertions(+), 224 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 113022960..3319f049b 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -153,9 +153,9 @@ export AbstractSem, start, Label, label, - get_identifier_indices, + params_to_indices, RAMMatrices, - identifier, + param_indices, fit_measures, AIC, BIC, diff --git a/src/additional_functions/identifier.jl b/src/additional_functions/identifier.jl index fefcc1be5..1b10357d6 100644 --- a/src/additional_functions/identifier.jl +++ b/src/additional_functions/identifier.jl @@ -1,59 +1,59 @@ ############################################################################################ -# get parameter identifier +# get a map from parameters to their indices ############################################################################################ -identifier(sem_fit::SemFit) = identifier(sem_fit.model) -identifier(model::AbstractSemSingle) = identifier(model.imply) -identifier(model::SemEnsemble) = model.identifier +param_indices(sem_fit::SemFit) = param_indices(sem_fit.model) +param_indices(model::AbstractSemSingle) = param_indices(model.imply) +param_indices(model::SemEnsemble) = model.param_indices ############################################################################################ -# construct identifier +# construct a map from parameters to indices ############################################################################################ -identifier(ram_matrices::RAMMatrices) = - Dict{Symbol, Int64}(ram_matrices.parameters .=> 1:length(ram_matrices.parameters)) -function identifier(partable::ParameterTable) - _, _, identifier = get_par_npar_identifier(partable) - return identifier +param_indices(ram_matrices::RAMMatrices) = + Dict(par => i for (i, par) in enumerate(ram_matrices.params)) +function param_indices(partable::ParameterTable) + _, _, param_indices = get_par_npar_indices(partable) + return param_indices end ############################################################################################ # get indices of a Vector of parameter labels ############################################################################################ -get_identifier_indices(parameters, identifier::Dict{Symbol, Int}) = - [identifier[par] for par in parameters] +params_to_indices(params, param_indices::Dict{Symbol, Int}) = + [param_indices[par] for par in params] -get_identifier_indices( - parameters, +params_to_indices( + params, obj::Union{SemFit, AbstractSemSingle, SemEnsemble, SemImply}, -) = get_identifier_indices(parameters, identifier(obj)) +) = params_to_indices(params, params(obj)) -function get_identifier_indices(parameters, obj::Union{ParameterTable, RAMMatrices}) +function params_to_indices(params, obj::Union{ParameterTable, RAMMatrices}) @warn "You are trying to find parameter indices from a ParameterTable or RAMMatrices object. \n If your model contains user-defined types, this may lead to wrong results. \n - To be on the safe side, try to reference parameters by labels or query the indices from - the constructed model (`get_identifier_indices(parameters, model)`)." maxlog = 1 - return get_identifier_indices(parameters, identifier(obj)) + To be on the safe side, try to reference parameters by labels or query the indices from + the constructed model (`params_to_indices(params, model)`)." maxlog = 1 + return params_to_indices(params, params(obj)) end ############################################################################################ # documentation ############################################################################################ """ - get_identifier_indices(parameters, model) + params_to_indices(params, model) -Returns the indices of `parameters`. +Returns the indices of `params`. # Arguments -- `parameters::Vector{Symbol}`: parameter labels +- `params::Vector{Symbol}`: parameter labels - `model`: either a SEM or a fitted SEM # Examples ```julia -parameter_indices = get_identifier_indices([:λ₁, λ₂], my_fitted_sem) +parameter_indices = params_to_indices([:λ₁, λ₂], my_fitted_sem) values = solution(my_fitted_sem)[parameter_indices] ``` """ -function get_identifier_indices end +function params_to_indices end diff --git a/src/additional_functions/parameters.jl b/src/additional_functions/parameters.jl index 8d01b3747..d6e8eb535 100644 --- a/src/additional_functions/parameters.jl +++ b/src/additional_functions/parameters.jl @@ -6,9 +6,9 @@ function fill_A_S_M!( A_indices::AbstractArrayParamsMap, S_indices::AbstractArrayParamsMap, M_indices::Union{AbstractArrayParamsMap, Nothing}, - parameters::AbstractVector, + params::AbstractVector, ) - @inbounds for (iA, iS, par) in zip(A_indices, S_indices, parameters) + @inbounds for (iA, iS, par) in zip(A_indices, S_indices, params) for index_A in iA A[index_A] = par end @@ -19,7 +19,7 @@ function fill_A_S_M!( end if !isnothing(M) - @inbounds for (iM, par) in zip(M_indices, parameters) + @inbounds for (iM, par) in zip(M_indices, params) for index_M in iM M[index_M] = par end @@ -30,10 +30,10 @@ end # build the map from the index of the parameter to the linear indices # of this parameter occurences in M # returns ArrayParamsMap object -function array_parameters_map(parameters::AbstractVector, M::AbstractArray) - params_index = Dict(param => i for (i, param) in enumerate(parameters)) +function array_params_map(params::AbstractVector, M::AbstractArray) + params_index = Dict(param => i for (i, param) in enumerate(params)) T = Base.eltype(eachindex(M)) - res = [Vector{T}() for _ in eachindex(parameters)] + res = [Vector{T}() for _ in eachindex(params)] for (i, val) in enumerate(M) par_ind = get(params_index, val, nothing) if !isnothing(par_ind) @@ -105,9 +105,9 @@ end function fill_matrix!( M::AbstractMatrix, M_indices::AbstractArrayParamsMap, - parameters::AbstractVector, + params::AbstractVector, ) - for (iM, par) in zip(M_indices, parameters) + for (iM, par) in zip(M_indices, params) for index_M in iM M[index_M] = par end diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index ee7dcb8cf..b56ee60a1 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -31,13 +31,13 @@ function start_fabin3(observed::SemObservedMissing, imply, optimizer, args...; k end function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) - A_ind, S_ind, F_ind, M_ind, parameters = ram_matrices.A_ind, + A_ind, S_ind, F_ind, M_ind, params = ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.F_ind, ram_matrices.M_ind, - ram_matrices.parameters + ram_matrices.params - n_par = length(parameters) + n_par = length(params) start_val = zeros(n_par) n_var, n_nod = ram_matrices.size_F n_latent = n_nod - n_var diff --git a/src/additional_functions/start_val/start_partable.jl b/src/additional_functions/start_val/start_partable.jl index 01d06ac71..6fb15e365 100644 --- a/src/additional_functions/start_val/start_partable.jl +++ b/src/additional_functions/start_val/start_partable.jl @@ -1,6 +1,6 @@ """ start_parameter_table(model; parameter_table) - + Return a vector of starting values taken from `parameter_table`. """ function start_parameter_table end @@ -28,10 +28,10 @@ function start_parameter_table( ) start_val = zeros(0) - for identifier_ram in ram_matrices.parameters + for param in ram_matrices.params found = false - for (i, identifier_table) in enumerate(parameter_table.identifier) - if identifier_ram == identifier_table + for (i, param_table) in enumerate(parameter_table.params) + if param == param_table push!(start_val, parameter_table.start[i]) found = true break diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 4c4645256..2c4f661c1 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -10,7 +10,7 @@ start_covariances_obs_lat = 0.0, start_means = 0.0, kwargs...) - + Return a vector of simple starting values. """ function start_simple end @@ -62,13 +62,13 @@ function start_simple( start_means = 0.0, kwargs..., ) - A_ind, S_ind, F_ind, M_ind, parameters = ram_matrices.A_ind, + A_ind, S_ind, F_ind, M_ind, params = ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.F_ind, ram_matrices.M_ind, - ram_matrices.parameters + ram_matrices.params - n_par = length(parameters) + n_par = length(params) start_val = zeros(n_par) n_var, n_nod = ram_matrices.size_F diff --git a/src/frontend/fit/fitmeasures/n_par.jl b/src/frontend/fit/fitmeasures/n_par.jl index 9cb2d3479..c8553572b 100644 --- a/src/frontend/fit/fitmeasures/n_par.jl +++ b/src/frontend/fit/fitmeasures/n_par.jl @@ -5,7 +5,7 @@ n_par(sem_fit::SemFit) n_par(model::AbstractSemSingle) n_par(model::SemEnsemble) - n_par(identifier::Dict) + n_par(param_indices::Dict) Return the number of parameters. """ @@ -15,6 +15,6 @@ n_par(fit::SemFit) = n_par(fit.model) n_par(model::AbstractSemSingle) = n_par(model.imply) -n_par(model::SemEnsemble) = n_par(model.identifier) +n_par(model::SemEnsemble) = n_par(model.param_indices) -n_par(identifier::Dict) = length(identifier) +n_par(param_indices::Dict) = length(param_indices) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 79283953f..29a6cf984 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -109,12 +109,12 @@ get_group(partable::EnsembleParameterTable, group) = get_group(partable.tables, # update generic --------------------------------------------------------------------------- function update_partable!( partable::EnsembleParameterTable, - model_identifier::AbstractDict, + param_indices::AbstractDict, vec, column, ) for k in keys(partable.tables) - update_partable!(partable.tables[k], model_identifier, vec, column) + update_partable!(partable.tables[k], param_indices, vec, column) end return partable end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 4fc9d1513..c0430625f 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -192,16 +192,16 @@ push!(partable::ParameterTable, d::Nothing) = nothing function update_partable!( partable::ParameterTable, - model_identifier::AbstractDict, - vec, + param_indices::AbstractDict, + values::AbstractVector, column, ) new_col = Vector{eltype(vec)}(undef, length(partable)) - for (i, identifier) in enumerate(partable.columns[:identifier]) - if !(identifier == :const) - new_col[i] = vec[model_identifier[identifier]] - elseif identifier == :const - new_col[i] = zero(eltype(vec)) + for (i, param) in enumerate(partable.columns[:identifier]) + if !(param == :const) + new_col[i] = values[param_indices[param]] + elseif param == :const + new_col[i] = zero(eltype(values)) end end push!(partable.columns, column => new_col) @@ -210,14 +210,14 @@ end """ update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, vec, column) - + Write `vec` to `column` of `partable`. # Arguments - `vec::Vector`: has to be in the same order as the `model` parameters """ update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, vec, column) = - update_partable!(partable, identifier(sem_fit), vec, column) + update_partable!(partable, param_indices(sem_fit), vec, column) # update estimates ------------------------------------------------------------------------- """ @@ -254,7 +254,7 @@ function update_start!( if !(start_val isa Vector) start_val = start_val(model; kwargs...) end - return update_partable!(partable, identifier(model), start_val, :start) + return update_partable!(partable, param_indices(model), start_val, :start) end # update partable standard errors ---------------------------------------------------------- diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 2faf84a8f..eb92c889b 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -11,7 +11,7 @@ struct RAMMatrices <: SemSpecification S_ind::ArrayParamsMap F_ind::Vector{Int} M_ind::Union{ArrayParamsMap, Nothing} - parameters::Any + params::Any colnames::Any constants::Any size_F::Any @@ -21,10 +21,10 @@ end ### Constructor ############################################################################################ -function RAMMatrices(; A, S, F, M = nothing, parameters, colnames) - A_indices = array_parameters_map(parameters, A) - S_indices = array_parameters_map(parameters, S) - M_indices = !isnothing(M) ? array_parameters_map(parameters, M) : nothing +function RAMMatrices(; A, S, F, M = nothing, params, colnames) + A_indices = array_params_map(params, A) + S_indices = array_params_map(params, S) + M_indices = !isnothing(M) ? array_params_map(params, M) : nothing F_indices = findall([any(isone.(col)) for col in eachcol(F)]) constants = get_RAMConstants(A, S, M) return RAMMatrices( @@ -32,7 +32,7 @@ function RAMMatrices(; A, S, F, M = nothing, parameters, colnames) S_indices, F_indices, M_indices, - parameters, + params, colnames, constants, size(F), @@ -107,10 +107,10 @@ end function RAMMatrices(partable::ParameterTable; par_id = nothing) if isnothing(par_id) - parameters, n_par, par_positions = get_par_npar_identifier(partable) + params, n_par, par_positions = get_par_npar_indices(partable) else - parameters, n_par, par_positions = - par_id[:parameters], par_id[:n_par], par_id[:par_positions] + params, n_par, par_positions = + par_id[:params], par_id[:n_par], par_id[:par_positions] end n_observed = size(partable.variables[:observed_vars], 1) @@ -169,7 +169,7 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) constants = Vector{RAMConstant}() for i in 1:length(partable) - from, parameter_type, to, free, value_fixed, identifier = partable[i] + from, parameter_type, to, free, value_fixed, param = partable[i] row_ind = positions[to] if from != Symbol("1") @@ -191,7 +191,7 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) ) end else - par_ind = par_positions[identifier] + par_ind = par_positions[param] if (parameter_type == :→) && (from == Symbol("1")) push!(M_ind[par_ind], row_ind) elseif parameter_type == :→ @@ -210,7 +210,7 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) S_ind, F_ind, M_ind, - parameters, + params, colnames, constants, (n_observed, n_node), @@ -243,7 +243,7 @@ function ParameterTable(ram_matrices::RAMMatrices) end # parameters - for (i, par) in enumerate(ram_matrices.parameters) + for (i, par) in enumerate(ram_matrices.params) push_partable_rows!( partable, position_names, @@ -266,9 +266,9 @@ end function RAMMatrices(partable::EnsembleParameterTable) ram_matrices = Dict{Symbol, RAMMatrices}() - parameters, n_par, par_positions = get_par_npar_identifier(partable) + params, n_par, par_positions = get_par_npar_indices(partable) par_id = - Dict(:parameters => parameters, :n_par => n_par, :par_positions => par_positions) + Dict(:params => params, :n_par => n_par, :par_positions => par_positions) for key in keys(partable.tables) ram_mat = RAMMatrices(partable.tables[key]; par_id = par_id) @@ -291,27 +291,27 @@ end ### Additional Functions ############################################################################################ -function get_par_npar_identifier(partable::ParameterTable) - parameters = unique(partable.columns[:identifier]) - filter!(x -> x != :const, parameters) - n_par = length(parameters) - par_positions = Dict(parameters .=> 1:n_par) - return parameters, n_par, par_positions +function get_par_npar_indices(partable::ParameterTable) + params = unique(partable.columns[:identifier]) + filter!(x -> x != :const, params) + n_par = length(params) + par_positions = Dict(params .=> 1:n_par) + return params, n_par, par_positions end -function get_par_npar_identifier(partable::EnsembleParameterTable) - parameters = Vector{Symbol}() +function get_par_npar_indices(partable::EnsembleParameterTable) + params = Vector{Symbol}() for key in keys(partable.tables) - append!(parameters, partable.tables[key].columns[:identifier]) + append!(params, partable.tables[key].columns[:identifier]) end - parameters = unique(parameters) - filter!(x -> x != :const, parameters) + params = unique(params) + filter!(x -> x != :const, params) - n_par = length(parameters) + n_par = length(params) - par_positions = Dict(parameters .=> 1:n_par) + par_positions = Dict(params .=> 1:n_par) - return parameters, n_par, par_positions + return params, n_par, par_positions end function get_partable_row(c::RAMConstant, position_names) @@ -330,7 +330,7 @@ function get_partable_row(c::RAMConstant, position_names) value_fixed = c.value start = 0.0 estimate = 0.0 - identifier = :const + return Dict( :from => from, :parameter_type => parameter_type, @@ -339,7 +339,7 @@ function get_partable_row(c::RAMConstant, position_names) :value_fixed => value_fixed, :start => start, :estimate => estimate, - :identifier => identifier, + :identifier => :const, ) end @@ -355,7 +355,7 @@ end cartesian_is_known(index, known_indices::Nothing) = false -function get_partable_row(par, position_names, index, matrix, n_nod, known_indices) +function get_partable_row(param, position_names, index, matrix, n_nod, known_indices) # variable names if matrix == :M @@ -387,7 +387,6 @@ function get_partable_row(par, position_names, index, matrix, n_nod, known_indic value_fixed = 0.0 start = 0.0 estimate = 0.0 - identifier = par return Dict( :from => from, @@ -397,7 +396,7 @@ function get_partable_row(par, position_names, index, matrix, n_nod, known_indic :value_fixed => value_fixed, :start => start, :estimate => estimate, - :identifier => identifier, + :identifier => param, ) end @@ -433,7 +432,7 @@ function ==(mat1::RAMMatrices, mat2::RAMMatrices) (mat1.S_ind == mat2.S_ind) && (mat1.F_ind == mat2.F_ind) && (mat1.M_ind == mat2.M_ind) && - (mat1.parameters == mat2.parameters) && + (mat1.params == mat2.params) && (mat1.colnames == mat2.colnames) && (mat1.size_F == mat2.size_F) && (mat1.constants == mat2.constants) diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 5c9ce7fdb..a581e9e5a 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -38,8 +38,8 @@ function ParameterTable(; graph, observed_vars, latent_vars, g = 1, parname = : value_fixed = zeros(n) start = zeros(n) estimate = zeros(n) - identifier = Vector{Symbol}(undef, n) - identifier .= Symbol("") + params = Vector{Symbol}(undef, n) + params .= Symbol("") # group = Vector{Symbol}(undef, n) # start_partable = zeros(Bool, n) @@ -80,7 +80,7 @@ function ParameterTable(; graph, observed_vars, latent_vars, g = 1, parname = : if modifier.value[g] == :NaN throw(DomainError(NaN, "NaN is not allowed as a parameter label.")) end - identifier[i] = modifier.value[g] + params[i] = modifier.value[g] end end end @@ -88,13 +88,13 @@ function ParameterTable(; graph, observed_vars, latent_vars, g = 1, parname = : # make identifiers for parameters that are not labeled current_id = 1 - for i in 1:length(identifier) - if (identifier[i] == Symbol("")) & free[i] - identifier[i] = Symbol(parname, :_, current_id) + for i in 1:length(params) + if (params[i] == Symbol("")) & free[i] + params[i] = Symbol(parname, :_, current_id) current_id += 1 - elseif (identifier[i] == Symbol("")) & !free[i] - identifier[i] = :const - elseif (identifier[i] != Symbol("")) & !free[i] + elseif (params[i] == Symbol("")) & !free[i] + params[i] = :const + elseif (params[i] != Symbol("")) & !free[i] @warn "You labeled a constant. Please check if the labels of your graph are correct." end end @@ -108,7 +108,7 @@ function ParameterTable(; graph, observed_vars, latent_vars, g = 1, parname = : :value_fixed => value_fixed, :start => start, :estimate => estimate, - :identifier => identifier, + :identifier => params, ), Dict( :latent_vars => latent_vars, diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl index e3be49971..27bedfea1 100644 --- a/src/frontend/specification/documentation.jl +++ b/src/frontend/specification/documentation.jl @@ -14,7 +14,7 @@ Return a `ParameterTable` constructed from (1) a graph or (2) RAM matrices. - `observed_vars::Vector{Symbol}`: observed variable names - `latent_vars::Vector{Symbol}`: latent variable names - `ram_matrices::RAMMatrices`: a `RAMMatrices` object - + # Examples See the online documentation on [Model specification](@ref) and the [ParameterTable interface](@ref). @@ -54,11 +54,11 @@ function EnsembleParameterTable end (1) RAMMatrices(partable::ParameterTable) - (2) RAMMatrices(;A, S, F, M = nothing, parameters, colnames) + (2) RAMMatrices(;A, S, F, M = nothing, params, colnames) (3) RAMMatrices(partable::EnsembleParameterTable) - -Return `RAMMatrices` constructed from (1) a parameter table or (2) individual matrices. + +Return `RAMMatrices` constructed from (1) a parameter table or (2) individual matrices. (3) Return a dictionary of `RAMMatrices` from an `EnsembleParameterTable` (keys are the group names). @@ -68,7 +68,7 @@ Return `RAMMatrices` constructed from (1) a parameter table or (2) individual ma - `S`: matrix of undirected effects - `F`: filter matrix - `M`: vector of mean effects -- `parameters::Vector{Symbol}`: parameter labels +- `params::Vector{Symbol}`: parameter labels - `colnames::Vector{Symbol}`: variable names corresponding to the A, S and F matrix columns # Examples @@ -79,7 +79,7 @@ function RAMMatrices end """ fixed(args...) -Fix parameters to a certain value. +Fix parameters to a certain value. For ensemble models, multiple values (one for each submodel/group) are needed. # Examples @@ -94,7 +94,7 @@ function fixed end """ start(args...) -Define starting values for parameters. +Define starting values for parameters. For ensemble models, multiple values (one for each submodel/group) are needed. # Examples diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index c14121bb4..e934f8b84 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -34,7 +34,7 @@ and for models with a meanstructure, the model implied means are computed as ``` ## Interfaces -- `identifier(::RAM) `-> Dict containing the parameter labels and their position +- `params(::RAM) `-> Dict containing the parameter labels and their position - `n_par(::RAM)` -> Number of parameters - `Σ(::RAM)` -> model implied covariance matrix @@ -111,7 +111,7 @@ mutable struct RAM{ ∇S::S2 ∇M::S3 - identifier::D + param_indices::D end using StructuralEquationModels @@ -128,12 +128,11 @@ function RAM(; kwargs..., ) ram_matrices = RAMMatrices(specification) - identifier = StructuralEquationModels.identifier(ram_matrices) + param_indices = SEM.param_indices(ram_matrices) # get dimensions of the model - n_par = length(ram_matrices.parameters) + n_par = length(ram_matrices.params) n_var, n_nod = ram_matrices.size_F - parameters = ram_matrices.parameters F = zeros(ram_matrices.size_F) F[CartesianIndex.(1:n_var, ram_matrices.F_ind)] .= 1.0 @@ -198,7 +197,7 @@ function RAM(; ∇A, ∇S, ∇M, - identifier, + param_indices, ) end @@ -213,7 +212,7 @@ gradient!(imply::RAM, par, model::AbstractSemSingle) = gradient!(imply, par, model, imply.has_meanstructure) # objective and gradient -function objective!(imply::RAM, parameters, model, has_meanstructure::Val{T}) where {T} +function objective!(imply::RAM, params, model, has_meanstructure::Val{T}) where {T} fill_A_S_M!( imply.A, imply.S, @@ -221,7 +220,7 @@ function objective!(imply::RAM, parameters, model, has_meanstructure::Val{T}) wh imply.A_indices, imply.S_indices, imply.M_indices, - parameters, + params, ) @. imply.I_A = -imply.A @@ -239,7 +238,7 @@ end function gradient!( imply::RAM, - parameters, + params, model::AbstractSemSingle, has_meanstructure::Val{T}, ) where {T} @@ -250,7 +249,7 @@ function gradient!( imply.A_indices, imply.S_indices, imply.M_indices, - parameters, + params, ) @. imply.I_A = -imply.A @@ -281,7 +280,7 @@ objective_gradient_hessian!(imply::RAM, par, model::AbstractSemSingle, has_means ### Recommended methods ############################################################################################ -identifier(imply::RAM) = imply.identifier +param_indices(imply::RAM) = imply.param_indices n_par(imply::RAM) = imply.n_par function update_observed(imply::RAM, observed::SemObserved; kwargs...) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index fae687e1d..ce381d2b4 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -29,7 +29,7 @@ Subtype of `SemImply` that implements the RAM notation with symbolic precomputat Subtype of `SemImply`. ## Interfaces -- `identifier(::RAMSymbolic) `-> Dict containing the parameter labels and their position +- `params(::RAMSymbolic) `-> Dict containing the parameter labels and their position - `n_par(::RAMSymbolic)` -> Number of parameters - `Σ(::RAMSymbolic)` -> model implied covariance matrix @@ -79,7 +79,7 @@ struct RAMSymbolic{F1, F2, F3, A1, A2, A3, S1, S2, S3, V, V2, F4, A4, F5, A5, D1 μ::A4 ∇μ_function::F5 ∇μ::A5 - identifier::D1 + param_indices::D1 has_meanstructure::B end @@ -98,9 +98,9 @@ function RAMSymbolic(; kwargs..., ) ram_matrices = RAMMatrices(specification) - identifier = StructuralEquationModels.identifier(ram_matrices) + param_indices = SEM.param_indices(ram_matrices) - n_par = length(ram_matrices.parameters) + n_par = length(ram_matrices.params) n_var, n_nod = ram_matrices.size_F par = (Symbolics.@variables θ[1:n_par])[1] @@ -201,7 +201,7 @@ function RAMSymbolic(; μ, ∇μ_function, ∇μ, - identifier, + param_indices, has_meanstructure, ) end @@ -240,7 +240,7 @@ objective_gradient_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, p ### Recommended methods ############################################################################################ -identifier(imply::RAMSymbolic) = imply.identifier +param_indices(imply::RAMSymbolic) = imply.param_indices n_par(imply::RAMSymbolic) = imply.n_par function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) diff --git a/src/imply/empty.jl b/src/imply/empty.jl index ba8580d16..1d0ea69ff 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -19,14 +19,14 @@ model per group and an additional model with `ImplyEmpty` and `SemRidge` for the # Extended help ## Interfaces -- `identifier(::RAMSymbolic) `-> Dict containing the parameter labels and their position +- `params(::RAMSymbolic) `-> Dict containing the parameter labels and their position - `n_par(::RAMSymbolic)` -> Number of parameters ## Implementation Subtype of `SemImply`. """ struct ImplyEmpty{V, V2} <: SemImply - identifier::V2 + param_indices::V2 n_par::V end @@ -37,9 +37,9 @@ end function ImplyEmpty(; specification, kwargs...) ram_matrices = RAMMatrices(specification) - n_par = length(ram_matrices.parameters) + n_par = length(ram_matrices.params) - return ImplyEmpty(identifier(ram_matrices), n_par) + return ImplyEmpty(param_indices(ram_matrices), n_par) end ############################################################################################ @@ -54,7 +54,7 @@ hessian!(imply::ImplyEmpty, par, model) = nothing ### Recommended methods ############################################################################################ -identifier(imply::ImplyEmpty) = imply.identifier +param_indices(imply::ImplyEmpty) = imply.param_indices n_par(imply::ImplyEmpty) = imply.n_par update_observed(imply::ImplyEmpty, observed::SemObserved; kwargs...) = imply diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 1cc7c123c..18cc88289 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -82,20 +82,20 @@ end ### methods ############################################################################################ -function objective!(semfiml::SemFIML, parameters, model) +function objective!(semfiml::SemFIML, params, model) if !check_fiml(semfiml, model) - return non_posdef_return(parameters) + return non_posdef_return(params) end prepare_SemFIML!(semfiml, model) - objective = F_FIML(rows(observed(model)), semfiml, model, parameters) + objective = F_FIML(rows(observed(model)), semfiml, model, params) return objective / n_obs(observed(model)) end -function gradient!(semfiml::SemFIML, parameters, model) +function gradient!(semfiml::SemFIML, params, model) if !check_fiml(semfiml, model) - return ones(eltype(parameters), size(parameters)) + return ones(eltype(params), size(params)) end prepare_SemFIML!(semfiml, model) @@ -104,15 +104,15 @@ function gradient!(semfiml::SemFIML, parameters, model) return gradient end -function objective_gradient!(semfiml::SemFIML, parameters, model) +function objective_gradient!(semfiml::SemFIML, params, model) if !check_fiml(semfiml, model) - return non_posdef_return(parameters), ones(eltype(parameters), size(parameters)) + return non_posdef_return(params), ones(eltype(params), size(params)) end prepare_SemFIML!(semfiml, model) objective = - F_FIML(rows(observed(model)), semfiml, model, parameters) / n_obs(observed(model)) + F_FIML(rows(observed(model)), semfiml, model, params) / n_obs(observed(model)) gradient = ∇F_FIML(rows(observed(model)), semfiml, model) / n_obs(observed(model)) return objective, gradient @@ -174,8 +174,8 @@ function ∇F_fiml_outer(JΣ, Jμ, imply, model, semfiml) return G end -function F_FIML(rows, semfiml, model, parameters) - F = zero(eltype(parameters)) +function F_FIML(rows, semfiml, model, params) + F = zero(eltype(params)) for i in 1:size(rows, 1) F += F_one_pattern( semfiml.meandiff[i], diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index ebf3e7bfe..0d9d10b4b 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -58,7 +58,7 @@ function SemRidge(; ), ) else - which_ridge = get_identifier_indices(which_ridge, imply) + which_ridge = params_to_indices(which_ridge, imply) end end which = [CartesianIndex(x) for x in which_ridge] diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index 53d68ec2c..61b78a54f 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -2,56 +2,56 @@ # methods for AbstractSem ############################################################################################ -function objective!(model::AbstractSemSingle, parameters) - objective!(imply(model), parameters, model) - return objective!(loss(model), parameters, model) +function objective!(model::AbstractSemSingle, params) + objective!(imply(model), params, model) + return objective!(loss(model), params, model) end -function gradient!(gradient, model::AbstractSemSingle, parameters) +function gradient!(gradient, model::AbstractSemSingle, params) fill!(gradient, zero(eltype(gradient))) - gradient!(imply(model), parameters, model) - gradient!(gradient, loss(model), parameters, model) + gradient!(imply(model), params, model) + gradient!(gradient, loss(model), params, model) end -function hessian!(hessian, model::AbstractSemSingle, parameters) +function hessian!(hessian, model::AbstractSemSingle, params) fill!(hessian, zero(eltype(hessian))) - hessian!(imply(model), parameters, model) - hessian!(hessian, loss(model), parameters, model) + hessian!(imply(model), params, model) + hessian!(hessian, loss(model), params, model) end -function objective_gradient!(gradient, model::AbstractSemSingle, parameters) +function objective_gradient!(gradient, model::AbstractSemSingle, params) fill!(gradient, zero(eltype(gradient))) - objective_gradient!(imply(model), parameters, model) - objective_gradient!(gradient, loss(model), parameters, model) + objective_gradient!(imply(model), params, model) + objective_gradient!(gradient, loss(model), params, model) end -function objective_hessian!(hessian, model::AbstractSemSingle, parameters) +function objective_hessian!(hessian, model::AbstractSemSingle, params) fill!(hessian, zero(eltype(hessian))) - objective_hessian!(imply(model), parameters, model) - objective_hessian!(hessian, loss(model), parameters, model) + objective_hessian!(imply(model), params, model) + objective_hessian!(hessian, loss(model), params, model) end -function gradient_hessian!(gradient, hessian, model::AbstractSemSingle, parameters) +function gradient_hessian!(gradient, hessian, model::AbstractSemSingle, params) fill!(gradient, zero(eltype(gradient))) fill!(hessian, zero(eltype(hessian))) - gradient_hessian!(imply(model), parameters, model) - gradient_hessian!(gradient, hessian, loss(model), parameters, model) + gradient_hessian!(imply(model), params, model) + gradient_hessian!(gradient, hessian, loss(model), params, model) end function objective_gradient_hessian!( gradient, hessian, model::AbstractSemSingle, - parameters, + params, ) fill!(gradient, zero(eltype(gradient))) fill!(hessian, zero(eltype(hessian))) - objective_gradient_hessian!(imply(model), parameters, model) - return objective_gradient_hessian!(gradient, hessian, loss(model), parameters, model) + objective_gradient_hessian!(imply(model), params, model) + return objective_gradient_hessian!(gradient, hessian, loss(model), params, model) end ############################################################################################ -# methods for SemFiniteDiff +# methods for SemFiniteDiff ############################################################################################ gradient!(gradient, model::SemFiniteDiff, par) = @@ -60,25 +60,25 @@ gradient!(gradient, model::SemFiniteDiff, par) = hessian!(hessian, model::SemFiniteDiff, par) = FiniteDiff.finite_difference_hessian!(hessian, x -> objective!(model, x), par) -function objective_gradient!(gradient, model::SemFiniteDiff, parameters) - gradient!(gradient, model, parameters) - return objective!(model, parameters) +function objective_gradient!(gradient, model::SemFiniteDiff, params) + gradient!(gradient, model, params) + return objective!(model, params) end # other methods -function gradient_hessian!(gradient, hessian, model::SemFiniteDiff, parameters) - gradient!(gradient, model, parameters) - hessian!(hessian, model, parameters) +function gradient_hessian!(gradient, hessian, model::SemFiniteDiff, params) + gradient!(gradient, model, params) + hessian!(hessian, model, params) end -function objective_hessian!(hessian, model::SemFiniteDiff, parameters) - hessian!(hessian, model, parameters) - return objective!(model, parameters) +function objective_hessian!(hessian, model::SemFiniteDiff, params) + hessian!(hessian, model, params) + return objective!(model, params) end -function objective_gradient_hessian!(gradient, hessian, model::SemFiniteDiff, parameters) - hessian!(hessian, model, parameters) - return objective_gradient!(gradient, model, parameters) +function objective_gradient_hessian!(gradient, hessian, model::SemFiniteDiff, params) + hessian!(hessian, model, params) + return objective_gradient!(gradient, model, params) end ############################################################################################ @@ -341,44 +341,44 @@ end # Documentation ############################################################################################ """ - objective!(model::AbstractSem, parameters) + objective!(model::AbstractSem, params) -Returns the objective value at `parameters`. +Returns the objective value at `params`. The model object can be modified. # Implementation To implement a new `SemImply` or `SemLossFunction` subtype, you need to add a method for - objective!(newtype::MyNewType, parameters, model::AbstractSemSingle) + objective!(newtype::MyNewType, params, model::AbstractSemSingle) To implement a new `AbstractSem` subtype, you need to add a method for - objective!(model::MyNewType, parameters) + objective!(model::MyNewType, params) """ function objective! end """ - gradient!(gradient, model::AbstractSem, parameters) + gradient!(gradient, model::AbstractSem, params) -Writes the gradient value at `parameters` to `gradient`. +Writes the gradient value at `params` to `gradient`. # Implementation To implement a new `SemImply` or `SemLossFunction` type, you can add a method for - gradient!(newtype::MyNewType, parameters, model::AbstractSemSingle) + gradient!(newtype::MyNewType, params, model::AbstractSemSingle) To implement a new `AbstractSem` subtype, you can add a method for - gradient!(gradient, model::MyNewType, parameters) + gradient!(gradient, model::MyNewType, params) """ function gradient! end """ - hessian!(hessian, model::AbstractSem, parameters) + hessian!(hessian, model::AbstractSem, params) -Writes the hessian value at `parameters` to `hessian`. +Writes the hessian value at `params` to `hessian`. # Implementation To implement a new `SemImply` or `SemLossFunction` type, you can add a method for - hessian!(newtype::MyNewType, parameters, model::AbstractSemSingle) + hessian!(newtype::MyNewType, params, model::AbstractSemSingle) To implement a new `AbstractSem` subtype, you can add a method for - hessian!(hessian, model::MyNewType, parameters) + hessian!(hessian, model::MyNewType, params) """ function hessian! end diff --git a/src/types.jl b/src/types.jl index 46b2781fb..f026b2cb0 100644 --- a/src/types.jl +++ b/src/types.jl @@ -153,14 +153,14 @@ Returns a SemEnsemble with fields - `sems::Tuple`: `AbstractSem`s. - `weights::Vector`: Weights for each model. - `optimizer::SemOptimizer`: Connects the model to the optimizer. See also [`SemOptimizer`](@ref). -- `identifier::Dict`: Stores parameter labels and their position. +- `param_indices::Dict`: Stores parameter labels and their position. """ struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, D, I} <: AbstractSemCollection n::N sems::T weights::V optimizer::D - identifier::I + param_indices::I end function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing, kwargs...) @@ -174,11 +174,11 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing weights = [n_obs(model) / nobs_total for model in models] end - # check identifier equality - id = identifier(models[1]) + # check parameters equality + par_indices = param_indices(models[1]) for model in models - if id != identifier(model) - throw(ErrorException("The identifier of your models do not match. \n + if par_indices != param_indices(model) + throw(ErrorException("The parameters of your models do not match. \n Maybe you tried to specify models of an ensemble via ParameterTables. \n In that case, you may use RAMMatrices instead.")) end @@ -189,7 +189,7 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing optimizer = optimizer(; kwargs...) end - return SemEnsemble(n, models, weights, optimizer, id) + return SemEnsemble(n, models, weights, optimizer, par_indices) end """ diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 3bb4e217a..4ba264e37 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,43 +1,43 @@ -function test_gradient(model, parameters; rtol = 1e-10, atol = 0) +function test_gradient(model, params; rtol = 1e-10, atol = 0) true_grad = - FiniteDiff.finite_difference_gradient(Base.Fix1(objective!, model), parameters) - gradient = similar(parameters) + FiniteDiff.finite_difference_gradient(Base.Fix1(objective!, model), params) + gradient = similar(params) # F and G fill!(gradient, NaN) - gradient!(gradient, model, parameters) + gradient!(gradient, model, params) @test gradient ≈ true_grad rtol = rtol atol = atol # only G fill!(gradient, NaN) - objective_gradient!(gradient, model, parameters) + objective_gradient!(gradient, model, params) @test gradient ≈ true_grad rtol = rtol atol = atol end -function test_hessian(model, parameters; rtol = 1e-4, atol = 0) +function test_hessian(model, params; rtol = 1e-4, atol = 0) true_hessian = - FiniteDiff.finite_difference_hessian(Base.Fix1(objective!, model), parameters) - hessian = similar(parameters, size(true_hessian)) - gradient = similar(parameters) + FiniteDiff.finite_difference_hessian(Base.Fix1(objective!, model), params) + hessian = similar(params, size(true_hessian)) + gradient = similar(params) # H fill!(hessian, NaN) - hessian!(hessian, model, parameters) + hessian!(hessian, model, params) @test hessian ≈ true_hessian rtol = rtol atol = atol # F and H fill!(hessian, NaN) - objective_hessian!(hessian, model, parameters) + objective_hessian!(hessian, model, params) @test hessian ≈ true_hessian rtol = rtol atol = atol # G and H fill!(hessian, NaN) - gradient_hessian!(gradient, hessian, model, parameters) + gradient_hessian!(gradient, hessian, model, params) @test hessian ≈ true_hessian rtol = rtol atol = atol # F, G and H fill!(hessian, NaN) - objective_gradient_hessian!(gradient, hessian, model, parameters) + objective_gradient_hessian!(gradient, hessian, model, params) @test hessian ≈ true_hessian rtol = rtol atol = atol end diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 9b97300df..8913860c8 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -122,7 +122,7 @@ struct UserSemML <: SemLossFunction end using LinearAlgebra: isposdef, logdet, tr, inv -function SEM.objective!(semml::UserSemML, parameters, model::AbstractSem) +function SEM.objective!(semml::UserSemML, params, model::AbstractSem) Σ = imply(model).Σ Σₒ = SEM.obs_cov(observed(model)) if !isposdef(Σ) diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 759c24eda..818f9afdc 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -56,7 +56,7 @@ specification_g1 = RAMMatrices(; A = A, S = S1, F = F, - parameters = x, + params = x, colnames = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], ) @@ -64,7 +64,7 @@ specification_g2 = RAMMatrices(; A = A, S = S2, F = F, - parameters = x, + params = x, colnames = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], ) diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 389800745..86b7e89bc 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -75,7 +75,7 @@ spec = RAMMatrices(; A = A, S = S, F = F, - parameters = x, + params = x, colnames = [ :x1, :x2, @@ -107,7 +107,7 @@ spec_mean = RAMMatrices(; S = S, F = F, M = M, - parameters = x, + params = x, colnames = [ :x1, :x2, diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 68e44ce20..1bd7136bc 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -40,7 +40,7 @@ A = [ 0 0 0 0 0 0 0 0 ] -ram_matrices = RAMMatrices(; A = A, S = S, F = F, parameters = x, colnames = nothing) +ram_matrices = RAMMatrices(; A = A, S = S, F = F, params = x, colnames = nothing) true_val = [ repeat([1], 8) diff --git a/test/unit_tests/specification.jl b/test/unit_tests/specification.jl index 0bfc0de2d..42ad5e431 100644 --- a/test/unit_tests/specification.jl +++ b/test/unit_tests/specification.jl @@ -3,18 +3,18 @@ @test ram_matrices == RAMMatrices(partable) end -@test get_identifier_indices([:x2, :x10, :x28], model_ml) == [2, 10, 28] +@test params_to_indices([:x2, :x10, :x28], model_ml) == [2, 10, 28] -@testset "get_identifier_indices" begin +@testset "params_to_indices" begin pars = [:θ_1, :θ_7, :θ_21] - @test get_identifier_indices(pars, model_ml) == get_identifier_indices(pars, partable) - @test get_identifier_indices(pars, model_ml) == - get_identifier_indices(pars, RAMMatrices(partable)) + @test params_to_indices(pars, model_ml) == params_to_indices(pars, partable) + @test params_to_indices(pars, model_ml) == + params_to_indices(pars, RAMMatrices(partable)) end # from docstrings: -parameter_indices = get_identifier_indices([:λ₁, λ₂], my_fitted_sem) -values = solution(my_fitted_sem)[parameter_indices] +param_indices = params_to_indices([:λ₁, λ₂], my_fitted_sem) +values = solution(my_fitted_sem)[param_indices] graph = @StenoGraph begin # measurement model From 61282bc00e70da6cf2f928fec7210af7561fc44f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 28 Apr 2024 13:23:59 -0700 Subject: [PATCH 011/364] ParTable: columns[:identifier] => columns[:param] --- src/frontend/fit/summary.jl | 10 +++++----- src/frontend/specification/ParameterTable.jl | 8 ++++---- src/frontend/specification/RAMMatrices.jl | 8 ++++---- src/frontend/specification/StenoGraphs.jl | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 4cda902d7..1d75eb826 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -91,7 +91,7 @@ function sem_summary( printstyled("Loadings: \n"; color = color) print("\n") - sorted_columns = [:to, :estimate, :identifier, :value_fixed, :start] + sorted_columns = [:to, :estimate, :param, :value_fixed, :start] loading_columns = sort_partially(sorted_columns, columns) header_cols = copy(loading_columns) replace!(header_cols, :parameter_type => :type) @@ -140,7 +140,7 @@ function sem_summary( ) sorted_columns = - [:from, :parameter_type, :to, :estimate, :identifier, :value_fixed, :start] + [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] regression_columns = sort_partially(sorted_columns, columns) regression_array = reduce( @@ -168,7 +168,7 @@ function sem_summary( ) sorted_columns = - [:from, :parameter_type, :to, :estimate, :identifier, :value_fixed, :start] + [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce( @@ -196,7 +196,7 @@ function sem_summary( ) sorted_columns = - [:from, :parameter_type, :to, :estimate, :identifier, :value_fixed, :start] + [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce( @@ -225,7 +225,7 @@ function sem_summary( printstyled("Means: \n"; color = color) sorted_columns = - [:from, :parameter_type, :to, :estimate, :identifier, :value_fixed, :start] + [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce( diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index c0430625f..aff0380f9 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -21,7 +21,7 @@ function ParameterTable(::Nothing) :value_fixed => Vector{Float64}(), :start => Vector{Float64}(), :estimate => Vector{Float64}(), - :identifier => Vector{Symbol}(), + :param => Vector{Symbol}(), :start => Vector{Float64}(), ) @@ -66,7 +66,7 @@ function Base.show(io::IO, partable::ParameterTable) :start, :estimate, :se, - :identifier, + :param, ] existing_columns = [haskey(partable.columns, key) for key in relevant_columns] @@ -102,7 +102,7 @@ Base.getindex(partable::ParameterTable, i::Int) = ( partable.columns[:to][i], partable.columns[:free][i], partable.columns[:value_fixed][i], - partable.columns[:identifier][i], + partable.columns[:param][i], ) function Base.length(partable::ParameterTable) @@ -197,7 +197,7 @@ function update_partable!( column, ) new_col = Vector{eltype(vec)}(undef, length(partable)) - for (i, param) in enumerate(partable.columns[:identifier]) + for (i, param) in enumerate(partable.columns[:param]) if !(param == :const) new_col[i] = values[param_indices[param]] elseif param == :const diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index eb92c889b..f6e56b950 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -292,7 +292,7 @@ end ############################################################################################ function get_par_npar_indices(partable::ParameterTable) - params = unique(partable.columns[:identifier]) + params = unique(partable.columns[:param]) filter!(x -> x != :const, params) n_par = length(params) par_positions = Dict(params .=> 1:n_par) @@ -302,7 +302,7 @@ end function get_par_npar_indices(partable::EnsembleParameterTable) params = Vector{Symbol}() for key in keys(partable.tables) - append!(params, partable.tables[key].columns[:identifier]) + append!(params, partable.tables[key].columns[:param]) end params = unique(params) filter!(x -> x != :const, params) @@ -339,7 +339,7 @@ function get_partable_row(c::RAMConstant, position_names) :value_fixed => value_fixed, :start => start, :estimate => estimate, - :identifier => :const, + :param => :const, ) end @@ -396,7 +396,7 @@ function get_partable_row(param, position_names, index, matrix, n_nod, known_ind :value_fixed => value_fixed, :start => start, :estimate => estimate, - :identifier => param, + :param => param, ) end diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index a581e9e5a..bebbdeb2e 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -108,7 +108,7 @@ function ParameterTable(; graph, observed_vars, latent_vars, g = 1, parname = : :value_fixed => value_fixed, :start => start, :estimate => estimate, - :identifier => params, + :param => params, ), Dict( :latent_vars => latent_vars, From a0e76cc5ea6b0675568f4c1b8893561c0d72fbfc Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 18:41:13 -0700 Subject: [PATCH 012/364] getindex(EnsParTable, i) instead of get_group() --- src/frontend/specification/EnsembleParameterTable.jl | 4 +--- src/frontend/specification/RAMMatrices.jl | 4 ---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 29a6cf984..0865a4dcc 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -98,9 +98,7 @@ end push!(partable::EnsembleParameterTable, d::Nothing, group) = nothing -# get group -------------------------------------------------------------------------------- - -get_group(partable::EnsembleParameterTable, group) = get_group(partable.tables, group) +Base.getindex(partable::EnsembleParameterTable, group) = partable.tables[group] ############################################################################################ ### Update Partable from Fitted Model diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index f6e56b950..db52defcd 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -439,7 +439,3 @@ function ==(mat1::RAMMatrices, mat2::RAMMatrices) ) return res end - -function get_group(d::Dict, group) - return d[group] -end From e177e29c73194c423ac60ec3b912690fb44629d4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 18:40:35 -0700 Subject: [PATCH 013/364] replace no-op ctors with convert(T, obj) convert() is a proper method to call to avoid unnecessary construction, ctor semantics requires that a new object is constructed --- src/frontend/specification/EnsembleParameterTable.jl | 6 ++---- src/frontend/specification/RAMMatrices.jl | 7 +++++-- src/imply/RAM/generic.jl | 2 +- src/imply/RAM/symbolic.jl | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 0865a4dcc..24a9a295a 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -20,10 +20,8 @@ end ### Convert to other types ############################################################################################ -import Base.Dict - -function Dict(partable::EnsembleParameterTable) - return partable.tables +function Base.convert(::Type{Dict}, partable::EnsembleParameterTable) + return convert(Dict, partable.tables) end #= function DataFrame( diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index db52defcd..6752d7c6b 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -39,8 +39,6 @@ function RAMMatrices(; A, S, F, M = nothing, params, colnames) ) end -RAMMatrices(a::RAMMatrices) = a - ############################################################################################ ### Constants ############################################################################################ @@ -217,6 +215,8 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) ) end +Base.convert(::Type{RAMMatrices}, partable::ParameterTable) = RAMMatrices(partable) + ############################################################################################ ### get parameter table from RAMMatrices ############################################################################################ @@ -259,6 +259,9 @@ function ParameterTable(ram_matrices::RAMMatrices) return partable end +Base.convert(::Type{<:ParameterTable}, ram_matrices::RAMMatrices) = + ParameterTable(ram_matrices) + ############################################################################################ ### get RAMMatrices from EnsembleParameterTable ############################################################################################ diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index e934f8b84..d43c8378e 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -127,7 +127,7 @@ function RAM(; meanstructure = false, kwargs..., ) - ram_matrices = RAMMatrices(specification) + ram_matrices = convert(RAMMatrices, specification) param_indices = SEM.param_indices(ram_matrices) # get dimensions of the model diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index ce381d2b4..0fe9c29bb 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -97,7 +97,7 @@ function RAMSymbolic(; approximate_hessian = false, kwargs..., ) - ram_matrices = RAMMatrices(specification) + ram_matrices = convert(RAMMatrices, specification) param_indices = SEM.param_indices(ram_matrices) n_par = length(ram_matrices.params) From 161dc584adbb5ea1fbc1a0de1583ebd92369058d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 27 May 2024 14:29:44 -0700 Subject: [PATCH 014/364] ParamTable: convert vars from Dict to fields make the type immutable --- src/frontend/fit/summary.jl | 34 +++--- src/frontend/specification/ParameterTable.jl | 107 ++++++++----------- src/frontend/specification/RAMMatrices.jl | 83 ++++++-------- src/loss/ML/FIML.jl | 4 +- src/observed/get_colnames.jl | 15 +-- test/examples/helper.jl | 7 +- test/unit_tests/data_input_formats.jl | 7 +- 7 files changed, 106 insertions(+), 151 deletions(-) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 1d75eb826..4d6bc6181 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -60,19 +60,18 @@ function sem_summary( ) print("\n") printstyled("Latent variables: "; color = color) - for var in partable.variables[:latent_vars] + for var in partable.latent_vars print("$var ") end print("\n") printstyled("Observed variables: "; color = color) - for var in partable.variables[:observed_vars] + for var in partable.observed_vars print("$var ") end print("\n") - if haskey(partable.variables, :sorted_vars) && - (length(partable.variables[:sorted_vars]) > 0) + if length(partable.sorted_vars) > 0 printstyled("Sorted variables: "; color = color) - for var in partable.variables[:sorted_vars] + for var in partable.sorted_vars print("$var ") end print("\n") @@ -96,11 +95,11 @@ function sem_summary( header_cols = copy(loading_columns) replace!(header_cols, :parameter_type => :type) - for var in partable.variables[:latent_vars] + for var in partable.latent_vars indicator_indices = findall( (partable.columns[:from] .== var) .& (partable.columns[:parameter_type] .== :→) .& - (partable.columns[:to] .∈ [partable.variables[:observed_vars]]), + (partable.columns[:to] .∈ [partable.observed_vars]), ) loading_array = reduce( hcat, @@ -125,16 +124,16 @@ function sem_summary( regression_indices = findall( (partable.columns[:parameter_type] .== :→) .& ( ( - (partable.columns[:to] .∈ [partable.variables[:observed_vars]]) .& - (partable.columns[:from] .∈ [partable.variables[:observed_vars]]) + (partable.columns[:to] .∈ [partable.observed_vars]) .& + (partable.columns[:from] .∈ [partable.observed_vars]) ) .| ( - (partable.columns[:to] .∈ [partable.variables[:latent_vars]]) .& - (partable.columns[:from] .∈ [partable.variables[:observed_vars]]) + (partable.columns[:to] .∈ [partable.latent_vars]) .& + (partable.columns[:from] .∈ [partable.observed_vars]) ) .| ( - (partable.columns[:to] .∈ [partable.variables[:latent_vars]]) .& - (partable.columns[:from] .∈ [partable.variables[:latent_vars]]) + (partable.columns[:to] .∈ [partable.latent_vars]) .& + (partable.columns[:from] .∈ [partable.latent_vars]) ) ), ) @@ -266,19 +265,18 @@ function sem_summary( print("\n") let partable = partable.tables[[keys(partable.tables)...][1]] printstyled("Latent variables: "; color = color) - for var in partable.variables[:latent_vars] + for var in partable.latent_vars print("$var ") end print("\n") printstyled("Observed variables: "; color = color) - for var in partable.variables[:observed_vars] + for var in partable.observed_vars print("$var ") end print("\n") - if haskey(partable.variables, :sorted_vars) && - (length(partable.variables[:sorted_vars]) > 0) + if length(partable.sorted_vars) > 0 printstyled("Sorted variables: "; color = color) - for var in partable.variables[:sorted_vars] + for var in partable.sorted_vars print("$var ") end print("\n") diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index aff0380f9..49ea4664b 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -2,9 +2,11 @@ ### Types ############################################################################################ -mutable struct ParameterTable{C, V} <: AbstractParameterTable +struct ParameterTable{C} <: AbstractParameterTable columns::C - variables::V + observed_vars::Vector{Symbol} + latent_vars::Vector{Symbol} + sorted_vars::Vector{Symbol} end ############################################################################################ @@ -12,7 +14,10 @@ end ############################################################################################ # constuct an empty table -function ParameterTable(::Nothing) +function ParameterTable(; + observed_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, + latent_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, +) columns = Dict{Symbol, Any}( :from => Vector{Symbol}(), :parameter_type => Vector{Symbol}(), @@ -25,13 +30,10 @@ function ParameterTable(::Nothing) :start => Vector{Float64}(), ) - variables = Dict{Symbol, Any}( - :latent_vars => Vector{Symbol}(), - :observed_vars => Vector{Symbol}(), - :sorted_vars => Vector{Symbol}(), - ) - - return ParameterTable(columns, variables) + return ParameterTable(columns, + !isnothing(observed_vars) ? copy(observed_vars) : Vector{Symbol}(), + !isnothing(latent_vars) ? copy(latent_vars) : Vector{Symbol}(), + Vector{Symbol}()) end ############################################################################################ @@ -68,26 +70,21 @@ function Base.show(io::IO, partable::ParameterTable) :se, :param, ] - existing_columns = [haskey(partable.columns, key) for key in relevant_columns] + shown_columns = filter!( + col -> haskey(partable.columns, col) && length(partable.columns[col]) > 0, + relevant_columns, + ) - as_matrix = - hcat([partable.columns[key] for key in relevant_columns[existing_columns]]...) + as_matrix = mapreduce(col -> partable.columns[col], hcat, shown_columns) pretty_table( io, as_matrix, - header = ( - relevant_columns[existing_columns], - eltype.([partable.columns[key] for key in relevant_columns[existing_columns]]), - ), + header = (shown_columns, [eltype(partable.columns[col]) for col in shown_columns]), tf = PrettyTables.tf_compact, ) - if haskey(partable.variables, :latent_vars) - print(io, "Latent Variables: $(partable.variables[:latent_vars]) \n") - end - if haskey(partable.variables, :observed_vars) - print(io, "Observed Variables: $(partable.variables[:observed_vars]) \n") - end + print(io, "Latent Variables: $(partable.latent_vars) \n") + print(io, "Observed Variables: $(partable.observed_vars) \n") end ############################################################################################ @@ -96,7 +93,7 @@ end # Iteration -------------------------------------------------------------------------------- -Base.getindex(partable::ParameterTable, i::Int) = ( +Base.getindex(partable::ParameterTable, i::Integer) = ( partable.columns[:from][i], partable.columns[:parameter_type][i], partable.columns[:to][i], @@ -105,14 +102,7 @@ Base.getindex(partable::ParameterTable, i::Int) = ( partable.columns[:param][i], ) -function Base.length(partable::ParameterTable) - len = missing - for key in keys(partable.columns) - len = length(partable.columns[key]) - break - end - return len -end +Base.length(partable::ParameterTable) = length(first(values(partable.columns))) # Sorting ---------------------------------------------------------------------------------- @@ -122,51 +112,46 @@ end Base.showerror(io::IO, e::CyclicModelError) = print(io, e.msg) -import Base.sort!, Base.sort - -function sort!(partable::ParameterTable) - variables = [partable.variables[:latent_vars]; partable.variables[:observed_vars]] +function Base.sort!(partable::ParameterTable) + vars = [ + partable.latent_vars + partable.observed_vars + ] - is_regression = - (partable.columns[:parameter_type] .== :→) .& - (partable.columns[:from] .!= Symbol("1")) + is_regression = [ + (partype == :→) && (from != Symbol("1")) for + (partype, from) in zip(partable.columns[:parameter_type], partable.columns[:from]) + ] to = partable.columns[:to][is_regression] from = partable.columns[:from][is_regression] - sorted_variables = Vector{Symbol}() + sorted_vars = Vector{Symbol}() - sorted = false - while !sorted + while !isempty(vars) acyclic = false - for (i, variable) in enumerate(variables) - if !(variable ∈ to) - push!(sorted_variables, variable) - deleteat!(variables, i) - delete_edges = from .!= variable + for (i, var) in enumerate(vars) + if !(var ∈ to) + push!(sorted_vars, var) + deleteat!(vars, i) + delete_edges = from .!= var to = to[delete_edges] from = from[delete_edges] acyclic = true end end - if !acyclic + acyclic || throw(CyclicModelError("your model is cyclic and therefore can not be ordered")) - end - acyclic = false - - if length(variables) == 0 - sorted = true - end end - push!(partable.variables, :sorted_vars => sorted_variables) + copyto!(resize!(partable.sorted_vars, length(sorted_vars)), sorted_vars) return partable end -function sort(partable::ParameterTable) +function Base.sort(partable::ParameterTable) new_partable = deepcopy(partable) sort!(new_partable) return new_partable @@ -174,15 +159,13 @@ end # add a row -------------------------------------------------------------------------------- -import Base.push! - -function push!(partable::ParameterTable, d::AbstractDict) - for key in keys(d) - push!(partable.columns[key], d[key]) +function Base.push!(partable::ParameterTable, d::AbstractDict{Symbol}) + for (key, val) in pairs(d) + push!(partable.columns[key], val) end end -push!(partable::ParameterTable, d::Nothing) = nothing +Base.push!(partable::ParameterTable, d::Nothing) = nothing ############################################################################################ ### Update Partable from Fitted Model diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 6752d7c6b..45bdfe57b 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -111,34 +111,24 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) par_id[:params], par_id[:n_par], par_id[:par_positions] end - n_observed = size(partable.variables[:observed_vars], 1) - n_latent = size(partable.variables[:latent_vars], 1) + n_observed = length(partable.observed_vars) + n_latent = length(partable.latent_vars) n_node = n_observed + n_latent # F indices - if length(partable.variables[:sorted_vars]) != 0 - F_ind = findall( - x -> x ∈ partable.variables[:observed_vars], - partable.variables[:sorted_vars], - ) - else - F_ind = 1:n_observed - end + F_ind = + length(partable.sorted_vars) != 0 ? + findall(∈(Set(partable.observed_vars)), partable.sorted_vars) : + 1:n_observed # indices of the colnames - if length(partable.variables[:sorted_vars]) != 0 - positions = - Dict(zip(partable.variables[:sorted_vars], collect(1:n_observed+n_latent))) - colnames = copy(partable.variables[:sorted_vars]) - else - positions = Dict( - zip( - [partable.variables[:observed_vars]; partable.variables[:latent_vars]], - collect(1:n_observed+n_latent), - ), - ) - colnames = [partable.variables[:observed_vars]; partable.variables[:latent_vars]] - end + colnames = + length(partable.sorted_vars) != 0 ? copy(partable.sorted_vars) : + [ + partable.observed_vars + partable.latent_vars + ] + col_indices = Dict(col => i for (i, col) in enumerate(colnames)) # fill Matrices # known_labels = Dict{Symbol, Int64}() @@ -154,51 +144,48 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) end # is there a meanstructure? - if any(partable.columns[:from] .== Symbol("1")) - M_ind = Vector{Vector{Int64}}(undef, n_par) - for i in 1:length(M_ind) - M_ind[i] = Vector{Int64}() - end - else - M_ind = nothing - end + M_ind = + any(==(Symbol("1")), partable.columns[:from]) ? [Vector{Int64}() for _ in 1:n_par] : + nothing - # handel constants + # handle constants constants = Vector{RAMConstant}() for i in 1:length(partable) from, parameter_type, to, free, value_fixed, param = partable[i] - row_ind = positions[to] - if from != Symbol("1") - col_ind = positions[from] - end + row_ind = col_indices[to] + col_ind = from != Symbol("1") ? col_indices[from] : nothing if !free - if (parameter_type == :→) & (from == Symbol("1")) + if (parameter_type == :→) && (from == Symbol("1")) push!(constants, RAMConstant(:M, row_ind, value_fixed)) elseif (parameter_type == :→) push!( constants, RAMConstant(:A, CartesianIndex(row_ind, col_ind), value_fixed), ) - else + elseif (parameter_type == :↔) push!( constants, RAMConstant(:S, CartesianIndex(row_ind, col_ind), value_fixed), ) + else + error("Unsupported parameter type: $(parameter_type)") end else par_ind = par_positions[param] if (parameter_type == :→) && (from == Symbol("1")) push!(M_ind[par_ind], row_ind) elseif parameter_type == :→ - push!(A_ind[par_ind], (row_ind + (col_ind - 1) * n_node)) - else + push!(A_ind[par_ind], row_ind + (col_ind - 1) * n_node) + elseif parameter_type == :↔ push!(S_ind[par_ind], row_ind + (col_ind - 1) * n_node) if row_ind != col_ind push!(S_ind[par_ind], col_ind + (row_ind - 1) * n_node) end + else + error("Unsupported parameter type: $(parameter_type)") end end end @@ -222,21 +209,15 @@ Base.convert(::Type{RAMMatrices}, partable::ParameterTable) = RAMMatrices(partab ############################################################################################ function ParameterTable(ram_matrices::RAMMatrices) - partable = ParameterTable(nothing) - colnames = ram_matrices.colnames - position_names = Dict{Int64, Symbol}(1:length(colnames) .=> colnames) - - # observed and latent variables - names_obs = colnames[ram_matrices.F_ind] - names_lat = colnames[findall(x -> !(x ∈ ram_matrices.F_ind), 1:length(colnames))] - partable.variables = Dict( - :sorted_vars => Vector{Symbol}(), - :observed_vars => names_obs, - :latent_vars => names_lat, + partable = ParameterTable( + observed_vars = colnames[ram_matrices.F_ind], + latent_vars = colnames[setdiff(eachindex(colnames), ram_matrices.F_ind)], ) + position_names = Dict{Int64, Symbol}(1:length(colnames) .=> colnames) + # constants for c in ram_matrices.constants push!(partable, get_partable_row(c, position_names)) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 18cc88289..9f5103f1f 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -252,5 +252,5 @@ end get_n_nodes(specification::RAMMatrices) = specification.size_F[2] get_n_nodes(specification::ParameterTable) = - length(specification.variables[:observed_vars]) + - length(specification.variables[:latent_vars]) + length(specification.observed_vars) + + length(specification.latent_vars) diff --git a/src/observed/get_colnames.jl b/src/observed/get_colnames.jl index d620de659..b8d89c3d0 100644 --- a/src/observed/get_colnames.jl +++ b/src/observed/get_colnames.jl @@ -1,15 +1,8 @@ -# specification colnames +# specification colnames (only observed) function get_colnames(specification::ParameterTable) - if !haskey(specification.variables, :sorted_vars) || - (length(specification.variables[:sorted_vars]) == 0) - colnames = specification.variables[:observed_vars] - else - is_obs = [ - var ∈ specification.variables[:observed_vars] for - var in specification.variables[:sorted_vars] - ] - colnames = specification.variables[:sorted_vars][is_obs] - end + colnames = + isempty(specification.sorted_vars) ? specification.observed_vars : + filter(in(Set(specification.observed_vars)), specification.sorted_vars) return colnames end diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 4ba264e37..e93a1437d 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -118,8 +118,7 @@ function compare_estimates( if type == :↔ type = "~~" elseif type == :→ - if (from ∈ partable.variables[:latent_vars]) & - (to ∈ partable.variables[:observed_vars]) + if (from ∈ partable.latent_vars) && (to ∈ partable.observed_vars) type = "=~" else type = "~" @@ -251,8 +250,8 @@ function compare_estimates( if type == :↔ type = "~~" elseif type == :→ - if (from ∈ partable.variables[:latent_vars]) & - (to ∈ partable.variables[:observed_vars]) + if (from ∈ partable.latent_vars) && + (to ∈ partable.observed_vars) type = "=~" else type = "~" diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 485cf82d2..7a048b280 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -2,9 +2,10 @@ using StructuralEquationModels, Test, Statistics using StructuralEquationModels: obs_cov, obs_mean, get_data ### model specification -------------------------------------------------------------------- -spec = ParameterTable(nothing) -spec.variables[:observed_vars] = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] -spec.variables[:latent_vars] = [:ind60, :dem60, :dem65] +spec = ParameterTable( + observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8], + latent_vars = [:ind60, :dem60, :dem65], +) ### data ----------------------------------------------------------------------------------- From afaff2c9512722342330e6931ed636539e054a14 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 26 May 2024 21:23:55 -0700 Subject: [PATCH 015/364] ParamTable: update StenGraph-based ctor * use graph as a main parameter * simplify rows processing * don't reallocate table.columns Co-authored-by: Maximilian-Stefan-Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- src/frontend/specification/StenoGraphs.jl | 124 +++++++++--------- test/examples/multigroup/multigroup.jl | 8 +- .../political_democracy.jl | 7 +- 3 files changed, 66 insertions(+), 73 deletions(-) diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index bebbdeb2e..1d3332cb9 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -4,6 +4,9 @@ ### Define Modifiers ############################################################################################ +#FIXME: remove when StenoGraphs.jl will provide AbstractStenoGraph +const AbstractStenoGraph = AbstractArray{T, 1} where {T <: StenoGraphs.AbstractEdge} + # fixed parameter values struct Fixed{N} <: EdgeModifier value::N @@ -28,59 +31,59 @@ label(args...) = Label(args) ### constructor for parameter table from graph ############################################################################################ -function ParameterTable(; graph, observed_vars, latent_vars, g = 1, parname = :θ) +function ParameterTable( + graph::AbstractStenoGraph; + observed_vars, + latent_vars, + group::Integer = 1, + param_prefix = :θ, +) graph = unique(graph) n = length(graph) - from = Vector{Symbol}(undef, n) - parameter_type = Vector{Symbol}(undef, n) - to = Vector{Symbol}(undef, n) - free = ones(Bool, n) - value_fixed = zeros(n) - start = zeros(n) - estimate = zeros(n) - params = Vector{Symbol}(undef, n) - params .= Symbol("") - # group = Vector{Symbol}(undef, n) - # start_partable = zeros(Bool, n) - sorted_vars = Vector{Symbol}() + partable = ParameterTable(latent_vars = latent_vars, observed_vars = observed_vars) + from = resize!(partable.columns[:from], n) + parameter_type = resize!(partable.columns[:parameter_type], n) + to = resize!(partable.columns[:to], n) + free = fill!(resize!(partable.columns[:free], n), true) + value_fixed = fill!(resize!(partable.columns[:value_fixed], n), NaN) + start = fill!(resize!(partable.columns[:start], n), NaN) + param_refs = fill!(resize!(partable.columns[:param], n), Symbol("")) + # group = Vector{Symbol}(undef, n) for (i, element) in enumerate(graph) - if element isa DirectedEdge - from[i] = element.src.node - to[i] = element.dst.node + edge = element isa ModifiedEdge ? element.edge : element + from[i] = edge.src.node + to[i] = edge.dst.node + if edge isa DirectedEdge parameter_type[i] = :→ - elseif element isa UndirectedEdge - from[i] = element.src.node - to[i] = element.dst.node + elseif edge isa UndirectedEdge parameter_type[i] = :↔ - elseif element isa ModifiedEdge - if element.edge isa DirectedEdge - from[i] = element.edge.src.node - to[i] = element.edge.dst.node - parameter_type[i] = :→ - elseif element.edge isa UndirectedEdge - from[i] = element.edge.src.node - to[i] = element.edge.dst.node - parameter_type[i] = :↔ - end + else + throw( + ArgumentError( + "The graph contains an unsupported edge of type $(typeof(edge)).", + ), + ) + end + if element isa ModifiedEdge for modifier in values(element.modifiers) + modval = modifier.value[group] if modifier isa Fixed - if modifier.value[g] == :NaN + if modval == :NaN free[i] = true value_fixed[i] = 0.0 else free[i] = false - value_fixed[i] = modifier.value[g] + value_fixed[i] = modval end elseif modifier isa Start - start_partable[i] = modifier.value[g] == :NaN - start[i] = modifier.value[g] + start[i] = modval elseif modifier isa Label - if modifier.value[g] == :NaN + if modval == :NaN throw(DomainError(NaN, "NaN is not allowed as a parameter label.")) end - params[i] = modifier.value[g] + param_refs[i] = modval end end end @@ -88,41 +91,32 @@ function ParameterTable(; graph, observed_vars, latent_vars, g = 1, parname = : # make identifiers for parameters that are not labeled current_id = 1 - for i in 1:length(params) - if (params[i] == Symbol("")) & free[i] - params[i] = Symbol(parname, :_, current_id) - current_id += 1 - elseif (params[i] == Symbol("")) & !free[i] - params[i] = :const - elseif (params[i] != Symbol("")) & !free[i] - @warn "You labeled a constant. Please check if the labels of your graph are correct." + for i in eachindex(param_refs) + if param_refs[i] == Symbol("") + if free[i] + param_refs[i] = Symbol(param_prefix, :_, current_id) + current_id += 1 + else + param_refs[i] = :const + end + elseif !free[i] + @warn "You labeled a constant ($(param_refs[i])=$(value_fixed[i])). Please check if the labels of your graph are correct." end end - return StructuralEquationModels.ParameterTable( - Dict( - :from => from, - :parameter_type => parameter_type, - :to => to, - :free => free, - :value_fixed => value_fixed, - :start => start, - :estimate => estimate, - :param => params, - ), - Dict( - :latent_vars => latent_vars, - :observed_vars => observed_vars, - :sorted_vars => sorted_vars, - ), - ) + return partable end ############################################################################################ ### constructor for EnsembleParameterTable from graph ############################################################################################ -function EnsembleParameterTable(; graph, observed_vars, latent_vars, groups) +function EnsembleParameterTable( + graph::AbstractStenoGraph; + observed_vars, + latent_vars, + groups +) graph = unique(graph) partable = EnsembleParameterTable(nothing) @@ -130,12 +124,12 @@ function EnsembleParameterTable(; graph, observed_vars, latent_vars, groups) for (i, group) in enumerate(groups) push!( partable.tables, - Symbol(group) => ParameterTable(; - graph = graph, + Symbol(group) => ParameterTable( + graph; observed_vars = observed_vars, latent_vars = latent_vars, - g = i, - parname = Symbol(:g, i), + group = i, + param_prefix = Symbol(:g, i), ), ) end diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 818f9afdc..0a648f2dc 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -111,8 +111,8 @@ graph = @StenoGraph begin _(latent_vars) ⇔ _(latent_vars) end -partable = EnsembleParameterTable(; - graph = graph, +partable = EnsembleParameterTable( + graph; observed_vars = observed_vars, latent_vars = latent_vars, groups = [:Pasteur, :Grant_White], @@ -140,8 +140,8 @@ graph = @StenoGraph begin Symbol("1") → _(observed_vars) end -partable_miss = EnsembleParameterTable(; - graph = graph, +partable_miss = EnsembleParameterTable( + graph; observed_vars = observed_vars, latent_vars = latent_vars, groups = [:Pasteur, :Grant_White], diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 86b7e89bc..9085531b0 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -136,6 +136,7 @@ semoptimizer = SemOptimizerOptim @testset "RAMMatrices | constructor | Optim" begin include("constructor.jl") end + semoptimizer = SemOptimizerNLopt @testset "RAMMatrices | constructor | NLopt" begin include("constructor.jl") @@ -212,8 +213,7 @@ graph = @StenoGraph begin y8 ↔ y4 + y6 end -spec = - ParameterTable(latent_vars = latent_vars, observed_vars = observed_vars, graph = graph) +spec = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) sort!(spec) @@ -244,8 +244,7 @@ graph = @StenoGraph begin Symbol("1") → fixed(0) * ind60 end -spec_mean = - ParameterTable(latent_vars = latent_vars, observed_vars = observed_vars, graph = graph) +spec_mean = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) sort!(spec_mean) From 6308cdda46914428bcf2e67454bb560d4d79bec4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 26 May 2024 21:28:45 -0700 Subject: [PATCH 016/364] rename Base.sort() to sort_vars() because the ParTable contains rows and columns, it is not clear, what sort() actually sorts. Co-authored-by: Maximilian-Stefan-Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- src/StructuralEquationModels.jl | 2 ++ .../specification/EnsembleParameterTable.jl | 18 ++++-------- src/frontend/specification/ParameterTable.jl | 29 +++++++++++++++---- test/examples/multigroup/build_models.jl | 2 +- .../political_democracy.jl | 4 +-- test/unit_tests/sorting.jl | 4 +-- 6 files changed, 36 insertions(+), 23 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 3319f049b..5f8bd070f 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -154,6 +154,8 @@ export AbstractSem, Label, label, params_to_indices, + sort_vars!, + sort_vars, RAMMatrices, param_indices, fit_measures, diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 24a9a295a..4653022cc 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -67,23 +67,17 @@ end ### Additional Methods ############################################################################################ -# Sorting ---------------------------------------------------------------------------------- +# Variables Sorting ------------------------------------------------------------------------ -# Sorting ---------------------------------------------------------------------------------- - -function sort!(ensemble_partable::EnsembleParameterTable) - for partable in values(ensemble_partable.tables) - sort!(partable) +function sort_vars!(partables::EnsembleParameterTable) + for partable in values(partables.tables) + sort_vars!(partable) end - return ensemble_partable + return partables end -function sort(partable::EnsembleParameterTable) - new_partable = deepcopy(partable) - sort!(new_partable) - return new_partable -end +sort_vars(partables::EnsembleParameterTable) = sort_vars!(deepcopy(partables)) # add a row -------------------------------------------------------------------------------- diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 49ea4664b..d82b22f1d 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -112,7 +112,18 @@ end Base.showerror(io::IO, e::CyclicModelError) = print(io, e.msg) -function Base.sort!(partable::ParameterTable) +""" + sort_vars!(partable::ParameterTable) + sort_vars!(partables::EnsembleParameterTable) + +Sort variables in `partable` so that all independent variables are +before the dependent variables and store it in `partable.sorted_vars`. + +If the relations between the variables are acyclic, sorting will +make the resulting `A` matrix in the *RAM* model lower triangular +and allow faster calculations. +""" +function sort_vars!(partable::ParameterTable) vars = [ partable.latent_vars partable.observed_vars @@ -151,11 +162,17 @@ function Base.sort!(partable::ParameterTable) return partable end -function Base.sort(partable::ParameterTable) - new_partable = deepcopy(partable) - sort!(new_partable) - return new_partable -end +""" + sort_vars(partable::ParameterTable) + sort_vars(partables::EnsembleParameterTable) + +Sort variables in `partable` so that all independent variables are +before the dependent variables, and return a copy of `partable` +where the sorted variables are in `partable.sorted_vars`. + +See [sort_vars!](@ref) for in-place version. +""" +sort_vars(partable::ParameterTable) = sort_vars!(deepcopy(partable)) # add a row -------------------------------------------------------------------------------- diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 8913860c8..e26facc5e 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -49,7 +49,7 @@ end # ML estimation - sorted ############################################################################################ -partable_s = sort(partable) +partable_s = sort_vars(partable) specification_s = RAMMatrices(partable_s) diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 9085531b0..d7fbb8f2c 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -215,7 +215,7 @@ end spec = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) -sort!(spec) +sort_vars!(spec) partable = spec @@ -246,7 +246,7 @@ end spec_mean = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) -sort!(spec_mean) +sort_vars!(spec_mean) partable_mean = spec_mean diff --git a/test/unit_tests/sorting.jl b/test/unit_tests/sorting.jl index e573c6d22..5ca890c51 100644 --- a/test/unit_tests/sorting.jl +++ b/test/unit_tests/sorting.jl @@ -1,8 +1,8 @@ ############################################################################ -### test sorting +### test variables sorting ############################################################################ -sort!(partable) +sort_vars!(partable) model_ml_sorted = Sem(specification = partable, data = dat) From 92730f8c616c165b16a52261b8a3e017729128a9 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 21:51:05 -0700 Subject: [PATCH 017/364] don't import == --- src/frontend/specification/RAMMatrices.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 45bdfe57b..f3c8f11fc 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -49,9 +49,7 @@ struct RAMConstant value::Any end -import Base.== - -function ==(c1::RAMConstant, c2::RAMConstant) +function Base.:(==)(c1::RAMConstant, c2::RAMConstant) res = ((c1.matrix == c2.matrix) && (c1.index == c2.index) && (c1.value == c2.value)) return res end @@ -410,7 +408,7 @@ function push_partable_rows!(partable, position_names, par, i, A_ind, S_ind, M_i return nothing end -function ==(mat1::RAMMatrices, mat2::RAMMatrices) +function Base.:(==)(mat1::RAMMatrices, mat2::RAMMatrices) res = ( (mat1.A_ind == mat2.A_ind) && (mat1.S_ind == mat2.S_ind) && From 83e782823685b3defa4b3b01c167f8e565e47a67 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 21:51:51 -0700 Subject: [PATCH 018/364] don't import push!() --- src/frontend/specification/EnsembleParameterTable.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 4653022cc..1ce7a0d59 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -82,13 +82,11 @@ sort_vars(partables::EnsembleParameterTable) = sort_vars!(deepcopy(partables)) # add a row -------------------------------------------------------------------------------- # do we really need this? -import Base.push! - -function push!(partable::EnsembleParameterTable, d::AbstractDict, group) +function Base.push!(partable::EnsembleParameterTable, d::AbstractDict, group) push!(partable.tables[group], d) end -push!(partable::EnsembleParameterTable, d::Nothing, group) = nothing +Base.push!(partable::EnsembleParameterTable, d::Nothing, group) = nothing Base.getindex(partable::EnsembleParameterTable, group) = partable.tables[group] From a2eed98e66235429d49462dc5bea27f46ea41c9a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 22:42:28 -0700 Subject: [PATCH 019/364] don't import DataFrame --- src/StructuralEquationModels.jl | 1 - src/frontend/specification/ParameterTable.jl | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 5f8bd070f..be9eccc07 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -16,7 +16,6 @@ using LinearAlgebra, DelimitedFiles, DataFrames -import DataFrames: DataFrame export StenoGraphs, @StenoGraph, meld const SEM = StructuralEquationModels diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index d82b22f1d..c4ebefef3 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -46,12 +46,14 @@ function Dict(partable::ParameterTable) return partable.columns end -function DataFrame(partable::ParameterTable; columns = nothing) +function DataFrames.DataFrame( + partable::ParameterTable; + columns::Union{AbstractVector{Symbol}, Nothing} = nothing, +) if isnothing(columns) columns = keys(partable.columns) end - out = DataFrame([key => partable.columns[key] for key in columns]) - return DataFrame(out) + return DataFrame([col => partable.columns[col] for col in columns]) end ############################################################################################ From 458f1cfd397d60f32822532655c859c27383da7a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 14:50:28 -0700 Subject: [PATCH 020/364] remove no-op push!() --- src/frontend/specification/EnsembleParameterTable.jl | 2 -- src/frontend/specification/ParameterTable.jl | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 1ce7a0d59..672b9c25b 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -86,8 +86,6 @@ function Base.push!(partable::EnsembleParameterTable, d::AbstractDict, group) push!(partable.tables[group], d) end -Base.push!(partable::EnsembleParameterTable, d::Nothing, group) = nothing - Base.getindex(partable::EnsembleParameterTable, group) = partable.tables[group] ############################################################################################ diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index c4ebefef3..41f02da9e 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -184,8 +184,6 @@ function Base.push!(partable::ParameterTable, d::AbstractDict{Symbol}) end end -Base.push!(partable::ParameterTable, d::Nothing) = nothing - ############################################################################################ ### Update Partable from Fitted Model ############################################################################################ From ff2140a9442bc31d84851f86d93512b8b5f0b8e7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 27 May 2024 14:30:12 -0700 Subject: [PATCH 021/364] ParTable ctor: simplify rows code * use named tuples * reduce code duplication * use colnames vector instead of position_names Dict --- src/frontend/specification/ParameterTable.jl | 2 +- src/frontend/specification/RAMMatrices.jl | 168 +++++++++---------- 2 files changed, 77 insertions(+), 93 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 41f02da9e..f73f80b77 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -178,7 +178,7 @@ sort_vars(partable::ParameterTable) = sort_vars!(deepcopy(partable)) # add a row -------------------------------------------------------------------------------- -function Base.push!(partable::ParameterTable, d::AbstractDict{Symbol}) +function Base.push!(partable::ParameterTable, d::Union{AbstractDict{Symbol}, NamedTuple}) for (key, val) in pairs(d) push!(partable.columns[key], val) end diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index f3c8f11fc..ba5af9243 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -214,18 +214,16 @@ function ParameterTable(ram_matrices::RAMMatrices) latent_vars = colnames[setdiff(eachindex(colnames), ram_matrices.F_ind)], ) - position_names = Dict{Int64, Symbol}(1:length(colnames) .=> colnames) - # constants for c in ram_matrices.constants - push!(partable, get_partable_row(c, position_names)) + push!(partable, partable_row(c, colnames)) end # parameters for (i, par) in enumerate(ram_matrices.params) - push_partable_rows!( + append_partable_rows!( partable, - position_names, + colnames, par, i, ram_matrices.A_ind, @@ -296,112 +294,98 @@ function get_par_npar_indices(partable::EnsembleParameterTable) return params, n_par, par_positions end -function get_partable_row(c::RAMConstant, position_names) - # variable names - from = position_names[c.index[2]] - to = position_names[c.index[1]] - # parameter type - if c.matrix == :A - parameter_type = :→ - elseif c.matrix == :S - parameter_type = :↔ - elseif c.matrix == :M - parameter_type = :→ - end - free = false - value_fixed = c.value - start = 0.0 - estimate = 0.0 - - return Dict( - :from => from, - :parameter_type => parameter_type, - :to => to, - :free => free, - :value_fixed => value_fixed, - :start => start, - :estimate => estimate, - :param => :const, - ) -end - -function cartesian_is_known(index, known_indices) - known = false - for k_in in known_indices - if (index == k_in) | ((index[1] == k_in[2]) & (index[2] == k_in[1])) - known = true - end +function matrix_to_parameter_type(matrix::Symbol) + if matrix == :A + return :→ + elseif matrix == :S + return :↔ + elseif matrix == :M + return :→ + else + throw( + ArgumentError( + "Unsupported matrix $matrix, supported matrices are :A, :S and :M", + ), + ) end - return known end -cartesian_is_known(index, known_indices::Nothing) = false - -function get_partable_row(param, position_names, index, matrix, n_nod, known_indices) +partable_row(c::RAMConstant, varnames::AbstractVector{Symbol}) = ( + from = varnames[c.index[2]], + parameter_type = matrix_to_parameter_type(c.matrix), + to = varnames[c.index[1]], + free = false, + value_fixed = c.value, + start = 0.0, + estimate = 0.0, + param = :const, +) + +function partable_row( + par::Symbol, + varnames::AbstractVector{Symbol}, + index::Integer, + matrix::Symbol, + n_nod::Integer, +) # variable names if matrix == :M from = Symbol("1") - to = position_names[index] + to = varnames[index] else - index = linear2cartesian(index, (n_nod, n_nod)) - - if (matrix == :S) & (cartesian_is_known(index, known_indices)) - return nothing - elseif matrix == :S - push!(known_indices, index) - end + cart_index = linear2cartesian(index, (n_nod, n_nod)) - from = position_names[index[2]] - to = position_names[index[1]] - end - - # parameter type - if matrix == :A - parameter_type = :→ - elseif matrix == :S - parameter_type = :↔ - elseif matrix == :M - parameter_type = :→ + from = varnames[cart_index[2]] + to = varnames[cart_index[1]] end - free = true - value_fixed = 0.0 - start = 0.0 - estimate = 0.0 - - return Dict( - :from => from, - :parameter_type => parameter_type, - :to => to, - :free => free, - :value_fixed => value_fixed, - :start => start, - :estimate => estimate, - :param => param, + return ( + from = from, + parameter_type = matrix_to_parameter_type(matrix), + to = to, + free = true, + value_fixed = 0.0, + start = 0.0, + estimate = 0.0, + param = par, ) end -function push_partable_rows!(partable, position_names, par, i, A_ind, S_ind, M_ind, n_nod) - A_ind = A_ind[i] - S_ind = S_ind[i] - isnothing(M_ind) || (M_ind = M_ind[i]) - - for ind in A_ind - push!(partable, get_partable_row(par, position_names, ind, :A, n_nod, nothing)) +function append_partable_rows!( + partable::ParameterTable, + varnames::AbstractVector{Symbol}, + par::Symbol, + par_index::Integer, + A_ind, + S_ind, + M_ind, + n_nod::Integer, +) + for ind in A_ind[par_index] + push!(partable, partable_row(par, varnames, ind, :A, n_nod)) end - known_indices = Vector{CartesianIndex}() - for ind in S_ind - push!( - partable, - get_partable_row(par, position_names, ind, :S, n_nod, known_indices), - ) + visited_S_indices = Set{Int}() + for ind in S_ind[par_index] + if ind ∉ visited_S_indices + push!(partable, partable_row(par, varnames, ind, :S, n_nod)) + # mark index and its symmetric as visited + push!(visited_S_indices, ind) + cart_index = linear2cartesian(ind, (n_nod, n_nod)) + push!( + visited_S_indices, + cartesian2linear( + CartesianIndex(cart_index[2], cart_index[1]), + (n_nod, n_nod), + ), + ) + end end if !isnothing(M_ind) - for ind in M_ind - push!(partable, get_partable_row(par, position_names, ind, :M, n_nod, nothing)) + for ind in M_ind[par_index] + push!(partable, partable_row(par, varnames, ind, :M, n_nod)) end end From edc8443e39338c32af665a5130ec713c27346558 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 27 May 2024 00:52:46 -0700 Subject: [PATCH 022/364] ParTable: full support for Iterator iface --- src/frontend/specification/ParameterTable.jl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index f73f80b77..13d6448df 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -94,6 +94,14 @@ end ############################################################################################ # Iteration -------------------------------------------------------------------------------- +ParameterTableRow = @NamedTuple begin + from::Symbol + parameter_type::Symbol + to::Symbol + free::Bool + value_fixed::Any + param::Symbol +end Base.getindex(partable::ParameterTable, i::Integer) = ( partable.columns[:from][i], @@ -104,7 +112,12 @@ Base.getindex(partable::ParameterTable, i::Integer) = ( partable.columns[:param][i], ) -Base.length(partable::ParameterTable) = length(first(values(partable.columns))) +Base.length(partable::ParameterTable) = length(partable.columns[:param]) +Base.eachindex(partable::ParameterTable) = Base.OneTo(length(partable)) + +Base.eltype(::Type{<:ParameterTable}) = ParameterTableRow +Base.iterate(partable::ParameterTable, i::Integer = 1) = + i > length(partable) ? nothing : (partable[i], i + 1) # Sorting ---------------------------------------------------------------------------------- From 018b077a73806c39670b1d0b44468e8a3aa8356f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:23:26 -0800 Subject: [PATCH 023/364] RAMConstant: simplify * declare RAMConstant field types * refactor constants collection to avoid code duplication --- src/frontend/specification/RAMMatrices.jl | 110 ++++++++++------------ 1 file changed, 50 insertions(+), 60 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index ba5af9243..8c6314316 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -1,3 +1,48 @@ +############################################################################################ +### Constants +############################################################################################ + +struct RAMConstant + matrix::Symbol + index::Union{Int, CartesianIndex{2}} + value::Any +end + +function Base.:(==)(c1::RAMConstant, c2::RAMConstant) + res = ((c1.matrix == c2.matrix) && (c1.index == c2.index) && (c1.value == c2.value)) + return res +end + +function append_RAMConstants!( + constants::AbstractVector{RAMConstant}, + mtx_name::Symbol, + mtx::AbstractArray, +) + for (index, val) in pairs(mtx) + if isa(val, Number) && !iszero(val) + push!(constants, RAMConstant(mtx_name, index, val)) + end + end + return constants +end + +function set_RAMConstant!(A, S, M, rc::RAMConstant) + if rc.matrix == :A + A[rc.index] = rc.value + elseif rc.matrix == :S + S[rc.index] = rc.value + S[rc.index[2], rc.index[1]] = rc.value # symmetric + elseif rc.matrix == :M + M[rc.index] = rc.value + end +end + +function set_RAMConstants!(A, S, M, rc_vec::Vector{RAMConstant}) + for rc in rc_vec + set_RAMConstant!(A, S, M, rc) + end +end + ############################################################################################ ### Type ############################################################################################ @@ -13,7 +58,7 @@ struct RAMMatrices <: SemSpecification M_ind::Union{ArrayParamsMap, Nothing} params::Any colnames::Any - constants::Any + constants::Vector{RAMConstant} size_F::Any end @@ -26,7 +71,10 @@ function RAMMatrices(; A, S, F, M = nothing, params, colnames) S_indices = array_params_map(params, S) M_indices = !isnothing(M) ? array_params_map(params, M) : nothing F_indices = findall([any(isone.(col)) for col in eachcol(F)]) - constants = get_RAMConstants(A, S, M) + constants = Vector{RAMConstant}() + append_RAMConstants!(constants, :A, A) + append_RAMConstants!(constants, :S, S) + isnothing(M) || append_RAMConstants!(constants, :M, M) return RAMMatrices( A_indices, S_indices, @@ -39,64 +87,6 @@ function RAMMatrices(; A, S, F, M = nothing, params, colnames) ) end -############################################################################################ -### Constants -############################################################################################ - -struct RAMConstant - matrix::Any - index::Any - value::Any -end - -function Base.:(==)(c1::RAMConstant, c2::RAMConstant) - res = ((c1.matrix == c2.matrix) && (c1.index == c2.index) && (c1.value == c2.value)) - return res -end - -function get_RAMConstants(A, S, M) - constants = Vector{RAMConstant}() - - for index in CartesianIndices(A) - if (A[index] isa Number) && !iszero(A[index]) - push!(constants, RAMConstant(:A, index, A[index])) - end - end - - for index in CartesianIndices(S) - if (S[index] isa Number) && !iszero(S[index]) - push!(constants, RAMConstant(:S, index, S[index])) - end - end - - if !isnothing(M) - for index in CartesianIndices(M) - if (M[index] isa Number) && !iszero(M[index]) - push!(constants, RAMConstant(:M, index, M[index])) - end - end - end - - return constants -end - -function set_RAMConstant!(A, S, M, rc::RAMConstant) - if rc.matrix == :A - A[rc.index] = rc.value - elseif rc.matrix == :S - S[rc.index] = rc.value - S[rc.index[2], rc.index[1]] = rc.value - elseif rc.matrix == :M - M[rc.index] = rc.value - end -end - -function set_RAMConstants!(A, S, M, rc_vec::Vector{RAMConstant}) - for rc in rc_vec - set_RAMConstant!(A, S, M, rc) - end -end - ############################################################################################ ### get RAMMatrices from parameter table ############################################################################################ From 70e6199b8c03949c0d429d7127df9261eb0abb2b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:24:39 -0800 Subject: [PATCH 024/364] RAMMatrices: optimize F_indices init --- src/frontend/specification/RAMMatrices.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 8c6314316..718e319a1 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -70,7 +70,7 @@ function RAMMatrices(; A, S, F, M = nothing, params, colnames) A_indices = array_params_map(params, A) S_indices = array_params_map(params, S) M_indices = !isnothing(M) ? array_params_map(params, M) : nothing - F_indices = findall([any(isone.(col)) for col in eachcol(F)]) + F_indices = [i for (i, col) in zip(axes(F, 2), eachcol(F)) if any(isone, col)] constants = Vector{RAMConstant}() append_RAMConstants!(constants, :A, A) append_RAMConstants!(constants, :S, S) From 9bf094e1780b6f63a12f41cd33f248fd1d7a7573 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:26:01 -0800 Subject: [PATCH 025/364] RAMMatrices: declare types for all fields --- src/frontend/specification/RAMMatrices.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 718e319a1..6f674b6bd 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -56,10 +56,10 @@ struct RAMMatrices <: SemSpecification S_ind::ArrayParamsMap F_ind::Vector{Int} M_ind::Union{ArrayParamsMap, Nothing} - params::Any - colnames::Any + params::Vector{Symbol} + colnames::Union{Vector{Symbol}, Nothing} constants::Vector{RAMConstant} - size_F::Any + size_F::Tuple{Int, Int} end ############################################################################################ From 9a5842ee287064763050363dfce88efe88d49419 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 12:14:33 -0700 Subject: [PATCH 026/364] RAMMatrices: option to keep zero constants --- src/frontend/specification/RAMMatrices.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 6f674b6bd..e605d432a 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -16,10 +16,11 @@ end function append_RAMConstants!( constants::AbstractVector{RAMConstant}, mtx_name::Symbol, - mtx::AbstractArray, + mtx::AbstractArray; + skip_zeros::Bool = true, ) for (index, val) in pairs(mtx) - if isa(val, Number) && !iszero(val) + if isa(val, Number) && !(skip_zeros && iszero(val)) push!(constants, RAMConstant(mtx_name, index, val)) end end From 296d827d9d5adf76b20a001f6df0092946d4f6ee Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 22:49:34 -0700 Subject: [PATCH 027/364] nonunique() helper function --- src/additional_functions/helper.jl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index b96813dc3..3e614e57b 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -142,3 +142,18 @@ function elimination_matrix(n::Integer) end return L end + +# returns the vector of non-unique values in the order of appearance +# each non-unique values is reported once +function nonunique(values::AbstractVector) + value_counts = Dict{eltype(values), Int}() + res = similar(values, 0) + for v in values + n = get!(value_counts, v, 0) + if n == 1 # second encounter + push!(res, v) + end + value_counts[v] = n + 1 + end + return res +end From dec1f4d533df23821bf6f35bfe69db5bcf74acbd Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 3 May 2024 07:37:14 -0700 Subject: [PATCH 028/364] add check_vars() and check_params() --- src/StructuralEquationModels.jl | 1 + src/frontend/specification/checks.jl | 42 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 src/frontend/specification/checks.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index be9eccc07..8fe2ff90e 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -30,6 +30,7 @@ include("additional_functions/commutation_matrix.jl") # fitted objects include("frontend/fit/SemFit.jl") # specification of models +include("frontend/specification/checks.jl") include("frontend/specification/ParameterTable.jl") include("frontend/specification/EnsembleParameterTable.jl") include("frontend/specification/RAMMatrices.jl") diff --git a/src/frontend/specification/checks.jl b/src/frontend/specification/checks.jl new file mode 100644 index 000000000..5326e535f --- /dev/null +++ b/src/frontend/specification/checks.jl @@ -0,0 +1,42 @@ +# check if params vector correctly matches the parameter references (from the ParameterTable) +function check_params( + params::AbstractVector{Symbol}, + param_refs::Union{AbstractVector{Symbol}, Nothing}, +) + dup_params = nonunique(params) + isempty(dup_params) || + throw(ArgumentError("Duplicate parameters detected: $(join(dup_params, ", "))")) + any(==(:const), params) && + throw(ArgumentError("Parameters constain reserved :const name")) + + if !isnothing(param_refs) + # check if all references parameters are present + all_refs = Set(id for id in param_refs if id != :const) + undecl_params = setdiff(all_refs, params) + if !isempty(undecl_params) + throw( + ArgumentError( + "The following $(length(undecl_params)) parameters present in the table, but are not declared: " * + join(sort!(collect(undecl_params))), + ), + ) + end + end + + return nothing +end + +function check_vars(vars::AbstractVector{Symbol}, nvars::Union{Integer, Nothing}) + isnothing(nvars) || + length(vars) == nvars || + throw( + DimensionMismatch( + "variables length ($(length(vars))) does not match the number of columns in A matrix ($nvars)", + ), + ) + dup_vars = nonunique(vars) + isempty(dup_vars) || + throw(ArgumentError("Duplicate variables detected: $(join(dup_vars, ", "))")) + + return nothing +end From a7a17dfed691b98d3c03f90088e716bcf66070cc Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 3 May 2024 07:43:25 -0700 Subject: [PATCH 029/364] RAMMatrices ctor: dims and vars checks --- src/frontend/specification/RAMMatrices.jl | 36 ++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index e605d432a..551cf4c60 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -67,7 +67,41 @@ end ### Constructor ############################################################################################ -function RAMMatrices(; A, S, F, M = nothing, params, colnames) +function RAMMatrices(; + A::AbstractMatrix, + S::AbstractMatrix, + F::AbstractMatrix, + M::Union{AbstractVector, Nothing} = nothing, + params::AbstractVector{Symbol}, + colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, +) + ncols = size(A, 2) + isnothing(colnames) || check_vars(colnames, ncols) + + size(A, 1) == size(A, 2) || throw(DimensionMismatch("A must be a square matrix")) + size(S, 1) == size(S, 2) || throw(DimensionMismatch("S must be a square matrix")) + size(A, 2) == ncols || throw( + DimensionMismatch( + "A should have as many rows and columns as colnames length ($ncols), $(size(A)) found", + ), + ) + size(S, 2) == ncols || throw( + DimensionMismatch( + "S should have as many rows and columns as colnames length ($ncols), $(size(S)) found", + ), + ) + size(F, 2) == ncols || throw( + DimensionMismatch( + "F should have as many columns as colnames length ($ncols), $(size(F, 2)) found", + ), + ) + if !isnothing(M) + length(M) == ncols || throw( + DimensionMismatch( + "M should have as many elements as colnames length ($ncols), $(length(M)) found", + ), + ) + end A_indices = array_params_map(params, A) S_indices = array_params_map(params, S) M_indices = !isnothing(M) ? array_params_map(params, M) : nothing From 25cd574d6495cd0df5246b02c9e72d63766c1889 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 3 May 2024 07:43:44 -0700 Subject: [PATCH 030/364] RAMMatrices: cleanup params index * simplify parameters() function to return just a vector of params * RAMMatrices ctor: use check_params() --- .../specification/EnsembleParameterTable.jl | 9 +++ src/frontend/specification/ParameterTable.jl | 8 +- src/frontend/specification/RAMMatrices.jl | 76 ++++--------------- 3 files changed, 31 insertions(+), 62 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 672b9c25b..6d8961523 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -67,6 +67,15 @@ end ### Additional Methods ############################################################################################ +# get the vector of all parameters in the table +# the position of the parameter is based on its first appearance in the table (and the ensemble) +function params(partable::EnsembleParameterTable) + params = mapreduce(vcat, values(partable.tables)) do tbl + tbl.columns[:param] + end + return filter!(!=(:const), unique!(params)) # exclude constants +end + # Variables Sorting ------------------------------------------------------------------------ function sort_vars!(partables::EnsembleParameterTable) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 13d6448df..5efa2905f 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -119,8 +119,14 @@ Base.eltype(::Type{<:ParameterTable}) = ParameterTableRow Base.iterate(partable::ParameterTable, i::Integer = 1) = i > length(partable) ? nothing : (partable[i], i + 1) -# Sorting ---------------------------------------------------------------------------------- +# get the vector of all parameters in the table +# the position of the parameter is based on its first appearance in the table (and the ensemble) +params(partable::ParameterTable) = + filter!(!=(:const), unique(partable.columns[:param])) + + +# Sorting ---------------------------------------------------------------------------------- struct CyclicModelError <: Exception msg::AbstractString end diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 551cf4c60..023f46342 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -102,6 +102,9 @@ function RAMMatrices(; ), ) end + + check_params(params, nothing) + A_indices = array_params_map(params, A) S_indices = array_params_map(params, S) M_indices = !isnothing(M) ? array_params_map(params, M) : nothing @@ -126,13 +129,13 @@ end ### get RAMMatrices from parameter table ############################################################################################ -function RAMMatrices(partable::ParameterTable; par_id = nothing) - if isnothing(par_id) - params, n_par, par_positions = get_par_npar_indices(partable) - else - params, n_par, par_positions = - par_id[:params], par_id[:n_par], par_id[:par_positions] - end +function RAMMatrices( + partable::ParameterTable; + params::Union{AbstractVector{Symbol}, Nothing} = nothing, +) + params = copy(isnothing(params) ? SEM.params(partable) : params) + check_params(params, partable.columns[:param]) + params_index = Dict(param => i for (i, param) in enumerate(params)) n_observed = length(partable.observed_vars) n_latent = length(partable.latent_vars) @@ -156,20 +159,13 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) # fill Matrices # known_labels = Dict{Symbol, Int64}() - A_ind = Vector{Vector{Int64}}(undef, n_par) - for i in 1:length(A_ind) - A_ind[i] = Vector{Int64}() - end - S_ind = Vector{Vector{Int64}}(undef, n_par) - S_ind .= [Vector{Int64}()] - for i in 1:length(S_ind) - S_ind[i] = Vector{Int64}() - end + A_ind = [Vector{Int64}() for _ in 1:length(params)] + S_ind = [Vector{Int64}() for _ in 1:length(params)] # is there a meanstructure? M_ind = - any(==(Symbol("1")), partable.columns[:from]) ? [Vector{Int64}() for _ in 1:n_par] : - nothing + any(==(Symbol("1")), partable.columns[:from]) ? + [Vector{Int64}() for _ in 1:length(params)] : nothing # handle constants constants = Vector{RAMConstant}() @@ -197,7 +193,7 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) error("Unsupported parameter type: $(parameter_type)") end else - par_ind = par_positions[param] + par_ind = params_index[param] if (parameter_type == :→) && (from == Symbol("1")) push!(M_ind[par_ind], row_ind) elseif parameter_type == :→ @@ -264,25 +260,6 @@ end Base.convert(::Type{<:ParameterTable}, ram_matrices::RAMMatrices) = ParameterTable(ram_matrices) -############################################################################################ -### get RAMMatrices from EnsembleParameterTable -############################################################################################ - -function RAMMatrices(partable::EnsembleParameterTable) - ram_matrices = Dict{Symbol, RAMMatrices}() - - params, n_par, par_positions = get_par_npar_indices(partable) - par_id = - Dict(:params => params, :n_par => n_par, :par_positions => par_positions) - - for key in keys(partable.tables) - ram_mat = RAMMatrices(partable.tables[key]; par_id = par_id) - push!(ram_matrices, key => ram_mat) - end - - return ram_matrices -end - ############################################################################################ ### Pretty Printing ############################################################################################ @@ -296,29 +273,6 @@ end ### Additional Functions ############################################################################################ -function get_par_npar_indices(partable::ParameterTable) - params = unique(partable.columns[:param]) - filter!(x -> x != :const, params) - n_par = length(params) - par_positions = Dict(params .=> 1:n_par) - return params, n_par, par_positions -end - -function get_par_npar_indices(partable::EnsembleParameterTable) - params = Vector{Symbol}() - for key in keys(partable.tables) - append!(params, partable.tables[key].columns[:param]) - end - params = unique(params) - filter!(x -> x != :const, params) - - n_par = length(params) - - par_positions = Dict(params .=> 1:n_par) - - return params, n_par, par_positions -end - function matrix_to_parameter_type(matrix::Symbol) if matrix == :A return :→ From 98b74d9cb7f1d38ac3a54921e4917338005ab9e3 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 17:19:57 -0700 Subject: [PATCH 031/364] include RAMMatrices before EnsParTable --- src/StructuralEquationModels.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 8fe2ff90e..a8598ebc7 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -32,8 +32,8 @@ include("frontend/fit/SemFit.jl") # specification of models include("frontend/specification/checks.jl") include("frontend/specification/ParameterTable.jl") -include("frontend/specification/EnsembleParameterTable.jl") include("frontend/specification/RAMMatrices.jl") +include("frontend/specification/EnsembleParameterTable.jl") include("frontend/specification/StenoGraphs.jl") include("frontend/fit/summary.jl") # pretty printing From 5cdbe7c8fd0d1ca9d711713a022af040832c50ad Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:35:14 -0700 Subject: [PATCH 032/364] fix EnsParTable to Dict{RAMMatrices} convert * this method is not RAMMatrices ctor, it is Dict{K, RAMMatrices} convert * use comprehension to construct dict --- .../specification/EnsembleParameterTable.jl | 13 +++++++++++++ src/frontend/specification/ParameterTable.jl | 4 +--- test/examples/multigroup/build_models.jl | 2 +- test/examples/multigroup/multigroup.jl | 4 ++-- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 6d8961523..8192ab6f8 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -24,6 +24,19 @@ function Base.convert(::Type{Dict}, partable::EnsembleParameterTable) return convert(Dict, partable.tables) end +function Base.convert( + ::Type{Dict{K, RAMMatrices}}, + partables::EnsembleParameterTable; + params::Union{AbstractVector{Symbol}, Nothing} = nothing, +) where {K} + isnothing(params) || (params = SEM.params(partables)) + + return Dict{K, RAMMatrices}( + K(key) => RAMMatrices(partable; params = params) for + (key, partable) in pairs(partables.tables) + ) +end + #= function DataFrame( partable::ParameterTable; columns = nothing) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 5efa2905f..5e4fe157f 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -40,9 +40,7 @@ end ### Convert to other types ############################################################################################ -import Base.Dict - -function Dict(partable::ParameterTable) +function Base.convert(::Type{Dict}, partable::ParameterTable) return partable.columns end diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index e26facc5e..23d429796 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -51,7 +51,7 @@ end partable_s = sort_vars(partable) -specification_s = RAMMatrices(partable_s) +specification_s = convert(Dict{Symbol, RAMMatrices}, partable_s) specification_g1_s = specification_s[:Pasteur] specification_g2_s = specification_s[:Grant_White] diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 0a648f2dc..552a65cfb 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -118,7 +118,7 @@ partable = EnsembleParameterTable( groups = [:Pasteur, :Grant_White], ) -specification = RAMMatrices(partable) +specification = convert(Dict{Symbol, RAMMatrices}, partable) specification_g1 = specification[:Pasteur] specification_g2 = specification[:Grant_White] @@ -147,7 +147,7 @@ partable_miss = EnsembleParameterTable( groups = [:Pasteur, :Grant_White], ) -specification_miss = RAMMatrices(partable_miss) +specification_miss = convert(Dict{Symbol, RAMMatrices}, partable_miss) specification_miss_g1 = specification_miss[:Pasteur] specification_miss_g2 = specification_miss[:Grant_White] From e71573b8ac67a3a7abbf37510201dcd049670a45 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 16:40:51 -0700 Subject: [PATCH 033/364] DataFrame(EnsParTable) --- .../specification/EnsembleParameterTable.jl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 8192ab6f8..4430651e9 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -37,13 +37,16 @@ function Base.convert( ) end -#= function DataFrame( - partable::ParameterTable; - columns = nothing) - if isnothing(columns) columns = keys(partable.columns) end - out = DataFrame([key => partable.columns[key] for key in columns]) - return DataFrame(out) -end =# +function DataFrames.DataFrame( + partables::EnsembleParameterTable; + columns::Union{AbstractVector{Symbol}, Nothing} = nothing, +) + mapreduce(vcat, pairs(partables.tables)) do (key, partable) + df = DataFrame(partable; columns = columns) + df[!, :group] .= key + return df + end +end ############################################################################################ ### get parameter table from RAMMatrices From d5ac0d19c49aa7aa60510121d5a581b28f41f569 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 16:45:38 -0700 Subject: [PATCH 034/364] params() API method * remove n_par.jl * remove identifier.jl --- src/StructuralEquationModels.jl | 6 +- src/additional_functions/identifier.jl | 59 ------------------- src/frontend/fit/SemFit.jl | 3 + src/frontend/fit/fitmeasures/n_par.jl | 20 ------- .../specification/EnsembleParameterTable.jl | 6 +- src/frontend/specification/ParameterTable.jl | 11 ++-- src/frontend/specification/RAMMatrices.jl | 2 + src/imply/RAM/generic.jl | 11 ---- src/imply/RAM/symbolic.jl | 10 +--- src/imply/empty.jl | 14 +---- src/loss/regularization/ridge.jl | 3 +- src/types.jl | 36 +++++++++-- test/unit_tests/specification.jl | 15 ++--- 13 files changed, 56 insertions(+), 140 deletions(-) delete mode 100644 src/additional_functions/identifier.jl delete mode 100644 src/frontend/fit/fitmeasures/n_par.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a8598ebc7..109a913e7 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -74,15 +74,12 @@ include("additional_functions/start_val/start_partable.jl") include("additional_functions/start_val/start_simple.jl") include("additional_functions/artifacts.jl") include("additional_functions/simulation.jl") -# identifier -include("additional_functions/identifier.jl") # fit measures include("frontend/fit/fitmeasures/AIC.jl") include("frontend/fit/fitmeasures/BIC.jl") include("frontend/fit/fitmeasures/chi2.jl") include("frontend/fit/fitmeasures/df.jl") include("frontend/fit/fitmeasures/minus2ll.jl") -include("frontend/fit/fitmeasures/n_par.jl") include("frontend/fit/fitmeasures/n_obs.jl") include("frontend/fit/fitmeasures/p.jl") include("frontend/fit/fitmeasures/RMSEA.jl") @@ -153,11 +150,10 @@ export AbstractSem, start, Label, label, - params_to_indices, sort_vars!, sort_vars, RAMMatrices, - param_indices, + params, fit_measures, AIC, BIC, diff --git a/src/additional_functions/identifier.jl b/src/additional_functions/identifier.jl deleted file mode 100644 index 1b10357d6..000000000 --- a/src/additional_functions/identifier.jl +++ /dev/null @@ -1,59 +0,0 @@ -############################################################################################ -# get a map from parameters to their indices -############################################################################################ - -param_indices(sem_fit::SemFit) = param_indices(sem_fit.model) -param_indices(model::AbstractSemSingle) = param_indices(model.imply) -param_indices(model::SemEnsemble) = model.param_indices - -############################################################################################ -# construct a map from parameters to indices -############################################################################################ - -param_indices(ram_matrices::RAMMatrices) = - Dict(par => i for (i, par) in enumerate(ram_matrices.params)) -function param_indices(partable::ParameterTable) - _, _, param_indices = get_par_npar_indices(partable) - return param_indices -end - -############################################################################################ -# get indices of a Vector of parameter labels -############################################################################################ - -params_to_indices(params, param_indices::Dict{Symbol, Int}) = - [param_indices[par] for par in params] - -params_to_indices( - params, - obj::Union{SemFit, AbstractSemSingle, SemEnsemble, SemImply}, -) = params_to_indices(params, params(obj)) - -function params_to_indices(params, obj::Union{ParameterTable, RAMMatrices}) - @warn "You are trying to find parameter indices from a ParameterTable or RAMMatrices object. \n - If your model contains user-defined types, this may lead to wrong results. \n - To be on the safe side, try to reference parameters by labels or query the indices from - the constructed model (`params_to_indices(params, model)`)." maxlog = 1 - return params_to_indices(params, params(obj)) -end - -############################################################################################ -# documentation -############################################################################################ -""" - params_to_indices(params, model) - -Returns the indices of `params`. - -# Arguments -- `params::Vector{Symbol}`: parameter labels -- `model`: either a SEM or a fitted SEM - -# Examples -```julia -parameter_indices = params_to_indices([:λ₁, λ₂], my_fitted_sem) - -values = solution(my_fitted_sem)[parameter_indices] -``` -""" -function params_to_indices end diff --git a/src/frontend/fit/SemFit.jl b/src/frontend/fit/SemFit.jl index 97cd9c5a6..19a6d4441 100644 --- a/src/frontend/fit/SemFit.jl +++ b/src/frontend/fit/SemFit.jl @@ -46,6 +46,9 @@ end # additional methods ############################################################################################ +params(fit::SemFit) = params(fit.model) +n_par(fit::SemFit) = n_par(fit.model) + # access fields minimum(sem_fit::SemFit) = sem_fit.minimum solution(sem_fit::SemFit) = sem_fit.solution diff --git a/src/frontend/fit/fitmeasures/n_par.jl b/src/frontend/fit/fitmeasures/n_par.jl deleted file mode 100644 index c8553572b..000000000 --- a/src/frontend/fit/fitmeasures/n_par.jl +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################################ -### get number of parameters -############################################################################################ -""" - n_par(sem_fit::SemFit) - n_par(model::AbstractSemSingle) - n_par(model::SemEnsemble) - n_par(param_indices::Dict) - -Return the number of parameters. -""" -function n_par end - -n_par(fit::SemFit) = n_par(fit.model) - -n_par(model::AbstractSemSingle) = n_par(model.imply) - -n_par(model::SemEnsemble) = n_par(model.param_indices) - -n_par(param_indices::Dict) = length(param_indices) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 4430651e9..37d7ef15c 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -120,12 +120,12 @@ Base.getindex(partable::EnsembleParameterTable, group) = partable.tables[group] # update generic --------------------------------------------------------------------------- function update_partable!( partable::EnsembleParameterTable, - param_indices::AbstractDict, - vec, + params::AbstractVector{Symbol}, + values::AbstractVector, column, ) for k in keys(partable.tables) - update_partable!(partable.tables[k], param_indices, vec, column) + update_partable!(partable.tables[k], params, values, column) end return partable end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 5e4fe157f..60cc93ec9 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -209,14 +209,15 @@ end function update_partable!( partable::ParameterTable, - param_indices::AbstractDict, + params::AbstractVector{Symbol}, values::AbstractVector, column, ) new_col = Vector{eltype(vec)}(undef, length(partable)) + params_index = Dict(param => i for (i, param) in enumerate(params)) for (i, param) in enumerate(partable.columns[:param]) if !(param == :const) - new_col[i] = values[param_indices[param]] + new_col[i] = values[params_index[param]] elseif param == :const new_col[i] = zero(eltype(values)) end @@ -233,8 +234,8 @@ Write `vec` to `column` of `partable`. # Arguments - `vec::Vector`: has to be in the same order as the `model` parameters """ -update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, vec, column) = - update_partable!(partable, param_indices(sem_fit), vec, column) +update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, values, column) = + update_partable!(partable, params(sem_fit), values, column) # update estimates ------------------------------------------------------------------------- """ @@ -271,7 +272,7 @@ function update_start!( if !(start_val isa Vector) start_val = start_val(model; kwargs...) end - return update_partable!(partable, param_indices(model), start_val, :start) + return update_partable!(partable, params(model), start_val, :start) end # update partable standard errors ---------------------------------------------------------- diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 023f46342..5d3abe295 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -63,6 +63,8 @@ struct RAMMatrices <: SemSpecification size_F::Tuple{Int, Int} end +n_par(ram::RAMMatrices) = length(ram.A_ind) + ############################################################################################ ### Constructor ############################################################################################ diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index d43c8378e..0988d8e99 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -72,7 +72,6 @@ mutable struct RAM{ A4, A5, A6, - V, V2, I1, I2, @@ -85,7 +84,6 @@ mutable struct RAM{ S2, S3, B, - D, } <: SemImply Σ::A1 A::A2 @@ -94,7 +92,6 @@ mutable struct RAM{ μ::A5 M::A6 - n_par::V ram_matrices::V2 has_meanstructure::B @@ -110,8 +107,6 @@ mutable struct RAM{ ∇A::S1 ∇S::S2 ∇M::S3 - - param_indices::D end using StructuralEquationModels @@ -128,7 +123,6 @@ function RAM(; kwargs..., ) ram_matrices = convert(RAMMatrices, specification) - param_indices = SEM.param_indices(ram_matrices) # get dimensions of the model n_par = length(ram_matrices.params) @@ -184,7 +178,6 @@ function RAM(; F, μ, M_pre, - n_par, ram_matrices, has_meanstructure, A_indices, @@ -197,7 +190,6 @@ function RAM(; ∇A, ∇S, ∇M, - param_indices, ) end @@ -280,9 +272,6 @@ objective_gradient_hessian!(imply::RAM, par, model::AbstractSemSingle, has_means ### Recommended methods ############################################################################################ -param_indices(imply::RAM) = imply.param_indices -n_par(imply::RAM) = imply.n_par - function update_observed(imply::RAM, observed::SemObserved; kwargs...) if n_man(observed) == size(imply.Σ, 1) return imply diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 0fe9c29bb..db5997d2b 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -62,7 +62,7 @@ and for models with a meanstructure, the model implied means are computed as \mu = F(I-A)^{-1}M ``` """ -struct RAMSymbolic{F1, F2, F3, A1, A2, A3, S1, S2, S3, V, V2, F4, A4, F5, A5, D1, B} <: +struct RAMSymbolic{F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5, B} <: SemImplySymbolic Σ_function::F1 ∇Σ_function::F2 @@ -73,13 +73,11 @@ struct RAMSymbolic{F1, F2, F3, A1, A2, A3, S1, S2, S3, V, V2, F4, A4, F5, A5, D1 Σ_symbolic::S1 ∇Σ_symbolic::S2 ∇²Σ_symbolic::S3 - n_par::V ram_matrices::V2 μ_function::F4 μ::A4 ∇μ_function::F5 ∇μ::A5 - param_indices::D1 has_meanstructure::B end @@ -98,7 +96,6 @@ function RAMSymbolic(; kwargs..., ) ram_matrices = convert(RAMMatrices, specification) - param_indices = SEM.param_indices(ram_matrices) n_par = length(ram_matrices.params) n_var, n_nod = ram_matrices.size_F @@ -195,13 +192,11 @@ function RAMSymbolic(; Σ_symbolic, ∇Σ_symbolic, ∇²Σ_symbolic, - n_par, ram_matrices, μ_function, μ, ∇μ_function, ∇μ, - param_indices, has_meanstructure, ) end @@ -240,9 +235,6 @@ objective_gradient_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, p ### Recommended methods ############################################################################################ -param_indices(imply::RAMSymbolic) = imply.param_indices -n_par(imply::RAMSymbolic) = imply.n_par - function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) if n_man(observed) == size(imply.Σ, 1) return imply diff --git a/src/imply/empty.jl b/src/imply/empty.jl index 1d0ea69ff..56297ea06 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -25,9 +25,8 @@ model per group and an additional model with `ImplyEmpty` and `SemRidge` for the ## Implementation Subtype of `SemImply`. """ -struct ImplyEmpty{V, V2} <: SemImply - param_indices::V2 - n_par::V +struct ImplyEmpty{V2} <: SemImply + ram_matrices::V2 end ############################################################################################ @@ -35,11 +34,7 @@ end ############################################################################################ function ImplyEmpty(; specification, kwargs...) - ram_matrices = RAMMatrices(specification) - - n_par = length(ram_matrices.params) - - return ImplyEmpty(param_indices(ram_matrices), n_par) + return ImplyEmpty(convert(RAMMatrices, specification)) end ############################################################################################ @@ -54,7 +49,4 @@ hessian!(imply::ImplyEmpty, par, model) = nothing ### Recommended methods ############################################################################################ -param_indices(imply::ImplyEmpty) = imply.param_indices -n_par(imply::ImplyEmpty) = imply.n_par - update_observed(imply::ImplyEmpty, observed::SemObserved; kwargs...) = imply diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index 0d9d10b4b..2d098d550 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -58,7 +58,8 @@ function SemRidge(; ), ) else - which_ridge = params_to_indices(which_ridge, imply) + par2ind = Dict(par => ind for (ind, par) in enumerate(params(imply))) + which_ridge = getindex.(Ref(par2ind), which_ridge) end end which = [CartesianIndex(x) for x in which_ridge] diff --git a/src/types.jl b/src/types.jl index f026b2cb0..70e0b4fbf 100644 --- a/src/types.jl +++ b/src/types.jl @@ -13,6 +13,23 @@ abstract type AbstractSemCollection <: AbstractSem end "Supertype for all loss functions of SEMs. If you want to implement a custom loss function, it should be a subtype of `SemLossFunction`." abstract type SemLossFunction end +""" + params(semobj) + +Return the vector of SEM model parameters. +""" +params(model::AbstractSem) = model.params + +""" + n_par(semobj) + +Return the number of SEM model parameters. +""" +n_par(model::AbstractSem) = length(params(model)) + +params(model::AbstractSemSingle) = params(model.imply) +n_par(model::AbstractSemSingle) = n_par(model.imply) + """ SemLoss(args...; loss_weights = nothing, ...) @@ -75,6 +92,9 @@ If you would like to implement a different notation, e.g. LISREL, you should imp """ abstract type SemImply end +params(imply::SemImply) = params(imply.ram_matrices) +n_par(imply::SemImply) = n_par(imply.ram_matrices) + "Subtype of SemImply for all objects that can serve as the imply field of a SEM and use some form of symbolic precomputation." abstract type SemImplySymbolic <: SemImply end @@ -153,14 +173,14 @@ Returns a SemEnsemble with fields - `sems::Tuple`: `AbstractSem`s. - `weights::Vector`: Weights for each model. - `optimizer::SemOptimizer`: Connects the model to the optimizer. See also [`SemOptimizer`](@ref). -- `param_indices::Dict`: Stores parameter labels and their position. +- `params::Vector`: Stores parameter labels and their position. """ struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, D, I} <: AbstractSemCollection n::N sems::T weights::V optimizer::D - param_indices::I + params::I end function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing, kwargs...) @@ -175,9 +195,9 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing end # check parameters equality - par_indices = param_indices(models[1]) + params = SEM.params(models[1]) for model in models - if par_indices != param_indices(model) + if params != SEM.params(model) throw(ErrorException("The parameters of your models do not match. \n Maybe you tried to specify models of an ensemble via ParameterTables. \n In that case, you may use RAMMatrices instead.")) @@ -189,9 +209,12 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing optimizer = optimizer(; kwargs...) end - return SemEnsemble(n, models, weights, optimizer, par_indices) + return SemEnsemble(n, models, weights, optimizer, params) end +params(ensemble::SemEnsemble) = ensemble.params +n_par(ensemble::SemEnsemble) = length(ensemble.params) + """ n_models(ensemble::SemEnsemble) -> Integer @@ -253,4 +276,7 @@ Base type for all SEM specifications. """ abstract type SemSpecification end +params(spec::SemSpecification) = spec.params +n_par(spec::SemSpecification) = length(params(spec)) + abstract type AbstractParameterTable <: SemSpecification end diff --git a/test/unit_tests/specification.jl b/test/unit_tests/specification.jl index 42ad5e431..c081dc0f9 100644 --- a/test/unit_tests/specification.jl +++ b/test/unit_tests/specification.jl @@ -3,19 +3,12 @@ @test ram_matrices == RAMMatrices(partable) end -@test params_to_indices([:x2, :x10, :x28], model_ml) == [2, 10, 28] - -@testset "params_to_indices" begin - pars = [:θ_1, :θ_7, :θ_21] - @test params_to_indices(pars, model_ml) == params_to_indices(pars, partable) - @test params_to_indices(pars, model_ml) == - params_to_indices(pars, RAMMatrices(partable)) +@testset "params()" begin + @test params(model_ml)[2, 10, 28] == [:x2, :x10, :x28] + @test params(model_ml) == params(partable) + @test params(model_ml) == params(RAMMatrices(partable)) end -# from docstrings: -param_indices = params_to_indices([:λ₁, λ₂], my_fitted_sem) -values = solution(my_fitted_sem)[param_indices] - graph = @StenoGraph begin # measurement model visual → fixed(1.0, 1.0) * x1 + fixed(0.5, 0.5) * x2 + fixed(0.6, 0.8) * x3 From 957aa8c5b1732cb31e648c94f4a3c38062744279 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 9 May 2024 09:41:38 -0700 Subject: [PATCH 035/364] EnsParTable ctor: enforce same params in tables * fix EnsParTable container to Dict{Symbol, ParTable} * don't use keywords for main params as it complicates dispatch Co-authored-by: Maximilian-Stefan-Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- .../specification/EnsembleParameterTable.jl | 62 ++++++++++--------- src/frontend/specification/StenoGraphs.jl | 28 ++++----- test/examples/multigroup/multigroup.jl | 4 +- 3 files changed, 46 insertions(+), 48 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 37d7ef15c..0161cc81c 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -2,8 +2,9 @@ ### Types ############################################################################################ -mutable struct EnsembleParameterTable{C} <: AbstractParameterTable - tables::C +struct EnsembleParameterTable <: AbstractParameterTable + tables::Dict{Symbol, ParameterTable} + params::Vector{Symbol} end ############################################################################################ @@ -11,9 +12,37 @@ end ############################################################################################ # constuct an empty table -function EnsembleParameterTable(::Nothing) - tables = Dict{Symbol, ParameterTable}() - return EnsembleParameterTable(tables) +EnsembleParameterTable(::Nothing; params::Union{Nothing, Vector{Symbol}} = nothing) = + EnsembleParameterTable( + Dict{Symbol, ParameterTable}(), + isnothing(params) ? Symbol[] : copy(params), + ) + +# dictionary of SEM specifications +function EnsembleParameterTable( + spec_ensemble::AbstractDict{K, V}; + params::Union{Nothing, Vector{Symbol}} = nothing, +) where {K, V <: SemSpecification} + partables = Dict{Symbol, ParameterTable}( + Symbol(group) => convert(ParameterTable, spec; params = params) for + (group, spec) in pairs(spec_ensemble) + ) + + if isnothing(params) + # collect all SEM parameters in ensemble if not specified + # and apply the set to all partables + params = + unique(mapreduce(SEM.params, vcat, values(partables), init = Vector{Symbol}())) + for partable in values(partables) + if partable.params != params + copyto!(resize!(partable.params, length(params)), params) + #throw(ArgumentError("The parameter sets of the SEM specifications in the ensemble do not match.")) + end + end + else + params = copy(params) + end + return EnsembleParameterTable(partables, params) end ############################################################################################ @@ -48,20 +77,6 @@ function DataFrames.DataFrame( end end -############################################################################################ -### get parameter table from RAMMatrices -############################################################################################ - -function EnsembleParameterTable(args...; groups) - partable = EnsembleParameterTable(nothing) - - for (group, ram_matrices) in zip(groups, args) - push!(partable.tables, group => ParameterTable(ram_matrices)) - end - - return partable -end - ############################################################################################ ### Pretty Printing ############################################################################################ @@ -83,15 +98,6 @@ end ### Additional Methods ############################################################################################ -# get the vector of all parameters in the table -# the position of the parameter is based on its first appearance in the table (and the ensemble) -function params(partable::EnsembleParameterTable) - params = mapreduce(vcat, values(partable.tables)) do tbl - tbl.columns[:param] - end - return filter!(!=(:const), unique!(params)) # exclude constants -end - # Variables Sorting ------------------------------------------------------------------------ function sort_vars!(partables::EnsembleParameterTable) diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 1d3332cb9..424878f2c 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -115,24 +115,18 @@ function EnsembleParameterTable( graph::AbstractStenoGraph; observed_vars, latent_vars, - groups + groups, ) graph = unique(graph) - partable = EnsembleParameterTable(nothing) - - for (i, group) in enumerate(groups) - push!( - partable.tables, - Symbol(group) => ParameterTable( - graph; - observed_vars = observed_vars, - latent_vars = latent_vars, - group = i, - param_prefix = Symbol(:g, i), - ), - ) - end - - return partable + partables = Dict( + group => ParameterTable( + graph; + observed_vars = observed_vars, + latent_vars = latent_vars, + group = i, + param_prefix = Symbol(:g, group), + ) for (i, group) in enumerate(groups) + ) + return EnsembleParameterTable(partables) end diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 552a65cfb..e428eba1d 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -69,9 +69,7 @@ specification_g2 = RAMMatrices(; ) partable = EnsembleParameterTable( - specification_g1, - specification_g2; - groups = [:Pasteur, :Grant_White], + Dict(:Pasteur => specification_g1, :Grant_White => specification_g2), ) specification_miss_g1 = nothing From 7def9bfb431543edc8636f0cd62209b0199fd4b1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 15:19:09 -0700 Subject: [PATCH 036/364] formatting fixes --- src/frontend/fit/summary.jl | 9 +++----- src/frontend/specification/ParameterTable.jl | 18 +++++----------- src/frontend/specification/RAMMatrices.jl | 3 +-- src/imply/RAM/generic.jl | 22 ++------------------ src/loss/ML/FIML.jl | 3 +-- src/objective_gradient_hessian.jl | 7 +------ test/examples/helper.jl | 6 ++---- 7 files changed, 15 insertions(+), 53 deletions(-) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 4d6bc6181..85c09590b 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -138,8 +138,7 @@ function sem_summary( ), ) - sorted_columns = - [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] regression_columns = sort_partially(sorted_columns, columns) regression_array = reduce( @@ -166,8 +165,7 @@ function sem_summary( (partable.columns[:to] .== partable.columns[:from]), ) - sorted_columns = - [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce( @@ -194,8 +192,7 @@ function sem_summary( (partable.columns[:to] .!= partable.columns[:from]), ) - sorted_columns = - [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce( diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 60cc93ec9..3b6ff7ed6 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -59,17 +59,8 @@ end ############################################################################################ function Base.show(io::IO, partable::ParameterTable) - relevant_columns = [ - :from, - :parameter_type, - :to, - :free, - :value_fixed, - :start, - :estimate, - :se, - :param, - ] + relevant_columns = + [:from, :parameter_type, :to, :free, :value_fixed, :start, :estimate, :se, :param] shown_columns = filter!( col -> haskey(partable.columns, col) && length(partable.columns[col]) > 0, relevant_columns, @@ -125,6 +116,7 @@ params(partable::ParameterTable) = # Sorting ---------------------------------------------------------------------------------- + struct CyclicModelError <: Exception msg::AbstractString end @@ -149,8 +141,8 @@ function sort_vars!(partable::ParameterTable) ] is_regression = [ - (partype == :→) && (from != Symbol("1")) for - (partype, from) in zip(partable.columns[:parameter_type], partable.columns[:from]) + (partype == :→) && (from != Symbol("1")) for (partype, from) in + zip(partable.columns[:parameter_type], partable.columns[:from]) ] to = partable.columns[:to][is_regression] diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 5d3abe295..138bc1c9b 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -146,8 +146,7 @@ function RAMMatrices( # F indices F_ind = length(partable.sorted_vars) != 0 ? - findall(∈(Set(partable.observed_vars)), partable.sorted_vars) : - 1:n_observed + findall(∈(Set(partable.observed_vars)), partable.sorted_vars) : 1:n_observed # indices of the colnames colnames = diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 0988d8e99..07aec6648 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -65,26 +65,8 @@ Additional interfaces Only available in gradient! calls: - `I_A⁻¹(::RAM)` -> ``(I-A)^{-1}`` """ -mutable struct RAM{ - A1, - A2, - A3, - A4, - A5, - A6, - V2, - I1, - I2, - I3, - M1, - M2, - M3, - M4, - S1, - S2, - S3, - B, -} <: SemImply +mutable struct RAM{A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S2, S3, B} <: + SemImply Σ::A1 A::A2 S::A3 diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 9f5103f1f..d4870ac1b 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -252,5 +252,4 @@ end get_n_nodes(specification::RAMMatrices) = specification.size_F[2] get_n_nodes(specification::ParameterTable) = - length(specification.observed_vars) + - length(specification.latent_vars) + length(specification.observed_vars) + length(specification.latent_vars) diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index 61b78a54f..2debbcd40 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -38,12 +38,7 @@ function gradient_hessian!(gradient, hessian, model::AbstractSemSingle, params) gradient_hessian!(gradient, hessian, loss(model), params, model) end -function objective_gradient_hessian!( - gradient, - hessian, - model::AbstractSemSingle, - params, -) +function objective_gradient_hessian!(gradient, hessian, model::AbstractSemSingle, params) fill!(gradient, zero(eltype(gradient))) fill!(hessian, zero(eltype(hessian))) objective_gradient_hessian!(imply(model), params, model) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index e93a1437d..0f10ce838 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,6 +1,5 @@ function test_gradient(model, params; rtol = 1e-10, atol = 0) - true_grad = - FiniteDiff.finite_difference_gradient(Base.Fix1(objective!, model), params) + true_grad = FiniteDiff.finite_difference_gradient(Base.Fix1(objective!, model), params) gradient = similar(params) # F and G @@ -250,8 +249,7 @@ function compare_estimates( if type == :↔ type = "~~" elseif type == :→ - if (from ∈ partable.latent_vars) && - (to ∈ partable.observed_vars) + if (from ∈ partable.latent_vars) && (to ∈ partable.observed_vars) type = "=~" else type = "~" From becc6b52a4ed34262bd9a9b3971f35173a0da963 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 15:15:09 -0700 Subject: [PATCH 037/364] ParTable ctor: allow providing columns data --- src/frontend/specification/ParameterTable.jl | 30 +++++++++++--------- src/frontend/specification/StenoGraphs.jl | 18 ++++++------ 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 3b6ff7ed6..38f84917b 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -13,23 +13,25 @@ end ### Constructors ############################################################################################ -# constuct an empty table -function ParameterTable(; +# construct a dictionary with the default partable columns +# optionally pre-allocate data for nrows +empty_partable_columns(nrows::Integer = 0) = Dict{Symbol, Vector}( + :from => fill(Symbol(), nrows), + :parameter_type => fill(Symbol(), nrows), + :to => fill(Symbol(), nrows), + :free => fill(true, nrows), + :value_fixed => fill(NaN, nrows), + :start => fill(NaN, nrows), + :estimate => fill(NaN, nrows), + :param => fill(Symbol(), nrows), +) + +# construct using the provided columns data or create and empty table +function ParameterTable( + columns::Dict{Symbol, Vector} = empty_partable_columns(); observed_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, latent_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, ) - columns = Dict{Symbol, Any}( - :from => Vector{Symbol}(), - :parameter_type => Vector{Symbol}(), - :to => Vector{Symbol}(), - :free => Vector{Bool}(), - :value_fixed => Vector{Float64}(), - :start => Vector{Float64}(), - :estimate => Vector{Float64}(), - :param => Vector{Symbol}(), - :start => Vector{Float64}(), - ) - return ParameterTable(columns, !isnothing(observed_vars) ? copy(observed_vars) : Vector{Symbol}(), !isnothing(latent_vars) ? copy(latent_vars) : Vector{Symbol}(), diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 424878f2c..42edd6a13 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -41,14 +41,14 @@ function ParameterTable( graph = unique(graph) n = length(graph) - partable = ParameterTable(latent_vars = latent_vars, observed_vars = observed_vars) - from = resize!(partable.columns[:from], n) - parameter_type = resize!(partable.columns[:parameter_type], n) - to = resize!(partable.columns[:to], n) - free = fill!(resize!(partable.columns[:free], n), true) - value_fixed = fill!(resize!(partable.columns[:value_fixed], n), NaN) - start = fill!(resize!(partable.columns[:start], n), NaN) - param_refs = fill!(resize!(partable.columns[:param], n), Symbol("")) + columns = empty_partable_columns(n) + from = columns[:from] + parameter_type = columns[:parameter_type] + to = columns[:to] + free = columns[:free] + value_fixed = columns[:value_fixed] + start = columns[:start] + param_refs = columns[:param] # group = Vector{Symbol}(undef, n) for (i, element) in enumerate(graph) @@ -104,7 +104,7 @@ function ParameterTable( end end - return partable + return ParameterTable(columns; latent_vars, observed_vars) end ############################################################################################ From c29b0c753a2e3a7cbce4fd3d2f19e917ee52d9da Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 15:28:45 -0700 Subject: [PATCH 038/364] update_partable!() cleanup + docstring --- src/frontend/specification/ParameterTable.jl | 39 ++++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 38f84917b..52344d72e 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -201,36 +201,35 @@ end # update generic --------------------------------------------------------------------------- +""" + update_partable!(partable::AbstractParameterTable, params::Vector{Symbol}, values, column) + +Write parameter `values` into `column` of `partable`. + +The `params` and `values` vectors define the pairs of value +parameters, which are being matched to the `:param` column +of the `partable`. +""" function update_partable!( partable::ParameterTable, params::AbstractVector{Symbol}, values::AbstractVector, - column, + column::Symbol, ) - new_col = Vector{eltype(vec)}(undef, length(partable)) - params_index = Dict(param => i for (i, param) in enumerate(params)) + length(params) == length(values) || throw( + ArgumentError( + "The length of `params` ($(length(params))) and their `values` ($(length(values))) must be the same", + ), + ) + coldata = get!(() -> Vector{eltype(values)}(), partable.columns, column) + resize!(coldata, length(partable)) + params_index = Dict(zip(params, eachindex(params))) for (i, param) in enumerate(partable.columns[:param]) - if !(param == :const) - new_col[i] = values[params_index[param]] - elseif param == :const - new_col[i] = zero(eltype(values)) - end + coldata[i] = param != :const ? values[params_index[param]] : zero(eltype(values)) end - push!(partable.columns, column => new_col) return partable end -""" - update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, vec, column) - -Write `vec` to `column` of `partable`. - -# Arguments -- `vec::Vector`: has to be in the same order as the `model` parameters -""" -update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, values, column) = - update_partable!(partable, params(sem_fit), values, column) - # update estimates ------------------------------------------------------------------------- """ update_estimate!( From b9d8bc5d5957121ef9c6cdd4fa17ab4c99bb3288 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 15:29:48 -0700 Subject: [PATCH 039/364] update_partable!(): SemFit methods use basic one --- src/frontend/specification/ParameterTable.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 52344d72e..4749ecc8b 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -239,7 +239,7 @@ end Write parameter estimates from `sem_fit` to the `:estimate` column of `partable` """ update_estimate!(partable::AbstractParameterTable, sem_fit::SemFit) = - update_partable!(partable, sem_fit, sem_fit.solution, :estimate) + update_partable!(partable, params(sem_fit), sem_fit.solution, :estimate) # update starting values ------------------------------------------------------------------- """ @@ -254,7 +254,7 @@ Write starting values from `sem_fit` or `start_val` to the `:estimate` column of - `kwargs...`: are passed to `start_val` """ update_start!(partable::AbstractParameterTable, sem_fit::SemFit) = - update_partable!(partable, sem_fit, sem_fit.start_val, :start) + update_partable!(partable, params(sem_fit), sem_fit.start_val, :start) function update_start!( partable::AbstractParameterTable, @@ -289,5 +289,5 @@ function update_se_hessian!( hessian = :finitediff, ) se = se_hessian(sem_fit; hessian = hessian) - return update_partable!(partable, sem_fit, se, :se) + return update_partable!(partable, params(sem_fit), se, :se) end From 6a484f1338ba010dfac6db641036237505581aee Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 15:34:16 -0700 Subject: [PATCH 040/364] ParTable: add explicit params field --- .../specification/EnsembleParameterTable.jl | 24 ++++------- src/frontend/specification/ParameterTable.jl | 43 +++++++++++++++---- src/frontend/specification/RAMMatrices.jl | 40 ++++++++++++++--- 3 files changed, 77 insertions(+), 30 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 0161cc81c..10b59fa15 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -23,25 +23,19 @@ function EnsembleParameterTable( spec_ensemble::AbstractDict{K, V}; params::Union{Nothing, Vector{Symbol}} = nothing, ) where {K, V <: SemSpecification} - partables = Dict{Symbol, ParameterTable}( - Symbol(group) => convert(ParameterTable, spec; params = params) for - (group, spec) in pairs(spec_ensemble) - ) - - if isnothing(params) + params = if isnothing(params) # collect all SEM parameters in ensemble if not specified # and apply the set to all partables - params = - unique(mapreduce(SEM.params, vcat, values(partables), init = Vector{Symbol}())) - for partable in values(partables) - if partable.params != params - copyto!(resize!(partable.params, length(params)), params) - #throw(ArgumentError("The parameter sets of the SEM specifications in the ensemble do not match.")) - end - end + unique(mapreduce(SEM.params, vcat, values(spec_ensemble), init = Vector{Symbol}())) else - params = copy(params) + copy(params) end + + # convert each model specification to ParameterTable + partables = Dict{Symbol, ParameterTable}( + Symbol(group) => convert(ParameterTable, spec; params) for + (group, spec) in pairs(spec_ensemble) + ) return EnsembleParameterTable(partables, params) end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 4749ecc8b..86811b0f4 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -7,6 +7,7 @@ struct ParameterTable{C} <: AbstractParameterTable observed_vars::Vector{Symbol} latent_vars::Vector{Symbol} sorted_vars::Vector{Symbol} + params::Vector{Symbol} end ############################################################################################ @@ -31,11 +32,32 @@ function ParameterTable( columns::Dict{Symbol, Vector} = empty_partable_columns(); observed_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, latent_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, + params::Union{AbstractVector{Symbol}, Nothing} = nothing, ) - return ParameterTable(columns, + params = isnothing(params) ? unique!(filter(!=(:const), columns[:param])) : copy(params) + check_params(params, columns[:param]) + return ParameterTable( + columns, !isnothing(observed_vars) ? copy(observed_vars) : Vector{Symbol}(), !isnothing(latent_vars) ? copy(latent_vars) : Vector{Symbol}(), - Vector{Symbol}()) + Vector{Symbol}(), + params, + ) +end + +# new parameter table with different parameters order +function ParameterTable( + partable::ParameterTable; + params::Union{AbstractVector{Symbol}, Nothing} = nothing, +) + isnothing(params) || check_params(params, partable.columns[:param]) + + return ParameterTable( + Dict(col => copy(values) for (col, values) in pairs(partable.columns)), + observed_vars = copy(partable.observed_vars), + latent_vars = copy(partable.latent_vars), + params = params, + ) end ############################################################################################ @@ -46,6 +68,15 @@ function Base.convert(::Type{Dict}, partable::ParameterTable) return partable.columns end +function Base.convert( + ::Type{ParameterTable}, + partable::ParameterTable; + params::Union{AbstractVector{Symbol}, Nothing} = nothing, +) + return isnothing(params) || partable.params == params ? partable : + ParameterTable(partable; params) +end + function DataFrames.DataFrame( partable::ParameterTable; columns::Union{AbstractVector{Symbol}, Nothing} = nothing, @@ -110,12 +141,8 @@ Base.eltype(::Type{<:ParameterTable}) = ParameterTableRow Base.iterate(partable::ParameterTable, i::Integer = 1) = i > length(partable) ? nothing : (partable[i], i + 1) - -# get the vector of all parameters in the table -# the position of the parameter is based on its first appearance in the table (and the ensemble) -params(partable::ParameterTable) = - filter!(!=(:const), unique(partable.columns[:param])) - +params(partable::ParameterTable) = partable.params +n_par(partable::ParameterTable) = length(params(partable)) # Sorting ---------------------------------------------------------------------------------- diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 138bc1c9b..0f79d592b 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -222,18 +222,40 @@ function RAMMatrices( ) end -Base.convert(::Type{RAMMatrices}, partable::ParameterTable) = RAMMatrices(partable) +Base.convert( + ::Type{RAMMatrices}, + partable::ParameterTable; + params::Union{AbstractVector{Symbol}, Nothing} = nothing, +) = RAMMatrices(partable; params) ############################################################################################ ### get parameter table from RAMMatrices ############################################################################################ -function ParameterTable(ram_matrices::RAMMatrices) - colnames = ram_matrices.colnames +function ParameterTable( + ram_matrices::RAMMatrices; + params::Union{AbstractVector{Symbol}, Nothing} = nothing, + observed_var_prefix::Symbol = :obs, + latent_var_prefix::Symbol = :var, +) + # defer parameter checks until we know which ones are used + if !isnothing(ram_matrices.colnames) + colnames = ram_matrices.colnames + observed_vars = colnames[ram_matrices.F_ind] + latent_vars = colnames[setdiff(eachindex(colnames), ram_matrices.F_ind)] + else + observed_vars = + [Symbol("$(observed_var_prefix)_$i") for i in 1:nobserved_vars(ram_matrices)] + latent_vars = + [Symbol("$(latent_var_prefix)_$i") for i in 1:nlatent_vars(ram_matrices)] + colnames = vcat(observed_vars, latent_vars) + end + # construct an empty table partable = ParameterTable( - observed_vars = colnames[ram_matrices.F_ind], - latent_vars = colnames[setdiff(eachindex(colnames), ram_matrices.F_ind)], + observed_vars = observed_vars, + latent_vars = latent_vars, + params = isnothing(params) ? SEM.params(ram_matrices) : params, ) # constants @@ -254,12 +276,16 @@ function ParameterTable(ram_matrices::RAMMatrices) ram_matrices.size_F[2], ) end + check_params(SEM.params(partable), partable.columns[:param]) return partable end -Base.convert(::Type{<:ParameterTable}, ram_matrices::RAMMatrices) = - ParameterTable(ram_matrices) +Base.convert( + ::Type{<:ParameterTable}, + ram::RAMMatrices; + params::Union{AbstractVector{Symbol}, Nothing} = nothing, +) = ParameterTable(ram; params) ############################################################################################ ### Pretty Printing From 3804cc3814f6e1dc70bad65254bc46a2fdde32a0 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:02:00 -0700 Subject: [PATCH 041/364] n_par() -> nparams() for clarity and aligning to Julia naming conventions --- docs/src/developer/imply.md | 4 ++-- src/StructuralEquationModels.jl | 2 +- src/additional_functions/simulation.jl | 2 +- src/frontend/fit/SemFit.jl | 2 +- src/frontend/fit/fitmeasures/AIC.jl | 2 +- src/frontend/fit/fitmeasures/BIC.jl | 2 +- src/frontend/fit/fitmeasures/df.jl | 2 +- src/frontend/fit/fitmeasures/fit_measures.jl | 2 +- src/frontend/fit/summary.jl | 2 +- src/frontend/specification/ParameterTable.jl | 2 +- src/frontend/specification/RAMMatrices.jl | 2 +- src/frontend/specification/Sem.jl | 2 +- src/imply/RAM/generic.jl | 6 +++--- src/imply/RAM/symbolic.jl | 7 +++---- src/imply/empty.jl | 4 ++-- src/loss/regularization/ridge.jl | 12 ++++++------ src/types.jl | 13 ++++++------- test/examples/helper.jl | 4 ++-- test/examples/political_democracy/by_parts.jl | 2 +- .../recover_parameters_twofact.jl | 2 +- 20 files changed, 37 insertions(+), 39 deletions(-) diff --git a/docs/src/developer/imply.md b/docs/src/developer/imply.md index 44e0f6ff4..cb30e40fe 100644 --- a/docs/src/developer/imply.md +++ b/docs/src/developer/imply.md @@ -30,10 +30,10 @@ To make stored computations available to loss functions, simply write a function Additionally, you can specify methods for `gradient` and `hessian` as well as the combinations described in [Custom loss functions](@ref). -The last thing nedded to make it work is a method for `n_par` that takes your imply type and returns the number of parameters of the model: +The last thing nedded to make it work is a method for `nparams` that takes your imply type and returns the number of parameters of the model: ```julia -n_par(imply::MyImply) = ... +nparams(imply::MyImply) = ... ``` Just as described in [Custom loss functions](@ref), you may define a constructor. Typically, this will depend on the `specification = ...` argument that can be a `ParameterTable` or a `RAMMatrices` object. diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 109a913e7..7812fa819 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -154,6 +154,7 @@ export AbstractSem, sort_vars, RAMMatrices, params, + nparams, fit_measures, AIC, BIC, @@ -161,7 +162,6 @@ export AbstractSem, df, fit_measures, minus2ll, - n_par, n_obs, p_value, RMSEA, diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 58e8432e1..0dda725c6 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -73,7 +73,7 @@ function swap_observed( # update imply imply = update_observed(imply, new_observed; kwargs...) kwargs[:imply] = imply - kwargs[:n_par] = n_par(imply) + kwargs[:nparams] = nparams(imply) # update loss loss = update_observed(loss, new_observed; kwargs...) diff --git a/src/frontend/fit/SemFit.jl b/src/frontend/fit/SemFit.jl index 19a6d4441..ace9ed320 100644 --- a/src/frontend/fit/SemFit.jl +++ b/src/frontend/fit/SemFit.jl @@ -47,7 +47,7 @@ end ############################################################################################ params(fit::SemFit) = params(fit.model) -n_par(fit::SemFit) = n_par(fit.model) +nparams(fit::SemFit) = nparams(fit.model) # access fields minimum(sem_fit::SemFit) = sem_fit.minimum diff --git a/src/frontend/fit/fitmeasures/AIC.jl b/src/frontend/fit/fitmeasures/AIC.jl index 519f7beb7..f26f1f4dc 100644 --- a/src/frontend/fit/fitmeasures/AIC.jl +++ b/src/frontend/fit/fitmeasures/AIC.jl @@ -3,4 +3,4 @@ Return the akaike information criterion. """ -AIC(sem_fit::SemFit) = minus2ll(sem_fit) + 2n_par(sem_fit) +AIC(sem_fit::SemFit) = minus2ll(sem_fit) + 2nparams(sem_fit) diff --git a/src/frontend/fit/fitmeasures/BIC.jl b/src/frontend/fit/fitmeasures/BIC.jl index 56200f32b..47bd12f1b 100644 --- a/src/frontend/fit/fitmeasures/BIC.jl +++ b/src/frontend/fit/fitmeasures/BIC.jl @@ -3,4 +3,4 @@ Return the bayesian information criterion. """ -BIC(sem_fit::SemFit) = minus2ll(sem_fit) + log(n_obs(sem_fit)) * n_par(sem_fit) +BIC(sem_fit::SemFit) = minus2ll(sem_fit) + log(n_obs(sem_fit)) * nparams(sem_fit) diff --git a/src/frontend/fit/fitmeasures/df.jl b/src/frontend/fit/fitmeasures/df.jl index f546bb000..d4a4376dd 100644 --- a/src/frontend/fit/fitmeasures/df.jl +++ b/src/frontend/fit/fitmeasures/df.jl @@ -8,7 +8,7 @@ function df end df(sem_fit::SemFit) = df(sem_fit.model) -df(model::AbstractSem) = n_dp(model) - n_par(model) +df(model::AbstractSem) = n_dp(model) - nparams(model) function n_dp(model::AbstractSemSingle) nman = n_man(model) diff --git a/src/frontend/fit/fitmeasures/fit_measures.jl b/src/frontend/fit/fitmeasures/fit_measures.jl index e3f85a0f2..40e3caae0 100644 --- a/src/frontend/fit/fitmeasures/fit_measures.jl +++ b/src/frontend/fit/fitmeasures/fit_measures.jl @@ -1,5 +1,5 @@ fit_measures(sem_fit) = - fit_measures(sem_fit, n_par, df, AIC, BIC, RMSEA, χ², p_value, minus2ll) + fit_measures(sem_fit, nparams, df, AIC, BIC, RMSEA, χ², p_value, minus2ll) function fit_measures(sem_fit, args...) measures = Dict{Symbol, Union{Float64, Missing}}() diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 85c09590b..a31d4796f 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -16,7 +16,7 @@ function sem_summary( println("Convergence: $(convergence(sem_fit))") println("No. iterations/evaluations: $(n_iterations(sem_fit))") print("\n") - println("Number of parameters: $(n_par(sem_fit))") + println("Number of parameters: $(nparams(sem_fit))") println("Number of observations: $(n_obs(sem_fit))") print("\n") printstyled( diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 86811b0f4..fd34570fb 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -142,7 +142,7 @@ Base.iterate(partable::ParameterTable, i::Integer = 1) = i > length(partable) ? nothing : (partable[i], i + 1) params(partable::ParameterTable) = partable.params -n_par(partable::ParameterTable) = length(params(partable)) +nparams(partable::ParameterTable) = length(params(partable)) # Sorting ---------------------------------------------------------------------------------- diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 0f79d592b..80ac34cf9 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -63,7 +63,7 @@ struct RAMMatrices <: SemSpecification size_F::Tuple{Int, Int} end -n_par(ram::RAMMatrices) = length(ram.A_ind) +nparams(ram::RAMMatrices) = length(ram.A_ind) ############################################################################################ ### Constructor diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 208ef3000..73d4e81da 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -73,7 +73,7 @@ function get_fields!(kwargs, observed, imply, loss, optimizer) end kwargs[:imply] = imply - kwargs[:n_par] = n_par(imply) + kwargs[:nparams] = nparams(imply) # loss loss = get_SemLoss(loss; kwargs...) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 07aec6648..9eb694d51 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -34,8 +34,8 @@ and for models with a meanstructure, the model implied means are computed as ``` ## Interfaces -- `params(::RAM) `-> Dict containing the parameter labels and their position -- `n_par(::RAM)` -> Number of parameters +- `params(::RAM) `-> vector of parameter labels +- `nparams(::RAM)` -> number of parameters - `Σ(::RAM)` -> model implied covariance matrix - `μ(::RAM)` -> model implied mean vector @@ -107,7 +107,7 @@ function RAM(; ram_matrices = convert(RAMMatrices, specification) # get dimensions of the model - n_par = length(ram_matrices.params) + n_par = nparams(ram_matrices) n_var, n_nod = ram_matrices.size_F F = zeros(ram_matrices.size_F) F[CartesianIndex.(1:n_var, ram_matrices.F_ind)] .= 1.0 diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index db5997d2b..6eb372d4d 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -29,8 +29,8 @@ Subtype of `SemImply` that implements the RAM notation with symbolic precomputat Subtype of `SemImply`. ## Interfaces -- `params(::RAMSymbolic) `-> Dict containing the parameter labels and their position -- `n_par(::RAMSymbolic)` -> Number of parameters +- `params(::RAMSymbolic) `-> vector of parameter ids +- `nparams(::RAMSymbolic)` -> number of parameters - `Σ(::RAMSymbolic)` -> model implied covariance matrix - `μ(::RAMSymbolic)` -> model implied mean vector @@ -97,7 +97,7 @@ function RAMSymbolic(; ) ram_matrices = convert(RAMMatrices, specification) - n_par = length(ram_matrices.params) + n_par = nparams(ram_matrices) n_var, n_nod = ram_matrices.size_F par = (Symbolics.@variables θ[1:n_par])[1] @@ -141,7 +141,6 @@ function RAMSymbolic(; if hessian & !approximate_hessian n_sig = length(Σ_symbolic) - n_par = size(par, 1) ∇²Σ_symbolic_vec = [Symbolics.sparsehessian(σᵢ, [par...]) for σᵢ in vec(Σ_symbolic)] @variables J[1:n_sig] diff --git a/src/imply/empty.jl b/src/imply/empty.jl index 56297ea06..f1af2ec42 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -19,8 +19,8 @@ model per group and an additional model with `ImplyEmpty` and `SemRidge` for the # Extended help ## Interfaces -- `params(::RAMSymbolic) `-> Dict containing the parameter labels and their position -- `n_par(::RAMSymbolic)` -> Number of parameters +- `params(::RAMSymbolic) `-> Vector of parameter labels +- `nparams(::RAMSymbolic)` -> Number of parameters ## Implementation Subtype of `SemImply`. diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index 2d098d550..a61dd2af0 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -8,18 +8,18 @@ Ridge regularization. # Constructor - SemRidge(;α_ridge, which_ridge, n_par, parameter_type = Float64, imply = nothing, kwargs...) + SemRidge(;α_ridge, which_ridge, nparams, parameter_type = Float64, imply = nothing, kwargs...) # Arguments - `α_ridge`: hyperparameter for penalty term - `which_ridge::Vector`: Vector of parameter labels (Symbols) or indices that indicate which parameters should be regularized. -- `n_par::Int`: number of parameters of the model +- `nparams::Int`: number of parameters of the model - `imply::SemImply`: imply part of the model - `parameter_type`: type of the parameters # Examples ```julia -my_ridge = SemRidge(;α_ridge = 0.02, which_ridge = [:λ₁, :λ₂, :ω₂₃], n_par = 30, imply = my_imply) +my_ridge = SemRidge(;α_ridge = 0.02, which_ridge = [:λ₁, :λ₂, :ω₂₃], nparams = 30, imply = my_imply) ``` # Interfaces @@ -45,7 +45,7 @@ end function SemRidge(; α_ridge, which_ridge, - n_par, + nparams, parameter_type = Float64, imply = nothing, kwargs..., @@ -68,8 +68,8 @@ function SemRidge(; α_ridge, which, which_H, - zeros(parameter_type, n_par), - zeros(parameter_type, n_par, n_par), + zeros(parameter_type, nparams), + zeros(parameter_type, nparams, nparams), ) end diff --git a/src/types.jl b/src/types.jl index 70e0b4fbf..9db48b6db 100644 --- a/src/types.jl +++ b/src/types.jl @@ -21,14 +21,14 @@ Return the vector of SEM model parameters. params(model::AbstractSem) = model.params """ - n_par(semobj) + nparams(semobj) Return the number of SEM model parameters. """ -n_par(model::AbstractSem) = length(params(model)) +nparams(model::AbstractSem) = length(params(model)) params(model::AbstractSemSingle) = params(model.imply) -n_par(model::AbstractSemSingle) = n_par(model.imply) +nparams(model::AbstractSemSingle) = nparams(model.imply) """ SemLoss(args...; loss_weights = nothing, ...) @@ -93,7 +93,7 @@ If you would like to implement a different notation, e.g. LISREL, you should imp abstract type SemImply end params(imply::SemImply) = params(imply.ram_matrices) -n_par(imply::SemImply) = n_par(imply.ram_matrices) +nparams(imply::SemImply) = nparams(imply.ram_matrices) "Subtype of SemImply for all objects that can serve as the imply field of a SEM and use some form of symbolic precomputation." abstract type SemImplySymbolic <: SemImply end @@ -185,7 +185,6 @@ end function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing, kwargs...) n = length(models) - npar = n_par(models[1]) # default weights @@ -213,7 +212,7 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing end params(ensemble::SemEnsemble) = ensemble.params -n_par(ensemble::SemEnsemble) = length(ensemble.params) +nparams(ensemble::SemEnsemble) = length(ensemble.params) """ n_models(ensemble::SemEnsemble) -> Integer @@ -277,6 +276,6 @@ Base type for all SEM specifications. abstract type SemSpecification end params(spec::SemSpecification) = spec.params -n_par(spec::SemSpecification) = length(params(spec)) +nparams(spec::SemSpecification) = length(params(spec)) abstract type AbstractParameterTable <: SemSpecification end diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 0f10ce838..0230dd497 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -46,7 +46,7 @@ fitmeasure_names_ml = Dict( :df => "df", :χ² => "chisq", :p_value => "pvalue", - :n_par => "npar", + :nparams => "npar", :RMSEA => "rmsea", ) @@ -54,7 +54,7 @@ fitmeasure_names_ls = Dict( :df => "df", :χ² => "chisq", :p_value => "pvalue", - :n_par => "npar", + :nparams => "npar", :RMSEA => "rmsea", ) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index c071d9e00..c06c9929d 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -15,7 +15,7 @@ ml = SemML(observed = observed) wls = SemWLS(observed = observed) -ridge = SemRidge(α_ridge = 0.001, which_ridge = 16:20, n_par = 31) +ridge = SemRidge(α_ridge = 0.001, which_ridge = 16:20, nparams = 31) constant = SemConstant(constant_loss = 3.465) diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 1bd7136bc..5aa79842c 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -63,7 +63,7 @@ Random.seed!(1234) x = transpose(rand(true_dist, 100000)) semobserved = SemObservedData(data = x, specification = nothing) -loss_ml = SemLoss(SemML(; observed = semobserved, n_par = length(start))) +loss_ml = SemLoss(SemML(; observed = semobserved, nparams = length(start))) optimizer = SemOptimizerOptim( BFGS(; linesearch = BackTracking(order = 3), alphaguess = InitialHagerZhang()),# m = 100), From ca9f860bb680e190ea992634ac1a7c31ad7037ed Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 26 May 2024 21:24:19 -0700 Subject: [PATCH 042/364] param_values(ParTable) Co-authored-by: Maximilian-Stefan-Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- src/frontend/specification/ParameterTable.jl | 56 ++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index fd34570fb..0e3125665 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -318,3 +318,59 @@ function update_se_hessian!( se = se_hessian(sem_fit; hessian = hessian) return update_partable!(partable, params(sem_fit), se, :se) end + +""" + param_values!(out::AbstractVector, partable::ParameterTable, + col::Symbol = :estimate) + +Extract parameter values from the `col` column of `partable` +into the `out` vector. + +The `out` vector should be of `nparams(partable)` length. +The *i*-th element of the `out` vector will contain the +value of the *i*-th parameter from `params(partable)`. + +Note that the function combines the duplicate occurences of the +same parameter in `partable` and will raise an error if the +values do not match. +""" +function param_values!( + out::AbstractVector, + partable::ParameterTable, + col::Symbol = :estimate, +) + (length(out) == nparams(partable)) || throw( + DimensionMismatch( + "The length of parameter values vector ($(length(out))) does not match the number of parameters ($(nparams(partable)))", + ), + ) + param_index = Dict(param => i for (i, param) in enumerate(params(partable))) + param_values_col = partable.columns[col] + for (i, param) in enumerate(partable.columns[:param]) + (param == :const) && continue + param_ind = get(param_index, param, nothing) + @assert !isnothing(param_ind) "Parameter table contains unregistered parameter :$param at row #$i" + val = param_values_col[i] + if !isnan(out[param_ind]) + @assert isequal(out[param_ind], val) "Parameter :$param value at row #$i ($val) differs from the earlier encountered value ($(out[param_ind]))" + else + out[param_ind] = val + end + end + return out +end + +""" + param_values(out::AbstractVector, col::Symbol = :estimate) + +Extract parameter values from the `col` column of `partable`. + +Returns the values vector. The *i*-th element corresponds to +the value of *i*-th parameter from `params(partable)`. + +Note that the function combines the duplicate occurences of the +same parameter in `partable` and will raise an error if the +values do not match. +""" +param_values(partable::ParameterTable, col::Symbol = :estimate) = + param_values!(fill(NaN, nparams(partable)), partable, col) From b117b34cb03b42fc848dd9886ec7b64ffbbd497f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 16:46:34 -0700 Subject: [PATCH 043/364] lavaan_param_values(lav_fit, partable) --- src/frontend/specification/ParameterTable.jl | 142 +++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 0e3125665..43d0e1b11 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -374,3 +374,145 @@ values do not match. """ param_values(partable::ParameterTable, col::Symbol = :estimate) = param_values!(fill(NaN, nparams(partable)), partable, col) + +""" + lavaan_param_values!(out::AbstractVector, partable_lav, + partable::ParameterTable, + lav_col::Symbol = :est, lav_group = nothing) + +Extract parameter values from the `partable_lav` lavaan model that +match the parameters of `partable` into the `out` vector. + +The method sets the *i*-th element of the `out` vector to +the value of *i*-th parameter from `params(partable)`. + +Note that the lavaan and `partable` models are matched by the +the names of variables in the tables (`from` and `to` columns) +as well as the type of their relationship (`relation` column), +and not by the names of the model parameters. +""" +function lavaan_param_values!( + out::AbstractVector, + partable_lav, + partable::ParameterTable, + lav_col::Symbol = :est, + lav_group = nothing, +) + + # find indices of all df row where f is true + findallrows(f::Function, df) = findall(f(r) for r in eachrow(df)) + + (length(out) == nparams(partable)) || throw( + DimensionMismatch( + "The length of parameter values vector ($(length(out))) does not match the number of parameters ($(nparams(partable)))", + ), + ) + partable_mask = findall(partable.columns[:free]) + param_index = Dict(param => i for (i, param) in enumerate(params(partable))) + + lav_values = partable_lav[:, lav_col] + for (from, to, type, id) in zip( + [ + view(partable.columns[k], partable_mask) for + k in [:from, :to, :parameter_type, :param] + ]..., + ) + lav_ind = nothing + + if from == Symbol("1") + lav_ind = findallrows( + r -> + r[:lhs] == String(to) && + r[:op] == "~1" && + (isnothing(lav_group) || r[:group] == lav_group), + partable_lav, + ) + else + if type == :↔ + lav_type = "~~" + elseif type == :→ + if (from ∈ partable.latent_vars) && (to ∈ partable.observed_vars) + lav_type = "=~" + else + lav_type = "~" + from, to = to, from + end + end + + if lav_type == "~~" + lav_ind = findallrows( + r -> + ( + (r[:lhs] == String(from) && r[:rhs] == String(to)) || + (r[:lhs] == String(to) && r[:rhs] == String(from)) + ) && + r[:op] == lav_type && + (isnothing(lav_group) || r[:group] == lav_group), + partable_lav, + ) + else + lav_ind = findallrows( + r -> + r[:lhs] == String(from) && + r[:rhs] == String(to) && + r[:op] == lav_type && + (isnothing(lav_group) || r[:group] == lav_group), + partable_lav, + ) + end + end + + if length(lav_ind) == 0 + throw( + ErrorException( + "Parameter $id ($from $type $to) could not be found in the lavaan solution", + ), + ) + elseif length(lav_ind) > 1 + throw( + ErrorException( + "At least one parameter was found twice in the lavaan solution", + ), + ) + end + + param_ind = param_index[id] + param_val = lav_values[lav_ind[1]] + if isnan(out[param_ind]) + out[param_ind] = param_val + else + @assert out[param_ind] ≈ param_val atol = 1E-10 "Parameter :$id value at row #$lav_ind ($param_val) differs from the earlier encountered value ($(out[param_ind]))" + end + end + + return out +end + +""" + lavaan_param_values(partable_lav, partable::ParameterTable, + lav_col::Symbol = :est, lav_group = nothing) + +Extract parameter values from the `partable_lav` lavaan model that +match the parameters of `partable`. + +The `out` vector should be of `nparams(partable)` length. +The *i*-th element of the `out` vector will contain the +value of the *i*-th parameter from `params(partable)`. + +Note that the lavaan and `partable` models are matched by the +the names of variables in the tables (`from` and `to` columns), +and the type of their relationship (`relation` column), +but not by the ids of the model parameters. +""" +lavaan_param_values( + partable_lav, + partable::ParameterTable, + lav_col::Symbol = :est, + lav_group = nothing, +) = lavaan_param_values!( + fill(NaN, nparams(partable)), + partable_lav, + partable, + lav_col, + lav_group, +) From 398870b23d9617752885b00eb26c87313e225ab0 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 16:46:54 -0700 Subject: [PATCH 044/364] compare_estimates() -> test_estimates() * do tests inside * use param_values()/lavaan_param_values() --- test/examples/helper.jl | 283 +++--------------- test/examples/multigroup/build_models.jl | 18 +- test/examples/political_democracy/by_parts.jl | 37 +-- .../political_democracy/constructor.jl | 37 +-- test/unit_tests/sorting.jl | 2 +- 5 files changed, 79 insertions(+), 298 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 0230dd497..d4c140d67 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,3 +1,5 @@ +using LinearAlgebra: norm + function test_gradient(model, params; rtol = 1e-10, atol = 0) true_grad = FiniteDiff.finite_difference_gradient(Base.Fix1(objective!, model), params) gradient = similar(params) @@ -71,262 +73,63 @@ function test_fitmeasures( end end -function compare_estimates( +function test_estimates( partable::ParameterTable, partable_lav; rtol = 1e-10, atol = 0, col = :estimate, lav_col = :est, + lav_group = nothing, + skip::Bool = false, ) - correct = [] - - for i in findall(partable.columns[:free]) - from = partable.columns[:from][i] - to = partable.columns[:to][i] - type = partable.columns[:parameter_type][i] - estimate = partable.columns[col][i] - - if from == Symbol("1") - lav_ind = - findall((partable_lav.lhs .== String(to)) .& (partable_lav.op .== "~1")) - - if length(lav_ind) == 0 - throw( - ErrorException( - "Parameter from: $from, to: $to, type: $type, could not be found in the lavaan solution", - ), - ) - elseif length(lav_ind) > 1 - throw( - ErrorException( - "At least one parameter was found twice in the lavaan solution", - ), - ) - else - is_correct = isapprox( - estimate, - partable_lav[:, lav_col][lav_ind[1]]; - rtol = rtol, - atol = atol, - ) - push!(correct, is_correct) - end - - else - if type == :↔ - type = "~~" - elseif type == :→ - if (from ∈ partable.latent_vars) && (to ∈ partable.observed_vars) - type = "=~" - else - type = "~" - from, to = to, from - end - end - - if type == "~~" - lav_ind = findall( - ( - ( - (partable_lav.lhs .== String(from)) .& - (partable_lav.rhs .== String(to)) - ) .| ( - (partable_lav.lhs .== String(to)) .& - (partable_lav.rhs .== String(from)) - ) - ) .& (partable_lav.op .== type), - ) - - if length(lav_ind) == 0 - throw( - ErrorException( - "Parameter from: $from, to: $to, type: $type, could not be found in the lavaan solution", - ), - ) - elseif length(lav_ind) > 1 - throw( - ErrorException( - "At least one parameter was found twice in the lavaan solution", - ), - ) - else - is_correct = isapprox( - estimate, - partable_lav[:, lav_col][lav_ind[1]]; - rtol = rtol, - atol = atol, - ) - push!(correct, is_correct) - end - - else - lav_ind = findall( - (partable_lav.lhs .== String(from)) .& - (partable_lav.rhs .== String(to)) .& - (partable_lav.op .== type), - ) - - if length(lav_ind) == 0 - throw( - ErrorException( - "Parameter from: $from, to: $to, type: $type, could not be found in the lavaan solution", - ), - ) - elseif length(lav_ind) > 1 - throw( - ErrorException( - "At least one parameter was found twice in the lavaan solution", - ), - ) - else - is_correct = isapprox( - estimate, - partable_lav[:, lav_col][lav_ind[1]]; - rtol = rtol, - atol = atol, - ) - push!(correct, is_correct) - end - end - end + actual = StructuralEquationModels.param_values(partable, col) + expected = StructuralEquationModels.lavaan_param_values( + partable_lav, + partable, + lav_col, + lav_group, + ) + @test !any(isnan, actual) + @test !any(isnan, expected) + + if skip # workaround skip=false not supported in earlier versions + @test actual ≈ expected rtol = rtol atol = atol norm = Base.Fix2(norm, Inf) skip = + skip + else + @test actual ≈ expected rtol = rtol atol = atol norm = Base.Fix2(norm, Inf) end - - return all(correct) end -function compare_estimates( +function test_estimates( ens_partable::EnsembleParameterTable, partable_lav; rtol = 1e-10, atol = 0, col = :estimate, lav_col = :est, - lav_groups, + lav_groups::AbstractDict, + skip::Bool = false, ) - correct = [] - - for key in keys(ens_partable.tables) - group = lav_groups[key] - partable = ens_partable.tables[key] - - for i in findall(partable.columns[:free]) - from = partable.columns[:from][i] - to = partable.columns[:to][i] - type = partable.columns[:parameter_type][i] - estimate = partable.columns[col][i] - - if from == Symbol("1") - lav_ind = findall( - (partable_lav.lhs .== String(to)) .& - (partable_lav.op .== "~1") .& - (partable_lav.group .== group), - ) - - if length(lav_ind) == 0 - throw( - ErrorException( - "Mean parameter of variable $to could not be found in the lavaan solution", - ), - ) - elseif length(lav_ind) > 1 - throw( - ErrorException( - "At least one parameter was found twice in the lavaan solution", - ), - ) - else - is_correct = isapprox( - estimate, - partable_lav[:, lav_col][lav_ind[1]]; - rtol = rtol, - atol = atol, - ) - push!(correct, is_correct) - end - - else - if type == :↔ - type = "~~" - elseif type == :→ - if (from ∈ partable.latent_vars) && (to ∈ partable.observed_vars) - type = "=~" - else - type = "~" - from, to = to, from - end - end - - if type == "~~" - lav_ind = findall( - ( - ( - (partable_lav.lhs .== String(from)) .& - (partable_lav.rhs .== String(to)) - ) .| ( - (partable_lav.lhs .== String(to)) .& - (partable_lav.rhs .== String(from)) - ) - ) .& - (partable_lav.op .== type) .& - (partable_lav.group .== group), - ) - - if length(lav_ind) == 0 - throw( - ErrorException( - "Parameter from: $from, to: $to, type: $type, could not be found in the lavaan solution", - ), - ) - elseif length(lav_ind) > 1 - throw( - ErrorException( - "At least one parameter was found twice in the lavaan solution", - ), - ) - else - is_correct = isapprox( - estimate, - partable_lav[:, lav_col][lav_ind[1]]; - rtol = rtol, - atol = atol, - ) - push!(correct, is_correct) - end - - else - lav_ind = findall( - (partable_lav.lhs .== String(from)) .& - (partable_lav.rhs .== String(to)) .& - (partable_lav.op .== type) .& - (partable_lav.group .== group), - ) - - if length(lav_ind) == 0 - throw( - ErrorException( - "Parameter $from $type $to could not be found in the lavaan solution", - ), - ) - elseif length(lav_ind) > 1 - throw( - ErrorException( - "At least one parameter was found twice in the lavaan solution", - ), - ) - else - is_correct = isapprox( - estimate, - partable_lav[:, lav_col][lav_ind[1]]; - rtol = rtol, - atol = atol, - ) - push!(correct, is_correct) - end - end - end - end + actual = fill(NaN, nparams(ens_partable)) + expected = fill(NaN, nparams(ens_partable)) + for (key, partable) in pairs(ens_partable.tables) + StructuralEquationModels.param_values!(actual, partable, col) + StructuralEquationModels.lavaan_param_values!( + expected, + partable_lav, + partable, + lav_col, + lav_groups[key], + ) + end + @test !any(isnan, actual) + @test !any(isnan, expected) + + if skip # workaround skip=false not supported in earlier versions + @test actual ≈ expected rtol = rtol atol = atol norm = Base.Fix2(norm, Inf) skip = + skip + else + @test actual ≈ expected rtol = rtol atol = atol norm = Base.Fix2(norm, Inf) end - - return all(correct) end diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 23d429796..70d2bb914 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -17,7 +17,7 @@ end @testset "ml_solution_multigroup" begin solution = sem_fit(model_ml_multigroup) update_estimate!(partable, solution) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ml]; atol = 1e-4, @@ -35,7 +35,7 @@ end ) update_se_hessian!(partable, solution_ml) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3, @@ -78,7 +78,7 @@ grad_fd = FiniteDiff.finite_difference_gradient( @testset "ml_solution_multigroup | sorted" begin solution = sem_fit(model_ml_multigroup) update_estimate!(partable_s, solution) - @test compare_estimates( + test_estimates( partable_s, solution_lav[:parameter_estimates_ml]; atol = 1e-4, @@ -96,7 +96,7 @@ end ) update_se_hessian!(partable_s, solution_ml) - @test compare_estimates( + test_estimates( partable_s, solution_lav[:parameter_estimates_ml]; atol = 1e-3, @@ -152,7 +152,7 @@ end @testset "solution_user_defined_loss" begin solution = sem_fit(model_ml_multigroup) update_estimate!(partable, solution) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ml]; atol = 1e-4, @@ -179,7 +179,7 @@ end @testset "ls_solution_multigroup" begin solution = sem_fit(model_ls_multigroup) update_estimate!(partable, solution) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ls]; atol = 1e-4, @@ -198,7 +198,7 @@ end ) update_se_hessian!(partable, solution_ls) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, @@ -266,7 +266,7 @@ if !isnothing(specification_miss_g1) @testset "fiml_solution_multigroup" begin solution = sem_fit(model_ml_multigroup) update_estimate!(partable_miss, solution) - @test compare_estimates( + test_estimates( partable_miss, solution_lav[:parameter_estimates_fiml]; atol = 1e-4, @@ -284,7 +284,7 @@ if !isnothing(specification_miss_g1) ) update_se_hessian!(partable_miss, solution) - @test compare_estimates( + test_estimates( partable_miss, solution_lav[:parameter_estimates_fiml]; atol = 1e-3, diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index c06c9929d..11953ccb6 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -73,7 +73,7 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) @testset "$(name)_solution" begin solution = sem_fit(model) update_estimate!(partable, solution) - @test compare_estimates(partable, solution_lav[solution_name]; atol = 1e-2) + test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) end catch end @@ -114,7 +114,7 @@ end test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) update_se_hessian!(partable, solution_ml) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3, @@ -135,7 +135,7 @@ end @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) update_se_hessian!(partable, solution_ls) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, @@ -178,21 +178,18 @@ if semoptimizer == SemOptimizerOptim @testset "ml_solution_hessian" begin solution = sem_fit(model_ml) update_estimate!(partable, solution) - @test compare_estimates( - partable, - solution_lav[:parameter_estimates_ml]; - atol = 1e-3, - ) + test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) end @testset "ls_solution_hessian" begin solution = sem_fit(model_ls) update_estimate!(partable, solution) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ls]; atol = 1e-3, - ) skip = true + skip = true, + ) end end @@ -260,7 +257,7 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) @testset "$(name)_solution_mean" begin solution = sem_fit(model) update_estimate!(partable_mean, solution) - @test compare_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) + test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) end catch end @@ -279,7 +276,7 @@ end ) update_se_hessian!(partable_mean, solution_ml) - @test compare_estimates( + test_estimates( partable_mean, solution_lav[:parameter_estimates_ml_mean]; atol = 0.002, @@ -300,7 +297,7 @@ end @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) update_se_hessian!(partable_mean, solution_ls) - @test compare_estimates( + test_estimates( partable_mean, solution_lav[:parameter_estimates_ls_mean]; atol = 1e-2, @@ -342,21 +339,13 @@ end @testset "fiml_solution" begin solution = sem_fit(model_ml) update_estimate!(partable_mean, solution) - @test compare_estimates( - partable_mean, - solution_lav[:parameter_estimates_fiml]; - atol = 1e-2, - ) + test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @testset "fiml_solution_symbolic" begin solution = sem_fit(model_ml_sym) update_estimate!(partable_mean, solution) - @test compare_estimates( - partable_mean, - solution_lav[:parameter_estimates_fiml]; - atol = 1e-2, - ) + test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end ############################################################################################ @@ -372,7 +361,7 @@ end ) update_se_hessian!(partable_mean, solution_ml) - @test compare_estimates( + test_estimates( partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-3, diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 4ca1994bd..5f1c838e8 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -87,7 +87,7 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) @testset "$(name)_solution" begin solution = sem_fit(model) update_estimate!(partable, solution) - @test compare_estimates(partable, solution_lav[solution_name]; atol = 1e-2) + test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) end catch end @@ -131,7 +131,7 @@ end test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) update_se_hessian!(partable, solution_ml) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3, @@ -152,7 +152,7 @@ end @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) update_se_hessian!(partable, solution_ls) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, @@ -199,22 +199,19 @@ if semoptimizer == SemOptimizerOptim @testset "ml_solution_hessian" begin solution = sem_fit(model_ml) update_estimate!(partable, solution) - @test compare_estimates( - partable, - solution_lav[:parameter_estimates_ml]; - atol = 1e-3, - ) + test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) end @testset "ls_solution_hessian" begin solution = sem_fit(model_ls) update_estimate!(partable, solution) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ls]; atol = 0.002, rtol = 0.0, - ) skip = true + skip = true, + ) end end @@ -286,7 +283,7 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) @testset "$(name)_solution_mean" begin solution = sem_fit(model) update_estimate!(partable_mean, solution) - @test compare_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) + test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) end catch end @@ -305,7 +302,7 @@ end ) update_se_hessian!(partable_mean, solution_ml) - @test compare_estimates( + test_estimates( partable_mean, solution_lav[:parameter_estimates_ml_mean]; atol = 0.002, @@ -326,7 +323,7 @@ end @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) update_se_hessian!(partable_mean, solution_ls) - @test compare_estimates( + test_estimates( partable_mean, solution_lav[:parameter_estimates_ls_mean]; atol = 1e-2, @@ -379,21 +376,13 @@ end @testset "fiml_solution" begin solution = sem_fit(model_ml) update_estimate!(partable_mean, solution) - @test compare_estimates( - partable_mean, - solution_lav[:parameter_estimates_fiml]; - atol = 1e-2, - ) + test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @testset "fiml_solution_symbolic" begin solution = sem_fit(model_ml_sym) update_estimate!(partable_mean, solution) - @test compare_estimates( - partable_mean, - solution_lav[:parameter_estimates_fiml]; - atol = 1e-2, - ) + test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end ############################################################################################ @@ -409,7 +398,7 @@ end ) update_se_hessian!(partable_mean, solution_ml) - @test compare_estimates( + test_estimates( partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 0.002, diff --git a/test/unit_tests/sorting.jl b/test/unit_tests/sorting.jl index 5ca890c51..f5bc38ae0 100644 --- a/test/unit_tests/sorting.jl +++ b/test/unit_tests/sorting.jl @@ -13,5 +13,5 @@ end @testset "ml_solution_sorted" begin solution_ml_sorted = sem_fit(model_ml_sorted) update_estimate!(partable, solution_ml_sorted) - @test SEM.compare_estimates(par_ml, partable, 0.01) + @test test_estimates(par_ml, partable, 0.01) end From a2518b718a1dbf9bf0592efc2113ff1e4c8195d8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 26 May 2024 21:25:03 -0700 Subject: [PATCH 045/364] update_partable!(): dict-based generic version Co-authored-by: Maximilian-Stefan-Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- .../specification/EnsembleParameterTable.jl | 23 ++++-- src/frontend/specification/ParameterTable.jl | 81 ++++++++++++++----- 2 files changed, 76 insertions(+), 28 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 10b59fa15..b0b50448b 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -117,15 +117,24 @@ Base.getindex(partable::EnsembleParameterTable, group) = partable.tables[group] ### Update Partable from Fitted Model ############################################################################################ -# update generic --------------------------------------------------------------------------- function update_partable!( - partable::EnsembleParameterTable, + partables::EnsembleParameterTable, + column::Symbol, + param_values::AbstractDict{Symbol}, + default::Any = nothing, +) + for partable in values(partables.tables) + update_partable!(partable, column, param_values, default) + end + return partables +end + +function update_partable!( + partables::EnsembleParameterTable, + column::Symbol, params::AbstractVector{Symbol}, values::AbstractVector, - column, + default::Any = nothing, ) - for k in keys(partable.tables) - update_partable!(partable.tables[k], params, values, column) - end - return partable + return update_partable!(partables, column, Dict(zip(params, values)), default) end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 43d0e1b11..6dcbbbf84 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -227,6 +227,32 @@ end ############################################################################################ # update generic --------------------------------------------------------------------------- +function update_partable!( + partable::ParameterTable, + column::Symbol, + param_values::AbstractDict{Symbol, T}, + default::Any = nothing, +) where {T} + coldata = get!(() -> Vector{T}(undef, length(partable)), partable.columns, column) + + isvec_def = (default isa AbstractVector) && (length(default) == length(partable)) + + for (i, par) in enumerate(partable.columns[:param]) + if par == :const + coldata[i] = !isnothing(default) ? (isvec_def ? default[i] : default) : zero(T) + elseif haskey(param_values, par) + coldata[i] = param_values[par] + else + if isnothing(default) + throw(KeyError(par)) + else + coldata[i] = isvec_def ? default[i] : default + end + end + end + + return partable +end """ update_partable!(partable::AbstractParameterTable, params::Vector{Symbol}, values, column) @@ -239,49 +265,62 @@ of the `partable`. """ function update_partable!( partable::ParameterTable, + column::Symbol, params::AbstractVector{Symbol}, values::AbstractVector, - column::Symbol, + default::Any = nothing, ) length(params) == length(values) || throw( ArgumentError( "The length of `params` ($(length(params))) and their `values` ($(length(values))) must be the same", ), ) - coldata = get!(() -> Vector{eltype(values)}(), partable.columns, column) - resize!(coldata, length(partable)) - params_index = Dict(zip(params, eachindex(params))) - for (i, param) in enumerate(partable.columns[:param]) - coldata[i] = param != :const ? values[params_index[param]] : zero(eltype(values)) + param_values = Dict(zip(params, values)) + if length(param_values) != length(params) + throw(ArgumentError("Duplicate parameter names in `params`")) end - return partable + update_partable!(partable, column, param_values, default) end # update estimates ------------------------------------------------------------------------- """ update_estimate!( partable::AbstractParameterTable, - sem_fit::SemFit) + fit::SemFit) -Write parameter estimates from `sem_fit` to the `:estimate` column of `partable` +Write parameter estimates from `fit` to the `:estimate` column of `partable` """ -update_estimate!(partable::AbstractParameterTable, sem_fit::SemFit) = - update_partable!(partable, params(sem_fit), sem_fit.solution, :estimate) +update_estimate!(partable::ParameterTable, fit::SemFit) = update_partable!( + partable, + :estimate, + params(fit), + fit.solution, + partable.columns[:value_fixed], +) + +# fallback method for ensemble +update_estimate!(partable::AbstractParameterTable, fit::SemFit) = + update_partable!(partable, :estimate, params(fit), fit.solution) # update starting values ------------------------------------------------------------------- """ - update_start!(partable::AbstractParameterTable, sem_fit::SemFit) + update_start!(partable::AbstractParameterTable, fit::SemFit) update_start!(partable::AbstractParameterTable, model::AbstractSem, start_val; kwargs...) -Write starting values from `sem_fit` or `start_val` to the `:estimate` column of `partable`. +Write starting values from `fit` or `start_val` to the `:estimate` column of `partable`. # Arguments - `start_val`: either a vector of starting values or a function to compute starting values from `model` - `kwargs...`: are passed to `start_val` """ -update_start!(partable::AbstractParameterTable, sem_fit::SemFit) = - update_partable!(partable, params(sem_fit), sem_fit.start_val, :start) +update_start!(partable::AbstractParameterTable, fit::SemFit) = update_partable!( + partable, + :start, + params(fit), + fit.start_val, + partable.columns[:value_fixed], +) function update_start!( partable::AbstractParameterTable, @@ -292,17 +331,17 @@ function update_start!( if !(start_val isa Vector) start_val = start_val(model; kwargs...) end - return update_partable!(partable, params(model), start_val, :start) + return update_partable!(partable, :start, params(model), start_val) end # update partable standard errors ---------------------------------------------------------- """ update_se_hessian!( partable::AbstractParameterTable, - sem_fit::SemFit; + fit::SemFit; hessian = :finitediff) -Write hessian standard errors computed for `sem_fit` to the `:se` column of `partable` +Write hessian standard errors computed for `fit` to the `:se` column of `partable` # Arguments - `hessian::Symbol`: how to compute the hessian, see [se_hessian](@ref) for more information. @@ -312,11 +351,11 @@ Write hessian standard errors computed for `sem_fit` to the `:se` column of `par """ function update_se_hessian!( partable::AbstractParameterTable, - sem_fit::SemFit; + fit::SemFit; hessian = :finitediff, ) - se = se_hessian(sem_fit; hessian = hessian) - return update_partable!(partable, params(sem_fit), se, :se) + se = se_hessian(fit; hessian = hessian) + return update_partable!(partable, :se, params(fit), se) end """ From 84945fa2dfdab324c8c0d65b8e806d45dabe91b3 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 21:19:28 -0700 Subject: [PATCH 046/364] ParTable: getindex() returns NamedTuple so the downstream code doesn't rely on the order of tuple elements --- src/frontend/specification/ParameterTable.jl | 12 +++---- src/frontend/specification/RAMMatrices.jl | 36 +++++++++----------- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 6dcbbbf84..d1590269c 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -126,12 +126,12 @@ ParameterTableRow = @NamedTuple begin end Base.getindex(partable::ParameterTable, i::Integer) = ( - partable.columns[:from][i], - partable.columns[:parameter_type][i], - partable.columns[:to][i], - partable.columns[:free][i], - partable.columns[:value_fixed][i], - partable.columns[:param][i], + from = partable.columns[:from][i], + parameter_type = partable.columns[:parameter_type][i], + to = partable.columns[:to][i], + free = partable.columns[:free][i], + value_fixed = partable.columns[:value_fixed][i], + param = partable.columns[:param][i], ) Base.length(partable::ParameterTable) = length(partable.columns[:param]) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 80ac34cf9..a900f53a9 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -171,41 +171,39 @@ function RAMMatrices( # handle constants constants = Vector{RAMConstant}() - for i in 1:length(partable) - from, parameter_type, to, free, value_fixed, param = partable[i] - - row_ind = col_indices[to] - col_ind = from != Symbol("1") ? col_indices[from] : nothing - - if !free - if (parameter_type == :→) && (from == Symbol("1")) - push!(constants, RAMConstant(:M, row_ind, value_fixed)) - elseif (parameter_type == :→) + for r in partable + row_ind = col_indices[r.to] + col_ind = r.from != Symbol("1") ? col_indices[r.from] : nothing + + if !r.free + if (r.parameter_type == :→) && (r.from == Symbol("1")) + push!(constants, RAMConstant(:M, row_ind, r.value_fixed)) + elseif r.parameter_type == :→ push!( constants, - RAMConstant(:A, CartesianIndex(row_ind, col_ind), value_fixed), + RAMConstant(:A, CartesianIndex(row_ind, col_ind), r.value_fixed), ) - elseif (parameter_type == :↔) + elseif r.parameter_type == :↔ push!( constants, - RAMConstant(:S, CartesianIndex(row_ind, col_ind), value_fixed), + RAMConstant(:S, CartesianIndex(row_ind, col_ind), r.value_fixed), ) else - error("Unsupported parameter type: $(parameter_type)") + error("Unsupported parameter type: $(r.parameter_type)") end else - par_ind = params_index[param] - if (parameter_type == :→) && (from == Symbol("1")) + par_ind = params_index[r.param] + if (r.parameter_type == :→) && (r.from == Symbol("1")) push!(M_ind[par_ind], row_ind) - elseif parameter_type == :→ + elseif r.parameter_type == :→ push!(A_ind[par_ind], row_ind + (col_ind - 1) * n_node) - elseif parameter_type == :↔ + elseif r.parameter_type == :↔ push!(S_ind[par_ind], row_ind + (col_ind - 1) * n_node) if row_ind != col_ind push!(S_ind[par_ind], col_ind + (row_ind - 1) * n_node) end else - error("Unsupported parameter type: $(parameter_type)") + error("Unsupported parameter type: $(r.parameter_type)") end end end From 60e864bb5258822d70f65fd2bca11f8c43c5c3e7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 21:42:33 -0700 Subject: [PATCH 047/364] ParTable: graph-based ctor supports params= kw --- src/frontend/specification/StenoGraphs.jl | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 42edd6a13..69b91eabf 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -33,8 +33,9 @@ label(args...) = Label(args) function ParameterTable( graph::AbstractStenoGraph; - observed_vars, - latent_vars, + observed_vars::AbstractVector{Symbol}, + latent_vars::AbstractVector{Symbol}, + params::Union{AbstractVector{Symbol}, Nothing} = nothing, group::Integer = 1, param_prefix = :θ, ) @@ -104,7 +105,7 @@ function ParameterTable( end end - return ParameterTable(columns; latent_vars, observed_vars) + return ParameterTable(columns; latent_vars, observed_vars, params) end ############################################################################################ @@ -113,8 +114,9 @@ end function EnsembleParameterTable( graph::AbstractStenoGraph; - observed_vars, - latent_vars, + observed_vars::AbstractVector{Symbol}, + latent_vars::AbstractVector{Symbol}, + params::Union{AbstractVector{Symbol}, Nothing} = nothing, groups, ) graph = unique(graph) @@ -122,11 +124,13 @@ function EnsembleParameterTable( partables = Dict( group => ParameterTable( graph; - observed_vars = observed_vars, - latent_vars = latent_vars, + observed_vars, + latent_vars, + params, group = i, param_prefix = Symbol(:g, group), ) for (i, group) in enumerate(groups) ) - return EnsembleParameterTable(partables) + + return EnsembleParameterTable(partables; params) end From 88ea6b6d4a31027beec92189d28de4cfb89a79c5 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 22:36:44 -0700 Subject: [PATCH 048/364] rename parameter_type to relation for clarity --- src/frontend/fit/summary.jl | 30 ++++++-------------- src/frontend/specification/ParameterTable.jl | 14 ++++----- src/frontend/specification/RAMMatrices.jl | 23 ++++++++------- src/frontend/specification/StenoGraphs.jl | 6 ++-- 4 files changed, 31 insertions(+), 42 deletions(-) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index a31d4796f..f7ecdb331 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -93,19 +93,13 @@ function sem_summary( sorted_columns = [:to, :estimate, :param, :value_fixed, :start] loading_columns = sort_partially(sorted_columns, columns) header_cols = copy(loading_columns) - replace!(header_cols, :parameter_type => :type) for var in partable.latent_vars indicator_indices = findall( (partable.columns[:from] .== var) .& - (partable.columns[:parameter_type] .== :→) .& + (partable.columns[:relation] .== :→) .& (partable.columns[:to] .∈ [partable.observed_vars]), ) - loading_array = reduce( - hcat, - check_round(partable.columns[c][indicator_indices]; digits = digits) for - c in loading_columns - ) printstyled(var; color = secondary_color) print("\n") @@ -122,7 +116,7 @@ function sem_summary( printstyled("Directed Effects: \n"; color = color) regression_indices = findall( - (partable.columns[:parameter_type] .== :→) .& ( + (partable.columns[:relation] .== :→) .& ( ( (partable.columns[:to] .∈ [partable.observed_vars]) .& (partable.columns[:from] .∈ [partable.observed_vars]) @@ -138,7 +132,7 @@ function sem_summary( ), ) - sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] regression_columns = sort_partially(sorted_columns, columns) regression_array = reduce( @@ -147,7 +141,6 @@ function sem_summary( c in regression_columns ) regression_columns[2] = Symbol("") - replace!(regression_columns, :parameter_type => :type) print("\n") pretty_table( @@ -161,11 +154,11 @@ function sem_summary( printstyled("Variances: \n"; color = color) variance_indices = findall( - (partable.columns[:parameter_type] .== :↔) .& + (partable.columns[:relation] .== :↔) .& (partable.columns[:to] .== partable.columns[:from]), ) - sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce( @@ -174,7 +167,6 @@ function sem_summary( c in variance_columns ) variance_columns[2] = Symbol("") - replace!(variance_columns, :parameter_type => :type) print("\n") pretty_table( @@ -188,11 +180,11 @@ function sem_summary( printstyled("Covariances: \n"; color = color) variance_indices = findall( - (partable.columns[:parameter_type] .== :↔) .& + (partable.columns[:relation] .== :↔) .& (partable.columns[:to] .!= partable.columns[:from]), ) - sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce( @@ -201,7 +193,6 @@ function sem_summary( c in variance_columns ) variance_columns[2] = Symbol("") - replace!(variance_columns, :parameter_type => :type) print("\n") pretty_table( @@ -213,15 +204,13 @@ function sem_summary( print("\n") mean_indices = findall( - (partable.columns[:parameter_type] .== :→) .& - (partable.columns[:from] .== Symbol("1")), + (partable.columns[:relation] .== :→) .& (partable.columns[:from] .== Symbol("1")), ) if length(mean_indices) > 0 printstyled("Means: \n"; color = color) - sorted_columns = - [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce( @@ -230,7 +219,6 @@ function sem_summary( c in variance_columns ) variance_columns[2] = Symbol("") - replace!(variance_columns, :parameter_type => :type) print("\n") pretty_table( diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index d1590269c..eeb749401 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -18,7 +18,7 @@ end # optionally pre-allocate data for nrows empty_partable_columns(nrows::Integer = 0) = Dict{Symbol, Vector}( :from => fill(Symbol(), nrows), - :parameter_type => fill(Symbol(), nrows), + :relation => fill(Symbol(), nrows), :to => fill(Symbol(), nrows), :free => fill(true, nrows), :value_fixed => fill(NaN, nrows), @@ -93,7 +93,7 @@ end function Base.show(io::IO, partable::ParameterTable) relevant_columns = - [:from, :parameter_type, :to, :free, :value_fixed, :start, :estimate, :se, :param] + [:from, :relation, :to, :free, :value_fixed, :start, :estimate, :se, :param] shown_columns = filter!( col -> haskey(partable.columns, col) && length(partable.columns[col]) > 0, relevant_columns, @@ -118,7 +118,7 @@ end # Iteration -------------------------------------------------------------------------------- ParameterTableRow = @NamedTuple begin from::Symbol - parameter_type::Symbol + relation::Symbol to::Symbol free::Bool value_fixed::Any @@ -127,7 +127,7 @@ end Base.getindex(partable::ParameterTable, i::Integer) = ( from = partable.columns[:from][i], - parameter_type = partable.columns[:parameter_type][i], + relation = partable.columns[:relation][i], to = partable.columns[:to][i], free = partable.columns[:free][i], value_fixed = partable.columns[:value_fixed][i], @@ -170,8 +170,8 @@ function sort_vars!(partable::ParameterTable) ] is_regression = [ - (partype == :→) && (from != Symbol("1")) for (partype, from) in - zip(partable.columns[:parameter_type], partable.columns[:from]) + (rel == :→) && (from != Symbol("1")) for + (rel, from) in zip(partable.columns[:relation], partable.columns[:from]) ] to = partable.columns[:to][is_regression] @@ -453,7 +453,7 @@ function lavaan_param_values!( for (from, to, type, id) in zip( [ view(partable.columns[k], partable_mask) for - k in [:from, :to, :parameter_type, :param] + k in [:from, :to, :relation, :param] ]..., ) lav_ind = nothing diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index a900f53a9..b52c15a51 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -176,34 +176,34 @@ function RAMMatrices( col_ind = r.from != Symbol("1") ? col_indices[r.from] : nothing if !r.free - if (r.parameter_type == :→) && (r.from == Symbol("1")) + if (r.relation == :→) && (r.from == Symbol("1")) push!(constants, RAMConstant(:M, row_ind, r.value_fixed)) - elseif r.parameter_type == :→ + elseif r.relation == :→ push!( constants, RAMConstant(:A, CartesianIndex(row_ind, col_ind), r.value_fixed), ) - elseif r.parameter_type == :↔ + elseif r.relation == :↔ push!( constants, RAMConstant(:S, CartesianIndex(row_ind, col_ind), r.value_fixed), ) else - error("Unsupported parameter type: $(r.parameter_type)") + error("Unsupported parameter type: $(r.relation)") end else par_ind = params_index[r.param] - if (r.parameter_type == :→) && (r.from == Symbol("1")) + if (r.relation == :→) && (r.from == Symbol("1")) push!(M_ind[par_ind], row_ind) - elseif r.parameter_type == :→ + elseif r.relation == :→ push!(A_ind[par_ind], row_ind + (col_ind - 1) * n_node) - elseif r.parameter_type == :↔ + elseif r.relation == :↔ push!(S_ind[par_ind], row_ind + (col_ind - 1) * n_node) if row_ind != col_ind push!(S_ind[par_ind], col_ind + (row_ind - 1) * n_node) end else - error("Unsupported parameter type: $(r.parameter_type)") + error("Unsupported parameter type: $(r.relation)") end end end @@ -298,7 +298,8 @@ end ### Additional Functions ############################################################################################ -function matrix_to_parameter_type(matrix::Symbol) +# return the `from □ to` variables relation symbol (□) given the name of the source RAM matrix +function matrix_to_relation(matrix::Symbol) if matrix == :A return :→ elseif matrix == :S @@ -316,7 +317,7 @@ end partable_row(c::RAMConstant, varnames::AbstractVector{Symbol}) = ( from = varnames[c.index[2]], - parameter_type = matrix_to_parameter_type(c.matrix), + relation = matrix_to_relation(c.matrix), to = varnames[c.index[1]], free = false, value_fixed = c.value, @@ -346,7 +347,7 @@ function partable_row( return ( from = from, - parameter_type = matrix_to_parameter_type(matrix), + relation = matrix_to_relation(matrix), to = to, free = true, value_fixed = 0.0, diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 69b91eabf..67bb7973c 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -44,7 +44,7 @@ function ParameterTable( columns = empty_partable_columns(n) from = columns[:from] - parameter_type = columns[:parameter_type] + relation = columns[:relation] to = columns[:to] free = columns[:free] value_fixed = columns[:value_fixed] @@ -57,9 +57,9 @@ function ParameterTable( from[i] = edge.src.node to[i] = edge.dst.node if edge isa DirectedEdge - parameter_type[i] = :→ + relation[i] = :→ elseif edge isa UndirectedEdge - parameter_type[i] = :↔ + relation[i] = :↔ else throw( ArgumentError( From d57c08be5f7291341bb834aee227ffbf42f66348 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 May 2024 22:40:59 -0700 Subject: [PATCH 049/364] sem_summary(): cleanup filters --- src/frontend/fit/summary.jl | 80 +++++++++++++++---------------------- 1 file changed, 32 insertions(+), 48 deletions(-) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index f7ecdb331..214eb58f1 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -96,9 +96,9 @@ function sem_summary( for var in partable.latent_vars indicator_indices = findall( - (partable.columns[:from] .== var) .& - (partable.columns[:relation] .== :→) .& - (partable.columns[:to] .∈ [partable.observed_vars]), + r -> + (r.from == var) && (r.relation == :→) && (r.to ∈ partable.observed_vars), + partable, ) printstyled(var; color = secondary_color) @@ -116,20 +116,13 @@ function sem_summary( printstyled("Directed Effects: \n"; color = color) regression_indices = findall( - (partable.columns[:relation] .== :→) .& ( - ( - (partable.columns[:to] .∈ [partable.observed_vars]) .& - (partable.columns[:from] .∈ [partable.observed_vars]) - ) .| - ( - (partable.columns[:to] .∈ [partable.latent_vars]) .& - (partable.columns[:from] .∈ [partable.observed_vars]) - ) .| - ( - (partable.columns[:to] .∈ [partable.latent_vars]) .& - (partable.columns[:from] .∈ [partable.latent_vars]) - ) - ), + r -> + (r.relation == :→) && ( + ((r.to ∈ partable.observed_vars) && (r.from ∈ partable.observed_vars)) || + ((r.to ∈ partable.latent_vars) && (r.from ∈ partable.observed_vars)) || + ((r.to ∈ partable.latent_vars) && (r.from ∈ partable.latent_vars)) + ), + partable, ) sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] @@ -153,25 +146,22 @@ function sem_summary( printstyled("Variances: \n"; color = color) - variance_indices = findall( - (partable.columns[:relation] .== :↔) .& - (partable.columns[:to] .== partable.columns[:from]), - ) + var_indices = findall(r -> r.relation == :↔ && r.to == r.from, partable) sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] - variance_columns = sort_partially(sorted_columns, columns) + var_columns = sort_partially(sorted_columns, columns) - variance_array = reduce( + var_array = reduce( hcat, - check_round(partable.columns[c][variance_indices]; digits = digits) for + check_round(partable.columns[c][var_indices]; digits = digits) for c in variance_columns ) - variance_columns[2] = Symbol("") + var_columns[2] = Symbol("") print("\n") pretty_table( - variance_array; - header = variance_columns, + var_array; + header = var_columns, tf = PrettyTables.tf_borderless, alignment = :l, ) @@ -179,51 +169,45 @@ function sem_summary( printstyled("Covariances: \n"; color = color) - variance_indices = findall( - (partable.columns[:relation] .== :↔) .& - (partable.columns[:to] .!= partable.columns[:from]), - ) + covar_indices = findall(r -> r.relation == :↔ && r.to != r.from, partable) - sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] - variance_columns = sort_partially(sorted_columns, columns) + covar_columns = sort_partially(sorted_columns, columns) - variance_array = reduce( + covar_array = reduce( hcat, - check_round(partable.columns[c][variance_indices]; digits = digits) for - c in variance_columns + check_round(partable.columns[c][covar_indices]; digits = digits) for + c in covar_columns ) - variance_columns[2] = Symbol("") + covar_columns[2] = Symbol("") print("\n") pretty_table( - variance_array; - header = variance_columns, + covar_array; + header = covar_columns, tf = PrettyTables.tf_borderless, alignment = :l, ) print("\n") - mean_indices = findall( - (partable.columns[:relation] .== :→) .& (partable.columns[:from] .== Symbol("1")), - ) + mean_indices = findall(r -> (r.relation == :→) && (r.from == Symbol("1")), partable) if length(mean_indices) > 0 printstyled("Means: \n"; color = color) sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] - variance_columns = sort_partially(sorted_columns, columns) + mean_columns = sort_partially(sorted_columns, columns) - variance_array = reduce( + mean_array = reduce( hcat, check_round(partable.columns[c][mean_indices]; digits = digits) for - c in variance_columns + c in mean_columns ) - variance_columns[2] = Symbol("") + mean_columns[2] = Symbol("") print("\n") pretty_table( - variance_array; - header = variance_columns, + mean_array; + header = mean_columns, tf = PrettyTables.tf_borderless, alignment = :l, ) From 14970a2086c9c86a3c6e1c488eaf5c996b6b32a7 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 19 May 2024 21:16:40 +0200 Subject: [PATCH 050/364] fix sem_summary method for partable --- src/frontend/fit/summary.jl | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 214eb58f1..621791211 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -100,6 +100,11 @@ function sem_summary( (r.from == var) && (r.relation == :→) && (r.to ∈ partable.observed_vars), partable, ) + loading_array = reduce( + hcat, + check_round(partable.columns[c][indicator_indices]; digits = digits) for + c in loading_columns + ) printstyled(var; color = secondary_color) print("\n") @@ -109,6 +114,7 @@ function sem_summary( header = header_cols, tf = PrettyTables.tf_borderless, alignment = :l, + formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, ) print("\n") end @@ -141,6 +147,7 @@ function sem_summary( header = regression_columns, tf = PrettyTables.tf_borderless, alignment = :l, + formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, ) print("\n") @@ -154,7 +161,7 @@ function sem_summary( var_array = reduce( hcat, check_round(partable.columns[c][var_indices]; digits = digits) for - c in variance_columns + c in var_columns ) var_columns[2] = Symbol("") @@ -164,6 +171,7 @@ function sem_summary( header = var_columns, tf = PrettyTables.tf_borderless, alignment = :l, + formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, ) print("\n") @@ -186,6 +194,7 @@ function sem_summary( header = covar_columns, tf = PrettyTables.tf_borderless, alignment = :l, + formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, ) print("\n") @@ -210,6 +219,7 @@ function sem_summary( header = mean_columns, tf = PrettyTables.tf_borderless, alignment = :l, + formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, ) print("\n") end @@ -290,6 +300,14 @@ function sort_partially(sorted, to_sort) return out end +function Base.findall(fun::Function, partable::ParameterTable) + rows = Int[] + for (i, r) in enumerate(partable) + fun(r) ? push!(rows, i) : nothing + end + return rows +end + """ (1) sem_summary(sem_fit::SemFit; show_fitmeasures = false) From 4ba5fa8081b4d4aa6c71c9bfd63cf48b171b832b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 27 May 2024 02:26:15 -0700 Subject: [PATCH 051/364] show(ParTable): suppress NaNs --- src/frontend/specification/ParameterTable.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index eeb749401..6455f3332 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -105,6 +105,8 @@ function Base.show(io::IO, partable::ParameterTable) as_matrix, header = (shown_columns, [eltype(partable.columns[col]) for col in shown_columns]), tf = PrettyTables.tf_compact, + # TODO switch to `missing` as non-specified values and suppress printing of `missing` instead + formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, ) print(io, "Latent Variables: $(partable.latent_vars) \n") From 10ada011d571e7cdb779361bc9add4048c3be374 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 19 May 2024 19:39:01 +0200 Subject: [PATCH 052/364] sort_vars!(ParTable): cleanup --- src/frontend/specification/ParameterTable.jl | 28 +++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 6455f3332..7f4eab486 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -171,13 +171,15 @@ function sort_vars!(partable::ParameterTable) partable.observed_vars ] - is_regression = [ - (rel == :→) && (from != Symbol("1")) for - (rel, from) in zip(partable.columns[:relation], partable.columns[:from]) + # regression edges (excluding intercept) + edges = [ + (from, to) for (rel, from, to) in zip( + partable.columns[:relation], + partable.columns[:from], + partable.columns[:to], + ) if (rel == :→) && (from != Symbol("1")) ] - - to = partable.columns[:to][is_regression] - from = partable.columns[:from][is_regression] + sort!(edges, by = last) # sort edges by target sorted_vars = Vector{Symbol}() @@ -185,21 +187,27 @@ function sort_vars!(partable::ParameterTable) acyclic = false for (i, var) in enumerate(vars) - if !(var ∈ to) + # check if var has any incoming edge + eix = searchsortedfirst(edges, (var, var), by = last) + if !(eix <= length(edges) && last(edges[eix]) == var) + # var is source, no edges to it push!(sorted_vars, var) deleteat!(vars, i) - delete_edges = from .!= var - to = to[delete_edges] - from = from[delete_edges] + # remove var outgoing edges + filter!(e -> e[1] != var, edges) acyclic = true + break end end + # if acyclic is false, all vars have incoming edge acyclic || throw(CyclicModelError("your model is cyclic and therefore can not be ordered")) end copyto!(resize!(partable.sorted_vars, length(sorted_vars)), sorted_vars) + @assert length(partable.sorted_vars) == + length(partable.observed_vars) + length(partable.latent_vars) return partable end From 42da8a8bd35b1c1f79c15c5d7cd60547e29dbfdc Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 11 May 2024 13:21:57 -0700 Subject: [PATCH 053/364] Project.toml: disable SymbolicUtils 1.6 causes problems with sparsehessian(). It is a temporary fix until the compatibility issues are resolved in Symbolics.jl --- Project.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Project.toml b/Project.toml index 2fa168a9e..3a44943a6 100644 --- a/Project.toml +++ b/Project.toml @@ -22,6 +22,7 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" StenoGraphs = "78862bba-adae-4a83-bb4d-33c106177f81" Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" +SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b" [compat] julia = "1.9, 1.10" @@ -36,6 +37,7 @@ Optim = "1" PrettyTables = "2" StatsBase = "0.33, 0.34" Symbolics = "4, 5" +SymbolicUtils = "1.4 - 1.5" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From 7892b6d6b19dd99c8033ebec706de418373fef45 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 11 May 2024 13:22:15 -0700 Subject: [PATCH 054/364] Project.toml: support StenoGraphs 0.3 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 3a44943a6..9edeb4536 100644 --- a/Project.toml +++ b/Project.toml @@ -26,7 +26,7 @@ SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b" [compat] julia = "1.9, 1.10" -StenoGraphs = "0.2" +StenoGraphs = "0.2, 0.3" DataFrames = "1" Distributions = "0.25" FiniteDiff = "2" From 63ae853d34a9ded04be2f866b86f78f69f8457fb Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 19 May 2024 22:15:15 +0200 Subject: [PATCH 055/364] RAM ctor: better error for missing meanstruct --- src/imply/RAM/generic.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 9eb694d51..bc81a71a0 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -143,6 +143,7 @@ function RAM(; # μ if meanstructure has_meanstructure = Val(true) + !isnothing(M_indices) || throw(ArgumentError("You set `meanstructure = true`, but your model specification contains no mean parameters.")) ∇M = gradient ? matrix_gradient(M_indices, n_nod) : nothing μ = zeros(n_var) else From 7a39a0dc249843abc6603fcca743133153e46172 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 19 May 2024 23:43:54 +0200 Subject: [PATCH 056/364] add function param_indices --- src/StructuralEquationModels.jl | 1 + src/types.jl | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 7812fa819..19c7653bc 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -155,6 +155,7 @@ export AbstractSem, RAMMatrices, params, nparams, + param_indices, fit_measures, AIC, BIC, diff --git a/src/types.jl b/src/types.jl index 9db48b6db..0be745e80 100644 --- a/src/types.jl +++ b/src/types.jl @@ -29,6 +29,23 @@ nparams(model::AbstractSem) = length(params(model)) params(model::AbstractSemSingle) = params(model.imply) nparams(model::AbstractSemSingle) = nparams(model.imply) +""" + param_indices(semobj) + param_indices(param_names, semobj) + +Returns either a dict of parameter names and their indices in `semobj`. +If `param_names` are provided, returns a vector their indices in `semobj` instead. + +# Examples +```julia +parind = param_indices(my_fitted_sem) +parind[:param_name] + +parind = param_indices([:param_name_1, param_name_2], my_fitted_sem) +``` +""" +param_indices(semobj) = Dict(params(semobj) .=> 1:nparams(semobj)) +param_indices(param_names, semobj) = getindex.([Dict(params(semobj) .=> 1:nparams(semobj))], param_names) """ SemLoss(args...; loss_weights = nothing, ...) From 79af8105eeae66519a060124207e9e9249cdc9b7 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 19 May 2024 23:45:48 +0200 Subject: [PATCH 057/364] start fixing docs --- docs/src/developer/loss.md | 9 ++--- docs/src/developer/sem.md | 3 +- docs/src/performance/simulation.md | 5 +-- docs/src/performance/sorting.md | 2 +- docs/src/tutorials/collection/multigroup.md | 6 ++-- docs/src/tutorials/constraints/constraints.md | 35 +++++++++++-------- .../tutorials/construction/build_by_parts.md | 4 +-- .../construction/outer_constructor.md | 2 +- docs/src/tutorials/first_model.md | 4 +-- docs/src/tutorials/inspection/inspection.md | 10 +++--- docs/src/tutorials/meanstructure.md | 8 ++--- .../regularization/regularization.md | 7 ++-- .../specification/graph_interface.md | 8 ++--- .../tutorials/specification/ram_matrices.md | 4 +-- .../tutorials/specification/specification.md | 4 +-- 15 files changed, 61 insertions(+), 50 deletions(-) diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index 8bd654bf1..e1137dbf1 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -60,11 +60,12 @@ graph = @StenoGraph begin end partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars +) -parameter_indices = get_identifier_indices([:a, :b, :c], partable) +parameter_indices = param_indices([:a, :b, :c], partable) myridge = Ridge(0.01, parameter_indices) model = SemFiniteDiff( @@ -269,4 +270,4 @@ model_ml = SemFiniteDiff( model_fit = sem_fit(model_ml) ``` -If you want to differentiate your own loss functions via automatic differentiation, check out the [AutoDiffSEM](https://github.com/StructuralEquationModels/AutoDiffSEM) package (spoiler allert: it's really easy). +If you want to differentiate your own loss functions via automatic differentiation, check out the [AutoDiffSEM](https://github.com/StructuralEquationModels/AutoDiffSEM) package. diff --git a/docs/src/developer/sem.md b/docs/src/developer/sem.md index c6b9f0523..528da88b8 100644 --- a/docs/src/developer/sem.md +++ b/docs/src/developer/sem.md @@ -11,7 +11,8 @@ struct SemFiniteDiff{ observed::O imply::I loss::L - optimizer::Dend + optimizer::D +end ``` Additionally, we need to define a method to compute at least the objective value, and if you want to use gradient based optimizers (which you most probably will), we need also to define a method to compute the gradient. For example, the respective fallback methods for all `AbstractSemSingle` models are defined as diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index 4b00df6a4..b8a5081fe 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -43,9 +43,10 @@ graph = @StenoGraph begin end partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars +) ``` ```@example swap_observed diff --git a/docs/src/performance/sorting.md b/docs/src/performance/sorting.md index 802720099..78fd09411 100644 --- a/docs/src/performance/sorting.md +++ b/docs/src/performance/sorting.md @@ -13,7 +13,7 @@ To automatically reorder your variables in a way that makes this optimization po We use it as ```julia -sort!(parameter_table) +sort_vars!(parameter_table) model = Sem( specification = parameter_table, diff --git a/docs/src/tutorials/collection/multigroup.md b/docs/src/tutorials/collection/multigroup.md index 4e6105128..399d89760 100644 --- a/docs/src/tutorials/collection/multigroup.md +++ b/docs/src/tutorials/collection/multigroup.md @@ -61,8 +61,8 @@ You can then use the resulting graph to specify an `EnsembleParameterTable` ```@example mg; ansicolor = true groups = [:Pasteur, :Grant_White] -partable = EnsembleParameterTable(; - graph = graph, +partable = EnsembleParameterTable( + graph, observed_vars = observed_vars, latent_vars = latent_vars, groups = groups) @@ -71,7 +71,7 @@ partable = EnsembleParameterTable(; The parameter table can be used to create a `Dict` of RAMMatrices with keys equal to the group names and parameter tables as values: ```@example mg; ansicolor = true -specification = RAMMatrices(partable) +specification = convert(Dict{Symbol, RAMMatrices}, partable) ``` That is, you can asses the group-specific `RAMMatrices` as `specification[:group_name]`. diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index 7e2ec53e1..a67ad7372 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -16,13 +16,13 @@ graph = @StenoGraph begin # loadings ind60 → fixed(1)*x1 + x2 + x3 - dem60 → fixed(1)*y1 + y2 + y3 + y4 + dem60 → fixed(1)*y1 + label(:λ₂)*y2 + label(:λ₃)*y3 + y4 dem65 → fixed(1)*y5 + y6 + y7 + y8 # latent regressions ind60 → dem60 dem60 → dem65 - ind60 → dem65 + ind60 → label(:λₗ)*dem65 # variances _(observed_vars) ↔ _(observed_vars) @@ -31,15 +31,15 @@ graph = @StenoGraph begin # covariances y1 ↔ y5 y2 ↔ y4 + y6 - y3 ↔ y7 - y8 ↔ y4 + y6 + y3 ↔ label(:y3y7)*y7 + y8 ↔ label(:y8y4)*y4 + y6 end partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars) data = example_data("political_democracy") @@ -64,17 +64,19 @@ Let's introduce some constraints: (Of course those constaints only serve an illustratory purpose.) -We first need to get the indices of the respective parameters that are invoved in the constraints. We can look up their labels in the output above, and retrieve their indices as +We first need to get the indices of the respective parameters that are invoved in the constraints. +We can look up their labels in the output above, and retrieve their indices as ```@example constraints -parameter_indices = get_identifier_indices([:θ_29, :θ_30, :θ_3, :θ_4, :θ_11], model) +parind = param_indices(model) +parind[:y3y7] # 29 ``` -The bound constraint is easy to specify: Just give a vector of upper or lower bounds that contains the bound for each parameter. In our example, only parameter number 11 has an upper bound, and the number of total parameters is `n_par(model) = 31`, so we define +The bound constraint is easy to specify: Just give a vector of upper or lower bounds that contains the bound for each parameter. In our example, only the parameter labeled `:λₗ` has an upper bound, and the number of total parameters is `n_par(model) = 31`, so we define ```@example constraints upper_bounds = fill(Inf, 31) -upper_bounds[11] = 0.5 +upper_bounds[parind[:λₗ]] = 0.5 ``` The equailty and inequality constraints have to be reformulated to be of the form `x = 0` or `x ≤ 0`: @@ -84,6 +86,8 @@ The equailty and inequality constraints have to be reformulated to be of the for Now they can be defined as functions of the parameter vector: ```@example constraints +parind[:y3y7] # 29 +parind[:y8y4] # 30 # θ[29] + θ[30] - 1 = 0.0 function eq_constraint(θ, gradient) if length(gradient) > 0 @@ -94,6 +98,8 @@ function eq_constraint(θ, gradient) return θ[29] + θ[30] - 1 end +parind[:λ₂] # 3 +parind[:λ₃] # 4 # θ[3] - θ[4] - 0.1 ≤ 0 function ineq_constraint(θ, gradient) if length(gradient) > 0 @@ -109,7 +115,7 @@ If the algorithm needs gradients at an iteration, it will pass the vector `gradi With `if length(gradient) > 0` we check if the algorithm needs gradients, and if it does, we fill the `gradient` vector with the gradients of the constraint w.r.t. the parameters. -In NLopt, vector-valued constraints are also possible, but we refer to the documentation fot that. +In NLopt, vector-valued constraints are also possible, but we refer to the documentation for that. ### Fit the model @@ -153,10 +159,11 @@ As you can see, the optimizer converged (`:XTOL_REACHED`) and investigating the ```@example constraints update_partable!( - partable, - model_fit_constrained, + partable, + :estimate_constr, + params(model_fit_constrained), solution(model_fit_constrained), - :estimate_constr) + ) sem_summary(partable) ``` diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 5a56f1ccf..779949d98 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -39,9 +39,9 @@ graph = @StenoGraph begin end partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars) ``` Now, we construct the different parts: diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md index 21f6bfd3f..f072b80bc 100644 --- a/docs/src/tutorials/construction/outer_constructor.md +++ b/docs/src/tutorials/construction/outer_constructor.md @@ -74,7 +74,7 @@ model = Sem( specification = partable, data = data, imply = RAMSymbolic, - loss = SemWLS + loss = SemWLS, wls_weight_matrix = W ) diff --git a/docs/src/tutorials/first_model.md b/docs/src/tutorials/first_model.md index b19a22200..7568a5917 100644 --- a/docs/src/tutorials/first_model.md +++ b/docs/src/tutorials/first_model.md @@ -83,9 +83,9 @@ We then use this graph to define a `ParameterTable` object ```@example high_level; ansicolor = true partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars) ``` load the example data diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md index 5bc7946ba..b2eefadb2 100644 --- a/docs/src/tutorials/inspection/inspection.md +++ b/docs/src/tutorials/inspection/inspection.md @@ -31,9 +31,9 @@ graph = @StenoGraph begin end partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars) data = example_data("political_democracy") @@ -87,8 +87,8 @@ We can also update the `ParameterTable` object with other information via [`upda se_bs = se_bootstrap(model_fit; n_boot = 20) se_he = se_hessian(model_fit) -update_partable!(partable, model_fit, se_he, :se_hessian) -update_partable!(partable, model_fit, se_bs, :se_bootstrap) +update_partable!(partable, :se_hessian, params(model_fit), se_he) +update_partable!(partable, :se_bootstrap, params(model_fit), se_bs) sem_summary(partable) ``` @@ -130,7 +130,7 @@ df minus2ll n_man n_obs -n_par +nparams p_value RMSEA ``` diff --git a/docs/src/tutorials/meanstructure.md b/docs/src/tutorials/meanstructure.md index 9f2c167df..c6ad692b6 100644 --- a/docs/src/tutorials/meanstructure.md +++ b/docs/src/tutorials/meanstructure.md @@ -39,9 +39,9 @@ graph = @StenoGraph begin end partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars) ``` ```julia @@ -77,9 +77,9 @@ graph = @StenoGraph begin end partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars) ``` that is, all observed variable means are estimated freely. diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index b7d9affab..ce554a91d 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -86,9 +86,10 @@ graph = @StenoGraph begin end partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars +) data = example_data("political_democracy") @@ -101,7 +102,7 @@ model = Sem( We labeled the covariances between the items because we want to regularize those: ```@example reg -ind = get_identifier_indices([:cov_15, :cov_24, :cov_26, :cov_37, :cov_48, :cov_68], model) +ind = param_indices([:cov_15, :cov_24, :cov_26, :cov_37, :cov_48, :cov_68], model) ``` In the following, we fit the same model with lasso regularization of those covariances. diff --git a/docs/src/tutorials/specification/graph_interface.md b/docs/src/tutorials/specification/graph_interface.md index 7a03083c7..609c844c3 100644 --- a/docs/src/tutorials/specification/graph_interface.md +++ b/docs/src/tutorials/specification/graph_interface.md @@ -16,9 +16,9 @@ observed_vars = ... latent_vars = ... partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars) model = Sem( specification = partable, @@ -65,9 +65,9 @@ As you saw above and in the [A first model](@ref) example, the graph object need ```julia partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars) ``` The `ParameterTable` constructor also needs you to specify a vector of observed and latent variables, in the example above this would correspond to diff --git a/docs/src/tutorials/specification/ram_matrices.md b/docs/src/tutorials/specification/ram_matrices.md index 8eea6967c..5f0757238 100644 --- a/docs/src/tutorials/specification/ram_matrices.md +++ b/docs/src/tutorials/specification/ram_matrices.md @@ -59,7 +59,7 @@ spec = RAMMatrices(; A = A, S = S, F = F, - parameters = θ, + params = θ, colnames = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) @@ -90,7 +90,7 @@ spec = RAMMatrices(; A = A, S = S, F = F, - parameters = θ, + params = θ, colnames = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) ``` diff --git a/docs/src/tutorials/specification/specification.md b/docs/src/tutorials/specification/specification.md index 88f19ce3d..c426443f4 100644 --- a/docs/src/tutorials/specification/specification.md +++ b/docs/src/tutorials/specification/specification.md @@ -18,9 +18,9 @@ graph = @StenoGraph begin end partable = ParameterTable( + graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars) model = Sem( specification = partable, From 7ba872abe81795fb0f092e5a3ed0260e384c52ee Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 20 May 2024 00:04:28 +0200 Subject: [PATCH 058/364] fix regularization docs --- docs/src/tutorials/regularization/regularization.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index ce554a91d..4aaff1d0a 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -146,7 +146,7 @@ fit = sem_fit(model) update_estimate!(partable, fit) -update_partable!(partable, fit_lasso, solution(fit_lasso), :estimate_lasso) +update_partable!(partable, :estimate_lasso, params(fit_lasso), solution(fit_lasso)) sem_summary(partable) ``` @@ -180,7 +180,7 @@ fit_mixed = sem_fit(model_mixed) Let's again compare the different results: ```@example reg -update_partable!(partable, fit_mixed, solution(fit_mixed), :estimate_mixed) +update_partable!(partable, :estimate_mixed, params(fit_mixed), solution(fit_mixed)) sem_summary(partable) ``` \ No newline at end of file From c3ee769fa9f12808f27806d90f3030b4e7760aab Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 20 May 2024 00:07:06 +0200 Subject: [PATCH 059/364] introduce formatting error --- src/additional_functions/helper.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 3e614e57b..138ae431e 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -41,7 +41,7 @@ function get_observed(rowind, data, semobserved; args = (), kwargs = NamedTuple( return observed_vec end -skipmissing_mean(mat::AbstractMatrix) = +skipmissing_mean(mat::AbstractMatrix) = [mean(skipmissing(coldata)) for coldata in eachcol(mat)] function F_one_person(imp_mean, meandiff, inverse, data, logdet) From 5bb89a0400440697aa94ace4e54e91c7f8287b6a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 26 May 2024 21:25:20 -0700 Subject: [PATCH 060/364] update_start(): fix docstring typo Co-authored-by: Maximilian-Stefan-Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- src/frontend/specification/ParameterTable.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 7f4eab486..4f1bf747d 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -317,7 +317,7 @@ update_estimate!(partable::AbstractParameterTable, fit::SemFit) = update_start!(partable::AbstractParameterTable, fit::SemFit) update_start!(partable::AbstractParameterTable, model::AbstractSem, start_val; kwargs...) -Write starting values from `fit` or `start_val` to the `:estimate` column of `partable`. +Write starting values from `fit` or `start_val` to the `:start` column of `partable`. # Arguments - `start_val`: either a vector of starting values or a function to compute starting values From 43dceff22f33d835296c7780e212d0bd7f2b2e26 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 26 May 2024 21:28:14 -0700 Subject: [PATCH 061/364] push!(::ParTable, Tuple): check keys compat Co-authored-by: Maximilian-Stefan-Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- src/frontend/specification/ParameterTable.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 4f1bf747d..03b111e87 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -227,6 +227,8 @@ sort_vars(partable::ParameterTable) = sort_vars!(deepcopy(partable)) # add a row -------------------------------------------------------------------------------- function Base.push!(partable::ParameterTable, d::Union{AbstractDict{Symbol}, NamedTuple}) + issetequal(keys(partable.columns), keys(d)) || + throw(ArgumentError("The new row needs to have the same keys as the columns of the parameter table.")) for (key, val) in pairs(d) push!(partable.columns[key], val) end From 165474ca7a1c7f6a4616d03d9e61b4b825b0e551 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 27 May 2024 14:38:02 -0700 Subject: [PATCH 062/364] SemObsCov ctor: restrict n_obs to integer don't allow missing n_obs --- src/observed/covariance.jl | 4 ++-- test/unit_tests/data_input_formats.jl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 8d73b1a99..9be35e510 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -47,13 +47,13 @@ struct SemObservedCovariance{B, C} <: SemObserved end function SemObservedCovariance(; - specification::Union{SemSpecification, Nothing}, + specification::Union{SemSpecification, Nothing} = nothing, obs_cov, obs_colnames = nothing, spec_colnames = nothing, obs_mean = nothing, meanstructure = false, - n_obs = nothing, + n_obs::Integer, kwargs..., ) if !meanstructure & !isnothing(obs_mean) diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 7a048b280..44656c331 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -209,7 +209,7 @@ end ) end -@test_throws UndefKeywordError(:specification) SemObservedCovariance(obs_cov = dat_cov) +@test_throws UndefKeywordError(:n_obs) SemObservedCovariance(obs_cov = dat_cov) @test_throws ArgumentError("no `obs_colnames` were specified") begin SemObservedCovariance(specification = spec, obs_cov = dat_cov, n_obs = 75) From b2012b09f5d4b2bac0b69381eb1d8878e25924fa Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 27 May 2024 14:04:48 -0700 Subject: [PATCH 063/364] fixup param_indices() --- src/types.jl | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/types.jl b/src/types.jl index 0be745e80..d194709b8 100644 --- a/src/types.jl +++ b/src/types.jl @@ -29,23 +29,19 @@ nparams(model::AbstractSem) = length(params(model)) params(model::AbstractSemSingle) = params(model.imply) nparams(model::AbstractSemSingle) = nparams(model.imply) + """ param_indices(semobj) - param_indices(param_names, semobj) -Returns either a dict of parameter names and their indices in `semobj`. -If `param_names` are provided, returns a vector their indices in `semobj` instead. +Returns a dict of parameter names and their indices in `semobj`. # Examples ```julia parind = param_indices(my_fitted_sem) parind[:param_name] - -parind = param_indices([:param_name_1, param_name_2], my_fitted_sem) ``` """ -param_indices(semobj) = Dict(params(semobj) .=> 1:nparams(semobj)) -param_indices(param_names, semobj) = getindex.([Dict(params(semobj) .=> 1:nparams(semobj))], param_names) +param_indices(semobj) = Dict(par => i for (i, par) in enumerate(params(semobj))) """ SemLoss(args...; loss_weights = nothing, ...) From 619d89a55e32ed1c80d1eda14c1653cfc3ae36e7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 29 Jul 2024 23:58:34 -0700 Subject: [PATCH 064/364] common.jl: common vars API methods --- src/StructuralEquationModels.jl | 1 + src/frontend/common.jl | 48 +++++++++++++++++++++++++++++++++ src/types.jl | 28 ------------------- 3 files changed, 49 insertions(+), 28 deletions(-) create mode 100644 src/frontend/common.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 19c7653bc..33ca5a24a 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -30,6 +30,7 @@ include("additional_functions/commutation_matrix.jl") # fitted objects include("frontend/fit/SemFit.jl") # specification of models +include("frontend/common.jl") include("frontend/specification/checks.jl") include("frontend/specification/ParameterTable.jl") include("frontend/specification/RAMMatrices.jl") diff --git a/src/frontend/common.jl b/src/frontend/common.jl new file mode 100644 index 000000000..c55acf1d1 --- /dev/null +++ b/src/frontend/common.jl @@ -0,0 +1,48 @@ +# API methods supported by multiple SEM.jl types + +""" + nparams(semobj) + +Return the number of parameters in a SEM model associated with `semobj`. + +See also [`params`](@ref). +""" +nparams(semobj) = length(params(semobj)) + +""" + nvars(semobj) + +Return the number of variables in a SEM model associated with `semobj`. + +See also [`vars`](@ref). +""" +nvars(semobj) = length(vars(semobj)) + +""" + nobserved_vars(semobj) + +Return the number of observed variables in a SEM model associated with `semobj`. +""" +nobserved_vars(semobj) = length(observed_vars(semobj)) + +""" + nlatent_vars(semobj) + +Return the number of latent variables in a SEM model associated with `semobj`. +""" +nlatent_vars(semobj) = length(latent_vars(semobj)) + +""" + param_indices(semobj) + +Returns a dict of parameter names and their indices in `semobj`. + +# Examples +```julia +parind = param_indices(my_fitted_sem) +parind[:param_name] +``` + +See also [`params`](@ref). +""" +param_indices(semobj) = Dict(par => i for (i, par) in enumerate(params(semobj))) \ No newline at end of file diff --git a/src/types.jl b/src/types.jl index d194709b8..d3e1cde25 100644 --- a/src/types.jl +++ b/src/types.jl @@ -20,29 +20,6 @@ Return the vector of SEM model parameters. """ params(model::AbstractSem) = model.params -""" - nparams(semobj) - -Return the number of SEM model parameters. -""" -nparams(model::AbstractSem) = length(params(model)) - -params(model::AbstractSemSingle) = params(model.imply) -nparams(model::AbstractSemSingle) = nparams(model.imply) - -""" - param_indices(semobj) - -Returns a dict of parameter names and their indices in `semobj`. - -# Examples -```julia -parind = param_indices(my_fitted_sem) -parind[:param_name] -``` -""" -param_indices(semobj) = Dict(par => i for (i, par) in enumerate(params(semobj))) - """ SemLoss(args...; loss_weights = nothing, ...) @@ -105,9 +82,6 @@ If you would like to implement a different notation, e.g. LISREL, you should imp """ abstract type SemImply end -params(imply::SemImply) = params(imply.ram_matrices) -nparams(imply::SemImply) = nparams(imply.ram_matrices) - "Subtype of SemImply for all objects that can serve as the imply field of a SEM and use some form of symbolic precomputation." abstract type SemImplySymbolic <: SemImply end @@ -225,7 +199,6 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing end params(ensemble::SemEnsemble) = ensemble.params -nparams(ensemble::SemEnsemble) = length(ensemble.params) """ n_models(ensemble::SemEnsemble) -> Integer @@ -289,6 +262,5 @@ Base type for all SEM specifications. abstract type SemSpecification end params(spec::SemSpecification) = spec.params -nparams(spec::SemSpecification) = length(params(spec)) abstract type AbstractParameterTable <: SemSpecification end From 6aa0fd9dcb5331c9f8dd2d470fbd958d0133a513 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 16 Jun 2024 20:58:57 -0700 Subject: [PATCH 065/364] SemSpecification: vars API --- src/frontend/specification/documentation.jl | 41 +++++++++++++++++++++ src/types.jl | 2 - 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl index 27bedfea1..464af144b 100644 --- a/src/frontend/specification/documentation.jl +++ b/src/frontend/specification/documentation.jl @@ -1,3 +1,44 @@ +""" + params(semobj) -> Vector{Symbol} + +Return the vector of SEM model parameter identifiers. +""" +function params end + +params(spec::SemSpecification) = spec.params + +""" + vars(semobj) -> Vector{Symbol} + +Return the vector of SEM model variables (both observed and latent) +in the order specified by the model. +""" +function vars end + +vars(spec::SemSpecification) = error("vars(spec::$(typeof(spec))) is not implemented") + +""" + observed_vars(semobj) -> Vector{Symbol} + +Return the vector of SEM model observed variable in the order specified by the +model, which also should match the order of variables in [`SemObserved`](@ref). +""" +function observed_vars end + +observed_vars(spec::SemSpecification) = + error("observed_vars(spec::$(typeof(spec))) is not implemented") + +""" + latent_vars(semobj) -> Vector{Symbol} + +Return the vector of SEM model latent variable in the order specified by the +model. +""" +function latent_vars end + +latent_vars(spec::SemSpecification) = + error("latent_vars(spec::$(typeof(spec))) is not implemented") + """ `ParameterTable`s contain the specification of a structural equation model. diff --git a/src/types.jl b/src/types.jl index d3e1cde25..99153622e 100644 --- a/src/types.jl +++ b/src/types.jl @@ -261,6 +261,4 @@ Base type for all SEM specifications. """ abstract type SemSpecification end -params(spec::SemSpecification) = spec.params - abstract type AbstractParameterTable <: SemSpecification end From 2e4784a3555af3d120a8e418336ae0df070751de Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 16:42:59 -0700 Subject: [PATCH 066/364] RAMMatrices: vars API --- src/frontend/specification/RAMMatrices.jl | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index b52c15a51..6ba6be3d0 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -65,6 +65,30 @@ end nparams(ram::RAMMatrices) = length(ram.A_ind) +nvars(ram::RAMMatrices) = ram.size_F[2] +nobserved_vars(ram::RAMMatrices) = ram.size_F[1] +nlatent_vars(ram::RAMMatrices) = nvars(ram) - nobserved_vars(ram) + +vars(ram::RAMMatrices) = ram.colnames + +function observed_vars(ram::RAMMatrices) + if isnothing(ram.colnames) + @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" + return nothing + else + return view(ram.colnames, ram.F_ind) + end +end + +function latent_vars(ram::RAMMatrices) + if isnothing(ram.colnames) + @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" + return nothing + else + return view(ram.colnames, setdiff(eachindex(ram.colnames), ram.F_ind)) + end +end + ############################################################################################ ### Constructor ############################################################################################ From 467c034e999c62d6010cff68bb131b09e18fcb9d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 27 May 2024 14:06:22 -0700 Subject: [PATCH 067/364] ParamTable: vars API --- src/frontend/specification/ParameterTable.jl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 03b111e87..91d55ce46 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -60,6 +60,15 @@ function ParameterTable( ) end +vars(partable::ParameterTable) = + !isempty(partable.sorted_vars) ? partable.sorted_vars : + vcat(partable.latent_vars, partable.observed_vars) +observed_vars(partable::ParameterTable) = partable.observed_vars +latent_vars(partable::ParameterTable) = partable.latent_vars + +nvars(partable::ParameterTable) = + length(partable.latent_vars) + length(partable.observed_vars) + ############################################################################################ ### Convert to other types ############################################################################################ @@ -206,8 +215,7 @@ function sort_vars!(partable::ParameterTable) end copyto!(resize!(partable.sorted_vars, length(sorted_vars)), sorted_vars) - @assert length(partable.sorted_vars) == - length(partable.observed_vars) + length(partable.latent_vars) + @assert length(partable.sorted_vars) == nvars(partable) return partable end From d6b14496ee7bf5c738ad52b3436fe13480ceabe4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 29 Jul 2024 23:57:48 -0700 Subject: [PATCH 068/364] SemImply: vars and params API --- src/StructuralEquationModels.jl | 1 + src/imply/abstract.jl | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 src/imply/abstract.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 33ca5a24a..a77bc8d94 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -49,6 +49,7 @@ include("observed/EM.jl") include("frontend/specification/Sem.jl") include("frontend/specification/documentation.jl") # imply +include("imply/abstract.jl") include("imply/RAM/symbolic.jl") include("imply/RAM/generic.jl") include("imply/empty.jl") diff --git a/src/imply/abstract.jl b/src/imply/abstract.jl new file mode 100644 index 000000000..6a3f84191 --- /dev/null +++ b/src/imply/abstract.jl @@ -0,0 +1,12 @@ + +# vars and params API methods for SemImply +vars(imply::SemImply) = vars(imply.ram_matrices) +observed_vars(imply::SemImply) = observed_vars(imply.ram_matrices) +latent_vars(imply::SemImply) = latent_vars(imply.ram_matrices) + +nvars(imply::SemImply) = nvars(imply.ram_matrices) +nobserved_vars(imply::SemImply) = nobserved_vars(imply.ram_matrices) +nlatent_vars(imply::SemImply) = nlatent_vars(imply.ram_matrices) + +params(imply::SemImply) = params(imply.ram_matrices) +nparams(imply::SemImply) = nparams(imply.ram_matrices) From 56e68e07ed3f1807c0c86e6c9778a895bb2d0cb7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 23:55:42 -0700 Subject: [PATCH 069/364] RAM imply: use vars API --- src/imply/RAM/generic.jl | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index bc81a71a0..8ace80759 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -108,7 +108,8 @@ function RAM(; # get dimensions of the model n_par = nparams(ram_matrices) - n_var, n_nod = ram_matrices.size_F + n_obs = nobserved_vars(ram_matrices) + n_var = nvars(ram_matrices) F = zeros(ram_matrices.size_F) F[CartesianIndex.(1:n_var, ram_matrices.F_ind)] .= 1.0 @@ -118,23 +119,23 @@ function RAM(; M_indices = !isnothing(ram_matrices.M_ind) ? copy(ram_matrices.M_ind) : nothing #preallocate arrays - A_pre = zeros(n_nod, n_nod) - S_pre = zeros(n_nod, n_nod) - !isnothing(M_indices) ? M_pre = zeros(n_nod) : M_pre = nothing + A_pre = zeros(n_var, n_var) + S_pre = zeros(n_var, n_var) + M_pre = !isnothing(M_indices) ? zeros(n_var) : nothing set_RAMConstants!(A_pre, S_pre, M_pre, ram_matrices.constants) A_pre = check_acyclic(A_pre, n_par, A_indices) # pre-allocate some matrices - Σ = zeros(n_var, n_var) - F⨉I_A⁻¹ = zeros(n_var, n_nod) - F⨉I_A⁻¹S = zeros(n_var, n_nod) + Σ = zeros(n_obs, n_obs) + F⨉I_A⁻¹ = zeros(n_obs, n_var) + F⨉I_A⁻¹S = zeros(n_obs, n_var) I_A = similar(A_pre) if gradient - ∇A = matrix_gradient(A_indices, n_nod^2) - ∇S = matrix_gradient(S_indices, n_nod^2) + ∇A = matrix_gradient(A_indices, n_var^2) + ∇S = matrix_gradient(S_indices, n_var^2) else ∇A = nothing ∇S = nothing @@ -144,7 +145,7 @@ function RAM(; if meanstructure has_meanstructure = Val(true) !isnothing(M_indices) || throw(ArgumentError("You set `meanstructure = true`, but your model specification contains no mean parameters.")) - ∇M = gradient ? matrix_gradient(M_indices, n_nod) : nothing + ∇M = gradient ? matrix_gradient(M_indices, n_var) : nothing μ = zeros(n_var) else has_meanstructure = Val(false) From e27f028c44211047fbb872e35a93e2fa6b28a465 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 23:55:42 -0700 Subject: [PATCH 070/364] RAMSymbolic: use vars API --- src/imply/RAM/symbolic.jl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 6eb372d4d..3c99053bf 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -98,15 +98,16 @@ function RAMSymbolic(; ram_matrices = convert(RAMMatrices, specification) n_par = nparams(ram_matrices) - n_var, n_nod = ram_matrices.size_F + n_obs = nobserved_vars(ram_matrices) + n_var = nvars(ram_matrices) par = (Symbolics.@variables θ[1:n_par])[1] - A = zeros(Num, n_nod, n_nod) - S = zeros(Num, n_nod, n_nod) - !isnothing(ram_matrices.M_ind) ? M = zeros(Num, n_nod) : M = nothing + A = zeros(Num, n_var, n_var) + S = zeros(Num, n_var, n_var) + !isnothing(ram_matrices.M_ind) ? M = zeros(Num, n_var) : M = nothing F = zeros(ram_matrices.size_F) - F[CartesianIndex.(1:n_var, ram_matrices.F_ind)] .= 1.0 + F[CartesianIndex.(1:n_obs, ram_matrices.F_ind)] .= 1.0 set_RAMConstants!(A, S, M, ram_matrices.constants) fill_A_S_M!(A, S, M, ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.M_ind, par) From 814d615bbf354f2b633a9ad9b61c907ae824a2c0 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 23:55:42 -0700 Subject: [PATCH 071/364] start_simple(): use vars API --- src/additional_functions/start_val/start_simple.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 2c4f661c1..8e3cb32cb 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -62,17 +62,17 @@ function start_simple( start_means = 0.0, kwargs..., ) - A_ind, S_ind, F_ind, M_ind, params = ram_matrices.A_ind, + A_ind, S_ind, F_ind, M_ind, n_par = ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.F_ind, ram_matrices.M_ind, - ram_matrices.params + nparams(ram_matrices) - n_par = length(params) start_val = zeros(n_par) - n_var, n_nod = ram_matrices.size_F + n_obs = nobserved_vars(ram_matrices) + n_var = nvars(ram_matrices) - C_indices = CartesianIndices((n_nod, n_nod)) + C_indices = CartesianIndices((n_var, n_var)) for i in 1:n_par if length(S_ind[i]) != 0 From 0ac47b42ff916be5d31c19847a6885aade83ab47 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 23:55:42 -0700 Subject: [PATCH 072/364] starts_fabin3: use vars API --- .../start_val/start_fabin3.jl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index b56ee60a1..081af3ba1 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -31,18 +31,18 @@ function start_fabin3(observed::SemObservedMissing, imply, optimizer, args...; k end function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) - A_ind, S_ind, F_ind, M_ind, params = ram_matrices.A_ind, + A_ind, S_ind, F_ind, M_ind, n_par = ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.F_ind, ram_matrices.M_ind, - ram_matrices.params + nparams(ram_matrices) - n_par = length(params) start_val = zeros(n_par) - n_var, n_nod = ram_matrices.size_F - n_latent = n_nod - n_var + n_obs = nobserved_vars(ram_matrices) + n_var = nvars(ram_matrices) + n_latent = nlatent_vars(ram_matrices) - C_indices = CartesianIndices((n_nod, n_nod)) + C_indices = CartesianIndices((n_var, n_var)) # check in which matrix each parameter appears @@ -50,7 +50,7 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) #= in_S = length.(S_ind) .!= 0 in_A = length.(A_ind) .!= 0 - A_ind_c = [linear2cartesian(ind, (n_nod, n_nod)) for ind in A_ind] + A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind] in_Λ = [any(ind[2] .∈ F_ind) for ind in A_ind_c] if !isnothing(M) @@ -77,7 +77,7 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) # set loadings constants = ram_matrices.constants - A_ind_c = [linear2cartesian(ind, (n_nod, n_nod)) for ind in A_ind] + A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind] # ind_Λ = findall([is_in_Λ(ind_vec, F_ind) for ind_vec in A_ind_c]) function calculate_lambda( @@ -99,7 +99,7 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) end end - for i in setdiff(1:n_nod, F_ind) + for i in setdiff(1:n_var, F_ind) reference = Int64[] indicators = Int64[] indicator2parampos = Dict{Int, Int}() From 0d45014da01a23da373b2d65596339334a9e3d0c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 11 Mar 2024 22:03:40 -0700 Subject: [PATCH 073/364] remove get_colnames() replaced by observed_vars() --- src/StructuralEquationModels.jl | 1 - src/observed/covariance.jl | 4 ++-- src/observed/data.jl | 4 ++-- src/observed/get_colnames.jl | 21 --------------------- src/observed/missing.jl | 4 ++-- 5 files changed, 6 insertions(+), 28 deletions(-) delete mode 100644 src/observed/get_colnames.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a77bc8d94..69038c0d0 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -40,7 +40,6 @@ include("frontend/fit/summary.jl") # pretty printing include("frontend/pretty_printing.jl") # observed -include("observed/get_colnames.jl") include("observed/covariance.jl") include("observed/data.jl") include("observed/missing.jl") diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 9be35e510..28430263f 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -63,8 +63,8 @@ function SemObservedCovariance(; throw(ArgumentError("`meanstructure = true`, but no observed means were passed")) end - if isnothing(spec_colnames) - spec_colnames = get_colnames(specification) + if isnothing(spec_colnames) && !isnothing(specification) + spec_colnames = observed_vars(specification) end if !isnothing(spec_colnames) & isnothing(obs_colnames) diff --git a/src/observed/data.jl b/src/observed/data.jl index 89deefd04..8886c18b3 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -64,8 +64,8 @@ function SemObservedData(; rowwise = false, kwargs..., ) - if isnothing(spec_colnames) - spec_colnames = get_colnames(specification) + if isnothing(spec_colnames) && !isnothing(specification) + spec_colnames = observed_vars(specification) end if !isnothing(spec_colnames) diff --git a/src/observed/get_colnames.jl b/src/observed/get_colnames.jl deleted file mode 100644 index b8d89c3d0..000000000 --- a/src/observed/get_colnames.jl +++ /dev/null @@ -1,21 +0,0 @@ -# specification colnames (only observed) -function get_colnames(specification::ParameterTable) - colnames = - isempty(specification.sorted_vars) ? specification.observed_vars : - filter(in(Set(specification.observed_vars)), specification.sorted_vars) - return colnames -end - -function get_colnames(specification::RAMMatrices) - if isnothing(specification.colnames) - @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" - return nothing - else - colnames = specification.colnames[specification.F_ind] - return colnames - end -end - -function get_colnames(specification::Nothing) - return nothing -end diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 439e3d837..af673becd 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -92,8 +92,8 @@ function SemObservedMissing(; spec_colnames = nothing, kwargs..., ) - if isnothing(spec_colnames) - spec_colnames = get_colnames(specification) + if isnothing(spec_colnames) && !isnothing(specification) + spec_colnames = observed_vars(specification) end if !isnothing(spec_colnames) From 09f5ecad39e678d3a5cd8d0a073b99ba522b94ad Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:40:33 -0700 Subject: [PATCH 074/364] remove get_n_nodes() replaced by nvars() --- src/loss/ML/FIML.jl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index d4870ac1b..135bb411b 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -73,7 +73,7 @@ function SemFIML(; observed, specification, kwargs...) meandiff, imp_inv, mult, - CommutationMatrix(get_n_nodes(specification)), + CommutationMatrix(nvars(specification)), nothing, ) end @@ -249,7 +249,3 @@ function check_fiml(semfiml, model) a = cholesky!(Symmetric(semfiml.imp_inv); check = false) return isposdef(a) end - -get_n_nodes(specification::RAMMatrices) = specification.size_F[2] -get_n_nodes(specification::ParameterTable) = - length(specification.observed_vars) + length(specification.latent_vars) From 3161745d40a66fb4db79390ee4fa6a7dca5ec7ac Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 16 Jun 2024 20:55:29 -0700 Subject: [PATCH 075/364] get_data() -> samples() and add default implementation samples(::SemObserved) --- src/StructuralEquationModels.jl | 1 + src/frontend/fit/standard_errors/bootstrap.jl | 2 +- src/observed/abstract.jl | 10 ++++++++++ src/observed/data.jl | 3 +-- src/observed/missing.jl | 3 +-- test/unit_tests/data_input_formats.jl | 18 +++++++++--------- 6 files changed, 23 insertions(+), 14 deletions(-) create mode 100644 src/observed/abstract.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 69038c0d0..89439ee9d 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -40,6 +40,7 @@ include("frontend/fit/summary.jl") # pretty printing include("frontend/pretty_printing.jl") # observed +include("observed/abstract.jl") include("observed/covariance.jl") include("observed/data.jl") include("observed/missing.jl") diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index 814f46e59..9695e4cb3 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -25,7 +25,7 @@ function se_bootstrap( end if isnothing(data) - data = get_data(observed(model(semfit))) + data = samples(observed(model(semfit))) end data = prepare_data_bootstrap(data) diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl new file mode 100644 index 000000000..90de8b5a6 --- /dev/null +++ b/src/observed/abstract.jl @@ -0,0 +1,10 @@ +""" + samples(observed::SemObservedData) + +Gets the matrix of observed data samples. +Rows are samples, columns are observed variables. + +## See Also +[`nsamples`](@ref), [`observed_vars`](@ref). +""" +samples(observed::SemObserved) = observed.data diff --git a/src/observed/data.jl b/src/observed/data.jl index 8886c18b3..f1c0ff9b1 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -21,7 +21,7 @@ For observed data without missings. - `n_obs(::SemObservedData)` -> number of observed data points - `n_man(::SemObservedData)` -> number of manifest variables -- `get_data(::SemObservedData)` -> observed data +- `samples(::SemObservedData)` -> observed data - `obs_cov(::SemObservedData)` -> observed.obs_cov - `obs_mean(::SemObservedData)` -> observed.obs_mean - `data_rowwise(::SemObservedData)` -> observed data, stored as vectors per observation @@ -124,7 +124,6 @@ n_man(observed::SemObservedData) = observed.n_man ### additional methods ############################################################################################ -get_data(observed::SemObservedData) = observed.data obs_cov(observed::SemObservedData) = observed.obs_cov obs_mean(observed::SemObservedData) = observed.obs_mean data_rowwise(observed::SemObservedData) = observed.data_rowwise diff --git a/src/observed/missing.jl b/src/observed/missing.jl index af673becd..159a4915c 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -30,7 +30,7 @@ For observed data with missing values. - `n_obs(::SemObservedMissing)` -> number of observed data points - `n_man(::SemObservedMissing)` -> number of manifest variables -- `get_data(::SemObservedMissing)` -> observed data +- `samples(::SemObservedMissing)` -> observed data - `data_rowwise(::SemObservedMissing)` -> observed data as vector per observation, with missing values deleted - `patterns(::SemObservedMissing)` -> indices of non-missing variables per missing patterns @@ -211,7 +211,6 @@ n_man(observed::SemObservedMissing) = observed.n_man ### Additional methods ############################################################################################ -get_data(observed::SemObservedMissing) = observed.data patterns(observed::SemObservedMissing) = observed.patterns patterns_not(observed::SemObservedMissing) = observed.patterns_not rows(observed::SemObservedMissing) = observed.rows diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 44656c331..070c19317 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -1,5 +1,5 @@ using StructuralEquationModels, Test, Statistics -using StructuralEquationModels: obs_cov, obs_mean, get_data +using StructuralEquationModels: obs_cov, obs_mean, samples ### model specification -------------------------------------------------------------------- spec = ParameterTable( @@ -64,8 +64,8 @@ all_equal_cov = (obs_cov(observed) == obs_cov(observed_matrix)) all_equal_data = - (get_data(observed) == get_data(observed_nospec)) & - (get_data(observed) == get_data(observed_matrix)) + (samples(observed) == samples(observed_nospec)) & + (samples(observed) == samples(observed_matrix)) @testset "unit tests | SemObservedData | input formats" begin @test all_equal_cov @@ -94,8 +94,8 @@ all_equal_cov_suffled = (obs_cov(observed) == obs_cov(observed_matrix_shuffle)) all_equal_data_suffled = - (get_data(observed) == get_data(observed_shuffle)) & - (get_data(observed) == get_data(observed_matrix_shuffle)) + (samples(observed) == samples(observed_shuffle)) & + (samples(observed) == samples(observed_matrix_shuffle)) @testset "unit tests | SemObservedData | input formats shuffled " begin @test all_equal_cov_suffled @@ -396,8 +396,8 @@ observed_matrix = SemObservedMissing( ) all_equal_data = - isequal(get_data(observed), get_data(observed_nospec)) & - isequal(get_data(observed), get_data(observed_matrix)) + isequal(samples(observed), samples(observed_nospec)) & + isequal(samples(observed), samples(observed_matrix)) @testset "unit tests | SemObservedMissing | input formats" begin @test all_equal_data @@ -421,8 +421,8 @@ observed_matrix_shuffle = SemObservedMissing( ) all_equal_data_shuffled = - isequal(get_data(observed), get_data(observed_shuffle)) & - isequal(get_data(observed), get_data(observed_matrix_shuffle)) + isequal(samples(observed), samples(observed_shuffle)) & + isequal(samples(observed), samples(observed_matrix_shuffle)) @testset "unit tests | SemObservedMissing | input formats shuffled " begin @test all_equal_data_suffled From 08044e1c9f6a8d62a8a15a2cb6b11c1e4821f01d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 18 Apr 2024 21:46:58 -0700 Subject: [PATCH 076/364] SemObsData: remove rowwise * it is unused * if ever rowwise access would be required, it could be done with eachrow(data) without allocation --- src/observed/data.jl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/observed/data.jl b/src/observed/data.jl index f1c0ff9b1..7573b9746 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -24,7 +24,6 @@ For observed data without missings. - `samples(::SemObservedData)` -> observed data - `obs_cov(::SemObservedData)` -> observed.obs_cov - `obs_mean(::SemObservedData)` -> observed.obs_mean -- `data_rowwise(::SemObservedData)` -> observed data, stored as vectors per observation ## Implementation Subtype of `SemObserved` @@ -37,15 +36,13 @@ use this if you are sure your observed data is in the right format. ## Additional keyword arguments: - `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object - `compute_covariance::Bool ) = true`: should the covariance of `data` be computed and stored? -- `rowwise::Bool = false`: should the data be stored also as vectors per observation """ -struct SemObservedData{A, B, C, R} <: SemObserved +struct SemObservedData{A, B, C} <: SemObserved data::A obs_cov::B obs_mean::C n_man::Int n_obs::Int - data_rowwise::R end # error checks @@ -61,7 +58,6 @@ function SemObservedData(; spec_colnames = nothing, meanstructure = false, compute_covariance = true, - rowwise = false, kwargs..., ) if isnothing(spec_colnames) && !isnothing(specification) @@ -109,7 +105,6 @@ function SemObservedData(; meanstructure ? vec(Statistics.mean(data, dims = 1)) : nothing, size(data, 2), size(data, 1), - rowwise ? [data[i, :] for i in axes(data, 1)] : nothing, ) end @@ -126,7 +121,6 @@ n_man(observed::SemObservedData) = observed.n_man obs_cov(observed::SemObservedData) = observed.obs_cov obs_mean(observed::SemObservedData) = observed.obs_mean -data_rowwise(observed::SemObservedData) = observed.data_rowwise ############################################################################################ ### Additional functions From 5c0c588b9f271591424b2f57131fab44bb7cee34 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 16 Jun 2024 20:39:53 -0700 Subject: [PATCH 077/364] AbstractSemSingle: vars API --- src/frontend/specification/Sem.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 73d4e81da..14f4d5a78 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -20,6 +20,14 @@ function Sem(; return sem end +nvars(sem::AbstractSemSingle) = nvars(sem.imply) +nobserved_vars(sem::AbstractSemSingle) = nobserved_vars(sem.imply) +nlatent_vars(sem::AbstractSemSingle) = nlatent_vars(sem.imply) + +vars(sem::AbstractSemSingle) = vars(sem.imply) +observed_vars(sem::AbstractSemSingle) = observed_vars(sem.imply) +latent_vars(sem::AbstractSemSingle) = latent_vars(sem.imply) + function SemFiniteDiff(; observed::O = SemObservedData, imply::I = RAM, From 913964be1e88287ab2ec6c1740185a7582825f3c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 16 Jun 2024 20:55:56 -0700 Subject: [PATCH 078/364] rename n_obs() -> nsamples() --- src/StructuralEquationModels.jl | 3 +- src/frontend/common.jl | 11 +++++++- src/frontend/fit/SemFit.jl | 1 + src/frontend/fit/fitmeasures/BIC.jl | 2 +- src/frontend/fit/fitmeasures/RMSEA.jl | 8 +++--- src/frontend/fit/fitmeasures/chi2.jl | 8 +++--- src/frontend/fit/fitmeasures/minus2ll.jl | 20 ++++++------- src/frontend/fit/fitmeasures/n_obs.jl | 16 ----------- src/frontend/fit/standard_errors/hessian.jl | 8 +++--- src/frontend/fit/summary.jl | 2 +- src/frontend/specification/Sem.jl | 5 ++++ src/loss/ML/FIML.jl | 14 +++++----- src/observed/EM.jl | 19 +++++++------ src/observed/covariance.jl | 14 +++++----- src/observed/data.jl | 6 ++-- src/observed/missing.jl | 26 ++++++++--------- src/types.jl | 4 +-- test/examples/political_democracy/by_parts.jl | 4 +-- .../political_democracy/constructor.jl | 8 +++--- test/unit_tests/data_input_formats.jl | 28 +++++++++---------- 20 files changed, 103 insertions(+), 104 deletions(-) delete mode 100644 src/frontend/fit/fitmeasures/n_obs.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 89439ee9d..1f69b2e8e 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -82,7 +82,6 @@ include("frontend/fit/fitmeasures/BIC.jl") include("frontend/fit/fitmeasures/chi2.jl") include("frontend/fit/fitmeasures/df.jl") include("frontend/fit/fitmeasures/minus2ll.jl") -include("frontend/fit/fitmeasures/n_obs.jl") include("frontend/fit/fitmeasures/p.jl") include("frontend/fit/fitmeasures/RMSEA.jl") include("frontend/fit/fitmeasures/n_man.jl") @@ -165,7 +164,7 @@ export AbstractSem, df, fit_measures, minus2ll, - n_obs, + nsamples, p_value, RMSEA, n_man, diff --git a/src/frontend/common.jl b/src/frontend/common.jl index c55acf1d1..2be13c113 100644 --- a/src/frontend/common.jl +++ b/src/frontend/common.jl @@ -45,4 +45,13 @@ parind[:param_name] See also [`params`](@ref). """ -param_indices(semobj) = Dict(par => i for (i, par) in enumerate(params(semobj))) \ No newline at end of file +param_indices(semobj) = Dict(par => i for (i, par) in enumerate(params(semobj))) + +""" + nsamples(semobj) + +Return the number of samples (observed data points). + +For ensemble models, return the sum over all submodels. +""" +function nsamples end diff --git a/src/frontend/fit/SemFit.jl b/src/frontend/fit/SemFit.jl index ace9ed320..84d2f502c 100644 --- a/src/frontend/fit/SemFit.jl +++ b/src/frontend/fit/SemFit.jl @@ -48,6 +48,7 @@ end params(fit::SemFit) = params(fit.model) nparams(fit::SemFit) = nparams(fit.model) +nsamples(fit::SemFit) = nsamples(fit.model) # access fields minimum(sem_fit::SemFit) = sem_fit.minimum diff --git a/src/frontend/fit/fitmeasures/BIC.jl b/src/frontend/fit/fitmeasures/BIC.jl index 47bd12f1b..20638f4e4 100644 --- a/src/frontend/fit/fitmeasures/BIC.jl +++ b/src/frontend/fit/fitmeasures/BIC.jl @@ -3,4 +3,4 @@ Return the bayesian information criterion. """ -BIC(sem_fit::SemFit) = minus2ll(sem_fit) + log(n_obs(sem_fit)) * nparams(sem_fit) +BIC(sem_fit::SemFit) = minus2ll(sem_fit) + log(nsamples(sem_fit)) * nparams(sem_fit) diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index 3b3eb384b..b91e81d3e 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -6,13 +6,13 @@ Return the RMSEA. function RMSEA end RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) = - RMSEA(df(sem_fit), χ²(sem_fit), n_obs(sem_fit)) + RMSEA(df(sem_fit), χ²(sem_fit), nsamples(sem_fit)) RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}) = - sqrt(length(sem_fit.model.sems)) * RMSEA(df(sem_fit), χ²(sem_fit), n_obs(sem_fit)) + sqrt(length(sem_fit.model.sems)) * RMSEA(df(sem_fit), χ²(sem_fit), nsamples(sem_fit)) -function RMSEA(df, chi2, n_obs) - rmsea = (chi2 - df) / (n_obs * df) +function RMSEA(df, chi2, nsamples) + rmsea = (chi2 - df) / (nsamples * df) rmsea > 0 ? nothing : rmsea = 0 return sqrt(rmsea) end diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index 51fe6f0cd..2abebd968 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -20,11 +20,11 @@ function χ² end # RAM + SemML χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) = - (n_obs(sem_fit) - 1) * (sem_fit.minimum - logdet(observed.obs_cov) - observed.n_man) + (nsamples(sem_fit) - 1) * (sem_fit.minimum - logdet(observed.obs_cov) - observed.n_man) # bollen, p. 115, only correct for GLS weight matrix χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) = - (n_obs(sem_fit) - 1) * sem_fit.minimum + (nsamples(sem_fit) - 1) * sem_fit.minimum # FIML function χ²(sem_fit::SemFit, observed::SemObservedMissing, imp, optimizer, loss_ml::SemFIML) @@ -45,7 +45,7 @@ end function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemWLS} check_ensemble_length(model) check_lossfun_types(model, L) - return (sum(n_obs.(model.sems)) - 1) * sem_fit.minimum + return (nsamples(model) - 1) * sem_fit.minimum end function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemML} @@ -56,7 +56,7 @@ function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemML w * (logdet(m.observed.obs_cov) + m.observed.n_man) for (w, m) in zip(model.weights, model.sems) ]) - return (sum(n_obs.(model.sems)) - 1) * F_G + return (nsamples(model) - 1) * F_G end function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemFIML} diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index c984555b3..67d69bbed 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -25,7 +25,7 @@ minus2ll(sem_fit::SemFit, obs, imp, optimizer, args...) = # SemML ------------------------------------------------------------------------------------ minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) = - n_obs(obs) * (minimum + log(2π) * n_man(obs)) + nsamples(obs) * (minimum + log(2π) * n_man(obs)) # WLS -------------------------------------------------------------------------------------- minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) = @@ -41,8 +41,8 @@ function minus2ll( loss_ml::SemFIML, ) F = minimum - F *= n_obs(observed) - F += sum(log(2π) * observed.pattern_n_obs .* observed.pattern_nvar_obs) + F *= nsamples(observed) + F += sum(log(2π) * observed.pattern_nsamples .* observed.pattern_nvar_obs) return F end @@ -53,12 +53,12 @@ function minus2ll(observed::SemObservedMissing) minus2ll( observed.em_model.μ, observed.em_model.Σ, - observed.n_obs, + nsamples(observed), observed.rows, observed.patterns, observed.obs_mean, observed.obs_cov, - observed.pattern_n_obs, + observed.pattern_nsamples, observed.pattern_nvar_obs, ) else @@ -66,12 +66,12 @@ function minus2ll(observed::SemObservedMissing) minus2ll( observed.em_model.μ, observed.em_model.Σ, - observed.n_obs, + nsamples(observed), observed.rows, observed.patterns, observed.obs_mean, observed.obs_cov, - observed.pattern_n_obs, + observed.pattern_nsamples, observed.pattern_nvar_obs, ) end @@ -85,13 +85,13 @@ function minus2ll( patterns, obs_mean, obs_cov, - pattern_n_obs, + pattern_nsamples, pattern_nvar_obs, ) F = 0.0 for i in 1:length(rows) - nᵢ = pattern_n_obs[i] + nᵢ = pattern_nsamples[i] # missing pattern pattern = patterns[i] # observed data @@ -106,7 +106,7 @@ function minus2ll( F += F_one_pattern(meandiffᵢ, Σᵢ⁻¹, Sᵢ, ld, nᵢ) end - F += sum(log(2π) * pattern_n_obs .* pattern_nvar_obs) + F += sum(log(2π) * pattern_nsamples .* pattern_nvar_obs) #F *= N return F diff --git a/src/frontend/fit/fitmeasures/n_obs.jl b/src/frontend/fit/fitmeasures/n_obs.jl deleted file mode 100644 index cd4bdca30..000000000 --- a/src/frontend/fit/fitmeasures/n_obs.jl +++ /dev/null @@ -1,16 +0,0 @@ -""" - n_obs(sem_fit::SemFit) - n_obs(model::AbstractSemSingle) - n_obs(model::SemEnsemble) - -Return the number of observed data points. - -For ensemble models, return the sum over all submodels. -""" -function n_obs end - -n_obs(sem_fit::SemFit) = n_obs(sem_fit.model) - -n_obs(model::AbstractSemSingle) = n_obs(model.observed) - -n_obs(model::SemEnsemble) = sum(n_obs, model.sems) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index 396d3b98c..afcb570bc 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -46,13 +46,13 @@ end H_scaling(model::AbstractSemSingle) = H_scaling(model, model.observed, model.imply, model.optimizer, model.loss.functions...) -H_scaling(model, obs, imp, optimizer, lossfun::SemML) = 2 / (n_obs(model) - 1) +H_scaling(model, obs, imp, optimizer, lossfun::SemML) = 2 / (nsamples(model) - 1) function H_scaling(model, obs, imp, optimizer, lossfun::SemWLS) @warn "Standard errors for WLS are only correct if a GLS weight matrix (the default) is used." - return 2 / (n_obs(model) - 1) + return 2 / (nsamples(model) - 1) end -H_scaling(model, obs, imp, optimizer, lossfun::SemFIML) = 2 / n_obs(model) +H_scaling(model, obs, imp, optimizer, lossfun::SemFIML) = 2 / nsamples(model) -H_scaling(model::SemEnsemble) = 2 / n_obs(model) +H_scaling(model::SemEnsemble) = 2 / nsamples(model) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 621791211..507e835fc 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -17,7 +17,7 @@ function sem_summary( println("No. iterations/evaluations: $(n_iterations(sem_fit))") print("\n") println("Number of parameters: $(nparams(sem_fit))") - println("Number of observations: $(n_obs(sem_fit))") + println("Number of data samples: $(nsamples(sem_fit))") print("\n") printstyled( "----------------------------------- Model ----------------------------------- \n"; diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 14f4d5a78..1befb9aad 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -28,6 +28,11 @@ vars(sem::AbstractSemSingle) = vars(sem.imply) observed_vars(sem::AbstractSemSingle) = observed_vars(sem.imply) latent_vars(sem::AbstractSemSingle) = latent_vars(sem.imply) +nsamples(sem::AbstractSemSingle) = nsamples(sem.observed) + +# sum of samples in all sub-models +nsamples(ensemble::SemEnsemble) = sum(nsamples, ensemble.sems) + function SemFiniteDiff(; observed::O = SemObservedData, imply::I = RAM, diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 135bb411b..6ff8e4f04 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -90,7 +90,7 @@ function objective!(semfiml::SemFIML, params, model) prepare_SemFIML!(semfiml, model) objective = F_FIML(rows(observed(model)), semfiml, model, params) - return objective / n_obs(observed(model)) + return objective / nsamples(observed(model)) end function gradient!(semfiml::SemFIML, params, model) @@ -100,7 +100,7 @@ function gradient!(semfiml::SemFIML, params, model) prepare_SemFIML!(semfiml, model) - gradient = ∇F_FIML(rows(observed(model)), semfiml, model) / n_obs(observed(model)) + gradient = ∇F_FIML(rows(observed(model)), semfiml, model) / nsamples(observed(model)) return gradient end @@ -112,8 +112,8 @@ function objective_gradient!(semfiml::SemFIML, params, model) prepare_SemFIML!(semfiml, model) objective = - F_FIML(rows(observed(model)), semfiml, model, params) / n_obs(observed(model)) - gradient = ∇F_FIML(rows(observed(model)), semfiml, model) / n_obs(observed(model)) + F_FIML(rows(observed(model)), semfiml, model, params) / nsamples(observed(model)) + gradient = ∇F_FIML(rows(observed(model)), semfiml, model) / nsamples(observed(model)) return objective, gradient end @@ -182,7 +182,7 @@ function F_FIML(rows, semfiml, model, params) semfiml.inverses[i], obs_cov(observed(model))[i], semfiml.logdets[i], - pattern_n_obs(observed(model))[i], + pattern_nsamples(observed(model))[i], ) end return F @@ -199,7 +199,7 @@ function ∇F_FIML(rows, semfiml, model) obs_cov(observed(model))[i], patterns(observed(model))[i], semfiml.∇ind[i], - pattern_n_obs(observed(model))[i], + pattern_nsamples(observed(model))[i], Jμ, JΣ, model, @@ -213,7 +213,7 @@ function prepare_SemFIML!(semfiml, model) batch_cholesky!(semfiml, model) #batch_sym_inv_update!(semfiml, model) batch_inv!(semfiml, model) - for i in 1:size(pattern_n_obs(observed(model)), 1) + for i in 1:size(pattern_nsamples(observed(model)), 1) semfiml.meandiff[i] .= obs_mean(observed(model))[i] - semfiml.imp_mean[i] end end diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 09dfbd82e..4640a7137 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -29,7 +29,8 @@ function em_mvn( rtol_em = 1e-4, kwargs..., ) - n_obs, n_man = observed.n_obs, Int(observed.n_man) + n_man = observed.n_man + nsamps = nsamples(observed) # preallocate stuff? 𝔼x_pre = zeros(n_man) @@ -44,8 +45,8 @@ function em_mvn( end end - # ess = 𝔼x, 𝔼xxᵀ, ismissing, missingRows, n_obs - # estepFn = (em_model, data) -> estep(em_model, data, EXsum, EXXsum, ismissing, missingRows, n_obs) + # ess = 𝔼x, 𝔼xxᵀ, ismissing, missingRows, nsamps + # estepFn = (em_model, data) -> estep(em_model, data, EXsum, EXXsum, ismissing, missingRows, nsamps) # initialize em_model = start_em(observed; kwargs...) @@ -57,7 +58,7 @@ function em_mvn( while !done em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xxᵀ_pre) - em_mvn_Mstep!(em_model, n_obs, 𝔼x, 𝔼xxᵀ) + em_mvn_Mstep!(em_model, nsamps, 𝔼x, 𝔼xxᵀ) if iter > max_iter_em done = true @@ -96,7 +97,7 @@ function em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xx Σ = em_model.Σ # Compute the expected sufficient statistics - for i in 2:length(observed.pattern_n_obs) + for i in 2:length(observed.pattern_nsamples) # observed and unobserved vars u = observed.patterns_not[i] @@ -125,9 +126,9 @@ function em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xx 𝔼xxᵀ .+= 𝔼xxᵀ_pre end -function em_mvn_Mstep!(em_model, n_obs, 𝔼x, 𝔼xxᵀ) - em_model.μ = 𝔼x / n_obs - Σ = Symmetric(𝔼xxᵀ / n_obs - em_model.μ * em_model.μ') +function em_mvn_Mstep!(em_model, nsamples, 𝔼x, 𝔼xxᵀ) + em_model.μ = 𝔼x / nsamples + Σ = Symmetric(𝔼xxᵀ / nsamples - em_model.μ * em_model.μ') # ridge Σ # while !isposdef(Σ) @@ -152,7 +153,7 @@ end # use μ and Σ of full cases function start_em_observed(observed::SemObservedMissing; kwargs...) - if (length(observed.patterns[1]) == observed.n_man) & (observed.pattern_n_obs[1] > 1) + if (length(observed.patterns[1]) == observed.n_man) & (observed.pattern_nsamples[1] > 1) μ = copy(observed.obs_mean[1]) Σ = copy(Symmetric(observed.obs_cov[1])) if !isposdef(Σ) diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 28430263f..a3ff822a3 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -9,7 +9,7 @@ For observed covariance matrices and means. obs_colnames = nothing, meanstructure = false, obs_mean = nothing, - n_obs = nothing, + nsamples = nothing, kwargs...) # Arguments @@ -18,11 +18,11 @@ For observed covariance matrices and means. - `obs_colnames::Vector{Symbol}`: column names of the covariance matrix - `meanstructure::Bool`: does the model have a meanstructure? - `obs_mean`: observed mean vector -- `n_obs::Number`: number of observed data points (necessary for fit statistics) +- `nsamples::Number`: number of samples (observed data points); necessary for fit statistics # Extended help ## Interfaces -- `n_obs(::SemObservedCovariance)` -> number of observed data points +- `nsamples(::SemObservedCovariance)`: number of samples (observed data points) - `n_man(::SemObservedCovariance)` -> number of manifest variables - `obs_cov(::SemObservedCovariance)` -> observed covariance matrix @@ -43,7 +43,7 @@ struct SemObservedCovariance{B, C} <: SemObserved obs_cov::B obs_mean::C n_man::Int - n_obs::Int + nsamples::Int end function SemObservedCovariance(; @@ -53,7 +53,7 @@ function SemObservedCovariance(; spec_colnames = nothing, obs_mean = nothing, meanstructure = false, - n_obs::Integer, + nsamples::Integer, kwargs..., ) if !meanstructure & !isnothing(obs_mean) @@ -80,14 +80,14 @@ function SemObservedCovariance(; (obs_mean = reorder_obs_mean(obs_mean, spec_colnames, obs_colnames)) end - return SemObservedCovariance(obs_cov, obs_mean, size(obs_cov, 1), n_obs) + return SemObservedCovariance(obs_cov, obs_mean, size(obs_cov, 1), nsamples) end ############################################################################################ ### Recommended methods ############################################################################################ -n_obs(observed::SemObservedCovariance) = observed.n_obs +nsamples(observed::SemObservedCovariance) = observed.nsamples n_man(observed::SemObservedCovariance) = observed.n_man ############################################################################################ diff --git a/src/observed/data.jl b/src/observed/data.jl index 7573b9746..c0a42d9c2 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -18,7 +18,7 @@ For observed data without missings. # Extended help ## Interfaces -- `n_obs(::SemObservedData)` -> number of observed data points +- `nsamples(::SemObservedData)` -> number of observed data points - `n_man(::SemObservedData)` -> number of manifest variables - `samples(::SemObservedData)` -> observed data @@ -42,7 +42,7 @@ struct SemObservedData{A, B, C} <: SemObserved obs_cov::B obs_mean::C n_man::Int - n_obs::Int + nsamples::Int end # error checks @@ -112,7 +112,7 @@ end ### Recommended methods ############################################################################################ -n_obs(observed::SemObservedData) = observed.n_obs +nsamples(observed::SemObservedData) = observed.nsamples n_man(observed::SemObservedData) = observed.n_man ############################################################################################ diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 159a4915c..6a78b5161 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -27,7 +27,7 @@ For observed data with missing values. # Extended help ## Interfaces -- `n_obs(::SemObservedMissing)` -> number of observed data points +- `nsamples(::SemObservedMissing)` -> number of observed data points - `n_man(::SemObservedMissing)` -> number of manifest variables - `samples(::SemObservedMissing)` -> observed data @@ -36,7 +36,7 @@ For observed data with missing values. - `patterns(::SemObservedMissing)` -> indices of non-missing variables per missing patterns - `patterns_not(::SemObservedMissing)` -> indices of missing variables per missing pattern - `rows(::SemObservedMissing)` -> row indices of observed data points that belong to each pattern -- `pattern_n_obs(::SemObservedMissing)` -> number of data points per pattern +- `pattern_nsamples(::SemObservedMissing)` -> number of data points per pattern - `pattern_nvar_obs(::SemObservedMissing)` -> number of non-missing observed variables per pattern - `obs_mean(::SemObservedMissing)` -> observed mean per pattern - `obs_cov(::SemObservedMissing)` -> observed covariance per pattern @@ -56,7 +56,7 @@ use this if you are sure your observed data is in the right format. mutable struct SemObservedMissing{ A <: AbstractArray, D <: AbstractFloat, - O <: AbstractFloat, + O <: Number, P <: Vector, P2 <: Vector, R <: Vector, @@ -69,12 +69,12 @@ mutable struct SemObservedMissing{ } <: SemObserved data::A n_man::D - n_obs::O + nsamples::O patterns::P # missing patterns patterns_not::P2 rows::R # coresponding rows in data_rowwise data_rowwise::PD # list of data - pattern_n_obs::PO # observed rows per pattern + pattern_nsamples::PO # observed rows per pattern pattern_nvar_obs::PVO # number of non-missing variables per pattern obs_mean::A2 obs_cov::A3 @@ -140,14 +140,14 @@ function SemObservedMissing(; end data = data[keep, :] - n_obs, n_man = size(data) + nsamples, n_man = size(data) # compute and store the different missing patterns with their rowindices missings = ismissing.(data) patterns = [missings[i, :] for i in 1:size(missings, 1)] patterns_cart = findall.(!, patterns) - data_rowwise = [data[i, patterns_cart[i]] for i in 1:n_obs] + data_rowwise = [data[i, patterns_cart[i]] for i in 1:nsamples] data_rowwise = convert.(Array{Float64}, data_rowwise) remember = Vector{BitArray{1}}() @@ -175,7 +175,7 @@ function SemObservedMissing(; remember_cart_not = findall.(remember) rows = rows[sort_n_miss] - pattern_n_obs = size.(rows, 1) + pattern_nsamples = size.(rows, 1) pattern_nvar_obs = length.(remember_cart) cov_mean = [cov_and_mean(data_rowwise[rows]) for rows in rows] @@ -186,13 +186,13 @@ function SemObservedMissing(; return SemObservedMissing( data, - Float64(n_man), - Float64(n_obs), + Float64(nobs_vars), + nsamples, remember_cart, remember_cart_not, rows, data_rowwise, - Float64.(pattern_n_obs), + pattern_nsamples, Float64.(pattern_nvar_obs), obs_mean, obs_cov, @@ -204,7 +204,7 @@ end ### Recommended methods ############################################################################################ -n_obs(observed::SemObservedMissing) = observed.n_obs +nsamples(observed::SemObservedMissing) = observed.nsamples n_man(observed::SemObservedMissing) = observed.n_man ############################################################################################ @@ -215,7 +215,7 @@ patterns(observed::SemObservedMissing) = observed.patterns patterns_not(observed::SemObservedMissing) = observed.patterns_not rows(observed::SemObservedMissing) = observed.rows data_rowwise(observed::SemObservedMissing) = observed.data_rowwise -pattern_n_obs(observed::SemObservedMissing) = observed.pattern_n_obs +pattern_nsamples(observed::SemObservedMissing) = observed.pattern_nsamples pattern_nvar_obs(observed::SemObservedMissing) = observed.pattern_nvar_obs obs_mean(observed::SemObservedMissing) = observed.obs_mean obs_cov(observed::SemObservedMissing) = observed.obs_cov diff --git a/src/types.jl b/src/types.jl index 99153622e..98d5f87ec 100644 --- a/src/types.jl +++ b/src/types.jl @@ -176,8 +176,8 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing # default weights if isnothing(weights) - nobs_total = sum(n_obs, models) - weights = [n_obs(model) / nobs_total for model in models] + nsamples_total = sum(nsamples, models) + weights = [nsamples(model) / nsamples_total for model in models] end # check parameters equality diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 11953ccb6..09d40cc28 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -41,7 +41,7 @@ model_ridge = Sem(observed, imply_ram, SemLoss(ml, ridge), optimizer_obj) model_constant = Sem(observed, imply_ram, SemLoss(ml, constant), optimizer_obj) model_ml_weighted = - Sem(observed, imply_ram, SemLoss(ml; loss_weights = [n_obs(model_ml)]), optimizer_obj) + Sem(observed, imply_ram, SemLoss(ml; loss_weights = [nsamples(model_ml)]), optimizer_obj) ############################################################################################ ### test gradients @@ -101,7 +101,7 @@ end solution_ml = sem_fit(model_ml) solution_ml_weighted = sem_fit(model_ml_weighted) @test solution(solution_ml) ≈ solution(solution_ml_weighted) rtol = 1e-3 - @test n_obs(model_ml) * StructuralEquationModels.minimum(solution_ml) ≈ + @test nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml) ≈ StructuralEquationModels.minimum(solution_ml_weighted) rtol = 1e-6 end diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 5f1c838e8..bf674dd73 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -12,7 +12,7 @@ model_ml_cov = Sem( obs_cov = cov(Matrix(dat)), obs_colnames = Symbol.(names(dat)), optimizer = semoptimizer, - n_obs = 75, + nsamples = 75, ) model_ls_sym = Sem( @@ -46,7 +46,7 @@ model_constant = Sem( model_ml_weighted = Sem( specification = partable, data = dat, - loss_weights = (n_obs(model_ml),), + loss_weights = (nsamples(model_ml),), optimizer = semoptimizer, ) @@ -116,7 +116,7 @@ end solution_ml_weighted = sem_fit(model_ml_weighted) @test isapprox(solution(solution_ml), solution(solution_ml_weighted), rtol = 1e-3) @test isapprox( - n_obs(model_ml) * StructuralEquationModels.minimum(solution_ml), + nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml), StructuralEquationModels.minimum(solution_ml_weighted), rtol = 1e-6, ) @@ -244,7 +244,7 @@ model_ml_cov = Sem( obs_colnames = Symbol.(names(dat)), meanstructure = true, optimizer = semoptimizer, - n_obs = 75, + nsamples = 75, ) model_ml_sym = Sem( diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 070c19317..f1adaf625 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -205,14 +205,14 @@ end specification = nothing, obs_cov = dat_cov, obs_mean = dat_mean, - n_obs = 75, + nsamples = 75, ) end -@test_throws UndefKeywordError(:n_obs) SemObservedCovariance(obs_cov = dat_cov) +@test_throws UndefKeywordError(:nsamples) SemObservedCovariance(obs_cov = dat_cov) @test_throws ArgumentError("no `obs_colnames` were specified") begin - SemObservedCovariance(specification = spec, obs_cov = dat_cov, n_obs = 75) + SemObservedCovariance(specification = spec, obs_cov = dat_cov, nsamples = 75) end @test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin @@ -220,7 +220,7 @@ end specification = spec, obs_cov = dat_cov, obs_colnames = names(dat), - n_obs = 75, + nsamples = 75, ) end @@ -229,18 +229,18 @@ observed = SemObservedCovariance( specification = spec, obs_cov = dat_cov, obs_colnames = obs_colnames = Symbol.(names(dat)), - n_obs = 75, + nsamples = 75, ) observed_nospec = - SemObservedCovariance(specification = nothing, obs_cov = dat_cov, n_obs = 75) + SemObservedCovariance(specification = nothing, obs_cov = dat_cov, nsamples = 75) all_equal_cov = (obs_cov(observed) == obs_cov(observed_nospec)) @testset "unit tests | SemObservedCovariance | input formats" begin @test all_equal_cov - @test n_obs(observed) == 75 - @test n_obs(observed_nospec) == 75 + @test nsamples(observed) == 75 + @test nsamples(observed_nospec) == 75 end # shuffle variables @@ -256,7 +256,7 @@ observed_shuffle = SemObservedCovariance( specification = spec, obs_cov = shuffle_dat_cov, obs_colnames = shuffle_names, - n_obs = 75, + nsamples = 75, ) all_equal_cov_suffled = (obs_cov(observed) ≈ obs_cov(observed_shuffle)) @@ -273,7 +273,7 @@ end specification = spec, obs_cov = dat_cov, meanstructure = true, - n_obs = 75, + nsamples = 75, ) end @@ -293,7 +293,7 @@ end obs_cov = dat_cov, obs_colnames = Symbol.(names(dat)), meanstructure = true, - n_obs = 75, + nsamples = 75, ) end @@ -303,7 +303,7 @@ observed = SemObservedCovariance( obs_cov = dat_cov, obs_mean = dat_mean, obs_colnames = Symbol.(names(dat)), - n_obs = 75, + nsamples = 75, meanstructure = true, ) @@ -312,7 +312,7 @@ observed_nospec = SemObservedCovariance( obs_cov = dat_cov, obs_mean = dat_mean, meanstructure = true, - n_obs = 75, + nsamples = 75, ) all_equal_mean = (obs_mean(observed) == obs_mean(observed_nospec)) @@ -338,7 +338,7 @@ observed_shuffle = SemObservedCovariance( obs_cov = shuffle_dat_cov, obs_mean = shuffle_dat_mean, obs_colnames = shuffle_names, - n_obs = 75, + nsamples = 75, meanstructure = true, ) From 35c0466eb1540f9316f136f5c6b798064516286b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 25 Jun 2024 17:43:36 -0700 Subject: [PATCH 079/364] rename n_man() -> nobserved_vars() for missing data pattern: nobserved_vars() -> nmeasured_vars(), obs_cov/obs_mean -> measured_cov/measured_mean --- src/StructuralEquationModels.jl | 2 -- src/frontend/fit/fitmeasures/chi2.jl | 5 +++-- src/frontend/fit/fitmeasures/df.jl | 6 +++--- src/frontend/fit/fitmeasures/minus2ll.jl | 12 ++++++------ src/frontend/fit/fitmeasures/n_man.jl | 11 ----------- src/imply/RAM/generic.jl | 6 +++--- src/imply/RAM/symbolic.jl | 2 +- src/loss/ML/FIML.jl | 16 ++++++++-------- src/loss/WLS/WLS.jl | 2 +- src/observed/EM.jl | 24 ++++++++++++------------ src/observed/covariance.jl | 4 ++-- src/observed/data.jl | 6 +++--- src/observed/missing.jl | 24 ++++++++++++------------ 13 files changed, 54 insertions(+), 66 deletions(-) delete mode 100644 src/frontend/fit/fitmeasures/n_man.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 1f69b2e8e..6d2a82823 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -84,7 +84,6 @@ include("frontend/fit/fitmeasures/df.jl") include("frontend/fit/fitmeasures/minus2ll.jl") include("frontend/fit/fitmeasures/p.jl") include("frontend/fit/fitmeasures/RMSEA.jl") -include("frontend/fit/fitmeasures/n_man.jl") include("frontend/fit/fitmeasures/fit_measures.jl") # standard errors include("frontend/fit/standard_errors/hessian.jl") @@ -167,7 +166,6 @@ export AbstractSem, nsamples, p_value, RMSEA, - n_man, EmMVNModel, se_hessian, se_bootstrap, diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index 2abebd968..df1027bd6 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -20,7 +20,8 @@ function χ² end # RAM + SemML χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) = - (nsamples(sem_fit) - 1) * (sem_fit.minimum - logdet(observed.obs_cov) - observed.n_man) + (nsamples(sem_fit) - 1) * + (sem_fit.minimum - logdet(observed.obs_cov) - nobserved_vars(observed)) # bollen, p. 115, only correct for GLS weight matrix χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) = @@ -53,7 +54,7 @@ function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemML check_lossfun_types(model, L) F_G = sem_fit.minimum F_G -= sum([ - w * (logdet(m.observed.obs_cov) + m.observed.n_man) for + w * (logdet(m.observed.obs_cov) + nobserved_vars(m.observed)) for (w, m) in zip(model.weights, model.sems) ]) return (nsamples(model) - 1) * F_G diff --git a/src/frontend/fit/fitmeasures/df.jl b/src/frontend/fit/fitmeasures/df.jl index d4a4376dd..e8e72d594 100644 --- a/src/frontend/fit/fitmeasures/df.jl +++ b/src/frontend/fit/fitmeasures/df.jl @@ -11,10 +11,10 @@ df(sem_fit::SemFit) = df(sem_fit.model) df(model::AbstractSem) = n_dp(model) - nparams(model) function n_dp(model::AbstractSemSingle) - nman = n_man(model) - ndp = 0.5(nman^2 + nman) + nvars = nobserved_vars(model) + ndp = 0.5(nvars^2 + nvars) if !isnothing(model.imply.μ) - ndp += n_man(model) + ndp += nvars end return ndp end diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 67d69bbed..bfd161142 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -25,7 +25,7 @@ minus2ll(sem_fit::SemFit, obs, imp, optimizer, args...) = # SemML ------------------------------------------------------------------------------------ minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) = - nsamples(obs) * (minimum + log(2π) * n_man(obs)) + nsamples(obs) * (minimum + log(2π) * nobserved_vars(obs)) # WLS -------------------------------------------------------------------------------------- minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) = @@ -42,7 +42,7 @@ function minus2ll( ) F = minimum F *= nsamples(observed) - F += sum(log(2π) * observed.pattern_nsamples .* observed.pattern_nvar_obs) + F += sum(log(2π) * observed.pattern_nsamples .* observed.pattern_nobs_vars) return F end @@ -59,7 +59,7 @@ function minus2ll(observed::SemObservedMissing) observed.obs_mean, observed.obs_cov, observed.pattern_nsamples, - observed.pattern_nvar_obs, + observed.pattern_nobs_vars, ) else em_mvn(observed) @@ -72,7 +72,7 @@ function minus2ll(observed::SemObservedMissing) observed.obs_mean, observed.obs_cov, observed.pattern_nsamples, - observed.pattern_nvar_obs, + observed.pattern_nobs_vars, ) end end @@ -86,7 +86,7 @@ function minus2ll( obs_mean, obs_cov, pattern_nsamples, - pattern_nvar_obs, + pattern_nobs_vars, ) F = 0.0 @@ -106,7 +106,7 @@ function minus2ll( F += F_one_pattern(meandiffᵢ, Σᵢ⁻¹, Sᵢ, ld, nᵢ) end - F += sum(log(2π) * pattern_nsamples .* pattern_nvar_obs) + F += sum(log(2π) * pattern_nsamples .* pattern_nobs_vars) #F *= N return F diff --git a/src/frontend/fit/fitmeasures/n_man.jl b/src/frontend/fit/fitmeasures/n_man.jl deleted file mode 100644 index 45a7d99de..000000000 --- a/src/frontend/fit/fitmeasures/n_man.jl +++ /dev/null @@ -1,11 +0,0 @@ -""" - n_man(sem_fit::SemFit) - n_man(model::AbstractSemSingle) - -Return the number of manifest variables. -""" -function n_man end - -n_man(sem_fit::SemFit) = n_man(sem_fit.model) - -n_man(model::AbstractSemSingle) = n_man(model.observed) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 8ace80759..e93f5c5c2 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -111,7 +111,7 @@ function RAM(; n_obs = nobserved_vars(ram_matrices) n_var = nvars(ram_matrices) F = zeros(ram_matrices.size_F) - F[CartesianIndex.(1:n_var, ram_matrices.F_ind)] .= 1.0 + F[CartesianIndex.(1:n_obs, ram_matrices.F_ind)] .= 1.0 # get indices A_indices = copy(ram_matrices.A_ind) @@ -146,7 +146,7 @@ function RAM(; has_meanstructure = Val(true) !isnothing(M_indices) || throw(ArgumentError("You set `meanstructure = true`, but your model specification contains no mean parameters.")) ∇M = gradient ? matrix_gradient(M_indices, n_var) : nothing - μ = zeros(n_var) + μ = zeros(n_obs) else has_meanstructure = Val(false) M_indices = nothing @@ -257,7 +257,7 @@ objective_gradient_hessian!(imply::RAM, par, model::AbstractSemSingle, has_means ############################################################################################ function update_observed(imply::RAM, observed::SemObserved; kwargs...) - if n_man(observed) == size(imply.Σ, 1) + if nobserved_vars(observed) == size(imply.Σ, 1) return imply else return RAM(; observed = observed, kwargs...) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 3c99053bf..b8da20148 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -236,7 +236,7 @@ objective_gradient_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, p ############################################################################################ function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) - if n_man(observed) == size(imply.Σ, 1) + if nobserved_vars(observed) == size(imply.Σ, 1) return imply else return RAMSymbolic(; observed = observed, kwargs...) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 6ff8e4f04..3f39245db 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -47,20 +47,20 @@ end ############################################################################################ function SemFIML(; observed, specification, kwargs...) - inverses = broadcast(x -> zeros(x, x), Int64.(pattern_nvar_obs(observed))) + inverses = broadcast(x -> zeros(x, x), pattern_nobs_vars(observed)) choleskys = Array{Cholesky{Float64, Array{Float64, 2}}, 1}(undef, length(inverses)) n_patterns = size(rows(observed), 1) logdets = zeros(n_patterns) - imp_mean = zeros.(Int64.(pattern_nvar_obs(observed))) - meandiff = zeros.(Int64.(pattern_nvar_obs(observed))) + imp_mean = zeros.(pattern_nobs_vars(observed)) + meandiff = zeros.(pattern_nobs_vars(observed)) - nman = Int64(n_man(observed)) - imp_inv = zeros(nman, nman) + nobs_vars = nobserved_vars(observed) + imp_inv = zeros(nobs_vars, nobs_vars) mult = similar.(inverses) - ∇ind = vec(CartesianIndices(Array{Float64}(undef, nman, nman))) + ∇ind = vec(CartesianIndices(Array{Float64}(undef, nobs_vars, nobs_vars))) ∇ind = [findall(x -> !(x[1] ∈ ind || x[2] ∈ ind), ∇ind) for ind in patterns_not(observed)] @@ -189,8 +189,8 @@ function F_FIML(rows, semfiml, model, params) end function ∇F_FIML(rows, semfiml, model) - Jμ = zeros(Int64(n_man(model))) - JΣ = zeros(Int64(n_man(model)^2)) + Jμ = zeros(nobserved_vars(model)) + JΣ = zeros(nobserved_vars(model)^2) for i in 1:size(rows, 1) ∇F_one_pattern( diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 61c89fc85..8fcc84a99 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -64,7 +64,7 @@ function SemWLS(; # compute V here if isnothing(wls_weight_matrix) - D = duplication_matrix(n_man(observed)) + D = duplication_matrix(nobserved_vars(observed)) S = inv(obs_cov(observed)) S = kron(S, S) wls_weight_matrix = 0.5 * (D' * S * D) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 4640a7137..2807a2816 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -29,15 +29,15 @@ function em_mvn( rtol_em = 1e-4, kwargs..., ) - n_man = observed.n_man + nvars = nobserved_vars(observed) nsamps = nsamples(observed) # preallocate stuff? - 𝔼x_pre = zeros(n_man) - 𝔼xxᵀ_pre = zeros(n_man, n_man) + 𝔼x_pre = zeros(nvars) + 𝔼xxᵀ_pre = zeros(nvars, nvars) ### precompute for full cases - if length(observed.patterns[1]) == observed.n_man + if length(observed.patterns[1]) == nvars for row in observed.rows[1] row = observed.data_rowwise[row] 𝔼x_pre += row @@ -50,11 +50,11 @@ function em_mvn( # initialize em_model = start_em(observed; kwargs...) - em_model_prev = EmMVNModel(zeros(n_man, n_man), zeros(n_man), false) + em_model_prev = EmMVNModel(zeros(nvars, nvars), zeros(nvars), false) iter = 1 done = false - 𝔼x = zeros(n_man) - 𝔼xxᵀ = zeros(n_man, n_man) + 𝔼x = zeros(nvars) + 𝔼xxᵀ = zeros(nvars, nvars) while !done em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xxᵀ_pre) @@ -153,7 +153,7 @@ end # use μ and Σ of full cases function start_em_observed(observed::SemObservedMissing; kwargs...) - if (length(observed.patterns[1]) == observed.n_man) & (observed.pattern_nsamples[1] > 1) + if (length(observed.patterns[1]) == nobserved_vars(observed)) & (observed.pattern_nsamples[1] > 1) μ = copy(observed.obs_mean[1]) Σ = copy(Symmetric(observed.obs_cov[1])) if !isposdef(Σ) @@ -167,11 +167,11 @@ end # use μ = O and Σ = I function start_em_simple(observed::SemObservedMissing; kwargs...) - n_man = Int(observed.n_man) - μ = zeros(n_man) - Σ = rand(n_man, n_man) + nvars = nobserved_vars(observed) + μ = zeros(nvars) + Σ = rand(nvars, nvars) Σ = Σ * Σ' - # Σ = Matrix(1.0I, n_man, n_man) + # Σ = Matrix(1.0I, nvars, nvars) return EmMVNModel(Σ, μ, false) end diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index a3ff822a3..f851fd5b5 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -42,7 +42,7 @@ use this if you are sure your covariance matrix is in the right format. struct SemObservedCovariance{B, C} <: SemObserved obs_cov::B obs_mean::C - n_man::Int + nobs_vars::Int nsamples::Int end @@ -88,7 +88,7 @@ end ############################################################################################ nsamples(observed::SemObservedCovariance) = observed.nsamples -n_man(observed::SemObservedCovariance) = observed.n_man +nobserved_vars(observed::SemObservedCovariance) = observed.nobs_vars ############################################################################################ ### additional methods diff --git a/src/observed/data.jl b/src/observed/data.jl index c0a42d9c2..c9b50e597 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -19,7 +19,7 @@ For observed data without missings. # Extended help ## Interfaces - `nsamples(::SemObservedData)` -> number of observed data points -- `n_man(::SemObservedData)` -> number of manifest variables +- `nobserved_vars(::SemObservedData)` -> number of observed (manifested) variables - `samples(::SemObservedData)` -> observed data - `obs_cov(::SemObservedData)` -> observed.obs_cov @@ -41,7 +41,7 @@ struct SemObservedData{A, B, C} <: SemObserved data::A obs_cov::B obs_mean::C - n_man::Int + nobs_vars::Int nsamples::Int end @@ -113,7 +113,7 @@ end ############################################################################################ nsamples(observed::SemObservedData) = observed.nsamples -n_man(observed::SemObservedData) = observed.n_man +nobserved_vars(observed::SemObservedData) = observed.nobs_vars ############################################################################################ ### additional methods diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 6a78b5161..859a9d197 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -28,7 +28,7 @@ For observed data with missing values. # Extended help ## Interfaces - `nsamples(::SemObservedMissing)` -> number of observed data points -- `n_man(::SemObservedMissing)` -> number of manifest variables +- `nobserved_vars(::SemObservedMissing)` -> number of manifest variables - `samples(::SemObservedMissing)` -> observed data - `data_rowwise(::SemObservedMissing)` -> observed data as vector per observation, with missing values deleted @@ -37,7 +37,7 @@ For observed data with missing values. - `patterns_not(::SemObservedMissing)` -> indices of missing variables per missing pattern - `rows(::SemObservedMissing)` -> row indices of observed data points that belong to each pattern - `pattern_nsamples(::SemObservedMissing)` -> number of data points per pattern -- `pattern_nvar_obs(::SemObservedMissing)` -> number of non-missing observed variables per pattern +- `pattern_nobs_vars(::SemObservedMissing)` -> number of non-missing observed variables per pattern - `obs_mean(::SemObservedMissing)` -> observed mean per pattern - `obs_cov(::SemObservedMissing)` -> observed covariance per pattern - `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via optimization maximization @@ -55,7 +55,7 @@ use this if you are sure your observed data is in the right format. """ mutable struct SemObservedMissing{ A <: AbstractArray, - D <: AbstractFloat, + D <: Number, O <: Number, P <: Vector, P2 <: Vector, @@ -68,14 +68,14 @@ mutable struct SemObservedMissing{ S <: EmMVNModel, } <: SemObserved data::A - n_man::D + nobs_vars::D nsamples::O patterns::P # missing patterns patterns_not::P2 rows::R # coresponding rows in data_rowwise data_rowwise::PD # list of data pattern_nsamples::PO # observed rows per pattern - pattern_nvar_obs::PVO # number of non-missing variables per pattern + pattern_nobs_vars::PVO # number of non-missing variables per pattern obs_mean::A2 obs_cov::A3 em_model::S @@ -140,7 +140,7 @@ function SemObservedMissing(; end data = data[keep, :] - nsamples, n_man = size(data) + nsamples, nobs_vars = size(data) # compute and store the different missing patterns with their rowindices missings = ismissing.(data) @@ -176,24 +176,24 @@ function SemObservedMissing(; rows = rows[sort_n_miss] pattern_nsamples = size.(rows, 1) - pattern_nvar_obs = length.(remember_cart) + pattern_nobs_vars = length.(remember_cart) cov_mean = [cov_and_mean(data_rowwise[rows]) for rows in rows] obs_cov = [cov_mean[1] for cov_mean in cov_mean] obs_mean = [cov_mean[2] for cov_mean in cov_mean] - em_model = EmMVNModel(zeros(n_man, n_man), zeros(n_man), false) + em_model = EmMVNModel(zeros(nobs_vars, nobs_vars), zeros(nobs_vars), false) return SemObservedMissing( data, - Float64(nobs_vars), + nobs_vars, nsamples, remember_cart, remember_cart_not, rows, data_rowwise, pattern_nsamples, - Float64.(pattern_nvar_obs), + pattern_nobs_vars, obs_mean, obs_cov, em_model, @@ -205,7 +205,7 @@ end ############################################################################################ nsamples(observed::SemObservedMissing) = observed.nsamples -n_man(observed::SemObservedMissing) = observed.n_man +nobserved_vars(observed::SemObservedMissing) = observed.nobs_vars ############################################################################################ ### Additional methods @@ -216,7 +216,7 @@ patterns_not(observed::SemObservedMissing) = observed.patterns_not rows(observed::SemObservedMissing) = observed.rows data_rowwise(observed::SemObservedMissing) = observed.data_rowwise pattern_nsamples(observed::SemObservedMissing) = observed.pattern_nsamples -pattern_nvar_obs(observed::SemObservedMissing) = observed.pattern_nvar_obs +pattern_nobs_vars(observed::SemObservedMissing) = observed.pattern_nobs_vars obs_mean(observed::SemObservedMissing) = observed.obs_mean obs_cov(observed::SemObservedMissing) = observed.obs_cov em_model(observed::SemObservedMissing) = observed.em_model From d22254c0bfa1671ed6b25fc87e4b6637bc76f017 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 16 Jun 2024 20:59:10 -0700 Subject: [PATCH 080/364] move Sem methods out of types.jl --- src/frontend/specification/Sem.jl | 33 +++++++++++++++++++++++++++ src/types.jl | 38 ------------------------------- 2 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 1befb9aad..758bc073d 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -30,9 +30,42 @@ latent_vars(sem::AbstractSemSingle) = latent_vars(sem.imply) nsamples(sem::AbstractSemSingle) = nsamples(sem.observed) +params(model::AbstractSem) = params(model.imply) + # sum of samples in all sub-models nsamples(ensemble::SemEnsemble) = sum(nsamples, ensemble.sems) +############################################################################################ +# additional methods +############################################################################################ +""" + observed(model::AbstractSemSingle) -> SemObserved + +Returns the observed part of a model. +""" +observed(model::AbstractSemSingle) = model.observed + +""" + imply(model::AbstractSemSingle) -> SemImply + +Returns the imply part of a model. +""" +imply(model::AbstractSemSingle) = model.imply + +""" + loss(model::AbstractSemSingle) -> SemLoss + +Returns the loss part of a model. +""" +loss(model::AbstractSemSingle) = model.loss + +""" + optimizer(model::AbstractSemSingle) -> SemOptimizer + +Returns the optimizer part of a model. +""" +optimizer(model::AbstractSemSingle) = model.optimizer + function SemFiniteDiff(; observed::O = SemObservedData, imply::I = RAM, diff --git a/src/types.jl b/src/types.jl index 98d5f87ec..0493da8fa 100644 --- a/src/types.jl +++ b/src/types.jl @@ -13,13 +13,6 @@ abstract type AbstractSemCollection <: AbstractSem end "Supertype for all loss functions of SEMs. If you want to implement a custom loss function, it should be a subtype of `SemLossFunction`." abstract type SemLossFunction end -""" - params(semobj) - -Return the vector of SEM model parameters. -""" -params(model::AbstractSem) = model.params - """ SemLoss(args...; loss_weights = nothing, ...) @@ -225,37 +218,6 @@ Returns the optimizer part of an ensemble model. """ optimizer(ensemble::SemEnsemble) = ensemble.optimizer -############################################################################################ -# additional methods -############################################################################################ -""" - observed(model::AbstractSemSingle) -> SemObserved - -Returns the observed part of a model. -""" -observed(model::AbstractSemSingle) = model.observed - -""" - imply(model::AbstractSemSingle) -> SemImply - -Returns the imply part of a model. -""" -imply(model::AbstractSemSingle) = model.imply - -""" - loss(model::AbstractSemSingle) -> SemLoss - -Returns the loss part of a model. -""" -loss(model::AbstractSemSingle) = model.loss - -""" - optimizer(model::AbstractSemSingle) -> SemOptimizer - -Returns the optimizer part of a model. -""" -optimizer(model::AbstractSemSingle) = model.optimizer - """ Base type for all SEM specifications. """ From 1b13c73bfbb10cb15e8be061ec7aeb7163793588 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 16 Jun 2024 20:58:22 -0700 Subject: [PATCH 081/364] rows(::SemObservedMissing) -> pattern_rows() --- src/frontend/fit/fitmeasures/minus2ll.jl | 4 ++-- src/loss/ML/FIML.jl | 10 +++++----- src/observed/EM.jl | 4 ++-- src/observed/missing.jl | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index bfd161142..88948d4d4 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -54,7 +54,7 @@ function minus2ll(observed::SemObservedMissing) observed.em_model.μ, observed.em_model.Σ, nsamples(observed), - observed.rows, + pattern_rows(observed), observed.patterns, observed.obs_mean, observed.obs_cov, @@ -67,7 +67,7 @@ function minus2ll(observed::SemObservedMissing) observed.em_model.μ, observed.em_model.Σ, nsamples(observed), - observed.rows, + pattern_rows(observed), observed.patterns, observed.obs_mean, observed.obs_cov, diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 3f39245db..5609224e7 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -50,7 +50,7 @@ function SemFIML(; observed, specification, kwargs...) inverses = broadcast(x -> zeros(x, x), pattern_nobs_vars(observed)) choleskys = Array{Cholesky{Float64, Array{Float64, 2}}, 1}(undef, length(inverses)) - n_patterns = size(rows(observed), 1) + n_patterns = size(pattern_rows(observed), 1) logdets = zeros(n_patterns) imp_mean = zeros.(pattern_nobs_vars(observed)) @@ -89,7 +89,7 @@ function objective!(semfiml::SemFIML, params, model) prepare_SemFIML!(semfiml, model) - objective = F_FIML(rows(observed(model)), semfiml, model, params) + objective = F_FIML(pattern_rows(observed(model)), semfiml, model, params) return objective / nsamples(observed(model)) end @@ -100,7 +100,7 @@ function gradient!(semfiml::SemFIML, params, model) prepare_SemFIML!(semfiml, model) - gradient = ∇F_FIML(rows(observed(model)), semfiml, model) / nsamples(observed(model)) + gradient = ∇F_FIML(pattern_rows(observed(model)), semfiml, model) / nsamples(observed(model)) return gradient end @@ -112,8 +112,8 @@ function objective_gradient!(semfiml::SemFIML, params, model) prepare_SemFIML!(semfiml, model) objective = - F_FIML(rows(observed(model)), semfiml, model, params) / nsamples(observed(model)) - gradient = ∇F_FIML(rows(observed(model)), semfiml, model) / nsamples(observed(model)) + F_FIML(pattern_rows(observed(model)), semfiml, model, params) / nsamples(observed(model)) + gradient = ∇F_FIML(pattern_rows(observed(model)), semfiml, model) / nsamples(observed(model)) return objective, gradient end diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 2807a2816..a681e0fc2 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -38,7 +38,7 @@ function em_mvn( ### precompute for full cases if length(observed.patterns[1]) == nvars - for row in observed.rows[1] + for row in pattern_rows(observed)[1] row = observed.data_rowwise[row] 𝔼x_pre += row 𝔼xxᵀ_pre += row * row' @@ -107,7 +107,7 @@ function em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xx V = Σ[u, u] - Σ[u, o] * (Σ[o, o] \ Σ[o, u]) # loop trough data - for row in observed.rows[i] + for row in pattern_rows(observed)[i] m = μ[u] + Σ[u, o] * (Σ[o, o] \ (observed.data_rowwise[row] - μ[o])) 𝔼xᵢ[u] = m diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 859a9d197..b628a313b 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -35,7 +35,7 @@ For observed data with missing values. - `patterns(::SemObservedMissing)` -> indices of non-missing variables per missing patterns - `patterns_not(::SemObservedMissing)` -> indices of missing variables per missing pattern -- `rows(::SemObservedMissing)` -> row indices of observed data points that belong to each pattern +- `pattern_rows(::SemObservedMissing)` -> row indices of observed data points that belong to each pattern - `pattern_nsamples(::SemObservedMissing)` -> number of data points per pattern - `pattern_nobs_vars(::SemObservedMissing)` -> number of non-missing observed variables per pattern - `obs_mean(::SemObservedMissing)` -> observed mean per pattern @@ -72,7 +72,7 @@ mutable struct SemObservedMissing{ nsamples::O patterns::P # missing patterns patterns_not::P2 - rows::R # coresponding rows in data_rowwise + pattern_rows::R # coresponding rows in data_rowwise data_rowwise::PD # list of data pattern_nsamples::PO # observed rows per pattern pattern_nobs_vars::PVO # number of non-missing variables per pattern @@ -213,7 +213,7 @@ nobserved_vars(observed::SemObservedMissing) = observed.nobs_vars patterns(observed::SemObservedMissing) = observed.patterns patterns_not(observed::SemObservedMissing) = observed.patterns_not -rows(observed::SemObservedMissing) = observed.rows +pattern_rows(observed::SemObservedMissing) = observed.pattern_rows data_rowwise(observed::SemObservedMissing) = observed.data_rowwise pattern_nsamples(observed::SemObservedMissing) = observed.pattern_nsamples pattern_nobs_vars(observed::SemObservedMissing) = observed.pattern_nobs_vars From aec435867be2002bbc3b3bf5e0db9424063d2eda Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 30 Jul 2024 00:02:02 -0700 Subject: [PATCH 082/364] fix formatting --- src/frontend/fit/summary.jl | 3 +-- src/frontend/specification/ParameterTable.jl | 7 +++++-- src/imply/RAM/generic.jl | 6 +++++- src/loss/ML/FIML.jl | 9 ++++++--- src/observed/EM.jl | 5 +++-- test/examples/political_democracy/by_parts.jl | 8 ++++++-- 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 507e835fc..a77f62c21 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -160,8 +160,7 @@ function sem_summary( var_array = reduce( hcat, - check_round(partable.columns[c][var_indices]; digits = digits) for - c in var_columns + check_round(partable.columns[c][var_indices]; digits) for c in var_columns ) var_columns[2] = Symbol("") diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 91d55ce46..8970b7430 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -235,8 +235,11 @@ sort_vars(partable::ParameterTable) = sort_vars!(deepcopy(partable)) # add a row -------------------------------------------------------------------------------- function Base.push!(partable::ParameterTable, d::Union{AbstractDict{Symbol}, NamedTuple}) - issetequal(keys(partable.columns), keys(d)) || - throw(ArgumentError("The new row needs to have the same keys as the columns of the parameter table.")) + issetequal(keys(partable.columns), keys(d)) || throw( + ArgumentError( + "The new row needs to have the same keys as the columns of the parameter table.", + ), + ) for (key, val) in pairs(d) push!(partable.columns[key], val) end diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index e93f5c5c2..9ff46bd2e 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -144,7 +144,11 @@ function RAM(; # μ if meanstructure has_meanstructure = Val(true) - !isnothing(M_indices) || throw(ArgumentError("You set `meanstructure = true`, but your model specification contains no mean parameters.")) + !isnothing(M_indices) || throw( + ArgumentError( + "You set `meanstructure = true`, but your model specification contains no mean parameters.", + ), + ) ∇M = gradient ? matrix_gradient(M_indices, n_var) : nothing μ = zeros(n_obs) else diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 5609224e7..cd5d0270f 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -100,7 +100,8 @@ function gradient!(semfiml::SemFIML, params, model) prepare_SemFIML!(semfiml, model) - gradient = ∇F_FIML(pattern_rows(observed(model)), semfiml, model) / nsamples(observed(model)) + gradient = + ∇F_FIML(pattern_rows(observed(model)), semfiml, model) / nsamples(observed(model)) return gradient end @@ -112,8 +113,10 @@ function objective_gradient!(semfiml::SemFIML, params, model) prepare_SemFIML!(semfiml, model) objective = - F_FIML(pattern_rows(observed(model)), semfiml, model, params) / nsamples(observed(model)) - gradient = ∇F_FIML(pattern_rows(observed(model)), semfiml, model) / nsamples(observed(model)) + F_FIML(pattern_rows(observed(model)), semfiml, model, params) / + nsamples(observed(model)) + gradient = + ∇F_FIML(pattern_rows(observed(model)), semfiml, model) / nsamples(observed(model)) return objective, gradient end diff --git a/src/observed/EM.jl b/src/observed/EM.jl index a681e0fc2..ef5da317d 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -62,7 +62,7 @@ function em_mvn( if iter > max_iter_em done = true - @warn "EM Algorithm for MVN missing data did not converge. Likelihood for FIML is not interpretable. + @warn "EM Algorithm for MVN missing data did not converge. Likelihood for FIML is not interpretable. Maybe try passing different starting values via 'start_em = ...' " elseif iter > 1 # done = isapprox(ll, ll_prev; rtol = rtol) @@ -153,7 +153,8 @@ end # use μ and Σ of full cases function start_em_observed(observed::SemObservedMissing; kwargs...) - if (length(observed.patterns[1]) == nobserved_vars(observed)) & (observed.pattern_nsamples[1] > 1) + if (length(observed.patterns[1]) == nobserved_vars(observed)) & + (observed.pattern_nsamples[1] > 1) μ = copy(observed.obs_mean[1]) Σ = copy(Symmetric(observed.obs_cov[1])) if !isposdef(Σ) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 09d40cc28..f50fb6dd0 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -40,8 +40,12 @@ model_ridge = Sem(observed, imply_ram, SemLoss(ml, ridge), optimizer_obj) model_constant = Sem(observed, imply_ram, SemLoss(ml, constant), optimizer_obj) -model_ml_weighted = - Sem(observed, imply_ram, SemLoss(ml; loss_weights = [nsamples(model_ml)]), optimizer_obj) +model_ml_weighted = Sem( + observed, + imply_ram, + SemLoss(ml; loss_weights = [nsamples(model_ml)]), + optimizer_obj, +) ############################################################################################ ### test gradients From 95dc47761d23d214c239131b871068678b92ea1f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 26 Jun 2024 13:47:42 -0700 Subject: [PATCH 083/364] samples(SemObsCov) throws an exception --- src/observed/covariance.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index f851fd5b5..b78f41833 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -90,6 +90,9 @@ end nsamples(observed::SemObservedCovariance) = observed.nsamples nobserved_vars(observed::SemObservedCovariance) = observed.nobs_vars +samples(observed::SemObservedCovariance) = + error("$(typeof(observed)) does not store data samples") + ############################################################################################ ### additional methods ############################################################################################ From 1f67019135b364edca24ff88f21247a3e8a99397 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 30 Jul 2024 00:01:03 -0700 Subject: [PATCH 084/364] SemObserved tests: refactor and add var API tests --- test/unit_tests/data_input_formats.jl | 755 +++++++++++++------------- test/unit_tests/unit_tests.jl | 8 +- 2 files changed, 372 insertions(+), 391 deletions(-) diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index f1adaf625..dd522fda1 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -1,5 +1,7 @@ using StructuralEquationModels, Test, Statistics -using StructuralEquationModels: obs_cov, obs_mean, samples +using StructuralEquationModels: + samples, nsamples, observed_vars, nobserved_vars, obs_cov, obs_mean + ### model specification -------------------------------------------------------------------- spec = ParameterTable( @@ -18,412 +20,391 @@ dat_missing_matrix = Matrix(dat_missing) dat_cov = Statistics.cov(dat_matrix) dat_mean = vcat(Statistics.mean(dat_matrix, dims = 1)...) -############################################################################################ -### tests - SemObservedData -############################################################################################ - -# w.o. means ------------------------------------------------------------------------------- - -# errors -@test_throws ArgumentError( - "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * - "Please make sure the column names of your data frame indicate the correct variables " * - "or pass your data in a different format.", -) begin - SemObservedData(specification = spec, data = dat, obs_colnames = Symbol.(names(dat))) -end - -@test_throws ArgumentError( - "Your `data` can not be indexed by symbols. " * - "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", -) begin - SemObservedData(specification = spec, data = dat_matrix) -end - -@test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin - SemObservedData(specification = spec, data = dat_matrix, obs_colnames = names(dat)) -end - -@test_throws UndefKeywordError(:data) SemObservedData(specification = spec) - -@test_throws UndefKeywordError(:specification) SemObservedData(data = dat_matrix) - -# should work -observed = SemObservedData(specification = spec, data = dat) - -observed_nospec = SemObservedData(specification = nothing, data = dat_matrix) - -observed_matrix = SemObservedData( - specification = spec, - data = dat_matrix, - obs_colnames = Symbol.(names(dat)), -) - -all_equal_cov = - (obs_cov(observed) == obs_cov(observed_nospec)) & - (obs_cov(observed) == obs_cov(observed_matrix)) - -all_equal_data = - (samples(observed) == samples(observed_nospec)) & - (samples(observed) == samples(observed_matrix)) - -@testset "unit tests | SemObservedData | input formats" begin - @test all_equal_cov - @test all_equal_data -end - -# shuffle variables -new_order = [3, 2, 7, 8, 5, 6, 9, 11, 1, 10, 4] - -shuffle_names = Symbol.(names(dat))[new_order] - -shuffle_dat = dat[:, new_order] - -shuffle_dat_matrix = dat_matrix[:, new_order] - -observed_shuffle = SemObservedData(specification = spec, data = shuffle_dat) - -observed_matrix_shuffle = SemObservedData( - specification = spec, - data = shuffle_dat_matrix, - obs_colnames = shuffle_names, -) - -all_equal_cov_suffled = - (obs_cov(observed) == obs_cov(observed_shuffle)) & - (obs_cov(observed) == obs_cov(observed_matrix_shuffle)) - -all_equal_data_suffled = - (samples(observed) == samples(observed_shuffle)) & - (samples(observed) == samples(observed_matrix_shuffle)) - -@testset "unit tests | SemObservedData | input formats shuffled " begin - @test all_equal_cov_suffled - @test all_equal_data_suffled -end - -# with means ------------------------------------------------------------------------------- - -# errors -@test_throws ArgumentError( - "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * - "Please make sure the column names of your data frame indicate the correct variables " * - "or pass your data in a different format.", -) begin - SemObservedData( - specification = spec, - data = dat, - obs_colnames = Symbol.(names(dat)), - meanstructure = true, - ) -end - -@test_throws ArgumentError( - "Your `data` can not be indexed by symbols. " * - "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", -) begin - SemObservedData(specification = spec, data = dat_matrix, meanstructure = true) -end - -@test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin - SemObservedData( - specification = spec, - data = dat_matrix, - obs_colnames = names(dat), - meanstructure = true, - ) -end - -@test_throws UndefKeywordError(:data) SemObservedData( - specification = spec, - meanstructure = true, -) - -@test_throws UndefKeywordError(:specification) SemObservedData( - data = dat_matrix, - meanstructure = true, -) - -# should work -observed = SemObservedData(specification = spec, data = dat, meanstructure = true) - -observed_nospec = - SemObservedData(specification = nothing, data = dat_matrix, meanstructure = true) - -observed_matrix = SemObservedData( - specification = spec, - data = dat_matrix, - obs_colnames = Symbol.(names(dat)), - meanstructure = true, -) - -all_equal_mean = - (obs_mean(observed) == obs_mean(observed_nospec)) & - (obs_mean(observed) == obs_mean(observed_matrix)) - -@testset "unit tests | SemObservedData | input formats - means" begin - @test all_equal_mean -end - -# shuffle variables -new_order = [3, 2, 7, 8, 5, 6, 9, 11, 1, 10, 4] - -shuffle_names = Symbol.(names(dat))[new_order] - -shuffle_dat = dat[:, new_order] - -shuffle_dat_matrix = dat_matrix[:, new_order] - -observed_shuffle = - SemObservedData(specification = spec, data = shuffle_dat, meanstructure = true) - -observed_matrix_shuffle = SemObservedData( - specification = spec, - data = shuffle_dat_matrix, - obs_colnames = shuffle_names, - meanstructure = true, -) - -all_equal_mean_suffled = - (obs_mean(observed) == obs_mean(observed_shuffle)) & - (obs_mean(observed) == obs_mean(observed_matrix_shuffle)) - -@testset "unit tests | SemObservedData | input formats shuffled - mean" begin - @test all_equal_mean_suffled -end - -############################################################################################ -### tests - SemObservedCovariance -############################################################################################ - -# w.o. means ------------------------------------------------------------------------------- - -# errors - -@test_throws ArgumentError("observed means were passed, but `meanstructure = false`") begin - SemObservedCovariance( - specification = nothing, - obs_cov = dat_cov, - obs_mean = dat_mean, - nsamples = 75, - ) -end - -@test_throws UndefKeywordError(:nsamples) SemObservedCovariance(obs_cov = dat_cov) - -@test_throws ArgumentError("no `obs_colnames` were specified") begin - SemObservedCovariance(specification = spec, obs_cov = dat_cov, nsamples = 75) -end - -@test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin - SemObservedCovariance( - specification = spec, - obs_cov = dat_cov, - obs_colnames = names(dat), - nsamples = 75, - ) -end - -# should work -observed = SemObservedCovariance( - specification = spec, - obs_cov = dat_cov, - obs_colnames = obs_colnames = Symbol.(names(dat)), - nsamples = 75, -) - -observed_nospec = - SemObservedCovariance(specification = nothing, obs_cov = dat_cov, nsamples = 75) - -all_equal_cov = (obs_cov(observed) == obs_cov(observed_nospec)) - -@testset "unit tests | SemObservedCovariance | input formats" begin - @test all_equal_cov - @test nsamples(observed) == 75 - @test nsamples(observed_nospec) == 75 -end - -# shuffle variables -new_order = [3, 2, 7, 8, 5, 6, 9, 11, 1, 10, 4] - -shuffle_names = Symbol.(names(dat))[new_order] - -shuffle_dat_matrix = dat_matrix[:, new_order] - -shuffle_dat_cov = Statistics.cov(shuffle_dat_matrix) - -observed_shuffle = SemObservedCovariance( - specification = spec, - obs_cov = shuffle_dat_cov, - obs_colnames = shuffle_names, - nsamples = 75, -) - -all_equal_cov_suffled = (obs_cov(observed) ≈ obs_cov(observed_shuffle)) - -@testset "unit tests | SemObservedCovariance | input formats shuffled " begin - @test all_equal_cov_suffled -end - -# with means ------------------------------------------------------------------------------- - -# errors -@test_throws ArgumentError("`meanstructure = true`, but no observed means were passed") begin - SemObservedCovariance( - specification = spec, - obs_cov = dat_cov, - meanstructure = true, - nsamples = 75, - ) -end - -@test_throws UndefKeywordError SemObservedCovariance( - data = dat_matrix, - meanstructure = true, -) - -@test_throws UndefKeywordError SemObservedCovariance( - obs_cov = dat_cov, - meanstructure = true, -) - -@test_throws ArgumentError("`meanstructure = true`, but no observed means were passed") begin - SemObservedCovariance( - specification = spec, - obs_cov = dat_cov, - obs_colnames = Symbol.(names(dat)), - meanstructure = true, - nsamples = 75, - ) -end - -# should work -observed = SemObservedCovariance( - specification = spec, - obs_cov = dat_cov, - obs_mean = dat_mean, - obs_colnames = Symbol.(names(dat)), - nsamples = 75, - meanstructure = true, -) - -observed_nospec = SemObservedCovariance( - specification = nothing, - obs_cov = dat_cov, - obs_mean = dat_mean, - meanstructure = true, - nsamples = 75, -) - -all_equal_mean = (obs_mean(observed) == obs_mean(observed_nospec)) - -@testset "unit tests | SemObservedCovariance | input formats - means" begin - @test all_equal_mean -end - # shuffle variables new_order = [3, 2, 7, 8, 5, 6, 9, 11, 1, 10, 4] shuffle_names = Symbol.(names(dat))[new_order] shuffle_dat = dat[:, new_order] +shuffle_dat_missing = dat_missing[:, new_order] shuffle_dat_matrix = dat_matrix[:, new_order] +shuffle_dat_missing_matrix = dat_missing_matrix[:, new_order] shuffle_dat_cov = Statistics.cov(shuffle_dat_matrix) shuffle_dat_mean = vcat(Statistics.mean(shuffle_dat_matrix, dims = 1)...) -observed_shuffle = SemObservedCovariance( - specification = spec, - obs_cov = shuffle_dat_cov, - obs_mean = shuffle_dat_mean, - obs_colnames = shuffle_names, - nsamples = 75, - meanstructure = true, +# common tests for SemObserved subtypes +function test_observed( + observed::SemObserved, + dat, + dat_matrix, + dat_cov, + dat_mean; + meanstructure::Bool, + approx_cov::Bool = false, ) - -all_equal_mean_suffled = (obs_mean(observed) == obs_mean(observed_shuffle)) - -@testset "unit tests | SemObservedCovariance | input formats shuffled - mean" begin - @test all_equal_mean_suffled + @test @inferred(nobserved_vars(observed)) == size(dat, 2) + # FIXME observed should provide names of observed variables + @test @inferred(observed_vars(observed)) == names(dat) broken = true + @test @inferred(nsamples(observed)) == size(dat, 1) + + hasmissing = + !isnothing(dat_matrix) && any(ismissing, dat_matrix) || + !isnothing(dat_cov) && any(ismissing, dat_cov) + + if !isnothing(dat_matrix) + if hasmissing + @test isequal(@inferred(samples(observed)), dat_matrix) + else + @test @inferred(samples(observed)) == dat_matrix + end + end + + if !isnothing(dat_cov) + if hasmissing + @test isequal(@inferred(obs_cov(observed)), dat_cov) + else + if approx_cov + @test @inferred(obs_cov(observed)) ≈ dat_cov + else + @test @inferred(obs_cov(observed)) == dat_cov + end + end + end + + # FIXME actually, SemObserved should not use meanstructure and always provide obs_mean() + # meanstructure is a part of SEM model + if meanstructure + if !isnothing(dat_mean) + if hasmissing + @test isequal(@inferred(obs_mean(observed)), dat_mean) + else + @test @inferred(obs_mean(observed)) == dat_mean + end + else + # FIXME if meanstructure is present, obs_mean() should provide something (currently Missing don't support it) + @test (@inferred(obs_mean(observed)) isa AbstractVector{Float64}) broken = true + end + else + @test @inferred(obs_mean(observed)) === nothing skip = true + end end ############################################################################################ -### tests - SemObservedMissing +@testset "SemObservedData" begin + + # errors + @test_throws ArgumentError( + "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * + "Please make sure the column names of your data frame indicate the correct variables " * + "or pass your data in a different format.", + ) begin + SemObservedData( + specification = spec, + data = dat, + obs_colnames = Symbol.(names(dat)), + ) + end + + @test_throws ArgumentError( + "Your `data` can not be indexed by symbols. " * + "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", + ) begin + SemObservedData(specification = spec, data = dat_matrix) + end + + @test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin + SemObservedData(specification = spec, data = dat_matrix, obs_colnames = names(dat)) + end + + @test_throws UndefKeywordError(:data) SemObservedData(specification = spec) + + @test_throws UndefKeywordError(:specification) SemObservedData(data = dat_matrix) + + @testset "meanstructure=$meanstructure" for meanstructure in (false, true) + observed = SemObservedData(specification = spec, data = dat; meanstructure) + + test_observed(observed, dat, dat_matrix, dat_cov, dat_mean; meanstructure) + + observed_nospec = + SemObservedData(specification = nothing, data = dat_matrix; meanstructure) + + test_observed(observed_nospec, dat, dat_matrix, dat_cov, dat_mean; meanstructure) + + observed_matrix = SemObservedData( + specification = spec, + data = dat_matrix, + obs_colnames = Symbol.(names(dat)), + meanstructure = meanstructure, + ) + + test_observed(observed_matrix, dat, dat_matrix, dat_cov, dat_mean; meanstructure) + + observed_shuffle = + SemObservedData(specification = spec, data = shuffle_dat; meanstructure) + + test_observed(observed_shuffle, dat, dat_matrix, dat_cov, dat_mean; meanstructure) + + observed_matrix_shuffle = SemObservedData( + specification = spec, + data = shuffle_dat_matrix, + obs_colnames = shuffle_names; + meanstructure, + ) + + test_observed( + observed_matrix_shuffle, + dat, + dat_matrix, + dat_cov, + dat_mean; + meanstructure, + ) + end # meanstructure +end # SemObservedData + ############################################################################################ -# errors -@test_throws ArgumentError( - "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * - "Please make sure the column names of your data frame indicate the correct variables " * - "or pass your data in a different format.", -) begin - SemObservedMissing( - specification = spec, - data = dat_missing, - obs_colnames = Symbol.(names(dat)), - ) -end +@testset "SemObservedCovariance" begin + + # errors + + @test_throws UndefKeywordError(:nsamples) SemObservedCovariance(obs_cov = dat_cov) + + @test_throws ArgumentError("no `obs_colnames` were specified") begin + SemObservedCovariance( + specification = spec, + obs_cov = dat_cov, + nsamples = size(dat, 1), + ) + end + + @test_throws ArgumentError("observed means were passed, but `meanstructure = false`") begin + SemObservedCovariance( + specification = nothing, + obs_cov = dat_cov, + obs_mean = dat_mean, + nsamples = size(dat, 1), + ) + end + + @test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin + SemObservedCovariance( + specification = spec, + obs_cov = dat_cov, + obs_colnames = names(dat), + nsamples = size(dat, 1), + meanstructure = false, + ) + end + + @test_throws ArgumentError("`meanstructure = true`, but no observed means were passed") begin + SemObservedCovariance( + specification = spec, + obs_cov = dat_cov, + obs_colnames = Symbol.(names(dat)), + meanstructure = true, + nsamples = size(dat, 1), + ) + end + + @testset "meanstructure=$meanstructure" for meanstructure in (false, true) + + # errors + @test_throws UndefKeywordError SemObservedCovariance( + obs_cov = dat_cov; + meanstructure, + ) + + @test_throws UndefKeywordError SemObservedCovariance( + data = dat_matrix; + meanstructure, + ) + + # should work + observed = SemObservedCovariance( + specification = spec, + obs_cov = dat_cov, + obs_mean = meanstructure ? dat_mean : nothing, + obs_colnames = obs_colnames = Symbol.(names(dat)), + nsamples = size(dat, 1), + meanstructure = meanstructure, + ) + + test_observed( + observed, + dat, + nothing, + dat_cov, + dat_mean; + meanstructure, + approx_cov = true, + ) + + @test_throws ErrorException samples(observed) + + observed_nospec = SemObservedCovariance( + specification = nothing, + obs_cov = dat_cov, + obs_mean = meanstructure ? dat_mean : nothing, + nsamples = size(dat, 1); + meanstructure, + ) + + test_observed( + observed_nospec, + dat, + nothing, + dat_cov, + dat_mean; + meanstructure, + approx_cov = true, + ) + + @test_throws ErrorException samples(observed_nospec) + + observed_shuffle = SemObservedCovariance( + specification = spec, + obs_cov = shuffle_dat_cov, + obs_mean = meanstructure ? dat_mean[new_order] : nothing, + obs_colnames = shuffle_names, + nsamples = size(dat, 1); + meanstructure, + ) + + test_observed( + observed_shuffle, + dat, + nothing, + dat_cov, + dat_mean; + meanstructure, + approx_cov = true, + ) + + @test_throws ErrorException samples(observed_shuffle) + + # respect specification order + @test @inferred(obs_cov(observed_shuffle)) ≈ obs_cov(observed) + @test @inferred(observed_vars(observed_shuffle)) == shuffle_names broken = true + end # meanstructure +end # SemObservedCovariance -@test_throws ArgumentError( - "Your `data` can not be indexed by symbols. " * - "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", -) begin - SemObservedMissing(specification = spec, data = dat_missing_matrix) -end +############################################################################################ -@test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin - SemObservedMissing( - specification = spec, +@testset "SemObservedMissing" begin + + # errors + @test_throws ArgumentError( + "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * + "Please make sure the column names of your data frame indicate the correct variables " * + "or pass your data in a different format.", + ) begin + SemObservedMissing( + specification = spec, + data = dat_missing, + obs_colnames = Symbol.(names(dat)), + ) + end + + @test_throws ArgumentError( + "Your `data` can not be indexed by symbols. " * + "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", + ) begin + SemObservedMissing(specification = spec, data = dat_missing_matrix) + end + + @test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin + SemObservedMissing( + specification = spec, + data = dat_missing_matrix, + obs_colnames = names(dat), + ) + end + + @test_throws UndefKeywordError(:data) SemObservedMissing(specification = spec) + + @test_throws UndefKeywordError(:specification) SemObservedMissing( data = dat_missing_matrix, - obs_colnames = names(dat), ) -end - -@test_throws UndefKeywordError(:data) SemObservedMissing(specification = spec) - -@test_throws UndefKeywordError(:specification) SemObservedMissing(data = dat_missing_matrix) - -# should work -observed = SemObservedMissing(specification = spec, data = dat_missing) - -observed_nospec = SemObservedMissing(specification = nothing, data = dat_missing_matrix) -observed_matrix = SemObservedMissing( - specification = spec, - data = dat_missing_matrix, - obs_colnames = Symbol.(names(dat)), -) - -all_equal_data = - isequal(samples(observed), samples(observed_nospec)) & - isequal(samples(observed), samples(observed_matrix)) - -@testset "unit tests | SemObservedMissing | input formats" begin - @test all_equal_data -end - -# shuffle variables -new_order = [3, 2, 7, 8, 5, 6, 9, 11, 1, 10, 4] - -shuffle_names = Symbol.(names(dat))[new_order] - -shuffle_dat_missing = dat_missing[:, new_order] - -shuffle_dat_missing_matrix = dat_missing_matrix[:, new_order] - -observed_shuffle = SemObservedMissing(specification = spec, data = shuffle_dat_missing) - -observed_matrix_shuffle = SemObservedMissing( - specification = spec, - data = shuffle_dat_missing_matrix, - obs_colnames = shuffle_names, -) - -all_equal_data_shuffled = - isequal(samples(observed), samples(observed_shuffle)) & - isequal(samples(observed), samples(observed_matrix_shuffle)) - -@testset "unit tests | SemObservedMissing | input formats shuffled " begin - @test all_equal_data_suffled -end + @testset "meanstructure=$meanstructure" for meanstructure in (false, true) + observed = + SemObservedMissing(specification = spec, data = dat_missing; meanstructure) + + test_observed( + observed, + dat_missing, + dat_missing_matrix, + nothing, + nothing; + meanstructure, + ) + + @test @inferred(length(StructuralEquationModels.patterns(observed))) == 55 + @test sum(@inferred(StructuralEquationModels.pattern_nsamples(observed))) == + size(dat_missing, 1) + @test all( + <=(size(dat_missing, 2)), + @inferred(StructuralEquationModels.pattern_nsamples(observed)) + ) + + observed_nospec = SemObservedMissing( + specification = nothing, + data = dat_missing_matrix; + meanstructure, + ) + + test_observed( + observed_nospec, + dat_missing, + dat_missing_matrix, + nothing, + nothing; + meanstructure, + ) + + observed_matrix = SemObservedMissing( + specification = spec, + data = dat_missing_matrix, + obs_colnames = Symbol.(names(dat)), + ) + + test_observed( + observed_matrix, + dat_missing, + dat_missing_matrix, + nothing, + nothing; + meanstructure, + ) + + observed_shuffle = + SemObservedMissing(specification = spec, data = shuffle_dat_missing) + + test_observed( + observed_shuffle, + dat_missing, + dat_missing_matrix, + nothing, + nothing; + meanstructure, + ) + + observed_matrix_shuffle = SemObservedMissing( + specification = spec, + data = shuffle_dat_missing_matrix, + obs_colnames = shuffle_names, + ) + + test_observed( + observed_matrix_shuffle, + dat_missing, + dat_missing_matrix, + nothing, + nothing; + meanstructure, + ) + end # meanstructure +end # SemObservedMissing diff --git a/test/unit_tests/unit_tests.jl b/test/unit_tests/unit_tests.jl index eb58650c1..b8400e542 100644 --- a/test/unit_tests/unit_tests.jl +++ b/test/unit_tests/unit_tests.jl @@ -4,10 +4,10 @@ using Test, SafeTestsets include("multithreading.jl") end -@safetestset "SemObs" begin - include("data_input_formats.jl") -end - @safetestset "Matrix algebra helper functions" begin include("matrix_helpers.jl") end + +@safetestset "SemObserved" begin + include("data_input_formats.jl") +end From 3e9aac37e8f6086c60ceae61c3e91eeb9b215470 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 26 Jun 2024 19:45:39 -0700 Subject: [PATCH 085/364] ParTable(graph): group is only valid for ensemble --- src/frontend/specification/StenoGraphs.jl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 67bb7973c..9f72f36a0 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -36,7 +36,7 @@ function ParameterTable( observed_vars::AbstractVector{Symbol}, latent_vars::AbstractVector{Symbol}, params::Union{AbstractVector{Symbol}, Nothing} = nothing, - group::Integer = 1, + group::Union{Integer, Nothing} = nothing, param_prefix = :θ, ) graph = unique(graph) @@ -69,7 +69,17 @@ function ParameterTable( end if element isa ModifiedEdge for modifier in values(element.modifiers) - modval = modifier.value[group] + if isnothing(group) && + modifier.value isa Union{AbstractVector, Tuple} && + length(modifier.value) > 1 + throw( + ArgumentError( + "The graph contains a group of parameters, ParameterTable expects a single value.\n" * + "For SEM ensembles, use EnsembleParameterTable instead.", + ), + ) + end + modval = modifier.value[something(group, 1)] if modifier isa Fixed if modval == :NaN free[i] = true From 7a0723342f1dff037acf275424aec21ab7c338df Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 26 Jun 2024 19:46:31 -0700 Subject: [PATCH 086/364] ParTable(graph): fix NaN modif detection --- src/frontend/specification/StenoGraphs.jl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 9f72f36a0..69e7bc94b 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -27,6 +27,11 @@ struct Label{N} <: EdgeModifier end label(args...) = Label(args) +# test whether the modifier is NaN +isnanmodval(val::Number) = isnan(val) +isnanmodval(val::Symbol) = val == :NaN +isnanmodval(val::SimpleNode{Symbol}) = val.node == :NaN + ############################################################################################ ### constructor for parameter table from graph ############################################################################################ @@ -81,7 +86,7 @@ function ParameterTable( end modval = modifier.value[something(group, 1)] if modifier isa Fixed - if modval == :NaN + if isnanmodval(modval) free[i] = true value_fixed[i] = 0.0 else @@ -89,9 +94,11 @@ function ParameterTable( value_fixed[i] = modval end elseif modifier isa Start - start[i] = modval + if !isnanmodval(modval) + start[i] = modval + end elseif modifier isa Label - if modval == :NaN + if isnanmodval(modval) throw(DomainError(NaN, "NaN is not allowed as a parameter label.")) end param_refs[i] = modval From c0e2c9e9d367e705981b60313a7c6404eaca23ea Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 30 Jul 2024 00:00:31 -0700 Subject: [PATCH 087/364] export vars, params and observed APIs --- src/StructuralEquationModels.jl | 14 ++++++++++++-- test/unit_tests/data_input_formats.jl | 2 -- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 6d2a82823..a032ab724 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -126,6 +126,10 @@ export AbstractSem, SemObservedCovariance, SemObservedMissing, observed, + obs_cov, + obs_mean, + nsamples, + samples, sem_fit, SemFit, minimum, @@ -138,6 +142,8 @@ export AbstractSem, objective_hessian!, gradient_hessian!, objective_gradient_hessian!, + SemSpecification, + RAMMatrices, ParameterTable, EnsembleParameterTable, update_partable!, @@ -150,9 +156,14 @@ export AbstractSem, start, Label, label, + nvars, + vars, + nlatent_vars, + latent_vars, + nobserved_vars, + observed_vars, sort_vars!, sort_vars, - RAMMatrices, params, nparams, param_indices, @@ -163,7 +174,6 @@ export AbstractSem, df, fit_measures, minus2ll, - nsamples, p_value, RMSEA, EmMVNModel, diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index dd522fda1..3fc255b84 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -1,6 +1,4 @@ using StructuralEquationModels, Test, Statistics -using StructuralEquationModels: - samples, nsamples, observed_vars, nobserved_vars, obs_cov, obs_mean ### model specification -------------------------------------------------------------------- From 86c172a51f028669bae5b90414c65daa2596893b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 30 Jul 2024 00:01:48 -0700 Subject: [PATCH 088/364] refactor SemSpec tests --- test/unit_tests/specification.jl | 131 ++++++++++++++++++++++++++++--- test/unit_tests/unit_tests.jl | 4 + 2 files changed, 124 insertions(+), 11 deletions(-) diff --git a/test/unit_tests/specification.jl b/test/unit_tests/specification.jl index c081dc0f9..e0a412e76 100644 --- a/test/unit_tests/specification.jl +++ b/test/unit_tests/specification.jl @@ -1,22 +1,131 @@ -@testset "ParameterTable - RAMMatrices conversion" begin - partable = ParameterTable(ram_matrices) - @test ram_matrices == RAMMatrices(partable) -end +using StructuralEquationModels -@testset "params()" begin - @test params(model_ml)[2, 10, 28] == [:x2, :x10, :x28] - @test params(model_ml) == params(partable) - @test params(model_ml) == params(RAMMatrices(partable)) -end +obs_vars = Symbol.("x", 1:9) +lat_vars = [:visual, :textual, :speed] graph = @StenoGraph begin + # measurement model + visual → fixed(1.0) * x1 + fixed(0.5) * x2 + fixed(0.6) * x3 + textual → fixed(1.0) * x4 + x5 + label(:a₁) * x6 + speed → fixed(1.0) * x7 + fixed(1.0) * x8 + label(:λ₉) * x9 + # variances and covariances + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ↔ _(lat_vars) + visual ↔ textual + speed + textual ↔ speed +end + +ens_graph = @StenoGraph begin # measurement model visual → fixed(1.0, 1.0) * x1 + fixed(0.5, 0.5) * x2 + fixed(0.6, 0.8) * x3 textual → fixed(1.0, 1.0) * x4 + x5 + label(:a₁, :a₂) * x6 speed → fixed(1.0, 1.0) * x7 + fixed(1.0, NaN) * x8 + label(:λ₉, :λ₉) * x9 # variances and covariances - _(observed_vars) ↔ _(observed_vars) - _(latent_vars) ↔ _(latent_vars) + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ↔ _(lat_vars) visual ↔ textual + speed textual ↔ speed end + +@testset "ParameterTable" begin + @testset "from StenoGraph" begin + @test_throws UndefKeywordError(:observed_vars) ParameterTable(graph) + @test_throws UndefKeywordError(:latent_vars) ParameterTable( + graph, + observed_vars = obs_vars, + ) + partable = @inferred( + ParameterTable(graph, observed_vars = obs_vars, latent_vars = lat_vars) + ) + + @test partable isa ParameterTable + + # vars API + @test observed_vars(partable) == obs_vars + @test nobserved_vars(partable) == length(obs_vars) + @test latent_vars(partable) == lat_vars + @test nlatent_vars(partable) == length(lat_vars) + @test nvars(partable) == length(obs_vars) + length(lat_vars) + @test issetequal(vars(partable), [obs_vars; lat_vars]) + + # params API + @test params(partable) == [[:θ_1, :a₁, :λ₉]; Symbol.("θ_", 2:16)] + @test nparams(partable) == 18 + + # don't allow constructing ParameterTable from a graph for an ensemble + @test_throws ArgumentError ParameterTable( + ens_graph, + observed_vars = obs_vars, + latent_vars = lat_vars, + ) + end + + @testset "from RAMMatrices" begin + partable_orig = + ParameterTable(graph, observed_vars = obs_vars, latent_vars = lat_vars) + ram_matrices = RAMMatrices(partable_orig) + + partable = @inferred(ParameterTable(ram_matrices)) + @test partable isa ParameterTable + @test issetequal(keys(partable.columns), keys(partable_orig.columns)) + # FIXME nrow()? + @test length(partable.columns[:from]) == length(partable_orig.columns[:from]) + @test partable == partable_orig broken = true + end +end + +@testset "EnsembleParameterTable" begin + groups = [:Pasteur, :Grant_White], + @test_throws UndefKeywordError(:observed_vars) EnsembleParameterTable(ens_graph) + @test_throws UndefKeywordError(:latent_vars) EnsembleParameterTable( + ens_graph, + observed_vars = obs_vars, + ) + @test_throws UndefKeywordError(:groups) EnsembleParameterTable( + ens_graph, + observed_vars = obs_vars, + latent_vars = lat_vars, + ) + + enspartable = @inferred( + EnsembleParameterTable( + ens_graph, + observed_vars = obs_vars, + latent_vars = lat_vars, + groups = [:Pasteur, :Grant_White], + ) + ) + @test enspartable isa EnsembleParameterTable + + @test nobserved_vars(enspartable) == length(obs_vars) broken = true + @test observed_vars(enspartable) == obs_vars broken = true + @test nlatent_vars(enspartable) == length(lat_vars) broken = true + @test latent_vars(enspartable) == lat_vars broken = true + @test nvars(enspartable) == length(obs_vars) + length(lat_vars) broken = true + @test issetequal(vars(enspartable), [obs_vars; lat_vars]) broken = true + + @test nparams(enspartable) == 36 + @test issetequal( + params(enspartable), + [Symbol.("gPasteur_", 1:16); Symbol.("gGrant_White_", 1:17); [:a₁, :a₂, :λ₉]], + ) +end + +@testset "RAMMatrices" begin + partable = ParameterTable(graph, observed_vars = obs_vars, latent_vars = lat_vars) + + ram_matrices = @inferred(RAMMatrices(partable)) + @test ram_matrices isa RAMMatrices + + # vars API + @test nobserved_vars(ram_matrices) == length(obs_vars) + @test observed_vars(ram_matrices) == obs_vars + @test nlatent_vars(ram_matrices) == length(lat_vars) + @test latent_vars(ram_matrices) == lat_vars + @test nvars(ram_matrices) == length(obs_vars) + length(lat_vars) + @test issetequal(vars(ram_matrices), [obs_vars; lat_vars]) + + # params API + @test nparams(ram_matrices) == nparams(partable) + @test params(ram_matrices) == params(partable) +end diff --git a/test/unit_tests/unit_tests.jl b/test/unit_tests/unit_tests.jl index b8400e542..c05051487 100644 --- a/test/unit_tests/unit_tests.jl +++ b/test/unit_tests/unit_tests.jl @@ -11,3 +11,7 @@ end @safetestset "SemObserved" begin include("data_input_formats.jl") end + +@safetestset "SemSpecification" begin + include("specification.jl") +end From 2f6e8b7b91b076e0c075b3a58b206411195541df Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 27 Jun 2024 00:06:46 -0700 Subject: [PATCH 089/364] add Sem unit tests --- test/unit_tests/model.jl | 75 +++++++++++++++++++++++++++++++++++ test/unit_tests/unit_tests.jl | 4 ++ 2 files changed, 79 insertions(+) create mode 100644 test/unit_tests/model.jl diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl new file mode 100644 index 000000000..e13327642 --- /dev/null +++ b/test/unit_tests/model.jl @@ -0,0 +1,75 @@ +using StructuralEquationModels, Test, Statistics + +dat = example_data("political_democracy") +dat_missing = example_data("political_democracy_missing")[:, names(dat)] + +obs_vars = [Symbol.("x", 1:3); Symbol.("y", 1:8)] +lat_vars = [:ind60, :dem60, :dem65] + +graph = @StenoGraph begin + # loadings + ind60 → fixed(1) * x1 + x2 + x3 + dem60 → fixed(1) * y1 + y2 + y3 + y4 + dem65 → fixed(1) * y5 + y6 + y7 + y8 + # latent regressions + label(:a) * dem60 ← ind60 + dem65 ← dem60 + dem65 ← ind60 + # variances + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ↔ _(lat_vars) + # covariances + y1 ↔ y5 + y2 ↔ y4 + y6 + y3 ↔ y7 + y8 ↔ y4 + y6 +end + +ram_matrices = + RAMMatrices(ParameterTable(graph, observed_vars = obs_vars, latent_vars = lat_vars)) + +obs = SemObservedData(specification = ram_matrices, data = dat) + +function test_vars_api(semobj, spec::SemSpecification) + @test @inferred(nobserved_vars(semobj)) == nobserved_vars(spec) + @test observed_vars(semobj) == observed_vars(spec) + + @test @inferred(nlatent_vars(semobj)) == nlatent_vars(spec) + @test latent_vars(semobj) == latent_vars(spec) + + @test @inferred(nvars(semobj)) == nvars(spec) + @test vars(semobj) == vars(spec) +end + +function test_params_api(semobj, spec::SemSpecification) + @test @inferred(nparams(semobj)) == nparams(spec) + @test @inferred(params(semobj)) == params(spec) +end + +@testset "Sem(imply=$implytype, loss=$losstype)" for implytype in (RAM, RAMSymbolic), + losstype in (SemML, SemWLS) + + model = Sem( + specification = ram_matrices, + observed = obs, + imply = implytype, + loss = losstype, + ) + + @test model isa Sem + @test @inferred(imply(model)) isa implytype + @test @inferred(observed(model)) isa SemObserved + @test @inferred(optimizer(model)) isa SemOptimizer + + test_vars_api(model, ram_matrices) + test_params_api(model, ram_matrices) + + test_vars_api(imply(model), ram_matrices) + test_params_api(imply(model), ram_matrices) + + @test @inferred(loss(model)) isa SemLoss + semloss = loss(model).functions[1] + @test semloss isa losstype + + @test @inferred(nsamples(model)) == nsamples(obs) +end diff --git a/test/unit_tests/unit_tests.jl b/test/unit_tests/unit_tests.jl index c05051487..a638b991d 100644 --- a/test/unit_tests/unit_tests.jl +++ b/test/unit_tests/unit_tests.jl @@ -15,3 +15,7 @@ end @safetestset "SemSpecification" begin include("specification.jl") end + +@safetestset "Sem model" begin + include("model.jl") +end From ae6255a30a93dfccdb8a52b20387235584e5dba5 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Wed, 7 Aug 2024 14:29:47 +0200 Subject: [PATCH 090/364] dont allow fixed and labeled parameters --- src/frontend/specification/StenoGraphs.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 67bb7973c..76bd69e06 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -68,6 +68,13 @@ function ParameterTable( ) end if element isa ModifiedEdge + if any(Base.Fix2(isa, Fixed), values(element.modifiers)) & any(Base.Fix2(isa, Label), values(element.modifiers)) + throw( + ArgumentError( + "It is not allowed to label fixed parameters." + ) + ) + end for modifier in values(element.modifiers) modval = modifier.value[group] if modifier isa Fixed From 93ee7729bac8f46aeaa05e4d8716c9e0a29d7943 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Wed, 7 Aug 2024 14:37:53 +0200 Subject: [PATCH 091/364] add test for labeled and fixed parameters --- test/unit_tests/specification.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/unit_tests/specification.jl b/test/unit_tests/specification.jl index e0a412e76..e307d60f2 100644 --- a/test/unit_tests/specification.jl +++ b/test/unit_tests/specification.jl @@ -27,6 +27,11 @@ ens_graph = @StenoGraph begin textual ↔ speed end +fixed_and_labeled_graph = @StenoGraph begin + # measurement model + visual → fixed(1.0)*label(:λ)*x1 +end + @testset "ParameterTable" begin @testset "from StenoGraph" begin @test_throws UndefKeywordError(:observed_vars) ParameterTable(graph) @@ -34,6 +39,11 @@ end graph, observed_vars = obs_vars, ) + @test_throws ArgumentError("It is not allowed to label fixed parameters.") ParameterTable( + fixed_and_labeled_graph, + observed_vars = obs_vars, + latent_vars = lat_vars + ) partable = @inferred( ParameterTable(graph, observed_vars = obs_vars, latent_vars = lat_vars) ) From 8119ad231d9da83139722920e5108e020468bb89 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 31 Jul 2024 21:27:09 -0700 Subject: [PATCH 092/364] remove get_observed() does not seem to be used anywhere; also the method signature does not match Julia conventions --- src/additional_functions/helper.jl | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 138ae431e..be559b0d9 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -33,15 +33,7 @@ function semvec(observed, imply, loss, optimizer) return sem_vec end -function get_observed(rowind, data, semobserved; args = (), kwargs = NamedTuple()) - observed_vec = Vector{semobserved}(undef, length(rowind)) - for i in 1:length(rowind) - observed_vec[i] = semobserved(args...; data = Matrix(data[rowind[i], :]), kwargs...) - end - return observed_vec -end - -skipmissing_mean(mat::AbstractMatrix) = +skipmissing_mean(mat::AbstractMatrix) = [mean(skipmissing(coldata)) for coldata in eachcol(mat)] function F_one_person(imp_mean, meandiff, inverse, data, logdet) From 1c179d47df5c298643779d3fd6c6c4736962ff88 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:10:30 -0700 Subject: [PATCH 093/364] fix ridge eval --- src/loss/regularization/ridge.jl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index a61dd2af0..66ce37428 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -62,11 +62,10 @@ function SemRidge(; which_ridge = getindex.(Ref(par2ind), which_ridge) end end - which = [CartesianIndex(x) for x in which_ridge] which_H = [CartesianIndex(x, x) for x in which_ridge] return SemRidge( α_ridge, - which, + which_ridge, which_H, zeros(parameter_type, nparams), zeros(parameter_type, nparams, nparams), @@ -77,15 +76,15 @@ end ### methods ############################################################################################ -objective!(ridge::SemRidge, par, model) = @views ridge.α * sum(x -> x^2, par[ridge.which]) +objective!(ridge::SemRidge, par, model) = @views ridge.α * sum(abs2, par[ridge.which]) function gradient!(ridge::SemRidge, par, model) - @views ridge.gradient[ridge.which] .= 2 * ridge.α * par[ridge.which] + @views ridge.gradient[ridge.which] .= (2 * ridge.α) * par[ridge.which] return ridge.gradient end function hessian!(ridge::SemRidge, par, model) - @views @. ridge.hessian[ridge.which_H] += ridge.α * 2.0 + @views @. ridge.hessian[ridge.which_H] .= 2 * ridge.α return ridge.hessian end From 22e76ebe3823d3757796fc99c096d772df6f11b0 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 18 Mar 2024 17:28:21 -0700 Subject: [PATCH 094/364] MeanStructure, HessianEvaluation traits * replace has_meanstrcture and approximate_hessian fields with trait-like typeparams * remove methods for has_meanstructure-based dispatch --- src/StructuralEquationModels.jl | 6 + src/imply/RAM/generic.jl | 53 ++---- src/imply/RAM/symbolic.jl | 35 ++-- src/imply/empty.jl | 2 +- src/loss/ML/FIML.jl | 2 +- src/loss/ML/ML.jl | 212 +++++++---------------- src/loss/WLS/WLS.jl | 133 ++++---------- src/loss/constant/constant.jl | 2 +- src/loss/regularization/ridge.jl | 2 +- src/types.jl | 33 +++- test/examples/multigroup/build_models.jl | 2 +- 11 files changed, 170 insertions(+), 312 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a032ab724..a171c29d0 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -95,6 +95,12 @@ export AbstractSem, Sem, SemFiniteDiff, SemEnsemble, + MeanStructure, + NoMeanStructure, + HasMeanStructure, + HessianEvaluation, + ExactHessian, + ApproximateHessian, SemImply, RAMSymbolic, RAM, diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 9ff46bd2e..c749e3ff0 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -65,8 +65,8 @@ Additional interfaces Only available in gradient! calls: - `I_A⁻¹(::RAM)` -> ``(I-A)^{-1}`` """ -mutable struct RAM{A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S2, S3, B} <: - SemImply +mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S2, S3} <: + SemImply{MS, ExactHessian} Σ::A1 A::A2 S::A3 @@ -75,7 +75,6 @@ mutable struct RAM{A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S M::A6 ram_matrices::V2 - has_meanstructure::B A_indices::I1 S_indices::I2 @@ -89,9 +88,10 @@ mutable struct RAM{A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S ∇A::S1 ∇S::S2 ∇M::S3 -end -using StructuralEquationModels + RAM{MS}(args...) where {MS <: MeanStructure} = + new{MS, map(typeof, args)...}(args...) +end ############################################################################################ ### Constructors @@ -143,7 +143,7 @@ function RAM(; # μ if meanstructure - has_meanstructure = Val(true) + MS = HasMeanStructure !isnothing(M_indices) || throw( ArgumentError( "You set `meanstructure = true`, but your model specification contains no mean parameters.", @@ -152,14 +152,14 @@ function RAM(; ∇M = gradient ? matrix_gradient(M_indices, n_var) : nothing μ = zeros(n_obs) else - has_meanstructure = Val(false) + MS = NoMeanStructure M_indices = nothing M_pre = nothing μ = nothing ∇M = nothing end - return RAM( + return RAM{MS}( Σ, A_pre, S_pre, @@ -167,7 +167,6 @@ function RAM(; μ, M_pre, ram_matrices, - has_meanstructure, A_indices, S_indices, M_indices, @@ -185,14 +184,8 @@ end ### methods ############################################################################################ -# dispatch on meanstructure -objective!(imply::RAM, par, model::AbstractSemSingle) = - objective!(imply, par, model, imply.has_meanstructure) -gradient!(imply::RAM, par, model::AbstractSemSingle) = - gradient!(imply, par, model, imply.has_meanstructure) - # objective and gradient -function objective!(imply::RAM, params, model, has_meanstructure::Val{T}) where {T} +function objective!(imply::RAM, params, model) fill_A_S_M!( imply.A, imply.S, @@ -211,17 +204,12 @@ function objective!(imply::RAM, params, model, has_meanstructure::Val{T}) where Σ_RAM!(imply.Σ, imply.F⨉I_A⁻¹, imply.S, imply.F⨉I_A⁻¹S) - if T + if MeanStructure(imply) === HasMeanStructure μ_RAM!(imply.μ, imply.F⨉I_A⁻¹, imply.M) end end -function gradient!( - imply::RAM, - params, - model::AbstractSemSingle, - has_meanstructure::Val{T}, -) where {T} +function gradient!(imply::RAM, params, model::AbstractSemSingle) fill_A_S_M!( imply.A, imply.S, @@ -240,21 +228,18 @@ function gradient!( Σ_RAM!(imply.Σ, imply.F⨉I_A⁻¹, imply.S, imply.F⨉I_A⁻¹S) - if T + if MeanStructure(imply) === HasMeanStructure μ_RAM!(imply.μ, imply.F⨉I_A⁻¹, imply.M) end end -hessian!(imply::RAM, par, model::AbstractSemSingle, has_meanstructure) = - gradient!(imply, par, model, has_meanstructure) -objective_gradient!(imply::RAM, par, model::AbstractSemSingle, has_meanstructure) = - gradient!(imply, par, model, has_meanstructure) -objective_hessian!(imply::RAM, par, model::AbstractSemSingle, has_meanstructure) = - gradient!(imply, par, model, has_meanstructure) -gradient_hessian!(imply::RAM, par, model::AbstractSemSingle, has_meanstructure) = - gradient!(imply, par, model, has_meanstructure) -objective_gradient_hessian!(imply::RAM, par, model::AbstractSemSingle, has_meanstructure) = - gradient!(imply, par, model, has_meanstructure) +hessian!(imply::RAM, par, model::AbstractSemSingle) = gradient!(imply, par, model) +objective_gradient!(imply::RAM, par, model::AbstractSemSingle) = + gradient!(imply, par, model) +objective_hessian!(imply::RAM, par, model::AbstractSemSingle) = gradient!(imply, par, model) +gradient_hessian!(imply::RAM, par, model::AbstractSemSingle) = gradient!(imply, par, model) +objective_gradient_hessian!(imply::RAM, par, model::AbstractSemSingle) = + gradient!(imply, par, model) ############################################################################################ ### Recommended methods diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index b8da20148..a0e68c298 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -62,8 +62,8 @@ and for models with a meanstructure, the model implied means are computed as \mu = F(I-A)^{-1}M ``` """ -struct RAMSymbolic{F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5, B} <: - SemImplySymbolic +struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5} <: + SemImplySymbolic{MS, ExactHessian} Σ_function::F1 ∇Σ_function::F2 ∇²Σ_function::F3 @@ -78,7 +78,9 @@ struct RAMSymbolic{F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5, B} <: μ::A4 ∇μ_function::F5 ∇μ::A5 - has_meanstructure::B + + RAMSymbolic{MS}(args...) where {MS <: MeanStructure} = + new{MS, map(typeof, args)...}(args...) end ############################################################################################ @@ -140,7 +142,7 @@ function RAMSymbolic(; ∇Σ = nothing end - if hessian & !approximate_hessian + if hessian && !approximate_hessian n_sig = length(Σ_symbolic) ∇²Σ_symbolic_vec = [Symbolics.sparsehessian(σᵢ, [par...]) for σᵢ in vec(Σ_symbolic)] @@ -161,7 +163,7 @@ function RAMSymbolic(; # μ if meanstructure - has_meanstructure = Val(true) + MS = HasMeanStructure μ_symbolic = eval_μ_symbolic(M, I_A⁻¹, F) μ_function = Symbolics.build_function(μ_symbolic, par, expression = Val{false})[2] μ = zeros(size(μ_symbolic)) @@ -175,14 +177,14 @@ function RAMSymbolic(; ∇μ = nothing end else - has_meanstructure = Val(false) + MS = NoMeanStructure μ_function = nothing μ = nothing ∇μ_function = nothing ∇μ = nothing end - return RAMSymbolic( + return RAMSymbolic{MS}( Σ_function, ∇Σ_function, ∇²Σ_function, @@ -197,7 +199,6 @@ function RAMSymbolic(; μ, ∇μ_function, ∇μ, - has_meanstructure, ) end @@ -205,23 +206,21 @@ end ### objective, gradient, hessian ############################################################################################ -# dispatch on meanstructure -objective!(imply::RAMSymbolic, par, model) = - objective!(imply, par, model, imply.has_meanstructure) -gradient!(imply::RAMSymbolic, par, model) = - gradient!(imply, par, model, imply.has_meanstructure) - # objective -function objective!(imply::RAMSymbolic, par, model, has_meanstructure::Val{T}) where {T} +function objective!(imply::RAMSymbolic, par, model) imply.Σ_function(imply.Σ, par) - T && imply.μ_function(imply.μ, par) + if MeanStructure(imply) === HasMeanStructure + imply.μ_function(imply.μ, par) + end end # gradient -function gradient!(imply::RAMSymbolic, par, model, has_meanstructure::Val{T}) where {T} +function gradient!(imply::RAMSymbolic, par, model) objective!(imply, par, model, imply.has_meanstructure) imply.∇Σ_function(imply.∇Σ, par) - T && imply.∇μ_function(imply.∇μ, par) + if MeanStructure(imply) === HasMeanStructure + imply.∇μ_function(imply.∇μ, par) + end end # other methods diff --git a/src/imply/empty.jl b/src/imply/empty.jl index f1af2ec42..cf5270599 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -25,7 +25,7 @@ model per group and an additional model with `ImplyEmpty` and `SemRidge` for the ## Implementation Subtype of `SemImply`. """ -struct ImplyEmpty{V2} <: SemImply +struct ImplyEmpty{V2} <: SemImply{NoMeanStructure, ExactHessian} ram_matrices::V2 end diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index cd5d0270f..4a6d6b5c3 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -24,7 +24,7 @@ Analytic gradients are available. ## Implementation Subtype of `SemLossFunction`. """ -mutable struct SemFIML{INV, C, L, O, M, IM, I, T, W} <: SemLossFunction +mutable struct SemFIML{INV, C, L, O, M, IM, I, T, W} <: SemLossFunction{ExactHessian} inverses::INV #preallocated inverses of imp_cov choleskys::C #preallocated choleskys logdets::L #logdets of implied covmats diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 7811cda7f..85d36ca78 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -27,26 +27,28 @@ Analytic gradients are available, and for models without a meanstructure, also a ## Implementation Subtype of `SemLossFunction`. """ -struct SemML{INV, M, M2, B, V} <: SemLossFunction +struct SemML{HE <: HessianEvaluation, INV, M, M2} <: SemLossFunction{HE} Σ⁻¹::INV Σ⁻¹Σₒ::M meandiff::M2 - approximate_hessian::B - has_meanstructure::V + + SemML{HE}(args...) where {HE <: HessianEvaluation} = + new{HE, map(typeof, args)...}(args...) end ############################################################################################ ### Constructors ############################################################################################ -function SemML(; observed, meanstructure = false, approximate_hessian = false, kwargs...) - isnothing(obs_mean(observed)) ? meandiff = nothing : meandiff = copy(obs_mean(observed)) - return SemML( - similar(obs_cov(observed)), - similar(obs_cov(observed)), +function SemML(; observed::SemObserved, approximate_hessian::Bool = false, kwargs...) + obsmean = obs_mean(observed) + obscov = obs_cov(observed) + meandiff = isnothing(obsmean) ? nothing : copy(obsmean) + + return SemML{approximate_hessian ? ApproximateHessian : ExactHessian}( + similar(obscov), + similar(obscov), meandiff, - approximate_hessian, - Val(meanstructure), ) end @@ -54,38 +56,26 @@ end ### objective, gradient, hessian methods ############################################################################################ -# first, dispatch for meanstructure +# dispatch for SemImply objective!(semml::SemML, par, model::AbstractSemSingle) = - objective!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) + objective!(semml, par, model, imply(model)) gradient!(semml::SemML, par, model::AbstractSemSingle) = - gradient!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) + gradient!(semml, par, model, imply(model)) hessian!(semml::SemML, par, model::AbstractSemSingle) = - hessian!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) + hessian!(semml, par, model, imply(model)) objective_gradient!(semml::SemML, par, model::AbstractSemSingle) = - objective_gradient!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) + objective_gradient!(semml, par, model, imply(model)) objective_hessian!(semml::SemML, par, model::AbstractSemSingle) = - objective_hessian!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) + objective_hessian!(semml, par, model, imply(model)) gradient_hessian!(semml::SemML, par, model::AbstractSemSingle) = - gradient_hessian!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) + gradient_hessian!(semml, par, model, imply(model)) objective_gradient_hessian!(semml::SemML, par, model::AbstractSemSingle) = - objective_gradient_hessian!( - semml::SemML, - par, - model, - semml.has_meanstructure, - imply(model), - ) + objective_gradient_hessian!(semml, par, model, imply(model)) ############################################################################################ ### Symbolic Imply Types -function objective!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, - imp::SemImplySymbolic, -) where {T} +function objective!(semml::SemML, par, model::AbstractSemSingle, imp::SemImplySymbolic) let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), @@ -100,7 +90,7 @@ function objective!( Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) #mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ return ld + dot(Σ⁻¹, Σₒ) + dot(μ₋, Σ⁻¹, μ₋) else @@ -109,13 +99,7 @@ function objective!( end end -function gradient!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, - imp::SemImplySymbolic, -) where {T} +function gradient!(semml::SemML, par, model::AbstractSemSingle, imp::SemImplySymbolic) let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), @@ -131,25 +115,22 @@ function gradient!( Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ μ₋ᵀΣ⁻¹ = μ₋' * Σ⁻¹ gradient = vec(Σ⁻¹ - Σ⁻¹Σₒ * Σ⁻¹ - μ₋ᵀΣ⁻¹'μ₋ᵀΣ⁻¹)' * ∇Σ - 2 * μ₋ᵀΣ⁻¹ * ∇μ - return gradient' else gradient = vec(Σ⁻¹ - Σ⁻¹Σₒ * Σ⁻¹)' * ∇Σ - return gradient' end + return gradient' end end -function hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{false}, - imp::SemImplySymbolic, -) +function hessian!(semml::SemML, par, model::AbstractSemSingle, imp::SemImplySymbolic) + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of ML + meanstructure is not available")) + end + let Σ = Σ(imply(model)), ∇Σ = ∇Σ(imply(model)), Σₒ = obs_cov(observed(model)), @@ -158,12 +139,7 @@ function hessian!( ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || return diagm(fill(one(eltype(par)), length(par))) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - - if semml.approximate_hessian + if HessianEvaluation(semml) === ApproximateHessian hessian = 2 * ∇Σ' * kron(Σ⁻¹, Σ⁻¹) * ∇Σ else mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) @@ -181,23 +157,12 @@ function hessian!( end end -function hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{true}, - imp::SemImplySymbolic, -) - throw(DomainError(H, "hessian of ML + meanstructure is not available")) -end - function objective_gradient!( semml::SemML, par, model::AbstractSemSingle, - has_meanstructure::Val{T}, imp::SemImplySymbolic, -) where {T} +) let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), @@ -216,18 +181,17 @@ function objective_gradient!( Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ μ₋ᵀΣ⁻¹ = μ₋' * Σ⁻¹ objective = ld + tr(Σ⁻¹Σₒ) + dot(μ₋, Σ⁻¹, μ₋) gradient = vec(Σ⁻¹ * (I - Σₒ * Σ⁻¹ - μ₋ * μ₋ᵀΣ⁻¹))' * ∇Σ - 2 * μ₋ᵀΣ⁻¹ * ∇μ - return objective, gradient' else objective = ld + tr(Σ⁻¹Σₒ) gradient = (vec(Σ⁻¹) - vec(Σ⁻¹Σₒ * Σ⁻¹))' * ∇Σ - return objective, gradient' end + return objective, gradient' end end end @@ -236,9 +200,11 @@ function objective_hessian!( semml::SemML, par, model::AbstractSemSingle, - has_meanstructure::Val{T}, imp::SemImplySymbolic, -) where {T} +) + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of ML + meanstructure is not available")) + end let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), @@ -258,7 +224,7 @@ function objective_hessian!( mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) objective = ld + tr(Σ⁻¹Σₒ) - if semml.approximate_hessian + if HessianEvaluation(semml) === ApproximateHessian hessian = 2 * ∇Σ' * kron(Σ⁻¹, Σ⁻¹) * ∇Σ else Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ * Σ⁻¹ @@ -276,23 +242,16 @@ function objective_hessian!( end end -function objective_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{true}, - imp::SemImplySymbolic, -) - throw(DomainError(H, "hessian of ML + meanstructure is not available")) -end - function gradient_hessian!( semml::SemML, par, model::AbstractSemSingle, - has_meanstructure::Val{false}, imp::SemImplySymbolic, ) + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of ML + meanstructure is not available")) + end + let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), @@ -314,7 +273,7 @@ function gradient_hessian!( J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' gradient = J * ∇Σ - if semml.approximate_hessian + if HessianEvaluation(semml) === ApproximateHessian hessian = 2 * ∇Σ' * kron(Σ⁻¹, Σ⁻¹) * ∇Σ else # inner @@ -329,23 +288,16 @@ function gradient_hessian!( end end -function gradient_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{true}, - imp::SemImplySymbolic, -) - throw(DomainError(H, "hessian of ML + meanstructure is not available")) -end - function objective_gradient_hessian!( semml::SemML, par, model::AbstractSemSingle, - has_meanstructure::Val{false}, imp::SemImplySymbolic, ) + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of ML + meanstructure is not available")) + end + let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), @@ -372,7 +324,7 @@ function objective_gradient_hessian!( J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' gradient = J * ∇Σ - if semml.approximate_hessian + if HessianEvaluation(semml) == ApproximateHessian hessian = 2 * ∇Σ' * kron(Σ⁻¹, Σ⁻¹) * ∇Σ else Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ * Σ⁻¹ @@ -388,64 +340,30 @@ function objective_gradient_hessian!( end end -function objective_gradient_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{true}, - imp::SemImplySymbolic, -) - throw(DomainError(H, "hessian of ML + meanstructure is not available")) -end - ############################################################################################ ### Non-Symbolic Imply Types # no hessians ------------------------------------------------------------------------------ -function hessian!(semml::SemML, par, model::AbstractSemSingle, has_meanstructure, imp::RAM) +function hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) end -function objective_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure, - imp::RAM, -) +function objective_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) end -function gradient_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure, - imp::RAM, -) +function gradient_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) end -function objective_gradient_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure, - imp::RAM, -) +function objective_gradient_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) end # objective, gradient ---------------------------------------------------------------------- -function objective!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, - imp::RAM, -) where {T} +function objective!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), @@ -460,7 +378,7 @@ function objective!( Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ return ld + tr(Σ⁻¹Σₒ) + dot(μ₋, Σ⁻¹, μ₋) else @@ -469,13 +387,7 @@ function objective!( end end -function gradient!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, - imp::RAM, -) where {T} +function gradient!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), @@ -499,7 +411,7 @@ function gradient!( C = F⨉I_A⁻¹' * (I - Σ⁻¹Σₒ) * Σ⁻¹ * F⨉I_A⁻¹ gradient = 2vec(C * S * I_A⁻¹')'∇A + vec(C)'∇S - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ μ₋ᵀΣ⁻¹ = μ₋' * Σ⁻¹ k = μ₋ᵀΣ⁻¹ * F⨉I_A⁻¹ @@ -511,13 +423,7 @@ function gradient!( end end -function objective_gradient!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, - imp::RAM, -) where {T} +function objective_gradient!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), @@ -547,7 +453,7 @@ function objective_gradient!( C = F⨉I_A⁻¹' * (I - Σ⁻¹Σₒ) * Σ⁻¹ * F⨉I_A⁻¹ gradient = 2vec(C * S * I_A⁻¹')'∇A + vec(C)'∇S - if T + if MeanStructure(semml) === HasMeanStructure μ₋ = μₒ - μ objective += dot(μ₋, Σ⁻¹, μ₋) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 8fcc84a99..b75d47454 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -38,18 +38,19 @@ Analytic gradients are available, and for models without a meanstructure, also a ## Implementation Subtype of `SemLossFunction`. """ -struct SemWLS{Vt, St, B, C, B2} <: SemLossFunction +struct SemWLS{HE <: HessianEvaluation, Vt, St, C} <: SemLossFunction{HE} V::Vt σₒ::St - approximate_hessian::B V_μ::C - has_meanstructure::B2 end ############################################################################################ ### Constructors ############################################################################################ +SemWLS{HE}(args...) where {HE <: HessianEvaluation} = + SemWLS{HE, map(typeof, args)...}(args...) + function SemWLS(; observed, wls_weight_matrix = nothing, @@ -77,43 +78,16 @@ function SemWLS(; else wls_weight_matrix_mean = nothing end + HE = approximate_hessian ? ApproximateHessian : AnalyticHessian - return SemWLS( - wls_weight_matrix, - s, - approximate_hessian, - wls_weight_matrix_mean, - Val(meanstructure), - ) + return SemWLS{HE}(wls_weight_matrix, s, wls_weight_matrix_mean) end ############################################################################ ### methods ############################################################################ -objective!(semwls::SemWLS, par, model::AbstractSemSingle) = - objective!(semwls::SemWLS, par, model, semwls.has_meanstructure) -gradient!(semwls::SemWLS, par, model::AbstractSemSingle) = - gradient!(semwls::SemWLS, par, model, semwls.has_meanstructure) -hessian!(semwls::SemWLS, par, model::AbstractSemSingle) = - hessian!(semwls::SemWLS, par, model, semwls.has_meanstructure) - -objective_gradient!(semwls::SemWLS, par, model::AbstractSemSingle) = - objective_gradient!(semwls::SemWLS, par, model, semwls.has_meanstructure) -objective_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) = - objective_hessian!(semwls::SemWLS, par, model, semwls.has_meanstructure) -gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) = - gradient_hessian!(semwls::SemWLS, par, model, semwls.has_meanstructure) - -objective_gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) = - objective_gradient_hessian!(semwls::SemWLS, par, model, semwls.has_meanstructure) - -function objective!( - semwls::SemWLS, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, -) where {T} +function objective!(semwls::SemWLS, par, model::AbstractSemSingle) let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, @@ -123,7 +97,7 @@ function objective!( σ₋ = σₒ - σ - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ return dot(σ₋, V, σ₋) + dot(μ₋, V_μ, μ₋) else @@ -132,12 +106,7 @@ function objective!( end end -function gradient!( - semwls::SemWLS, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, -) where {T} +function gradient!(semwls::SemWLS, par, model::AbstractSemSingle) let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, @@ -149,7 +118,7 @@ function gradient!( σ₋ = σₒ - σ - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ return -2 * (σ₋' * V * ∇σ + μ₋' * V_μ * ∇μ)' else @@ -158,12 +127,7 @@ function gradient!( end end -function hessian!( - semwls::SemWLS, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, -) where {T} +function hessian!(semwls::SemWLS, par, model::AbstractSemSingle) let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, @@ -173,11 +137,11 @@ function hessian!( σ₋ = σₒ - σ - if T + if MeanStructure(imply(model)) === HasMeanStructure throw(DomainError(H, "hessian of WLS with meanstructure is not available")) else hessian = 2 * ∇σ' * V * ∇σ - if !semwls.approximate_hessian + if HessianEvaluation(semwls) === ExactHessian J = -2 * (σ₋' * semwls.V)' ∇²Σ_function!(∇²Σ, J, par) hessian .+= ∇²Σ @@ -187,12 +151,7 @@ function hessian!( end end -function objective_gradient!( - semwls::SemWLS, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, -) where {T} +function objective_gradient!(semwls::SemWLS, par, model::AbstractSemSingle) let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, @@ -204,9 +163,9 @@ function objective_gradient!( σ₋ = σₒ - σ - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ - objective = dot(σ₋, V, σ₋) + dot(μ₋', V_μ, μ₋) + objective = dot(σ₋, V, σ₋) + dot(μ₋, V_μ, μ₋) gradient = -2 * (σ₋' * V * ∇σ + μ₋' * V_μ * ∇μ)' return objective, gradient else @@ -217,12 +176,11 @@ function objective_gradient!( end end -function objective_hessian!( - semwls::SemWLS, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, -) where {T} +function objective_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of WLS with meanstructure is not available")) + end + let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, @@ -235,7 +193,7 @@ function objective_hessian!( objective = dot(σ₋, V, σ₋) hessian = 2 * ∇σ' * V * ∇σ - if !semwls.approximate_hessian + if HessianEvaluation(semwls) === ExactHessian J = -2 * (σ₋' * semwls.V)' ∇²Σ_function!(∇²Σ, J, par) hessian .+= ∇²Σ @@ -245,19 +203,11 @@ function objective_hessian!( end end -objective_hessian!( - semwls::SemWLS, - par, - model::AbstractSemSingle, - has_meanstructure::Val{true}, -) = throw(DomainError(H, "hessian of WLS with meanstructure is not available")) - -function gradient_hessian!( - semwls::SemWLS, - par, - model::AbstractSemSingle, - has_meanstructure::Val{false}, -) +function gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of WLS with meanstructure is not available")) + end + let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, @@ -270,7 +220,7 @@ function gradient_hessian!( gradient = -2 * (σ₋' * V * ∇σ)' hessian = 2 * ∇σ' * V * ∇σ - if !semwls.approximate_hessian + if HessianEvaluation(semwls) === ExactHessian J = -2 * (σ₋' * semwls.V)' ∇²Σ_function!(∇²Σ, J, par) hessian .+= ∇²Σ @@ -280,19 +230,11 @@ function gradient_hessian!( end end -gradient_hessian!( - semwls::SemWLS, - par, - model::AbstractSemSingle, - has_meanstructure::Val{true}, -) = throw(DomainError(H, "hessian of WLS with meanstructure is not available")) - -function objective_gradient_hessian!( - semwls::SemWLS, - par, - model::AbstractSemSingle, - has_meanstructure::Val{false}, -) +function objective_gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of WLS with meanstructure is not available")) + end + let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, @@ -305,7 +247,7 @@ function objective_gradient_hessian!( objective = dot(σ₋, V, σ₋) gradient = -2 * (σ₋' * V * ∇σ)' hessian = 2 * ∇σ' * V * ∇σ - if !semwls.approximate_hessian + if HessianEvaluation(semwls) === ExactHessian J = -2 * (σ₋' * semwls.V)' ∇²Σ_function!(∇²Σ, J, par) hessian .+= ∇²Σ @@ -314,13 +256,6 @@ function objective_gradient_hessian!( end end -objective_gradient_hessian!( - semwls::SemWLS, - par, - model::AbstractSemSingle, - has_meanstructure::Val{true}, -) = throw(DomainError(H, "hessian of WLS with meanstructure is not available")) - ############################################################################################ ### Recommended methods ############################################################################################ diff --git a/src/loss/constant/constant.jl b/src/loss/constant/constant.jl index f3165b541..9b3dfcd34 100644 --- a/src/loss/constant/constant.jl +++ b/src/loss/constant/constant.jl @@ -25,7 +25,7 @@ Analytic gradients and hessians are available. ## Implementation Subtype of `SemLossFunction`. """ -struct SemConstant{C} <: SemLossFunction +struct SemConstant{C} <: SemLossFunction{ExactHessian} c::C end diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index 66ce37428..e89ceeed7 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -29,7 +29,7 @@ Analytic gradients and hessians are available. ## Implementation Subtype of `SemLossFunction`. """ -struct SemRidge{P, W1, W2, GT, HT} <: SemLossFunction +struct SemRidge{P, W1, W2, GT, HT} <: SemLossFunction{ExactHessian} α::P which::W1 which_H::W2 diff --git a/src/types.jl b/src/types.jl index 0493da8fa..6cdf9bead 100644 --- a/src/types.jl +++ b/src/types.jl @@ -10,8 +10,32 @@ abstract type AbstractSemSingle{O, I, L, D} <: AbstractSem end "Supertype for all collections of multiple SEMs" abstract type AbstractSemCollection <: AbstractSem end +"Meanstructure trait for `SemImply` subtypes" +abstract type MeanStructure end +"Indicates that `SemImply` subtype supports meanstructure" +struct HasMeanStructure <: MeanStructure end +"Indicates that `SemImply` subtype does not support meanstructure" +struct NoMeanStructure <: MeanStructure end + +# fallback implementation +MeanStructure(::Type{T}) where {T} = + error("Objects of type $T do not support MeanStructure trait") +MeanStructure(semobj) = MeanStructure(typeof(semobj)) + +"Hessian Evaluation trait for `SemImply` and `SemLossFunction` subtypes" +abstract type HessianEvaluation end +struct ApproximateHessian <: HessianEvaluation end +struct ExactHessian <: HessianEvaluation end + +# fallback implementation +HessianEvaluation(::Type{T}) where {T} = + error("Objects of type $T do not support HessianEvaluation trait") +HessianEvaluation(semobj) = HessianEvaluation(typeof(semobj)) + "Supertype for all loss functions of SEMs. If you want to implement a custom loss function, it should be a subtype of `SemLossFunction`." -abstract type SemLossFunction end +abstract type SemLossFunction{HE <: HessianEvaluation} end + +HessianEvaluation(::Type{<:SemLossFunction{HE}}) where {HE <: HessianEvaluation} = HE """ SemLoss(args...; loss_weights = nothing, ...) @@ -73,10 +97,13 @@ Computed model-implied values that should be compared with the observed data to e. g. the model implied covariance or mean. If you would like to implement a different notation, e.g. LISREL, you should implement a subtype of SemImply. """ -abstract type SemImply end +abstract type SemImply{MS <: MeanStructure, HE <: HessianEvaluation} end + +MeanStructure(::Type{<:SemImply{MS}}) where {MS <: MeanStructure} = MS +HessianEvaluation(::Type{<:SemImply{MS, HE}}) where {MS, HE <: MeanStructure} = HE "Subtype of SemImply for all objects that can serve as the imply field of a SEM and use some form of symbolic precomputation." -abstract type SemImplySymbolic <: SemImply end +abstract type SemImplySymbolic{MS, HE} <: SemImply{MS, HE} end """ Sem(;observed = SemObservedData, imply = RAM, loss = SemML, optimizer = SemOptimizerOptim, kwargs...) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 70d2bb914..265ab178a 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -114,7 +114,7 @@ end # ML estimation - user defined loss function ############################################################################################ -struct UserSemML <: SemLossFunction end +struct UserSemML <: SemLossFunction{ExactHessian} end ############################################################################################ ### functors From af09c79613856adc7a5fba9d0f09cf92f303b313 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 11 Aug 2024 14:05:22 -0700 Subject: [PATCH 095/364] obj/grad/hess: refactor evaluation API the intent of this commit is to refactor the API for objective, gradient and hessian evaluation, such that the evaluation code does not have to be duplicates across functions that calculate different combinations of those functions * introduce EvaluationTargets class that handles selection of what to evaluate * add evaluate!(EvalTargets, ...) methods for loss and imply objs that evaluate only what is required * objective!(), obj_grad!() etc calls are just a wrapper of evaluate!() with proper targets --- src/frontend/fit/standard_errors/hessian.jl | 2 +- src/imply/RAM/generic.jl | 116 ++--- src/imply/RAM/symbolic.jl | 46 +- src/imply/empty.jl | 4 +- src/loss/ML/FIML.jl | 90 ++-- src/loss/ML/ML.jl | 475 +++++--------------- src/loss/WLS/WLS.jl | 199 ++------ src/loss/constant/constant.jl | 7 +- src/loss/regularization/ridge.jl | 7 +- src/objective_gradient_hessian.jl | 442 ++++++------------ test/examples/multigroup/build_models.jl | 4 +- 11 files changed, 377 insertions(+), 1015 deletions(-) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index afcb570bc..e71e601fb 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -17,7 +17,7 @@ function se_hessian(sem_fit::SemFit; hessian = :finitediff) hessian!(H, sem_fit.model, sem_fit.solution) elseif hessian == :finitediff H = FiniteDiff.finite_difference_hessian( - Base.Fix1(objective!, sem_fit.model), + p -> evaluate!(zero(eltype(sem_fit.solution)), nothing, nothing, fit.model, p), sem_fit.solution, ) elseif hessian == :optimizer diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index c749e3ff0..85cbc0220 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -65,8 +65,26 @@ Additional interfaces Only available in gradient! calls: - `I_A⁻¹(::RAM)` -> ``(I-A)^{-1}`` """ -mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S2, S3} <: - SemImply{MS, ExactHessian} +mutable struct RAM{ + MS, + A1, + A2, + A3, + A4, + A5, + A6, + V2, + I1, + I2, + I3, + M1, + M2, + M3, + M4, + S1, + S2, + S3, +} <: SemImply{MS, ExactHessian} Σ::A1 A::A2 S::A3 @@ -89,8 +107,7 @@ mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S ∇S::S2 ∇M::S3 - RAM{MS}(args...) where {MS <: MeanStructure} = - new{MS, map(typeof, args)...}(args...) + RAM{MS}(args...) where {MS <: MeanStructure} = new{MS, map(typeof, args)...}(args...) end ############################################################################################ @@ -100,7 +117,7 @@ end function RAM(; specification::SemSpecification, #vech = false, - gradient = true, + gradient_required = true, meanstructure = false, kwargs..., ) @@ -133,7 +150,7 @@ function RAM(; F⨉I_A⁻¹S = zeros(n_obs, n_var) I_A = similar(A_pre) - if gradient + if gradient_required ∇A = matrix_gradient(A_indices, n_var^2) ∇S = matrix_gradient(S_indices, n_var^2) else @@ -149,7 +166,7 @@ function RAM(; "You set `meanstructure = true`, but your model specification contains no mean parameters.", ), ) - ∇M = gradient ? matrix_gradient(M_indices, n_var) : nothing + ∇M = gradient_required ? matrix_gradient(M_indices, n_var) : nothing μ = zeros(n_obs) else MS = NoMeanStructure @@ -184,8 +201,7 @@ end ### methods ############################################################################################ -# objective and gradient -function objective!(imply::RAM, params, model) +function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingle, params) fill_A_S_M!( imply.A, imply.S, @@ -199,48 +215,22 @@ function objective!(imply::RAM, params, model) @. imply.I_A = -imply.A @view(imply.I_A[diagind(imply.I_A)]) .+= 1 - copyto!(imply.F⨉I_A⁻¹, imply.F) - rdiv!(imply.F⨉I_A⁻¹, factorize(imply.I_A)) - - Σ_RAM!(imply.Σ, imply.F⨉I_A⁻¹, imply.S, imply.F⨉I_A⁻¹S) - - if MeanStructure(imply) === HasMeanStructure - μ_RAM!(imply.μ, imply.F⨉I_A⁻¹, imply.M) + if is_gradient_required(targets) || is_hessian_required(targets) + imply.I_A⁻¹ = LinearAlgebra.inv!(factorize(imply.I_A)) + mul!(imply.F⨉I_A⁻¹, imply.F, imply.I_A⁻¹) + else + copyto!(imply.F⨉I_A⁻¹, imply.F) + rdiv!(imply.F⨉I_A⁻¹, factorize(imply.I_A)) end -end - -function gradient!(imply::RAM, params, model::AbstractSemSingle) - fill_A_S_M!( - imply.A, - imply.S, - imply.M, - imply.A_indices, - imply.S_indices, - imply.M_indices, - params, - ) - - @. imply.I_A = -imply.A - @view(imply.I_A[diagind(imply.I_A)]) .+= 1 - - imply.I_A⁻¹ = LinearAlgebra.inv!(factorize(imply.I_A)) - mul!(imply.F⨉I_A⁻¹, imply.F, imply.I_A⁻¹) - Σ_RAM!(imply.Σ, imply.F⨉I_A⁻¹, imply.S, imply.F⨉I_A⁻¹S) + mul!(imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹, imply.S) + mul!(imply.Σ, imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹') if MeanStructure(imply) === HasMeanStructure - μ_RAM!(imply.μ, imply.F⨉I_A⁻¹, imply.M) + mul!(imply.μ, imply.F⨉I_A⁻¹, imply.M) end end -hessian!(imply::RAM, par, model::AbstractSemSingle) = gradient!(imply, par, model) -objective_gradient!(imply::RAM, par, model::AbstractSemSingle) = - gradient!(imply, par, model) -objective_hessian!(imply::RAM, par, model::AbstractSemSingle) = gradient!(imply, par, model) -gradient_hessian!(imply::RAM, par, model::AbstractSemSingle) = gradient!(imply, par, model) -objective_gradient_hessian!(imply::RAM, par, model::AbstractSemSingle) = - gradient!(imply, par, model) - ############################################################################################ ### Recommended methods ############################################################################################ @@ -253,48 +243,10 @@ function update_observed(imply::RAM, observed::SemObserved; kwargs...) end end -############################################################################################ -### additional methods -############################################################################################ - -Σ(imply::RAM) = imply.Σ -μ(imply::RAM) = imply.μ - -A(imply::RAM) = imply.A -S(imply::RAM) = imply.S -F(imply::RAM) = imply.F -M(imply::RAM) = imply.M - -∇A(imply::RAM) = imply.∇A -∇S(imply::RAM) = imply.∇S -∇M(imply::RAM) = imply.∇M - -A_indices(imply::RAM) = imply.A_indices -S_indices(imply::RAM) = imply.S_indices -M_indices(imply::RAM) = imply.M_indices - -F⨉I_A⁻¹(imply::RAM) = imply.F⨉I_A⁻¹ -F⨉I_A⁻¹S(imply::RAM) = imply.F⨉I_A⁻¹S -I_A(imply::RAM) = imply.I_A -I_A⁻¹(imply::RAM) = imply.I_A⁻¹ # only for gradient available! - -has_meanstructure(imply::RAM) = imply.has_meanstructure - -ram_matrices(imply::RAM) = imply.ram_matrices - ############################################################################################ ### additional functions ############################################################################################ -function Σ_RAM!(Σ, F⨉I_A⁻¹, S, pre2) - mul!(pre2, F⨉I_A⁻¹, S) - mul!(Σ, pre2, F⨉I_A⁻¹') -end - -function μ_RAM!(μ, F⨉I_A⁻¹, M) - mul!(μ, F⨉I_A⁻¹, M) -end - function check_acyclic(A_pre, n_par, A_indices) # fill copy of A-matrix with random parameters A_rand = copy(A_pre) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index a0e68c298..d79454f3f 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -206,30 +206,25 @@ end ### objective, gradient, hessian ############################################################################################ -# objective -function objective!(imply::RAMSymbolic, par, model) +function update!( + targets::EvaluationTargets, + imply::RAMSymbolic, + model::AbstractSemSingle, + par, +) imply.Σ_function(imply.Σ, par) if MeanStructure(imply) === HasMeanStructure imply.μ_function(imply.μ, par) end -end -# gradient -function gradient!(imply::RAMSymbolic, par, model) - objective!(imply, par, model, imply.has_meanstructure) - imply.∇Σ_function(imply.∇Σ, par) - if MeanStructure(imply) === HasMeanStructure - imply.∇μ_function(imply.∇μ, par) + if is_gradient_required(targets) || is_hessian_required(targets) + imply.∇Σ_function(imply.∇Σ, par) + if MeanStructure(imply) === HasMeanStructure + imply.∇μ_function(imply.∇μ, par) + end end end -# other methods -hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, par, model) -objective_gradient!(imply::RAMSymbolic, par, model) = gradient!(imply, par, model) -objective_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, par, model) -gradient_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, par, model) -objective_gradient_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, par, model) - ############################################################################################ ### Recommended methods ############################################################################################ @@ -242,25 +237,6 @@ function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) end end -############################################################################################ -### additional methods -############################################################################################ - -Σ(imply::RAMSymbolic) = imply.Σ -∇Σ(imply::RAMSymbolic) = imply.∇Σ -∇²Σ(imply::RAMSymbolic) = imply.∇²Σ - -μ(imply::RAMSymbolic) = imply.μ -∇μ(imply::RAMSymbolic) = imply.∇μ - -Σ_function(imply::RAMSymbolic) = imply.Σ_function -∇Σ_function(imply::RAMSymbolic) = imply.∇Σ_function -∇²Σ_function(imply::RAMSymbolic) = imply.∇²Σ_function - -has_meanstructure(imply::RAMSymbolic) = imply.has_meanstructure - -ram_matrices(imply::RAMSymbolic) = imply.ram_matrices - ############################################################################################ ### additional functions ############################################################################################ diff --git a/src/imply/empty.jl b/src/imply/empty.jl index cf5270599..8b23194ac 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -41,9 +41,7 @@ end ### methods ############################################################################################ -objective!(imply::ImplyEmpty, par, model) = nothing -gradient!(imply::ImplyEmpty, par, model) = nothing -hessian!(imply::ImplyEmpty, par, model) = nothing +update!(targets::EvaluationTargets, imply::ImplyEmpty, par, model) = nothing ############################################################################################ ### Recommended methods diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 4a6d6b5c3..92ecf73ca 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -82,43 +82,32 @@ end ### methods ############################################################################################ -function objective!(semfiml::SemFIML, params, model) - if !check_fiml(semfiml, model) - return non_posdef_return(params) - end - - prepare_SemFIML!(semfiml, model) - - objective = F_FIML(pattern_rows(observed(model)), semfiml, model, params) - return objective / nsamples(observed(model)) -end - -function gradient!(semfiml::SemFIML, params, model) - if !check_fiml(semfiml, model) - return ones(eltype(params), size(params)) - end - - prepare_SemFIML!(semfiml, model) - - gradient = - ∇F_FIML(pattern_rows(observed(model)), semfiml, model) / nsamples(observed(model)) - return gradient -end +function evaluate!( + objective, + gradient, + hessian, + semfiml::SemFIML, + implied::SemImply, + model::AbstractSemSingle, + params, +) + isnothing(hessian) || error("Hessian not implemented for FIML") -function objective_gradient!(semfiml::SemFIML, params, model) if !check_fiml(semfiml, model) - return non_posdef_return(params), ones(eltype(params), size(params)) + isnothing(objective) || (objective = non_posdef_return(params)) + isnothing(gradient) || fill!(gradient, 1) + return objective end prepare_SemFIML!(semfiml, model) - objective = - F_FIML(pattern_rows(observed(model)), semfiml, model, params) / - nsamples(observed(model)) - gradient = - ∇F_FIML(pattern_rows(observed(model)), semfiml, model) / nsamples(observed(model)) + scale = inv(nsamples(observed(model))) + obs_rows = pattern_rows(observed(model)) + isnothing(objective) || (objective = scale * F_FIML(obs_rows, semfiml, model, params)) + isnothing(gradient) || + (∇F_FIML!(gradient, obs_rows, semfiml, model); gradient .*= scale) - return objective, gradient + return objective end ############################################################################################ @@ -133,13 +122,11 @@ update_observed(lossfun::SemFIML, observed::SemObserved; kwargs...) = ############################################################################################ function F_one_pattern(meandiff, inverse, obs_cov, logdet, N) - F = logdet - F += meandiff' * inverse * meandiff + F = logdet + dot(meandiff, inverse, meandiff) if N > one(N) F += dot(obs_cov, inverse) end - F = N * F - return F + return F * N end function ∇F_one_pattern(μ_diff, Σ⁻¹, S, pattern, ∇ind, N, Jμ, JΣ, model) @@ -155,26 +142,23 @@ function ∇F_one_pattern(μ_diff, Σ⁻¹, S, pattern, ∇ind, N, Jμ, JΣ, mod end end -function ∇F_fiml_outer(JΣ, Jμ, imply::SemImplySymbolic, model, semfiml) - G = transpose(JΣ' * ∇Σ(imply) - Jμ' * ∇μ(imply)) - return G +function ∇F_fiml_outer!(G, JΣ, Jμ, imply::SemImplySymbolic, model, semfiml) + mul!(G, imply.∇Σ', JΣ) # should be transposed + G .-= imply.∇μ' * Jμ end -function ∇F_fiml_outer(JΣ, Jμ, imply, model, semfiml) - Iₙ = sparse(1.0I, size(A(imply))...) - P = kron(F⨉I_A⁻¹(imply), F⨉I_A⁻¹(imply)) - Q = kron(S(imply) * I_A⁻¹(imply)', Iₙ) +function ∇F_fiml_outer!(G, JΣ, Jμ, imply, model, semfiml) + Iₙ = sparse(1.0I, size(imply.A)...) + P = kron(imply.F⨉I_A⁻¹, imply.F⨉I_A⁻¹) + Q = kron(imply.S * imply.I_A⁻¹', Iₙ) Q .+= semfiml.commutator * Q - ∇Σ = P * (∇S(imply) + Q * ∇A(imply)) - - ∇μ = - F⨉I_A⁻¹(imply) * ∇M(imply) + - kron((I_A⁻¹(imply) * M(imply))', F⨉I_A⁻¹(imply)) * ∇A(imply) + ∇Σ = P * (imply.∇S + Q * imply.∇A) - G = transpose(JΣ' * ∇Σ - Jμ' * ∇μ) + ∇μ = imply.F⨉I_A⁻¹ * imply.∇M + kron((imply.I_A⁻¹ * imply.M)', imply.F⨉I_A⁻¹) * imply.∇A - return G + mul!(G, ∇Σ', JΣ) # actually transposed + G .-= ∇μ' * Jμ end function F_FIML(rows, semfiml, model, params) @@ -191,7 +175,7 @@ function F_FIML(rows, semfiml, model, params) return F end -function ∇F_FIML(rows, semfiml, model) +function ∇F_FIML!(G, rows, semfiml, model) Jμ = zeros(nobserved_vars(model)) JΣ = zeros(nobserved_vars(model)^2) @@ -208,7 +192,7 @@ function ∇F_FIML(rows, semfiml, model) model, ) end - return ∇F_fiml_outer(JΣ, Jμ, imply(model), model, semfiml) + return ∇F_fiml_outer!(G, JΣ, Jμ, imply(model), model, semfiml) end function prepare_SemFIML!(semfiml, model) @@ -233,9 +217,9 @@ end copy_per_pattern!(semfiml, model::M where {M <: AbstractSem}) = copy_per_pattern!( semfiml.inverses, - Σ(imply(model)), + imply(model).Σ, semfiml.imp_mean, - μ(imply(model)), + imply(model).μ, patterns(observed(model)), ) @@ -248,7 +232,7 @@ function batch_cholesky!(semfiml, model) end function check_fiml(semfiml, model) - copyto!(semfiml.imp_inv, Σ(imply(model))) + copyto!(semfiml.imp_inv, imply(model).Σ) a = cholesky!(Symmetric(semfiml.imp_inv); check = false) return isposdef(a) end diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 85d36ca78..445a557a7 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -56,415 +56,149 @@ end ### objective, gradient, hessian methods ############################################################################################ -# dispatch for SemImply -objective!(semml::SemML, par, model::AbstractSemSingle) = - objective!(semml, par, model, imply(model)) -gradient!(semml::SemML, par, model::AbstractSemSingle) = - gradient!(semml, par, model, imply(model)) -hessian!(semml::SemML, par, model::AbstractSemSingle) = - hessian!(semml, par, model, imply(model)) -objective_gradient!(semml::SemML, par, model::AbstractSemSingle) = - objective_gradient!(semml, par, model, imply(model)) -objective_hessian!(semml::SemML, par, model::AbstractSemSingle) = - objective_hessian!(semml, par, model, imply(model)) -gradient_hessian!(semml::SemML, par, model::AbstractSemSingle) = - gradient_hessian!(semml, par, model, imply(model)) -objective_gradient_hessian!(semml::SemML, par, model::AbstractSemSingle) = - objective_gradient_hessian!(semml, par, model, imply(model)) - ############################################################################################ ### Symbolic Imply Types -function objective!(semml::SemML, par, model::AbstractSemSingle, imp::SemImplySymbolic) - let Σ = Σ(imply(model)), - Σₒ = obs_cov(observed(model)), - Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), - μ = μ(imply(model)), - μₒ = obs_mean(observed(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || return non_posdef_return(par) - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - #mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - return ld + dot(Σ⁻¹, Σₒ) + dot(μ₋, Σ⁻¹, μ₋) - else - return ld + dot(Σ⁻¹, Σₒ) - end - end -end - -function gradient!(semml::SemML, par, model::AbstractSemSingle, imp::SemImplySymbolic) - let Σ = Σ(imply(model)), - Σₒ = obs_cov(observed(model)), - Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), - ∇Σ = ∇Σ(imply(model)), - μ = μ(imply(model)), - ∇μ = ∇μ(imply(model)), - μₒ = obs_mean(observed(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || return ones(eltype(par), size(par)) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - μ₋ᵀΣ⁻¹ = μ₋' * Σ⁻¹ - gradient = vec(Σ⁻¹ - Σ⁻¹Σₒ * Σ⁻¹ - μ₋ᵀΣ⁻¹'μ₋ᵀΣ⁻¹)' * ∇Σ - 2 * μ₋ᵀΣ⁻¹ * ∇μ - else - gradient = vec(Σ⁻¹ - Σ⁻¹Σₒ * Σ⁻¹)' * ∇Σ - end - return gradient' - end -end - -function hessian!(semml::SemML, par, model::AbstractSemSingle, imp::SemImplySymbolic) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of ML + meanstructure is not available")) - end - - let Σ = Σ(imply(model)), - ∇Σ = ∇Σ(imply(model)), - Σₒ = obs_cov(observed(model)), - Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), - ∇²Σ_function! = ∇²Σ_function(imply(model)), - ∇²Σ = ∇²Σ(imply(model)) - - if HessianEvaluation(semml) === ApproximateHessian - hessian = 2 * ∇Σ' * kron(Σ⁻¹, Σ⁻¹) * ∇Σ - else - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ * Σ⁻¹ - # inner - J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' - ∇²Σ_function!(∇²Σ, J, par) - # outer - H_outer = kron(2Σ⁻¹ΣₒΣ⁻¹ - Σ⁻¹, Σ⁻¹) - hessian = ∇Σ' * H_outer * ∇Σ - hessian .+= ∇²Σ - end - - return hessian - end -end - -function objective_gradient!( +function evaluate!( + objective, + gradient, + hessian, semml::SemML, - par, + implied::SemImplySymbolic, model::AbstractSemSingle, - imp::SemImplySymbolic, -) - let Σ = Σ(imply(model)), - Σₒ = obs_cov(observed(model)), - Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), - μ = μ(imply(model)), - μₒ = obs_mean(observed(model)), - ∇Σ = ∇Σ(imply(model)), - ∇μ = ∇μ(imply(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - if !isposdef(Σ_chol) - return non_posdef_return(par), ones(eltype(par), size(par)) - else - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - μ₋ᵀΣ⁻¹ = μ₋' * Σ⁻¹ - - objective = ld + tr(Σ⁻¹Σₒ) + dot(μ₋, Σ⁻¹, μ₋) - gradient = vec(Σ⁻¹ * (I - Σₒ * Σ⁻¹ - μ₋ * μ₋ᵀΣ⁻¹))' * ∇Σ - 2 * μ₋ᵀΣ⁻¹ * ∇μ - else - objective = ld + tr(Σ⁻¹Σₒ) - gradient = (vec(Σ⁻¹) - vec(Σ⁻¹Σₒ * Σ⁻¹))' * ∇Σ - end - return objective, gradient' - end - end -end - -function objective_hessian!( - semml::SemML, par, - model::AbstractSemSingle, - imp::SemImplySymbolic, ) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of ML + meanstructure is not available")) - end - let Σ = Σ(imply(model)), - Σₒ = obs_cov(observed(model)), - Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), - ∇Σ = ∇Σ(imply(model)), - ∇μ = ∇μ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), - ∇²Σ = ∇²Σ(imply(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - if !isposdef(Σ_chol) - return non_posdef_return(par), diagm(fill(one(eltype(par)), length(par))) - else - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - objective = ld + tr(Σ⁻¹Σₒ) + if !isnothing(hessian) + (MeanStructure(implied) === HasMeanStructure) && + throw(DomainError(H, "hessian of ML + meanstructure is not available")) + end + + Σ = implied.Σ + Σₒ = obs_cov(observed(model)) + Σ⁻¹Σₒ = semml.Σ⁻¹Σₒ + Σ⁻¹ = semml.Σ⁻¹ + + copyto!(Σ⁻¹, Σ) + Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) + if !isposdef(Σ_chol) + #@warn "∑⁻¹ is not positive definite" + isnothing(objective) || (objective = non_posdef_return(par)) + isnothing(gradient) || fill!(gradient, 1) + isnothing(hessian) || copyto!(hessian, I) + return objective + end + ld = logdet(Σ_chol) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) + mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) + isnothing(objective) || (objective = ld + tr(Σ⁻¹Σₒ)) + + if MeanStructure(implied) === HasMeanStructure + μ = implied.μ + μₒ = obs_mean(observed(model)) + μ₋ = μₒ - μ + isnothing(objective) || (objective += dot(μ₋, Σ⁻¹, μ₋)) + if !isnothing(gradient) + ∇Σ = implied.∇Σ + ∇μ = implied.∇μ + μ₋ᵀΣ⁻¹ = μ₋' * Σ⁻¹ + gradient .= (vec(Σ⁻¹ - Σ⁻¹Σₒ * Σ⁻¹ - μ₋ᵀΣ⁻¹'μ₋ᵀΣ⁻¹)' * ∇Σ)' + gradient .-= (2 * μ₋ᵀΣ⁻¹ * ∇μ)' + end + elseif !isnothing(gradient) || !isnothing(hessian) + ∇Σ = implied.∇Σ + Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ * Σ⁻¹ + J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' + if !isnothing(gradient) + gradient .= (J * ∇Σ)' + end + if !isnothing(hessian) if HessianEvaluation(semml) === ApproximateHessian - hessian = 2 * ∇Σ' * kron(Σ⁻¹, Σ⁻¹) * ∇Σ + mul!(hessian, 2 * ∇Σ' * kron(Σ⁻¹, Σ⁻¹), ∇Σ) else - Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ * Σ⁻¹ + ∇²Σ_function! = implied.∇²Σ_function + ∇²Σ = implied.∇²Σ # inner - J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' ∇²Σ_function!(∇²Σ, J, par) # outer H_outer = kron(2Σ⁻¹ΣₒΣ⁻¹ - Σ⁻¹, Σ⁻¹) - hessian = ∇Σ' * H_outer * ∇Σ + mul!(hessian, ∇Σ' * H_outer, ∇Σ) hessian .+= ∇²Σ end - - return objective, hessian end end + return objective end -function gradient_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - imp::SemImplySymbolic, -) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of ML + meanstructure is not available")) - end - - let Σ = Σ(imply(model)), - Σₒ = obs_cov(observed(model)), - Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), - ∇Σ = ∇Σ(imply(model)), - ∇μ = ∇μ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), - ∇²Σ = ∇²Σ(imply(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || - return ones(eltype(par), size(par)), diagm(fill(one(eltype(par)), length(par))) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - - Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ * Σ⁻¹ - - J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' - gradient = J * ∇Σ - - if HessianEvaluation(semml) === ApproximateHessian - hessian = 2 * ∇Σ' * kron(Σ⁻¹, Σ⁻¹) * ∇Σ - else - # inner - ∇²Σ_function!(∇²Σ, J, par) - # outer - H_outer = kron(2Σ⁻¹ΣₒΣ⁻¹ - Σ⁻¹, Σ⁻¹) - hessian = ∇Σ' * H_outer * ∇Σ - hessian .+= ∇²Σ - end - - return gradient', hessian - end -end +############################################################################################ +### Non-Symbolic Imply Types -function objective_gradient_hessian!( +function evaluate!( + objective, + gradient, + hessian, semml::SemML, - par, + implied::RAM, model::AbstractSemSingle, - imp::SemImplySymbolic, + par, ) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of ML + meanstructure is not available")) + if !isnothing(hessian) + error("hessian of ML + non-symbolic imply type is not available") end - let Σ = Σ(imply(model)), - Σₒ = obs_cov(observed(model)), - Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), - ∇Σ = ∇Σ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), - ∇²Σ = ∇²Σ(imply(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - if !isposdef(Σ_chol) - objective = non_posdef_return(par) - gradient = ones(eltype(par), size(par)) - hessian = diagm(fill(one(eltype(par)), length(par))) - return objective, gradient, hessian - end - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - objective = ld + tr(Σ⁻¹Σₒ) + Σ = implied.Σ + Σₒ = obs_cov(observed(model)) + Σ⁻¹Σₒ = semml.Σ⁻¹Σₒ + Σ⁻¹ = semml.Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ * Σ⁻¹ - - J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' - gradient = J * ∇Σ - - if HessianEvaluation(semml) == ApproximateHessian - hessian = 2 * ∇Σ' * kron(Σ⁻¹, Σ⁻¹) * ∇Σ - else - Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ * Σ⁻¹ - # inner - ∇²Σ_function!(∇²Σ, J, par) - # outer - H_outer = kron(2Σ⁻¹ΣₒΣ⁻¹ - Σ⁻¹, Σ⁻¹) - hessian = ∇Σ' * H_outer * ∇Σ - hessian .+= ∇²Σ - end - - return objective, gradient', hessian + copyto!(Σ⁻¹, Σ) + Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) + if !isposdef(Σ_chol) + #@warn "Σ⁻¹ is not positive definite" + isnothing(objective) || (objective = non_posdef_return(par)) + isnothing(gradient) || fill!(gradient, 1) + isnothing(hessian) || copyto!(hessian, I) + return objective end -end - -############################################################################################ -### Non-Symbolic Imply Types - -# no hessians ------------------------------------------------------------------------------ - -function hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) -end - -function objective_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) -end - -function gradient_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) -end + ld = logdet(Σ_chol) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) + mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) -function objective_gradient_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) -end - -# objective, gradient ---------------------------------------------------------------------- - -function objective!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - let Σ = Σ(imply(model)), - Σₒ = obs_cov(observed(model)), - Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), - μ = μ(imply(model)), - μₒ = obs_mean(observed(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || return non_posdef_return(par) - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) + if !isnothing(objective) + objective = ld + tr(Σ⁻¹Σₒ) - if MeanStructure(imply(model)) === HasMeanStructure + if MeanStructure(implied) === HasMeanStructure + μ = implied.μ + μₒ = obs_mean(observed(model)) μ₋ = μₒ - μ - return ld + tr(Σ⁻¹Σₒ) + dot(μ₋, Σ⁻¹, μ₋) - else - return ld + tr(Σ⁻¹Σₒ) + objective += dot(μ₋, Σ⁻¹, μ₋) end end -end -function gradient!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - let Σ = Σ(imply(model)), - Σₒ = obs_cov(observed(model)), - Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), - S = S(imply(model)), - M = M(imply(model)), - F⨉I_A⁻¹ = F⨉I_A⁻¹(imply(model)), - I_A⁻¹ = I_A⁻¹(imply(model)), - ∇A = ∇A(imply(model)), - ∇S = ∇S(imply(model)), - ∇M = ∇M(imply(model)), - μ = μ(imply(model)), - μₒ = obs_mean(observed(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || return ones(eltype(par), size(par)) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) + if !isnothing(gradient) + S = implied.S + F⨉I_A⁻¹ = implied.F⨉I_A⁻¹ + I_A⁻¹ = implied.I_A⁻¹ + ∇A = implied.∇A + ∇S = implied.∇S C = F⨉I_A⁻¹' * (I - Σ⁻¹Σₒ) * Σ⁻¹ * F⨉I_A⁻¹ - gradient = 2vec(C * S * I_A⁻¹')'∇A + vec(C)'∇S + gradᵀ = 2vec(C * S * I_A⁻¹')'∇A + vec(C)'∇S - if MeanStructure(imply(model)) === HasMeanStructure + if MeanStructure(implied) === HasMeanStructure + μ = implied.μ + μₒ = obs_mean(observed(model)) + ∇M = implied.∇M + M = implied.M μ₋ = μₒ - μ μ₋ᵀΣ⁻¹ = μ₋' * Σ⁻¹ k = μ₋ᵀΣ⁻¹ * F⨉I_A⁻¹ - - gradient .+= -2k * ∇M - 2vec(k' * (M' + k * S) * I_A⁻¹')'∇A - vec(k'k)'∇S + gradᵀ .+= -2k * ∇M - 2vec(k' * (M' + k * S) * I_A⁻¹')'∇A - vec(k'k)'∇S end - - return gradient' + copyto!(gradient, gradᵀ') end -end - -function objective_gradient!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - let Σ = Σ(imply(model)), - Σₒ = obs_cov(observed(model)), - Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), - S = S(imply(model)), - M = M(imply(model)), - F⨉I_A⁻¹ = F⨉I_A⁻¹(imply(model)), - I_A⁻¹ = I_A⁻¹(imply(model)), - ∇A = ∇A(imply(model)), - ∇S = ∇S(imply(model)), - ∇M = ∇M(imply(model)), - μ = μ(imply(model)), - μₒ = obs_mean(observed(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - if !isposdef(Σ_chol) - objective = non_posdef_return(par) - gradient = ones(eltype(par), size(par)) - return objective, gradient - else - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - objective = ld + tr(Σ⁻¹Σₒ) - - C = F⨉I_A⁻¹' * (I - Σ⁻¹Σₒ) * Σ⁻¹ * F⨉I_A⁻¹ - gradient = 2vec(C * S * I_A⁻¹')'∇A + vec(C)'∇S - - if MeanStructure(semml) === HasMeanStructure - μ₋ = μₒ - μ - objective += dot(μ₋, Σ⁻¹, μ₋) - - μ₋ᵀΣ⁻¹ = μ₋' * Σ⁻¹ - k = μ₋ᵀΣ⁻¹ * F⨉I_A⁻¹ - gradient .+= -2k * ∇M - 2vec(k' * (M' + k * S) * I_A⁻¹')'∇A - vec(k'k)'∇S - end - return objective, gradient' - end - end + return objective end ############################################################################################ @@ -484,7 +218,7 @@ end ############################################################################################ update_observed(lossfun::SemML, observed::SemObservedMissing; kwargs...) = - throw(ArgumentError("ML estimation does not work with missing data - use FIML instead")) + error("ML estimation does not work with missing data - use FIML instead") function update_observed(lossfun::SemML, observed::SemObserved; kwargs...) if size(lossfun.Σ⁻¹) == size(obs_cov(observed)) @@ -493,10 +227,3 @@ function update_observed(lossfun::SemML, observed::SemObserved; kwargs...) return SemML(; observed = observed, kwargs...) end end - -############################################################################################ -### additional methods -############################################################################################ - -Σ⁻¹(semml::SemML) = semml.Σ⁻¹ -Σ⁻¹Σₒ(semml::SemML) = semml.Σ⁻¹Σₒ diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index b75d47454..60a454e37 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -78,7 +78,7 @@ function SemWLS(; else wls_weight_matrix_mean = nothing end - HE = approximate_hessian ? ApproximateHessian : AnalyticHessian + HE = approximate_hessian ? ApproximateHessian : ExactHessian return SemWLS{HE}(wls_weight_matrix, s, wls_weight_matrix_mean) end @@ -87,173 +87,58 @@ end ### methods ############################################################################ -function objective!(semwls::SemWLS, par, model::AbstractSemSingle) - let σ = Σ(imply(model)), - μ = μ(imply(model)), - σₒ = semwls.σₒ, - μₒ = obs_mean(observed(model)), - V = semwls.V, - V_μ = semwls.V_μ, - - σ₋ = σₒ - σ - - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - return dot(σ₋, V, σ₋) + dot(μ₋, V_μ, μ₋) - else - return dot(σ₋, V, σ₋) - end - end -end - -function gradient!(semwls::SemWLS, par, model::AbstractSemSingle) - let σ = Σ(imply(model)), - μ = μ(imply(model)), - σₒ = semwls.σₒ, - μₒ = obs_mean(observed(model)), - V = semwls.V, - V_μ = semwls.V_μ, - ∇σ = ∇Σ(imply(model)), - ∇μ = ∇μ(imply(model)) - - σ₋ = σₒ - σ - - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - return -2 * (σ₋' * V * ∇σ + μ₋' * V_μ * ∇μ)' - else - return -2 * (σ₋' * V * ∇σ)' - end - end -end - -function hessian!(semwls::SemWLS, par, model::AbstractSemSingle) - let σ = Σ(imply(model)), - σₒ = semwls.σₒ, - V = semwls.V, - ∇σ = ∇Σ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), - ∇²Σ = ∇²Σ(imply(model)) - - σ₋ = σₒ - σ - - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of WLS with meanstructure is not available")) - else - hessian = 2 * ∇σ' * V * ∇σ - if HessianEvaluation(semwls) === ExactHessian - J = -2 * (σ₋' * semwls.V)' - ∇²Σ_function!(∇²Σ, J, par) - hessian .+= ∇²Σ - end - return hessian - end +function evaluate!( + objective, + gradient, + hessian, + semwls::SemWLS, + implied::SemImplySymbolic, + model::AbstractSemSingle, + par, +) + if !isnothing(hessian) && (MeanStructure(implied) === HasMeanStructure) + error("hessian of WLS with meanstructure is not available") end -end -function objective_gradient!(semwls::SemWLS, par, model::AbstractSemSingle) - let σ = Σ(imply(model)), - μ = μ(imply(model)), - σₒ = semwls.σₒ, - μₒ = obs_mean(observed(model)), - V = semwls.V, - V_μ = semwls.V_μ, - ∇σ = ∇Σ(imply(model)), - ∇μ = ∇μ(imply(model)) + V = semwls.V + ∇σ = implied.∇Σ - σ₋ = σₒ - σ + σ = implied.Σ + σₒ = semwls.σₒ + σ₋ = σₒ - σ - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - objective = dot(σ₋, V, σ₋) + dot(μ₋, V_μ, μ₋) - gradient = -2 * (σ₋' * V * ∇σ + μ₋' * V_μ * ∇μ)' - return objective, gradient - else - objective = dot(σ₋, V, σ₋) - gradient = -2 * (σ₋' * V * ∇σ)' - return objective, gradient + isnothing(objective) || (objective = dot(σ₋, V, σ₋)) + if !isnothing(gradient) + if issparse(∇σ) + gradient .= (σ₋' * V * ∇σ)' + else # save one allocation + mul!(gradient, σ₋' * V, ∇σ) # actually transposed, but should be fine for vectors end + gradient .*= -2 end -end - -function objective_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of WLS with meanstructure is not available")) + isnothing(hessian) || (mul!(hessian, ∇σ' * V, ∇σ); + hessian .*= 2) + if !isnothing(hessian) && (HessianEvaluation(semwls) === ExactHessian) + ∇²Σ_function! = implied.∇²Σ_function + ∇²Σ = implied.∇²Σ + J = -2 * (σ₋' * semwls.V)' + ∇²Σ_function!(∇²Σ, J, par) + hessian .+= ∇²Σ end - - let σ = Σ(imply(model)), - σₒ = semwls.σₒ, - V = semwls.V, - ∇σ = ∇Σ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), - ∇²Σ = ∇²Σ(imply(model)) - - σ₋ = σₒ - σ - - objective = dot(σ₋, V, σ₋) - - hessian = 2 * ∇σ' * V * ∇σ - if HessianEvaluation(semwls) === ExactHessian - J = -2 * (σ₋' * semwls.V)' - ∇²Σ_function!(∇²Σ, J, par) - hessian .+= ∇²Σ + if MeanStructure(implied) === HasMeanStructure + μ = implied.μ + μₒ = obs_mean(observed(model)) + μ₋ = μₒ - μ + V_μ = semwls.V_μ + if !isnothing(objective) + objective += dot(μ₋, V_μ, μ₋) end - - return objective, hessian - end -end - -function gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of WLS with meanstructure is not available")) - end - - let σ = Σ(imply(model)), - σₒ = semwls.σₒ, - V = semwls.V, - ∇σ = ∇Σ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), - ∇²Σ = ∇²Σ(imply(model)) - - σ₋ = σₒ - σ - - gradient = -2 * (σ₋' * V * ∇σ)' - - hessian = 2 * ∇σ' * V * ∇σ - if HessianEvaluation(semwls) === ExactHessian - J = -2 * (σ₋' * semwls.V)' - ∇²Σ_function!(∇²Σ, J, par) - hessian .+= ∇²Σ + if !isnothing(gradient) + gradient .-= 2 * (μ₋' * V_μ * implied.∇μ)' end - - return gradient, hessian - end -end - -function objective_gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of WLS with meanstructure is not available")) end - let σ = Σ(imply(model)), - σₒ = semwls.σₒ, - V = semwls.V, - ∇σ = ∇Σ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), - ∇²Σ = ∇²Σ(imply(model)) - - σ₋ = σₒ - σ - - objective = dot(σ₋, V, σ₋) - gradient = -2 * (σ₋' * V * ∇σ)' - hessian = 2 * ∇σ' * V * ∇σ - if HessianEvaluation(semwls) === ExactHessian - J = -2 * (σ₋' * semwls.V)' - ∇²Σ_function!(∇²Σ, J, par) - hessian .+= ∇²Σ - end - return objective, gradient, hessian - end + return objective end ############################################################################################ diff --git a/src/loss/constant/constant.jl b/src/loss/constant/constant.jl index 9b3dfcd34..639864610 100644 --- a/src/loss/constant/constant.jl +++ b/src/loss/constant/constant.jl @@ -41,9 +41,10 @@ end ### methods ############################################################################################ -objective!(constant::SemConstant, par, model) = constant.c -gradient!(constant::SemConstant, par, model) = zero(par) -hessian!(constant::SemConstant, par, model) = zeros(eltype(par), length(par), length(par)) +objective(constant::SemConstant, model::AbstractSem, par) = constant.c +gradient(constant::SemConstant, model::AbstractSem, par) = zero(par) +hessian(constant::SemConstant, model::AbstractSem, par) = + zeros(eltype(par), length(par), length(par)) ############################################################################################ ### Recommended methods diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index e89ceeed7..be9b14fa5 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -76,14 +76,15 @@ end ### methods ############################################################################################ -objective!(ridge::SemRidge, par, model) = @views ridge.α * sum(abs2, par[ridge.which]) +objective(ridge::SemRidge, model::AbstractSem, par) = + @views ridge.α * sum(abs2, par[ridge.which]) -function gradient!(ridge::SemRidge, par, model) +function gradient(ridge::SemRidge, model::AbstractSem, par) @views ridge.gradient[ridge.which] .= (2 * ridge.α) * par[ridge.which] return ridge.gradient end -function hessian!(ridge::SemRidge, par, model) +function hessian(ridge::SemRidge, model::AbstractSem, par) @views @. ridge.hessian[ridge.which_H] .= 2 * ridge.α return ridge.hessian end diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index 2debbcd40..f07b572aa 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -1,298 +1,150 @@ -############################################################################################ -# methods for AbstractSem -############################################################################################ - -function objective!(model::AbstractSemSingle, params) - objective!(imply(model), params, model) - return objective!(loss(model), params, model) -end - -function gradient!(gradient, model::AbstractSemSingle, params) - fill!(gradient, zero(eltype(gradient))) - gradient!(imply(model), params, model) - gradient!(gradient, loss(model), params, model) -end - -function hessian!(hessian, model::AbstractSemSingle, params) - fill!(hessian, zero(eltype(hessian))) - hessian!(imply(model), params, model) - hessian!(hessian, loss(model), params, model) -end - -function objective_gradient!(gradient, model::AbstractSemSingle, params) - fill!(gradient, zero(eltype(gradient))) - objective_gradient!(imply(model), params, model) - objective_gradient!(gradient, loss(model), params, model) -end - -function objective_hessian!(hessian, model::AbstractSemSingle, params) - fill!(hessian, zero(eltype(hessian))) - objective_hessian!(imply(model), params, model) - objective_hessian!(hessian, loss(model), params, model) -end - -function gradient_hessian!(gradient, hessian, model::AbstractSemSingle, params) - fill!(gradient, zero(eltype(gradient))) - fill!(hessian, zero(eltype(hessian))) - gradient_hessian!(imply(model), params, model) - gradient_hessian!(gradient, hessian, loss(model), params, model) -end - -function objective_gradient_hessian!(gradient, hessian, model::AbstractSemSingle, params) - fill!(gradient, zero(eltype(gradient))) - fill!(hessian, zero(eltype(hessian))) - objective_gradient_hessian!(imply(model), params, model) - return objective_gradient_hessian!(gradient, hessian, loss(model), params, model) -end +"Specifies whether objective (O), gradient (G) or hessian (H) evaluation is required" +struct EvaluationTargets{O, G, H} end + +EvaluationTargets(objective, gradient, hessian) = + EvaluationTargets{!isnothing(objective), !isnothing(gradient), !isnothing(hessian)}() + +# convenience methods to check type params +is_objective_required(::EvaluationTargets{O}) where {O} = O +is_gradient_required(::EvaluationTargets{<:Any, G}) where {G} = G +is_hessian_required(::EvaluationTargets{<:Any, <:Any, H}) where {H} = H + +# return the tuple of the required results +(::EvaluationTargets{true, false, false})(objective, gradient, hessian) = objective +(::EvaluationTargets{false, true, false})(objective, gradient, hessian) = gradient +(::EvaluationTargets{false, false, true})(objective, gradient, hessian) = hessian +(::EvaluationTargets{true, true, false})(objective, gradient, hessian) = + (objective, gradient) +(::EvaluationTargets{true, false, true})(objective, gradient, hessian) = + (objective, hessian) +(::EvaluationTargets{false, true, true})(objective, gradient, hessian) = (gradient, hessian) +(::EvaluationTargets{true, true, true})(objective, gradient, hessian) = + (objective, gradient, hessian) + +(targets::EvaluationTargets)(arg_tuple::Tuple) = targets(arg_tuple...) + +# dispatch on SemImply +evaluate!(objective, gradient, hessian, loss::SemLossFunction, model::AbstractSem, params) = + evaluate!(objective, gradient, hessian, loss, imply(model), model, params) + +# fallback method +function evaluate!(obj, grad, hess, loss::SemLossFunction, imply::SemImply, model, params) + isnothing(obj) || (obj = objective(loss, imply, model, params)) + isnothing(grad) || copyto!(grad, gradient(loss, imply, model, params)) + isnothing(hess) || copyto!(hess, hessian(loss, imply, model, params)) + return obj +end + +# fallback methods +objective(f::SemLossFunction, imply::SemImply, model, params) = objective(f, model, params) +gradient(f::SemLossFunction, imply::SemImply, model, params) = gradient(f, model, params) +hessian(f::SemLossFunction, imply::SemImply, model, params) = hessian(f, model, params) + +# fallback method for SemImply that calls update_xxx!() methods +function update!(targets::EvaluationTargets, imply::SemImply, model, params) + is_objective_required(targets) && update_objective!(imply, model, params) + is_gradient_required(targets) && update_gradient!(imply, model, params) + is_hessian_required(targets) && update_hessian!(imply, model, params) +end + +# guess objective type +objective_type(model::AbstractSem, params::Any) = Float64 +objective_type(model::AbstractSem, params::AbstractVector{T}) where {T <: Number} = T +objective_zero(model::AbstractSem, params::Any) = zero(objective_type(model, params)) + +objective_type(objective::T, gradient, hessian) where {T <: Number} = T +objective_type( + objective::Nothing, + gradient::AbstractArray{T}, + hessian, +) where {T <: Number} = T +objective_type( + objective::Nothing, + gradient::Nothing, + hessian::AbstractArray{T}, +) where {T <: Number} = T +objective_zero(objective, gradient, hessian) = + zero(objective_type(objective, gradient, hessian)) ############################################################################################ -# methods for SemFiniteDiff +# methods for AbstractSem ############################################################################################ -gradient!(gradient, model::SemFiniteDiff, par) = - FiniteDiff.finite_difference_gradient!(gradient, x -> objective!(model, x), par) - -hessian!(hessian, model::SemFiniteDiff, par) = - FiniteDiff.finite_difference_hessian!(hessian, x -> objective!(model, x), par) - -function objective_gradient!(gradient, model::SemFiniteDiff, params) - gradient!(gradient, model, params) - return objective!(model, params) -end - -# other methods -function gradient_hessian!(gradient, hessian, model::SemFiniteDiff, params) - gradient!(gradient, model, params) - hessian!(hessian, model, params) -end - -function objective_hessian!(hessian, model::SemFiniteDiff, params) - hessian!(hessian, model, params) - return objective!(model, params) -end - -function objective_gradient_hessian!(gradient, hessian, model::SemFiniteDiff, params) - hessian!(hessian, model, params) - return objective_gradient!(gradient, model, params) +function evaluate!(objective, gradient, hessian, model::AbstractSemSingle, params) + targets = EvaluationTargets(objective, gradient, hessian) + # update imply state, its gradient and hessian (if required) + update!(targets, imply(model), model, params) + return evaluate!( + !isnothing(objective) ? zero(objective) : nothing, + gradient, + hessian, + loss(model), + model, + params, + ) end ############################################################################################ -# methods for SemLoss +# methods for SemFiniteDiff (approximate gradient and hessian with finite differences of objective) ############################################################################################ -function objective!(loss::SemLoss, par, model) - return mapreduce( - (fun, weight) -> weight * objective!(fun, par, model), - +, - loss.functions, - loss.weights, - ) -end - -function gradient!(gradient, loss::SemLoss, par, model) - for (lossfun, w) in zip(loss.functions, loss.weights) - new_gradient = gradient!(lossfun, par, model) - gradient .+= w * new_gradient - end -end - -function hessian!(hessian, loss::SemLoss, par, model) - for (lossfun, w) in zip(loss.functions, loss.weights) - hessian .+= w * hessian!(lossfun, par, model) - end -end - -function objective_gradient!(gradient, loss::SemLoss, par, model) - return mapreduce( - (fun, weight) -> objective_gradient_wrap_(gradient, fun, par, model, weight), - +, - loss.functions, - loss.weights, - ) -end - -function objective_hessian!(hessian, loss::SemLoss, par, model) - return mapreduce( - (fun, weight) -> objective_hessian_wrap_(hessian, fun, par, model, weight), - +, - loss.functions, - loss.weights, - ) -end - -function gradient_hessian!(gradient, hessian, loss::SemLoss, par, model) - for (lossfun, w) in zip(loss.functions, loss.weights) - new_gradient, new_hessian = gradient_hessian!(lossfun, par, model) - gradient .+= w * new_gradient - hessian .+= w * new_hessian +function evaluate!(objective, gradient, hessian, model::SemFiniteDiff, params) + function obj(p) + # recalculate imply state for p + update!(EvaluationTargets{true, false, false}(), imply(model), model, p) + evaluate!( + objective_zero(objective, gradient, hessian), + nothing, + nothing, + loss(model), + model, + p, + ) end + isnothing(gradient) || FiniteDiff.finite_difference_gradient!(gradient, obj, params) + isnothing(hessian) || FiniteDiff.finite_difference_hessian!(hessian, obj, params) + return !isnothing(objective) ? obj(params) : nothing end -function objective_gradient_hessian!(gradient, hessian, loss::SemLoss, par, model) - return mapreduce( - (fun, weight) -> - objective_gradient_hessian_wrap_(gradient, hessian, fun, par, model, weight), - +, - loss.functions, - loss.weights, - ) -end - -# wrapper to update gradient/hessian and return objective value -function objective_gradient_wrap_(gradient, lossfun, par, model, w) - new_objective, new_gradient = objective_gradient!(lossfun, par, model) - gradient .+= w * new_gradient - return w * new_objective -end - -function objective_hessian_wrap_(hessian, lossfun, par, model, w) - new_objective, new_hessian = objective_hessian!(lossfun, par, model) - hessian .+= w * new_hessian - return w * new_objective -end - -function objective_gradient_hessian_wrap_(gradient, hessian, lossfun, par, model, w) - new_objective, new_gradient, new_hessian = - objective_gradient_hessian!(lossfun, par, model) - gradient .+= w * new_gradient - hessian .+= w * new_hessian - return w * new_objective -end +objective(model::AbstractSem, params) = + evaluate!(objective_zero(model, params), nothing, nothing, model, params) ############################################################################################ -# methods for SemEnsemble +# methods for SemLoss (weighted sum of individual SemLossFunctions) ############################################################################################ -function objective!(ensemble::SemEnsemble, par) - return mapreduce( - (model, weight) -> weight * objective!(model, par), - +, - ensemble.sems, - ensemble.weights, - ) -end - -function gradient!(gradient, ensemble::SemEnsemble, par) - fill!(gradient, zero(eltype(gradient))) - for (model, w) in zip(ensemble.sems, ensemble.weights) - gradient_new = similar(gradient) - gradient!(gradient_new, model, par) - gradient .+= w * gradient_new - end -end - -function hessian!(hessian, ensemble::SemEnsemble, par) - fill!(hessian, zero(eltype(hessian))) - for (model, w) in zip(ensemble.sems, ensemble.weights) - hessian_new = similar(hessian) - hessian!(hessian_new, model, par) - hessian .+= w * hessian_new - end -end - -function objective_gradient!(gradient, ensemble::SemEnsemble, par) - fill!(gradient, zero(eltype(gradient))) - return mapreduce( - (model, weight) -> objective_gradient_wrap_(gradient, model, par, weight), - +, - ensemble.sems, - ensemble.weights, - ) -end - -function objective_hessian!(hessian, ensemble::SemEnsemble, par) - fill!(hessian, zero(eltype(hessian))) - return mapreduce( - (model, weight) -> objective_hessian_wrap_(hessian, model, par, weight), - +, - ensemble.sems, - ensemble.weights, - ) -end - -function gradient_hessian!(gradient, hessian, ensemble::SemEnsemble, par) - fill!(gradient, zero(eltype(gradient))) - fill!(hessian, zero(eltype(hessian))) - for (model, w) in zip(ensemble.sems, ensemble.weights) - new_gradient = similar(gradient) - new_hessian = similar(hessian) - - gradient_hessian!(new_gradient, new_hessian, model, par) - - gradient .+= w * new_gradient - hessian .+= w * new_hessian +function evaluate!(objective, gradient, hessian, loss::SemLoss, model::AbstractSem, params) + isnothing(objective) || (objective = zero(objective)) + isnothing(gradient) || fill!(gradient, zero(eltype(gradient))) + isnothing(hessian) || fill!(hessian, zero(eltype(hessian))) + f_grad = isnothing(gradient) ? nothing : similar(gradient) + f_hess = isnothing(hessian) ? nothing : similar(hessian) + for (f, weight) in zip(loss.functions, loss.weights) + f_obj = evaluate!(objective, f_grad, f_hess, f, model, params) + isnothing(objective) || (objective += weight * f_obj) + isnothing(gradient) || (gradient .+= weight * f_grad) + isnothing(hessian) || (hessian .+= weight * f_hess) end -end - -function objective_gradient_hessian!(gradient, hessian, ensemble::SemEnsemble, par) - fill!(gradient, zero(eltype(gradient))) - fill!(hessian, zero(eltype(hessian))) - return mapreduce( - (model, weight) -> - objective_gradient_hessian_wrap_(gradient, hessian, model, par, model, weight), - +, - ensemble.sems, - ensemble.weights, - ) -end - -# wrapper to update gradient/hessian and return objective value -function objective_gradient_wrap_(gradient, model::AbstractSemSingle, par, w) - gradient_pre = similar(gradient) - new_objective = objective_gradient!(gradient_pre, model, par) - gradient .+= w * gradient_pre - return w * new_objective -end - -function objective_hessian_wrap_(hessian, model::AbstractSemSingle, par, w) - hessian_pre = similar(hessian) - new_objective = objective_hessian!(hessian_pre, model, par) - hessian .+= w * new_hessian - return w * new_objective -end - -function objective_gradient_hessian_wrap_( - gradient, - hessian, - model::AbstractSemSingle, - par, - w, -) - gradient_pre = similar(gradient) - hessian_pre = similar(hessian) - new_objective = objective_gradient_hessian!(gradient_pre, hessian_pre, model, par) - gradient .+= w * new_gradient - hessian .+= w * new_hessian - return w * new_objective + return objective end ############################################################################################ -# generic methods for loss functions +# methods for SemEnsemble (weighted sum of individual AbstractSemSingle models) ############################################################################################ -function objective_gradient!(lossfun::SemLossFunction, par, model) - objective = objective!(lossfun::SemLossFunction, par, model) - gradient = gradient!(lossfun::SemLossFunction, par, model) - return objective, gradient -end - -function objective_hessian!(lossfun::SemLossFunction, par, model) - objective = objective!(lossfun::SemLossFunction, par, model) - hessian = hessian!(lossfun::SemLossFunction, par, model) - return objective, hessian -end - -function gradient_hessian!(lossfun::SemLossFunction, par, model) - gradient = gradient!(lossfun::SemLossFunction, par, model) - hessian = hessian!(lossfun::SemLossFunction, par, model) - return gradient, hessian -end - -function objective_gradient_hessian!(lossfun::SemLossFunction, par, model) - objective = objective!(lossfun::SemLossFunction, par, model) - gradient = gradient!(lossfun::SemLossFunction, par, model) - hessian = hessian!(lossfun::SemLossFunction, par, model) - return objective, gradient, hessian +function evaluate!(objective, gradient, hessian, ensemble::SemEnsemble, params) + isnothing(objective) || (objective = zero(objective)) + isnothing(gradient) || fill!(gradient, zero(eltype(gradient))) + isnothing(hessian) || fill!(hessian, zero(eltype(hessian))) + sem_grad = isnothing(gradient) ? nothing : similar(gradient) + sem_hess = isnothing(hessian) ? nothing : similar(hessian) + for (sem, weight) in zip(ensemble.sems, ensemble.weights) + sem_obj = evaluate!(objective, sem_grad, sem_hess, sem, params) + isnothing(objective) || (objective += weight * sem_obj) + isnothing(gradient) || (gradient .+= weight * sem_grad) + isnothing(hessian) || (hessian .+= weight * sem_hess) + end + return objective end # throw an error by default if gradient! and hessian! are not implemented @@ -303,35 +155,6 @@ end hessian!(lossfun::SemLossFunction, par, model) = throw(ArgumentError("hessian for $(typeof(lossfun).name.wrapper) is not available")) =# -############################################################################################ -# generic methods for imply -############################################################################################ - -function objective_gradient!(semimp::SemImply, par, model) - objective!(semimp::SemImply, par, model) - gradient!(semimp::SemImply, par, model) - return nothing -end - -function objective_hessian!(semimp::SemImply, par, model) - objective!(semimp::SemImply, par, model) - hessian!(semimp::SemImply, par, model) - return nothing -end - -function gradient_hessian!(semimp::SemImply, par, model) - gradient!(semimp::SemImply, par, model) - hessian!(semimp::SemImply, par, model) - return nothing -end - -function objective_gradient_hessian!(semimp::SemImply, par, model) - objective!(semimp::SemImply, par, model) - gradient!(semimp::SemImply, par, model) - hessian!(semimp::SemImply, par, model) - return nothing -end - ############################################################################################ # Documentation ############################################################################################ @@ -377,3 +200,18 @@ To implement a new `AbstractSem` subtype, you can add a method for hessian!(hessian, model::MyNewType, params) """ function hessian! end + +objective!(model::AbstractSem, params) = + evaluate!(objective_zero(model, params), nothing, nothing, model, params) +gradient!(gradient, model::AbstractSem, params) = + evaluate!(nothing, gradient, nothing, model, params) +hessian!(hessian, model::AbstractSem, params) = + evaluate!(nothing, nothing, hessian, model, params) +objective_gradient!(gradient, model::AbstractSem, params) = + evaluate!(objective_zero(model, params), gradient, nothing, model, params) +objective_hessian!(hessian, model::AbstractSem, params) = + evaluate!(objective_zero(model, params), nothing, hessian, model, params) +gradient_hessian!(gradient, hessian, model::AbstractSem, params) = + evaluate!(nothing, gradient, hessian, model, params) +objective_gradient_hessian!(gradient, hessian, model::AbstractSem, params) = + evaluate!(objective_zero(model, params), gradient, hessian, model, params) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 265ab178a..4790c9d36 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -70,7 +70,7 @@ end grad = similar(start_test) gradient!(grad, model_ml_multigroup, rand(36)) grad_fd = FiniteDiff.finite_difference_gradient( - x -> objective!(model_ml_multigroup, x), + Base.Fix1(SEM.objective, model_ml_multigroup), start_test, ) @@ -122,7 +122,7 @@ struct UserSemML <: SemLossFunction{ExactHessian} end using LinearAlgebra: isposdef, logdet, tr, inv -function SEM.objective!(semml::UserSemML, params, model::AbstractSem) +function SEM.objective(ml::UserSemML, model::AbstractSem, params) Σ = imply(model).Σ Σₒ = SEM.obs_cov(observed(model)) if !isposdef(Σ) From 3c533b6c75ccb46721bf18d079449a7faa96d3d2 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 19 Mar 2024 20:36:06 -0700 Subject: [PATCH 096/364] se_hessian(): rename hessian -> method for clarity --- src/frontend/fit/standard_errors/hessian.jl | 47 +++++++++----------- src/frontend/specification/ParameterTable.jl | 8 ++-- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index e71e601fb..8a4be88e3 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -1,39 +1,32 @@ """ - se_hessian(semfit::SemFit; hessian = :finitediff) + se_hessian(fit::SemFit; method = :finitediff) -Return hessian based standard errors. +Return hessian-based standard errors. # Arguments -- `hessian`: how to compute the hessian. Options are +- `method`: how to compute the hessian. Options are - `:analytic`: (only if an analytic hessian for the model can be computed) - `:finitediff`: for finite difference approximation """ -function se_hessian(sem_fit::SemFit; hessian = :finitediff) - c = H_scaling(sem_fit.model) - - if hessian == :analytic - par = solution(sem_fit) - H = zeros(eltype(par), length(par), length(par)) - hessian!(H, sem_fit.model, sem_fit.solution) - elseif hessian == :finitediff - H = FiniteDiff.finite_difference_hessian( - p -> evaluate!(zero(eltype(sem_fit.solution)), nothing, nothing, fit.model, p), - sem_fit.solution, - ) - elseif hessian == :optimizer - throw( - ArgumentError( - "standard errors from the optimizer hessian are not implemented yet", - ), - ) - elseif hessian == :expected - throw( - ArgumentError( - "standard errors based on the expected hessian are not implemented yet", - ), +function se_hessian(fit::SemFit; method = :finitediff) + c = H_scaling(fit.model) + params = solution(fit) + H = similar(params, (length(params), length(params))) + + if method == :analytic + evaluate!(nothing, nothing, H, fit.model, params) + elseif method == :finitediff + FiniteDiff.finite_difference_hessian!( + H, + p -> evaluate!(zero(eltype(H)), nothing, nothing, fit.model, p), + params, ) + elseif method == :optimizer + error("Standard errors from the optimizer hessian are not implemented yet") + elseif method == :expected + error("Standard errors based on the expected hessian are not implemented yet") else - throw(ArgumentError("I don't know how to compute `$hessian` standard-errors")) + throw(ArgumentError("Unsupported hessian calculation method :$method")) end invH = c * inv(H) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 8970b7430..687b712ba 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -362,12 +362,12 @@ end update_se_hessian!( partable::AbstractParameterTable, fit::SemFit; - hessian = :finitediff) + method = :finitediff) Write hessian standard errors computed for `fit` to the `:se` column of `partable` # Arguments -- `hessian::Symbol`: how to compute the hessian, see [se_hessian](@ref) for more information. +- `method::Symbol`: how to compute the hessian, see [se_hessian](@ref) for more information. # Examples @@ -375,9 +375,9 @@ Write hessian standard errors computed for `fit` to the `:se` column of `partabl function update_se_hessian!( partable::AbstractParameterTable, fit::SemFit; - hessian = :finitediff, + method = :finitediff, ) - se = se_hessian(fit; hessian = hessian) + se = se_hessian(fit; method) return update_partable!(partable, :se, params(fit), se) end From 9e33add58c7a9e7a4e1c21c26543aa089dd68584 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 14:20:11 -0700 Subject: [PATCH 097/364] se_hessian!(): optimize calc * explicitly use Cholesky factorization --- src/frontend/fit/standard_errors/hessian.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index 8a4be88e3..4de2db2f7 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -29,10 +29,9 @@ function se_hessian(fit::SemFit; method = :finitediff) throw(ArgumentError("Unsupported hessian calculation method :$method")) end - invH = c * inv(H) - se = sqrt.(diag(invH)) - - return se + H_chol = cholesky!(Symmetric(H)) + H_inv = LinearAlgebra.inv!(H_chol) + return [sqrt(c * H_inv[i]) for i in diagind(H_inv)] end # Addition functions ------------------------------------------------------------- From 5ad013e01309a9b09ce8690e5820845e8b77ec92 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 19 Mar 2024 20:38:00 -0700 Subject: [PATCH 098/364] H_scaling(): cleanup remove unnecesary arguments --- src/frontend/fit/standard_errors/hessian.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index 4de2db2f7..6ae53407f 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -35,16 +35,20 @@ function se_hessian(fit::SemFit; method = :finitediff) end # Addition functions ------------------------------------------------------------- -H_scaling(model::AbstractSemSingle) = - H_scaling(model, model.observed, model.imply, model.optimizer, model.loss.functions...) +function H_scaling(model::AbstractSemSingle) + if length(model.loss.functions) > 1 + @warn "Hessian scaling for multiple loss functions is not implemented yet" + end + return H_scaling(model.loss.functions[1], model) +end -H_scaling(model, obs, imp, optimizer, lossfun::SemML) = 2 / (nsamples(model) - 1) +H_scaling(lossfun::SemML, model::AbstractSemSingle) = 2 / (nsamples(model) - 1) -function H_scaling(model, obs, imp, optimizer, lossfun::SemWLS) +function H_scaling(lossfun::SemWLS, model::AbstractSemSingle) @warn "Standard errors for WLS are only correct if a GLS weight matrix (the default) is used." return 2 / (nsamples(model) - 1) end -H_scaling(model, obs, imp, optimizer, lossfun::SemFIML) = 2 / nsamples(model) +H_scaling(lossfun::SemFIML, model::AbstractSemSingle) = 2 / nsamples(model) H_scaling(model::SemEnsemble) = 2 / nsamples(model) From a32903e7d6397aac49245ec927db8b4f11d372c8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 18:35:50 -0700 Subject: [PATCH 099/364] SemOptOptim: remove redundant sem_fit() by dispatching over optimizer --- src/optimizer/documentation.jl | 7 +++++++ src/optimizer/optim.jl | 25 ++++--------------------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/optimizer/documentation.jl b/src/optimizer/documentation.jl index 83b4f7a98..7c17e6ce2 100644 --- a/src/optimizer/documentation.jl +++ b/src/optimizer/documentation.jl @@ -20,3 +20,10 @@ sem_fit( ``` """ function sem_fit end + +# dispatch on optimizer +sem_fit(model::AbstractSem; kwargs...) = sem_fit(model.optimizer, model; kwargs...) + +# fallback method +sem_fit(optimizer::SemOptimizer, model::AbstractSem; kwargs...) = + error("Optimizer $(optimizer) support not implemented.") diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 68617fdb8..6acf665e6 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -45,29 +45,12 @@ n_iterations(res::Optim.MultivariateOptimizationResults) = Optim.iterations(res) convergence(res::Optim.MultivariateOptimizationResults) = Optim.converged(res) function sem_fit( - model::AbstractSemSingle{O, I, L, D}; + optim::SemOptimizerOptim, + model::AbstractSem; start_val = start_val, kwargs..., -) where {O, I, L, D <: SemOptimizerOptim} - if !isa(start_val, Vector) - start_val = start_val(model; kwargs...) - end - - result = Optim.optimize( - Optim.only_fgh!((F, G, H, par) -> sem_wrap_optim(par, F, G, H, model)), - start_val, - model.optimizer.algorithm, - model.optimizer.options, - ) - return SemFit(result, model, start_val) -end - -function sem_fit( - model::SemEnsemble{N, T, V, D, S}; - start_val = start_val, - kwargs..., -) where {N, T, V, D <: SemOptimizerOptim, S} - if !isa(start_val, Vector) +) + if !isa(start_val, AbstractVector) start_val = start_val(model; kwargs...) end From 7ab2dcbc92b50fff9f0feb64d19976a2609883ad Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 18:37:05 -0700 Subject: [PATCH 100/364] SemOptNLopt: remove redundant sem_fit() by dispatching over optimizer --- src/optimizer/NLopt.jl | 42 ++++-------------------------------------- 1 file changed, 4 insertions(+), 38 deletions(-) diff --git a/src/optimizer/NLopt.jl b/src/optimizer/NLopt.jl index ffe2ffed0..1fa475ab4 100644 --- a/src/optimizer/NLopt.jl +++ b/src/optimizer/NLopt.jl @@ -34,48 +34,14 @@ end # sem_fit method function sem_fit( - model::Sem{O, I, L, D}; + optimizer::SemOptimizerNLopt, + model::AbstractSem; start_val = start_val, kwargs..., -) where {O, I, L, D <: SemOptimizerNLopt} +) # starting values - if !isa(start_val, Vector) - start_val = start_val(model; kwargs...) - end - - # construct the NLopt problem - opt = construct_NLopt_problem( - model.optimizer.algorithm, - model.optimizer.options, - length(start_val), - ) - set_NLopt_constraints!(opt, model.optimizer) - opt.min_objective = (par, G) -> sem_wrap_nlopt(par, G, model) - - if !isnothing(model.optimizer.local_algorithm) - opt_local = construct_NLopt_problem( - model.optimizer.local_algorithm, - model.optimizer.local_options, - length(start_val), - ) - opt.local_optimizer = opt_local - end - - # fit - result = NLopt.optimize(opt, start_val) - - return SemFit_NLopt(result, model, start_val, opt) -end - -function sem_fit( - model::SemEnsemble{N, T, V, D, S}; - start_val = start_val, - kwargs..., -) where {N, T, V, D <: SemOptimizerNLopt, S} - - # starting values - if !isa(start_val, Vector) + if !isa(start_val, AbstractVector) start_val = start_val(model; kwargs...) end From 65d111236ec9cf83b0c7ca6a8cb44a050f6a2275 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 18:37:45 -0700 Subject: [PATCH 101/364] SemOptOptim: use evaluate!() directly no wrapper required --- src/optimizer/optim.jl | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 6acf665e6..bb1bf507e 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -1,30 +1,4 @@ ## connect to Optim.jl as backend -function sem_wrap_optim(par, F, G, H, model::AbstractSem) - if !isnothing(F) - if !isnothing(G) - if !isnothing(H) - return objective_gradient_hessian!(G, H, model, par) - else - return objective_gradient!(G, model, par) - end - else - if !isnothing(H) - return objective_hessian!(H, model, par) - else - return objective!(model, par) - end - end - else - if !isnothing(G) - if !isnothing(H) - gradient_hessian!(G, H, model, par) - else - gradient!(G, model, par) - end - end - end - return nothing -end function SemFit( optimization_result::Optim.MultivariateOptimizationResults, @@ -55,7 +29,7 @@ function sem_fit( end result = Optim.optimize( - Optim.only_fgh!((F, G, H, par) -> sem_wrap_optim(par, F, G, H, model)), + Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), start_val, model.optimizer.algorithm, model.optimizer.options, From 9ac8f8846503d788745f670925444f0345a6dfc7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 18:38:30 -0700 Subject: [PATCH 102/364] SemOptNLopt: use evaluate!() directly --- src/optimizer/NLopt.jl | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/optimizer/NLopt.jl b/src/optimizer/NLopt.jl index 1fa475ab4..7f4f61e1e 100644 --- a/src/optimizer/NLopt.jl +++ b/src/optimizer/NLopt.jl @@ -2,16 +2,6 @@ ### connect to NLopt.jl as backend ############################################################################################ -# wrapper to define the objective -function sem_wrap_nlopt(par, G, model::AbstractSem) - need_gradient = length(G) != 0 - if need_gradient - return objective_gradient!(G, model, par) - else - return objective!(model, par) - end -end - mutable struct NLoptResult result::Any problem::Any @@ -52,7 +42,14 @@ function sem_fit( length(start_val), ) set_NLopt_constraints!(opt, model.optimizer) - opt.min_objective = (par, G) -> sem_wrap_nlopt(par, G, model) + opt.min_objective = + (par, G) -> evaluate!( + eltype(par), + !isnothing(G) && !isempty(G) ? G : nothing, + nothing, + model, + par, + ) if !isnothing(model.optimizer.local_algorithm) opt_local = construct_NLopt_problem( From cc778e28c9df7e03bab3710a916c19b3b6ae636c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 00:46:20 -0700 Subject: [PATCH 103/364] SemWLS: dim checks --- src/loss/WLS/WLS.jl | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 60a454e37..fa193a565 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -59,23 +59,34 @@ function SemWLS(; meanstructure = false, kwargs..., ) - ind = CartesianIndices(obs_cov(observed)) - ind = filter(x -> (x[1] >= x[2]), ind) - s = obs_cov(observed)[ind] + nobs_vars = nobserved_vars(observed) + tril_ind = filter(x -> (x[1] >= x[2]), CartesianIndices(obs_cov(observed))) + s = obs_cov(observed)[tril_ind] # compute V here if isnothing(wls_weight_matrix) - D = duplication_matrix(nobserved_vars(observed)) + D = duplication_matrix(nobs_vars) S = inv(obs_cov(observed)) S = kron(S, S) wls_weight_matrix = 0.5 * (D' * S * D) + else + size(wls_weight_matrix) == (length(tril_ind), length(tril_ind)) || + DimensionMismatch( + "wls_weight_matrix has to be of size $(length(tril_ind))×$(length(tril_ind))", + ) end if meanstructure if isnothing(wls_weight_matrix_mean) wls_weight_matrix_mean = inv(obs_cov(observed)) + else + size(wls_weight_matrix_mean) == (nobs_vars, nobs_vars) || DimensionMismatch( + "wls_weight_matrix_mean has to be of size $(nobs_vars)×$(nobs_vars)", + ) end else + isnothing(wls_weight_matrix_mean) || + @warn "Ignoring wls_weight_matrix_mean since meanstructure is disabled" wls_weight_matrix_mean = nothing end HE = approximate_hessian ? ApproximateHessian : ExactHessian From 0d33ba410f6b4ff38998caa768f037ea7525ddc0 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 11 Aug 2024 13:50:38 -0700 Subject: [PATCH 104/364] fixup formatting --- src/frontend/specification/StenoGraphs.jl | 9 +++------ test/unit_tests/specification.jl | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 035d9588b..64a33f13e 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -73,12 +73,9 @@ function ParameterTable( ) end if element isa ModifiedEdge - if any(Base.Fix2(isa, Fixed), values(element.modifiers)) & any(Base.Fix2(isa, Label), values(element.modifiers)) - throw( - ArgumentError( - "It is not allowed to label fixed parameters." - ) - ) + if any(Base.Fix2(isa, Fixed), values(element.modifiers)) && + any(Base.Fix2(isa, Label), values(element.modifiers)) + throw(ArgumentError("It is not allowed to label fixed parameters.")) end for modifier in values(element.modifiers) if isnothing(group) && diff --git a/test/unit_tests/specification.jl b/test/unit_tests/specification.jl index e307d60f2..ef9fc73a1 100644 --- a/test/unit_tests/specification.jl +++ b/test/unit_tests/specification.jl @@ -29,7 +29,7 @@ end fixed_and_labeled_graph = @StenoGraph begin # measurement model - visual → fixed(1.0)*label(:λ)*x1 + visual → fixed(1.0) * label(:λ) * x1 end @testset "ParameterTable" begin @@ -42,7 +42,7 @@ end @test_throws ArgumentError("It is not allowed to label fixed parameters.") ParameterTable( fixed_and_labeled_graph, observed_vars = obs_vars, - latent_vars = lat_vars + latent_vars = lat_vars, ) partable = @inferred( ParameterTable(graph, observed_vars = obs_vars, latent_vars = lat_vars) From 1c376a585e656d8fe5f46bad9cd7da02cf9aa89e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 12:24:56 -0700 Subject: [PATCH 105/364] WLS: use 5-arg mul!() to reduce allocations --- src/loss/WLS/WLS.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index fa193a565..3cac6ee12 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -127,8 +127,7 @@ function evaluate!( end gradient .*= -2 end - isnothing(hessian) || (mul!(hessian, ∇σ' * V, ∇σ); - hessian .*= 2) + isnothing(hessian) || (mul!(hessian, ∇σ' * V, ∇σ, 2, 0)) if !isnothing(hessian) && (HessianEvaluation(semwls) === ExactHessian) ∇²Σ_function! = implied.∇²Σ_function ∇²Σ = implied.∇²Σ @@ -145,7 +144,7 @@ function evaluate!( objective += dot(μ₋, V_μ, μ₋) end if !isnothing(gradient) - gradient .-= 2 * (μ₋' * V_μ * implied.∇μ)' + mul!(gradient, (V_μ * implied.∇μ)', μ₋, -2, 1) end end From 4a6f51b6a9a53cb52a9e6b0de68a804142d5d9f1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 16:10:26 -0700 Subject: [PATCH 106/364] ML: use 5-arg mul!() to reduce allocations --- src/loss/ML/ML.jl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 445a557a7..33b6319aa 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -102,19 +102,19 @@ function evaluate!( ∇Σ = implied.∇Σ ∇μ = implied.∇μ μ₋ᵀΣ⁻¹ = μ₋' * Σ⁻¹ - gradient .= (vec(Σ⁻¹ - Σ⁻¹Σₒ * Σ⁻¹ - μ₋ᵀΣ⁻¹'μ₋ᵀΣ⁻¹)' * ∇Σ)' - gradient .-= (2 * μ₋ᵀΣ⁻¹ * ∇μ)' + mul!(gradient, ∇Σ', vec(Σ⁻¹ - Σ⁻¹Σₒ * Σ⁻¹ - μ₋ᵀΣ⁻¹'μ₋ᵀΣ⁻¹)) + mul!(gradient, ∇μ', μ₋ᵀΣ⁻¹', -2, 1) end elseif !isnothing(gradient) || !isnothing(hessian) ∇Σ = implied.∇Σ Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ * Σ⁻¹ J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' if !isnothing(gradient) - gradient .= (J * ∇Σ)' + mul!(gradient, ∇Σ', J') end if !isnothing(hessian) if HessianEvaluation(semml) === ApproximateHessian - mul!(hessian, 2 * ∇Σ' * kron(Σ⁻¹, Σ⁻¹), ∇Σ) + mul!(hessian, ∇Σ' * kron(Σ⁻¹, Σ⁻¹), ∇Σ, 2, 0) else ∇²Σ_function! = implied.∇²Σ_function ∇²Σ = implied.∇²Σ @@ -183,7 +183,8 @@ function evaluate!( ∇S = implied.∇S C = F⨉I_A⁻¹' * (I - Σ⁻¹Σₒ) * Σ⁻¹ * F⨉I_A⁻¹ - gradᵀ = 2vec(C * S * I_A⁻¹')'∇A + vec(C)'∇S + mul!(gradient, ∇A', vec(C * S * I_A⁻¹'), 2, 0) + mul!(gradient, ∇S', vec(C), 1, 1) if MeanStructure(implied) === HasMeanStructure μ = implied.μ @@ -193,9 +194,10 @@ function evaluate!( μ₋ = μₒ - μ μ₋ᵀΣ⁻¹ = μ₋' * Σ⁻¹ k = μ₋ᵀΣ⁻¹ * F⨉I_A⁻¹ - gradᵀ .+= -2k * ∇M - 2vec(k' * (M' + k * S) * I_A⁻¹')'∇A - vec(k'k)'∇S + mul!(gradient, ∇M', k', -2, 1) + mul!(gradient, ∇A', vec(k' * (I_A⁻¹ * (M + S * k'))'), -2, 1) + mul!(gradient, ∇S', vec(k'k), -1, 1) end - copyto!(gradient, gradᵀ') end return objective From d2b7e8c68a86ed9bce7e8b776a3eaa834cb862cd Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 12:25:31 -0700 Subject: [PATCH 107/364] FIML: use 5-arg mul! to avoid extra allocation --- src/loss/ML/FIML.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 92ecf73ca..20e837997 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -144,7 +144,7 @@ end function ∇F_fiml_outer!(G, JΣ, Jμ, imply::SemImplySymbolic, model, semfiml) mul!(G, imply.∇Σ', JΣ) # should be transposed - G .-= imply.∇μ' * Jμ + mul!(G, imply.∇μ', Jμ, -1, 1) end function ∇F_fiml_outer!(G, JΣ, Jμ, imply, model, semfiml) @@ -158,7 +158,7 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, imply, model, semfiml) ∇μ = imply.F⨉I_A⁻¹ * imply.∇M + kron((imply.I_A⁻¹ * imply.M)', imply.F⨉I_A⁻¹) * imply.∇A mul!(G, ∇Σ', JΣ) # actually transposed - G .-= ∇μ' * Jμ + mul!(G, ∇μ', Jμ, -1, 1) end function F_FIML(rows, semfiml, model, params) From d0ea4066bd36c1cbacf3b46812a16ebe3da9b8d7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 14 Aug 2024 09:29:51 -0700 Subject: [PATCH 108/364] fix the error message Co-authored-by: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- src/types.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.jl b/src/types.jl index 6cdf9bead..5ae337e11 100644 --- a/src/types.jl +++ b/src/types.jl @@ -19,7 +19,7 @@ struct NoMeanStructure <: MeanStructure end # fallback implementation MeanStructure(::Type{T}) where {T} = - error("Objects of type $T do not support MeanStructure trait") + error("Objects of type $T do not support the MeanStructure trait") MeanStructure(semobj) = MeanStructure(typeof(semobj)) "Hessian Evaluation trait for `SemImply` and `SemLossFunction` subtypes" From 56ec1c9856484bd92279c6b1ac35daaac96f04cc Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 8 Oct 2024 01:07:22 -0700 Subject: [PATCH 109/364] HessianEvaluation -> HessianEval --- src/StructuralEquationModels.jl | 4 ++-- src/loss/ML/ML.jl | 8 ++++---- src/loss/WLS/WLS.jl | 8 ++++---- src/types.jl | 20 ++++++++++---------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a171c29d0..2a469dd91 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -98,9 +98,9 @@ export AbstractSem, MeanStructure, NoMeanStructure, HasMeanStructure, - HessianEvaluation, + HessianEval, ExactHessian, - ApproximateHessian, + ApproxHessian, SemImply, RAMSymbolic, RAM, diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 33b6319aa..261b260d6 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -27,12 +27,12 @@ Analytic gradients are available, and for models without a meanstructure, also a ## Implementation Subtype of `SemLossFunction`. """ -struct SemML{HE <: HessianEvaluation, INV, M, M2} <: SemLossFunction{HE} +struct SemML{HE <: HessianEval, INV, M, M2} <: SemLossFunction{HE} Σ⁻¹::INV Σ⁻¹Σₒ::M meandiff::M2 - SemML{HE}(args...) where {HE <: HessianEvaluation} = + SemML{HE}(args...) where {HE <: HessianEval} = new{HE, map(typeof, args)...}(args...) end @@ -45,7 +45,7 @@ function SemML(; observed::SemObserved, approximate_hessian::Bool = false, kwarg obscov = obs_cov(observed) meandiff = isnothing(obsmean) ? nothing : copy(obsmean) - return SemML{approximate_hessian ? ApproximateHessian : ExactHessian}( + return SemML{approximate_hessian ? ApproxHessian : ExactHessian}( similar(obscov), similar(obscov), meandiff, @@ -113,7 +113,7 @@ function evaluate!( mul!(gradient, ∇Σ', J') end if !isnothing(hessian) - if HessianEvaluation(semml) === ApproximateHessian + if HessianEval(semml) === ApproxHessian mul!(hessian, ∇Σ' * kron(Σ⁻¹, Σ⁻¹), ∇Σ, 2, 0) else ∇²Σ_function! = implied.∇²Σ_function diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 3cac6ee12..2345f859d 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -38,7 +38,7 @@ Analytic gradients are available, and for models without a meanstructure, also a ## Implementation Subtype of `SemLossFunction`. """ -struct SemWLS{HE <: HessianEvaluation, Vt, St, C} <: SemLossFunction{HE} +struct SemWLS{HE <: HessianEval, Vt, St, C} <: SemLossFunction{HE} V::Vt σₒ::St V_μ::C @@ -48,7 +48,7 @@ end ### Constructors ############################################################################################ -SemWLS{HE}(args...) where {HE <: HessianEvaluation} = +SemWLS{HE}(args...) where {HE <: HessianEval} = SemWLS{HE, map(typeof, args)...}(args...) function SemWLS(; @@ -89,7 +89,7 @@ function SemWLS(; @warn "Ignoring wls_weight_matrix_mean since meanstructure is disabled" wls_weight_matrix_mean = nothing end - HE = approximate_hessian ? ApproximateHessian : ExactHessian + HE = approximate_hessian ? ApproxHessian : ExactHessian return SemWLS{HE}(wls_weight_matrix, s, wls_weight_matrix_mean) end @@ -128,7 +128,7 @@ function evaluate!( gradient .*= -2 end isnothing(hessian) || (mul!(hessian, ∇σ' * V, ∇σ, 2, 0)) - if !isnothing(hessian) && (HessianEvaluation(semwls) === ExactHessian) + if !isnothing(hessian) && (HessianEval(semwls) === ExactHessian) ∇²Σ_function! = implied.∇²Σ_function ∇²Σ = implied.∇²Σ J = -2 * (σ₋' * semwls.V)' diff --git a/src/types.jl b/src/types.jl index 5ae337e11..12082be12 100644 --- a/src/types.jl +++ b/src/types.jl @@ -23,19 +23,19 @@ MeanStructure(::Type{T}) where {T} = MeanStructure(semobj) = MeanStructure(typeof(semobj)) "Hessian Evaluation trait for `SemImply` and `SemLossFunction` subtypes" -abstract type HessianEvaluation end -struct ApproximateHessian <: HessianEvaluation end -struct ExactHessian <: HessianEvaluation end +abstract type HessianEval end +struct ApproxHessian <: HessianEval end +struct ExactHessian <: HessianEval end # fallback implementation -HessianEvaluation(::Type{T}) where {T} = - error("Objects of type $T do not support HessianEvaluation trait") -HessianEvaluation(semobj) = HessianEvaluation(typeof(semobj)) +HessianEval(::Type{T}) where {T} = + error("Objects of type $T do not support HessianEval trait") +HessianEval(semobj) = HessianEval(typeof(semobj)) "Supertype for all loss functions of SEMs. If you want to implement a custom loss function, it should be a subtype of `SemLossFunction`." -abstract type SemLossFunction{HE <: HessianEvaluation} end +abstract type SemLossFunction{HE <: HessianEval} end -HessianEvaluation(::Type{<:SemLossFunction{HE}}) where {HE <: HessianEvaluation} = HE +HessianEval(::Type{<:SemLossFunction{HE}}) where {HE <: HessianEval} = HE """ SemLoss(args...; loss_weights = nothing, ...) @@ -97,10 +97,10 @@ Computed model-implied values that should be compared with the observed data to e. g. the model implied covariance or mean. If you would like to implement a different notation, e.g. LISREL, you should implement a subtype of SemImply. """ -abstract type SemImply{MS <: MeanStructure, HE <: HessianEvaluation} end +abstract type SemImply{MS <: MeanStructure, HE <: HessianEval} end MeanStructure(::Type{<:SemImply{MS}}) where {MS <: MeanStructure} = MS -HessianEvaluation(::Type{<:SemImply{MS, HE}}) where {MS, HE <: MeanStructure} = HE +HessianEval(::Type{<:SemImply{MS, HE}}) where {MS, HE <: MeanStructure} = HE "Subtype of SemImply for all objects that can serve as the imply field of a SEM and use some form of symbolic precomputation." abstract type SemImplySymbolic{MS, HE} <: SemImply{MS, HE} end From c673cd1d8eed836ac09812547ee5220fc9251a5f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 8 Oct 2024 01:12:51 -0700 Subject: [PATCH 110/364] MeanStructure -> MeanStruct --- src/StructuralEquationModels.jl | 6 +++--- src/imply/RAM/generic.jl | 8 ++++---- src/imply/RAM/symbolic.jl | 10 +++++----- src/imply/empty.jl | 2 +- src/loss/ML/ML.jl | 8 ++++---- src/loss/WLS/WLS.jl | 4 ++-- src/types.jl | 22 +++++++++++----------- 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 2a469dd91..944542379 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -95,9 +95,9 @@ export AbstractSem, Sem, SemFiniteDiff, SemEnsemble, - MeanStructure, - NoMeanStructure, - HasMeanStructure, + MeanStruct, + NoMeanStruct, + HasMeanStruct, HessianEval, ExactHessian, ApproxHessian, diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 85cbc0220..d7b0f8097 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -107,7 +107,7 @@ mutable struct RAM{ ∇S::S2 ∇M::S3 - RAM{MS}(args...) where {MS <: MeanStructure} = new{MS, map(typeof, args)...}(args...) + RAM{MS}(args...) where {MS <: MeanStruct} = new{MS, map(typeof, args)...}(args...) end ############################################################################################ @@ -160,7 +160,7 @@ function RAM(; # μ if meanstructure - MS = HasMeanStructure + MS = HasMeanStruct !isnothing(M_indices) || throw( ArgumentError( "You set `meanstructure = true`, but your model specification contains no mean parameters.", @@ -169,7 +169,7 @@ function RAM(; ∇M = gradient_required ? matrix_gradient(M_indices, n_var) : nothing μ = zeros(n_obs) else - MS = NoMeanStructure + MS = NoMeanStruct M_indices = nothing M_pre = nothing μ = nothing @@ -226,7 +226,7 @@ function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingl mul!(imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹, imply.S) mul!(imply.Σ, imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹') - if MeanStructure(imply) === HasMeanStructure + if MeanStruct(imply) === HasMeanStruct mul!(imply.μ, imply.F⨉I_A⁻¹, imply.M) end end diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index d79454f3f..3e2fc0ad3 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -79,7 +79,7 @@ struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5} < ∇μ_function::F5 ∇μ::A5 - RAMSymbolic{MS}(args...) where {MS <: MeanStructure} = + RAMSymbolic{MS}(args...) where {MS <: MeanStruct} = new{MS, map(typeof, args)...}(args...) end @@ -163,7 +163,7 @@ function RAMSymbolic(; # μ if meanstructure - MS = HasMeanStructure + MS = HasMeanStruct μ_symbolic = eval_μ_symbolic(M, I_A⁻¹, F) μ_function = Symbolics.build_function(μ_symbolic, par, expression = Val{false})[2] μ = zeros(size(μ_symbolic)) @@ -177,7 +177,7 @@ function RAMSymbolic(; ∇μ = nothing end else - MS = NoMeanStructure + MS = NoMeanStruct μ_function = nothing μ = nothing ∇μ_function = nothing @@ -213,13 +213,13 @@ function update!( par, ) imply.Σ_function(imply.Σ, par) - if MeanStructure(imply) === HasMeanStructure + if MeanStruct(imply) === HasMeanStruct imply.μ_function(imply.μ, par) end if is_gradient_required(targets) || is_hessian_required(targets) imply.∇Σ_function(imply.∇Σ, par) - if MeanStructure(imply) === HasMeanStructure + if MeanStruct(imply) === HasMeanStruct imply.∇μ_function(imply.∇μ, par) end end diff --git a/src/imply/empty.jl b/src/imply/empty.jl index 8b23194ac..6716e2c05 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -25,7 +25,7 @@ model per group and an additional model with `ImplyEmpty` and `SemRidge` for the ## Implementation Subtype of `SemImply`. """ -struct ImplyEmpty{V2} <: SemImply{NoMeanStructure, ExactHessian} +struct ImplyEmpty{V2} <: SemImply{NoMeanStruct, ExactHessian} ram_matrices::V2 end diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 261b260d6..20028ad77 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -69,7 +69,7 @@ function evaluate!( par, ) if !isnothing(hessian) - (MeanStructure(implied) === HasMeanStructure) && + (MeanStruct(implied) === HasMeanStruct) && throw(DomainError(H, "hessian of ML + meanstructure is not available")) end @@ -92,7 +92,7 @@ function evaluate!( mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) isnothing(objective) || (objective = ld + tr(Σ⁻¹Σₒ)) - if MeanStructure(implied) === HasMeanStructure + if MeanStruct(implied) === HasMeanStruct μ = implied.μ μₒ = obs_mean(observed(model)) μ₋ = μₒ - μ @@ -167,7 +167,7 @@ function evaluate!( if !isnothing(objective) objective = ld + tr(Σ⁻¹Σₒ) - if MeanStructure(implied) === HasMeanStructure + if MeanStruct(implied) === HasMeanStruct μ = implied.μ μₒ = obs_mean(observed(model)) μ₋ = μₒ - μ @@ -186,7 +186,7 @@ function evaluate!( mul!(gradient, ∇A', vec(C * S * I_A⁻¹'), 2, 0) mul!(gradient, ∇S', vec(C), 1, 1) - if MeanStructure(implied) === HasMeanStructure + if MeanStruct(implied) === HasMeanStruct μ = implied.μ μₒ = obs_mean(observed(model)) ∇M = implied.∇M diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 2345f859d..620784620 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -107,7 +107,7 @@ function evaluate!( model::AbstractSemSingle, par, ) - if !isnothing(hessian) && (MeanStructure(implied) === HasMeanStructure) + if !isnothing(hessian) && (MeanStruct(implied) === HasMeanStruct) error("hessian of WLS with meanstructure is not available") end @@ -135,7 +135,7 @@ function evaluate!( ∇²Σ_function!(∇²Σ, J, par) hessian .+= ∇²Σ end - if MeanStructure(implied) === HasMeanStructure + if MeanStruct(implied) === HasMeanStruct μ = implied.μ μₒ = obs_mean(observed(model)) μ₋ = μₒ - μ diff --git a/src/types.jl b/src/types.jl index 12082be12..53eec1496 100644 --- a/src/types.jl +++ b/src/types.jl @@ -11,16 +11,16 @@ abstract type AbstractSemSingle{O, I, L, D} <: AbstractSem end abstract type AbstractSemCollection <: AbstractSem end "Meanstructure trait for `SemImply` subtypes" -abstract type MeanStructure end -"Indicates that `SemImply` subtype supports meanstructure" -struct HasMeanStructure <: MeanStructure end -"Indicates that `SemImply` subtype does not support meanstructure" -struct NoMeanStructure <: MeanStructure end +abstract type MeanStruct end +"Indicates that `SemImply` subtype supports mean structure" +struct HasMeanStruct <: MeanStruct end +"Indicates that `SemImply` subtype does not support mean structure" +struct NoMeanStruct <: MeanStruct end # fallback implementation -MeanStructure(::Type{T}) where {T} = - error("Objects of type $T do not support the MeanStructure trait") -MeanStructure(semobj) = MeanStructure(typeof(semobj)) +MeanStruct(::Type{T}) where {T} = + error("Objects of type $T do not support MeanStruct trait") +MeanStruct(semobj) = MeanStruct(typeof(semobj)) "Hessian Evaluation trait for `SemImply` and `SemLossFunction` subtypes" abstract type HessianEval end @@ -97,10 +97,10 @@ Computed model-implied values that should be compared with the observed data to e. g. the model implied covariance or mean. If you would like to implement a different notation, e.g. LISREL, you should implement a subtype of SemImply. """ -abstract type SemImply{MS <: MeanStructure, HE <: HessianEval} end +abstract type SemImply{MS <: MeanStruct, HE <: HessianEval} end -MeanStructure(::Type{<:SemImply{MS}}) where {MS <: MeanStructure} = MS -HessianEval(::Type{<:SemImply{MS, HE}}) where {MS, HE <: MeanStructure} = HE +MeanStruct(::Type{<:SemImply{MS}}) where {MS <: MeanStruct} = MS +HessianEval(::Type{<:SemImply{MS, HE}}) where {MS, HE <: MeanStruct} = HE "Subtype of SemImply for all objects that can serve as the imply field of a SEM and use some form of symbolic precomputation." abstract type SemImplySymbolic{MS, HE} <: SemImply{MS, HE} end From 0cecaa857821eba2a619419ca04392b0c3db0c9b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 8 Oct 2024 01:14:48 -0700 Subject: [PATCH 111/364] SemImply: replace common type params with fields --- src/imply/RAM/generic.jl | 8 ++++++-- src/imply/RAM/symbolic.jl | 6 ++++-- src/imply/empty.jl | 6 ++++-- src/loss/ML/FIML.jl | 4 +++- src/loss/ML/ML.jl | 5 +++-- src/loss/WLS/WLS.jl | 5 +++-- src/loss/constant/constant.jl | 5 +++-- src/loss/regularization/ridge.jl | 4 +++- src/types.jl | 19 +++++++++---------- test/examples/multigroup/build_models.jl | 6 +++++- 10 files changed, 43 insertions(+), 25 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index d7b0f8097..e7e0b36f5 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -84,7 +84,10 @@ mutable struct RAM{ S1, S2, S3, -} <: SemImply{MS, ExactHessian} +} <: SemImply + meanstruct::MS + hessianeval::ExactHessian + Σ::A1 A::A2 S::A3 @@ -107,7 +110,8 @@ mutable struct RAM{ ∇S::S2 ∇M::S3 - RAM{MS}(args...) where {MS <: MeanStruct} = new{MS, map(typeof, args)...}(args...) + RAM{MS}(args...) where {MS <: MeanStruct} = + new{MS, map(typeof, args)...}(MS(), ExactHessian(), args...) end ############################################################################################ diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 3e2fc0ad3..9a96942ae 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -63,7 +63,9 @@ and for models with a meanstructure, the model implied means are computed as ``` """ struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5} <: - SemImplySymbolic{MS, ExactHessian} + SemImplySymbolic + meanstruct::MS + hessianeval::ExactHessian Σ_function::F1 ∇Σ_function::F2 ∇²Σ_function::F3 @@ -80,7 +82,7 @@ struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5} < ∇μ::A5 RAMSymbolic{MS}(args...) where {MS <: MeanStruct} = - new{MS, map(typeof, args)...}(args...) + new{MS, map(typeof, args)...}(MS(), ExactHessian(), args...) end ############################################################################################ diff --git a/src/imply/empty.jl b/src/imply/empty.jl index 6716e2c05..66373bc1b 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -25,7 +25,9 @@ model per group and an additional model with `ImplyEmpty` and `SemRidge` for the ## Implementation Subtype of `SemImply`. """ -struct ImplyEmpty{V2} <: SemImply{NoMeanStruct, ExactHessian} +struct ImplyEmpty{V2} <: SemImply + hessianeval::ExactHessian + meanstruct::NoMeanStruct ram_matrices::V2 end @@ -34,7 +36,7 @@ end ############################################################################################ function ImplyEmpty(; specification, kwargs...) - return ImplyEmpty(convert(RAMMatrices, specification)) + return ImplyEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) end ############################################################################################ diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 20e837997..20c81b831 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -24,7 +24,8 @@ Analytic gradients are available. ## Implementation Subtype of `SemLossFunction`. """ -mutable struct SemFIML{INV, C, L, O, M, IM, I, T, W} <: SemLossFunction{ExactHessian} +mutable struct SemFIML{INV, C, L, O, M, IM, I, T, W} <: SemLossFunction + hessianeval::ExactHessian inverses::INV #preallocated inverses of imp_cov choleskys::C #preallocated choleskys logdets::L #logdets of implied covmats @@ -65,6 +66,7 @@ function SemFIML(; observed, specification, kwargs...) [findall(x -> !(x[1] ∈ ind || x[2] ∈ ind), ∇ind) for ind in patterns_not(observed)] return SemFIML( + ExactHessian(), inverses, choleskys, logdets, diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 20028ad77..e81d27de7 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -27,13 +27,14 @@ Analytic gradients are available, and for models without a meanstructure, also a ## Implementation Subtype of `SemLossFunction`. """ -struct SemML{HE <: HessianEval, INV, M, M2} <: SemLossFunction{HE} +struct SemML{HE <: HessianEval, INV, M, M2} <: SemLossFunction + hessianeval::HE Σ⁻¹::INV Σ⁻¹Σₒ::M meandiff::M2 SemML{HE}(args...) where {HE <: HessianEval} = - new{HE, map(typeof, args)...}(args...) + new{HE, map(typeof, args)...}(HE(), args...) end ############################################################################################ diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 620784620..9702a9cf4 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -38,7 +38,8 @@ Analytic gradients are available, and for models without a meanstructure, also a ## Implementation Subtype of `SemLossFunction`. """ -struct SemWLS{HE <: HessianEval, Vt, St, C} <: SemLossFunction{HE} +struct SemWLS{HE <: HessianEval, Vt, St, C} <: SemLossFunction + hessianeval::HE V::Vt σₒ::St V_μ::C @@ -49,7 +50,7 @@ end ############################################################################################ SemWLS{HE}(args...) where {HE <: HessianEval} = - SemWLS{HE, map(typeof, args)...}(args...) + SemWLS{HE, map(typeof, args)...}(HE(), args...) function SemWLS(; observed, diff --git a/src/loss/constant/constant.jl b/src/loss/constant/constant.jl index 639864610..cb5157346 100644 --- a/src/loss/constant/constant.jl +++ b/src/loss/constant/constant.jl @@ -25,7 +25,8 @@ Analytic gradients and hessians are available. ## Implementation Subtype of `SemLossFunction`. """ -struct SemConstant{C} <: SemLossFunction{ExactHessian} +struct SemConstant{C} <: SemLossFunction + hessianeval::ExactHessian c::C end @@ -34,7 +35,7 @@ end ############################################################################################ function SemConstant(; constant_loss, kwargs...) - return SemConstant(constant_loss) + return SemConstant(ExactHessian(), constant_loss) end ############################################################################################ diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index be9b14fa5..6ec59ec39 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -29,7 +29,8 @@ Analytic gradients and hessians are available. ## Implementation Subtype of `SemLossFunction`. """ -struct SemRidge{P, W1, W2, GT, HT} <: SemLossFunction{ExactHessian} +struct SemRidge{P, W1, W2, GT, HT} <: SemLossFunction + hessianeval::ExactHessian α::P which::W1 which_H::W2 @@ -64,6 +65,7 @@ function SemRidge(; end which_H = [CartesianIndex(x, x) for x in which_ridge] return SemRidge( + ExactHessian(), α_ridge, which_ridge, which_H, diff --git a/src/types.jl b/src/types.jl index 53eec1496..020f6e77d 100644 --- a/src/types.jl +++ b/src/types.jl @@ -17,9 +17,11 @@ struct HasMeanStruct <: MeanStruct end "Indicates that `SemImply` subtype does not support mean structure" struct NoMeanStruct <: MeanStruct end -# fallback implementation +# default implementation MeanStruct(::Type{T}) where {T} = + hasfield(T, :meanstruct) ? fieldtype(T, :meanstruct) : error("Objects of type $T do not support MeanStruct trait") + MeanStruct(semobj) = MeanStruct(typeof(semobj)) "Hessian Evaluation trait for `SemImply` and `SemLossFunction` subtypes" @@ -27,15 +29,15 @@ abstract type HessianEval end struct ApproxHessian <: HessianEval end struct ExactHessian <: HessianEval end -# fallback implementation +# default implementation HessianEval(::Type{T}) where {T} = + hasfield(T, :hessianeval) ? fieldtype(T, :hessianeval) : error("Objects of type $T do not support HessianEval trait") + HessianEval(semobj) = HessianEval(typeof(semobj)) "Supertype for all loss functions of SEMs. If you want to implement a custom loss function, it should be a subtype of `SemLossFunction`." -abstract type SemLossFunction{HE <: HessianEval} end - -HessianEval(::Type{<:SemLossFunction{HE}}) where {HE <: HessianEval} = HE +abstract type SemLossFunction end """ SemLoss(args...; loss_weights = nothing, ...) @@ -97,13 +99,10 @@ Computed model-implied values that should be compared with the observed data to e. g. the model implied covariance or mean. If you would like to implement a different notation, e.g. LISREL, you should implement a subtype of SemImply. """ -abstract type SemImply{MS <: MeanStruct, HE <: HessianEval} end - -MeanStruct(::Type{<:SemImply{MS}}) where {MS <: MeanStruct} = MS -HessianEval(::Type{<:SemImply{MS, HE}}) where {MS, HE <: MeanStruct} = HE +abstract type SemImply end "Subtype of SemImply for all objects that can serve as the imply field of a SEM and use some form of symbolic precomputation." -abstract type SemImplySymbolic{MS, HE} <: SemImply{MS, HE} end +abstract type SemImplySymbolic <: SemImply end """ Sem(;observed = SemObservedData, imply = RAM, loss = SemML, optimizer = SemOptimizerOptim, kwargs...) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 4790c9d36..2e1af38a2 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -114,7 +114,11 @@ end # ML estimation - user defined loss function ############################################################################################ -struct UserSemML <: SemLossFunction{ExactHessian} end +struct UserSemML <: SemLossFunction + hessianeval::ExactHessian + + UserSemML() = new(ExactHessian()) +end ############################################################################################ ### functors From d42f4e6dfbea2c941bce299ae46784e882f5b1f6 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 29 Oct 2024 17:16:35 +0100 Subject: [PATCH 112/364] close #216 --- src/additional_functions/start_val/start_simple.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 8e3cb32cb..3b29ec178 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -21,7 +21,7 @@ function start_simple(model::AbstractSemSingle; kwargs...) model.observed, model.imply, model.optimizer, - model.loss.functions..., + model.loss.functions...; kwargs..., ) end From 23d0ace0baeaba6cafaf0a0f71cef7ec332d200c Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 29 Oct 2024 17:50:44 +0100 Subject: [PATCH 113/364] close #205 --- src/frontend/specification/EnsembleParameterTable.jl | 4 ++++ test/examples/multigroup/multigroup.jl | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index b0b50448b..1b5237e2d 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -18,6 +18,10 @@ EnsembleParameterTable(::Nothing; params::Union{Nothing, Vector{Symbol}} = nothi isnothing(params) ? Symbol[] : copy(params), ) +# convert pairs to dict +EnsembleParameterTable(ps::Pair{K, V}...; params = nothing) where {K, V} = + EnsembleParameterTable(Dict(ps...); params = params) + # dictionary of SEM specifications function EnsembleParameterTable( spec_ensemble::AbstractDict{K, V}; diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index e428eba1d..950ec5305 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -69,7 +69,8 @@ specification_g2 = RAMMatrices(; ) partable = EnsembleParameterTable( - Dict(:Pasteur => specification_g1, :Grant_White => specification_g2), + :Pasteur => specification_g1, + :Grant_White => specification_g2 ) specification_miss_g1 = nothing From 8a06e9ae3d471ca8c806abe281c725d488a675fb Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 29 Oct 2024 18:20:12 +0100 Subject: [PATCH 114/364] update EnsembleParameterTable docs and add methods for par table equality --- src/frontend/specification/EnsembleParameterTable.jl | 11 +++++++++++ src/frontend/specification/ParameterTable.jl | 11 +++++++++++ src/frontend/specification/documentation.jl | 8 ++++---- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 1b5237e2d..b1c8fb8e6 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -142,3 +142,14 @@ function update_partable!( ) return update_partable!(partables, column, Dict(zip(params, values)), default) end + +############################################################################################ +### Additional methods +############################################################################################ + +function Base.:(==)(p1::EnsembleParameterTable, p2::EnsembleParameterTable) + out = + (p1.tables == p2.tables) && + (p1.params == p2.params) + return out +end \ No newline at end of file diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 687b712ba..df2cc165b 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -126,6 +126,17 @@ end ### Additional Methods ############################################################################################ +# Equality -------------------------------------------------------------------------------- +function Base.:(==)(p1::ParameterTable, p2::ParameterTable) + out = + (p1.columns == p2.columns) && + (p1.observed_vars == p2.observed_vars) && + (p1.latent_vars == p2.latent_vars) && + (p1.sorted_vars == p2.sorted_vars) && + (p1.params == p2.params) + return out +end + # Iteration -------------------------------------------------------------------------------- ParameterTableRow = @NamedTuple begin from::Symbol diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl index 464af144b..e869dd43f 100644 --- a/src/frontend/specification/documentation.jl +++ b/src/frontend/specification/documentation.jl @@ -72,16 +72,16 @@ function ParameterTable end (1) EnsembleParameterTable(;graph, observed_vars, latent_vars, groups) - (2) EnsembleParameterTable(args...; groups) + (2) EnsembleParameterTable(ps::Pair...; params = nothing) -Return an `EnsembleParameterTable` constructed from (1) a graph or (2) multiple RAM matrices. +Return an `EnsembleParameterTable` constructed from (1) a graph or (2) multiple specifications. # Arguments - `graph`: graph defined via `@StenoGraph` - `observed_vars::Vector{Symbol}`: observed variable names - `latent_vars::Vector{Symbol}`: latent variable names -- `groups::Vector{Symbol}`: group names -- `args...`: `RAMMatrices` for each model +- `params::Vector{Symbol}`: (optional) a vector of parameter names +- `ps::Pair...`: `:group_name => specification`, where `specification` is either a `ParameterTable` or `RAMMatrices` # Examples See the online documentation on [Multigroup models](@ref). From 1244d20ae83c6fbe3ba1582d307b0615c645c03b Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 29 Oct 2024 18:59:26 +0100 Subject: [PATCH 115/364] close #213 --- src/frontend/fit/summary.jl | 56 +++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index a77f62c21..e6026e5f4 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -51,6 +51,7 @@ function sem_summary( secondary_color = :light_yellow, digits = 2, show_variables = true, + show_columns = nothing ) if show_variables print("\n") @@ -86,13 +87,19 @@ function sem_summary( print("\n") columns = keys(partable.columns) + show_columns = isnothing(show_columns) ? nothing : intersect(show_columns, columns) printstyled("Loadings: \n"; color = color) print("\n") - sorted_columns = [:to, :estimate, :param, :value_fixed, :start] - loading_columns = sort_partially(sorted_columns, columns) - header_cols = copy(loading_columns) + if isnothing(show_columns) + sorted_columns = [:to, :estimate, :param, :value_fixed, :start] + loading_columns = sort_partially(sorted_columns, columns) + header_cols = copy(loading_columns) + else + loading_columns = copy(show_columns) + header_cols = copy(loading_columns) + end for var in partable.latent_vars indicator_indices = findall( @@ -131,15 +138,19 @@ function sem_summary( partable, ) - sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] - regression_columns = sort_partially(sorted_columns, columns) + if isnothing(show_columns) + sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] + regression_columns = sort_partially(sorted_columns, columns) + else + regression_columns = copy(show_columns) + end regression_array = reduce( hcat, check_round(partable.columns[c][regression_indices]; digits = digits) for c in regression_columns ) - regression_columns[2] = Symbol("") + regression_columns[2] = regression_columns[2] == :relation ? Symbol("") : regression_columns[2] print("\n") pretty_table( @@ -155,14 +166,18 @@ function sem_summary( var_indices = findall(r -> r.relation == :↔ && r.to == r.from, partable) - sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] - var_columns = sort_partially(sorted_columns, columns) + if isnothing(show_columns) + sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] + var_columns = sort_partially(sorted_columns, columns) + else + var_columns = copy(show_columns) + end var_array = reduce( hcat, check_round(partable.columns[c][var_indices]; digits) for c in var_columns ) - var_columns[2] = Symbol("") + var_columns[2] = var_columns[2] == :relation ? Symbol("") : var_columns[2] print("\n") pretty_table( @@ -178,14 +193,18 @@ function sem_summary( covar_indices = findall(r -> r.relation == :↔ && r.to != r.from, partable) - covar_columns = sort_partially(sorted_columns, columns) + if isnothing(show_columns) + covar_columns = sort_partially(sorted_columns, columns) + else + covar_columns = copy(show_columns) + end covar_array = reduce( hcat, check_round(partable.columns[c][covar_indices]; digits = digits) for c in covar_columns ) - covar_columns[2] = Symbol("") + covar_columns[2] = covar_columns[2] == :relation ? Symbol("") : covar_columns[2] print("\n") pretty_table( @@ -202,15 +221,19 @@ function sem_summary( if length(mean_indices) > 0 printstyled("Means: \n"; color = color) - sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] - mean_columns = sort_partially(sorted_columns, columns) + if isnothing(show_columns) + sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] + mean_columns = sort_partially(sorted_columns, columns) + else + mean_columns = copy(show_columns) + end mean_array = reduce( hcat, check_round(partable.columns[c][mean_indices]; digits = digits) for c in mean_columns ) - mean_columns[2] = Symbol("") + mean_columns[2] = mean_columns[2] == :relation ? Symbol("") : mean_columns[2] print("\n") pretty_table( @@ -233,6 +256,7 @@ function sem_summary( secondary_color = :light_yellow, digits = 2, show_variables = true, + show_columns = nothing ) if show_variables print("\n") @@ -273,6 +297,7 @@ function sem_summary( secondary_color = secondary_color, digits = digits, show_variables = false, + show_columns = show_columns ) end @@ -310,7 +335,7 @@ end """ (1) sem_summary(sem_fit::SemFit; show_fitmeasures = false) - (2) sem_summary(partable::AbstractParameterTable) + (2) sem_summary(partable::AbstractParameterTable; ...) Print information about (1) a fitted SEM or (2) a parameter table to stdout. @@ -320,5 +345,6 @@ Print information about (1) a fitted SEM or (2) a parameter table to stdout. - `color = :light_cyan`: color of some parts of the printed output. Can be adjusted for readability. - `secondary_color = :light_yellow` - `show_variables = true` +- `show_columns = nothing`: columns names to include in the output e.g.`[:from, :to, :estimate]`) """ function sem_summary end From 28ee1ae5aea0c57c8fac2b4ffe38cbd8338aae39 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 29 Oct 2024 22:34:00 +0100 Subject: [PATCH 116/364] close #157 --- docs/src/tutorials/collection/multigroup.md | 39 ++++-------- src/types.jl | 35 ++++++++++- test/examples/multigroup/build_models.jl | 68 +++++++++++++++++++++ test/examples/multigroup/multigroup.jl | 3 + 4 files changed, 114 insertions(+), 31 deletions(-) diff --git a/docs/src/tutorials/collection/multigroup.md b/docs/src/tutorials/collection/multigroup.md index 399d89760..5ee88e936 100644 --- a/docs/src/tutorials/collection/multigroup.md +++ b/docs/src/tutorials/collection/multigroup.md @@ -6,20 +6,17 @@ using StructuralEquationModels As an example, we will fit the model from [the `lavaan` tutorial](https://lavaan.ugent.be/tutorial/groups.html) with loadings constrained to equality across groups. -We first load the example data and split it between groups: +We first load the example data. +We have to make sure that the column indicating the group (here called `school`) is a vector of `Symbol`s, not strings - so we convert it. ```@setup mg dat = example_data("holzinger_swineford") - -dat_g1 = dat[dat.school .== "Pasteur", :] -dat_g2 = dat[dat.school .== "Grant-White", :] +dat.school = ifelse.(dat.school .== "Pasteur", :Pasteur, :Grant_White) ``` ```julia dat = example_data("holzinger_swineford") - -dat_g1 = dat[dat.school .== "Pasteur", :] -dat_g2 = dat[dat.school .== "Grant-White", :] +dat.school = ifelse.(dat.school .== "Pasteur", :Pasteur, :Grant_White) ``` We then specify our model via the graph interface: @@ -68,32 +65,18 @@ partable = EnsembleParameterTable( groups = groups) ``` -The parameter table can be used to create a `Dict` of RAMMatrices with keys equal to the group names and parameter tables as values: +The parameter table can be used to create a `SemEnsemble` model: ```@example mg; ansicolor = true -specification = convert(Dict{Symbol, RAMMatrices}, partable) +model_ml_multigroup = SemEnsemble( + specification = partable, + data = dat, + column = :school, + groups = groups) ``` -That is, you can asses the group-specific `RAMMatrices` as `specification[:group_name]`. - !!! note "A different way to specify" - Instead of choosing the workflow "Graph -> EnsembleParameterTable -> RAMMatrices", you may also directly specify RAMMatrices for each group (for an example see [this test](https://github.com/StructuralEquationModels/StructuralEquationModels.jl/blob/main/test/examples/multigroup/multigroup.jl)). - -The next step is to construct the model: - -```@example mg; ansicolor = true -model_g1 = Sem( - specification = specification[:Pasteur], - data = dat_g1 -) - -model_g2 = Sem( - specification = specification[:Grant_White], - data = dat_g2 -) - -model_ml_multigroup = SemEnsemble(model_g1, model_g2) -``` + Instead of choosing the workflow "Graph -> EnsembleParameterTable -> model", you may also directly specify RAMMatrices for each group (for an example see [this test](https://github.com/StructuralEquationModels/StructuralEquationModels.jl/blob/main/test/examples/multigroup/multigroup.jl)). We now fit the model and inspect the parameter estimates: diff --git a/src/types.jl b/src/types.jl index 020f6e77d..576252726 100644 --- a/src/types.jl +++ b/src/types.jl @@ -163,16 +163,22 @@ end # ensemble models ############################################################################################ """ - SemEnsemble(models..., optimizer = SemOptimizerOptim, weights = nothing, kwargs...) + (1) SemEnsemble(models..., optimizer = SemOptimizerOptim, weights = nothing, kwargs...) -Constructor for ensemble models. + (2) SemEnsemble(;specification, data, groups, column = :group, optimizer = SemOptimizerOptim, kwargs...) + +Constructor for ensemble models. (2) can be used to conveniently specify multigroup models. # Arguments - `models...`: `AbstractSem`s. - `optimizer`: object of subtype `SemOptimizer` or a constructor. - `weights::Vector`: Weights for each model. Defaults to the number of observed data points. +- `specification::EnsembleParameterTable`: Model specification. +- `data::DataFrame`: Observed data. Must contain a `column` of type `Vector{Symbol}` that contains the group. +- `groups::Vector{Symbol}`: Group names. +- `column::Symbol`: Name of the column in `data` that contains the group. -All additional kwargs are passed down to the constructor for the optimizer field. +All additional kwargs are passed down to the model parts. Returns a SemEnsemble with fields - `n::Int`: Number of models. @@ -189,6 +195,7 @@ struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, D, I} <: AbstractSemColle params::I end +# constructor from multiple models function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing, kwargs...) n = length(models) @@ -217,6 +224,28 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing return SemEnsemble(n, models, weights, optimizer, params) end +# constructor from EnsembleParameterTable and data set +function SemEnsemble(;specification, data, groups, column = :group, optimizer = SemOptimizerOptim, kwargs...) + if specification isa EnsembleParameterTable + specification = convert(Dict{Symbol, RAMMatrices}, specification) + end + models = [] + for group in groups + ram_matrices = specification[group] + data_group = select(filter(r -> r[column] == group, data), Not(column)) + if iszero(nrow(data_group)) + error("Your data does not contain any observations from group `$(group)`.") + end + model = Sem(; + specification = ram_matrices, + data = data_group, + kwargs... + ) + push!(models, model) + end + return SemEnsemble(models...; optimizer = optimizer, weights = nothing, kwargs...) +end + params(ensemble::SemEnsemble) = ensemble.params """ diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 2e1af38a2..4b5afd58e 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -6,11 +6,21 @@ model_g1 = Sem(specification = specification_g1, data = dat_g1, imply = RAMSymbo model_g2 = Sem(specification = specification_g2, data = dat_g2, imply = RAM) +# test the different constructors model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) +model_ml_multigroup2 = SemEnsemble( + specification = partable, + data = dat, + column = :school, + groups = [:Pasteur, :Grant_White], + loss = SemML +) + # gradients @testset "ml_gradients_multigroup" begin test_gradient(model_ml_multigroup, start_test; atol = 1e-9) + test_gradient(model_ml_multigroup2, start_test; atol = 1e-9) end # fit @@ -23,6 +33,14 @@ end atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) + solution = sem_fit(model_ml_multigroup2) + update_estimate!(partable, solution) + test_estimates( + partable, + solution_lav[:parameter_estimates_ml]; + atol = 1e-4, + lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), + ) end @testset "fitmeasures/se_ml" begin @@ -33,7 +51,23 @@ end rtol = 1e-2, atol = 1e-7, ) + update_se_hessian!(partable, solution_ml) + test_estimates( + partable, + solution_lav[:parameter_estimates_ml]; + atol = 1e-3, + col = :se, + lav_col = :se, + lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), + ) + solution_ml = sem_fit(model_ml_multigroup2) + test_fitmeasures( + fit_measures(solution_ml), + solution_lav[:fitmeasures_ml]; + rtol = 1e-2, + atol = 1e-7, + ) update_se_hessian!(partable, solution_ml) test_estimates( partable, @@ -238,6 +272,15 @@ if !isnothing(specification_miss_g1) ) model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) + model_ml_multigroup2 = SemEnsemble( + specification = partable_miss, + data = dat_missing, + column = :school, + groups = [:Pasteur, :Grant_White], + loss = SemFIML, + observed = SemObservedMissing, + meanstructure = true + ) ############################################################################################ ### test gradients @@ -265,6 +308,7 @@ if !isnothing(specification_miss_g1) @testset "fiml_gradients_multigroup" begin test_gradient(model_ml_multigroup, start_test; atol = 1e-7) + test_gradient(model_ml_multigroup2, start_test; atol = 1e-7) end @testset "fiml_solution_multigroup" begin @@ -276,6 +320,14 @@ if !isnothing(specification_miss_g1) atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) + solution = sem_fit(model_ml_multigroup2) + update_estimate!(partable_miss, solution) + test_estimates( + partable_miss, + solution_lav[:parameter_estimates_fiml]; + atol = 1e-4, + lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), + ) end @testset "fitmeasures/se_fiml" begin @@ -286,7 +338,23 @@ if !isnothing(specification_miss_g1) rtol = 1e-3, atol = 0, ) + update_se_hessian!(partable_miss, solution) + test_estimates( + partable_miss, + solution_lav[:parameter_estimates_fiml]; + atol = 1e-3, + col = :se, + lav_col = :se, + lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), + ) + solution = sem_fit(model_ml_multigroup2) + test_fitmeasures( + fit_measures(solution), + solution_lav[:fitmeasures_fiml]; + rtol = 1e-3, + atol = 0, + ) update_se_hessian!(partable_miss, solution) test_estimates( partable_miss, diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 950ec5305..a2f277d91 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -15,6 +15,9 @@ dat_g2 = dat[dat.school.=="Grant-White", :] dat_miss_g1 = dat_missing[dat_missing.school.=="Pasteur", :] dat_miss_g2 = dat_missing[dat_missing.school.=="Grant-White", :] +dat.school = ifelse.(dat.school .== "Pasteur", :Pasteur, :Grant_White) +dat_missing.school = ifelse.(dat_missing.school .== "Pasteur", :Pasteur, :Grant_White) + ############################################################################################ ### specification - RAMMatrices ############################################################################################ From 1fd1a6a3458c0a48442d56e5973a046a103414fd Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 21 Nov 2024 11:51:24 +0100 Subject: [PATCH 117/364] add method for --- src/additional_functions/simulation.jl | 46 ++++++++++- .../political_democracy/constructor.jl | 79 +++++++++++++++++++ .../recover_parameters_twofact.jl | 2 +- 3 files changed, 125 insertions(+), 2 deletions(-) diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 0dda725c6..68ec62142 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -6,7 +6,7 @@ Return a new model with swaped observed part. # Arguments -- `model::AbstractSemSingle`: optimization algorithm. +- `model::AbstractSemSingle`: model to swap the observed part of. - `kwargs`: additional keyword arguments; typically includes `data = ...` - `observed`: Either an object of subtype of `SemObserved` or a subtype of `SemObserved` @@ -98,3 +98,47 @@ function update_observed(loss::SemLoss, new_observed; kwargs...) ) return SemLoss(new_functions, loss.weights) end + + +############################################################################################ +# simulate data +############################################################################################ +""" + (1) rand(model::AbstractSemSingle, params, n) + + (2) rand(model::AbstractSemSingle, n) + +Sample normally distributed data from the model-implied covariance matrix and mean vector. + +# Arguments +- `model::AbstractSemSingle`: model to simulate from. +- `params`: parameter values to simulate from. +- `n::Integer`: Number of samples. + +# Examples +```julia +rand(model, start_simple(model), 100) +``` +""" +function Distributions.rand( + model::AbstractSemSingle{O, I, L, D}, + params, + n::Integer) where {O, I <: Union{RAM, RAMSymbolic}, L, D} + update!( + EvaluationTargets{true, false, false}(), + model.imply, + model, + params) + return rand(model, n) +end + +function Distributions.rand( + model::AbstractSemSingle{O, I, L, D}, + n::Integer) where {O, I <: Union{RAM, RAMSymbolic}, L, D} + if MeanStruct(model.imply) === NoMeanStruct + data = permutedims(rand(MvNormal(Symmetric(model.imply.Σ)), n)) + elseif MeanStruct(model.imply) === HasMeanStruct + data = permutedims(rand(MvNormal(model.imply.μ, Symmetric(model.imply.Σ)), n)) + end + return data +end \ No newline at end of file diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index bf674dd73..3fc99289a 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -1,4 +1,5 @@ using Statistics: cov, mean +using Random ############################################################################################ ### models w.o. meanstructure @@ -161,6 +162,43 @@ end ) end +############################################################################################ +### data simulation +############################################################################################ + +@testset "data_simulation_wo_mean" begin + # parameters to recover + params = start_simple( + model_ml; + start_loadings = 0.5, + start_regressions = 0.5, + start_variances_observed = 0.5, + start_variances_latent = 1.0, + start_covariances_observed = 0.2) + # set seed for simulation + Random.seed!(83472834) + colnames = Symbol.(names(example_data("political_democracy"))) + # simulate data + model_ml_new = swap_observed( + model_ml, + data = rand(model_ml, params, 100_000), + specification = spec, + obs_colnames = colnames + ) + model_ml_sym_new = swap_observed( + model_ml_sym, + data = rand(model_ml_sym, params, 100_000), + specification = spec, + obs_colnames = colnames + ) + # fit models + sol_ml = solution(sem_fit(model_ml_new)) + sol_ml_sym = solution(sem_fit(model_ml_sym_new)) + # check solution + @test maximum(abs.(sol_ml - params)) < 0.01 + @test maximum(abs.(sol_ml_sym - params)) < 0.01 +end + ############################################################################################ ### test hessians ############################################################################################ @@ -332,6 +370,47 @@ end ) end + +############################################################################################ +### data simulation +############################################################################################ + +@testset "data_simulation_with_mean" begin + # parameters to recover + params = start_simple( + model_ml; + start_loadings = 0.5, + start_regressions = 0.5, + start_variances_observed = 0.5, + start_variances_latent = 1.0, + start_covariances_observed = 0.2, + start_means = 0.5) + # set seed for simulation + Random.seed!(83472834) + colnames = Symbol.(names(example_data("political_democracy"))) + # simulate data + model_ml_new = swap_observed( + model_ml, + data = rand(model_ml, params, 100_000), + specification = spec, + obs_colnames = colnames, + meanstructure = true + ) + model_ml_sym_new = swap_observed( + model_ml_sym, + data = rand(model_ml_sym, params, 100_000), + specification = spec, + obs_colnames = colnames, + meanstructure = true + ) + # fit models + sol_ml = solution(sem_fit(model_ml_new)) + sol_ml_sym = solution(sem_fit(model_ml_sym_new)) + # check solution + @test maximum(abs.(sol_ml - params)) < 0.01 + @test maximum(abs.(sol_ml_sym - params)) < 0.01 +end + ############################################################################################ ### fiml ############################################################################################ diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 5aa79842c..f00187fac 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -60,7 +60,7 @@ imply_ml.Σ_function(imply_ml.Σ, true_val) true_dist = MultivariateNormal(imply_ml.Σ) Random.seed!(1234) -x = transpose(rand(true_dist, 100000)) +x = transpose(rand(true_dist, 100_000)) semobserved = SemObservedData(data = x, specification = nothing) loss_ml = SemLoss(SemML(; observed = semobserved, nparams = length(start))) From 071005bb9d6b04301e9d1ff0d3fc0129ed38d4ec Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 21 Nov 2024 12:14:38 +0100 Subject: [PATCH 118/364] format --- src/additional_functions/simulation.jl | 21 ++++++++----------- .../political_democracy/constructor.jl | 15 ++++++------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 68ec62142..f1e41f360 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -99,7 +99,6 @@ function update_observed(loss::SemLoss, new_observed; kwargs...) return SemLoss(new_functions, loss.weights) end - ############################################################################################ # simulate data ############################################################################################ @@ -121,24 +120,22 @@ rand(model, start_simple(model), 100) ``` """ function Distributions.rand( - model::AbstractSemSingle{O, I, L, D}, - params, - n::Integer) where {O, I <: Union{RAM, RAMSymbolic}, L, D} - update!( - EvaluationTargets{true, false, false}(), - model.imply, - model, - params) + model::AbstractSemSingle{O, I, L, D}, + params, + n::Integer, +) where {O, I <: Union{RAM, RAMSymbolic}, L, D} + update!(EvaluationTargets{true, false, false}(), model.imply, model, params) return rand(model, n) end function Distributions.rand( - model::AbstractSemSingle{O, I, L, D}, - n::Integer) where {O, I <: Union{RAM, RAMSymbolic}, L, D} + model::AbstractSemSingle{O, I, L, D}, + n::Integer, +) where {O, I <: Union{RAM, RAMSymbolic}, L, D} if MeanStruct(model.imply) === NoMeanStruct data = permutedims(rand(MvNormal(Symmetric(model.imply.Σ)), n)) elseif MeanStruct(model.imply) === HasMeanStruct data = permutedims(rand(MvNormal(model.imply.μ, Symmetric(model.imply.Σ)), n)) end return data -end \ No newline at end of file +end diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 3fc99289a..6e16553f7 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -174,7 +174,8 @@ end start_regressions = 0.5, start_variances_observed = 0.5, start_variances_latent = 1.0, - start_covariances_observed = 0.2) + start_covariances_observed = 0.2, + ) # set seed for simulation Random.seed!(83472834) colnames = Symbol.(names(example_data("political_democracy"))) @@ -183,13 +184,13 @@ end model_ml, data = rand(model_ml, params, 100_000), specification = spec, - obs_colnames = colnames + obs_colnames = colnames, ) model_ml_sym_new = swap_observed( model_ml_sym, data = rand(model_ml_sym, params, 100_000), specification = spec, - obs_colnames = colnames + obs_colnames = colnames, ) # fit models sol_ml = solution(sem_fit(model_ml_new)) @@ -370,7 +371,6 @@ end ) end - ############################################################################################ ### data simulation ############################################################################################ @@ -384,7 +384,8 @@ end start_variances_observed = 0.5, start_variances_latent = 1.0, start_covariances_observed = 0.2, - start_means = 0.5) + start_means = 0.5, + ) # set seed for simulation Random.seed!(83472834) colnames = Symbol.(names(example_data("political_democracy"))) @@ -394,14 +395,14 @@ end data = rand(model_ml, params, 100_000), specification = spec, obs_colnames = colnames, - meanstructure = true + meanstructure = true, ) model_ml_sym_new = swap_observed( model_ml_sym, data = rand(model_ml_sym, params, 100_000), specification = spec, obs_colnames = colnames, - meanstructure = true + meanstructure = true, ) # fit models sol_ml = solution(sem_fit(model_ml_new)) From b7c111df9a6f075782ff222b1d30ed3556bee22c Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 21 Nov 2024 12:36:21 +0100 Subject: [PATCH 119/364] increase test sample size --- test/examples/political_democracy/constructor.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 6e16553f7..99ef06b3a 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -182,13 +182,13 @@ end # simulate data model_ml_new = swap_observed( model_ml, - data = rand(model_ml, params, 100_000), + data = rand(model_ml, params, 1_000_000), specification = spec, obs_colnames = colnames, ) model_ml_sym_new = swap_observed( model_ml_sym, - data = rand(model_ml_sym, params, 100_000), + data = rand(model_ml_sym, params, 1_000_000), specification = spec, obs_colnames = colnames, ) @@ -392,14 +392,14 @@ end # simulate data model_ml_new = swap_observed( model_ml, - data = rand(model_ml, params, 100_000), + data = rand(model_ml, params, 1_000_000), specification = spec, obs_colnames = colnames, meanstructure = true, ) model_ml_sym_new = swap_observed( model_ml_sym, - data = rand(model_ml_sym, params, 100_000), + data = rand(model_ml_sym, params, 1_000_000), specification = spec, obs_colnames = colnames, meanstructure = true, From 3bda87d61858cc890ac19ebfc8f37b83417142d7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 13 Sep 2024 17:55:58 -0700 Subject: [PATCH 120/364] Project.toml: update Symbolics deps --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 9edeb4536..b038c3364 100644 --- a/Project.toml +++ b/Project.toml @@ -36,8 +36,8 @@ NLopt = "0.6, 1" Optim = "1" PrettyTables = "2" StatsBase = "0.33, 0.34" -Symbolics = "4, 5" -SymbolicUtils = "1.4 - 1.5" +Symbolics = "4, 5, 6" +SymbolicUtils = "1.4 - 1.5, 1.7, 2, 3" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From 70a4e1f58f56651450fe57d3ed3ba8a75410e76d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 15 Mar 2024 08:36:18 -0700 Subject: [PATCH 121/364] tests/examples: import -> using no declarations, so import is not required --- test/examples/multigroup/build_models.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 4b5afd58e..3f29a6898 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -1,3 +1,5 @@ +const SEM = StructuralEquationModels + ############################################################################################ # ML estimation ############################################################################################ From 56a1b0426461a7ec4767f4a73ddfd17f32b575e4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Nov 2024 11:28:41 -0800 Subject: [PATCH 122/364] add ParamsArray replaces RAMMatrices indices and constants vectors with dedicated class that incapsulate this logic, resulting in overall cleaner interface A_ind, S_ind, M_ind become ParamsArray F_ind becomes SparseMatrixCSC parameters.jl is not longer required and is removed --- src/StructuralEquationModels.jl | 2 +- src/additional_functions/parameters.jl | 137 ------- src/additional_functions/params_array.jl | 204 ++++++++++ .../start_val/start_fabin3.jl | 152 ++++--- .../start_val/start_simple.jl | 20 +- src/frontend/specification/RAMMatrices.jl | 381 ++++++++---------- src/imply/RAM/generic.jl | 61 +-- src/imply/RAM/symbolic.jl | 21 +- 8 files changed, 486 insertions(+), 492 deletions(-) delete mode 100644 src/additional_functions/parameters.jl create mode 100644 src/additional_functions/params_array.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 944542379..6172af1ea 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -26,6 +26,7 @@ include("objective_gradient_hessian.jl") # helper objects and functions include("additional_functions/commutation_matrix.jl") +include("additional_functions/params_array.jl") # fitted objects include("frontend/fit/SemFit.jl") @@ -69,7 +70,6 @@ include("optimizer/optim.jl") include("optimizer/NLopt.jl") # helper functions include("additional_functions/helper.jl") -include("additional_functions/parameters.jl") include("additional_functions/start_val/start_val.jl") include("additional_functions/start_val/start_fabin3.jl") include("additional_functions/start_val/start_partable.jl") diff --git a/src/additional_functions/parameters.jl b/src/additional_functions/parameters.jl deleted file mode 100644 index d6e8eb535..000000000 --- a/src/additional_functions/parameters.jl +++ /dev/null @@ -1,137 +0,0 @@ -# fill A, S, and M matrices with the parameter values according to the parameters map -function fill_A_S_M!( - A::AbstractMatrix, - S::AbstractMatrix, - M::Union{AbstractVector, Nothing}, - A_indices::AbstractArrayParamsMap, - S_indices::AbstractArrayParamsMap, - M_indices::Union{AbstractArrayParamsMap, Nothing}, - params::AbstractVector, -) - @inbounds for (iA, iS, par) in zip(A_indices, S_indices, params) - for index_A in iA - A[index_A] = par - end - - for index_S in iS - S[index_S] = par - end - end - - if !isnothing(M) - @inbounds for (iM, par) in zip(M_indices, params) - for index_M in iM - M[index_M] = par - end - end - end -end - -# build the map from the index of the parameter to the linear indices -# of this parameter occurences in M -# returns ArrayParamsMap object -function array_params_map(params::AbstractVector, M::AbstractArray) - params_index = Dict(param => i for (i, param) in enumerate(params)) - T = Base.eltype(eachindex(M)) - res = [Vector{T}() for _ in eachindex(params)] - for (i, val) in enumerate(M) - par_ind = get(params_index, val, nothing) - if !isnothing(par_ind) - push!(res[par_ind], i) - end - end - return res -end - -function eachindex_lower(M; linear_indices = false, kwargs...) - indices = CartesianIndices(M) - indices = filter(x -> (x[1] >= x[2]), indices) - - if linear_indices - indices = cartesian2linear(indices, M) - end - - return indices -end - -function cartesian2linear(ind_cart, dims) - ind_lin = LinearIndices(dims)[ind_cart] - return ind_lin -end - -function linear2cartesian(ind_lin, dims) - ind_cart = CartesianIndices(dims)[ind_lin] - return ind_cart -end - -function set_constants!(M, M_pre) - for index in eachindex(M) - δ = tryparse(Float64, string(M[index])) - - if !iszero(M[index]) & (δ !== nothing) - M_pre[index] = δ - end - end -end - -function check_constants(M) - for index in eachindex(M) - δ = tryparse(Float64, string(M[index])) - - if !iszero(M[index]) & (δ !== nothing) - return true - end - end - - return false -end - -# construct length(M)×length(parameters) sparse matrix of 1s at the positions, -# where the corresponding parameter occurs in the M matrix -function matrix_gradient(M_indices::ArrayParamsMap, M_length::Integer) - rowval = reduce(vcat, M_indices) - colptr = - pushfirst!(accumulate((ptr, M_ind) -> ptr + length(M_ind), M_indices, init = 1), 1) - return SparseMatrixCSC( - M_length, - length(M_indices), - colptr, - rowval, - ones(length(rowval)), - ) -end - -# fill M with parameters -function fill_matrix!( - M::AbstractMatrix, - M_indices::AbstractArrayParamsMap, - params::AbstractVector, -) - for (iM, par) in zip(M_indices, params) - for index_M in iM - M[index_M] = par - end - end - return M -end - -# range of parameters that are referenced in the matrix -function param_range(mtx_indices::AbstractArrayParamsMap) - first_i = findfirst(!isempty, mtx_indices) - last_i = findlast(!isempty, mtx_indices) - - if !isnothing(first_i) && !isnothing(last_i) - for i in first_i:last_i - if isempty(mtx_indices[i]) - # TODO show which parameter is missing in which matrix - throw( - ErrorException( - "Your parameter vector is not partitioned into directed and undirected effects", - ), - ) - end - end - end - - return first_i:last_i -end diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl new file mode 100644 index 000000000..f20a6518b --- /dev/null +++ b/src/additional_functions/params_array.jl @@ -0,0 +1,204 @@ +""" +Array with partially parameterized elements. +""" +struct ParamsArray{T, N} <: AbstractArray{T, N} + linear_indices::Vector{Int} + param_ptr::Vector{Int} + constants::Vector{Pair{Int, T}} + size::NTuple{N, Int} +end + +ParamsVector{T} = ParamsArray{T, 1} +ParamsMatrix{T} = ParamsArray{T, 2} + +function ParamsArray{T, N}( + params_map::AbstractVector{<:AbstractVector{Int}}, + constants::Vector{Pair{Int, T}}, + size::NTuple{N, Int}, +) where {T, N} + params_ptr = + pushfirst!(accumulate((ptr, inds) -> ptr + length(inds), params_map, init = 1), 1) + return ParamsArray{T, N}( + reduce(vcat, params_map, init = Vector{Int}()), + params_ptr, + constants, + size, + ) +end + +function ParamsArray{T, N}( + arr::AbstractArray{<:Any, N}, + params::AbstractVector{Symbol}; + skip_zeros::Bool = true, +) where {T, N} + params_index = Dict(param => i for (i, param) in enumerate(params)) + constants = Vector{Pair{Int, T}}() + params_map = [Vector{Int}() for _ in eachindex(params)] + arr_ixs = CartesianIndices(arr) + for (i, val) in pairs(vec(arr)) + ismissing(val) && continue + if isa(val, Number) + (skip_zeros && iszero(val)) || push!(constants, i => val) + else + par_ind = get(params_index, val, nothing) + if !isnothing(par_ind) + push!(params_map[par_ind], i) + else + throw(KeyError("Unrecognized parameter $val at position $(arr_ixs[i])")) + end + end + end + return ParamsArray{T, N}(params_map, constants, size(arr)) +end + +ParamsArray{T}( + arr::AbstractArray{<:Any, N}, + params::AbstractVector{Symbol}; + kwargs..., +) where {T, N} = ParamsArray{T, N}(arr, params; kwargs...) + +nparams(arr::ParamsArray) = length(arr.param_ptr) - 1 + +Base.size(arr::ParamsArray) = arr.size +Base.size(arr::ParamsArray, i::Integer) = arr.size[i] + +Base.:(==)(a::ParamsArray, b::ParamsArray) = return eltype(a) == eltype(b) && + size(a) == size(b) && + a.constants == b.constants && + a.param_ptr == b.param_ptr && + a.linear_indices == b.linear_indices + +# the range of arr.param_ptr indices that correspond to i-th parameter +param_occurences_range(arr::ParamsArray, i::Integer) = + arr.param_ptr[i]:(arr.param_ptr[i+1]-1) + +""" + param_occurences(arr::ParamsArray, i::Integer) + +Get the linear indices of the elements in `arr` that correspond to the +`i`-th parameter. +""" +param_occurences(arr::ParamsArray, i::Integer) = + view(arr.linear_indices, arr.param_ptr[i]:(arr.param_ptr[i+1]-1)) + +""" + materialize!(dest::AbstractArray{<:Any, N}, src::ParamsArray{<:Any, N}, + param_values::AbstractVector; + set_constants::Bool = true, + set_zeros::Bool = false) + +Materialize the parameterized array `src` into `dest` by substituting the parameter +references with the parameter values from `param_values`. +""" +function materialize!( + dest::AbstractArray{<:Any, N}, + src::ParamsArray{<:Any, N}, + param_values::AbstractVector; + set_constants::Bool = true, + set_zeros::Bool = false, +) where {N} + size(dest) == size(src) || throw( + DimensionMismatch( + "Parameters ($(size(params_arr))) and destination ($(size(dest))) array sizes don't match", + ), + ) + nparams(src) == length(param_values) || throw( + DimensionMismatch( + "Number of values ($(length(param_values))) does not match the number of parameters ($(nparams(src)))", + ), + ) + Z = eltype(dest) <: Number ? eltype(dest) : eltype(src) + set_zeros && fill!(dest, zero(Z)) + if set_constants + @inbounds for (i, val) in src.constants + dest[i] = val + end + end + @inbounds for (i, val) in enumerate(param_values) + for j in param_occurences_range(src, i) + dest[src.linear_indices[j]] = val + end + end + return dest +end + +""" + materialize([T], src::ParamsArray{<:Any, N}, + param_values::AbstractVector{T}) where T + +Materialize the parameterized array `src` into a new array of type `T` +by substituting the parameter references with the parameter values from `param_values`. +""" +materialize(::Type{T}, arr::ParamsArray, param_values::AbstractVector) where {T} = + materialize!(similar(arr, T), arr, param_values, set_constants = true, set_zeros = true) + +materialize(arr::ParamsArray, param_values::AbstractVector{T}) where {T} = + materialize(Union{T, eltype(arr)}, arr, param_values) + +function sparse_materialize( + ::Type{T}, + arr::ParamsMatrix, + param_values::AbstractVector, +) where {T} + nparams(arr) == length(param_values) || throw( + DimensionMismatch( + "Number of values ($(length(param)values))) does not match the number of parameter ($(nparams(arr)))", + ), + ) + # constant values in sparse matrix + cvals = [T(v) for (_, v) in arr.constants] + # parameter values in sparse matrix + parvals = Vector{T}(undef, length(arr.linear_indices)) + @inbounds for (i, val) in enumerate(param_values) + for j in param_occurences_range(arr, i) + parvals[j] = val + end + end + nzixs = [first.(arr.constants); arr.linear_indices] + ixorder = sortperm(nzixs) + nzixs = nzixs[ixorder] + nzvals = [cvals; parvals][ixorder] + arr_ixs = CartesianIndices(size(arr)) + return sparse( + [arr_ixs[i][1] for i in nzixs], + [arr_ixs[i][2] for i in nzixs], + nzvals, + size(arr)..., + ) +end + +sparse_materialize(arr::ParamsArray, params::AbstractVector{T}) where {T} = + sparse_materialize(Union{T, eltype(arr)}, arr, params) + +# construct length(M)×length(params) sparse matrix of 1s at the positions, +# where the corresponding parameter occurs in the arr +sparse_gradient(::Type{T}, arr::ParamsArray) where {T} = SparseMatrixCSC( + length(arr), + nparams(arr), + arr.param_ptr, + arr.linear_indices, + ones(T, length(arr.linear_indices)), +) + +sparse_gradient(arr::ParamsArray{T}) where {T} = sparse_gradient(T, arr) + +# range of parameters that are referenced in the matrix +function params_range(arr::ParamsArray; allow_gaps::Bool = false) + first_i = findfirst(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:nparams(arr)-1) + last_i = findlast(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:nparams(arr)-1) + + if !allow_gaps && !isnothing(first_i) && !isnothing(last_i) + for i in first_i:last_i + if isempty(param_occurences_range(arr, i)) + # TODO show which parameter is missing in which matrix + throw( + ErrorException( + "Parameter vector is not partitioned into directed and undirected effects", + ), + ) + end + end + end + + return first_i:last_i +end diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 081af3ba1..9d692437e 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -31,23 +31,20 @@ function start_fabin3(observed::SemObservedMissing, imply, optimizer, args...; k end function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) - A_ind, S_ind, F_ind, M_ind, n_par = ram_matrices.A_ind, - ram_matrices.S_ind, - ram_matrices.F_ind, - ram_matrices.M_ind, + A, S, F, M, n_par = ram_matrices.A, + ram_matrices.S, + ram_matrices.F, + ram_matrices.M, nparams(ram_matrices) start_val = zeros(n_par) - n_obs = nobserved_vars(ram_matrices) - n_var = nvars(ram_matrices) - n_latent = nlatent_vars(ram_matrices) - - C_indices = CartesianIndices((n_var, n_var)) + F_var2obs = Dict( + i => F.rowval[F.colptr[i]] for i in axes(F, 2) if isobserved_var(ram_matrices, i) + ) + @assert length(F_var2obs) == size(F, 1) # check in which matrix each parameter appears - indices = Vector{CartesianIndex{2}}(undef, n_par) - #= in_S = length.(S_ind) .!= 0 in_A = length.(A_ind) .!= 0 A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind] @@ -65,26 +62,53 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) end =# # set undirected parameters in S - for (i, S_ind) in enumerate(S_ind) - for c_ind in C_indices[S_ind] - (c_ind[1] == c_ind[2]) || continue # covariances stay 0 - pos = searchsortedfirst(F_ind, c_ind[1]) - start_val[i] = - (pos <= length(F_ind)) && (F_ind[pos] == c_ind[1]) ? Σ[pos, pos] / 2 : 0.05 - break # i-th parameter initialized + S_indices = CartesianIndices(S) + for j in 1:nparams(S) + for lin_ind in param_occurences(S, j) + to, from = Tuple(S_indices[lin_ind]) + if (to == from) # covariances start with 0 + # half of observed variance for observed, 0.05 for latent + obs = get(F_var2obs, to, nothing) + start_val[j] = !isnothing(obs) ? Σ[obs, obs] / 2 : 0.05 + break # j-th parameter initialized + end end end # set loadings - constants = ram_matrices.constants - A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind] + A_indices = CartesianIndices(A) # ind_Λ = findall([is_in_Λ(ind_vec, F_ind) for ind_vec in A_ind_c]) - function calculate_lambda( - ref::Integer, - indicator::Integer, - indicators::AbstractVector{<:Integer}, - ) + # collect latent variable indicators in A + # maps latent parameter to the vector of dependent vars + # the 2nd index in the pair specified the parameter index, + # 0 if no parameter (constant), -1 if constant=1 + var2indicators = Dict{Int, Vector{Pair{Int, Int}}}() + for j in 1:nparams(A) + for lin_ind in param_occurences(A, j) + to, from = Tuple(A_indices[lin_ind]) + haskey(F_var2obs, from) && continue # skip observed + obs = get(F_var2obs, to, nothing) + if !isnothing(obs) + indicators = get!(() -> Vector{Pair{Int, Int}}(), var2indicators, from) + push!(indicators, obs => j) + end + end + end + + for (lin_ind, val) in A.constants + iszero(val) && continue # only non-zero loadings + to, from = Tuple(A_indices[lin_ind]) + haskey(F_var2obs, from) && continue # skip observed + obs = get(F_var2obs, to, nothing) + if !isnothing(obs) + indicators = get!(() -> Vector{Pair{Int, Int}}(), var2indicators, from) + push!(indicators, obs => ifelse(isone(val), -1, 0)) # no parameter associated, -1 = reference, 0 = indicator + end + end + + # calculate starting values for parameters of latent regression vars + function calculate_lambda(ref::Integer, indicator::Integer, indicators::AbstractVector) instruments = filter(i -> (i != ref) && (i != indicator), indicators) if length(instruments) == 1 s13 = Σ[ref, instruments[1]] @@ -99,61 +123,33 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) end end - for i in setdiff(1:n_var, F_ind) - reference = Int64[] - indicators = Int64[] - indicator2parampos = Dict{Int, Int}() - - for (j, Aj_ind_c) in enumerate(A_ind_c) - for ind_c in Aj_ind_c - (ind_c[2] == i) || continue - ind_pos = searchsortedfirst(F_ind, ind_c[1]) - if (ind_pos <= length(F_ind)) && (F_ind[ind_pos] == ind_c[1]) - push!(indicators, ind_pos) - indicator2parampos[ind_pos] = j - end - end - end - - for ram_const in constants - if (ram_const.matrix == :A) && (ram_const.index[2] == i) - ind_pos = searchsortedfirst(F_ind, ram_const.index[1]) - if (ind_pos <= length(F_ind)) && (F_ind[ind_pos] == ram_const.index[1]) - if isone(ram_const.value) - push!(reference, ind_pos) - else - push!(indicators, ind_pos) - # no parameter associated - end - end - end - end - + for (i, indicators) in pairs(var2indicators) + reference = [obs for (obs, param) in indicators if param == -1] + indicator_obs = first.(indicators) # is there at least one reference indicator? if length(reference) > 0 - if (length(reference) > 1) && isempty(indicator2parampos) # don't warn if entire column is fixed + if (length(reference) > 1) && any(((obs, param),) -> param > 0, indicators) # don't warn if entire column is fixed @warn "You have more than 1 scaling indicator for $(ram_matrices.colnames[i])" end ref = reference[1] - for (j, indicator) in enumerate(indicators) - if (indicator != ref) && - (parampos = get(indicator2parampos, indicator, 0)) != 0 - start_val[parampos] = calculate_lambda(ref, indicator, indicators) + for (indicator, param) in indicators + if (indicator != ref) && (param > 0) + start_val[param] = calculate_lambda(ref, indicator, indicator_obs) end end # no reference indicator: - elseif length(indicators) > 0 - ref = indicators[1] - λ = Vector{Float64}(undef, length(indicators)) + else + ref = indicator_obs[1] + λ = Vector{Float64}(undef, length(indicator_obs)) λ[1] = 1.0 - for (j, indicator) in enumerate(indicators) + for (j, indicator) in enumerate(indicator_obs) if indicator != ref - λ[j] = calculate_lambda(ref, indicator, indicators) + λ[j] = calculate_lambda(ref, indicator, indicator_obs) end end - Σ_λ = Σ[indicators, indicators] + Σ_λ = Σ[indicator_obs, indicator_obs] l₂ = sum(abs2, λ) D = λ * λ' ./ l₂ θ = (I - D .^ 2) \ (diag(Σ_λ - D * Σ_λ * D)) @@ -164,24 +160,22 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) λ .*= sign(Ψ) * sqrt(abs(Ψ)) - for (j, indicator) in enumerate(indicators) - if (parampos = get(indicator2parampos, indicator, 0)) != 0 - start_val[parampos] = λ[j] + for (j, (_, param)) in enumerate(indicators) + if param > 0 + start_val[param] = λ[j] end end - else - @warn "No scaling indicators for $(ram_matrices.colnames[i])" end end - # set means - if !isnothing(M_ind) - for (i, M_ind) in enumerate(M_ind) - if length(M_ind) != 0 - ind = M_ind[1] - pos = searchsortedfirst(F_ind, ind[1]) - if (pos <= length(F_ind)) && (F_ind[pos] == ind[1]) - start_val[i] = μ[pos] + if !isnothing(M) + # set starting values of the observed means + for j in 1:nparams(M) + M_ind = param_occurences(M, j) + if !isempty(M_ind) + obs = get(F_var2obs, M_ind[1], nothing) + if !isnothing(obs) + start_val[j] = μ[obs] end # latent means stay 0 end end diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 3b29ec178..1f73a3583 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -62,10 +62,10 @@ function start_simple( start_means = 0.0, kwargs..., ) - A_ind, S_ind, F_ind, M_ind, n_par = ram_matrices.A_ind, - ram_matrices.S_ind, - ram_matrices.F_ind, - ram_matrices.M_ind, + A, S, F_ind, M, n_par = ram_matrices.A, + ram_matrices.S, + observed_var_indices(ram_matrices), + ram_matrices.M, nparams(ram_matrices) start_val = zeros(n_par) @@ -75,9 +75,11 @@ function start_simple( C_indices = CartesianIndices((n_var, n_var)) for i in 1:n_par - if length(S_ind[i]) != 0 + Si_ind = param_occurences(S, i) + Ai_ind = param_occurences(A, i) + if length(Si_ind) != 0 # use the first occurence of the parameter to determine starting value - c_ind = C_indices[S_ind[i][1]] + c_ind = C_indices[Si_ind[1]] if c_ind[1] == c_ind[2] if c_ind[1] ∈ F_ind start_val[i] = start_variances_observed @@ -95,14 +97,14 @@ function start_simple( start_val[i] = start_covariances_obs_lat end end - elseif length(A_ind[i]) != 0 - c_ind = C_indices[A_ind[i][1]] + elseif length(Ai_ind) != 0 + c_ind = C_indices[Ai_ind[1]] if (c_ind[1] ∈ F_ind) & !(c_ind[2] ∈ F_ind) start_val[i] = start_loadings else start_val[i] = start_regressions end - elseif !isnothing(M_ind) && (length(M_ind[i]) != 0) + elseif !isnothing(M) && (length(param_occurences(M, i)) != 0) start_val[i] = start_means end end diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 6ba6be3d0..b140ae026 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -1,82 +1,56 @@ -############################################################################################ -### Constants -############################################################################################ - -struct RAMConstant - matrix::Symbol - index::Union{Int, CartesianIndex{2}} - value::Any -end - -function Base.:(==)(c1::RAMConstant, c2::RAMConstant) - res = ((c1.matrix == c2.matrix) && (c1.index == c2.index) && (c1.value == c2.value)) - return res -end - -function append_RAMConstants!( - constants::AbstractVector{RAMConstant}, - mtx_name::Symbol, - mtx::AbstractArray; - skip_zeros::Bool = true, -) - for (index, val) in pairs(mtx) - if isa(val, Number) && !(skip_zeros && iszero(val)) - push!(constants, RAMConstant(mtx_name, index, val)) - end - end - return constants -end - -function set_RAMConstant!(A, S, M, rc::RAMConstant) - if rc.matrix == :A - A[rc.index] = rc.value - elseif rc.matrix == :S - S[rc.index] = rc.value - S[rc.index[2], rc.index[1]] = rc.value # symmetric - elseif rc.matrix == :M - M[rc.index] = rc.value - end -end - -function set_RAMConstants!(A, S, M, rc_vec::Vector{RAMConstant}) - for rc in rc_vec - set_RAMConstant!(A, S, M, rc) - end -end ############################################################################################ ### Type ############################################################################################ -# map from parameter index to linear indices of matrix/vector positions where it occurs -AbstractArrayParamsMap = AbstractVector{<:AbstractVector{<:Integer}} -ArrayParamsMap = Vector{Vector{Int}} - struct RAMMatrices <: SemSpecification - A_ind::ArrayParamsMap - S_ind::ArrayParamsMap - F_ind::Vector{Int} - M_ind::Union{ArrayParamsMap, Nothing} + A::ParamsMatrix{Float64} + S::ParamsMatrix{Float64} + F::SparseMatrixCSC{Float64} + M::Union{ParamsVector{Float64}, Nothing} params::Vector{Symbol} - colnames::Union{Vector{Symbol}, Nothing} - constants::Vector{RAMConstant} - size_F::Tuple{Int, Int} + colnames::Union{Vector{Symbol}, Nothing} # better call it "variables": it's a mixture of observed and latent (and it gets confusing with get_colnames()) end -nparams(ram::RAMMatrices) = length(ram.A_ind) - -nvars(ram::RAMMatrices) = ram.size_F[2] -nobserved_vars(ram::RAMMatrices) = ram.size_F[1] +nparams(ram::RAMMatrices) = nparams(ram.A) +nvars(ram::RAMMatrices) = size(ram.F, 2) +nobserved_vars(ram::RAMMatrices) = size(ram.F, 1) nlatent_vars(ram::RAMMatrices) = nvars(ram) - nobserved_vars(ram) vars(ram::RAMMatrices) = ram.colnames +isobserved_var(ram::RAMMatrices, i::Integer) = ram.F.colptr[i+1] > ram.F.colptr[i] +islatent_var(ram::RAMMatrices, i::Integer) = ram.F.colptr[i+1] == ram.F.colptr[i] + +# indices of observed variables in the order as they appear in ram.F rows +function observed_var_indices(ram::RAMMatrices) + obs_inds = Vector{Int}(undef, nobserved_vars(ram)) + @inbounds for i in 1:nvars(ram) + colptr = ram.F.colptr[i] + if ram.F.colptr[i+1] > colptr # is observed + obs_inds[ram.F.rowval[colptr]] = i + end + end + return obs_inds +end + +latent_var_indices(ram::RAMMatrices) = + [i for i in axes(ram.F, 2) if islatent_var(ram, i)] + +# observed variables in the order as they appear in ram.F rows function observed_vars(ram::RAMMatrices) if isnothing(ram.colnames) @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" return nothing else - return view(ram.colnames, ram.F_ind) + obs_vars = Vector{Symbol}(undef, nobserved_vars(ram)) + @inbounds for (i, v) in enumerate(vars(ram)) + colptr = ram.F.colptr[i] + if ram.F.colptr[i+1] > colptr # is observed + obs_vars[ram.F.rowval[colptr]] = v + end + end + return obs_vars end end @@ -85,7 +59,7 @@ function latent_vars(ram::RAMMatrices) @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" return nothing else - return view(ram.colnames, setdiff(eachindex(ram.colnames), ram.F_ind)) + return [col for (i, col) in enumerate(ram.colnames) if islatent_var(ram, i)] end end @@ -128,27 +102,16 @@ function RAMMatrices(; ), ) end - check_params(params, nothing) - A_indices = array_params_map(params, A) - S_indices = array_params_map(params, S) - M_indices = !isnothing(M) ? array_params_map(params, M) : nothing - F_indices = [i for (i, col) in zip(axes(F, 2), eachcol(F)) if any(isone, col)] - constants = Vector{RAMConstant}() - append_RAMConstants!(constants, :A, A) - append_RAMConstants!(constants, :S, S) - isnothing(M) || append_RAMConstants!(constants, :M, M) - return RAMMatrices( - A_indices, - S_indices, - F_indices, - M_indices, - params, - colnames, - constants, - size(F), - ) + A = ParamsMatrix{Float64}(A, params) + S = ParamsMatrix{Float64}(S, params) + M = !isnothing(M) ? ParamsVector{Float64}(M, params) : nothing + spF = sparse(F) + if any(!isone, spF.nzval) + throw(ArgumentError("F should contain only 0s and 1s")) + end + return RAMMatrices(A, S, F, M, params, colnames) end ############################################################################################ @@ -165,83 +128,102 @@ function RAMMatrices( n_observed = length(partable.observed_vars) n_latent = length(partable.latent_vars) - n_node = n_observed + n_latent - - # F indices - F_ind = - length(partable.sorted_vars) != 0 ? - findall(∈(Set(partable.observed_vars)), partable.sorted_vars) : 1:n_observed - - # indices of the colnames - colnames = - length(partable.sorted_vars) != 0 ? copy(partable.sorted_vars) : - [ - partable.observed_vars - partable.latent_vars - ] - col_indices = Dict(col => i for (i, col) in enumerate(colnames)) + n_vars = n_observed + n_latent + + if length(partable.sorted_vars) != 0 + @assert length(partable.sorted_vars) == nvars(partable) + vars_sorted = copy(partable.sorted_vars) + else + vars_sorted = [partable.observed_vars + partable.latent_vars] + end + + # indices of the vars (A/S/M rows or columns) + vars_index = Dict(col => i for (i, col) in enumerate(vars_sorted)) # fill Matrices # known_labels = Dict{Symbol, Int64}() - A_ind = [Vector{Int64}() for _ in 1:length(params)] - S_ind = [Vector{Int64}() for _ in 1:length(params)] - + T = nonmissingtype(eltype(partable.columns[:value_fixed])) + A_inds = [Vector{Int64}() for _ in 1:length(params)] + A_lin_ixs = LinearIndices((n_vars, n_vars)) + S_inds = [Vector{Int64}() for _ in 1:length(params)] + S_lin_ixs = LinearIndices((n_vars, n_vars)) + A_consts = Vector{Pair{Int, T}}() + S_consts = Vector{Pair{Int, T}}() # is there a meanstructure? - M_ind = + M_inds = any(==(Symbol("1")), partable.columns[:from]) ? [Vector{Int64}() for _ in 1:length(params)] : nothing - - # handle constants - constants = Vector{RAMConstant}() + M_consts = !isnothing(M_inds) ? Vector{Pair{Int, T}}() : nothing for r in partable - row_ind = col_indices[r.to] - col_ind = r.from != Symbol("1") ? col_indices[r.from] : nothing + row_ind = vars_index[r.to] + col_ind = r.from != Symbol("1") ? vars_index[r.from] : nothing if !r.free if (r.relation == :→) && (r.from == Symbol("1")) - push!(constants, RAMConstant(:M, row_ind, r.value_fixed)) + push!(M_consts, row_ind => r.value_fixed) elseif r.relation == :→ push!( - constants, - RAMConstant(:A, CartesianIndex(row_ind, col_ind), r.value_fixed), + A_consts, + A_lin_ixs[CartesianIndex(row_ind, col_ind)] => r.value_fixed, ) elseif r.relation == :↔ push!( - constants, - RAMConstant(:S, CartesianIndex(row_ind, col_ind), r.value_fixed), + S_consts, + S_lin_ixs[CartesianIndex(row_ind, col_ind)] => r.value_fixed, ) + if row_ind != col_ind # symmetric + push!( + S_consts, + S_lin_ixs[CartesianIndex(col_ind, row_ind)] => r.value_fixed, + ) + end else - error("Unsupported parameter type: $(r.relation)") + error("Unsupported relation: $(r.relation)") end else par_ind = params_index[r.param] if (r.relation == :→) && (r.from == Symbol("1")) - push!(M_ind[par_ind], row_ind) + push!(M_inds[par_ind], row_ind) elseif r.relation == :→ - push!(A_ind[par_ind], row_ind + (col_ind - 1) * n_node) + push!(A_inds[par_ind], A_lin_ixs[CartesianIndex(row_ind, col_ind)]) elseif r.relation == :↔ - push!(S_ind[par_ind], row_ind + (col_ind - 1) * n_node) - if row_ind != col_ind - push!(S_ind[par_ind], col_ind + (row_ind - 1) * n_node) + push!(S_inds[par_ind], S_lin_ixs[CartesianIndex(row_ind, col_ind)]) + if row_ind != col_ind # symmetric + push!(S_inds[par_ind], S_lin_ixs[CartesianIndex(col_ind, row_ind)]) end else - error("Unsupported parameter type: $(r.relation)") + error("Unsupported relation: $(r.relation)") end end end + # sort linear indices + for A_ind in A_inds + sort!(A_ind) + end + for S_ind in S_inds + unique!(sort!(S_ind)) # also symmetric duplicates + end + if !isnothing(M_inds) + for M_ind in M_inds + sort!(M_ind) + end + end + sort!(A_consts, by = first) + sort!(S_consts, by = first) + if !isnothing(M_consts) + sort!(M_consts, by = first) + end - return RAMMatrices( - A_ind, - S_ind, - F_ind, - M_ind, - params, - colnames, - constants, - (n_observed, n_node), - ) + return RAMMatrices(ParamsMatrix{T}(A_inds, A_consts, (n_vars, n_vars)), + ParamsMatrix{T}(S_inds, S_consts, (n_vars, n_vars)), + sparse(1:n_observed, + [vars_index[var] for var in partable.observed_vars], + ones(T, n_observed), n_observed, n_vars), + !isnothing(M_inds) ? ParamsVector{T}(M_inds, M_consts, (n_vars,)) : nothing, + params, vars_sorted) end Base.convert( @@ -255,21 +237,20 @@ Base.convert( ############################################################################################ function ParameterTable( - ram_matrices::RAMMatrices; + ram::RAMMatrices; params::Union{AbstractVector{Symbol}, Nothing} = nothing, observed_var_prefix::Symbol = :obs, latent_var_prefix::Symbol = :var, ) # defer parameter checks until we know which ones are used - if !isnothing(ram_matrices.colnames) - colnames = ram_matrices.colnames - observed_vars = colnames[ram_matrices.F_ind] - latent_vars = colnames[setdiff(eachindex(colnames), ram_matrices.F_ind)] + + if !isnothing(ram.colnames) + latent_vars = SEM.latent_vars(ram) + observed_vars = SEM.observed_vars(ram) + colnames = ram.colnames else - observed_vars = - [Symbol("$(observed_var_prefix)_$i") for i in 1:nobserved_vars(ram_matrices)] - latent_vars = - [Symbol("$(latent_var_prefix)_$i") for i in 1:nlatent_vars(ram_matrices)] + observed_vars = [Symbol("$(observed_var_prefix)_$i") for i in 1:nobserved_vars(ram)] + latent_vars = [Symbol("$(latent_var_prefix)_$i") for i in 1:nlatent_vars(ram)] colnames = vcat(observed_vars, latent_vars) end @@ -277,27 +258,16 @@ function ParameterTable( partable = ParameterTable( observed_vars = observed_vars, latent_vars = latent_vars, - params = isnothing(params) ? SEM.params(ram_matrices) : params, + params = isnothing(params) ? SEM.params(ram) : params, ) - # constants - for c in ram_matrices.constants - push!(partable, partable_row(c, colnames)) + # fill the table + append_rows!(partable, ram.S, :S, ram.params, colnames, skip_symmetric = true) + append_rows!(partable, ram.A, :A, ram.params, colnames) + if !isnothing(ram.M) + append_rows!(partable, ram.M, :M, ram.params, colnames) end - # parameters - for (i, par) in enumerate(ram_matrices.params) - append_partable_rows!( - partable, - colnames, - par, - i, - ram_matrices.A_ind, - ram_matrices.S_ind, - ram_matrices.M_ind, - ram_matrices.size_F[2], - ) - end check_params(SEM.params(partable), partable.columns[:param]) return partable @@ -339,23 +309,13 @@ function matrix_to_relation(matrix::Symbol) end end -partable_row(c::RAMConstant, varnames::AbstractVector{Symbol}) = ( - from = varnames[c.index[2]], - relation = matrix_to_relation(c.matrix), - to = varnames[c.index[1]], - free = false, - value_fixed = c.value, - start = 0.0, - estimate = 0.0, - param = :const, -) - +# generates a ParTable row NamedTuple for a given element of RAM matrix function partable_row( - par::Symbol, - varnames::AbstractVector{Symbol}, - index::Integer, + val, + index, matrix::Symbol, - n_nod::Integer, + varnames::AbstractVector{Symbol}; + free::Bool = true, ) # variable names @@ -363,58 +323,65 @@ function partable_row( from = Symbol("1") to = varnames[index] else - cart_index = linear2cartesian(index, (n_nod, n_nod)) - - from = varnames[cart_index[2]] - to = varnames[cart_index[1]] + from = varnames[index[2]] + to = varnames[index[1]] end return ( from = from, relation = matrix_to_relation(matrix), to = to, - free = true, - value_fixed = 0.0, + free = free, + value_fixed = free ? 0.0 : val, start = 0.0, estimate = 0.0, - param = par, + param = free ? val : :const, ) end -function append_partable_rows!( +function append_rows!( partable::ParameterTable, - varnames::AbstractVector{Symbol}, - par::Symbol, - par_index::Integer, - A_ind, - S_ind, - M_ind, - n_nod::Integer, + arr::ParamsArray, + arr_name::Symbol, + params::AbstractVector, + varnames::AbstractVector{Symbol}; + skip_symmetric::Bool = false, ) - for ind in A_ind[par_index] - push!(partable, partable_row(par, varnames, ind, :A, n_nod)) - end + nparams(arr) == length(params) || throw( + ArgumentError( + "Length of parameters vector ($(length(params))) does not match the number of parameters in the matrix ($(nparams(arr)))", + ), + ) + arr_ixs = eachindex(arr) + + # add parameters + visited_indices = Set{eltype(arr_ixs)}() + for (i, par) in enumerate(params) + for j in param_occurences_range(arr, i) + arr_ix = arr_ixs[arr.linear_indices[j]] + skip_symmetric && (arr_ix ∈ visited_indices) && continue - visited_S_indices = Set{Int}() - for ind in S_ind[par_index] - if ind ∉ visited_S_indices - push!(partable, partable_row(par, varnames, ind, :S, n_nod)) - # mark index and its symmetric as visited - push!(visited_S_indices, ind) - cart_index = linear2cartesian(ind, (n_nod, n_nod)) push!( - visited_S_indices, - cartesian2linear( - CartesianIndex(cart_index[2], cart_index[1]), - (n_nod, n_nod), - ), + partable, + partable_row(par, arr_ix, arr_name, varnames, free = true), ) + if skip_symmetric + # mark index and its symmetric as visited + push!(visited_indices, arr_ix) + push!(visited_indices, CartesianIndex(arr_ix[2], arr_ix[1])) + end end end - if !isnothing(M_ind) - for ind in M_ind[par_index] - push!(partable, partable_row(par, varnames, ind, :M, n_nod)) + # add constants + for (i, val) in arr.constants + arr_ix = arr_ixs[i] + skip_symmetric && (arr_ix ∈ visited_indices) && continue + push!(partable, partable_row(val, arr_ix, arr_name, varnames, free = false)) + if skip_symmetric + # mark index and its symmetric as visited + push!(visited_indices, arr_ix) + push!(visited_indices, CartesianIndex(arr_ix[2], arr_ix[1])) end end @@ -423,14 +390,12 @@ end function Base.:(==)(mat1::RAMMatrices, mat2::RAMMatrices) res = ( - (mat1.A_ind == mat2.A_ind) && - (mat1.S_ind == mat2.S_ind) && - (mat1.F_ind == mat2.F_ind) && - (mat1.M_ind == mat2.M_ind) && + (mat1.A == mat2.A) && + (mat1.S == mat2.S) && + (mat1.F == mat2.F) && + (mat1.M == mat2.M) && (mat1.params == mat2.params) && - (mat1.colnames == mat2.colnames) && - (mat1.size_F == mat2.size_F) && - (mat1.constants == mat2.constants) + (mat1.colnames == mat2.colnames) ) return res end diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index e7e0b36f5..a16aac179 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -74,9 +74,6 @@ mutable struct RAM{ A5, A6, V2, - I1, - I2, - I3, M1, M2, M3, @@ -97,10 +94,6 @@ mutable struct RAM{ ram_matrices::V2 - A_indices::I1 - S_indices::I2 - M_indices::I3 - F⨉I_A⁻¹::M1 F⨉I_A⁻¹S::M2 I_A::M3 @@ -131,22 +124,14 @@ function RAM(; n_par = nparams(ram_matrices) n_obs = nobserved_vars(ram_matrices) n_var = nvars(ram_matrices) - F = zeros(ram_matrices.size_F) - F[CartesianIndex.(1:n_obs, ram_matrices.F_ind)] .= 1.0 - - # get indices - A_indices = copy(ram_matrices.A_ind) - S_indices = copy(ram_matrices.S_ind) - M_indices = !isnothing(ram_matrices.M_ind) ? copy(ram_matrices.M_ind) : nothing #preallocate arrays - A_pre = zeros(n_var, n_var) - S_pre = zeros(n_var, n_var) - M_pre = !isnothing(M_indices) ? zeros(n_var) : nothing - - set_RAMConstants!(A_pre, S_pre, M_pre, ram_matrices.constants) + nan_params = fill(NaN, n_par) + A_pre = materialize(ram_matrices.A, nan_params) + S_pre = materialize(ram_matrices.S, nan_params) + F = Matrix(ram_matrices.F) - A_pre = check_acyclic(A_pre, n_par, A_indices) + A_pre = check_acyclic(A_pre, ram_matrices.A) # pre-allocate some matrices Σ = zeros(n_obs, n_obs) @@ -155,8 +140,8 @@ function RAM(; I_A = similar(A_pre) if gradient_required - ∇A = matrix_gradient(A_indices, n_var^2) - ∇S = matrix_gradient(S_indices, n_var^2) + ∇A = sparse_gradient(ram_matrices.A) + ∇S = sparse_gradient(ram_matrices.S) else ∇A = nothing ∇S = nothing @@ -165,16 +150,16 @@ function RAM(; # μ if meanstructure MS = HasMeanStruct - !isnothing(M_indices) || throw( + !isnothing(ram_matrices.M) || throw( ArgumentError( "You set `meanstructure = true`, but your model specification contains no mean parameters.", ), ) - ∇M = gradient_required ? matrix_gradient(M_indices, n_var) : nothing + M_pre = materialize(ram_matrices.M, nan_params) + ∇M = gradient_required ? sparse_gradient(ram_matrices.M) : nothing μ = zeros(n_obs) else MS = NoMeanStruct - M_indices = nothing M_pre = nothing μ = nothing ∇M = nothing @@ -188,9 +173,6 @@ function RAM(; μ, M_pre, ram_matrices, - A_indices, - S_indices, - M_indices, F⨉I_A⁻¹, F⨉I_A⁻¹S, I_A, @@ -206,15 +188,11 @@ end ############################################################################################ function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingle, params) - fill_A_S_M!( - imply.A, - imply.S, - imply.M, - imply.A_indices, - imply.S_indices, - imply.M_indices, - params, - ) + materialize!(imply.A, imply.ram_matrices.A, params) + materialize!(imply.S, imply.ram_matrices.S, params) + if !isnothing(imply.M) + materialize!(imply.M, imply.ram_matrices.M, params) + end @. imply.I_A = -imply.A @view(imply.I_A[diagind(imply.I_A)]) .+= 1 @@ -251,12 +229,9 @@ end ### additional functions ############################################################################################ -function check_acyclic(A_pre, n_par, A_indices) - # fill copy of A-matrix with random parameters - A_rand = copy(A_pre) - randpar = rand(n_par) - - fill_matrix!(A_rand, A_indices, randpar) +function check_acyclic(A_pre::AbstractMatrix, A::ParamsMatrix) + # fill copy of A with random parameters + A_rand = materialize(A, rand(nparams(A))) # check if the model is acyclic acyclic = isone(det(I - A_rand)) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 9a96942ae..32ffcc068 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -102,24 +102,15 @@ function RAMSymbolic(; ram_matrices = convert(RAMMatrices, specification) n_par = nparams(ram_matrices) - n_obs = nobserved_vars(ram_matrices) - n_var = nvars(ram_matrices) - par = (Symbolics.@variables θ[1:n_par])[1] - A = zeros(Num, n_var, n_var) - S = zeros(Num, n_var, n_var) - !isnothing(ram_matrices.M_ind) ? M = zeros(Num, n_var) : M = nothing - F = zeros(ram_matrices.size_F) - F[CartesianIndex.(1:n_obs, ram_matrices.F_ind)] .= 1.0 - - set_RAMConstants!(A, S, M, ram_matrices.constants) - fill_A_S_M!(A, S, M, ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.M_ind, par) - - A, S, F = sparse(A), sparse(S), sparse(F) + A = sparse_materialize(Num, ram_matrices.A, par) + S = sparse_materialize(Num, ram_matrices.S, par) + M = !isnothing(ram_matrices.M) ? materialize(Num, ram_matrices.M, par) : nothing + F = ram_matrices.F - if !isnothing(loss_types) - any(loss_types .<: SemWLS) ? vech = true : nothing + if !isnothing(loss_types) && any(T -> T <: SemWLS, loss_types) + vech = true end I_A⁻¹ = neumann_series(A) From 81d0ab7f89bffb3e2792573a803dbc49abfc4629 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 15:02:40 -0700 Subject: [PATCH 123/364] materialize!(Symm/LowTri/UpTri) --- src/additional_functions/params_array.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index f20a6518b..b79b6454f 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -135,6 +135,14 @@ materialize(::Type{T}, arr::ParamsArray, param_values::AbstractVector) where {T} materialize(arr::ParamsArray, param_values::AbstractVector{T}) where {T} = materialize(Union{T, eltype(arr)}, arr, param_values) +# the hack to update the structured matrix (should be fine since the structure is imposed by ParamsMatrix) +materialize!( + dest::Union{Symmetric, LowerTriangular, UpperTriangular}, + src::ParamsMatrix{<:Any}, + param_values::AbstractVector; + kwargs..., +) = materialize!(parent(dest), src, param_values; kwargs...) + function sparse_materialize( ::Type{T}, arr::ParamsMatrix, From fd13c740b6efd4c1d7406d4547ee93a0e4508acd Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 16:13:34 -0700 Subject: [PATCH 124/364] ParamsArray: faster sparse materialize! --- src/additional_functions/params_array.jl | 91 ++++++++++++++++++----- src/frontend/specification/RAMMatrices.jl | 2 +- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index b79b6454f..13ae2eeaf 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -2,10 +2,14 @@ Array with partially parameterized elements. """ struct ParamsArray{T, N} <: AbstractArray{T, N} - linear_indices::Vector{Int} - param_ptr::Vector{Int} - constants::Vector{Pair{Int, T}} - size::NTuple{N, Int} + linear_indices::Vector{Int} # linear indices of the parameter refs in the destination array + nz_indices::Vector{Int} # indices of the parameters refs in nonzero elements vector + # (including the constants) ordered by the linear index + param_ptr::Vector{Int} # i-th element marks the start of the range in linear/nonzero + # indices arrays that corresponds to the i-th parameter + # (nparams + 1 elements) + constants::Vector{Tuple{Int, Int, T}} # linear index, index in nonzero vector, value + size::NTuple{N, Int} # size of the destination array end ParamsVector{T} = ParamsArray{T, 1} @@ -18,10 +22,16 @@ function ParamsArray{T, N}( ) where {T, N} params_ptr = pushfirst!(accumulate((ptr, inds) -> ptr + length(inds), params_map, init = 1), 1) + param_lin_inds = reduce(vcat, params_map, init = Vector{Int}()) + nz_lin_inds = unique!(sort!([param_lin_inds; first.(constants)])) + if length(nz_lin_inds) < length(param_lin_inds) + length(constants) + throw(ArgumentError("Duplicate linear indices in the parameterized array")) + end return ParamsArray{T, N}( - reduce(vcat, params_map, init = Vector{Int}()), + param_lin_inds, + searchsortedfirst.(Ref(nz_lin_inds), param_lin_inds), params_ptr, - constants, + [(c[1], searchsortedfirst(nz_lin_inds, c[1]), c[2]) for c in constants], size, ) end @@ -58,6 +68,7 @@ ParamsArray{T}( ) where {T, N} = ParamsArray{T, N}(arr, params; kwargs...) nparams(arr::ParamsArray) = length(arr.param_ptr) - 1 +SparseArrays.nnz(arr::ParamsArray) = length(arr.linear_indices) + length(arr.constants) Base.size(arr::ParamsArray) = arr.size Base.size(arr::ParamsArray, i::Integer) = arr.size[i] @@ -110,7 +121,7 @@ function materialize!( Z = eltype(dest) <: Number ? eltype(dest) : eltype(src) set_zeros && fill!(dest, zero(Z)) if set_constants - @inbounds for (i, val) in src.constants + @inbounds for (i, _, val) in src.constants dest[i] = val end end @@ -122,6 +133,43 @@ function materialize!( return dest end +function materialize!( + dest::SparseMatrixCSC, + src::ParamsMatrix, + param_values::AbstractVector; + set_constants::Bool = true, + set_zeros::Bool = false, +) + set_zeros && throw(ArgumentError("Cannot set zeros for sparse matrix")) + size(dest) == size(src) || throw( + DimensionMismatch( + "Parameters ($(size(params_arr))) and destination ($(size(dest))) array sizes don't match", + ), + ) + nparams(src) == length(param_values) || throw( + DimensionMismatch( + "Number of values ($(length(param_values))) does not match the number of parameters ($(nparams(src)))", + ), + ) + + nnz(dest) == nnz(src) || throw( + DimensionMismatch( + "Number of non-zero elements ($(nnz(dest))) does not match the number of parameter references and constants ($(nnz(src)))", + ), + ) + if set_constants + @inbounds for (_, j, val) in src.constants + dest.nzval[j] = val + end + end + @inbounds for (i, val) in enumerate(param_values) + for j in param_occurences_range(src, i) + dest.nzval[src.nz_indices[j]] = val + end + end + return dest +end + """ materialize([T], src::ParamsArray{<:Any, N}, param_values::AbstractVector{T}) where T @@ -150,27 +198,30 @@ function sparse_materialize( ) where {T} nparams(arr) == length(param_values) || throw( DimensionMismatch( - "Number of values ($(length(param)values))) does not match the number of parameter ($(nparams(arr)))", + "Number of values ($(length(param_values))) does not match the number of parameter ($(nparams(arr)))", ), ) - # constant values in sparse matrix - cvals = [T(v) for (_, v) in arr.constants] - # parameter values in sparse matrix - parvals = Vector{T}(undef, length(arr.linear_indices)) + + nz_vals = Vector{T}(undef, nnz(arr)) + nz_lininds = Vector{Int}(undef, nnz(arr)) + # fill constants + @inbounds for (lin_ind, nz_ind, val) in arr.constants + nz_vals[nz_ind] = val + nz_lininds[nz_ind] = lin_ind + end + # fill parameters @inbounds for (i, val) in enumerate(param_values) for j in param_occurences_range(arr, i) - parvals[j] = val + nz_ind = arr.nz_indices[j] + nz_vals[nz_ind] = val + nz_lininds[nz_ind] = arr.linear_indices[j] end end - nzixs = [first.(arr.constants); arr.linear_indices] - ixorder = sortperm(nzixs) - nzixs = nzixs[ixorder] - nzvals = [cvals; parvals][ixorder] arr_ixs = CartesianIndices(size(arr)) return sparse( - [arr_ixs[i][1] for i in nzixs], - [arr_ixs[i][2] for i in nzixs], - nzvals, + [arr_ixs[i][1] for i in nz_lininds], + [arr_ixs[i][2] for i in nz_lininds], + nz_vals, size(arr)..., ) end diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index b140ae026..ee487c25e 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -374,7 +374,7 @@ function append_rows!( end # add constants - for (i, val) in arr.constants + for (i, _, val) in arr.constants arr_ix = arr_ixs[i] skip_symmetric && (arr_ix ∈ visited_indices) && continue push!(partable, partable_row(val, arr_ix, arr_name, varnames, free = false)) From 19497d5bd7fbc8450ea861a80bb0d9c8321c8d2a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 2 Jul 2024 17:23:27 -0700 Subject: [PATCH 125/364] ParamsArray: use Iterators.flatten() (faster) --- src/additional_functions/params_array.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index 13ae2eeaf..bbee1dcf9 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -22,7 +22,7 @@ function ParamsArray{T, N}( ) where {T, N} params_ptr = pushfirst!(accumulate((ptr, inds) -> ptr + length(inds), params_map, init = 1), 1) - param_lin_inds = reduce(vcat, params_map, init = Vector{Int}()) + param_lin_inds = collect(Iterators.flatten(params_map)) nz_lin_inds = unique!(sort!([param_lin_inds; first.(constants)])) if length(nz_lin_inds) < length(param_lin_inds) + length(constants) throw(ArgumentError("Duplicate linear indices in the parameterized array")) From 139338d6884e45d898b6624cf43a0bbbe84d6eb7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 11 Aug 2024 13:07:50 -0700 Subject: [PATCH 126/364] Base.hash(::ParamsArray) --- src/additional_functions/params_array.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index bbee1dcf9..3a58171aa 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -79,6 +79,14 @@ Base.:(==)(a::ParamsArray, b::ParamsArray) = return eltype(a) == eltype(b) && a.param_ptr == b.param_ptr && a.linear_indices == b.linear_indices +Base.hash(a::ParamsArray, h::UInt) = hash( + typeof(a), + hash( + eltype(a), + hash(size(a), hash(a.constants, hash(a.param_ptr, hash(a.linear_indices, h)))), + ), +) + # the range of arr.param_ptr indices that correspond to i-th parameter param_occurences_range(arr::ParamsArray, i::Integer) = arr.param_ptr[i]:(arr.param_ptr[i+1]-1) From 58507840210ed7d7cc31c4f9c7d9f7036d1ca8cc Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 11 Aug 2024 13:07:38 -0700 Subject: [PATCH 127/364] colnames -> vars --- .../start_val/start_fabin3.jl | 2 +- src/frontend/specification/RAMMatrices.jl | 45 +++++++++---------- src/frontend/specification/documentation.jl | 4 +- test/examples/multigroup/multigroup.jl | 4 +- .../political_democracy.jl | 4 +- .../recover_parameters_twofact.jl | 2 +- 6 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 9d692437e..d86b992da 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -129,7 +129,7 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) # is there at least one reference indicator? if length(reference) > 0 if (length(reference) > 1) && any(((obs, param),) -> param > 0, indicators) # don't warn if entire column is fixed - @warn "You have more than 1 scaling indicator for $(ram_matrices.colnames[i])" + @warn "You have more than 1 scaling indicator for $(ram_matrices.vars[i])" end ref = reference[1] diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index ee487c25e..451f5fd69 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -9,7 +9,7 @@ struct RAMMatrices <: SemSpecification F::SparseMatrixCSC{Float64} M::Union{ParamsVector{Float64}, Nothing} params::Vector{Symbol} - colnames::Union{Vector{Symbol}, Nothing} # better call it "variables": it's a mixture of observed and latent (and it gets confusing with get_colnames()) + vars::Union{Vector{Symbol}, Nothing} # better call it "variables": it's a mixture of observed and latent (and it gets confusing with get_vars()) end nparams(ram::RAMMatrices) = nparams(ram.A) @@ -17,7 +17,7 @@ nvars(ram::RAMMatrices) = size(ram.F, 2) nobserved_vars(ram::RAMMatrices) = size(ram.F, 1) nlatent_vars(ram::RAMMatrices) = nvars(ram) - nobserved_vars(ram) -vars(ram::RAMMatrices) = ram.colnames +vars(ram::RAMMatrices) = ram.vars isobserved_var(ram::RAMMatrices, i::Integer) = ram.F.colptr[i+1] > ram.F.colptr[i] islatent_var(ram::RAMMatrices, i::Integer) = ram.F.colptr[i+1] == ram.F.colptr[i] @@ -34,13 +34,12 @@ function observed_var_indices(ram::RAMMatrices) return obs_inds end -latent_var_indices(ram::RAMMatrices) = - [i for i in axes(ram.F, 2) if islatent_var(ram, i)] +latent_var_indices(ram::RAMMatrices) = [i for i in axes(ram.F, 2) if islatent_var(ram, i)] # observed variables in the order as they appear in ram.F rows function observed_vars(ram::RAMMatrices) - if isnothing(ram.colnames) - @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" + if isnothing(ram.vars) + @warn "Your RAMMatrices do not contain variable names. Please make sure the order of variables in your data is correct!" return nothing else obs_vars = Vector{Symbol}(undef, nobserved_vars(ram)) @@ -55,11 +54,11 @@ function observed_vars(ram::RAMMatrices) end function latent_vars(ram::RAMMatrices) - if isnothing(ram.colnames) - @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" + if isnothing(ram.vars) + @warn "Your RAMMatrices do not contain variable names. Please make sure the order of variables in your data is correct!" return nothing else - return [col for (i, col) in enumerate(ram.colnames) if islatent_var(ram, i)] + return [col for (i, col) in enumerate(ram.vars) if islatent_var(ram, i)] end end @@ -73,32 +72,32 @@ function RAMMatrices(; F::AbstractMatrix, M::Union{AbstractVector, Nothing} = nothing, params::AbstractVector{Symbol}, - colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, + vars::Union{AbstractVector{Symbol}, Nothing} = nothing, ) ncols = size(A, 2) - isnothing(colnames) || check_vars(colnames, ncols) + isnothing(vars) || check_vars(vars, ncols) size(A, 1) == size(A, 2) || throw(DimensionMismatch("A must be a square matrix")) size(S, 1) == size(S, 2) || throw(DimensionMismatch("S must be a square matrix")) size(A, 2) == ncols || throw( DimensionMismatch( - "A should have as many rows and columns as colnames length ($ncols), $(size(A)) found", + "A should have as many rows and columns as vars length ($ncols), $(size(A)) found", ), ) size(S, 2) == ncols || throw( DimensionMismatch( - "S should have as many rows and columns as colnames length ($ncols), $(size(S)) found", + "S should have as many rows and columns as vars length ($ncols), $(size(S)) found", ), ) size(F, 2) == ncols || throw( DimensionMismatch( - "F should have as many columns as colnames length ($ncols), $(size(F, 2)) found", + "F should have as many columns as vars length ($ncols), $(size(F, 2)) found", ), ) if !isnothing(M) length(M) == ncols || throw( DimensionMismatch( - "M should have as many elements as colnames length ($ncols), $(length(M)) found", + "M should have as many elements as vars length ($ncols), $(length(M)) found", ), ) end @@ -111,7 +110,7 @@ function RAMMatrices(; if any(!isone, spF.nzval) throw(ArgumentError("F should contain only 0s and 1s")) end - return RAMMatrices(A, S, F, M, params, colnames) + return RAMMatrices(A, S, F, M, params, vars) end ############################################################################################ @@ -244,14 +243,14 @@ function ParameterTable( ) # defer parameter checks until we know which ones are used - if !isnothing(ram.colnames) + if !isnothing(ram.vars) latent_vars = SEM.latent_vars(ram) observed_vars = SEM.observed_vars(ram) - colnames = ram.colnames + vars = ram.vars else observed_vars = [Symbol("$(observed_var_prefix)_$i") for i in 1:nobserved_vars(ram)] latent_vars = [Symbol("$(latent_var_prefix)_$i") for i in 1:nlatent_vars(ram)] - colnames = vcat(observed_vars, latent_vars) + vars = vcat(observed_vars, latent_vars) end # construct an empty table @@ -262,10 +261,10 @@ function ParameterTable( ) # fill the table - append_rows!(partable, ram.S, :S, ram.params, colnames, skip_symmetric = true) - append_rows!(partable, ram.A, :A, ram.params, colnames) + append_rows!(partable, ram.S, :S, ram.params, vars, skip_symmetric = true) + append_rows!(partable, ram.A, :A, ram.params, vars) if !isnothing(ram.M) - append_rows!(partable, ram.M, :M, ram.params, colnames) + append_rows!(partable, ram.M, :M, ram.params, vars) end check_params(SEM.params(partable), partable.columns[:param]) @@ -395,7 +394,7 @@ function Base.:(==)(mat1::RAMMatrices, mat2::RAMMatrices) (mat1.F == mat2.F) && (mat1.M == mat2.M) && (mat1.params == mat2.params) && - (mat1.colnames == mat2.colnames) + (mat1.vars == mat2.vars) ) return res end diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl index e869dd43f..46135ead0 100644 --- a/src/frontend/specification/documentation.jl +++ b/src/frontend/specification/documentation.jl @@ -95,7 +95,7 @@ function EnsembleParameterTable end (1) RAMMatrices(partable::ParameterTable) - (2) RAMMatrices(;A, S, F, M = nothing, params, colnames) + (2) RAMMatrices(;A, S, F, M = nothing, params, vars) (3) RAMMatrices(partable::EnsembleParameterTable) @@ -110,7 +110,7 @@ Return `RAMMatrices` constructed from (1) a parameter table or (2) individual ma - `F`: filter matrix - `M`: vector of mean effects - `params::Vector{Symbol}`: parameter labels -- `colnames::Vector{Symbol}`: variable names corresponding to the A, S and F matrix columns +- `vars::Vector{Symbol}`: variable names corresponding to the A, S and F matrix columns # Examples See the online documentation on [Model specification](@ref) and the [RAMMatrices interface](@ref). diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index a2f277d91..caaa5c3f7 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -60,7 +60,7 @@ specification_g1 = RAMMatrices(; S = S1, F = F, params = x, - colnames = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], + vars = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], ) specification_g2 = RAMMatrices(; @@ -68,7 +68,7 @@ specification_g2 = RAMMatrices(; S = S2, F = F, params = x, - colnames = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], + vars = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], ) partable = EnsembleParameterTable( diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index d7fbb8f2c..2f570302a 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -76,7 +76,7 @@ spec = RAMMatrices(; S = S, F = F, params = x, - colnames = [ + vars = [ :x1, :x2, :x3, @@ -108,7 +108,7 @@ spec_mean = RAMMatrices(; F = F, M = M, params = x, - colnames = [ + vars = [ :x1, :x2, :x3, diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index f00187fac..89c1225e2 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -40,7 +40,7 @@ A = [ 0 0 0 0 0 0 0 0 ] -ram_matrices = RAMMatrices(; A = A, S = S, F = F, params = x, colnames = nothing) +ram_matrices = RAMMatrices(; A = A, S = S, F = F, params = x, vars = nothing) true_val = [ repeat([1], 8) From 0f747b7ec789d5aceb4eed3b2b4612de2f7c60f8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 22:50:18 -0700 Subject: [PATCH 128/364] update_partable!(): better params unique check --- src/frontend/specification/ParameterTable.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index df2cc165b..05350fb12 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -309,10 +309,10 @@ function update_partable!( "The length of `params` ($(length(params))) and their `values` ($(length(values))) must be the same", ), ) + dup_params = nonunique(params) + isempty(dup_params) || + throw(ArgumentError("Duplicate parameters detected: $(join(dup_params, ", "))")) param_values = Dict(zip(params, values)) - if length(param_values) != length(params) - throw(ArgumentError("Duplicate parameter names in `params`")) - end update_partable!(partable, column, param_values, default) end From aa34d5352979399de3f40226bf05b5690b777b9f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 31 Jul 2024 21:35:50 -0700 Subject: [PATCH 129/364] start_fabin3: check obs_mean data & meanstructure --- src/additional_functions/start_val/start_fabin3.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index d86b992da..53cf7cff6 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -30,13 +30,21 @@ function start_fabin3(observed::SemObservedMissing, imply, optimizer, args...; k return start_fabin3(imply.ram_matrices, observed.em_model.Σ, observed.em_model.μ) end -function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) +function start_fabin3( + ram_matrices::RAMMatrices, + Σ::AbstractMatrix, + μ::Union{AbstractVector, Nothing}, +) A, S, F, M, n_par = ram_matrices.A, ram_matrices.S, ram_matrices.F, ram_matrices.M, nparams(ram_matrices) + if !isnothing(M) && isnothing(μ) + throw(ArgumentError("RAM has meanstructure, but no observed means provided.")) + end + start_val = zeros(n_par) F_var2obs = Dict( i => F.rowval[F.colptr[i]] for i in axes(F, 2) if isobserved_var(ram_matrices, i) From 13aacd0cf87c1f8946f97bc27eb2cfe57f2e21cd Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Nov 2024 11:28:11 -0800 Subject: [PATCH 130/364] params/vars API tweaks and tests --- .../start_val/start_partable.jl | 31 +++++++------------ src/frontend/specification/ParameterTable.jl | 4 +-- src/frontend/specification/RAMMatrices.jl | 2 +- src/frontend/specification/StenoGraphs.jl | 2 +- test/examples/helper.jl | 2 ++ test/examples/multigroup/build_models.jl | 2 ++ .../political_democracy/constructor.jl | 1 + .../political_democracy.jl | 12 +++++-- 8 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/additional_functions/start_val/start_partable.jl b/src/additional_functions/start_val/start_partable.jl index 6fb15e365..15f863f5b 100644 --- a/src/additional_functions/start_val/start_partable.jl +++ b/src/additional_functions/start_val/start_partable.jl @@ -21,28 +21,19 @@ function start_parameter_table(observed, imply, optimizer, args...; kwargs...) return start_parameter_table(ram_matrices(imply); kwargs...) end -function start_parameter_table( - ram_matrices::RAMMatrices; - parameter_table::ParameterTable, - kwargs..., -) +function start_parameter_table(ram::RAMMatrices; partable::ParameterTable, kwargs...) start_val = zeros(0) - for param in ram_matrices.params - found = false - for (i, param_table) in enumerate(parameter_table.params) - if param == param_table - push!(start_val, parameter_table.start[i]) - found = true - break - end - end - if !found - throw( - ErrorException( - "At least one parameter could not be found in the parameter table.", - ), - ) + param_indices = Dict(param => i for (i, param) in enumerate(params(ram))) + start_col = partable.columns[:start] + + for (i, param) in enumerate(partable.columns[:param]) + par_ind = get(param_indices, param, nothing) + if !isnothing(par_ind) + par_start = start_col[i] + isfinite(par_start) && (start_val[i] = par_start) + else + throw(ErrorException("Parameter $(param) is not in the parameter table.")) end end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 05350fb12..8b7cc0973 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -54,8 +54,8 @@ function ParameterTable( return ParameterTable( Dict(col => copy(values) for (col, values) in pairs(partable.columns)), - observed_vars = copy(partable.observed_vars), - latent_vars = copy(partable.latent_vars), + observed_vars = copy(observed_vars(partable)), + latent_vars = copy(latent_vars(partable)), params = params, ) end diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 451f5fd69..0c5722f57 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -110,7 +110,7 @@ function RAMMatrices(; if any(!isone, spF.nzval) throw(ArgumentError("F should contain only 0s and 1s")) end - return RAMMatrices(A, S, F, M, params, vars) + return RAMMatrices(A, S, F, M, copy(params), vars) end ############################################################################################ diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 64a33f13e..5cf87c07a 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -42,7 +42,7 @@ function ParameterTable( latent_vars::AbstractVector{Symbol}, params::Union{AbstractVector{Symbol}, Nothing} = nothing, group::Union{Integer, Nothing} = nothing, - param_prefix = :θ, + param_prefix::Symbol = :θ, ) graph = unique(graph) n = length(graph) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index d4c140d67..042f7005f 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,6 +1,8 @@ using LinearAlgebra: norm function test_gradient(model, params; rtol = 1e-10, atol = 0) + @test nparams(model) == length(params) + true_grad = FiniteDiff.finite_difference_gradient(Base.Fix1(objective!, model), params) gradient = similar(params) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 3f29a6898..6991dd479 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -8,6 +8,8 @@ model_g1 = Sem(specification = specification_g1, data = dat_g1, imply = RAMSymbo model_g2 = Sem(specification = specification_g2, data = dat_g2, imply = RAM) +@test SEM.params(model_g1.imply.ram_matrices) == SEM.params(model_g2.imply.ram_matrices) + # test the different constructors model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) model_ml_multigroup2 = SemEnsemble( diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 99ef06b3a..bebabf6e0 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -6,6 +6,7 @@ using Random ############################################################################################ model_ml = Sem(specification = spec, data = dat, optimizer = semoptimizer) +@test SEM.params(model_ml.imply.ram_matrices) == SEM.params(spec) model_ml_cov = Sem( specification = spec, diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 2f570302a..2265e2a59 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -1,5 +1,7 @@ using StructuralEquationModels, Test, FiniteDiff +SEM = StructuralEquationModels + include( joinpath( chop(dirname(pathof(StructuralEquationModels)), tail = 3), @@ -96,9 +98,9 @@ spec = RAMMatrices(; partable = ParameterTable(spec) -# w. meanstructure ------------------------------------------------------------------------- +@test SEM.params(spec) == SEM.params(partable) -x = Symbol.("x" .* string.(1:38)) +# w. meanstructure ------------------------------------------------------------------------- M = [:x32; :x33; :x34; :x35; :x36; :x37; :x38; :x35; :x36; :x37; :x38; 0.0; 0.0; 0.0] @@ -107,7 +109,7 @@ spec_mean = RAMMatrices(; S = S, F = F, M = M, - params = x, + params = [SEM.params(spec); Symbol.("x", string.(32:38))], vars = [ :x1, :x2, @@ -128,6 +130,8 @@ spec_mean = RAMMatrices(; partable_mean = ParameterTable(spec_mean) +@test SEM.params(partable_mean) == SEM.params(spec_mean) + start_test = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3)] start_test_mean = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3); fill(0.1, 7)] @@ -164,6 +168,8 @@ end spec = ParameterTable(spec) spec_mean = ParameterTable(spec_mean) +@test SEM.params(spec) == SEM.params(partable) + partable = spec partable_mean = spec_mean From 8c26c351c5f440df66bc6cb78a92b43007b66c00 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 15:10:43 -0700 Subject: [PATCH 131/364] generic imply: keep F sparse --- src/imply/RAM/generic.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index a16aac179..850934a9c 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -129,7 +129,7 @@ function RAM(; nan_params = fill(NaN, n_par) A_pre = materialize(ram_matrices.A, nan_params) S_pre = materialize(ram_matrices.S, nan_params) - F = Matrix(ram_matrices.F) + F = copy(ram_matrices.F) A_pre = check_acyclic(A_pre, ram_matrices.A) From 3816539d9d6cacb0ee9740e6cb27108343f00b4f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 12 Mar 2024 16:49:33 -0700 Subject: [PATCH 132/364] tests helper: is_extended_tests() to consolidate ENV variable check --- test/examples/helper.jl | 4 ++++ test/examples/political_democracy/political_democracy.jl | 6 +++--- test/runtests.jl | 3 --- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 042f7005f..f35d2cac6 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,5 +1,9 @@ using LinearAlgebra: norm +function is_extended_tests() + return lowercase(get(ENV, "JULIA_EXTENDED_TESTS", "false")) == "true" +end + function test_gradient(model, params; rtol = 1e-10, atol = 0) @test nparams(model) == length(params) diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 2265e2a59..6754c29c3 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -146,7 +146,7 @@ semoptimizer = SemOptimizerNLopt include("constructor.jl") end -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" +if is_extended_tests() semoptimizer = SemOptimizerOptim @testset "RAMMatrices | parts | Optim" begin include("by_parts.jl") @@ -182,7 +182,7 @@ semoptimizer = SemOptimizerNLopt include("constructor.jl") end -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" +if is_extended_tests() semoptimizer = SemOptimizerOptim @testset "RAMMatrices → ParameterTable | parts | Optim" begin include("by_parts.jl") @@ -269,7 +269,7 @@ semoptimizer = SemOptimizerNLopt include("constructor.jl") end -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" +if is_extended_tests() semoptimizer = SemOptimizerOptim @testset "Graph → ParameterTable | parts | Optim" begin include("by_parts.jl") diff --git a/test/runtests.jl b/test/runtests.jl index c3b15475f..28d2142b1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,6 +11,3 @@ end @time @safetestset "Example Models" begin include("examples/examples.jl") end - -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" -end From b8d9a8fa9d60b8779a9a5a086f553cbb184fd08c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 25 May 2024 17:05:25 -0700 Subject: [PATCH 133/364] Optim sem_fit(): use provided optimizer --- src/optimizer/optim.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index bb1bf507e..7951e6b14 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -31,8 +31,8 @@ function sem_fit( result = Optim.optimize( Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), start_val, - model.optimizer.algorithm, - model.optimizer.options, + optim.algorithm, + optim.options, ) return SemFit(result, model, start_val) end From dd275d57dca12c473fe5d85b38db00493e15dd4e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 14:40:52 -0800 Subject: [PATCH 134/364] prepare_start_params(): arg-dependent dispatch * convert to argument type-dependent dispatch * replace start_val() function with prepare_start_params() * refactor start_parameter_table() into prepare_start_params(start_val::ParameterTable, ...) and use the SEM model param indices * unify processing of starting values by all optimizers * support dictionaries of values --- src/StructuralEquationModels.jl | 3 - .../start_val/start_partable.jl | 41 ------------- .../start_val/start_val.jl | 26 -------- src/optimizer/NLopt.jl | 17 ++---- src/optimizer/documentation.jl | 60 +++++++++++++++++-- src/optimizer/optim.jl | 12 ++-- 6 files changed, 65 insertions(+), 94 deletions(-) delete mode 100644 src/additional_functions/start_val/start_partable.jl delete mode 100644 src/additional_functions/start_val/start_val.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 6172af1ea..3f68dd95f 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -70,9 +70,7 @@ include("optimizer/optim.jl") include("optimizer/NLopt.jl") # helper functions include("additional_functions/helper.jl") -include("additional_functions/start_val/start_val.jl") include("additional_functions/start_val/start_fabin3.jl") -include("additional_functions/start_val/start_partable.jl") include("additional_functions/start_val/start_simple.jl") include("additional_functions/artifacts.jl") include("additional_functions/simulation.jl") @@ -109,7 +107,6 @@ export AbstractSem, start_val, start_fabin3, start_simple, - start_parameter_table, SemLoss, SemLossFunction, SemML, diff --git a/src/additional_functions/start_val/start_partable.jl b/src/additional_functions/start_val/start_partable.jl deleted file mode 100644 index 15f863f5b..000000000 --- a/src/additional_functions/start_val/start_partable.jl +++ /dev/null @@ -1,41 +0,0 @@ -""" - start_parameter_table(model; parameter_table) - -Return a vector of starting values taken from `parameter_table`. -""" -function start_parameter_table end - -# splice model and loss functions -function start_parameter_table(model::AbstractSemSingle; kwargs...) - return start_parameter_table( - model.observed, - model.imply, - model.optimizer, - model.loss.functions...; - kwargs..., - ) -end - -# RAM(Symbolic) -function start_parameter_table(observed, imply, optimizer, args...; kwargs...) - return start_parameter_table(ram_matrices(imply); kwargs...) -end - -function start_parameter_table(ram::RAMMatrices; partable::ParameterTable, kwargs...) - start_val = zeros(0) - - param_indices = Dict(param => i for (i, param) in enumerate(params(ram))) - start_col = partable.columns[:start] - - for (i, param) in enumerate(partable.columns[:param]) - par_ind = get(param_indices, param, nothing) - if !isnothing(par_ind) - par_start = start_col[i] - isfinite(par_start) && (start_val[i] = par_start) - else - throw(ErrorException("Parameter $(param) is not in the parameter table.")) - end - end - - return start_val -end diff --git a/src/additional_functions/start_val/start_val.jl b/src/additional_functions/start_val/start_val.jl deleted file mode 100644 index 8b6402efa..000000000 --- a/src/additional_functions/start_val/start_val.jl +++ /dev/null @@ -1,26 +0,0 @@ -""" - start_val(model) - -Return a vector of starting values. -Defaults are FABIN 3 starting values for single models and simple starting values for -ensemble models. -""" -function start_val end -# Single Models ---------------------------------------------------------------------------- - -# splice model and loss functions -start_val(model::AbstractSemSingle; kwargs...) = start_val( - model, - model.observed, - model.imply, - model.optimizer, - model.loss.functions...; - kwargs..., -) - -# Fabin 3 starting values for RAM(Symbolic) -start_val(model, observed, imply, optimizer, args...; kwargs...) = - start_fabin3(model; kwargs...) - -# Ensemble Models -------------------------------------------------------------------------- -start_val(model::SemEnsemble; kwargs...) = start_simple(model; kwargs...) diff --git a/src/optimizer/NLopt.jl b/src/optimizer/NLopt.jl index 7f4f61e1e..6b03a676c 100644 --- a/src/optimizer/NLopt.jl +++ b/src/optimizer/NLopt.jl @@ -25,21 +25,16 @@ end # sem_fit method function sem_fit( optimizer::SemOptimizerNLopt, - model::AbstractSem; - start_val = start_val, + model::AbstractSem, + start_params::AbstractVector; kwargs..., ) - # starting values - if !isa(start_val, AbstractVector) - start_val = start_val(model; kwargs...) - end - # construct the NLopt problem opt = construct_NLopt_problem( model.optimizer.algorithm, model.optimizer.options, - length(start_val), + length(start_params), ) set_NLopt_constraints!(opt, model.optimizer) opt.min_objective = @@ -55,15 +50,15 @@ function sem_fit( opt_local = construct_NLopt_problem( model.optimizer.local_algorithm, model.optimizer.local_options, - length(start_val), + length(start_params), ) opt.local_optimizer = opt_local end # fit - result = NLopt.optimize(opt, start_val) + result = NLopt.optimize(opt, start_params) - return SemFit_NLopt(result, model, start_val, opt) + return SemFit_NLopt(result, model, start_params, opt) end ############################################################################################ diff --git a/src/optimizer/documentation.jl b/src/optimizer/documentation.jl index 7c17e6ce2..a369fba77 100644 --- a/src/optimizer/documentation.jl +++ b/src/optimizer/documentation.jl @@ -5,16 +5,17 @@ Return the fitted `model`. # Arguments - `model`: `AbstractSem` to fit -- `start_val`: vector of starting values or function to compute starting values (1) +- `start_val`: a vector or a dictionary of starting parameter values, + or function to compute them (1) - `kwargs...`: keyword arguments, passed to starting value functions -(1) available options are `start_fabin3`, `start_simple` and `start_partable`. +(1) available functions are `start_fabin3`, `start_simple` and `start_partable`. For more information, we refer to the individual documentations and the online documentation on [Starting values](@ref). # Examples ```julia sem_fit( - my_model; + my_model; start_val = start_simple, start_covariances_latent = 0.5) ``` @@ -22,8 +23,57 @@ sem_fit( function sem_fit end # dispatch on optimizer -sem_fit(model::AbstractSem; kwargs...) = sem_fit(model.optimizer, model; kwargs...) +function sem_fit(model::AbstractSem; start_val = nothing, kwargs...) + start_params = prepare_start_params(start_val, model; kwargs...) + @assert start_params isa AbstractVector + @assert length(start_params) == nparams(model) + + sem_fit(model.optimizer, model, start_params; kwargs...) +end # fallback method -sem_fit(optimizer::SemOptimizer, model::AbstractSem; kwargs...) = +sem_fit(optimizer::SemOptimizer, model::AbstractSem, start_params; kwargs...) = error("Optimizer $(optimizer) support not implemented.") + +# FABIN3 is the default method for single models +prepare_start_params(start_val::Nothing, model::AbstractSemSingle; kwargs...) = + start_fabin3(model; kwargs...) + +# simple algorithm is the default method for ensembles +prepare_start_params(start_val::Nothing, model::AbstractSem; kwargs...) = + start_simple(model; kwargs...) + +function prepare_start_params(start_val::AbstractVector, model::AbstractSem; kwargs...) + (length(start_val) == nparams(model)) || throw( + DimensionMismatch( + "The length of `start_val` vector ($(length(start_val))) does not match the number of model parameters ($(nparams(model))).", + ), + ) + return start_val +end + +function prepare_start_params(start_val::AbstractDict, model::AbstractSem; kwargs...) + return [start_val[param] for param in params(model)] # convert to a vector +end + +# get from the ParameterTable (potentially from a different model with match param names) +# TODO: define kwargs that instruct to get values from "estimate" and "fixed" +function prepare_start_params(start_val::ParameterTable, model::AbstractSem; kwargs...) + res = zeros(eltype(start_val.columns[:start]), nparams(model)) + param_indices = Dict(param => i for (i, param) in enumerate(params(model))) + + for (param, startval) in zip(start_val.columns[:param], start_val.columns[:start]) + (param == :const) && continue + par_ind = get(param_indices, param, nothing) + if !isnothing(par_ind) + isfinite(startval) && (res[par_ind] = startval) + else + throw( + ErrorException( + "Model parameter $(param) not found in the parameter table.", + ), + ) + end + end + return res +end diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 7951e6b14..b2adfe03a 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -20,19 +20,15 @@ convergence(res::Optim.MultivariateOptimizationResults) = Optim.converged(res) function sem_fit( optim::SemOptimizerOptim, - model::AbstractSem; - start_val = start_val, + model::AbstractSem, + start_params::AbstractVector; kwargs..., ) - if !isa(start_val, AbstractVector) - start_val = start_val(model; kwargs...) - end - result = Optim.optimize( Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), - start_val, + start_params, optim.algorithm, optim.options, ) - return SemFit(result, model, start_val) + return SemFit(result, model, start_params) end From 0131bb784d49286b527f06f251f59f9b9b11cd55 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 14:45:37 -0800 Subject: [PATCH 135/364] prepare_param_bounds() API for optim --- src/optimizer/documentation.jl | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/optimizer/documentation.jl b/src/optimizer/documentation.jl index a369fba77..cf6aaa312 100644 --- a/src/optimizer/documentation.jl +++ b/src/optimizer/documentation.jl @@ -77,3 +77,40 @@ function prepare_start_params(start_val::ParameterTable, model::AbstractSem; kwa end return res end + +# prepare a vector of model parameter bounds (BOUND=:lower or BOUND=:lower): +# use the user-specified "bounds" vector "as is" +function prepare_param_bounds( + ::Val{BOUND}, + bounds::AbstractVector{<:Number}, + model::AbstractSem; + default::Number, # unused for vector bounds + variance_default::Number, # unused for vector bounds +) where {BOUND} + length(bounds) == nparams(model) || throw( + DimensionMismatch( + "The length of `bounds` vector ($(length(bounds))) does not match the number of model parameters ($(nparams(model))).", + ), + ) + return bounds +end + +# prepare a vector of model parameter bounds +# given the "bounds" dictionary and default values +function prepare_param_bounds( + ::Val{BOUND}, + bounds::Union{AbstractDict, Nothing}, + model::AbstractSem; + default::Number, + variance_default::Number, +) where {BOUND} + varparams = Set(variance_params(model.imply.ram_matrices)) + res = [ + begin + def = in(p, varparams) ? variance_default : default + isnothing(bounds) ? def : get(bounds, p, def) + end for p in SEM.params(model) + ] + + return res +end From fbdcc7f9b8caacfe8b4d553e3d323bbf848852f9 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 14:45:37 -0800 Subject: [PATCH 136/364] u/l_bounds support for Optim.jl --- src/optimizer/optim.jl | 45 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index b2adfe03a..19623b965 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -22,13 +22,46 @@ function sem_fit( optim::SemOptimizerOptim, model::AbstractSem, start_params::AbstractVector; + lower_bounds::Union{AbstractVector, AbstractDict, Nothing} = nothing, + upper_bounds::Union{AbstractVector, AbstractDict, Nothing} = nothing, + lower_bound = -Inf, + upper_bound = Inf, + variance_lower_bound::Number = 0.0, + variance_upper_bound::Number = Inf, kwargs..., ) - result = Optim.optimize( - Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), - start_params, - optim.algorithm, - optim.options, - ) + # setup lower/upper bounds if the algorithm supports it + if optim.algorithm isa Optim.Fminbox || optim.algorithm isa Optim.SAMIN + lbounds = prepare_param_bounds( + Val(:lower), + lower_bounds, + model, + default = lower_bound, + variance_default = variance_lower_bound, + ) + ubounds = prepare_param_bounds( + Val(:upper), + upper_bounds, + model, + default = upper_bound, + variance_default = variance_upper_bound, + ) + start_params = clamp.(start_params, lbounds, ubounds) + result = Optim.optimize( + Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), + lbounds, + ubounds, + start_params, + optim.algorithm, + optim.options, + ) + else + result = Optim.optimize( + Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), + start_params, + optim.algorithm, + optim.options, + ) + end return SemFit(result, model, start_params) end From d1f323a7b8f3f35d813ec130056509b16975c9d9 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 14 Apr 2024 15:52:01 -0700 Subject: [PATCH 137/364] SemOptimizer(engine = ...) ctor --- src/diff/Empty.jl | 4 ++-- src/diff/NLopt.jl | 4 +++- src/diff/optim.jl | 4 +++- src/types.jl | 13 +++++++++- test/Project.toml | 1 + test/examples/political_democracy/by_parts.jl | 9 +++---- .../political_democracy/constraints.jl | 11 +++++---- .../political_democracy/constructor.jl | 6 +++-- .../political_democracy.jl | 24 +++++++++---------- 9 files changed, 49 insertions(+), 27 deletions(-) diff --git a/src/diff/Empty.jl b/src/diff/Empty.jl index 57fa9ee98..45a20db55 100644 --- a/src/diff/Empty.jl +++ b/src/diff/Empty.jl @@ -15,13 +15,13 @@ an optimizer part. Subtype of `SemOptimizer`. """ -struct SemOptimizerEmpty <: SemOptimizer end +struct SemOptimizerEmpty <: SemOptimizer{:Empty} end ############################################################################################ ### Constructor ############################################################################################ -# SemOptimizerEmpty(;kwargs...) = SemOptimizerEmpty() +SemOptimizer{:Empty}() = SemOptimizerEmpty() ############################################################################################ ### Recommended methods diff --git a/src/diff/NLopt.jl b/src/diff/NLopt.jl index 12fcd7e0f..f0e4cea5b 100644 --- a/src/diff/NLopt.jl +++ b/src/diff/NLopt.jl @@ -56,7 +56,7 @@ see [Constrained optimization](@ref) in our online documentation. Subtype of `SemOptimizer`. """ -struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer +struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} algorithm::A local_algorithm::A2 options::B @@ -97,6 +97,8 @@ function SemOptimizerNLopt(; ) end +SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) + ############################################################################################ ### Recommended methods ############################################################################################ diff --git a/src/diff/optim.jl b/src/diff/optim.jl index 4e4b04e9f..5b8845275 100644 --- a/src/diff/optim.jl +++ b/src/diff/optim.jl @@ -44,11 +44,13 @@ my_newton_optimizer = SemOptimizerOptim( Subtype of `SemOptimizer`. """ -mutable struct SemOptimizerOptim{A, B} <: SemOptimizer +mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} algorithm::A options::B end +SemOptimizer{:Optim}(args...; kwargs...) = SemOptimizerOptim(args...; kwargs...) + SemOptimizerOptim(; algorithm = LBFGS(), options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), diff --git a/src/types.jl b/src/types.jl index 576252726..90b648ac8 100644 --- a/src/types.jl +++ b/src/types.jl @@ -84,7 +84,18 @@ Supertype of all objects that can serve as the `optimizer` field of a SEM. Connects the SEM to its optimization backend and controls options like the optimization algorithm. If you want to connect the SEM package to a new optimization backend, you should implement a subtype of SemOptimizer. """ -abstract type SemOptimizer end +abstract type SemOptimizer{E} end + +engine(::Type{SemOptimizer{E}}) where {E} = E +engine(optimizer::SemOptimizer) = engine(typeof(optimizer)) + +SemOptimizer(args...; engine::Symbol = :Optim, kwargs...) = + SemOptimizer{engine}(args...; kwargs...) + +# fallback optimizer constructor +function SemOptimizer{E}(args...; kwargs...) where {E} + throw(ErrorException("$E optimizer is not supported.")) +end """ Supertype of all objects that can serve as the observed field of a SEM. diff --git a/test/Project.toml b/test/Project.toml index c5124c659..5867c1f40 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -6,6 +6,7 @@ JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899" LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index f50fb6dd0..87e5fb733 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -25,7 +25,7 @@ loss_ml = SemLoss(ml) loss_wls = SemLoss(wls) # optimizer ------------------------------------------------------------------------------------- -optimizer_obj = semoptimizer() +optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- @@ -152,10 +152,11 @@ end ### test hessians ############################################################################################ -if semoptimizer == SemOptimizerOptim +if opt_engine == :Optim using Optim, LineSearches - optimizer_obj = SemOptimizerOptim( + optimizer_obj = SemOptimizer( + engine = opt_engine, algorithm = Newton(; linesearch = BackTracking(order = 3), alphaguess = InitialHagerZhang(), @@ -220,7 +221,7 @@ loss_ml = SemLoss(ml) loss_wls = SemLoss(wls) # optimizer ------------------------------------------------------------------------------------- -optimizer_obj = semoptimizer() +optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- model_ml = Sem(observed, imply_ram, loss_ml, optimizer_obj) diff --git a/test/examples/political_democracy/constraints.jl b/test/examples/political_democracy/constraints.jl index e5cd96ab9..47f27582a 100644 --- a/test/examples/political_democracy/constraints.jl +++ b/test/examples/political_democracy/constraints.jl @@ -1,4 +1,5 @@ # NLopt constraints ------------------------------------------------------------------------ +using NLopt # 1.5*x1 == x2 (aka 1.5*x1 - x2 == 0) #= function eq_constraint(x, grad) @@ -20,12 +21,13 @@ function ineq_constraint(x, grad) 0.6 - x[30] * x[31] end -constrained_optimizer = SemOptimizerNLopt(; +constrained_optimizer = SemOptimizer(; + engine = :NLopt, algorithm = :AUGLAG, local_algorithm = :LD_LBFGS, options = Dict(:xtol_rel => 1e-4), - # equality_constraints = NLoptConstraint(;f = eq_constraint, tol = 1e-14), - inequality_constraints = NLoptConstraint(; f = ineq_constraint, tol = 1e-8), + # equality_constraints = (f = eq_constraint, tol = 1e-14), + inequality_constraints = (f = ineq_constraint, tol = 0.0), ) model_ml_constrained = @@ -38,7 +40,8 @@ solution_constrained = sem_fit(model_ml_constrained) model_ml_maxeval = Sem( specification = spec, data = dat, - optimizer = SemOptimizerNLopt, + optimizer = SemOptimizer, + engine = :NLopt, options = Dict(:maxeval => 10), ) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index bebabf6e0..5ed576dc1 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -1,10 +1,12 @@ using Statistics: cov, mean -using Random +using Random, NLopt ############################################################################################ ### models w.o. meanstructure ############################################################################################ +semoptimizer = SemOptimizer(engine = opt_engine) + model_ml = Sem(specification = spec, data = dat, optimizer = semoptimizer) @test SEM.params(model_ml.imply.ram_matrices) == SEM.params(spec) @@ -205,7 +207,7 @@ end ### test hessians ############################################################################################ -if semoptimizer == SemOptimizerOptim +if opt_engine == :Optim using Optim, LineSearches model_ls = Sem( diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 6754c29c3..a2e5089bb 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -136,22 +136,22 @@ start_test = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0. start_test_mean = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3); fill(0.1, 7)] -semoptimizer = SemOptimizerOptim +opt_engine = :Optim @testset "RAMMatrices | constructor | Optim" begin include("constructor.jl") end -semoptimizer = SemOptimizerNLopt +opt_engine = :NLopt @testset "RAMMatrices | constructor | NLopt" begin include("constructor.jl") end if is_extended_tests() - semoptimizer = SemOptimizerOptim + opt_engine = :Optim @testset "RAMMatrices | parts | Optim" begin include("by_parts.jl") end - semoptimizer = SemOptimizerNLopt + opt_engine = :NLopt @testset "RAMMatrices | parts | NLopt" begin include("by_parts.jl") end @@ -173,21 +173,21 @@ spec_mean = ParameterTable(spec_mean) partable = spec partable_mean = spec_mean -semoptimizer = SemOptimizerOptim +opt_engine = :Optim @testset "RAMMatrices → ParameterTable | constructor | Optim" begin include("constructor.jl") end -semoptimizer = SemOptimizerNLopt +opt_engine = :NLopt @testset "RAMMatrices → ParameterTable | constructor | NLopt" begin include("constructor.jl") end if is_extended_tests() - semoptimizer = SemOptimizerOptim + opt_engine = :Optim @testset "RAMMatrices → ParameterTable | parts | Optim" begin include("by_parts.jl") end - semoptimizer = SemOptimizerNLopt + opt_engine = :NLopt @testset "RAMMatrices → ParameterTable | parts | NLopt" begin include("by_parts.jl") end @@ -260,21 +260,21 @@ start_test = [fill(0.5, 8); fill(0.05, 3); fill(1.0, 11); fill(0.05, 9)] start_test_mean = [fill(0.5, 8); fill(0.05, 3); fill(1.0, 11); fill(0.05, 3); fill(0.05, 13)] -semoptimizer = SemOptimizerOptim +opt_engine = :Optim @testset "Graph → ParameterTable | constructor | Optim" begin include("constructor.jl") end -semoptimizer = SemOptimizerNLopt +opt_engine = :NLopt @testset "Graph → ParameterTable | constructor | NLopt" begin include("constructor.jl") end if is_extended_tests() - semoptimizer = SemOptimizerOptim + opt_engine = :Optim @testset "Graph → ParameterTable | parts | Optim" begin include("by_parts.jl") end - semoptimizer = SemOptimizerNLopt + opt_engine = :NLopt @testset "Graph → ParameterTable | parts | NLopt" begin include("by_parts.jl") end From 0a6b073d2ac2ef4d361faf0f7d1d623a5889d72d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 2 Apr 2024 18:33:50 -0700 Subject: [PATCH 138/364] SEMNLOptExt for NLopt --- Project.toml | 7 ++++++- ext/SEMNLOptExt.jl | 12 +++++++++++ {src => ext}/diff/NLopt.jl | 37 ++++++++++++++++++++------------- {src => ext}/optimizer/NLopt.jl | 20 +++++++++--------- src/StructuralEquationModels.jl | 5 ----- 5 files changed, 50 insertions(+), 31 deletions(-) create mode 100644 ext/SEMNLOptExt.jl rename {src => ext}/diff/NLopt.jl (76%) rename {src => ext}/optimizer/NLopt.jl (84%) diff --git a/Project.toml b/Project.toml index b038c3364..21bd43814 100644 --- a/Project.toml +++ b/Project.toml @@ -12,7 +12,6 @@ LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" NLSolversBase = "d41bc354-129a-5804-8e4c-c37616107c6c" -NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" @@ -44,3 +43,9 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test"] + +[weakdeps] +NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" + +[extensions] +SEMNLOptExt = "NLopt" diff --git a/ext/SEMNLOptExt.jl b/ext/SEMNLOptExt.jl new file mode 100644 index 000000000..dfc3bbb42 --- /dev/null +++ b/ext/SEMNLOptExt.jl @@ -0,0 +1,12 @@ +module SEMNLOptExt + +using StructuralEquationModels, NLopt + +SEM = StructuralEquationModels + +export SemOptimizerNLopt, NLoptConstraint + +include("diff/NLopt.jl") +include("optimizer/NLopt.jl") + +end diff --git a/src/diff/NLopt.jl b/ext/diff/NLopt.jl similarity index 76% rename from src/diff/NLopt.jl rename to ext/diff/NLopt.jl index f0e4cea5b..8267cf4bc 100644 --- a/src/diff/NLopt.jl +++ b/ext/diff/NLopt.jl @@ -9,10 +9,10 @@ Connects to `NLopt.jl` as the optimization backend. SemOptimizerNLopt(; algorithm = :LD_LBFGS, options = Dict{Symbol, Any}(), - local_algorithm = nothing, - local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), + local_algorithm = nothing, + local_options = Dict{Symbol, Any}(), + equality_constraints = Vector{NLoptConstraint}(), + inequality_constraints = Vector{NLoptConstraint}(), kwargs...) # Arguments @@ -37,9 +37,9 @@ my_constrained_optimizer = SemOptimizerNLopt(; ``` # Usage -All algorithms and options from the NLopt library are available, for more information see +All algorithms and options from the NLopt library are available, for more information see the NLopt.jl package and the NLopt online documentation. -For information on how to use inequality and equality constraints, +For information on how to use inequality and equality constraints, see [Constrained optimization](@ref) in our online documentation. # Extended help @@ -65,11 +65,16 @@ struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} inequality_constraints::C end -Base.@kwdef mutable struct NLoptConstraint +Base.@kwdef struct NLoptConstraint f::Any tol = 0.0 end +Base.convert( + ::Type{NLoptConstraint}, + tuple::NamedTuple{(:f, :tol), Tuple{F, T}}, +) where {F, T} = NLoptConstraint(tuple.f, tuple.tol) + ############################################################################################ ### Constructor ############################################################################################ @@ -83,35 +88,37 @@ function SemOptimizerNLopt(; inequality_constraints = Vector{NLoptConstraint}(), kwargs..., ) - applicable(iterate, equality_constraints) || + applicable(iterate, equality_constraints) && !isa(equality_constraints, NamedTuple) || (equality_constraints = [equality_constraints]) - applicable(iterate, inequality_constraints) || + applicable(iterate, inequality_constraints) && + !isa(inequality_constraints, NamedTuple) || (inequality_constraints = [inequality_constraints]) return SemOptimizerNLopt( algorithm, local_algorithm, options, local_options, - equality_constraints, - inequality_constraints, + convert.(NLoptConstraint, equality_constraints), + convert.(NLoptConstraint, inequality_constraints), ) end -SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) +SEM.SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) ############################################################################################ ### Recommended methods ############################################################################################ -update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) = optimizer +SEM.update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) = + optimizer ############################################################################################ ### additional methods ############################################################################################ -algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm +SEM.algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm local_algorithm(optimizer::SemOptimizerNLopt) = optimizer.local_algorithm -options(optimizer::SemOptimizerNLopt) = optimizer.options +SEM.options(optimizer::SemOptimizerNLopt) = optimizer.options local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints diff --git a/src/optimizer/NLopt.jl b/ext/optimizer/NLopt.jl similarity index 84% rename from src/optimizer/NLopt.jl rename to ext/optimizer/NLopt.jl index 6b03a676c..1abdac053 100644 --- a/src/optimizer/NLopt.jl +++ b/ext/optimizer/NLopt.jl @@ -7,9 +7,9 @@ mutable struct NLoptResult problem::Any end -optimizer(res::NLoptResult) = res.problem.algorithm -n_iterations(res::NLoptResult) = res.problem.numevals -convergence(res::NLoptResult) = res.result[3] +SEM.optimizer(res::NLoptResult) = res.problem.algorithm +SEM.n_iterations(res::NLoptResult) = res.problem.numevals +SEM.convergence(res::NLoptResult) = res.result[3] # construct SemFit from fitted NLopt object function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, opt) @@ -23,7 +23,7 @@ function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, opt) end # sem_fit method -function sem_fit( +function SEM.sem_fit( optimizer::SemOptimizerNLopt, model::AbstractSem, start_params::AbstractVector; @@ -38,7 +38,7 @@ function sem_fit( ) set_NLopt_constraints!(opt, model.optimizer) opt.min_objective = - (par, G) -> evaluate!( + (par, G) -> SEM.evaluate!( eltype(par), !isnothing(G) && !isempty(G) ? G : nothing, nothing, @@ -68,19 +68,19 @@ end function construct_NLopt_problem(algorithm, options, npar) opt = Opt(algorithm, npar) - for key in keys(options) - setproperty!(opt, key, options[key]) + for (key, val) in pairs(options) + setproperty!(opt, key, val) end return opt end -function set_NLopt_constraints!(opt, optimizer::SemOptimizerNLopt) +function set_NLopt_constraints!(opt::Opt, optimizer::SemOptimizerNLopt) for con in optimizer.inequality_constraints - inequality_constraint!(opt::Opt, con.f, con.tol) + inequality_constraint!(opt, con.f, con.tol) end for con in optimizer.equality_constraints - equality_constraint!(opt::Opt, con.f, con.tol) + equality_constraint!(opt, con.f, con.tol) end end diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 3f68dd95f..ec2abf31c 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -7,7 +7,6 @@ using LinearAlgebra, StatsBase, SparseArrays, Symbolics, - NLopt, FiniteDiff, PrettyTables, Distributions, @@ -62,12 +61,10 @@ include("loss/WLS/WLS.jl") include("loss/constant/constant.jl") # optimizer include("diff/optim.jl") -include("diff/NLopt.jl") include("diff/Empty.jl") # optimizer include("optimizer/documentation.jl") include("optimizer/optim.jl") -include("optimizer/NLopt.jl") # helper functions include("additional_functions/helper.jl") include("additional_functions/start_val/start_fabin3.jl") @@ -119,8 +116,6 @@ export AbstractSem, SemOptimizer, SemOptimizerEmpty, SemOptimizerOptim, - SemOptimizerNLopt, - NLoptConstraint, optimizer, n_iterations, convergence, From 730eadccbfbd760eaf29ecff5f04fe5031e43228 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Nov 2024 18:07:38 -0800 Subject: [PATCH 139/364] NLopt: sem_fit(): use provided optimizer --- ext/optimizer/NLopt.jl | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/ext/optimizer/NLopt.jl b/ext/optimizer/NLopt.jl index 1abdac053..94da98361 100644 --- a/ext/optimizer/NLopt.jl +++ b/ext/optimizer/NLopt.jl @@ -24,32 +24,28 @@ end # sem_fit method function SEM.sem_fit( - optimizer::SemOptimizerNLopt, + optim::SemOptimizerNLopt, model::AbstractSem, start_params::AbstractVector; kwargs..., ) # construct the NLopt problem - opt = construct_NLopt_problem( - model.optimizer.algorithm, - model.optimizer.options, - length(start_params), - ) - set_NLopt_constraints!(opt, model.optimizer) + opt = construct_NLopt_problem(optim.algorithm, optim.options, length(start_params)) + set_NLopt_constraints!(opt, optim) opt.min_objective = (par, G) -> SEM.evaluate!( - eltype(par), + zero(eltype(par)), !isnothing(G) && !isempty(G) ? G : nothing, nothing, model, par, ) - if !isnothing(model.optimizer.local_algorithm) + if !isnothing(optim.local_algorithm) opt_local = construct_NLopt_problem( - model.optimizer.local_algorithm, - model.optimizer.local_options, + optim.local_algorithm, + optim.local_options, length(start_params), ) opt.local_optimizer = opt_local From 23e226500c217932a543f758c3cc0cabe454766c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 14:46:34 -0800 Subject: [PATCH 140/364] SEMProximalOptExt for Proximal opt --- Project.toml | 4 ++ ext/SEMProximalOptExt.jl | 15 ++++++++ ext/diff/Proximal.jl | 39 +++++++++++++++++++ ext/optimizer/ProximalAlgorithms.jl | 59 +++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 ext/SEMProximalOptExt.jl create mode 100644 ext/diff/Proximal.jl create mode 100644 ext/optimizer/ProximalAlgorithms.jl diff --git a/Project.toml b/Project.toml index 21bd43814..1bd335f19 100644 --- a/Project.toml +++ b/Project.toml @@ -46,6 +46,10 @@ test = ["Test"] [weakdeps] NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" +ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" +ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" +ProximalOperators = "f3b72e0c-5f3e-4b3e-8f3e-3f4f3e3e3e3e" [extensions] SEMNLOptExt = "NLopt" +SEMProximalOptExt = ["ProximalCore", "ProximalAlgorithms", "ProximalOperators"] diff --git a/ext/SEMProximalOptExt.jl b/ext/SEMProximalOptExt.jl new file mode 100644 index 000000000..fb9f3c410 --- /dev/null +++ b/ext/SEMProximalOptExt.jl @@ -0,0 +1,15 @@ +module SEMProximalOptExt + +using StructuralEquationModels +using ProximalCore, ProximalAlgorithms, ProximalOperators + +export SemOptimizerProximal + +SEM = StructuralEquationModels + +#ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) + +include("diff/Proximal.jl") +include("optimizer/ProximalAlgorithms.jl") + +end diff --git a/ext/diff/Proximal.jl b/ext/diff/Proximal.jl new file mode 100644 index 000000000..9c84c725a --- /dev/null +++ b/ext/diff/Proximal.jl @@ -0,0 +1,39 @@ +mutable struct SemOptimizerProximal{A, B, C, D} <: SemOptimizer{:Proximal} + algorithm::A + options::B + operator_g::C + operator_h::D +end + +SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) + +SemOptimizerProximal(; + algorithm = ProximalAlgorithms.PANOC(), + options = Dict{Symbol, Any}(), + operator_g, + operator_h = nothing, + kwargs..., +) = SemOptimizerProximal(algorithm, options, operator_g, operator_h) + +############################################################################################ +### Recommended methods +############################################################################################ + +SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwargs...) = + optimizer + +############################################################################################ +### additional methods +############################################################################################ + +SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm +SEM.options(optimizer::SemOptimizerProximal) = optimizer.options + +############################################################################ +### Pretty Printing +############################################################################ + +function Base.show(io::IO, struct_inst::SemOptimizerProximal) + print_type_name(io, struct_inst) + print_field_types(io, struct_inst) +end diff --git a/ext/optimizer/ProximalAlgorithms.jl b/ext/optimizer/ProximalAlgorithms.jl new file mode 100644 index 000000000..379b0a209 --- /dev/null +++ b/ext/optimizer/ProximalAlgorithms.jl @@ -0,0 +1,59 @@ +## connect do ProximalAlgorithms.jl as backend +ProximalCore.gradient!(grad, model::AbstractSem, parameters) = + objective_gradient!(grad, model::AbstractSem, parameters) + +mutable struct ProximalResult + result::Any +end + +function SEM.sem_fit( + optim::SemOptimizerProximal, + model::AbstractSem, + start_params::AbstractVector; + kwargs..., +) + if isnothing(optim.operator_h) + solution, iterations = + optim.algorithm(x0 = start_params, f = model, g = optim.operator_g) + else + solution, iterations = optim.algorithm( + x0 = start_params, + f = model, + g = optim.operator_g, + h = optim.operator_h, + ) + end + + minimum = objective!(model, solution) + + optimization_result = Dict( + :minimum => minimum, + :iterations => iterations, + :algorithm => optim.algorithm, + :operator_g => optim.operator_g, + ) + + isnothing(optim.operator_h) || + push!(optimization_result, :operator_h => optim.operator_h) + + return SemFit( + minimum, + solution, + start_params, + model, + ProximalResult(optimization_result), + ) +end + +############################################################################################ +# pretty printing +############################################################################################ + +function Base.show(io::IO, result::ProximalResult) + print(io, "Minimum: $(round(result.result[:minimum]; digits = 2)) \n") + print(io, "No. evaluations: $(result.result[:iterations]) \n") + print(io, "Operator: $(nameof(typeof(result.result[:operator_g]))) \n") + if haskey(result.result, :operator_h) + print(io, "Second Operator: $(nameof(typeof(result.result[:operator_h]))) \n") + end +end From 8a98831cb10aa6f2287a2883d88b7a11f7dbc4de Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 16:22:27 -0800 Subject: [PATCH 141/364] merge diff/*.jl optimizer code into optimizer/*.jl --- ext/SEMNLOptExt.jl | 1 - ext/SEMProximalOptExt.jl | 1 - ext/diff/NLopt.jl | 124 ---------------------------- ext/diff/Proximal.jl | 39 --------- ext/optimizer/NLopt.jl | 123 ++++++++++++++++++++++++++- ext/optimizer/ProximalAlgorithms.jl | 61 ++++++++++++++ src/StructuralEquationModels.jl | 4 +- src/diff/optim.jl | 71 ---------------- src/{diff => optimizer}/Empty.jl | 0 src/optimizer/optim.jl | 74 +++++++++++++++++ 10 files changed, 258 insertions(+), 240 deletions(-) delete mode 100644 ext/diff/NLopt.jl delete mode 100644 ext/diff/Proximal.jl delete mode 100644 src/diff/optim.jl rename src/{diff => optimizer}/Empty.jl (100%) diff --git a/ext/SEMNLOptExt.jl b/ext/SEMNLOptExt.jl index dfc3bbb42..a727b82f1 100644 --- a/ext/SEMNLOptExt.jl +++ b/ext/SEMNLOptExt.jl @@ -6,7 +6,6 @@ SEM = StructuralEquationModels export SemOptimizerNLopt, NLoptConstraint -include("diff/NLopt.jl") include("optimizer/NLopt.jl") end diff --git a/ext/SEMProximalOptExt.jl b/ext/SEMProximalOptExt.jl index fb9f3c410..e81760acb 100644 --- a/ext/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt.jl @@ -9,7 +9,6 @@ SEM = StructuralEquationModels #ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) -include("diff/Proximal.jl") include("optimizer/ProximalAlgorithms.jl") end diff --git a/ext/diff/NLopt.jl b/ext/diff/NLopt.jl deleted file mode 100644 index 8267cf4bc..000000000 --- a/ext/diff/NLopt.jl +++ /dev/null @@ -1,124 +0,0 @@ -############################################################################################ -### Types -############################################################################################ -""" -Connects to `NLopt.jl` as the optimization backend. - -# Constructor - - SemOptimizerNLopt(; - algorithm = :LD_LBFGS, - options = Dict{Symbol, Any}(), - local_algorithm = nothing, - local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), - kwargs...) - -# Arguments -- `algorithm`: optimization algorithm. -- `options::Dict{Symbol, Any}`: options for the optimization algorithm -- `local_algorithm`: local optimization algorithm -- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm -- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints -- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints - -# Example -```julia -my_optimizer = SemOptimizerNLopt() - -# constrained optimization with augmented lagrangian -my_constrained_optimizer = SemOptimizerNLopt(; - algorithm = :AUGLAG, - local_algorithm = :LD_LBFGS, - local_options = Dict(:ftol_rel => 1e-6), - inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0), -) -``` - -# Usage -All algorithms and options from the NLopt library are available, for more information see -the NLopt.jl package and the NLopt online documentation. -For information on how to use inequality and equality constraints, -see [Constrained optimization](@ref) in our online documentation. - -# Extended help - -## Interfaces -- `algorithm(::SemOptimizerNLopt)` -- `local_algorithm(::SemOptimizerNLopt)` -- `options(::SemOptimizerNLopt)` -- `local_options(::SemOptimizerNLopt)` -- `equality_constraints(::SemOptimizerNLopt)` -- `inequality_constraints(::SemOptimizerNLopt)` - -## Implementation - -Subtype of `SemOptimizer`. -""" -struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} - algorithm::A - local_algorithm::A2 - options::B - local_options::B2 - equality_constraints::C - inequality_constraints::C -end - -Base.@kwdef struct NLoptConstraint - f::Any - tol = 0.0 -end - -Base.convert( - ::Type{NLoptConstraint}, - tuple::NamedTuple{(:f, :tol), Tuple{F, T}}, -) where {F, T} = NLoptConstraint(tuple.f, tuple.tol) - -############################################################################################ -### Constructor -############################################################################################ - -function SemOptimizerNLopt(; - algorithm = :LD_LBFGS, - local_algorithm = nothing, - options = Dict{Symbol, Any}(), - local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), - kwargs..., -) - applicable(iterate, equality_constraints) && !isa(equality_constraints, NamedTuple) || - (equality_constraints = [equality_constraints]) - applicable(iterate, inequality_constraints) && - !isa(inequality_constraints, NamedTuple) || - (inequality_constraints = [inequality_constraints]) - return SemOptimizerNLopt( - algorithm, - local_algorithm, - options, - local_options, - convert.(NLoptConstraint, equality_constraints), - convert.(NLoptConstraint, inequality_constraints), - ) -end - -SEM.SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) - -############################################################################################ -### Recommended methods -############################################################################################ - -SEM.update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) = - optimizer - -############################################################################################ -### additional methods -############################################################################################ - -SEM.algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm -local_algorithm(optimizer::SemOptimizerNLopt) = optimizer.local_algorithm -SEM.options(optimizer::SemOptimizerNLopt) = optimizer.options -local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options -equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints -inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints diff --git a/ext/diff/Proximal.jl b/ext/diff/Proximal.jl deleted file mode 100644 index 9c84c725a..000000000 --- a/ext/diff/Proximal.jl +++ /dev/null @@ -1,39 +0,0 @@ -mutable struct SemOptimizerProximal{A, B, C, D} <: SemOptimizer{:Proximal} - algorithm::A - options::B - operator_g::C - operator_h::D -end - -SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) - -SemOptimizerProximal(; - algorithm = ProximalAlgorithms.PANOC(), - options = Dict{Symbol, Any}(), - operator_g, - operator_h = nothing, - kwargs..., -) = SemOptimizerProximal(algorithm, options, operator_g, operator_h) - -############################################################################################ -### Recommended methods -############################################################################################ - -SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwargs...) = - optimizer - -############################################################################################ -### additional methods -############################################################################################ - -SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm -SEM.options(optimizer::SemOptimizerProximal) = optimizer.options - -############################################################################ -### Pretty Printing -############################################################################ - -function Base.show(io::IO, struct_inst::SemOptimizerProximal) - print_type_name(io, struct_inst) - print_field_types(io, struct_inst) -end diff --git a/ext/optimizer/NLopt.jl b/ext/optimizer/NLopt.jl index 94da98361..959380292 100644 --- a/ext/optimizer/NLopt.jl +++ b/ext/optimizer/NLopt.jl @@ -1,6 +1,127 @@ ############################################################################################ -### connect to NLopt.jl as backend +### Types ############################################################################################ +""" +Connects to `NLopt.jl` as the optimization backend. + +# Constructor + + SemOptimizerNLopt(; + algorithm = :LD_LBFGS, + options = Dict{Symbol, Any}(), + local_algorithm = nothing, + local_options = Dict{Symbol, Any}(), + equality_constraints = Vector{NLoptConstraint}(), + inequality_constraints = Vector{NLoptConstraint}(), + kwargs...) + +# Arguments +- `algorithm`: optimization algorithm. +- `options::Dict{Symbol, Any}`: options for the optimization algorithm +- `local_algorithm`: local optimization algorithm +- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm +- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints +- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints + +# Example +```julia +my_optimizer = SemOptimizerNLopt() + +# constrained optimization with augmented lagrangian +my_constrained_optimizer = SemOptimizerNLopt(; + algorithm = :AUGLAG, + local_algorithm = :LD_LBFGS, + local_options = Dict(:ftol_rel => 1e-6), + inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0), +) +``` + +# Usage +All algorithms and options from the NLopt library are available, for more information see +the NLopt.jl package and the NLopt online documentation. +For information on how to use inequality and equality constraints, +see [Constrained optimization](@ref) in our online documentation. + +# Extended help + +## Interfaces +- `algorithm(::SemOptimizerNLopt)` +- `local_algorithm(::SemOptimizerNLopt)` +- `options(::SemOptimizerNLopt)` +- `local_options(::SemOptimizerNLopt)` +- `equality_constraints(::SemOptimizerNLopt)` +- `inequality_constraints(::SemOptimizerNLopt)` + +## Implementation + +Subtype of `SemOptimizer`. +""" +struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} + algorithm::A + local_algorithm::A2 + options::B + local_options::B2 + equality_constraints::C + inequality_constraints::C +end + +Base.@kwdef struct NLoptConstraint + f::Any + tol = 0.0 +end + +Base.convert( + ::Type{NLoptConstraint}, + tuple::NamedTuple{(:f, :tol), Tuple{F, T}}, +) where {F, T} = NLoptConstraint(tuple.f, tuple.tol) + +############################################################################################ +### Constructor +############################################################################################ + +function SemOptimizerNLopt(; + algorithm = :LD_LBFGS, + local_algorithm = nothing, + options = Dict{Symbol, Any}(), + local_options = Dict{Symbol, Any}(), + equality_constraints = Vector{NLoptConstraint}(), + inequality_constraints = Vector{NLoptConstraint}(), + kwargs..., +) + applicable(iterate, equality_constraints) && !isa(equality_constraints, NamedTuple) || + (equality_constraints = [equality_constraints]) + applicable(iterate, inequality_constraints) && + !isa(inequality_constraints, NamedTuple) || + (inequality_constraints = [inequality_constraints]) + return SemOptimizerNLopt( + algorithm, + local_algorithm, + options, + local_options, + convert.(NLoptConstraint, equality_constraints), + convert.(NLoptConstraint, inequality_constraints), + ) +end + +SEM.SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) + +############################################################################################ +### Recommended methods +############################################################################################ + +SEM.update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) = + optimizer + +############################################################################################ +### additional methods +############################################################################################ + +SEM.algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm +local_algorithm(optimizer::SemOptimizerNLopt) = optimizer.local_algorithm +SEM.options(optimizer::SemOptimizerNLopt) = optimizer.options +local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options +equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints +inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints mutable struct NLoptResult result::Any diff --git a/ext/optimizer/ProximalAlgorithms.jl b/ext/optimizer/ProximalAlgorithms.jl index 379b0a209..8d7cc5b2d 100644 --- a/ext/optimizer/ProximalAlgorithms.jl +++ b/ext/optimizer/ProximalAlgorithms.jl @@ -1,3 +1,64 @@ +############################################################################################ +### Types +############################################################################################ +""" +Connects to `ProximalAlgorithms.jl` as the optimization backend. + +# Constructor + + SemOptimizerProximal(; + algorithm = ProximalAlgorithms.PANOC(), + options = Dict{Symbol, Any}(), + operator_g, + operator_h = nothing, + kwargs..., + +# Arguments +- `algorithm`: optimization algorithm. +- `options::Dict{Symbol, Any}`: options for the optimization algorithm +- `operator_g`: gradient of the objective function +- `operator_h`: optional hessian of the objective function +""" +mutable struct SemOptimizerProximal{A, B, C, D} <: SemOptimizer{:Proximal} + algorithm::A + options::B + operator_g::C + operator_h::D +end + +SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) + +SemOptimizerProximal(; + algorithm = ProximalAlgorithms.PANOC(), + options = Dict{Symbol, Any}(), + operator_g, + operator_h = nothing, + kwargs..., +) = SemOptimizerProximal(algorithm, options, operator_g, operator_h) + +############################################################################################ +### Recommended methods +############################################################################################ + +SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwargs...) = + optimizer + +############################################################################################ +### additional methods +############################################################################################ + +SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm +SEM.options(optimizer::SemOptimizerProximal) = optimizer.options + +############################################################################ +### Pretty Printing +############################################################################ + +function Base.show(io::IO, struct_inst::SemOptimizerProximal) + print_type_name(io, struct_inst) + print_field_types(io, struct_inst) +end + ## connect do ProximalAlgorithms.jl as backend ProximalCore.gradient!(grad, model::AbstractSem, parameters) = objective_gradient!(grad, model::AbstractSem, parameters) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index ec2abf31c..ca1ae61f0 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -60,10 +60,8 @@ include("loss/regularization/ridge.jl") include("loss/WLS/WLS.jl") include("loss/constant/constant.jl") # optimizer -include("diff/optim.jl") -include("diff/Empty.jl") -# optimizer include("optimizer/documentation.jl") +include("optimizer/Empty.jl") include("optimizer/optim.jl") # helper functions include("additional_functions/helper.jl") diff --git a/src/diff/optim.jl b/src/diff/optim.jl deleted file mode 100644 index 5b8845275..000000000 --- a/src/diff/optim.jl +++ /dev/null @@ -1,71 +0,0 @@ -############################################################################################ -### Types and Constructor -############################################################################################ -""" -Connects to `Optim.jl` as the optimization backend. - -# Constructor - - SemOptimizerOptim(; - algorithm = LBFGS(), - options = Optim.Options(;f_tol = 1e-10, x_tol = 1.5e-8), - kwargs...) - -# Arguments -- `algorithm`: optimization algorithm. -- `options::Optim.Options`: options for the optimization algorithm - -# Usage -All algorithms and options from the Optim.jl library are available, for more information see -the Optim.jl online documentation. - -# Examples -```julia -my_optimizer = SemOptimizerOptim() - -# hessian based optimization with backtracking linesearch and modified initial step size -using Optim, LineSearches - -my_newton_optimizer = SemOptimizerOptim( - algorithm = Newton( - ;linesearch = BackTracking(order=3), - alphaguess = InitialHagerZhang() - ) -) -``` - -# Extended help - -## Interfaces -- `algorithm(::SemOptimizerOptim)` -- `options(::SemOptimizerOptim)` - -## Implementation - -Subtype of `SemOptimizer`. -""" -mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} - algorithm::A - options::B -end - -SemOptimizer{:Optim}(args...; kwargs...) = SemOptimizerOptim(args...; kwargs...) - -SemOptimizerOptim(; - algorithm = LBFGS(), - options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), - kwargs..., -) = SemOptimizerOptim(algorithm, options) - -############################################################################################ -### Recommended methods -############################################################################################ - -update_observed(optimizer::SemOptimizerOptim, observed::SemObserved; kwargs...) = optimizer - -############################################################################################ -### additional methods -############################################################################################ - -algorithm(optimizer::SemOptimizerOptim) = optimizer.algorithm -options(optimizer::SemOptimizerOptim) = optimizer.options diff --git a/src/diff/Empty.jl b/src/optimizer/Empty.jl similarity index 100% rename from src/diff/Empty.jl rename to src/optimizer/Empty.jl diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 19623b965..4031f2e4a 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -1,5 +1,79 @@ ## connect to Optim.jl as backend +############################################################################################ +### Types and Constructor +############################################################################################ +""" + SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} + +Connects to `Optim.jl` as the optimization backend. + +# Constructor + + SemOptimizerOptim(; + algorithm = LBFGS(), + options = Optim.Options(;f_tol = 1e-10, x_tol = 1.5e-8), + kwargs...) + +# Arguments +- `algorithm`: optimization algorithm. +- `options::Optim.Options`: options for the optimization algorithm + +# Usage +All algorithms and options from the Optim.jl library are available, for more information see +the Optim.jl online documentation. + +# Examples +```julia +my_optimizer = SemOptimizerOptim() + +# hessian based optimization with backtracking linesearch and modified initial step size +using Optim, LineSearches + +my_newton_optimizer = SemOptimizerOptim( + algorithm = Newton( + ;linesearch = BackTracking(order=3), + alphaguess = InitialHagerZhang() + ) +) +``` + +# Extended help + +## Interfaces +- `algorithm(::SemOptimizerOptim)` +- `options(::SemOptimizerOptim)` + +## Implementation + +Subtype of `SemOptimizer`. +""" +mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} + algorithm::A + options::B +end + +SemOptimizer{:Optim}(args...; kwargs...) = SemOptimizerOptim(args...; kwargs...) + +SemOptimizerOptim(; + algorithm = LBFGS(), + options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), + kwargs..., +) = SemOptimizerOptim(algorithm, options) + +############################################################################################ +### Recommended methods +############################################################################################ + +update_observed(optimizer::SemOptimizerOptim, observed::SemObserved; kwargs...) = optimizer + +############################################################################################ +### additional methods +############################################################################################ + +algorithm(optimizer::SemOptimizerOptim) = optimizer.algorithm +options(optimizer::SemOptimizerOptim) = optimizer.options + function SemFit( optimization_result::Optim.MultivariateOptimizationResults, model::AbstractSem, From 9e2672dcb6aabc9634f13e3e566e3a975775a782 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 20 Dec 2024 15:05:44 -0800 Subject: [PATCH 142/364] Optim: document u/l bounds --- src/optimizer/optim.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 4031f2e4a..cec37a77a 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -40,6 +40,16 @@ my_newton_optimizer = SemOptimizerOptim( # Extended help +## Constrained optimization + +When using the `Fminbox` or `SAMIN` constrained optimization algorithms, +the vector or dictionary of lower and upper bounds for each model parameter can be specified +via `lower_bounds` and `upper_bounds` keyword arguments. +Alternatively, the `lower_bound` and `upper_bound` keyword arguments can be used to specify +the default bound for all non-variance model parameters, +and the `variance_lower_bound` and `variance_upper_bound` keyword -- +for the variance parameters (the diagonal of the *S* matrix). + ## Interfaces - `algorithm(::SemOptimizerOptim)` - `options(::SemOptimizerOptim)` From d6188981543669d649b668394782638091665013 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 19 Dec 2024 15:00:34 +0100 Subject: [PATCH 143/364] remove unused options field from Proximal optimizer --- ext/optimizer/ProximalAlgorithms.jl | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/ext/optimizer/ProximalAlgorithms.jl b/ext/optimizer/ProximalAlgorithms.jl index 8d7cc5b2d..13debf79d 100644 --- a/ext/optimizer/ProximalAlgorithms.jl +++ b/ext/optimizer/ProximalAlgorithms.jl @@ -8,33 +8,29 @@ Connects to `ProximalAlgorithms.jl` as the optimization backend. SemOptimizerProximal(; algorithm = ProximalAlgorithms.PANOC(), - options = Dict{Symbol, Any}(), operator_g, operator_h = nothing, kwargs..., # Arguments - `algorithm`: optimization algorithm. -- `options::Dict{Symbol, Any}`: options for the optimization algorithm - `operator_g`: gradient of the objective function - `operator_h`: optional hessian of the objective function """ -mutable struct SemOptimizerProximal{A, B, C, D} <: SemOptimizer{:Proximal} +mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} algorithm::A - options::B - operator_g::C - operator_h::D + operator_g::B + operator_h::C end SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) SemOptimizerProximal(; algorithm = ProximalAlgorithms.PANOC(), - options = Dict{Symbol, Any}(), operator_g, operator_h = nothing, kwargs..., -) = SemOptimizerProximal(algorithm, options, operator_g, operator_h) +) = SemOptimizerProximal(algorithm, operator_g, operator_h) ############################################################################################ ### Recommended methods @@ -48,7 +44,6 @@ SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwar ############################################################################################ SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm -SEM.options(optimizer::SemOptimizerProximal) = optimizer.options ############################################################################ ### Pretty Printing From d055c78b430ed24e6f9aceb2be3fc96672c67750 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 21:38:12 -0800 Subject: [PATCH 144/364] decouple optimizer from Sem model Co-authored-by: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- src/additional_functions/simulation.jl | 15 +--- .../start_val/start_fabin3.jl | 12 +-- .../start_val/start_simple.jl | 10 +-- src/frontend/fit/fitmeasures/chi2.jl | 7 +- src/frontend/fit/fitmeasures/minus2ll.jl | 19 ++-- src/frontend/specification/Sem.jl | 44 +++------- src/optimizer/documentation.jl | 26 +++--- src/types.jl | 62 ++++--------- test/examples/political_democracy/by_parts.jl | 65 +++++++------- .../political_democracy/constraints.jl | 19 +--- .../political_democracy/constructor.jl | 86 ++++++------------- .../recover_parameters_twofact.jl | 7 +- test/unit_tests/model.jl | 1 - 13 files changed, 130 insertions(+), 243 deletions(-) diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index f1e41f360..0b2626b15 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -47,7 +47,6 @@ swap_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = observed(model), imply(model), loss(model), - optimizer(model), new_observed; kwargs..., ) @@ -57,7 +56,6 @@ function swap_observed( old_observed, imply, loss, - optimizer, new_observed::SemObserved; kwargs..., ) @@ -68,7 +66,6 @@ function swap_observed( kwargs[:old_observed_type] = typeof(old_observed) kwargs[:imply_type] = typeof(imply) kwargs[:loss_types] = [typeof(lossfun) for lossfun in loss.functions] - kwargs[:optimizer_type] = typeof(optimizer) # update imply imply = update_observed(imply, new_observed; kwargs...) @@ -79,16 +76,12 @@ function swap_observed( loss = update_observed(loss, new_observed; kwargs...) kwargs[:loss] = loss - # update optimizer - optimizer = update_observed(optimizer, new_observed; kwargs...) - #new_imply = update_observed(model.imply, new_observed; kwargs...) return Sem( new_observed, update_observed(model.imply, new_observed; kwargs...), update_observed(model.loss, new_observed; kwargs...), - update_observed(model.optimizer, new_observed; kwargs...), ) end @@ -120,18 +113,18 @@ rand(model, start_simple(model), 100) ``` """ function Distributions.rand( - model::AbstractSemSingle{O, I, L, D}, + model::AbstractSemSingle{O, I, L}, params, n::Integer, -) where {O, I <: Union{RAM, RAMSymbolic}, L, D} +) where {O, I <: Union{RAM, RAMSymbolic}, L} update!(EvaluationTargets{true, false, false}(), model.imply, model, params) return rand(model, n) end function Distributions.rand( - model::AbstractSemSingle{O, I, L, D}, + model::AbstractSemSingle{O, I, L}, n::Integer, -) where {O, I <: Union{RAM, RAMSymbolic}, L, D} +) where {O, I <: Union{RAM, RAMSymbolic}, L} if MeanStruct(model.imply) === NoMeanStruct data = permutedims(rand(MvNormal(Symmetric(model.imply.Σ)), n)) elseif MeanStruct(model.imply) === HasMeanStruct diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 53cf7cff6..dd8d61fd9 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -8,21 +8,15 @@ function start_fabin3 end # splice model and loss functions function start_fabin3(model::AbstractSemSingle; kwargs...) - return start_fabin3( - model.observed, - model.imply, - model.optimizer, - model.loss.functions..., - kwargs..., - ) + return start_fabin3(model.observed, model.imply, model.loss.functions..., kwargs...) end -function start_fabin3(observed, imply, optimizer, args...; kwargs...) +function start_fabin3(observed, imply, args...; kwargs...) return start_fabin3(imply.ram_matrices, obs_cov(observed), obs_mean(observed)) end # SemObservedMissing -function start_fabin3(observed::SemObservedMissing, imply, optimizer, args...; kwargs...) +function start_fabin3(observed::SemObservedMissing, imply, args...; kwargs...) if !observed.em_model.fitted em_mvn(observed; kwargs...) end diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 1f73a3583..1f16b094c 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -17,16 +17,10 @@ function start_simple end # Single Models ---------------------------------------------------------------------------- function start_simple(model::AbstractSemSingle; kwargs...) - return start_simple( - model.observed, - model.imply, - model.optimizer, - model.loss.functions...; - kwargs..., - ) + return start_simple(model.observed, model.imply, model.loss.functions...; kwargs...) end -function start_simple(observed, imply, optimizer, args...; kwargs...) +function start_simple(observed, imply, args...; kwargs...) return start_simple(imply.ram_matrices; kwargs...) end diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index df1027bd6..12bc1d880 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -14,21 +14,20 @@ function χ² end sem_fit, sem_fit.model.observed, sem_fit.model.imply, - sem_fit.model.optimizer, sem_fit.model.loss.functions..., ) # RAM + SemML -χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) = +χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemML) = (nsamples(sem_fit) - 1) * (sem_fit.minimum - logdet(observed.obs_cov) - nobserved_vars(observed)) # bollen, p. 115, only correct for GLS weight matrix -χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) = +χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemWLS) = (nsamples(sem_fit) - 1) * sem_fit.minimum # FIML -function χ²(sem_fit::SemFit, observed::SemObservedMissing, imp, optimizer, loss_ml::SemFIML) +function χ²(sem_fit::SemFit, observed::SemObservedMissing, imp, loss_ml::SemFIML) ll_H0 = minus2ll(sem_fit) ll_H1 = minus2ll(observed) chi2 = ll_H0 - ll_H1 diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 88948d4d4..54a4ce12d 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -16,30 +16,21 @@ minus2ll( sem_fit, sem_fit.model.observed, sem_fit.model.imply, - sem_fit.model.optimizer, sem_fit.model.loss.functions..., ) -minus2ll(sem_fit::SemFit, obs, imp, optimizer, args...) = - minus2ll(sem_fit.minimum, obs, imp, optimizer, args...) +minus2ll(sem_fit::SemFit, obs, imp, args...) = minus2ll(sem_fit.minimum, obs, imp, args...) # SemML ------------------------------------------------------------------------------------ -minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) = +minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, loss_ml::SemML) = nsamples(obs) * (minimum + log(2π) * nobserved_vars(obs)) # WLS -------------------------------------------------------------------------------------- -minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) = - missing +minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, loss_ml::SemWLS) = missing # compute likelihood for missing data - H0 ------------------------------------------------- # -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n -function minus2ll( - minimum::Number, - observed, - imp::Union{RAM, RAMSymbolic}, - optimizer, - loss_ml::SemFIML, -) +function minus2ll(minimum::Number, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemFIML) F = minimum F *= nsamples(observed) F += sum(log(2π) * observed.pattern_nsamples .* observed.pattern_nobs_vars) @@ -117,7 +108,7 @@ end ############################################################################################ minus2ll(minimum, model::AbstractSemSingle) = - minus2ll(minimum, model.observed, model.imply, model.optimizer, model.loss.functions...) + minus2ll(minimum, model.observed, model.imply, model.loss.functions...) function minus2ll( sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}, diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 758bc073d..741d5f3c6 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -6,16 +6,15 @@ function Sem(; observed::O = SemObservedData, imply::I = RAM, loss::L = SemML, - optimizer::D = SemOptimizerOptim, kwargs..., -) where {O, I, L, D} +) where {O, I, L} kwdict = Dict{Symbol, Any}(kwargs...) - set_field_type_kwargs!(kwdict, observed, imply, loss, optimizer, O, I, D) + set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) - observed, imply, loss, optimizer = get_fields!(kwdict, observed, imply, loss, optimizer) + observed, imply, loss = get_fields!(kwdict, observed, imply, loss) - sem = Sem(observed, imply, loss, optimizer) + sem = Sem(observed, imply, loss) return sem end @@ -59,27 +58,19 @@ Returns the loss part of a model. """ loss(model::AbstractSemSingle) = model.loss -""" - optimizer(model::AbstractSemSingle) -> SemOptimizer - -Returns the optimizer part of a model. -""" -optimizer(model::AbstractSemSingle) = model.optimizer - function SemFiniteDiff(; observed::O = SemObservedData, imply::I = RAM, loss::L = SemML, - optimizer::D = SemOptimizerOptim, kwargs..., -) where {O, I, L, D} +) where {O, I, L} kwdict = Dict{Symbol, Any}(kwargs...) - set_field_type_kwargs!(kwdict, observed, imply, loss, optimizer, O, I, D) + set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) - observed, imply, loss, optimizer = get_fields!(kwdict, observed, imply, loss, optimizer) + observed, imply, loss = get_fields!(kwdict, observed, imply, loss) - sem = SemFiniteDiff(observed, imply, loss, optimizer) + sem = SemFiniteDiff(observed, imply, loss) return sem end @@ -88,7 +79,7 @@ end # functions ############################################################################################ -function set_field_type_kwargs!(kwargs, observed, imply, loss, optimizer, O, I, D) +function set_field_type_kwargs!(kwargs, observed, imply, loss, O, I) kwargs[:observed_type] = O <: Type ? observed : typeof(observed) kwargs[:imply_type] = I <: Type ? imply : typeof(imply) if loss isa SemLoss @@ -102,11 +93,10 @@ function set_field_type_kwargs!(kwargs, observed, imply, loss, optimizer, O, I, else kwargs[:loss_types] = [loss isa SemLossFunction ? typeof(loss) : loss] end - kwargs[:optimizer_type] = D <: Type ? optimizer : typeof(optimizer) end # construct Sem fields -function get_fields!(kwargs, observed, imply, loss, optimizer) +function get_fields!(kwargs, observed, imply, loss) # observed if !isa(observed, SemObserved) observed = observed(; kwargs...) @@ -125,12 +115,7 @@ function get_fields!(kwargs, observed, imply, loss, optimizer) loss = get_SemLoss(loss; kwargs...) kwargs[:loss] = loss - # optimizer - if !isa(optimizer, SemOptimizer) - optimizer = optimizer(; kwargs...) - end - - return observed, imply, loss, optimizer + return observed, imply, loss end # construct loss field @@ -167,7 +152,7 @@ end print(io, "Sem{$(nameof(O)), $(nameof(I)), $lossfuntypes, $(nameof(D))}") end =# -function Base.show(io::IO, sem::Sem{O, I, L, D}) where {O, I, L, D} +function Base.show(io::IO, sem::Sem{O, I, L}) where {O, I, L} lossfuntypes = @. string(nameof(typeof(sem.loss.functions))) lossfuntypes = " " .* lossfuntypes .* ("\n") print(io, "Structural Equation Model \n") @@ -176,10 +161,9 @@ function Base.show(io::IO, sem::Sem{O, I, L, D}) where {O, I, L, D} print(io, "- Fields \n") print(io, " observed: $(nameof(O)) \n") print(io, " imply: $(nameof(I)) \n") - print(io, " optimizer: $(nameof(D)) \n") end -function Base.show(io::IO, sem::SemFiniteDiff{O, I, L, D}) where {O, I, L, D} +function Base.show(io::IO, sem::SemFiniteDiff{O, I, L}) where {O, I, L} lossfuntypes = @. string(nameof(typeof(sem.loss.functions))) lossfuntypes = " " .* lossfuntypes .* ("\n") print(io, "Structural Equation Model : Finite Diff Approximation\n") @@ -188,7 +172,6 @@ function Base.show(io::IO, sem::SemFiniteDiff{O, I, L, D}) where {O, I, L, D} print(io, "- Fields \n") print(io, " observed: $(nameof(O)) \n") print(io, " imply: $(nameof(I)) \n") - print(io, " optimizer: $(nameof(D)) \n") end function Base.show(io::IO, loss::SemLoss) @@ -211,7 +194,6 @@ function Base.show(io::IO, models::SemEnsemble) print(io, "SemEnsemble \n") print(io, "- Number of Models: $(models.n) \n") print(io, "- Weights: $(round.(models.weights, digits = 2)) \n") - print(io, "- optimizer: $(nameof(typeof(optimizer(models)))) \n") print(io, "\n", "Models: \n") print(io, "===============================================", "\n") diff --git a/src/optimizer/documentation.jl b/src/optimizer/documentation.jl index cf6aaa312..c6669aa12 100644 --- a/src/optimizer/documentation.jl +++ b/src/optimizer/documentation.jl @@ -1,16 +1,22 @@ """ - sem_fit(model::AbstractSem; start_val = start_val, kwargs...) + sem_fit([optim::SemOptimizer], model::AbstractSem; + [engine::Symbol], start_val = start_val, kwargs...) Return the fitted `model`. # Arguments +- `optim`: [`SemOptimizer`](@ref) to use for fitting. + If omitted, a new optimizer is constructed as `SemOptimizer(; engine, kwargs...)`. - `model`: `AbstractSem` to fit +- `engine`: the optimization engine to use, default is `:Optim` - `start_val`: a vector or a dictionary of starting parameter values, or function to compute them (1) -- `kwargs...`: keyword arguments, passed to starting value functions +- `kwargs...`: keyword arguments, passed to optimization engine constructor and + `start_val` function (1) available functions are `start_fabin3`, `start_simple` and `start_partable`. -For more information, we refer to the individual documentations and the online documentation on [Starting values](@ref). +For more information, we refer to the individual documentations and +the online documentation on [Starting values](@ref). # Examples ```julia @@ -20,20 +26,20 @@ sem_fit( start_covariances_latent = 0.5) ``` """ -function sem_fit end - -# dispatch on optimizer -function sem_fit(model::AbstractSem; start_val = nothing, kwargs...) +function sem_fit(optim::SemOptimizer, model::AbstractSem; start_val = nothing, kwargs...) start_params = prepare_start_params(start_val, model; kwargs...) @assert start_params isa AbstractVector @assert length(start_params) == nparams(model) - sem_fit(model.optimizer, model, start_params; kwargs...) + sem_fit(optim, model, start_params; kwargs...) end +sem_fit(model::AbstractSem; engine::Symbol = :Optim, start_val = nothing, kwargs...) = + sem_fit(SemOptimizer(; engine, kwargs...), model; start_val, kwargs...) + # fallback method -sem_fit(optimizer::SemOptimizer, model::AbstractSem, start_params; kwargs...) = - error("Optimizer $(optimizer) support not implemented.") +sem_fit(optim::SemOptimizer, model::AbstractSem, start_params; kwargs...) = + error("Optimizer $(optim) support not implemented.") # FABIN3 is the default method for single models prepare_start_params(start_val::Nothing, model::AbstractSemSingle; kwargs...) = diff --git a/src/types.jl b/src/types.jl index 90b648ac8..cfe916d9e 100644 --- a/src/types.jl +++ b/src/types.jl @@ -4,8 +4,8 @@ "Most abstract supertype for all SEMs" abstract type AbstractSem end -"Supertype for all single SEMs, e.g. SEMs that have at least the fields `observed`, `imply`, `loss` and `optimizer`" -abstract type AbstractSemSingle{O, I, L, D} <: AbstractSem end +"Supertype for all single SEMs, e.g. SEMs that have at least the fields `observed`, `imply`, `loss`" +abstract type AbstractSemSingle{O, I, L} <: AbstractSem end "Supertype for all collections of multiple SEMs" abstract type AbstractSemCollection <: AbstractSem end @@ -116,73 +116,66 @@ abstract type SemImply end abstract type SemImplySymbolic <: SemImply end """ - Sem(;observed = SemObservedData, imply = RAM, loss = SemML, optimizer = SemOptimizerOptim, kwargs...) + Sem(;observed = SemObservedData, imply = RAM, loss = SemML, kwargs...) Constructor for the basic `Sem` type. -All additional kwargs are passed down to the constructors for the observed, imply, loss and optimizer fields. +All additional kwargs are passed down to the constructors for the observed, imply, and loss fields. # Arguments - `observed`: object of subtype `SemObserved` or a constructor. - `imply`: object of subtype `SemImply` or a constructor. - `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such. -- `optimizer`: object of subtype `SemOptimizer` or a constructor. Returns a Sem with fields - `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref). - `imply::SemImply`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImply`](@ref). - `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref). -- `optimizer::SemOptimizer`: Connects the model to the optimizer. See also [`SemOptimizer`](@ref). """ -mutable struct Sem{O <: SemObserved, I <: SemImply, L <: SemLoss, D <: SemOptimizer} <: - AbstractSemSingle{O, I, L, D} +mutable struct Sem{O <: SemObserved, I <: SemImply, L <: SemLoss} <: + AbstractSemSingle{O, I, L} observed::O imply::I loss::L - optimizer::D end ############################################################################################ # automatic differentiation ############################################################################################ """ - SemFiniteDiff(;observed = SemObservedData, imply = RAM, loss = SemML, optimizer = SemOptimizerOptim, kwargs...) + SemFiniteDiff(;observed = SemObservedData, imply = RAM, loss = SemML, kwargs...) -Constructor for `SemFiniteDiff`. -All additional kwargs are passed down to the constructors for the observed, imply, loss and optimizer fields. +A wrapper around [`Sem`](@ref) that substitutes dedicated evaluation of gradient and hessian with +finite difference approximation. # Arguments - `observed`: object of subtype `SemObserved` or a constructor. - `imply`: object of subtype `SemImply` or a constructor. - `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such. -- `optimizer`: object of subtype `SemOptimizer` or a constructor. Returns a Sem with fields - `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref). - `imply::SemImply`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImply`](@ref). - `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref). -- `optimizer::SemOptimizer`: Connects the model to the optimizer. See also [`SemOptimizer`](@ref). """ -struct SemFiniteDiff{O <: SemObserved, I <: SemImply, L <: SemLoss, D <: SemOptimizer} <: - AbstractSemSingle{O, I, L, D} +struct SemFiniteDiff{O <: SemObserved, I <: SemImply, L <: SemLoss} <: + AbstractSemSingle{O, I, L} observed::O imply::I loss::L - optimizer::D end ############################################################################################ # ensemble models ############################################################################################ """ - (1) SemEnsemble(models..., optimizer = SemOptimizerOptim, weights = nothing, kwargs...) + (1) SemEnsemble(models..., weights = nothing, kwargs...) - (2) SemEnsemble(;specification, data, groups, column = :group, optimizer = SemOptimizerOptim, kwargs...) + (2) SemEnsemble(;specification, data, groups, column = :group, kwargs...) Constructor for ensemble models. (2) can be used to conveniently specify multigroup models. # Arguments - `models...`: `AbstractSem`s. -- `optimizer`: object of subtype `SemOptimizer` or a constructor. - `weights::Vector`: Weights for each model. Defaults to the number of observed data points. - `specification::EnsembleParameterTable`: Model specification. - `data::DataFrame`: Observed data. Must contain a `column` of type `Vector{Symbol}` that contains the group. @@ -195,19 +188,17 @@ Returns a SemEnsemble with fields - `n::Int`: Number of models. - `sems::Tuple`: `AbstractSem`s. - `weights::Vector`: Weights for each model. -- `optimizer::SemOptimizer`: Connects the model to the optimizer. See also [`SemOptimizer`](@ref). - `params::Vector`: Stores parameter labels and their position. """ -struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, D, I} <: AbstractSemCollection +struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, I} <: AbstractSemCollection n::N sems::T weights::V - optimizer::D params::I end # constructor from multiple models -function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing, kwargs...) +function SemEnsemble(models...; weights = nothing, kwargs...) n = length(models) # default weights @@ -227,16 +218,11 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing end end - # optimizer - if !isa(optimizer, SemOptimizer) - optimizer = optimizer(; kwargs...) - end - - return SemEnsemble(n, models, weights, optimizer, params) + return SemEnsemble(n, models, weights, params) end # constructor from EnsembleParameterTable and data set -function SemEnsemble(;specification, data, groups, column = :group, optimizer = SemOptimizerOptim, kwargs...) +function SemEnsemble(; specification, data, groups, column = :group, kwargs...) if specification isa EnsembleParameterTable specification = convert(Dict{Symbol, RAMMatrices}, specification) end @@ -247,14 +233,10 @@ function SemEnsemble(;specification, data, groups, column = :group, optimizer = if iszero(nrow(data_group)) error("Your data does not contain any observations from group `$(group)`.") end - model = Sem(; - specification = ram_matrices, - data = data_group, - kwargs... - ) + model = Sem(; specification = ram_matrices, data = data_group, kwargs...) push!(models, model) end - return SemEnsemble(models...; optimizer = optimizer, weights = nothing, kwargs...) + return SemEnsemble(models...; weights = nothing, kwargs...) end params(ensemble::SemEnsemble) = ensemble.params @@ -277,12 +259,6 @@ models(ensemble::SemEnsemble) = ensemble.sems Returns the weights of an ensemble model. """ weights(ensemble::SemEnsemble) = ensemble.weights -""" - optimizer(ensemble::SemEnsemble) -> SemOptimizer - -Returns the optimizer part of an ensemble model. -""" -optimizer(ensemble::SemEnsemble) = ensemble.optimizer """ Base type for all SEM specifications. diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 87e5fb733..5e5244f91 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -29,23 +29,18 @@ optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- -model_ml = Sem(observed, imply_ram, loss_ml, optimizer_obj) +model_ml = Sem(observed, imply_ram, loss_ml) -model_ls_sym = - Sem(observed, RAMSymbolic(specification = spec, vech = true), loss_wls, optimizer_obj) +model_ls_sym = Sem(observed, RAMSymbolic(specification = spec, vech = true), loss_wls) -model_ml_sym = Sem(observed, imply_ram_sym, loss_ml, optimizer_obj) +model_ml_sym = Sem(observed, imply_ram_sym, loss_ml) -model_ridge = Sem(observed, imply_ram, SemLoss(ml, ridge), optimizer_obj) +model_ridge = Sem(observed, imply_ram, SemLoss(ml, ridge)) -model_constant = Sem(observed, imply_ram, SemLoss(ml, constant), optimizer_obj) +model_constant = Sem(observed, imply_ram, SemLoss(ml, constant)) -model_ml_weighted = Sem( - observed, - imply_ram, - SemLoss(ml; loss_weights = [nsamples(model_ml)]), - optimizer_obj, -) +model_ml_weighted = + Sem(observed, imply_ram, SemLoss(ml; loss_weights = [nsamples(model_ml)])) ############################################################################################ ### test gradients @@ -75,7 +70,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ls", "ml", "ml"]) for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution" begin - solution = sem_fit(model) + solution = sem_fit(optimizer_obj, model) update_estimate!(partable, solution) test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) end @@ -84,9 +79,9 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) end @testset "ridge_solution" begin - solution_ridge = sem_fit(model_ridge) - solution_ml = sem_fit(model_ml) - # solution_ridge_id = sem_fit(model_ridge_id) + solution_ridge = sem_fit(optimizer_obj, model_ridge) + solution_ml = sem_fit(optimizer_obj, model_ml) + # solution_ridge_id = sem_fit(optimizer_obj, model_ridge_id) @test solution_ridge.minimum < solution_ml.minimum + 1 end @@ -102,8 +97,8 @@ end end @testset "ml_solution_weighted" begin - solution_ml = sem_fit(model_ml) - solution_ml_weighted = sem_fit(model_ml_weighted) + solution_ml = sem_fit(optimizer_obj, model_ml) + solution_ml_weighted = sem_fit(optimizer_obj, model_ml_weighted) @test solution(solution_ml) ≈ solution(solution_ml_weighted) rtol = 1e-3 @test nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml) ≈ StructuralEquationModels.minimum(solution_ml_weighted) rtol = 1e-6 @@ -114,7 +109,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(optimizer_obj, model_ml) test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) update_se_hessian!(partable, solution_ml) @@ -128,7 +123,7 @@ end end @testset "fitmeasures/se_ls" begin - solution_ls = sem_fit(model_ls_sym) + solution_ls = sem_fit(optimizer_obj, model_ls_sym) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -167,10 +162,9 @@ if opt_engine == :Optim imply_sym_hessian = RAMSymbolic(specification = spec, hessian = true) - model_ls = Sem(observed, imply_sym_hessian_vech, loss_wls, optimizer_obj) + model_ls = Sem(observed, imply_sym_hessian_vech, loss_wls) - model_ml = - Sem(observed, imply_sym_hessian, loss_ml, SemOptimizerOptim(algorithm = Newton())) + model_ml = Sem(observed, imply_sym_hessian, loss_ml) @testset "ml_hessians" begin test_hessian(model_ml, start_test; atol = 1e-4) @@ -181,13 +175,13 @@ if opt_engine == :Optim end @testset "ml_solution_hessian" begin - solution = sem_fit(model_ml) + solution = sem_fit(optimizer_obj, model_ml) update_estimate!(partable, solution) test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) end @testset "ls_solution_hessian" begin - solution = sem_fit(model_ls) + solution = sem_fit(optimizer_obj, model_ls) update_estimate!(partable, solution) test_estimates( partable, @@ -224,16 +218,15 @@ loss_wls = SemLoss(wls) optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- -model_ml = Sem(observed, imply_ram, loss_ml, optimizer_obj) +model_ml = Sem(observed, imply_ram, loss_ml) model_ls = Sem( observed, RAMSymbolic(specification = spec_mean, meanstructure = true, vech = true), loss_wls, - optimizer_obj, ) -model_ml_sym = Sem(observed, imply_ram_sym, loss_ml, optimizer_obj) +model_ml_sym = Sem(observed, imply_ram_sym, loss_ml) ############################################################################################ ### test gradients @@ -260,7 +253,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ls", "ml"] .* "_mean" for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution_mean" begin - solution = sem_fit(model) + solution = sem_fit(optimizer_obj, model) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) end @@ -273,7 +266,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml_mean" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(optimizer_obj, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; @@ -291,7 +284,7 @@ end end @testset "fitmeasures/se_ls_mean" begin - solution_ls = sem_fit(model_ls) + solution_ls = sem_fit(optimizer_obj, model_ls) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -321,9 +314,9 @@ fiml = SemFIML(observed = observed, specification = spec_mean) loss_fiml = SemLoss(fiml) -model_ml = Sem(observed, imply_ram, loss_fiml, optimizer_obj) +model_ml = Sem(observed, imply_ram, loss_fiml) -model_ml_sym = Sem(observed, imply_ram_sym, loss_fiml, optimizer_obj) +model_ml_sym = Sem(observed, imply_ram_sym, loss_fiml) ############################################################################################ ### test gradients @@ -342,13 +335,13 @@ end ############################################################################################ @testset "fiml_solution" begin - solution = sem_fit(model_ml) + solution = sem_fit(optimizer_obj, model_ml) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @testset "fiml_solution_symbolic" begin - solution = sem_fit(model_ml_sym) + solution = sem_fit(optimizer_obj, model_ml_sym) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @@ -358,7 +351,7 @@ end ############################################################################################ @testset "fitmeasures/se_fiml" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(optimizer_obj, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; diff --git a/test/examples/political_democracy/constraints.jl b/test/examples/political_democracy/constraints.jl index 47f27582a..ef5692f27 100644 --- a/test/examples/political_democracy/constraints.jl +++ b/test/examples/political_democracy/constraints.jl @@ -30,33 +30,22 @@ constrained_optimizer = SemOptimizer(; inequality_constraints = (f = ineq_constraint, tol = 0.0), ) -model_ml_constrained = - Sem(specification = spec, data = dat, optimizer = constrained_optimizer) - -solution_constrained = sem_fit(model_ml_constrained) - # NLopt option setting --------------------------------------------------------------------- -model_ml_maxeval = Sem( - specification = spec, - data = dat, - optimizer = SemOptimizer, - engine = :NLopt, - options = Dict(:maxeval => 10), -) - ############################################################################################ ### test solution ############################################################################################ @testset "ml_solution_maxeval" begin - solution_maxeval = sem_fit(model_ml_maxeval) + solution_maxeval = sem_fit(model_ml, engine = :NLopt, options = Dict(:maxeval => 10)) + @test solution_maxeval.optimization_result.problem.numevals == 10 @test solution_maxeval.optimization_result.result[3] == :MAXEVAL_REACHED end @testset "ml_solution_constrained" begin - solution_constrained = sem_fit(model_ml_constrained) + solution_constrained = sem_fit(constrained_optimizer, model_ml) + @test solution_constrained.solution[31] * solution_constrained.solution[30] >= (0.6 - 1e-8) @test all(abs.(solution_constrained.solution) .< 10) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 5ed576dc1..cba86aef0 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -7,7 +7,7 @@ using Random, NLopt semoptimizer = SemOptimizer(engine = opt_engine) -model_ml = Sem(specification = spec, data = dat, optimizer = semoptimizer) +model_ml = Sem(specification = spec, data = dat) @test SEM.params(model_ml.imply.ram_matrices) == SEM.params(spec) model_ml_cov = Sem( @@ -15,20 +15,12 @@ model_ml_cov = Sem( observed = SemObservedCovariance, obs_cov = cov(Matrix(dat)), obs_colnames = Symbol.(names(dat)), - optimizer = semoptimizer, nsamples = 75, ) -model_ls_sym = Sem( - specification = spec, - data = dat, - imply = RAMSymbolic, - loss = SemWLS, - optimizer = semoptimizer, -) +model_ls_sym = Sem(specification = spec, data = dat, imply = RAMSymbolic, loss = SemWLS) -model_ml_sym = - Sem(specification = spec, data = dat, imply = RAMSymbolic, optimizer = semoptimizer) +model_ml_sym = Sem(specification = spec, data = dat, imply = RAMSymbolic) model_ridge = Sem( specification = spec, @@ -36,7 +28,6 @@ model_ridge = Sem( loss = (SemML, SemRidge), α_ridge = 0.001, which_ridge = 16:20, - optimizer = semoptimizer, ) model_constant = Sem( @@ -44,15 +35,10 @@ model_constant = Sem( data = dat, loss = (SemML, SemConstant), constant_loss = 3.465, - optimizer = semoptimizer, ) -model_ml_weighted = Sem( - specification = partable, - data = dat, - loss_weights = (nsamples(model_ml),), - optimizer = semoptimizer, -) +model_ml_weighted = + Sem(specification = partable, data = dat, loss_weights = (nsamples(model_ml),)) ############################################################################################ ### test gradients @@ -89,7 +75,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ml", "ls", "ml", "ml" for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution" begin - solution = sem_fit(model) + solution = sem_fit(semoptimizer, model) update_estimate!(partable, solution) test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) end @@ -98,9 +84,9 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) end @testset "ridge_solution" begin - solution_ridge = sem_fit(model_ridge) - solution_ml = sem_fit(model_ml) - # solution_ridge_id = sem_fit(model_ridge_id) + solution_ridge = sem_fit(semoptimizer, model_ridge) + solution_ml = sem_fit(semoptimizer, model_ml) + # solution_ridge_id = sem_fit(semoptimizer, model_ridge_id) @test abs(solution_ridge.minimum - solution_ml.minimum) < 1 end @@ -116,8 +102,8 @@ end end @testset "ml_solution_weighted" begin - solution_ml = sem_fit(model_ml) - solution_ml_weighted = sem_fit(model_ml_weighted) + solution_ml = sem_fit(semoptimizer, model_ml) + solution_ml_weighted = sem_fit(semoptimizer, model_ml_weighted) @test isapprox(solution(solution_ml), solution(solution_ml_weighted), rtol = 1e-3) @test isapprox( nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml), @@ -131,7 +117,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(semoptimizer, model_ml) test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) update_se_hessian!(partable, solution_ml) @@ -145,7 +131,7 @@ end end @testset "fitmeasures/se_ls" begin - solution_ls = sem_fit(model_ls_sym) + solution_ls = sem_fit(semoptimizer, model_ls_sym) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -196,8 +182,8 @@ end obs_colnames = colnames, ) # fit models - sol_ml = solution(sem_fit(model_ml_new)) - sol_ml_sym = solution(sem_fit(model_ml_sym_new)) + sol_ml = solution(sem_fit(semoptimizer, model_ml_new)) + sol_ml_sym = solution(sem_fit(semoptimizer, model_ml_sym_new)) # check solution @test maximum(abs.(sol_ml - params)) < 0.01 @test maximum(abs.(sol_ml_sym - params)) < 0.01 @@ -239,13 +225,13 @@ if opt_engine == :Optim end @testset "ml_solution_hessian" begin - solution = sem_fit(model_ml) + solution = sem_fit(semoptimizer, model_ml) update_estimate!(partable, solution) test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) end @testset "ls_solution_hessian" begin - solution = sem_fit(model_ls) + solution = sem_fit(semoptimizer, model_ls) update_estimate!(partable, solution) test_estimates( partable, @@ -268,15 +254,9 @@ model_ls = Sem( imply = RAMSymbolic, loss = SemWLS, meanstructure = true, - optimizer = semoptimizer, ) -model_ml = Sem( - specification = spec_mean, - data = dat, - meanstructure = true, - optimizer = semoptimizer, -) +model_ml = Sem(specification = spec_mean, data = dat, meanstructure = true) model_ml_cov = Sem( specification = spec_mean, @@ -285,18 +265,11 @@ model_ml_cov = Sem( obs_mean = vcat(mean(Matrix(dat), dims = 1)...), obs_colnames = Symbol.(names(dat)), meanstructure = true, - optimizer = semoptimizer, nsamples = 75, ) -model_ml_sym = Sem( - specification = spec_mean, - data = dat, - imply = RAMSymbolic, - meanstructure = true, - start_val = start_test_mean, - optimizer = semoptimizer, -) +model_ml_sym = + Sem(specification = spec_mean, data = dat, imply = RAMSymbolic, meanstructure = true) ############################################################################################ ### test gradients @@ -323,7 +296,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ml", "ls", "ml"] .* " for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution_mean" begin - solution = sem_fit(model) + solution = sem_fit(semoptimizer, model) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) end @@ -336,7 +309,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml_mean" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(semoptimizer, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; @@ -354,7 +327,7 @@ end end @testset "fitmeasures/se_ls_mean" begin - solution_ls = sem_fit(model_ls) + solution_ls = sem_fit(semoptimizer, model_ls) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -408,8 +381,8 @@ end meanstructure = true, ) # fit models - sol_ml = solution(sem_fit(model_ml_new)) - sol_ml_sym = solution(sem_fit(model_ml_sym_new)) + sol_ml = solution(sem_fit(semoptimizer, model_ml_new)) + sol_ml_sym = solution(sem_fit(semoptimizer, model_ml_sym_new)) # check solution @test maximum(abs.(sol_ml - params)) < 0.01 @test maximum(abs.(sol_ml_sym - params)) < 0.01 @@ -425,7 +398,6 @@ model_ml = Sem( data = dat_missing, observed = SemObservedMissing, loss = SemFIML, - optimizer = semoptimizer, meanstructure = true, ) @@ -435,8 +407,6 @@ model_ml_sym = Sem( observed = SemObservedMissing, imply = RAMSymbolic, loss = SemFIML, - start_val = start_test_mean, - optimizer = semoptimizer, meanstructure = true, ) @@ -457,13 +427,13 @@ end ############################################################################################ @testset "fiml_solution" begin - solution = sem_fit(model_ml) + solution = sem_fit(semoptimizer, model_ml) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @testset "fiml_solution_symbolic" begin - solution = sem_fit(model_ml_sym) + solution = sem_fit(semoptimizer, model_ml_sym) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @@ -473,7 +443,7 @@ end ############################################################################################ @testset "fitmeasures/se_fiml" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(semoptimizer, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 89c1225e2..4b968bc49 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -65,13 +65,14 @@ semobserved = SemObservedData(data = x, specification = nothing) loss_ml = SemLoss(SemML(; observed = semobserved, nparams = length(start))) +model_ml = Sem(semobserved, imply_ml, loss_ml) +objective!(model_ml, true_val) + optimizer = SemOptimizerOptim( BFGS(; linesearch = BackTracking(order = 3), alphaguess = InitialHagerZhang()),# m = 100), Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), ) -model_ml = Sem(semobserved, imply_ml, loss_ml, optimizer) -objective!(model_ml, true_val) -solution_ml = sem_fit(model_ml) +solution_ml = sem_fit(optimizer, model_ml) @test true_val ≈ solution(solution_ml) atol = 0.05 diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index e13327642..bf44091d2 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -59,7 +59,6 @@ end @test model isa Sem @test @inferred(imply(model)) isa implytype @test @inferred(observed(model)) isa SemObserved - @test @inferred(optimizer(model)) isa SemOptimizer test_vars_api(model, ram_matrices) test_params_api(model, ram_matrices) From 71bced74dfbddc449dea51c2cbad394cff429b4e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Nov 2024 22:58:09 -0800 Subject: [PATCH 145/364] fix inequality constraints test NLopt minimum was 18.11, below what the test expected --- test/examples/political_democracy/constraints.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/examples/political_democracy/constraints.jl b/test/examples/political_democracy/constraints.jl index ef5692f27..fb2116023 100644 --- a/test/examples/political_democracy/constraints.jl +++ b/test/examples/political_democracy/constraints.jl @@ -30,6 +30,8 @@ constrained_optimizer = SemOptimizer(; inequality_constraints = (f = ineq_constraint, tol = 0.0), ) +@test constrained_optimizer isa SemOptimizer{:NLopt} + # NLopt option setting --------------------------------------------------------------------- ############################################################################################ @@ -49,6 +51,6 @@ end @test solution_constrained.solution[31] * solution_constrained.solution[30] >= (0.6 - 1e-8) @test all(abs.(solution_constrained.solution) .< 10) - @test solution_constrained.optimization_result.result[3] == :FTOL_REACHED skip = true - @test abs(solution_constrained.minimum - 21.21) < 0.01 + @test solution_constrained.optimization_result.result[3] == :FTOL_REACHED + @test solution_constrained.minimum <= 21.21 + 0.01 end From 928af39fee7f4c894f293cb8d9da39c46edb018b Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 20 Dec 2024 09:02:16 -0800 Subject: [PATCH 146/364] add ProximalSEM tests --- test/Project.toml | 3 ++ test/examples/examples.jl | 3 ++ test/examples/proximal/l0.jl | 67 ++++++++++++++++++++++++++++++ test/examples/proximal/lasso.jl | 64 ++++++++++++++++++++++++++++ test/examples/proximal/proximal.jl | 9 ++++ test/examples/proximal/ridge.jl | 61 +++++++++++++++++++++++++++ 6 files changed, 207 insertions(+) create mode 100644 test/examples/proximal/l0.jl create mode 100644 test/examples/proximal/lasso.jl create mode 100644 test/examples/proximal/proximal.jl create mode 100644 test/examples/proximal/ridge.jl diff --git a/test/Project.toml b/test/Project.toml index 5867c1f40..14bd0bece 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -9,6 +9,9 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" +ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" +ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" diff --git a/test/examples/examples.jl b/test/examples/examples.jl index a1e0f2c28..e088ffa92 100644 --- a/test/examples/examples.jl +++ b/test/examples/examples.jl @@ -9,3 +9,6 @@ end @safetestset "Multigroup" begin include("multigroup/multigroup.jl") end +@safetestset "Proximal" begin + include("proximal/proximal.jl") +end diff --git a/test/examples/proximal/l0.jl b/test/examples/proximal/l0.jl new file mode 100644 index 000000000..e8874fd51 --- /dev/null +++ b/test/examples/proximal/l0.jl @@ -0,0 +1,67 @@ +using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators + +# load data +dat = example_data("political_democracy") + +############################################################################ +### define models +############################################################################ + +observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +latent_vars = [:ind60, :dem60, :dem65] + +graph = @StenoGraph begin + ind60 → fixed(1) * x1 + x2 + x3 + dem60 → fixed(1) * y1 + y2 + y3 + y4 + dem65 → fixed(1) * y5 + y6 + y7 + y8 + + dem60 ← ind60 + dem65 ← dem60 + dem65 ← ind60 + + _(observed_vars) ↔ _(observed_vars) + _(latent_vars) ↔ _(latent_vars) + + y1 ↔ label(:cov_15) * y5 + y2 ↔ label(:cov_24) * y4 + label(:cov_26) * y6 + y3 ↔ label(:cov_37) * y7 + y4 ↔ label(:cov_48) * y8 + y6 ↔ label(:cov_68) * y8 +end + +partable = ParameterTable(graph; latent_vars = latent_vars, observed_vars = observed_vars) + +ram_mat = RAMMatrices(partable) + +model = Sem(specification = partable, data = dat, loss = SemML) + +fit = sem_fit(model) + +# use l0 from ProximalSEM +# regularized +prox_operator = + SlicedSeparableSum((NormL0(0.0), NormL0(0.02)), ([vcat(1:15, 21:31)], [12:20])) + +model_prox = Sem(specification = partable, data = dat, loss = SemML) + +fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = prox_operator) + +@testset "l0 | solution_unregularized" begin + @test fit_prox.optimization_result.result[:iterations] < 1000 + @test maximum(abs.(solution(fit) - solution(fit_prox))) < 0.002 +end + +# regularized +prox_operator = SlicedSeparableSum((NormL0(0.0), NormL0(100.0)), ([1:30], [31])) + +model_prox = Sem(specification = partable, data = dat, loss = SemML) + +fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = prox_operator) + +@testset "l0 | solution_regularized" begin + @test fit_prox.optimization_result.result[:iterations] < 1000 + @test solution(fit_prox)[31] == 0.0 + @test abs( + StructuralEquationModels.minimum(fit_prox) - StructuralEquationModels.minimum(fit), + ) < 1.0 +end diff --git a/test/examples/proximal/lasso.jl b/test/examples/proximal/lasso.jl new file mode 100644 index 000000000..31a4073f9 --- /dev/null +++ b/test/examples/proximal/lasso.jl @@ -0,0 +1,64 @@ +using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators + +# load data +dat = example_data("political_democracy") + +############################################################################ +### define models +############################################################################ + +observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +latent_vars = [:ind60, :dem60, :dem65] + +graph = @StenoGraph begin + ind60 → fixed(1) * x1 + x2 + x3 + dem60 → fixed(1) * y1 + y2 + y3 + y4 + dem65 → fixed(1) * y5 + y6 + y7 + y8 + + dem60 ← ind60 + dem65 ← dem60 + dem65 ← ind60 + + _(observed_vars) ↔ _(observed_vars) + _(latent_vars) ↔ _(latent_vars) + + y1 ↔ label(:cov_15) * y5 + y2 ↔ label(:cov_24) * y4 + label(:cov_26) * y6 + y3 ↔ label(:cov_37) * y7 + y4 ↔ label(:cov_48) * y8 + y6 ↔ label(:cov_68) * y8 +end + +partable = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) + +ram_mat = RAMMatrices(partable) + +model = Sem(specification = partable, data = dat, loss = SemML) + +fit = sem_fit(model) + +# use lasso from ProximalSEM +λ = zeros(31) + +model_prox = Sem(specification = partable, data = dat, loss = SemML) + +fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = NormL1(λ)) + +@testset "lasso | solution_unregularized" begin + @test fit_prox.optimization_result.result[:iterations] < 1000 + @test maximum(abs.(solution(fit) - solution(fit_prox))) < 0.002 +end + +λ = zeros(31); +λ[16:20] .= 0.02; + +model_prox = Sem(specification = partable, data = dat, loss = SemML) + +fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = NormL1(λ)) + +@testset "lasso | solution_regularized" begin + @test fit_prox.optimization_result.result[:iterations] < 1000 + @test all(solution(fit_prox)[16:20] .< solution(fit)[16:20]) + @test StructuralEquationModels.minimum(fit_prox) - + StructuralEquationModels.minimum(fit) < 0.03 +end diff --git a/test/examples/proximal/proximal.jl b/test/examples/proximal/proximal.jl new file mode 100644 index 000000000..40e72a1ef --- /dev/null +++ b/test/examples/proximal/proximal.jl @@ -0,0 +1,9 @@ +@testset "Ridge" begin + include("ridge.jl") +end +@testset "Lasso" begin + include("lasso.jl") +end +@testset "L0" begin + include("l0.jl") +end diff --git a/test/examples/proximal/ridge.jl b/test/examples/proximal/ridge.jl new file mode 100644 index 000000000..120910234 --- /dev/null +++ b/test/examples/proximal/ridge.jl @@ -0,0 +1,61 @@ +using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators + +# load data +dat = example_data("political_democracy") + +############################################################################ +### define models +############################################################################ + +observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +latent_vars = [:ind60, :dem60, :dem65] + +graph = @StenoGraph begin + ind60 → fixed(1) * x1 + x2 + x3 + dem60 → fixed(1) * y1 + y2 + y3 + y4 + dem65 → fixed(1) * y5 + y6 + y7 + y8 + + dem60 ← ind60 + dem65 ← dem60 + dem65 ← ind60 + + _(observed_vars) ↔ _(observed_vars) + _(latent_vars) ↔ _(latent_vars) + + y1 ↔ label(:cov_15) * y5 + y2 ↔ label(:cov_24) * y4 + label(:cov_26) * y6 + y3 ↔ label(:cov_37) * y7 + y4 ↔ label(:cov_48) * y8 + y6 ↔ label(:cov_68) * y8 +end + +partable = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) + +ram_mat = RAMMatrices(partable) + +model = Sem(specification = partable, data = dat, loss = SemML) + +fit = sem_fit(model) + +# use ridge from StructuralEquationModels +model_ridge = Sem( + specification = partable, + data = dat, + loss = (SemML, SemRidge), + α_ridge = 0.02, + which_ridge = 16:20, +) + +solution_ridge = sem_fit(model_ridge) + +# use ridge from ProximalSEM; SqrNormL2 uses λ/2 as penalty +λ = zeros(31); +λ[16:20] .= 0.04; + +model_prox = Sem(specification = partable, data = dat, loss = SemML) + +solution_prox = sem_fit(model_prox, engine = :Proximal, operator_g = SqrNormL2(λ)) + +@testset "ridge_solution" begin + @test isapprox(solution_prox.solution, solution_ridge.solution; rtol = 1e-4) +end From c19c4a7cb625d1afc48e372c75d2db0f4ddb508a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 16:34:10 -0800 Subject: [PATCH 147/364] optim/documentation.jl: rename to abstract.jl --- src/StructuralEquationModels.jl | 2 +- src/optimizer/{documentation.jl => abstract.jl} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/optimizer/{documentation.jl => abstract.jl} (100%) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index ca1ae61f0..9e0fc3669 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -60,7 +60,7 @@ include("loss/regularization/ridge.jl") include("loss/WLS/WLS.jl") include("loss/constant/constant.jl") # optimizer -include("optimizer/documentation.jl") +include("optimizer/abstract.jl") include("optimizer/Empty.jl") include("optimizer/optim.jl") # helper functions diff --git a/src/optimizer/documentation.jl b/src/optimizer/abstract.jl similarity index 100% rename from src/optimizer/documentation.jl rename to src/optimizer/abstract.jl From c5b48c73bea5caf6aa5d5bd369e385e0ddec3272 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 20 Dec 2024 13:20:28 -0800 Subject: [PATCH 148/364] ext: change folder layout --- ext/{optimizer => SEMNLOptExt}/NLopt.jl | 0 ext/{ => SEMNLOptExt}/SEMNLOptExt.jl | 2 +- ext/{optimizer => SEMProximalOptExt}/ProximalAlgorithms.jl | 0 ext/{ => SEMProximalOptExt}/SEMProximalOptExt.jl | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename ext/{optimizer => SEMNLOptExt}/NLopt.jl (100%) rename ext/{ => SEMNLOptExt}/SEMNLOptExt.jl (82%) rename ext/{optimizer => SEMProximalOptExt}/ProximalAlgorithms.jl (100%) rename ext/{ => SEMProximalOptExt}/SEMProximalOptExt.jl (85%) diff --git a/ext/optimizer/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl similarity index 100% rename from ext/optimizer/NLopt.jl rename to ext/SEMNLOptExt/NLopt.jl diff --git a/ext/SEMNLOptExt.jl b/ext/SEMNLOptExt/SEMNLOptExt.jl similarity index 82% rename from ext/SEMNLOptExt.jl rename to ext/SEMNLOptExt/SEMNLOptExt.jl index a727b82f1..a159f6dc8 100644 --- a/ext/SEMNLOptExt.jl +++ b/ext/SEMNLOptExt/SEMNLOptExt.jl @@ -6,6 +6,6 @@ SEM = StructuralEquationModels export SemOptimizerNLopt, NLoptConstraint -include("optimizer/NLopt.jl") +include("NLopt.jl") end diff --git a/ext/optimizer/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl similarity index 100% rename from ext/optimizer/ProximalAlgorithms.jl rename to ext/SEMProximalOptExt/ProximalAlgorithms.jl diff --git a/ext/SEMProximalOptExt.jl b/ext/SEMProximalOptExt/SEMProximalOptExt.jl similarity index 85% rename from ext/SEMProximalOptExt.jl rename to ext/SEMProximalOptExt/SEMProximalOptExt.jl index e81760acb..8f91e03b0 100644 --- a/ext/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt/SEMProximalOptExt.jl @@ -9,6 +9,6 @@ SEM = StructuralEquationModels #ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) -include("optimizer/ProximalAlgorithms.jl") +include("ProximalAlgorithms.jl") end From d5357f0240543d44272150604e6f272da914802c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 24 Dec 2024 11:05:25 -0800 Subject: [PATCH 149/364] Project.toml: fix ProximalOperators ID --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 1bd335f19..ed5239c94 100644 --- a/Project.toml +++ b/Project.toml @@ -48,7 +48,7 @@ test = ["Test"] NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" -ProximalOperators = "f3b72e0c-5f3e-4b3e-8f3e-3f4f3e3e3e3e" +ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" [extensions] SEMNLOptExt = "NLopt" From 48a744f8976e3d055aaa7b29529a96e518597dc8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 31 Jul 2024 20:55:35 -0700 Subject: [PATCH 150/364] docs: fix nsamples, nobserved_vars --- docs/src/developer/observed.md | 8 ++++---- docs/src/tutorials/inspection/inspection.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/src/developer/observed.md b/docs/src/developer/observed.md index 2b695e597..93eca6ed9 100644 --- a/docs/src/developer/observed.md +++ b/docs/src/developer/observed.md @@ -22,10 +22,10 @@ end To compute some fit indices, you need to provide methods for ```julia -# Number of observed datapoints -n_obs(observed::MyObserved) = ... -# Number of manifest variables -n_man(observed::MyObserved) = ... +# Number of samples (observations) in the dataset +nsamples(observed::MyObserved) = ... +# Number of observed variables +nobserved_vars(observed::MyObserved) = ... ``` As always, you can add additional methods for properties that imply types and loss function want to access, for example (from the `SemObservedCommon` implementation): diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md index b2eefadb2..88caf5812 100644 --- a/docs/src/tutorials/inspection/inspection.md +++ b/docs/src/tutorials/inspection/inspection.md @@ -1,7 +1,7 @@ # Model inspection ```@setup colored -using StructuralEquationModels +using StructuralEquationModels observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] latent_vars = [:ind60, :dem60, :dem65] @@ -32,7 +32,7 @@ end partable = ParameterTable( graph, - latent_vars = latent_vars, + latent_vars = latent_vars, observed_vars = observed_vars) data = example_data("political_democracy") @@ -128,8 +128,8 @@ BIC χ² df minus2ll -n_man -n_obs +nobserved_vars +nsamples nparams p_value RMSEA From 5faf1160d94ee62e1b84729f58a17f7da9653565 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 11:25:21 -0700 Subject: [PATCH 151/364] cleanup data columns reordering define a single source_to_dest_perm() function --- src/observed/abstract.jl | 30 ++++++++++++++++++++++++++++++ src/observed/covariance.jl | 33 +++------------------------------ src/observed/data.jl | 17 +---------------- src/observed/missing.jl | 2 +- 4 files changed, 35 insertions(+), 47 deletions(-) diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index 90de8b5a6..71e87466a 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -8,3 +8,33 @@ Rows are samples, columns are observed variables. [`nsamples`](@ref), [`observed_vars`](@ref). """ samples(observed::SemObserved) = observed.data + +############################################################################################ +### Additional functions +############################################################################################ + +# compute the permutation that subsets and reorders source elements +# to match the destination order. +# if multiple identical elements are present in the source, the last one is used. +# if one_to_one is true, checks that the source and destination have the same length. +function source_to_dest_perm( + src::AbstractVector, + dest::AbstractVector; + one_to_one::Bool = false, + entities::String = "elements", +) + if dest == src # exact match + return eachindex(dest) + else + one_to_one && + length(dest) != length(src) && + throw( + DimensionMismatch( + "The length of the new $entities order ($(length(dest))) " * + "does not match the number of $entities ($(length(src)))", + ), + ) + src_inds = Dict(el => i for (i, el) in enumerate(src)) + return [src_inds[el] for el in dest] + end +end diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index b78f41833..860391e21 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -75,9 +75,9 @@ function SemObservedCovariance(; end if !isnothing(spec_colnames) - obs_cov = reorder_obs_cov(obs_cov, spec_colnames, obs_colnames) - isnothing(obs_mean) || - (obs_mean = reorder_obs_mean(obs_mean, spec_colnames, obs_colnames)) + obs2spec_perm = source_to_dest_perm(obs_colnames, spec_colnames) + obs_cov = obs_cov[obs2spec_perm, obs2spec_perm] + isnothing(obs_mean) || (obs_mean = obs_mean[obs2spec_perm]) end return SemObservedCovariance(obs_cov, obs_mean, size(obs_cov, 1), nsamples) @@ -99,30 +99,3 @@ samples(observed::SemObservedCovariance) = obs_cov(observed::SemObservedCovariance) = observed.obs_cov obs_mean(observed::SemObservedCovariance) = observed.obs_mean - -############################################################################################ -### Additional functions -############################################################################################ - -# reorder covariance matrices -------------------------------------------------------------- -function reorder_obs_cov(obs_cov, spec_colnames, obs_colnames) - if spec_colnames == obs_colnames - return obs_cov - else - new_position = [findfirst(==(x), obs_colnames) for x in spec_colnames] - obs_cov = obs_cov[new_position, new_position] - return obs_cov - end -end - -# reorder means ---------------------------------------------------------------------------- - -function reorder_obs_mean(obs_mean, spec_colnames, obs_colnames) - if spec_colnames == obs_colnames - return obs_mean - else - new_position = [findfirst(==(x), obs_colnames) for x in spec_colnames] - obs_mean = obs_mean[new_position] - return obs_mean - end -end diff --git a/src/observed/data.jl b/src/observed/data.jl index c9b50e597..ff68b450a 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -91,7 +91,7 @@ function SemObservedData(; throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end - data = reorder_data(data, spec_colnames, obs_colnames) + data = data[:, source_to_dest_perm(obs_colnames, spec_colnames)] end end @@ -121,18 +121,3 @@ nobserved_vars(observed::SemObservedData) = observed.nobs_vars obs_cov(observed::SemObservedData) = observed.obs_cov obs_mean(observed::SemObservedData) = observed.obs_mean - -############################################################################################ -### Additional functions -############################################################################################ - -# reorder data ----------------------------------------------------------------------------- -function reorder_data(data::AbstractArray, spec_colnames, obs_colnames) - if spec_colnames == obs_colnames - return data - else - obs_positions = Dict(col => i for (i, col) in enumerate(obs_colnames)) - new_positions = [obs_positions[col] for col in spec_colnames] - return data[:, new_positions] - end -end diff --git a/src/observed/missing.jl b/src/observed/missing.jl index b628a313b..1eafab8f8 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -123,7 +123,7 @@ function SemObservedMissing(; throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end - data = reorder_data(data, spec_colnames, obs_colnames) + data = data[:, source_to_dest_perm(obs_colnames, spec_colnames)] end end From 30e0b240d234a5600206abe6a1417fa84c6503d6 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 Jan 2025 11:57:28 -0800 Subject: [PATCH 152/364] SemObservedCov: def as an alias of SemObservedData reduces code duplication; also annotate types of ctor args now samples(SemObsCov) returns nothing --- src/StructuralEquationModels.jl | 2 +- src/observed/covariance.jl | 53 ++++++++------------------- test/unit_tests/data_input_formats.jl | 6 +-- 3 files changed, 20 insertions(+), 41 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 9e0fc3669..1caf1f5b4 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -41,8 +41,8 @@ include("frontend/fit/summary.jl") include("frontend/pretty_printing.jl") # observed include("observed/abstract.jl") -include("observed/covariance.jl") include("observed/data.jl") +include("observed/covariance.jl") include("observed/missing.jl") include("observed/EM.jl") # constructor diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 860391e21..195b84050 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -1,3 +1,10 @@ +""" +Type alias for [`SemObservedData`](@ref) that has mean and covariance, but no actual data. + +For instances of `SemObservedCovariance` [`samples`](@ref) returns `nothing`. +""" +const SemObservedCovariance{B, C} = SemObservedData{Nothing, B, C} + """ For observed covariance matrices and means. @@ -39,27 +46,19 @@ use this if you are sure your covariance matrix is in the right format. ## Additional keyword arguments: - `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object """ -struct SemObservedCovariance{B, C} <: SemObserved - obs_cov::B - obs_mean::C - nobs_vars::Int - nsamples::Int -end - function SemObservedCovariance(; specification::Union{SemSpecification, Nothing} = nothing, - obs_cov, - obs_colnames = nothing, - spec_colnames = nothing, - obs_mean = nothing, - meanstructure = false, + obs_cov::AbstractMatrix, + obs_colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, + spec_colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, + obs_mean::Union{AbstractVector, Nothing} = nothing, + meanstructure::Bool = false, nsamples::Integer, kwargs..., ) - if !meanstructure & !isnothing(obs_mean) + if !meanstructure && !isnothing(obs_mean) throw(ArgumentError("observed means were passed, but `meanstructure = false`")) - - elseif meanstructure & isnothing(obs_mean) + elseif meanstructure && isnothing(obs_mean) throw(ArgumentError("`meanstructure = true`, but no observed means were passed")) end @@ -67,11 +66,8 @@ function SemObservedCovariance(; spec_colnames = observed_vars(specification) end - if !isnothing(spec_colnames) & isnothing(obs_colnames) + if !isnothing(spec_colnames) && isnothing(obs_colnames) throw(ArgumentError("no `obs_colnames` were specified")) - - elseif !isnothing(spec_colnames) & !(eltype(obs_colnames) <: Symbol) - throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end if !isnothing(spec_colnames) @@ -80,22 +76,5 @@ function SemObservedCovariance(; isnothing(obs_mean) || (obs_mean = obs_mean[obs2spec_perm]) end - return SemObservedCovariance(obs_cov, obs_mean, size(obs_cov, 1), nsamples) + return SemObservedData(nothing, obs_cov, obs_mean, size(obs_cov, 1), nsamples) end - -############################################################################################ -### Recommended methods -############################################################################################ - -nsamples(observed::SemObservedCovariance) = observed.nsamples -nobserved_vars(observed::SemObservedCovariance) = observed.nobs_vars - -samples(observed::SemObservedCovariance) = - error("$(typeof(observed)) does not store data samples") - -############################################################################################ -### additional methods -############################################################################################ - -obs_cov(observed::SemObservedCovariance) = observed.obs_cov -obs_mean(observed::SemObservedCovariance) = observed.obs_mean diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 3fc255b84..9ab0c0af0 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -240,7 +240,7 @@ end # SemObservedData approx_cov = true, ) - @test_throws ErrorException samples(observed) + @test @inferred(samples(observed)) === nothing observed_nospec = SemObservedCovariance( specification = nothing, @@ -260,7 +260,7 @@ end # SemObservedData approx_cov = true, ) - @test_throws ErrorException samples(observed_nospec) + @test @inferred(samples(observed_nospec)) === nothing observed_shuffle = SemObservedCovariance( specification = spec, @@ -281,7 +281,7 @@ end # SemObservedData approx_cov = true, ) - @test_throws ErrorException samples(observed_shuffle) + @test @inferred(samples(observed_shuffle)) === nothing # respect specification order @test @inferred(obs_cov(observed_shuffle)) ≈ obs_cov(observed) From 86c5e2d8ba8a194f051a3de235f4509cd88e37b1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 8 May 2024 18:18:01 -0700 Subject: [PATCH 153/364] SemObserved: store observed_vars add observed_vars(data::SemObserved) --- src/observed/abstract.jl | 2 ++ src/observed/covariance.jl | 3 ++- src/observed/data.jl | 9 +++++---- src/observed/missing.jl | 10 +++++----- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index 71e87466a..62e88681b 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -9,6 +9,8 @@ Rows are samples, columns are observed variables. """ samples(observed::SemObserved) = observed.data +observed_vars(observed::SemObserved) = observed.observed_vars + ############################################################################################ ### Additional functions ############################################################################################ diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 195b84050..195d55b4e 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -72,9 +72,10 @@ function SemObservedCovariance(; if !isnothing(spec_colnames) obs2spec_perm = source_to_dest_perm(obs_colnames, spec_colnames) + obs_colnames = obs_colnames[obs2spec_perm] obs_cov = obs_cov[obs2spec_perm, obs2spec_perm] isnothing(obs_mean) || (obs_mean = obs_mean[obs2spec_perm]) end - return SemObservedData(nothing, obs_cov, obs_mean, size(obs_cov, 1), nsamples) + return SemObservedData(nothing, Symbol.(obs_colnames), obs_cov, obs_mean, nsamples) end diff --git a/src/observed/data.jl b/src/observed/data.jl index ff68b450a..700155924 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -39,9 +39,9 @@ use this if you are sure your observed data is in the right format. """ struct SemObservedData{A, B, C} <: SemObserved data::A + observed_vars::Vector{Symbol} obs_cov::B obs_mean::C - nobs_vars::Int nsamples::Int end @@ -68,6 +68,7 @@ function SemObservedData(; if isnothing(obs_colnames) try data = data[:, spec_colnames] + obs_colnames = spec_colnames catch throw( ArgumentError( @@ -91,7 +92,8 @@ function SemObservedData(; throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end - data = data[:, source_to_dest_perm(obs_colnames, spec_colnames)] + obs_colnames = obs_colnames[source_to_dest_perm(obs_colnames, spec_colnames)] + data = data[:, obs_colnames] end end @@ -101,9 +103,9 @@ function SemObservedData(; return SemObservedData( data, + Symbol.(obs_colnames), compute_covariance ? Statistics.cov(data) : nothing, meanstructure ? vec(Statistics.mean(data, dims = 1)) : nothing, - size(data, 2), size(data, 1), ) end @@ -113,7 +115,6 @@ end ############################################################################################ nsamples(observed::SemObservedData) = observed.nsamples -nobserved_vars(observed::SemObservedData) = observed.nobs_vars ############################################################################################ ### additional methods diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 1eafab8f8..76dd70cbb 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -55,7 +55,6 @@ use this if you are sure your observed data is in the right format. """ mutable struct SemObservedMissing{ A <: AbstractArray, - D <: Number, O <: Number, P <: Vector, P2 <: Vector, @@ -68,7 +67,7 @@ mutable struct SemObservedMissing{ S <: EmMVNModel, } <: SemObserved data::A - nobs_vars::D + observed_vars::Vector{Symbol} nsamples::O patterns::P # missing patterns patterns_not::P2 @@ -100,6 +99,7 @@ function SemObservedMissing(; if isnothing(obs_colnames) try data = data[:, spec_colnames] + obs_colnames = spec_colnames catch throw( ArgumentError( @@ -123,7 +123,8 @@ function SemObservedMissing(; throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end - data = data[:, source_to_dest_perm(obs_colnames, spec_colnames)] + obs_colnames = obs_colnames[source_to_dest_perm(obs_colnames, spec_colnames)] + data = data[:, obs_colnames] end end @@ -186,7 +187,7 @@ function SemObservedMissing(; return SemObservedMissing( data, - nobs_vars, + Symbol.(obs_colnames), nsamples, remember_cart, remember_cart_not, @@ -205,7 +206,6 @@ end ############################################################################################ nsamples(observed::SemObservedMissing) = observed.nsamples -nobserved_vars(observed::SemObservedMissing) = observed.nobs_vars ############################################################################################ ### Additional methods From ef1861e16f204bd39ac91e3f2361e1ecb8703fa9 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 8 May 2024 18:18:01 -0700 Subject: [PATCH 154/364] nsamples(observed::SemObserved): unify --- src/observed/abstract.jl | 1 + src/observed/data.jl | 2 -- src/observed/missing.jl | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index 62e88681b..816dd9e80 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -8,6 +8,7 @@ Rows are samples, columns are observed variables. [`nsamples`](@ref), [`observed_vars`](@ref). """ samples(observed::SemObserved) = observed.data +nsamples(observed::SemObserved) = observed.nsamples observed_vars(observed::SemObserved) = observed.observed_vars diff --git a/src/observed/data.jl b/src/observed/data.jl index 700155924..ce4ce4bce 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -114,8 +114,6 @@ end ### Recommended methods ############################################################################################ -nsamples(observed::SemObservedData) = observed.nsamples - ############################################################################################ ### additional methods ############################################################################################ diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 76dd70cbb..0f95037be 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -205,8 +205,6 @@ end ### Recommended methods ############################################################################################ -nsamples(observed::SemObservedMissing) = observed.nsamples - ############################################################################################ ### Additional methods ############################################################################################ From 1d573a3e66a5384c4b03a3f980ead66f0e930e28 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 25 Dec 2024 13:21:53 -0800 Subject: [PATCH 155/364] FIML: simplify index generation --- src/loss/ML/FIML.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 20c81b831..d88288453 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -61,9 +61,11 @@ function SemFIML(; observed, specification, kwargs...) imp_inv = zeros(nobs_vars, nobs_vars) mult = similar.(inverses) - ∇ind = vec(CartesianIndices(Array{Float64}(undef, nobs_vars, nobs_vars))) - ∇ind = - [findall(x -> !(x[1] ∈ ind || x[2] ∈ ind), ∇ind) for ind in patterns_not(observed)] + # linear indicies of co-observed variable pairs for each pattern + Σ_linind = LinearIndices((nobs_vars, nobs_vars)) + ∇ind = map(patterns_not(observed)) do pat_vars + vec(Σ_linind[pat_vars, pat_vars]) + end return SemFIML( ExactHessian(), From b8256678665846716fcb501542eebaa35738eca7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 25 Dec 2024 13:04:22 -0800 Subject: [PATCH 156/364] SemObservedMissing: refactor * use SemObsMissingPattern struct to simplify code * replace O(Nvars^2) common pattern detection with Dict{} * don't store row-wise, store sub-matrices of non-missing data instead * use StatsBase.mean_and_cov() --- src/StructuralEquationModels.jl | 1 + src/frontend/fit/fitmeasures/minus2ll.jl | 71 +++--------- src/loss/ML/FIML.jl | 72 ++++++------- src/observed/EM.jl | 34 +++--- src/observed/missing.jl | 131 ++++++----------------- src/observed/missing_pattern.jl | 45 ++++++++ test/unit_tests/data_input_formats.jl | 9 +- 7 files changed, 147 insertions(+), 216 deletions(-) create mode 100644 src/observed/missing_pattern.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 1caf1f5b4..a6677a4ed 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -43,6 +43,7 @@ include("frontend/pretty_printing.jl") include("observed/abstract.jl") include("observed/data.jl") include("observed/covariance.jl") +include("observed/missing_pattern.jl") include("observed/missing.jl") include("observed/EM.jl") # constructor diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 54a4ce12d..1cddee71d 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -31,74 +31,33 @@ minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, loss_ml::SemWLS) = # compute likelihood for missing data - H0 ------------------------------------------------- # -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n function minus2ll(minimum::Number, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemFIML) - F = minimum - F *= nsamples(observed) - F += sum(log(2π) * observed.pattern_nsamples .* observed.pattern_nobs_vars) + F = minimum * nsamples(observed) + F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), observed.patterns) return F end # compute likelihood for missing data - H1 ------------------------------------------------- # -2ll = ∑ log(2π)*(nᵢ + mᵢ) + ln(Σᵢ) + (mᵢ - μᵢ)ᵀ Σᵢ⁻¹ (mᵢ - μᵢ)) + tr(SᵢΣᵢ) function minus2ll(observed::SemObservedMissing) - if observed.em_model.fitted - minus2ll( - observed.em_model.μ, - observed.em_model.Σ, - nsamples(observed), - pattern_rows(observed), - observed.patterns, - observed.obs_mean, - observed.obs_cov, - observed.pattern_nsamples, - observed.pattern_nobs_vars, - ) - else - em_mvn(observed) - minus2ll( - observed.em_model.μ, - observed.em_model.Σ, - nsamples(observed), - pattern_rows(observed), - observed.patterns, - observed.obs_mean, - observed.obs_cov, - observed.pattern_nsamples, - observed.pattern_nobs_vars, - ) - end -end - -function minus2ll( - μ, - Σ, - N, - rows, - patterns, - obs_mean, - obs_cov, - pattern_nsamples, - pattern_nobs_vars, -) - F = 0.0 + # fit EM-based mean and cov if not yet fitted + # FIXME EM could be very computationally expensive + observed.em_model.fitted || em_mvn(observed) - for i in 1:length(rows) - nᵢ = pattern_nsamples[i] - # missing pattern - pattern = patterns[i] - # observed data - Sᵢ = obs_cov[i] + Σ = observed.em_model.Σ + μ = observed.em_model.μ + F = sum(observed.patterns) do pat # implied covariance/mean - Σᵢ = Σ[pattern, pattern] - ld = logdet(Σᵢ) - Σᵢ⁻¹ = inv(cholesky(Σᵢ)) - meandiffᵢ = obs_mean[i] - μ[pattern] + Σᵢ = Σ[pat.measured_mask, pat.measured_mask] + Σᵢ_chol = cholesky!(Σᵢ) + ld = logdet(Σᵢ_chol) + Σᵢ⁻¹ = LinearAlgebra.inv!(Σᵢ_chol) + meandiffᵢ = pat.measured_mean - μ[pat.measured_mask] - F += F_one_pattern(meandiffᵢ, Σᵢ⁻¹, Sᵢ, ld, nᵢ) + F_one_pattern(meandiffᵢ, Σᵢ⁻¹, pat.measured_cov, ld, nsamples(pat)) end - F += sum(log(2π) * pattern_nsamples .* pattern_nobs_vars) - #F *= N + F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), observed.patterns) return F end diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index d88288453..bf020d561 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -47,23 +47,25 @@ end ### Constructors ############################################################################################ -function SemFIML(; observed, specification, kwargs...) - inverses = broadcast(x -> zeros(x, x), pattern_nobs_vars(observed)) +function SemFIML(; observed::SemObservedMissing, specification, kwargs...) + inverses = + [zeros(nmeasured_vars(pat), nmeasured_vars(pat)) for pat in observed.patterns] choleskys = Array{Cholesky{Float64, Array{Float64, 2}}, 1}(undef, length(inverses)) - n_patterns = size(pattern_rows(observed), 1) + n_patterns = length(observed.patterns) logdets = zeros(n_patterns) - imp_mean = zeros.(pattern_nobs_vars(observed)) - meandiff = zeros.(pattern_nobs_vars(observed)) + imp_mean = [zeros(nmeasured_vars(pat)) for pat in observed.patterns] + meandiff = [zeros(nmeasured_vars(pat)) for pat in observed.patterns] nobs_vars = nobserved_vars(observed) imp_inv = zeros(nobs_vars, nobs_vars) mult = similar.(inverses) - # linear indicies of co-observed variable pairs for each pattern + # generate linear indicies of co-observed variable pairs for each pattern Σ_linind = LinearIndices((nobs_vars, nobs_vars)) - ∇ind = map(patterns_not(observed)) do pat_vars + ∇ind = map(observed.patterns) do pat + pat_vars = findall(pat.measured_mask) vec(Σ_linind[pat_vars, pat_vars]) end @@ -106,10 +108,10 @@ function evaluate!( prepare_SemFIML!(semfiml, model) scale = inv(nsamples(observed(model))) - obs_rows = pattern_rows(observed(model)) - isnothing(objective) || (objective = scale * F_FIML(obs_rows, semfiml, model, params)) + isnothing(objective) || + (objective = scale * F_FIML(observed(model), semfiml, model, params)) isnothing(gradient) || - (∇F_FIML!(gradient, obs_rows, semfiml, model); gradient .*= scale) + (∇F_FIML!(gradient, observed(model), semfiml, model); gradient .*= scale) return objective end @@ -133,16 +135,16 @@ function F_one_pattern(meandiff, inverse, obs_cov, logdet, N) return F * N end -function ∇F_one_pattern(μ_diff, Σ⁻¹, S, pattern, ∇ind, N, Jμ, JΣ, model) +function ∇F_one_pattern(μ_diff, Σ⁻¹, S, obs_mask, ∇ind, N, Jμ, JΣ, model) diff⨉inv = μ_diff' * Σ⁻¹ if N > one(N) JΣ[∇ind] .+= N * vec(Σ⁻¹ * (I - S * Σ⁻¹ - μ_diff * diff⨉inv)) - @. Jμ[pattern] += (N * 2 * diff⨉inv)' + @. Jμ[obs_mask] += (N * 2 * diff⨉inv)' else JΣ[∇ind] .+= vec(Σ⁻¹ * (I - μ_diff * diff⨉inv)) - @. Jμ[pattern] += (2 * diff⨉inv)' + @. Jμ[obs_mask] += (2 * diff⨉inv)' end end @@ -165,32 +167,32 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, imply, model, semfiml) mul!(G, ∇μ', Jμ, -1, 1) end -function F_FIML(rows, semfiml, model, params) +function F_FIML(observed::SemObservedMissing, semfiml, model, params) F = zero(eltype(params)) - for i in 1:size(rows, 1) + for (i, pat) in enumerate(observed.patterns) F += F_one_pattern( semfiml.meandiff[i], semfiml.inverses[i], - obs_cov(observed(model))[i], + pat.measured_cov, semfiml.logdets[i], - pattern_nsamples(observed(model))[i], + nsamples(pat), ) end return F end -function ∇F_FIML!(G, rows, semfiml, model) +function ∇F_FIML!(G, observed::SemObservedMissing, semfiml, model) Jμ = zeros(nobserved_vars(model)) JΣ = zeros(nobserved_vars(model)^2) - for i in 1:size(rows, 1) + for (i, pat) in enumerate(observed.patterns) ∇F_one_pattern( semfiml.meandiff[i], semfiml.inverses[i], - obs_cov(observed(model))[i], - patterns(observed(model))[i], + pat.measured_cov, + pat.measured_mask, semfiml.∇ind[i], - pattern_nsamples(observed(model))[i], + nsamples(pat), Jμ, JΣ, model, @@ -204,29 +206,21 @@ function prepare_SemFIML!(semfiml, model) batch_cholesky!(semfiml, model) #batch_sym_inv_update!(semfiml, model) batch_inv!(semfiml, model) - for i in 1:size(pattern_nsamples(observed(model)), 1) - semfiml.meandiff[i] .= obs_mean(observed(model))[i] - semfiml.imp_mean[i] + for (i, pat) in enumerate(observed(model).patterns) + semfiml.meandiff[i] .= pat.measured_mean .- semfiml.imp_mean[i] end end -function copy_per_pattern!(inverses, source_inverses, means, source_means, patterns) - @views for i in 1:size(patterns, 1) - inverses[i] .= source_inverses[patterns[i], patterns[i]] - end - - @views for i in 1:size(patterns, 1) - means[i] .= source_means[patterns[i]] +function copy_per_pattern!(fiml::SemFIML, model::AbstractSem) + Σ = imply(model).Σ + μ = imply(model).μ + data = observed(model) + @inbounds @views for (i, pat) in enumerate(data.patterns) + fiml.inverses[i] .= Σ[pat.measured_mask, pat.measured_mask] + fiml.imp_mean[i] .= μ[pat.measured_mask] end end -copy_per_pattern!(semfiml, model::M where {M <: AbstractSem}) = copy_per_pattern!( - semfiml.inverses, - imply(model).Σ, - semfiml.imp_mean, - imply(model).μ, - patterns(observed(model)), -) - function batch_cholesky!(semfiml, model) for i in 1:size(semfiml.inverses, 1) semfiml.choleskys[i] = cholesky!(Symmetric(semfiml.inverses[i])) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index ef5da317d..beac45ca8 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -37,9 +37,9 @@ function em_mvn( 𝔼xxᵀ_pre = zeros(nvars, nvars) ### precompute for full cases - if length(observed.patterns[1]) == nvars - for row in pattern_rows(observed)[1] - row = observed.data_rowwise[row] + fullpat = observed.patterns[1] + if nmissed_vars(fullpat) == 0 + for row in eachrow(fullpat.data) 𝔼x_pre += row 𝔼xxᵀ_pre += row * row' end @@ -97,21 +97,27 @@ function em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xx Σ = em_model.Σ # Compute the expected sufficient statistics - for i in 2:length(observed.pattern_nsamples) + for pat in observed.patterns + (nmissed_vars(pat) == 0) && continue # skip full cases # observed and unobserved vars - u = observed.patterns_not[i] - o = observed.patterns[i] + u = pat.miss_mask + o = pat.measured_mask # precompute for pattern - V = Σ[u, u] - Σ[u, o] * (Σ[o, o] \ Σ[o, u]) + Σoo = Σ[o, o] + Σuo = Σ[u, o] + μu = μ[u] + μo = μ[o] + + V = Σ[u, u] - Σuo * (Σoo \ Σ[o, u]) # loop trough data - for row in pattern_rows(observed)[i] - m = μ[u] + Σ[u, o] * (Σ[o, o] \ (observed.data_rowwise[row] - μ[o])) + for rowdata in eachrow(pat.data) + m = μu + Σuo * (Σoo \ (rowdata - μo)) 𝔼xᵢ[u] = m - 𝔼xᵢ[o] = observed.data_rowwise[row] + 𝔼xᵢ[o] = rowdata 𝔼xxᵀᵢ[u, u] = 𝔼xᵢ[u] * 𝔼xᵢ[u]' + V 𝔼xxᵀᵢ[o, o] = 𝔼xᵢ[o] * 𝔼xᵢ[o]' 𝔼xxᵀᵢ[o, u] = 𝔼xᵢ[o] * 𝔼xᵢ[u]' @@ -153,10 +159,10 @@ end # use μ and Σ of full cases function start_em_observed(observed::SemObservedMissing; kwargs...) - if (length(observed.patterns[1]) == nobserved_vars(observed)) & - (observed.pattern_nsamples[1] > 1) - μ = copy(observed.obs_mean[1]) - Σ = copy(Symmetric(observed.obs_cov[1])) + fullpat = observed.patterns[1] + if (nmissed_vars(fullpat) == 0) && (nobserved_vars(fullpat) > 1) + μ = copy(fullpat.measured_mean) + Σ = copy(Symmetric(fullpat.measured_cov)) if !isposdef(Σ) Σ = Matrix(Diagonal(Σ)) end diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 0f95037be..96b027ae6 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -9,6 +9,10 @@ mutable struct EmMVNModel{A, b, B} fitted::B end +# FIXME type unstable +obs_mean(em::EmMVNModel) = ifelse(em.fitted, em.μ, nothing) +obs_cov(em::EmMVNModel) = ifelse(em.fitted, em.Σ, nothing) + """ For observed data with missing values. @@ -31,16 +35,7 @@ For observed data with missing values. - `nobserved_vars(::SemObservedMissing)` -> number of manifest variables - `samples(::SemObservedMissing)` -> observed data -- `data_rowwise(::SemObservedMissing)` -> observed data as vector per observation, with missing values deleted - -- `patterns(::SemObservedMissing)` -> indices of non-missing variables per missing patterns -- `patterns_not(::SemObservedMissing)` -> indices of missing variables per missing pattern -- `pattern_rows(::SemObservedMissing)` -> row indices of observed data points that belong to each pattern -- `pattern_nsamples(::SemObservedMissing)` -> number of data points per pattern -- `pattern_nobs_vars(::SemObservedMissing)` -> number of non-missing observed variables per pattern -- `obs_mean(::SemObservedMissing)` -> observed mean per pattern -- `obs_cov(::SemObservedMissing)` -> observed covariance per pattern -- `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via optimization maximization +- `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via expectation maximization ## Implementation Subtype of `SemObserved` @@ -53,31 +48,17 @@ use this if you are sure your observed data is in the right format. ## Additional keyword arguments: - `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object """ -mutable struct SemObservedMissing{ - A <: AbstractArray, - O <: Number, - P <: Vector, - P2 <: Vector, - R <: Vector, - PD <: AbstractArray, - PO <: AbstractArray, - PVO <: AbstractArray, - A2 <: AbstractArray, - A3 <: AbstractArray, - S <: EmMVNModel, +struct SemObservedMissing{ + T <: Real, + S <: Real, + E <: EmMVNModel, } <: SemObserved - data::A + data::Matrix{Union{T, Missing}} observed_vars::Vector{Symbol} - nsamples::O - patterns::P # missing patterns - patterns_not::P2 - pattern_rows::R # coresponding rows in data_rowwise - data_rowwise::PD # list of data - pattern_nsamples::PO # observed rows per pattern - pattern_nobs_vars::PVO # number of non-missing variables per pattern - obs_mean::A2 - obs_cov::A3 - em_model::S + nsamples::Int + patterns::Vector{SemObservedMissingPattern{T, S}} + + em_model::E end ############################################################################################ @@ -132,73 +113,27 @@ function SemObservedMissing(; data = Matrix(data) end - # remove persons with only missings - keep = Vector{Int64}() - for i in 1:size(data, 1) - if any(.!ismissing.(data[i, :])) - push!(keep, i) - end - end - data = data[keep, :] - nsamples, nobs_vars = size(data) - # compute and store the different missing patterns with their rowindices - missings = ismissing.(data) - patterns = [missings[i, :] for i in 1:size(missings, 1)] - - patterns_cart = findall.(!, patterns) - data_rowwise = [data[i, patterns_cart[i]] for i in 1:nsamples] - data_rowwise = convert.(Array{Float64}, data_rowwise) - - remember = Vector{BitArray{1}}() - rows = [Vector{Int64}(undef, 0) for i in 1:size(patterns, 1)] - for i in 1:size(patterns, 1) - unknown = true - for j in 1:size(remember, 1) - if patterns[i] == remember[j] - push!(rows[j], i) - unknown = false - end - end - if unknown - push!(remember, patterns[i]) - push!(rows[size(remember, 1)], i) + # detect all different missing patterns with their row indices + pattern_to_rows = Dict{BitVector, Vector{Int}}() + for (i, datarow) in zip(axes(data, 1), eachrow(data)) + pattern = BitVector(.!ismissing.(datarow)) + if sum(pattern) > 0 # skip all-missing rows + pattern_rows = get!(() -> Vector{Int}(), pattern_to_rows, pattern) + push!(pattern_rows, i) end end - rows = rows[1:length(remember)] - n_patterns = size(rows, 1) - - # sort by number of missings - sort_n_miss = sortperm(sum.(remember)) - remember = remember[sort_n_miss] - remember_cart = findall.(!, remember) - remember_cart_not = findall.(remember) - rows = rows[sort_n_miss] - - pattern_nsamples = size.(rows, 1) - pattern_nobs_vars = length.(remember_cart) - - cov_mean = [cov_and_mean(data_rowwise[rows]) for rows in rows] - obs_cov = [cov_mean[1] for cov_mean in cov_mean] - obs_mean = [cov_mean[2] for cov_mean in cov_mean] + # process each pattern and sort from most to least number of observed vars + patterns = [ + SemObservedMissingPattern(pat, rows, data) for (pat, rows) in pairs(pattern_to_rows) + ] + sort!(patterns, by = nmissed_vars) + # allocate EM model (but don't fit) em_model = EmMVNModel(zeros(nobs_vars, nobs_vars), zeros(nobs_vars), false) - return SemObservedMissing( - data, - Symbol.(obs_colnames), - nsamples, - remember_cart, - remember_cart_not, - rows, - data_rowwise, - pattern_nsamples, - pattern_nobs_vars, - obs_mean, - obs_cov, - em_model, - ) + return SemObservedMissing(data, Symbol.(obs_colnames), nsamples, patterns, em_model) end ############################################################################################ @@ -209,12 +144,6 @@ end ### Additional methods ############################################################################################ -patterns(observed::SemObservedMissing) = observed.patterns -patterns_not(observed::SemObservedMissing) = observed.patterns_not -pattern_rows(observed::SemObservedMissing) = observed.pattern_rows -data_rowwise(observed::SemObservedMissing) = observed.data_rowwise -pattern_nsamples(observed::SemObservedMissing) = observed.pattern_nsamples -pattern_nobs_vars(observed::SemObservedMissing) = observed.pattern_nobs_vars -obs_mean(observed::SemObservedMissing) = observed.obs_mean -obs_cov(observed::SemObservedMissing) = observed.obs_cov em_model(observed::SemObservedMissing) = observed.em_model +obs_mean(observed::SemObservedMissing) = obs_mean(em_model(observed)) +obs_cov(observed::SemObservedMissing) = obs_cov(em_model(observed)) diff --git a/src/observed/missing_pattern.jl b/src/observed/missing_pattern.jl new file mode 100644 index 000000000..6ac6a360b --- /dev/null +++ b/src/observed/missing_pattern.jl @@ -0,0 +1,45 @@ +# data associated with the observed variables that all share the same missingness pattern +# variables that have values within that pattern are termed "measured" +# variables that have no measurements are termed "missing" +struct SemObservedMissingPattern{T, S} + measured_mask::BitVector # measured vars mask + miss_mask::BitVector # missing vars mask + rows::Vector{Int} # rows in original data + data::Matrix{T} # non-missing submatrix of data + + measured_mean::Vector{S} # means of measured vars + measured_cov::Matrix{S} # covariance of measured vars +end + +function SemObservedMissingPattern( + measured_mask::BitVector, + rows::AbstractVector{<:Integer}, + data::AbstractMatrix, +) + T = nonmissingtype(eltype(data)) + + pat_data = convert(Matrix{T}, view(data, rows, measured_mask)) + if size(pat_data, 1) > 1 + pat_mean, pat_cov = mean_and_cov(pat_data, 1, corrected = false) + @assert size(pat_cov) == (size(pat_data, 2), size(pat_data, 2)) + else + pat_mean = reshape(pat_data[1, :], 1, :) + # 1x1 covariance matrix since it is not meant to be used + pat_cov = fill(zero(T), 1, 1) + end + + return SemObservedMissingPattern{T, eltype(pat_mean)}( + measured_mask, + .!measured_mask, + rows, + pat_data, + dropdims(pat_mean, dims = 1), + pat_cov, + ) +end + +nobserved_vars(pat::SemObservedMissingPattern) = length(pat.measured_mask) +nsamples(pat::SemObservedMissingPattern) = length(pat.rows) + +nmeasured_vars(pat::SemObservedMissingPattern) = length(pat.measured_mean) +nmissed_vars(pat::SemObservedMissingPattern) = nobserved_vars(pat) - nmeasured_vars(pat) diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 9ab0c0af0..8791ebc12 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -340,13 +340,10 @@ end # SemObservedCovariance meanstructure, ) - @test @inferred(length(StructuralEquationModels.patterns(observed))) == 55 - @test sum(@inferred(StructuralEquationModels.pattern_nsamples(observed))) == + @test @inferred(length(observed.patterns)) == 55 + @test sum(@inferred(nsamples(pat)) for pat in observed.patterns) == size(dat_missing, 1) - @test all( - <=(size(dat_missing, 2)), - @inferred(StructuralEquationModels.pattern_nsamples(observed)) - ) + @test all(nsamples(pat) <= size(dat_missing, 2) for pat in observed.patterns) observed_nospec = SemObservedMissing( specification = nothing, From a848ed36763fe99734d7c95c97a57ac07b289490 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 18 Mar 2024 00:31:00 -0700 Subject: [PATCH 157/364] remove cov_and_mean(): not used anymore StatsBase.mean_and_cov() is used instead --- src/additional_functions/helper.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index be559b0d9..71b2559a8 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -98,11 +98,6 @@ function sparse_outer_mul!(C, A, B::Vector, ind) #computes A*S*B -> C, where ind end end -function cov_and_mean(rows; corrected = false) - obs_mean, obs_cov = StatsBase.mean_and_cov(reduce(hcat, rows), 2, corrected = corrected) - return obs_cov, vec(obs_mean) -end - # n²×(n(n+1)/2) matrix to transform a vector of lower # triangular entries into a vectorized form of a n×n symmetric matrix, # opposite of elimination_matrix() From c952792f6f2aa7c3e27223e754f37287a25795cd Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 Jan 2025 11:57:28 -0800 Subject: [PATCH 158/364] SemObserved: unify data preparation - SemObservedData: parameterize by cov/mean eltype instead of the whole container types Co-authored-by: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- src/frontend/specification/Sem.jl | 18 ++++-- src/observed/abstract.jl | 102 ++++++++++++++++++++++++++++++ src/observed/covariance.jl | 96 ++++++++++++++-------------- src/observed/data.jl | 93 +++++---------------------- src/observed/missing.jl | 83 ++++++------------------ 5 files changed, 194 insertions(+), 198 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 741d5f3c6..28984dbe9 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -3,6 +3,7 @@ ############################################################################################ function Sem(; + specification = ParameterTable, observed::O = SemObservedData, imply::I = RAM, loss::L = SemML, @@ -12,7 +13,7 @@ function Sem(; set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) - observed, imply, loss = get_fields!(kwdict, observed, imply, loss) + observed, imply, loss = get_fields!(kwdict, specification, observed, imply, loss) sem = Sem(observed, imply, loss) @@ -59,6 +60,7 @@ Returns the loss part of a model. loss(model::AbstractSemSingle) = model.loss function SemFiniteDiff(; + specification = ParameterTable, observed::O = SemObservedData, imply::I = RAM, loss::L = SemML, @@ -68,7 +70,7 @@ function SemFiniteDiff(; set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) - observed, imply, loss = get_fields!(kwdict, observed, imply, loss) + observed, imply, loss = get_fields!(kwdict, specification, observed, imply, loss) sem = SemFiniteDiff(observed, imply, loss) @@ -96,23 +98,27 @@ function set_field_type_kwargs!(kwargs, observed, imply, loss, O, I) end # construct Sem fields -function get_fields!(kwargs, observed, imply, loss) +function get_fields!(kwargs, specification, observed, imply, loss) + if !isa(specification, SemSpecification) + specification = specification(; kwargs...) + end + # observed if !isa(observed, SemObserved) - observed = observed(; kwargs...) + observed = observed(; specification, kwargs...) end kwargs[:observed] = observed # imply if !isa(imply, SemImply) - imply = imply(; kwargs...) + imply = imply(; specification, kwargs...) end kwargs[:imply] = imply kwargs[:nparams] = nparams(imply) # loss - loss = get_SemLoss(loss; kwargs...) + loss = get_SemLoss(loss; specification, kwargs...) kwargs[:loss] = loss return observed, imply, loss diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index 816dd9e80..53c0849c5 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -41,3 +41,105 @@ function source_to_dest_perm( return [src_inds[el] for el in dest] end end + +# function to prepare input data shared by SemObserved implementations +# returns tuple of +# 1) the matrix of data +# 2) the observed variable symbols that match matrix columns +# 3) the permutation of the original observed_vars (nothing if no reordering) +# If observed_vars is not specified, the vars order is taken from the specification. +# If both observed_vars and specification are provided, the observed_vars are used to match +# the column of the user-provided data matrix, and observed_vars(specification) is used to +# reorder the columns of the data to match the speciation. +# If no variable names are provided at all, generates the symbols in the form +# Symbol(observed_var_prefix, i) for i=1:nobserved_vars. +function prepare_data( + data::Union{AbstractDataFrame, AbstractMatrix, NTuple{2, Integer}, Nothing}, + observed_vars::Union{AbstractVector, Nothing}, + spec::Union{SemSpecification, Nothing}, +) + obs_vars = nothing + obs_vars_perm = nothing + if !isnothing(observed_vars) + obs_vars = Symbol.(observed_vars) + if !isnothing(spec) + obs_vars_spec = SEM.observed_vars(spec) + try + obs_vars_perm = source_to_dest_perm( + obs_vars, + obs_vars_spec, + one_to_one = false, + entities = "observed_vars", + ) + catch err + if isa(err, KeyError) + throw( + ArgumentError( + "observed_var \"$(err.key)\" from SEM specification is not listed in observed_vars argument", + ), + ) + else + rethrow(err) + end + end + # ignore trivial reorder + if obs_vars_perm == eachindex(obs_vars) + obs_vars_perm = nothing + end + end + elseif !isnothing(spec) + obs_vars = SEM.observed_vars(spec) + end + # observed vars in the order that matches the specification + obs_vars_reordered = isnothing(obs_vars_perm) ? obs_vars : obs_vars[obs_vars_perm] + + # subset the data, check that obs_vars matches data or guess the obs_vars + if data isa AbstractDataFrame + if !isnothing(obs_vars_reordered) # subset/reorder columns + data = data[:, obs_vars_reordered] + else # default symbol names + obs_vars = obs_vars_reordered = Symbol.(names(data)) + end + data_mtx = Matrix(data) + elseif data isa AbstractMatrix + if !isnothing(obs_vars) + size(data, 2) == length(obs_vars) || DimensionMismatch( + "The number of columns in the data matrix ($(size(data, 2))) does not match the length of observed_vars ($(length(obs_vars))).", + ) + # reorder columns to match the spec + data_ordered = !isnothing(obs_vars_perm) ? data[:, obs_vars_perm] : data + else + obs_vars = + obs_vars_reordered = + [Symbol(i) for i in axes(data, 2)] + data_ordered = data + end + # make sure data_mtx is a dense matrix (required for methods like mean_and_cov()) + data_mtx = convert(Matrix, data_ordered) + elseif data isa NTuple{2, Integer} # given the dimensions of the data matrix, but no data itself + data_mtx = nothing + nobs_vars = data[2] + if isnothing(obs_vars) + obs_vars = + obs_vars_reordered = [Symbol(i) for i in 1:nobs_vars] + elseif length(obs_vars) != nobs_vars + throw( + DimensionMismatch( + "The length of observed_vars ($(length(obs_vars))) does not match the data matrix columns ($(nobs_vars)).", + ), + ) + end + elseif isnothing(data) + data_mtx = nothing + if isnothing(obs_vars) + throw( + ArgumentError( + "No data, specification or observed_vars provided. Cannot infer observed_vars from provided inputs", + ), + ) + end + else + throw(ArgumentError("Unsupported data type: $(typeof(data))")) + end + return data_mtx, obs_vars_reordered, obs_vars_perm +end diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 195d55b4e..08917116f 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -3,79 +3,77 @@ Type alias for [`SemObservedData`](@ref) that has mean and covariance, but no ac For instances of `SemObservedCovariance` [`samples`](@ref) returns `nothing`. """ -const SemObservedCovariance{B, C} = SemObservedData{Nothing, B, C} +const SemObservedCovariance{S} = SemObservedData{Nothing, S} """ -For observed covariance matrices and means. - -# Constructor - SemObservedCovariance(; specification, obs_cov, obs_colnames = nothing, meanstructure = false, obs_mean = nothing, - nsamples = nothing, + nsamples::Integer, kwargs...) -# Arguments -- `specification`: either a `RAMMatrices` or `ParameterTable` object (1) -- `obs_cov`: observed covariance matrix -- `obs_colnames::Vector{Symbol}`: column names of the covariance matrix -- `meanstructure::Bool`: does the model have a meanstructure? -- `obs_mean`: observed mean vector -- `nsamples::Number`: number of samples (observed data points); necessary for fit statistics - -# Extended help -## Interfaces -- `nsamples(::SemObservedCovariance)`: number of samples (observed data points) -- `n_man(::SemObservedCovariance)` -> number of manifest variables - -- `obs_cov(::SemObservedCovariance)` -> observed covariance matrix -- `obs_mean(::SemObservedCovariance)` -> observed means - -## Implementation -Subtype of `SemObserved` +Construct [`SemObserved`](@ref) without providing the observations data, +but with the covariations (`obs_cov`) and the means (`obs_means`) of the observed variables. -## Remarks -(1) the `specification` argument can also be `nothing`, but this turns of checking whether -the observed data/covariance columns are in the correct order! As a result, you should only -use this if you are sure your covariance matrix is in the right format. +Returns [`SemObservedCovariance`](@ref) object. -## Additional keyword arguments: -- `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object +# Arguments +- `obs_cov`: pre-computed covariations of the observed variables +- `obs_mean`: optional pre-computed means of the observed variables +- `observed_vars::AbstractVector`: IDs of the observed variables (rows and columns of the `obs_cov` matrix) +- `specification`: optional SEM specification ([`SemSpecification`](@ref)) +- `nsamples::Number`: number of samples (observed data points) used to compute `obs_cov` and `obs_means` + necessary for calculating fit statistics """ function SemObservedCovariance(; - specification::Union{SemSpecification, Nothing} = nothing, obs_cov::AbstractMatrix, - obs_colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, - spec_colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, obs_mean::Union{AbstractVector, Nothing} = nothing, - meanstructure::Bool = false, + observed_vars::Union{AbstractVector, Nothing} = nothing, + specification::Union{SemSpecification, Nothing} = nothing, nsamples::Integer, kwargs..., ) - if !meanstructure && !isnothing(obs_mean) - throw(ArgumentError("observed means were passed, but `meanstructure = false`")) - elseif meanstructure && isnothing(obs_mean) - throw(ArgumentError("`meanstructure = true`, but no observed means were passed")) - end + nvars = size(obs_cov, 1) + size(obs_cov, 2) == nvars || throw( + DimensionMismatch( + "The covariance matrix should be square, $(size(obs_cov)) was found.", + ), + ) + S = eltype(obs_cov) - if isnothing(spec_colnames) && !isnothing(specification) - spec_colnames = observed_vars(specification) + if isnothing(obs_mean) + obs_mean = zeros(S, nvars) + else + length(obs_mean) == nvars || throw( + DimensionMismatch( + "The length of the mean vector $(length(obs_mean)) does not match the size of the covariance matrix $(size(obs_cov))", + ), + ) + S = promote_type(S, eltype(obs_mean)) end - if !isnothing(spec_colnames) && isnothing(obs_colnames) - throw(ArgumentError("no `obs_colnames` were specified")) + obs_cov = convert(Matrix{S}, obs_cov) + obs_mean = convert(Vector{S}, obs_mean) + + if !isnothing(observed_vars) + length(observed_vars) == nvars || throw( + DimensionMismatch( + "The length of the observed_vars $(length(observed_vars)) does not match the size of the covariance matrix $(size(obs_cov))", + ), + ) end - if !isnothing(spec_colnames) - obs2spec_perm = source_to_dest_perm(obs_colnames, spec_colnames) - obs_colnames = obs_colnames[obs2spec_perm] - obs_cov = obs_cov[obs2spec_perm, obs2spec_perm] - isnothing(obs_mean) || (obs_mean = obs_mean[obs2spec_perm]) + _, obs_vars, obs_vars_perm = + prepare_data((nsamples, nvars), observed_vars, specification) + + # reorder to match the specification + if !isnothing(obs_vars_perm) + obs_cov = obs_cov[obs_vars_perm, obs_vars_perm] + obs_mean = obs_mean[obs_vars_perm] end - return SemObservedData(nothing, Symbol.(obs_colnames), obs_cov, obs_mean, nsamples) + return SemObservedData(nothing, obs_vars, obs_cov, obs_mean, nsamples) end diff --git a/src/observed/data.jl b/src/observed/data.jl index ce4ce4bce..4af00e5a3 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -4,17 +4,15 @@ For observed data without missings. # Constructor SemObservedData(; - specification, data, - meanstructure = false, - obs_colnames = nothing, + observed_vars = nothing, + specification = nothing, kwargs...) # Arguments -- `specification`: either a `RAMMatrices` or `ParameterTable` object (1) -- `data`: observed data -- `meanstructure::Bool`: does the model have a meanstructure? -- `obs_colnames::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) +- `specification`: optional SEM specification ([`SemSpecification`](@ref)) +- `data`: observed data -- *DataFrame* or *Matrix* +- `observed_vars::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) # Extended help ## Interfaces @@ -27,87 +25,26 @@ For observed data without missings. ## Implementation Subtype of `SemObserved` - -## Remarks -(1) the `specification` argument can also be `nothing`, but this turns of checking whether -the observed data/covariance columns are in the correct order! As a result, you should only -use this if you are sure your observed data is in the right format. - -## Additional keyword arguments: -- `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object -- `compute_covariance::Bool ) = true`: should the covariance of `data` be computed and stored? """ -struct SemObservedData{A, B, C} <: SemObserved - data::A +struct SemObservedData{D <: Union{Nothing, AbstractMatrix}, S <: Number} <: SemObserved + data::D observed_vars::Vector{Symbol} - obs_cov::B - obs_mean::C + obs_cov::Matrix{S} + obs_mean::Vector{S} nsamples::Int end -# error checks -function check_arguments_SemObservedData(kwargs...) - # data is a data frame, - -end - function SemObservedData(; - specification::Union{SemSpecification, Nothing}, data, - obs_colnames = nothing, - spec_colnames = nothing, - meanstructure = false, - compute_covariance = true, + observed_vars::Union{AbstractVector, Nothing} = nothing, + specification::Union{SemSpecification, Nothing} = nothing, kwargs..., ) - if isnothing(spec_colnames) && !isnothing(specification) - spec_colnames = observed_vars(specification) - end + data, obs_vars, _ = + prepare_data(data, observed_vars, specification) + obs_mean, obs_cov = mean_and_cov(data, 1) - if !isnothing(spec_colnames) - if isnothing(obs_colnames) - try - data = data[:, spec_colnames] - obs_colnames = spec_colnames - catch - throw( - ArgumentError( - "Your `data` can not be indexed by symbols. " * - "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", - ), - ) - end - else - if data isa DataFrame - throw( - ArgumentError( - "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * - "Please make sure the column names of your data frame indicate the correct variables " * - "or pass your data in a different format.", - ), - ) - end - - if !(eltype(obs_colnames) <: Symbol) - throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) - end - - obs_colnames = obs_colnames[source_to_dest_perm(obs_colnames, spec_colnames)] - data = data[:, obs_colnames] - end - end - - if data isa DataFrame - data = Matrix(data) - end - - return SemObservedData( - data, - Symbol.(obs_colnames), - compute_covariance ? Statistics.cov(data) : nothing, - meanstructure ? vec(Statistics.mean(data, dims = 1)) : nothing, - size(data, 1), - ) + return SemObservedData(data, obs_vars, obs_cov, vec(obs_mean), size(data, 1)) end ############################################################################################ diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 96b027ae6..de1c93c95 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -19,40 +19,28 @@ For observed data with missing values. # Constructor SemObservedMissing(; - specification, data, - obs_colnames = nothing, + observed_vars = nothing, + specification = nothing, kwargs...) # Arguments -- `specification`: either a `RAMMatrices` or `ParameterTable` object (1) +- `specification`: optional SEM model specification ([`SemSpecification`](@ref)) - `data`: observed data -- `obs_colnames::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) +- `observed_vars::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) # Extended help ## Interfaces -- `nsamples(::SemObservedMissing)` -> number of observed data points -- `nobserved_vars(::SemObservedMissing)` -> number of manifest variables +- `nsamples(::SemObservedMissing)` -> number of samples (data points) +- `nobserved_vars(::SemObservedMissing)` -> number of observed variables -- `samples(::SemObservedMissing)` -> observed data +- `samples(::SemObservedMissing)` -> data matrix (contains both measured and missing values) - `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via expectation maximization ## Implementation Subtype of `SemObserved` - -## Remarks -(1) the `specification` argument can also be `nothing`, but this turns of checking whether -the observed data/covariance columns are in the correct order! As a result, you should only -use this if you are sure your observed data is in the right format. - -## Additional keyword arguments: -- `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object """ -struct SemObservedMissing{ - T <: Real, - S <: Real, - E <: EmMVNModel, -} <: SemObserved +struct SemObservedMissing{T <: Real, S <: Real, E <: EmMVNModel} <: SemObserved data::Matrix{Union{T, Missing}} observed_vars::Vector{Symbol} nsamples::Int @@ -66,53 +54,12 @@ end ############################################################################################ function SemObservedMissing(; - specification::Union{SemSpecification, Nothing}, data, - obs_colnames = nothing, - spec_colnames = nothing, + observed_vars::Union{AbstractVector, Nothing} = nothing, + specification::Union{SemSpecification, Nothing} = nothing, kwargs..., ) - if isnothing(spec_colnames) && !isnothing(specification) - spec_colnames = observed_vars(specification) - end - - if !isnothing(spec_colnames) - if isnothing(obs_colnames) - try - data = data[:, spec_colnames] - obs_colnames = spec_colnames - catch - throw( - ArgumentError( - "Your `data` can not be indexed by symbols. " * - "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", - ), - ) - end - else - if data isa DataFrame - throw( - ArgumentError( - "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * - "Please make sure the column names of your data frame indicate the correct variables " * - "or pass your data in a different format.", - ), - ) - end - - if !(eltype(obs_colnames) <: Symbol) - throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) - end - - obs_colnames = obs_colnames[source_to_dest_perm(obs_colnames, spec_colnames)] - data = data[:, obs_colnames] - end - end - - if data isa DataFrame - data = Matrix(data) - end - + data, obs_vars, _ = prepare_data(data, observed_vars, specification) nsamples, nobs_vars = size(data) # detect all different missing patterns with their row indices @@ -133,7 +80,13 @@ function SemObservedMissing(; # allocate EM model (but don't fit) em_model = EmMVNModel(zeros(nobs_vars, nobs_vars), zeros(nobs_vars), false) - return SemObservedMissing(data, Symbol.(obs_colnames), nsamples, patterns, em_model) + return SemObservedMissing( + convert(Matrix{Union{nonmissingtype(eltype(data)), Missing}}, data), + obs_vars, + nsamples, + patterns, + em_model, + ) end ############################################################################################ From 2596c61ff889e0f4fd1a29cd7cc65d58694130d1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 28 Dec 2024 12:18:27 -0800 Subject: [PATCH 159/364] tests: update SemObserved tests to match the update data preparation behaviour --- test/unit_tests/data_input_formats.jl | 349 ++++++++++++++++---------- 1 file changed, 213 insertions(+), 136 deletions(-) diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 8791ebc12..d93d02ad6 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -7,11 +7,19 @@ spec = ParameterTable( latent_vars = [:ind60, :dem60, :dem65], ) +# specification with non-existent observed var z1 +wrong_spec = ParameterTable( + observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :z1], + latent_vars = [:ind60, :dem60, :dem65], +) + ### data ----------------------------------------------------------------------------------- dat = example_data("political_democracy") dat_missing = example_data("political_democracy_missing")[:, names(dat)] +@assert Symbol.(names(dat)) == observed_vars(spec) + dat_matrix = Matrix(dat) dat_missing_matrix = Matrix(dat_missing) @@ -21,7 +29,12 @@ dat_mean = vcat(Statistics.mean(dat_matrix, dims = 1)...) # shuffle variables new_order = [3, 2, 7, 8, 5, 6, 9, 11, 1, 10, 4] -shuffle_names = Symbol.(names(dat))[new_order] +shuffle_names = names(dat)[new_order] + +shuffle_spec = ParameterTable( + observed_vars = Symbol.(shuffle_names), + latent_vars = [:ind60, :dem60, :dem65], +) shuffle_dat = dat[:, new_order] shuffle_dat_missing = dat_missing[:, new_order] @@ -29,8 +42,8 @@ shuffle_dat_missing = dat_missing[:, new_order] shuffle_dat_matrix = dat_matrix[:, new_order] shuffle_dat_missing_matrix = dat_missing_matrix[:, new_order] -shuffle_dat_cov = Statistics.cov(shuffle_dat_matrix) -shuffle_dat_mean = vcat(Statistics.mean(shuffle_dat_matrix, dims = 1)...) +shuffle_dat_cov = cov(shuffle_dat_matrix) +shuffle_dat_mean = vec(mean(shuffle_dat_matrix, dims = 1)) # common tests for SemObserved subtypes function test_observed( @@ -42,17 +55,16 @@ function test_observed( meanstructure::Bool, approx_cov::Bool = false, ) - @test @inferred(nobserved_vars(observed)) == size(dat, 2) - # FIXME observed should provide names of observed variables - @test @inferred(observed_vars(observed)) == names(dat) broken = true - @test @inferred(nsamples(observed)) == size(dat, 1) - - hasmissing = - !isnothing(dat_matrix) && any(ismissing, dat_matrix) || - !isnothing(dat_cov) && any(ismissing, dat_cov) + if !isnothing(dat) + @test @inferred(nsamples(observed)) == size(dat, 1) + @test @inferred(nobserved_vars(observed)) == size(dat, 2) + @test @inferred(observed_vars(observed)) == Symbol.(names(dat)) + end if !isnothing(dat_matrix) - if hasmissing + @test @inferred(nsamples(observed)) == size(dat_matrix, 1) + + if any(ismissing, dat_matrix) @test isequal(@inferred(samples(observed)), dat_matrix) else @test @inferred(samples(observed)) == dat_matrix @@ -60,7 +72,7 @@ function test_observed( end if !isnothing(dat_cov) - if hasmissing + if any(ismissing, dat_cov) @test isequal(@inferred(obs_cov(observed)), dat_cov) else if approx_cov @@ -72,17 +84,17 @@ function test_observed( end # FIXME actually, SemObserved should not use meanstructure and always provide obs_mean() - # meanstructure is a part of SEM model + # since meanstructure belongs to the implied part of a SEM model if meanstructure if !isnothing(dat_mean) - if hasmissing + if any(ismissing, dat_mean) @test isequal(@inferred(obs_mean(observed)), dat_mean) else @test @inferred(obs_mean(observed)) == dat_mean end else - # FIXME if meanstructure is present, obs_mean() should provide something (currently Missing don't support it) - @test (@inferred(obs_mean(observed)) isa AbstractVector{Float64}) broken = true + # FIXME @inferred is broken for EM cov/mean since it may return nothing if EM was not run + @test @inferred(obs_mean(observed)) isa AbstractVector{Float64} broken = true # EM-based means end else @test @inferred(obs_mean(observed)) === nothing skip = true @@ -93,32 +105,25 @@ end @testset "SemObservedData" begin # errors - @test_throws ArgumentError( - "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * - "Please make sure the column names of your data frame indicate the correct variables " * - "or pass your data in a different format.", - ) begin - SemObservedData( - specification = spec, - data = dat, - obs_colnames = Symbol.(names(dat)), - ) - end + obs_data_redundant = SemObservedData( + specification = spec, + data = dat, + observed_vars = Symbol.(names(dat)), + ) + @test observed_vars(obs_data_redundant) == Symbol.(names(dat)) + @test observed_vars(obs_data_redundant) == observed_vars(spec) - @test_throws ArgumentError( - "Your `data` can not be indexed by symbols. " * - "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", - ) begin - SemObservedData(specification = spec, data = dat_matrix) - end + obs_data_spec = SemObservedData(specification = spec, data = dat_matrix) + @test observed_vars(obs_data_spec) == observed_vars(spec) - @test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin - SemObservedData(specification = spec, data = dat_matrix, obs_colnames = names(dat)) - end + obs_data_strnames = + SemObservedData(specification = spec, data = dat_matrix, observed_vars = names(dat)) + @test observed_vars(obs_data_strnames) == Symbol.(names(dat)) @test_throws UndefKeywordError(:data) SemObservedData(specification = spec) - @test_throws UndefKeywordError(:specification) SemObservedData(data = dat_matrix) + obs_data_nonames = SemObservedData(data = dat_matrix) + @test observed_vars(obs_data_nonames) == Symbol.(1:size(dat_matrix, 2)) @testset "meanstructure=$meanstructure" for meanstructure in (false, true) observed = SemObservedData(specification = spec, data = dat; meanstructure) @@ -128,35 +133,92 @@ end observed_nospec = SemObservedData(specification = nothing, data = dat_matrix; meanstructure) - test_observed(observed_nospec, dat, dat_matrix, dat_cov, dat_mean; meanstructure) + test_observed( + observed_nospec, + nothing, + dat_matrix, + dat_cov, + dat_mean; + meanstructure, + ) observed_matrix = SemObservedData( specification = spec, data = dat_matrix, - obs_colnames = Symbol.(names(dat)), - meanstructure = meanstructure, + observed_vars = Symbol.(names(dat)); + meanstructure, ) test_observed(observed_matrix, dat, dat_matrix, dat_cov, dat_mean; meanstructure) + # detect non-existing column + @test_throws "ArgumentError: column name \"z1\"" SemObservedData( + specification = wrong_spec, + data = shuffle_dat, + ) + + # detect non-existing observed_var + @test_throws "ArgumentError: observed_var \"z1\"" SemObservedData( + specification = wrong_spec, + data = shuffle_dat_matrix, + observed_vars = shuffle_names, + ) + + # cannot infer observed_vars + @test_throws "No data, specification or observed_vars provided" SemObservedData( + data = nothing, + ) + + if false # FIXME data = nothing is for simulation studies + # no data, just observed_vars + observed_nodata = + SemObservedData(data = nothing, observed_vars = Symbol.(names(dat))) + @test observed_nodata isa SemObservedData + @test @inferred(samples(observed_nodata)) === nothing + @test observed_vars(observed_nodata) == Symbol.(names(dat)) + end + + # spec takes precedence in obs_vars order + observed_spec = SemObservedData( + specification = spec, + data = shuffle_dat, + observed_vars = shuffle_names, + ) + + test_observed( + observed_spec, + dat, + dat_matrix, + dat_cov, + meanstructure ? dat_mean : nothing; + meanstructure, + ) + observed_shuffle = - SemObservedData(specification = spec, data = shuffle_dat; meanstructure) + SemObservedData(specification = shuffle_spec, data = shuffle_dat; meanstructure) - test_observed(observed_shuffle, dat, dat_matrix, dat_cov, dat_mean; meanstructure) + test_observed( + observed_shuffle, + shuffle_dat, + shuffle_dat_matrix, + shuffle_dat_cov, + meanstructure ? shuffle_dat_mean : nothing; + meanstructure, + ) observed_matrix_shuffle = SemObservedData( - specification = spec, + specification = shuffle_spec, data = shuffle_dat_matrix, - obs_colnames = shuffle_names; + observed_vars = shuffle_names; meanstructure, ) test_observed( observed_matrix_shuffle, - dat, - dat_matrix, - dat_cov, - dat_mean; + shuffle_dat, + shuffle_dat_matrix, + shuffle_dat_cov, + meanstructure ? shuffle_dat_mean : nothing; meanstructure, ) end # meanstructure @@ -170,43 +232,6 @@ end # SemObservedData @test_throws UndefKeywordError(:nsamples) SemObservedCovariance(obs_cov = dat_cov) - @test_throws ArgumentError("no `obs_colnames` were specified") begin - SemObservedCovariance( - specification = spec, - obs_cov = dat_cov, - nsamples = size(dat, 1), - ) - end - - @test_throws ArgumentError("observed means were passed, but `meanstructure = false`") begin - SemObservedCovariance( - specification = nothing, - obs_cov = dat_cov, - obs_mean = dat_mean, - nsamples = size(dat, 1), - ) - end - - @test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin - SemObservedCovariance( - specification = spec, - obs_cov = dat_cov, - obs_colnames = names(dat), - nsamples = size(dat, 1), - meanstructure = false, - ) - end - - @test_throws ArgumentError("`meanstructure = true`, but no observed means were passed") begin - SemObservedCovariance( - specification = spec, - obs_cov = dat_cov, - obs_colnames = Symbol.(names(dat)), - meanstructure = true, - nsamples = size(dat, 1), - ) - end - @testset "meanstructure=$meanstructure" for meanstructure in (false, true) # errors @@ -220,12 +245,25 @@ end # SemObservedData meanstructure, ) - # should work + # default vars + observed_nonames = SemObservedCovariance( + obs_cov = dat_cov, + obs_mean = meanstructure ? dat_mean : nothing, + nsamples = size(dat, 1), + ) + @test observed_vars(observed_nonames) == Symbol.("obs", 1:size(dat_cov, 2)) + + @test_throws DimensionMismatch SemObservedCovariance( + obs_cov = dat_cov, + observed_vars = Symbol.("obs", 1:(size(dat_cov, 2)+1)), + nsamples = size(dat, 1), + ) + observed = SemObservedCovariance( specification = spec, obs_cov = dat_cov, obs_mean = meanstructure ? dat_mean : nothing, - obs_colnames = obs_colnames = Symbol.(names(dat)), + observed_vars = Symbol.(names(dat)), nsamples = size(dat, 1), meanstructure = meanstructure, ) @@ -252,7 +290,7 @@ end # SemObservedData test_observed( observed_nospec, - dat, + nothing, nothing, dat_cov, dat_mean; @@ -262,30 +300,51 @@ end # SemObservedData @test @inferred(samples(observed_nospec)) === nothing - observed_shuffle = SemObservedCovariance( + # detect non-existing observed_var + @test_throws "ArgumentError: observed_var \"z1\"" SemObservedCovariance( + specification = wrong_spec, + obs_cov = shuffle_dat_cov, + observed_vars = shuffle_names, + nsamples = size(dat, 1), + ) + + # spec takes precedence in obs_vars order + observed_spec = SemObservedCovariance( specification = spec, obs_cov = shuffle_dat_cov, - obs_mean = meanstructure ? dat_mean[new_order] : nothing, - obs_colnames = shuffle_names, - nsamples = size(dat, 1); - meanstructure, + obs_mean = meanstructure ? shuffle_dat_mean : nothing, + observed_vars = shuffle_names, + nsamples = size(dat, 1), ) test_observed( - observed_shuffle, + observed_spec, dat, nothing, dat_cov, - dat_mean; + meanstructure ? dat_mean : nothing; meanstructure, approx_cov = true, ) - @test @inferred(samples(observed_shuffle)) === nothing + observed_shuffle = SemObservedCovariance( + specification = shuffle_spec, + obs_cov = shuffle_dat_cov, + obs_mean = meanstructure ? shuffle_dat_mean : nothing, + observed_vars = shuffle_names, + nsamples = size(dat, 1); + meanstructure, + ) - # respect specification order - @test @inferred(obs_cov(observed_shuffle)) ≈ obs_cov(observed) - @test @inferred(observed_vars(observed_shuffle)) == shuffle_names broken = true + test_observed( + observed_shuffle, + shuffle_dat, + nothing, + shuffle_dat_cov, + meanstructure ? shuffle_dat_mean : nothing; + meanstructure, + approx_cov = true, + ) end # meanstructure end # SemObservedCovariance @@ -294,38 +353,27 @@ end # SemObservedCovariance @testset "SemObservedMissing" begin # errors - @test_throws ArgumentError( - "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * - "Please make sure the column names of your data frame indicate the correct variables " * - "or pass your data in a different format.", - ) begin - SemObservedMissing( - specification = spec, - data = dat_missing, - obs_colnames = Symbol.(names(dat)), - ) - end + observed_redundant_names = SemObservedMissing( + specification = spec, + data = dat_missing, + observed_vars = Symbol.(names(dat)), + ) + @test observed_vars(observed_redundant_names) == Symbol.(names(dat)) - @test_throws ArgumentError( - "Your `data` can not be indexed by symbols. " * - "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", - ) begin - SemObservedMissing(specification = spec, data = dat_missing_matrix) - end + observed_spec_only = SemObservedMissing(specification = spec, data = dat_missing_matrix) + @test observed_vars(observed_spec_only) == observed_vars(spec) - @test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin - SemObservedMissing( - specification = spec, - data = dat_missing_matrix, - obs_colnames = names(dat), - ) - end + observed_str_colnames = SemObservedMissing( + specification = spec, + data = dat_missing_matrix, + observed_vars = names(dat), + ) + @test observed_vars(observed_str_colnames) == Symbol.(names(dat)) @test_throws UndefKeywordError(:data) SemObservedMissing(specification = spec) - @test_throws UndefKeywordError(:specification) SemObservedMissing( - data = dat_missing_matrix, - ) + observed_no_names = SemObservedMissing(data = dat_missing_matrix) + @test observed_vars(observed_no_names) == Symbol.(1:size(dat_missing_matrix, 2)) @testset "meanstructure=$meanstructure" for meanstructure in (false, true) observed = @@ -353,7 +401,7 @@ end # SemObservedCovariance test_observed( observed_nospec, - dat_missing, + nothing, dat_missing_matrix, nothing, nothing; @@ -363,7 +411,7 @@ end # SemObservedCovariance observed_matrix = SemObservedMissing( specification = spec, data = dat_missing_matrix, - obs_colnames = Symbol.(names(dat)), + observed_vars = Symbol.(names(dat)), ) test_observed( @@ -375,11 +423,28 @@ end # SemObservedCovariance meanstructure, ) - observed_shuffle = - SemObservedMissing(specification = spec, data = shuffle_dat_missing) + # detect non-existing column + @test_throws "ArgumentError: column name \"z1\"" SemObservedMissing( + specification = wrong_spec, + data = shuffle_dat, + ) + + # detect non-existing observed_var + @test_throws "ArgumentError: observed_var \"z1\"" SemObservedMissing( + specification = wrong_spec, + data = shuffle_dat_missing_matrix, + observed_vars = shuffle_names, + ) + + # spec takes precedence in obs_vars order + observed_spec = SemObservedMissing( + specification = spec, + observed_vars = shuffle_names, + data = shuffle_dat_missing, + ) test_observed( - observed_shuffle, + observed_spec, dat_missing, dat_missing_matrix, nothing, @@ -387,16 +452,28 @@ end # SemObservedCovariance meanstructure, ) + observed_shuffle = + SemObservedMissing(specification = shuffle_spec, data = shuffle_dat_missing) + + test_observed( + observed_shuffle, + shuffle_dat_missing, + shuffle_dat_missing_matrix, + nothing, + nothing; + meanstructure, + ) + observed_matrix_shuffle = SemObservedMissing( - specification = spec, + specification = shuffle_spec, data = shuffle_dat_missing_matrix, - obs_colnames = shuffle_names, + observed_vars = shuffle_names, ) test_observed( observed_matrix_shuffle, - dat_missing, - dat_missing_matrix, + shuffle_dat_missing, + shuffle_dat_missing_matrix, nothing, nothing; meanstructure, From 80a64c90667c119f3b513fdef70daf3513f18c97 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 Jan 2025 13:06:40 -0800 Subject: [PATCH 160/364] prep_data: warn if obs_vars order don't match spec --- src/observed/abstract.jl | 3 +++ test/unit_tests/data_input_formats.jl | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index 53c0849c5..fb31d9752 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -97,6 +97,9 @@ function prepare_data( if data isa AbstractDataFrame if !isnothing(obs_vars_reordered) # subset/reorder columns data = data[:, obs_vars_reordered] + if obs_vars_reordered != obs_vars + @warn "The order of variables in observed_vars argument does not match the order of observed_vars(specification). The specification order is used." + end else # default symbol names obs_vars = obs_vars_reordered = Symbol.(names(data)) end diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index d93d02ad6..fe9421f55 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -178,6 +178,12 @@ end @test observed_vars(observed_nodata) == Symbol.(names(dat)) end + @test_warn "The order of variables in observed_vars" SemObservedData( + specification = spec, + data = shuffle_dat, + observed_vars = shuffle_names, + ) + # spec takes precedence in obs_vars order observed_spec = SemObservedData( specification = spec, From 673aa2b9cf272a371e039f71b3789d685ce13cd8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 25 Dec 2024 13:11:47 -0800 Subject: [PATCH 161/364] SemObsData: observed_var_prefix kwarg to specify the prefix of the generated observed_vars if none provided could be inferred, defaults to :obs --- src/observed/abstract.jl | 11 ++++++++--- src/observed/covariance.jl | 3 ++- src/observed/data.jl | 3 ++- src/observed/missing.jl | 4 +++- test/unit_tests/data_input_formats.jl | 12 ++++++++++-- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index fb31d9752..bb92ea12e 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -16,6 +16,10 @@ observed_vars(observed::SemObserved) = observed.observed_vars ### Additional functions ############################################################################################ +# generate default observed variable names if none provided +default_observed_vars(nobserved_vars::Integer, prefix::Union{Symbol, AbstractString}) = + Symbol.(prefix, 1:nobserved_vars) + # compute the permutation that subsets and reorders source elements # to match the destination order. # if multiple identical elements are present in the source, the last one is used. @@ -56,7 +60,8 @@ end function prepare_data( data::Union{AbstractDataFrame, AbstractMatrix, NTuple{2, Integer}, Nothing}, observed_vars::Union{AbstractVector, Nothing}, - spec::Union{SemSpecification, Nothing}, + spec::Union{SemSpecification, Nothing}; + observed_var_prefix::Union{Symbol, AbstractString}, ) obs_vars = nothing obs_vars_perm = nothing @@ -114,7 +119,7 @@ function prepare_data( else obs_vars = obs_vars_reordered = - [Symbol(i) for i in axes(data, 2)] + default_observed_vars(size(data, 2), observed_var_prefix) data_ordered = data end # make sure data_mtx is a dense matrix (required for methods like mean_and_cov()) @@ -124,7 +129,7 @@ function prepare_data( nobs_vars = data[2] if isnothing(obs_vars) obs_vars = - obs_vars_reordered = [Symbol(i) for i in 1:nobs_vars] + obs_vars_reordered = default_observed_vars(nobs_vars, observed_var_prefix) elseif length(obs_vars) != nobs_vars throw( DimensionMismatch( diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 08917116f..221ef5ca3 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -34,6 +34,7 @@ function SemObservedCovariance(; observed_vars::Union{AbstractVector, Nothing} = nothing, specification::Union{SemSpecification, Nothing} = nothing, nsamples::Integer, + observed_var_prefix::Union{Symbol, AbstractString} = :obs, kwargs..., ) nvars = size(obs_cov, 1) @@ -67,7 +68,7 @@ function SemObservedCovariance(; end _, obs_vars, obs_vars_perm = - prepare_data((nsamples, nvars), observed_vars, specification) + prepare_data((nsamples, nvars), observed_vars, specification; observed_var_prefix) # reorder to match the specification if !isnothing(obs_vars_perm) diff --git a/src/observed/data.jl b/src/observed/data.jl index 4af00e5a3..b6ddaa43d 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -38,10 +38,11 @@ function SemObservedData(; data, observed_vars::Union{AbstractVector, Nothing} = nothing, specification::Union{SemSpecification, Nothing} = nothing, + observed_var_prefix::Union{Symbol, AbstractString} = :obs, kwargs..., ) data, obs_vars, _ = - prepare_data(data, observed_vars, specification) + prepare_data(data, observed_vars, specification; observed_var_prefix) obs_mean, obs_cov = mean_and_cov(data, 1) return SemObservedData(data, obs_vars, obs_cov, vec(obs_mean), size(data, 1)) diff --git a/src/observed/missing.jl b/src/observed/missing.jl index de1c93c95..cf699252e 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -57,9 +57,11 @@ function SemObservedMissing(; data, observed_vars::Union{AbstractVector, Nothing} = nothing, specification::Union{SemSpecification, Nothing} = nothing, + observed_var_prefix::Union{Symbol, AbstractString} = :obs, kwargs..., ) - data, obs_vars, _ = prepare_data(data, observed_vars, specification) + data, obs_vars, _ = + prepare_data(data, observed_vars, specification; observed_var_prefix) nsamples, nobs_vars = size(data) # detect all different missing patterns with their row indices diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index fe9421f55..cc72673a6 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -123,7 +123,11 @@ end @test_throws UndefKeywordError(:data) SemObservedData(specification = spec) obs_data_nonames = SemObservedData(data = dat_matrix) - @test observed_vars(obs_data_nonames) == Symbol.(1:size(dat_matrix, 2)) + @test observed_vars(obs_data_nonames) == Symbol.("obs", 1:size(dat_matrix, 2)) + + obs_data_nonames2 = + SemObservedData(data = dat_matrix, observed_var_prefix = "observed_") + @test observed_vars(obs_data_nonames2) == Symbol.("observed_", 1:size(dat_matrix, 2)) @testset "meanstructure=$meanstructure" for meanstructure in (false, true) observed = SemObservedData(specification = spec, data = dat; meanstructure) @@ -379,7 +383,11 @@ end # SemObservedCovariance @test_throws UndefKeywordError(:data) SemObservedMissing(specification = spec) observed_no_names = SemObservedMissing(data = dat_missing_matrix) - @test observed_vars(observed_no_names) == Symbol.(1:size(dat_missing_matrix, 2)) + @test observed_vars(observed_no_names) == Symbol.(:obs, 1:size(dat_missing_matrix, 2)) + + observed_no_names2 = + SemObservedMissing(data = dat_missing_matrix, observed_var_prefix = "observed_") + @test observed_vars(observed_no_names2) == Symbol.("observed_", 1:size(dat_matrix, 2)) @testset "meanstructure=$meanstructure" for meanstructure in (false, true) observed = From 8c7cd1430fd0295a2d2eaf74f1ae098b63157174 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 2 Jan 2025 10:42:01 +0100 Subject: [PATCH 162/364] ParTable: add graph-based kw-only constructor --- src/frontend/specification/ParameterTable.jl | 4 ++-- src/frontend/specification/StenoGraphs.jl | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 8b7cc0973..07c24e46e 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -27,9 +27,9 @@ empty_partable_columns(nrows::Integer = 0) = Dict{Symbol, Vector}( :param => fill(Symbol(), nrows), ) -# construct using the provided columns data or create and empty table +# construct using the provided columns data or create an empty table function ParameterTable( - columns::Dict{Symbol, Vector} = empty_partable_columns(); + columns::Dict{Symbol, Vector}; observed_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, latent_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, params::Union{AbstractVector{Symbol}, Nothing} = nothing, diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 5cf87c07a..65bace302 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -129,6 +129,17 @@ function ParameterTable( return ParameterTable(columns; latent_vars, observed_vars, params) end +############################################################################################ +### keyword only constructor (for call in `Sem` constructor) +############################################################################################ + +# FIXME: this kw-only ctor conflicts with the empty ParTable constructor; +# it is left here for compatibility with the current Sem construction API, +# the proper fix would be to move away from kw-only ctors in general +ParameterTable(; graph::Union{AbstractStenoGraph, Nothing} = nothing, kwargs...) = + !isnothing(graph) ? ParameterTable(graph; kwargs...) : + ParameterTable(empty_partable_columns(); kwargs...) + ############################################################################################ ### constructor for EnsembleParameterTable from graph ############################################################################################ From ff67cf71d00ab226bc6de86a3b6ef6d8ba190f21 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 6 Jan 2025 11:42:01 -0800 Subject: [PATCH 163/364] Project.toml: fix ProximalAlgorithms to 0.5 v0.7 changed the diff interface (v0.6 was skipped) --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index ed5239c94..b0e421f21 100644 --- a/Project.toml +++ b/Project.toml @@ -34,6 +34,7 @@ NLSolversBase = "7" NLopt = "0.6, 1" Optim = "1" PrettyTables = "2" +ProximalAlgorithms = "0.5" StatsBase = "0.33, 0.34" Symbolics = "4, 5, 6" SymbolicUtils = "1.4 - 1.5, 1.7, 2, 3" From e63d5d8cba8fbdf90309a65b2480d0f6fa6b67a8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 6 Jan 2025 12:57:22 -0800 Subject: [PATCH 164/364] switch to ProximalAlgorithms.jl v0.7 also drop ProximalOperators and ProximalCore weak deps --- Project.toml | 6 ++---- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 11 ++++++++--- ext/SEMProximalOptExt/SEMProximalOptExt.jl | 4 +--- test/Project.toml | 1 - test/examples/proximal/l0.jl | 2 +- test/examples/proximal/lasso.jl | 2 +- test/examples/proximal/ridge.jl | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Project.toml b/Project.toml index b0e421f21..5937930d3 100644 --- a/Project.toml +++ b/Project.toml @@ -34,7 +34,7 @@ NLSolversBase = "7" NLopt = "0.6, 1" Optim = "1" PrettyTables = "2" -ProximalAlgorithms = "0.5" +ProximalAlgorithms = "0.7" StatsBase = "0.33, 0.34" Symbolics = "4, 5, 6" SymbolicUtils = "1.4 - 1.5, 1.7, 2, 3" @@ -48,9 +48,7 @@ test = ["Test"] [weakdeps] NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" -ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" -ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" [extensions] SEMNLOptExt = "NLopt" -SEMProximalOptExt = ["ProximalCore", "ProximalAlgorithms", "ProximalOperators"] +SEMProximalOptExt = "ProximalAlgorithms" diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index 13debf79d..f82c2b005 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -54,9 +54,14 @@ function Base.show(io::IO, struct_inst::SemOptimizerProximal) print_field_types(io, struct_inst) end -## connect do ProximalAlgorithms.jl as backend -ProximalCore.gradient!(grad, model::AbstractSem, parameters) = - objective_gradient!(grad, model::AbstractSem, parameters) +## connect to ProximalAlgorithms.jl +function ProximalAlgorithms.value_and_gradient(model::AbstractSem, params) + grad = similar(params) + obj = SEM.evaluate!(zero(eltype(params)), grad, nothing, model, params) + return obj, grad +end + +#ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) mutable struct ProximalResult result::Any diff --git a/ext/SEMProximalOptExt/SEMProximalOptExt.jl b/ext/SEMProximalOptExt/SEMProximalOptExt.jl index 8f91e03b0..156311367 100644 --- a/ext/SEMProximalOptExt/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt/SEMProximalOptExt.jl @@ -1,14 +1,12 @@ module SEMProximalOptExt using StructuralEquationModels -using ProximalCore, ProximalAlgorithms, ProximalOperators +using ProximalAlgorithms export SemOptimizerProximal SEM = StructuralEquationModels -#ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) - include("ProximalAlgorithms.jl") end diff --git a/test/Project.toml b/test/Project.toml index 14bd0bece..59db0b155 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -10,7 +10,6 @@ NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" -ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" diff --git a/test/examples/proximal/l0.jl b/test/examples/proximal/l0.jl index e8874fd51..da20f3901 100644 --- a/test/examples/proximal/l0.jl +++ b/test/examples/proximal/l0.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators +using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators # load data dat = example_data("political_democracy") diff --git a/test/examples/proximal/lasso.jl b/test/examples/proximal/lasso.jl index 31a4073f9..314453df4 100644 --- a/test/examples/proximal/lasso.jl +++ b/test/examples/proximal/lasso.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators +using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators # load data dat = example_data("political_democracy") diff --git a/test/examples/proximal/ridge.jl b/test/examples/proximal/ridge.jl index 120910234..16a318a12 100644 --- a/test/examples/proximal/ridge.jl +++ b/test/examples/proximal/ridge.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators +using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators # load data dat = example_data("political_democracy") From e2d6aa1b5baf0f119cb688313a1f24c86ef81492 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 11 Aug 2024 12:07:08 -0700 Subject: [PATCH 165/364] move params() to common.jl it is available for many SEM types, not just SemSpec --- src/frontend/common.jl | 7 +++++++ src/frontend/specification/documentation.jl | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/frontend/common.jl b/src/frontend/common.jl index 2be13c113..41d03effb 100644 --- a/src/frontend/common.jl +++ b/src/frontend/common.jl @@ -1,5 +1,12 @@ # API methods supported by multiple SEM.jl types +""" + params(semobj) -> Vector{Symbol} + +Return the vector of SEM model parameter identifiers. +""" +function params end + """ nparams(semobj) diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl index 46135ead0..72d95c6b4 100644 --- a/src/frontend/specification/documentation.jl +++ b/src/frontend/specification/documentation.jl @@ -1,10 +1,3 @@ -""" - params(semobj) -> Vector{Symbol} - -Return the vector of SEM model parameter identifiers. -""" -function params end - params(spec::SemSpecification) = spec.params """ From 92b5741840caa9e5ff307529eff7944c95941b83 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 1 Apr 2024 10:01:45 -0700 Subject: [PATCH 166/364] RAM ctor: use random parameters instead of NaNs to initialize RAM matrices simplify check_acyclic() --- src/imply/RAM/generic.jl | 41 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 850934a9c..69cbc517d 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -126,13 +126,11 @@ function RAM(; n_var = nvars(ram_matrices) #preallocate arrays - nan_params = fill(NaN, n_par) - A_pre = materialize(ram_matrices.A, nan_params) - S_pre = materialize(ram_matrices.S, nan_params) + rand_params = randn(Float64, n_par) + A_pre = check_acyclic(materialize(ram_matrices.A, rand_params)) + S_pre = materialize(ram_matrices.S, rand_params) F = copy(ram_matrices.F) - A_pre = check_acyclic(A_pre, ram_matrices.A) - # pre-allocate some matrices Σ = zeros(n_obs, n_obs) F⨉I_A⁻¹ = zeros(n_obs, n_var) @@ -155,7 +153,7 @@ function RAM(; "You set `meanstructure = true`, but your model specification contains no mean parameters.", ), ) - M_pre = materialize(ram_matrices.M, nan_params) + M_pre = materialize(ram_matrices.M, rand_params) ∇M = gradient_required ? sparse_gradient(ram_matrices.M) : nothing μ = zeros(n_obs) else @@ -229,22 +227,21 @@ end ### additional functions ############################################################################################ -function check_acyclic(A_pre::AbstractMatrix, A::ParamsMatrix) - # fill copy of A with random parameters - A_rand = materialize(A, rand(nparams(A))) - - # check if the model is acyclic - acyclic = isone(det(I - A_rand)) - +# checks if the A matrix is acyclic +# wraps A in LowerTriangular/UpperTriangular if it is triangular +function check_acyclic(A::AbstractMatrix) # check if A is lower or upper triangular - if istril(A_rand) - A_pre = LowerTriangular(A_pre) - elseif istriu(A_rand) - A_pre = UpperTriangular(A_pre) - elseif acyclic - @info "Your model is acyclic, specifying the A Matrix as either Upper or Lower Triangular can have great performance benefits.\n" maxlog = - 1 + if istril(A) + return LowerTriangular(A) + elseif istriu(A) + return UpperTriangular(A) + else + # check if non-triangular matrix is acyclic + acyclic = isone(det(I - A)) + if acyclic + @info "The matrix is acyclic. Reordering variables in the model to make the A matrix either Upper or Lower Triangular can significantly improve performance.\n" maxlog = + 1 + end + return A end - - return A_pre end From ae2291a3fa8342836f0e16e5904b93dc8219aa29 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 13 Jun 2024 00:13:08 -0700 Subject: [PATCH 167/364] move check_acyclic() to abstract.jl add verbose parameter --- src/imply/RAM/generic.jl | 23 ----------------------- src/imply/abstract.jl | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 69cbc517d..56960e4ff 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -222,26 +222,3 @@ function update_observed(imply::RAM, observed::SemObserved; kwargs...) return RAM(; observed = observed, kwargs...) end end - -############################################################################################ -### additional functions -############################################################################################ - -# checks if the A matrix is acyclic -# wraps A in LowerTriangular/UpperTriangular if it is triangular -function check_acyclic(A::AbstractMatrix) - # check if A is lower or upper triangular - if istril(A) - return LowerTriangular(A) - elseif istriu(A) - return UpperTriangular(A) - else - # check if non-triangular matrix is acyclic - acyclic = isone(det(I - A)) - if acyclic - @info "The matrix is acyclic. Reordering variables in the model to make the A matrix either Upper or Lower Triangular can significantly improve performance.\n" maxlog = - 1 - end - return A - end -end diff --git a/src/imply/abstract.jl b/src/imply/abstract.jl index 6a3f84191..37834415d 100644 --- a/src/imply/abstract.jl +++ b/src/imply/abstract.jl @@ -10,3 +10,24 @@ nlatent_vars(imply::SemImply) = nlatent_vars(imply.ram_matrices) params(imply::SemImply) = params(imply.ram_matrices) nparams(imply::SemImply) = nparams(imply.ram_matrices) + +# checks if the A matrix is acyclic +# wraps A in LowerTriangular/UpperTriangular if it is triangular +function check_acyclic(A::AbstractMatrix; verbose::Bool = false) + # check if A is lower or upper triangular + if istril(A) + verbose && @info "A matrix is lower triangular" + return LowerTriangular(A) + elseif istriu(A) + verbose && @info "A matrix is upper triangular" + return UpperTriangular(A) + else + # check if non-triangular matrix is acyclic + acyclic = isone(det(I - A)) + if acyclic + verbose && @info "The matrix is acyclic. Reordering variables in the model to make the A matrix either Upper or Lower Triangular can significantly improve performance.\n" maxlog = + 1 + end + return A + end +end From 317525774e812ef1549c44fa7d0203b89013af2a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 22 Dec 2024 12:44:19 -0800 Subject: [PATCH 168/364] AbstractSem: improve imply/observed API redirect --- src/frontend/specification/Sem.jl | 40 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 28984dbe9..d9b4a6e4e 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -20,45 +20,43 @@ function Sem(; return sem end -nvars(sem::AbstractSemSingle) = nvars(sem.imply) -nobserved_vars(sem::AbstractSemSingle) = nobserved_vars(sem.imply) -nlatent_vars(sem::AbstractSemSingle) = nlatent_vars(sem.imply) +""" + imply(model::AbstractSemSingle) -> SemImply -vars(sem::AbstractSemSingle) = vars(sem.imply) -observed_vars(sem::AbstractSemSingle) = observed_vars(sem.imply) -latent_vars(sem::AbstractSemSingle) = latent_vars(sem.imply) +Returns the [*implied*](@ref SemImply) part of a model. +""" +imply(model::AbstractSemSingle) = model.imply -nsamples(sem::AbstractSemSingle) = nsamples(sem.observed) +nvars(model::AbstractSemSingle) = nvars(imply(model)) +nobserved_vars(model::AbstractSemSingle) = nobserved_vars(imply(model)) +nlatent_vars(model::AbstractSemSingle) = nlatent_vars(imply(model)) -params(model::AbstractSem) = params(model.imply) +vars(model::AbstractSemSingle) = vars(imply(model)) +observed_vars(model::AbstractSemSingle) = observed_vars(imply(model)) +latent_vars(model::AbstractSemSingle) = latent_vars(imply(model)) -# sum of samples in all sub-models -nsamples(ensemble::SemEnsemble) = sum(nsamples, ensemble.sems) +params(model::AbstractSemSingle) = params(imply(model)) +nparams(model::AbstractSemSingle) = nparams(imply(model)) -############################################################################################ -# additional methods -############################################################################################ """ observed(model::AbstractSemSingle) -> SemObserved -Returns the observed part of a model. +Returns the [*observed*](@ref SemObserved) part of a model. """ observed(model::AbstractSemSingle) = model.observed -""" - imply(model::AbstractSemSingle) -> SemImply - -Returns the imply part of a model. -""" -imply(model::AbstractSemSingle) = model.imply +nsamples(model::AbstractSemSingle) = nsamples(observed(model)) """ loss(model::AbstractSemSingle) -> SemLoss -Returns the loss part of a model. +Returns the [*loss*](@ref SemLoss) function of a model. """ loss(model::AbstractSemSingle) = model.loss +# sum of samples in all sub-models +nsamples(ensemble::SemEnsemble) = sum(nsamples, ensemble.sems) + function SemFiniteDiff(; specification = ParameterTable, observed::O = SemObservedData, From c8b1645f118576c7835b6a486888285682f285a6 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 9 Jan 2025 14:24:53 -0800 Subject: [PATCH 169/364] imply -> implied, SemImply -> SemImplied --- docs/src/developer/imply.md | 38 ++++++------ docs/src/developer/loss.md | 10 ++-- docs/src/developer/observed.md | 2 +- docs/src/developer/sem.md | 14 ++--- docs/src/internals/files.md | 2 +- docs/src/internals/types.md | 2 +- docs/src/performance/symbolic.md | 2 +- docs/src/tutorials/concept.md | 26 ++++---- .../tutorials/construction/build_by_parts.md | 8 +-- .../construction/outer_constructor.md | 12 ++-- docs/src/tutorials/fitting/fitting.md | 2 +- docs/src/tutorials/meanstructure.md | 4 +- src/StructuralEquationModels.jl | 8 +-- src/additional_functions/helper.jl | 8 +-- src/additional_functions/simulation.jl | 30 +++++----- .../start_val/start_fabin3.jl | 10 ++-- .../start_val/start_simple.jl | 6 +- src/frontend/fit/fitmeasures/chi2.jl | 2 +- src/frontend/fit/fitmeasures/df.jl | 2 +- src/frontend/fit/fitmeasures/minus2ll.jl | 4 +- src/frontend/pretty_printing.jl | 4 +- src/frontend/specification/Sem.jl | 60 +++++++++---------- src/imply/RAM/generic.jl | 56 +++++++---------- src/imply/RAM/symbolic.jl | 26 ++++---- src/imply/abstract.jl | 18 +++--- src/imply/empty.jl | 18 +++--- src/loss/ML/FIML.jl | 28 ++++----- src/loss/ML/ML.jl | 8 +-- src/loss/WLS/WLS.jl | 2 +- src/loss/regularization/ridge.jl | 14 ++--- src/objective_gradient_hessian.jl | 44 +++++++------- src/optimizer/abstract.jl | 2 +- src/types.jl | 42 ++++++------- test/examples/multigroup/build_models.jl | 26 ++++---- test/examples/political_democracy/by_parts.jl | 38 ++++++------ .../political_democracy/constructor.jl | 16 ++--- .../recover_parameters_twofact.jl | 8 +-- test/unit_tests/model.jl | 10 ++-- test/unit_tests/sorting.jl | 2 +- 39 files changed, 299 insertions(+), 315 deletions(-) diff --git a/docs/src/developer/imply.md b/docs/src/developer/imply.md index cb30e40fe..403ecfa84 100644 --- a/docs/src/developer/imply.md +++ b/docs/src/developer/imply.md @@ -1,11 +1,11 @@ -# Custom imply types +# Custom implied types We recommend to first read the part [Custom loss functions](@ref), as the overall implementation is the same and we will describe it here more briefly. -Imply types are of subtype `SemImply`. To implement your own imply type, you should define a struct +Implied types are of subtype `SemImplied`. To implement your own implied type, you should define a struct ```julia -struct MyImply <: SemImply +struct MyImplied <: SemImplied ... end ``` @@ -15,37 +15,37 @@ and at least a method to compute the objective ```julia import StructuralEquationModels: objective! -function objective!(imply::MyImply, par, model::AbstractSemSingle) +function objective!(implied::MyImplied, par, model::AbstractSemSingle) ... return nothing end ``` -This method should compute and store things you want to make available to the loss functions, and returns `nothing`. For example, as we have seen in [Second example - maximum likelihood](@ref), the `RAM` imply type computes the model-implied covariance matrix and makes it available via `Σ(imply)`. -To make stored computations available to loss functions, simply write a function - for example, for the `RAM` imply type we defined +This method should compute and store things you want to make available to the loss functions, and returns `nothing`. For example, as we have seen in [Second example - maximum likelihood](@ref), the `RAM` implied type computes the model-implied covariance matrix and makes it available via `Σ(implied)`. +To make stored computations available to loss functions, simply write a function - for example, for the `RAM` implied type we defined ```julia -Σ(imply::RAM) = imply.Σ +Σ(implied::RAM) = implied.Σ ``` Additionally, you can specify methods for `gradient` and `hessian` as well as the combinations described in [Custom loss functions](@ref). -The last thing nedded to make it work is a method for `nparams` that takes your imply type and returns the number of parameters of the model: +The last thing nedded to make it work is a method for `nparams` that takes your implied type and returns the number of parameters of the model: ```julia -nparams(imply::MyImply) = ... +nparams(implied::MyImplied) = ... ``` Just as described in [Custom loss functions](@ref), you may define a constructor. Typically, this will depend on the `specification = ...` argument that can be a `ParameterTable` or a `RAMMatrices` object. -We implement an `ImplyEmpty` type in our package that does nothing but serving as an imply field in case you are using a loss function that does not need any imply type at all. You may use it as a template for defining your own imply type, as it also shows how to handle the specification objects: +We implement an `ImpliedEmpty` type in our package that does nothing but serving as an `implied` field in case you are using a loss function that does not need any implied type at all. You may use it as a template for defining your own implied type, as it also shows how to handle the specification objects: ```julia ############################################################################ ### Types ############################################################################ -struct ImplyEmpty{V, V2} <: SemImply +struct ImpliedEmpty{V, V2} <: SemImplied identifier::V2 n_par::V end @@ -54,7 +54,7 @@ end ### Constructors ############################################################################ -function ImplyEmpty(; +function ImpliedEmpty(; specification, kwargs...) @@ -63,25 +63,25 @@ function ImplyEmpty(; n_par = length(ram_matrices.parameters) - return ImplyEmpty(identifier, n_par) + return ImpliedEmpty(identifier, n_par) end ############################################################################ ### methods ############################################################################ -objective!(imply::ImplyEmpty, par, model) = nothing -gradient!(imply::ImplyEmpty, par, model) = nothing -hessian!(imply::ImplyEmpty, par, model) = nothing +objective!(implied::ImpliedEmpty, par, model) = nothing +gradient!(implied::ImpliedEmpty, par, model) = nothing +hessian!(implied::ImpliedEmpty, par, model) = nothing ############################################################################ ### Recommended methods ############################################################################ -identifier(imply::ImplyEmpty) = imply.identifier -n_par(imply::ImplyEmpty) = imply.n_par +identifier(implied::ImpliedEmpty) = implied.identifier +n_par(implied::ImpliedEmpty) = implied.n_par -update_observed(imply::ImplyEmpty, observed::SemObserved; kwargs...) = imply +update_observed(implied::ImpliedEmpty, observed::SemObserved; kwargs...) = implied ``` As you see, similar to [Custom loss functions](@ref) we implement a method for `update_observed`. Additionally, you should store the `identifier` from the specification object and write a method for `identifier`, as this will make it possible to access parameter indices by label. \ No newline at end of file diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index e1137dbf1..4f42a4700 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -171,7 +171,7 @@ function MyLoss(;arg1 = ..., arg2, kwargs...) end ``` -All keyword arguments that a user passes to the Sem constructor are passed to your loss function. In addition, all previously constructed parts of the model (imply and observed part) are passed as keyword arguments as well as the number of parameters `n_par = ...`, so your constructor may depend on those. For example, the constructor for `SemML` in our package depends on the additional argument `meanstructure` as well as the observed part of the model to pre-allocate arrays of the same size as the observed covariance matrix and the observed mean vector: +All keyword arguments that a user passes to the Sem constructor are passed to your loss function. In addition, all previously constructed parts of the model (implied and observed part) are passed as keyword arguments as well as the number of parameters `n_par = ...`, so your constructor may depend on those. For example, the constructor for `SemML` in our package depends on the additional argument `meanstructure` as well as the observed part of the model to pre-allocate arrays of the same size as the observed covariance matrix and the observed mean vector: ```julia function SemML(;observed, meanstructure = false, approx_H = false, kwargs...) @@ -221,9 +221,9 @@ To keep it simple, we only cover models without a meanstructure. The maximum lik F_{ML} = \log \det \Sigma_i + \mathrm{tr}\left(\Sigma_{i}^{-1} \Sigma_o \right) ``` -where ``\Sigma_i`` is the model implied covariance matrix and ``\Sigma_o`` is the observed covariance matrix. We can query the model implied covariance matrix from the `imply` par of our model, and the observed covariance matrix from the `observed` path of our model. +where ``\Sigma_i`` is the model implied covariance matrix and ``\Sigma_o`` is the observed covariance matrix. We can query the model implied covariance matrix from the `implied` par of our model, and the observed covariance matrix from the `observed` path of our model. -To get information on what we can access from a certain `imply` or `observed` type, we can check it`s documentation an the pages [API - model parts](@ref) or via the help mode of the REPL: +To get information on what we can access from a certain `implied` or `observed` type, we can check it`s documentation an the pages [API - model parts](@ref) or via the help mode of the REPL: ```julia julia>? @@ -233,7 +233,7 @@ help?> RAM help?> SemObservedCommon ``` -We see that the model implied covariance matrix can be assessed as `Σ(imply)` and the observed covariance matrix as `obs_cov(observed)`. +We see that the model implied covariance matrix can be assessed as `Σ(implied)` and the observed covariance matrix as `obs_cov(observed)`. With this information, we write can implement maximum likelihood optimization as @@ -245,7 +245,7 @@ import StructuralEquationModels: Σ, obs_cov, objective! function objective!(semml::MaximumLikelihood, parameters, model::AbstractSem) # access the model implied and observed covariance matrices - Σᵢ = Σ(imply(model)) + Σᵢ = Σ(implied(model)) Σₒ = obs_cov(observed(model)) # compute the objective if isposdef(Symmetric(Σᵢ)) # is the model implied covariance matrix positive definite? diff --git a/docs/src/developer/observed.md b/docs/src/developer/observed.md index 93eca6ed9..240c1c34f 100644 --- a/docs/src/developer/observed.md +++ b/docs/src/developer/observed.md @@ -28,7 +28,7 @@ nsamples(observed::MyObserved) = ... nobserved_vars(observed::MyObserved) = ... ``` -As always, you can add additional methods for properties that imply types and loss function want to access, for example (from the `SemObservedCommon` implementation): +As always, you can add additional methods for properties that implied types and loss function want to access, for example (from the `SemObservedCommon` implementation): ```julia obs_cov(observed::SemObservedCommon) = observed.obs_cov diff --git a/docs/src/developer/sem.md b/docs/src/developer/sem.md index 528da88b8..0063a85cf 100644 --- a/docs/src/developer/sem.md +++ b/docs/src/developer/sem.md @@ -1,15 +1,15 @@ # Custom model types -The abstract supertype for all models is `AbstractSem`, which has two subtypes, `AbstractSemSingle{O, I, L, D}` and `AbstractSemCollection`. Currently, there are 2 subtypes of `AbstractSemSingle`: `Sem`, `SemFiniteDiff`. All subtypes of `AbstractSemSingle` should have at least observed, imply, loss and optimizer fields, and share their types (`{O, I, L, D}`) with the parametric abstract supertype. For example, the `SemFiniteDiff` type is implemented as +The abstract supertype for all models is `AbstractSem`, which has two subtypes, `AbstractSemSingle{O, I, L, D}` and `AbstractSemCollection`. Currently, there are 2 subtypes of `AbstractSemSingle`: `Sem`, `SemFiniteDiff`. All subtypes of `AbstractSemSingle` should have at least observed, implied, loss and optimizer fields, and share their types (`{O, I, L, D}`) with the parametric abstract supertype. For example, the `SemFiniteDiff` type is implemented as ```julia struct SemFiniteDiff{ - O <: SemObserved, - I <: SemImply, - L <: SemLoss, + O <: SemObserved, + I <: SemImplied, + L <: SemLoss, D <: SemOptimizer} <: AbstractSemSingle{O, I, L, D} observed::O - imply::I + implied::I loss::L optimizer::D end @@ -19,13 +19,13 @@ Additionally, we need to define a method to compute at least the objective value ```julia function objective!(model::AbstractSemSingle, parameters) - objective!(imply(model), parameters, model) + objective!(implied(model), parameters, model) return objective!(loss(model), parameters, model) end function gradient!(gradient, model::AbstractSemSingle, parameters) fill!(gradient, zero(eltype(gradient))) - gradient!(imply(model), parameters, model) + gradient!(implied(model), parameters, model) gradient!(gradient, loss(model), parameters, model) end ``` diff --git a/docs/src/internals/files.md b/docs/src/internals/files.md index 06c73444d..9cf455fdc 100644 --- a/docs/src/internals/files.md +++ b/docs/src/internals/files.md @@ -10,7 +10,7 @@ All source code is in the `"src"` folder: - `"StructuralEquationModels.jl"` defines the module and the exported objects - `"types.jl"` defines all abstract types and the basic type hierarchy - `"objective_gradient_hessian.jl"` contains methods for computing objective, gradient and hessian values for different model types as well as generic fallback methods -- The four folders `"observed"`, `"imply"`, `"loss"` and `"diff"` contain implementations of specific subtypes (for example, the `"loss"` folder contains a file `"ML.jl"` that implements the `SemML` loss function). +- The four folders `"observed"`, `"implied"`, `"loss"` and `"diff"` contain implementations of specific subtypes (for example, the `"loss"` folder contains a file `"ML.jl"` that implements the `SemML` loss function). - `"optimizer"` contains connections to different optimization backends (aka methods for `sem_fit`) - `"optim.jl"`: connection to the `Optim.jl` package - `"NLopt.jl"`: connection to the `NLopt.jl` package diff --git a/docs/src/internals/types.md b/docs/src/internals/types.md index 488127b29..980d0f42f 100644 --- a/docs/src/internals/types.md +++ b/docs/src/internals/types.md @@ -8,6 +8,6 @@ The type hierarchy is implemented in `"src/types.jl"`. - `SemFiniteDiff`: models whose gradients and/or hessians should be computed via finite difference approximation - `AbstractSemCollection <: AbstractSem` is an abstract supertype of all models that contain multiple `AbstractSem` submodels -Every `AbstractSemSingle` has to have `SemObserved`, `SemImply`, `SemLoss` and `SemOptimizer` fields (and can have additional fields). +Every `AbstractSemSingle` has to have `SemObserved`, `SemImplied`, `SemLoss` and `SemOptimizer` fields (and can have additional fields). `SemLoss` is a container for multiple `SemLossFunctions`. \ No newline at end of file diff --git a/docs/src/performance/symbolic.md b/docs/src/performance/symbolic.md index 597d2c484..05729526e 100644 --- a/docs/src/performance/symbolic.md +++ b/docs/src/performance/symbolic.md @@ -13,6 +13,6 @@ If the model is acyclic, we can compute ``` for some ``n < \infty``. -Typically, the ``S`` and ``A`` matrices are sparse. In our package, we offer symbolic precomputation of ``\Sigma``, ``\nabla\Sigma`` and even ``\nabla^2\Sigma`` for acyclic models to optimally exploit this sparsity. To use this feature, simply use the `RAMSymbolic` imply type for your model. +Typically, the ``S`` and ``A`` matrices are sparse. In our package, we offer symbolic precomputation of ``\Sigma``, ``\nabla\Sigma`` and even ``\nabla^2\Sigma`` for acyclic models to optimally exploit this sparsity. To use this feature, simply use the `RAMSymbolic` implied type for your model. This can decrase model fitting time, but will also increase model building time (as we have to carry out the symbolic computations and compile specialised functions). As a result, this is probably not beneficial to use if you only fit a single model, but can lead to great improvements if you fit the same modle to multiple datasets (e.g. to compute bootstrap standard errors). \ No newline at end of file diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index c63c15941..b8d094abc 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -4,9 +4,9 @@ In our package, every Structural Equation Model (`Sem`) consists of four parts: ![SEM concept](../assets/concept.svg) -Those parts are interchangable building blocks (like 'Legos'), i.e. there are different pieces available you can choose as the 'observed' slot of the model, and stick them together with other pieces that can serve as the 'imply' part. +Those parts are interchangable building blocks (like 'Legos'), i.e. there are different pieces available you can choose as the `observed` slot of the model, and stick them together with other pieces that can serve as the `implied` part. -The 'observed' part is for observed data, the imply part is what the model implies about your data (e.g. the model implied covariance matrix), the loss part compares the observed data and implied properties (e.g. weighted least squares difference between the observed and implied covariance matrix) and the optimizer part connects to the optimization backend (e.g. the type of optimization algorithm used). +The `observed` part is for observed data, the `implied` part is what the model implies about your data (e.g. the model implied covariance matrix), the loss part compares the observed data and implied properties (e.g. weighted least squares difference between the observed and implied covariance matrix) and the optimizer part connects to the optimization backend (e.g. the type of optimization algorithm used). For example, to build a model for maximum likelihood estimation with the NLopt optimization suite as a backend you would choose `SemML` as a loss function and `SemOptimizerNLopt` as the optimizer. @@ -20,24 +20,24 @@ So everything that can be used as the 'observed' part has to be of type `SemObse Here is an overview on the available building blocks: -|[`SemObserved`](@ref) | [`SemImply`](@ref) | [`SemLossFunction`](@ref) | [`SemOptimizer`](@ref) | +|[`SemObserved`](@ref) | [`SemImplied`](@ref) | [`SemLossFunction`](@ref) | [`SemOptimizer`](@ref) | |---------------------------------|-----------------------|---------------------------|-------------------------------| | [`SemObservedData`](@ref) | [`RAM`](@ref) | [`SemML`](@ref) | [`SemOptimizerOptim`](@ref) | | [`SemObservedCovariance`](@ref) | [`RAMSymbolic`](@ref) | [`SemWLS`](@ref) | [`SemOptimizerNLopt`](@ref) | -| [`SemObservedMissing`](@ref) | [`ImplyEmpty`](@ref) | [`SemFIML`](@ref) | | -| | | [`SemRidge`](@ref) | | -| | | [`SemConstant`](@ref) | | +| [`SemObservedMissing`](@ref) | [`ImpliedEmpty`](@ref)| [`SemFIML`](@ref) | | +| | | [`SemRidge`](@ref) | | +| | | [`SemConstant`](@ref) | | The rest of this page explains the building blocks for each part. First, we explain every part and give an overview on the different options that are available. After that, the [API - model parts](@ref) section serves as a reference for detailed explanations about the different options. (How to stick them together to a final model is explained in the section on [Model Construction](@ref).) ## The observed part aka [`SemObserved`](@ref) -The 'observed' part contains all necessary information about the observed data. Currently, we have three options: [`SemObservedData`](@ref) for fully observed datasets, [`SemObservedCovariance`](@ref) for observed covariances (and means) and [`SemObservedMissing`](@ref) for data that contains missing values. +The *observed* part contains all necessary information about the observed data. Currently, we have three options: [`SemObservedData`](@ref) for fully observed datasets, [`SemObservedCovariance`](@ref) for observed covariances (and means) and [`SemObservedMissing`](@ref) for data that contains missing values. -## The imply part aka [`SemImply`](@ref) -The imply part is what your model implies about the data, for example, the model-implied covariance matrix. -There are two options at the moment: [`RAM`](@ref), which uses the reticular action model to compute the model implied covariance matrix, and [`RAMSymbolic`](@ref) which does the same but symbolically pre-computes part of the model, which increases subsequent performance in model fitting (see [Symbolic precomputation](@ref)). There is also a third option, [`ImplyEmpty`](@ref) that can serve as a 'placeholder' for models that do not need an imply part. +## The implied part aka [`SemImplied`](@ref) +The *implied* part is what your model implies about the data, for example, the model-implied covariance matrix. +There are two options at the moment: [`RAM`](@ref), which uses the reticular action model to compute the model implied covariance matrix, and [`RAMSymbolic`](@ref) which does the same but symbolically pre-computes part of the model, which increases subsequent performance in model fitting (see [Symbolic precomputation](@ref)). There is also a third option, [`ImpliedEmpty`](@ref) that can serve as a 'placeholder' for models that do not need an implied part. ## The loss part aka `SemLoss` The loss part specifies the objective that is optimized to find the parameter estimates. @@ -73,13 +73,13 @@ SemObservedCovariance SemObservedMissing ``` -## imply +## implied ```@docs -SemImply +SemImplied RAM RAMSymbolic -ImplyEmpty +ImpliedEmpty ``` ## loss functions diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 779949d98..071750a8c 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -1,6 +1,6 @@ # Build by parts -You can always build a model by parts - that is, you construct the observed, imply, loss and optimizer part seperately. +You can always build a model by parts - that is, you construct the observed, implied, loss and optimizer part seperately. As an example on how this works, we will build [A first model](@ref) in parts. @@ -50,8 +50,8 @@ Now, we construct the different parts: # observed --------------------------------------------------------------------------------- observed = SemObservedData(specification = partable, data = data) -# imply ------------------------------------------------------------------------------------ -imply_ram = RAM(specification = partable) +# implied ------------------------------------------------------------------------------------ +implied_ram = RAM(specification = partable) # loss ------------------------------------------------------------------------------------- ml = SemML(observed = observed) @@ -63,5 +63,5 @@ optimizer = SemOptimizerOptim() # model ------------------------------------------------------------------------------------ -model_ml = Sem(observed, imply_ram, loss_ml, optimizer) +model_ml = Sem(observed, implied_ram, loss_ml, optimizer) ``` \ No newline at end of file diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md index f072b80bc..0979f684a 100644 --- a/docs/src/tutorials/construction/outer_constructor.md +++ b/docs/src/tutorials/construction/outer_constructor.md @@ -15,13 +15,13 @@ Structural Equation Model SemML - Fields observed: SemObservedCommon - imply: RAM + implied: RAM optimizer: SemOptimizerOptim ``` -The output of this call tells you exactly what model you just constructed (i.e. what the loss functions, observed, imply and optimizer parts are). +The output of this call tells you exactly what model you just constructed (i.e. what the loss functions, observed, implied and optimizer parts are). -As you can see, by default, we use maximum likelihood estimation, the RAM imply type and the `Optim.jl` optimization backend. +As you can see, by default, we use maximum likelihood estimation, the RAM implied type and the `Optim.jl` optimization backend. To choose something different, you can provide it as a keyword argument: ```julia @@ -29,7 +29,7 @@ model = Sem( specification = partable, data = data, observed = ..., - imply = ..., + implied = ..., loss = ..., optimizer = ... ) @@ -41,7 +41,7 @@ For example, to construct a model for weighted least squares estimation that use model = Sem( specification = partable, data = data, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = SemWLS, optimizer = SemOptimizerNLopt ) @@ -73,7 +73,7 @@ W = ... model = Sem( specification = partable, data = data, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = SemWLS, wls_weight_matrix = W ) diff --git a/docs/src/tutorials/fitting/fitting.md b/docs/src/tutorials/fitting/fitting.md index f78a6c0db..b534ad754 100644 --- a/docs/src/tutorials/fitting/fitting.md +++ b/docs/src/tutorials/fitting/fitting.md @@ -16,7 +16,7 @@ Structural Equation Model SemML - Fields observed: SemObservedData - imply: RAM + implied: RAM optimizer: SemOptimizerOptim ------------- Optimization result ------------- diff --git a/docs/src/tutorials/meanstructure.md b/docs/src/tutorials/meanstructure.md index c6ad692b6..692f6cebc 100644 --- a/docs/src/tutorials/meanstructure.md +++ b/docs/src/tutorials/meanstructure.md @@ -106,11 +106,11 @@ For our example, ```@example meanstructure observed = SemObservedData(specification = partable, data = data, meanstructure = true) -imply_ram = RAM(specification = partable, meanstructure = true) +implied_ram = RAM(specification = partable, meanstructure = true) ml = SemML(observed = observed, meanstructure = true) -model = Sem(observed, imply_ram, SemLoss(ml), SemOptimizerOptim()) +model = Sem(observed, implied_ram, SemLoss(ml), SemOptimizerOptim()) sem_fit(model) ``` \ No newline at end of file diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a6677a4ed..a9a5af0d7 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -49,7 +49,7 @@ include("observed/EM.jl") # constructor include("frontend/specification/Sem.jl") include("frontend/specification/documentation.jl") -# imply +# implied include("imply/abstract.jl") include("imply/RAM/symbolic.jl") include("imply/RAM/generic.jl") @@ -95,11 +95,11 @@ export AbstractSem, HessianEval, ExactHessian, ApproxHessian, - SemImply, + SemImplied, RAMSymbolic, RAM, - ImplyEmpty, - imply, + ImpliedEmpty, + implied, start_val, start_fabin3, start_simple, diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 71b2559a8..5559034e0 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -21,14 +21,14 @@ function make_onelement_array(A) end =# -function semvec(observed, imply, loss, optimizer) +function semvec(observed, implied, loss, optimizer) observed = make_onelement_array(observed) - imply = make_onelement_array(imply) + implied = make_onelement_array(implied) loss = make_onelement_array(loss) optimizer = make_onelement_array(optimizer) - #sem_vec = Array{AbstractSem}(undef, maximum(length.([observed, imply, loss, optimizer]))) - sem_vec = Sem.(observed, imply, loss, optimizer) + #sem_vec = Array{AbstractSem}(undef, maximum(length.([observed, implied, loss, optimizer]))) + sem_vec = Sem.(observed, implied, loss, optimizer) return sem_vec end diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 0b2626b15..8c1a093a6 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -18,7 +18,7 @@ function swap_observed end """ update_observed(to_update, observed::SemObserved; kwargs...) -Update a `SemImply`, `SemLossFunction` or `SemOptimizer` object to use a `SemObserved` object. +Update a `SemImplied`, `SemLossFunction` or `SemOptimizer` object to use a `SemObserved` object. # Examples See the online documentation on [Swap observed data](@ref). @@ -45,7 +45,7 @@ swap_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = swap_observed( model, observed(model), - imply(model), + implied(model), loss(model), new_observed; kwargs..., @@ -54,7 +54,7 @@ swap_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = function swap_observed( model::AbstractSemSingle, old_observed, - imply, + implied, loss, new_observed::SemObserved; kwargs..., @@ -64,23 +64,23 @@ function swap_observed( # get field types kwargs[:observed_type] = typeof(new_observed) kwargs[:old_observed_type] = typeof(old_observed) - kwargs[:imply_type] = typeof(imply) + kwargs[:implied_type] = typeof(implied) kwargs[:loss_types] = [typeof(lossfun) for lossfun in loss.functions] - # update imply - imply = update_observed(imply, new_observed; kwargs...) - kwargs[:imply] = imply - kwargs[:nparams] = nparams(imply) + # update implied + implied = update_observed(implied, new_observed; kwargs...) + kwargs[:implied] = implied + kwargs[:nparams] = nparams(implied) # update loss loss = update_observed(loss, new_observed; kwargs...) kwargs[:loss] = loss - #new_imply = update_observed(model.imply, new_observed; kwargs...) + #new_implied = update_observed(model.implied, new_observed; kwargs...) return Sem( new_observed, - update_observed(model.imply, new_observed; kwargs...), + update_observed(model.implied, new_observed; kwargs...), update_observed(model.loss, new_observed; kwargs...), ) end @@ -117,7 +117,7 @@ function Distributions.rand( params, n::Integer, ) where {O, I <: Union{RAM, RAMSymbolic}, L} - update!(EvaluationTargets{true, false, false}(), model.imply, model, params) + update!(EvaluationTargets{true, false, false}(), model.implied, model, params) return rand(model, n) end @@ -125,10 +125,10 @@ function Distributions.rand( model::AbstractSemSingle{O, I, L}, n::Integer, ) where {O, I <: Union{RAM, RAMSymbolic}, L} - if MeanStruct(model.imply) === NoMeanStruct - data = permutedims(rand(MvNormal(Symmetric(model.imply.Σ)), n)) - elseif MeanStruct(model.imply) === HasMeanStruct - data = permutedims(rand(MvNormal(model.imply.μ, Symmetric(model.imply.Σ)), n)) + if MeanStruct(model.implied) === NoMeanStruct + data = permutedims(rand(MvNormal(Symmetric(model.implied.Σ)), n)) + elseif MeanStruct(model.implied) === HasMeanStruct + data = permutedims(rand(MvNormal(model.implied.μ, Symmetric(model.implied.Σ)), n)) end return data end diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index dd8d61fd9..bd55f21d7 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -8,20 +8,20 @@ function start_fabin3 end # splice model and loss functions function start_fabin3(model::AbstractSemSingle; kwargs...) - return start_fabin3(model.observed, model.imply, model.loss.functions..., kwargs...) + return start_fabin3(model.observed, model.implied, model.loss.functions..., kwargs...) end -function start_fabin3(observed, imply, args...; kwargs...) - return start_fabin3(imply.ram_matrices, obs_cov(observed), obs_mean(observed)) +function start_fabin3(observed, implied, args...; kwargs...) + return start_fabin3(implied.ram_matrices, obs_cov(observed), obs_mean(observed)) end # SemObservedMissing -function start_fabin3(observed::SemObservedMissing, imply, args...; kwargs...) +function start_fabin3(observed::SemObservedMissing, implied, args...; kwargs...) if !observed.em_model.fitted em_mvn(observed; kwargs...) end - return start_fabin3(imply.ram_matrices, observed.em_model.Σ, observed.em_model.μ) + return start_fabin3(implied.ram_matrices, observed.em_model.Σ, observed.em_model.μ) end function start_fabin3( diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 1f16b094c..ad5148e3f 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -17,11 +17,11 @@ function start_simple end # Single Models ---------------------------------------------------------------------------- function start_simple(model::AbstractSemSingle; kwargs...) - return start_simple(model.observed, model.imply, model.loss.functions...; kwargs...) + return start_simple(model.observed, model.implied, model.loss.functions...; kwargs...) end -function start_simple(observed, imply, args...; kwargs...) - return start_simple(imply.ram_matrices; kwargs...) +function start_simple(observed, implied, args...; kwargs...) + return start_simple(implied.ram_matrices; kwargs...) end # Ensemble Models -------------------------------------------------------------------------- diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index 12bc1d880..333783f95 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -13,7 +13,7 @@ function χ² end χ²(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) = χ²( sem_fit, sem_fit.model.observed, - sem_fit.model.imply, + sem_fit.model.implied, sem_fit.model.loss.functions..., ) diff --git a/src/frontend/fit/fitmeasures/df.jl b/src/frontend/fit/fitmeasures/df.jl index e8e72d594..4d9025601 100644 --- a/src/frontend/fit/fitmeasures/df.jl +++ b/src/frontend/fit/fitmeasures/df.jl @@ -13,7 +13,7 @@ df(model::AbstractSem) = n_dp(model) - nparams(model) function n_dp(model::AbstractSemSingle) nvars = nobserved_vars(model) ndp = 0.5(nvars^2 + nvars) - if !isnothing(model.imply.μ) + if !isnothing(model.implied.μ) ndp += nvars end return ndp diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 1cddee71d..2cb87d79c 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -15,7 +15,7 @@ minus2ll( ) = minus2ll( sem_fit, sem_fit.model.observed, - sem_fit.model.imply, + sem_fit.model.implied, sem_fit.model.loss.functions..., ) @@ -67,7 +67,7 @@ end ############################################################################################ minus2ll(minimum, model::AbstractSemSingle) = - minus2ll(minimum, model.observed, model.imply, model.loss.functions...) + minus2ll(minimum, model.observed, model.implied, model.loss.functions...) function minus2ll( sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}, diff --git a/src/frontend/pretty_printing.jl b/src/frontend/pretty_printing.jl index 5b732c980..c1cd72c2f 100644 --- a/src/frontend/pretty_printing.jl +++ b/src/frontend/pretty_printing.jl @@ -25,7 +25,7 @@ function print_type(io::IO, struct_instance) end ############################################################## -# Loss Functions, Imply, +# Loss Functions, Implied, ############################################################## function Base.show(io::IO, struct_inst::SemLossFunction) @@ -33,7 +33,7 @@ function Base.show(io::IO, struct_inst::SemLossFunction) print_field_types(io, struct_inst) end -function Base.show(io::IO, struct_inst::SemImply) +function Base.show(io::IO, struct_inst::SemImplied) print_type_name(io, struct_inst) print_field_types(io, struct_inst) end diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index d9b4a6e4e..33440e257 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -5,38 +5,38 @@ function Sem(; specification = ParameterTable, observed::O = SemObservedData, - imply::I = RAM, + implied::I = RAM, loss::L = SemML, kwargs..., ) where {O, I, L} kwdict = Dict{Symbol, Any}(kwargs...) - set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) + set_field_type_kwargs!(kwdict, observed, implied, loss, O, I) - observed, imply, loss = get_fields!(kwdict, specification, observed, imply, loss) + observed, implied, loss = get_fields!(kwdict, specification, observed, implied, loss) - sem = Sem(observed, imply, loss) + sem = Sem(observed, implied, loss) return sem end """ - imply(model::AbstractSemSingle) -> SemImply + implied(model::AbstractSemSingle) -> SemImplied -Returns the [*implied*](@ref SemImply) part of a model. +Returns the [*implied*](@ref SemImplied) part of a model. """ -imply(model::AbstractSemSingle) = model.imply +implied(model::AbstractSemSingle) = model.implied -nvars(model::AbstractSemSingle) = nvars(imply(model)) -nobserved_vars(model::AbstractSemSingle) = nobserved_vars(imply(model)) -nlatent_vars(model::AbstractSemSingle) = nlatent_vars(imply(model)) +nvars(model::AbstractSemSingle) = nvars(implied(model)) +nobserved_vars(model::AbstractSemSingle) = nobserved_vars(implied(model)) +nlatent_vars(model::AbstractSemSingle) = nlatent_vars(implied(model)) -vars(model::AbstractSemSingle) = vars(imply(model)) -observed_vars(model::AbstractSemSingle) = observed_vars(imply(model)) -latent_vars(model::AbstractSemSingle) = latent_vars(imply(model)) +vars(model::AbstractSemSingle) = vars(implied(model)) +observed_vars(model::AbstractSemSingle) = observed_vars(implied(model)) +latent_vars(model::AbstractSemSingle) = latent_vars(implied(model)) -params(model::AbstractSemSingle) = params(imply(model)) -nparams(model::AbstractSemSingle) = nparams(imply(model)) +params(model::AbstractSemSingle) = params(implied(model)) +nparams(model::AbstractSemSingle) = nparams(implied(model)) """ observed(model::AbstractSemSingle) -> SemObserved @@ -60,17 +60,17 @@ nsamples(ensemble::SemEnsemble) = sum(nsamples, ensemble.sems) function SemFiniteDiff(; specification = ParameterTable, observed::O = SemObservedData, - imply::I = RAM, + implied::I = RAM, loss::L = SemML, kwargs..., ) where {O, I, L} kwdict = Dict{Symbol, Any}(kwargs...) - set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) + set_field_type_kwargs!(kwdict, observed, implied, loss, O, I) - observed, imply, loss = get_fields!(kwdict, specification, observed, imply, loss) + observed, implied, loss = get_fields!(kwdict, specification, observed, implied, loss) - sem = SemFiniteDiff(observed, imply, loss) + sem = SemFiniteDiff(observed, implied, loss) return sem end @@ -79,9 +79,9 @@ end # functions ############################################################################################ -function set_field_type_kwargs!(kwargs, observed, imply, loss, O, I) +function set_field_type_kwargs!(kwargs, observed, implied, loss, O, I) kwargs[:observed_type] = O <: Type ? observed : typeof(observed) - kwargs[:imply_type] = I <: Type ? imply : typeof(imply) + kwargs[:implied_type] = I <: Type ? implied : typeof(implied) if loss isa SemLoss kwargs[:loss_types] = [ lossfun isa SemLossFunction ? typeof(lossfun) : lossfun for @@ -96,7 +96,7 @@ function set_field_type_kwargs!(kwargs, observed, imply, loss, O, I) end # construct Sem fields -function get_fields!(kwargs, specification, observed, imply, loss) +function get_fields!(kwargs, specification, observed, implied, loss) if !isa(specification, SemSpecification) specification = specification(; kwargs...) end @@ -107,19 +107,19 @@ function get_fields!(kwargs, specification, observed, imply, loss) end kwargs[:observed] = observed - # imply - if !isa(imply, SemImply) - imply = imply(; specification, kwargs...) + # implied + if !isa(implied, SemImplied) + implied = implied(; specification, kwargs...) end - kwargs[:imply] = imply - kwargs[:nparams] = nparams(imply) + kwargs[:implied] = implied + kwargs[:nparams] = nparams(implied) # loss loss = get_SemLoss(loss; specification, kwargs...) kwargs[:loss] = loss - return observed, imply, loss + return observed, implied, loss end # construct loss field @@ -164,7 +164,7 @@ function Base.show(io::IO, sem::Sem{O, I, L}) where {O, I, L} print(io, lossfuntypes...) print(io, "- Fields \n") print(io, " observed: $(nameof(O)) \n") - print(io, " imply: $(nameof(I)) \n") + print(io, " implied: $(nameof(I)) \n") end function Base.show(io::IO, sem::SemFiniteDiff{O, I, L}) where {O, I, L} @@ -175,7 +175,7 @@ function Base.show(io::IO, sem::SemFiniteDiff{O, I, L}) where {O, I, L} print(io, lossfuntypes...) print(io, "- Fields \n") print(io, " observed: $(nameof(O)) \n") - print(io, " imply: $(nameof(I)) \n") + print(io, " implied: $(nameof(I)) \n") end function Base.show(io::IO, loss::SemLoss) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 56960e4ff..30bd29bf4 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -20,7 +20,7 @@ Model implied covariance and means via RAM notation. # Extended help ## Implementation -Subtype of `SemImply`. +Subtype of `SemImplied`. ## RAM notation @@ -65,23 +65,7 @@ Additional interfaces Only available in gradient! calls: - `I_A⁻¹(::RAM)` -> ``(I-A)^{-1}`` """ -mutable struct RAM{ - MS, - A1, - A2, - A3, - A4, - A5, - A6, - V2, - M1, - M2, - M3, - M4, - S1, - S2, - S3, -} <: SemImply +mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, M1, M2, M3, M4, S1, S2, S3} <: SemImplied meanstruct::MS hessianeval::ExactHessian @@ -185,29 +169,29 @@ end ### methods ############################################################################################ -function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingle, params) - materialize!(imply.A, imply.ram_matrices.A, params) - materialize!(imply.S, imply.ram_matrices.S, params) - if !isnothing(imply.M) - materialize!(imply.M, imply.ram_matrices.M, params) +function update!(targets::EvaluationTargets, implied::RAM, model::AbstractSemSingle, params) + materialize!(implied.A, implied.ram_matrices.A, params) + materialize!(implied.S, implied.ram_matrices.S, params) + if !isnothing(implied.M) + materialize!(implied.M, implied.ram_matrices.M, params) end - @. imply.I_A = -imply.A - @view(imply.I_A[diagind(imply.I_A)]) .+= 1 + parent(implied.I_A) .= .-implied.A + @view(implied.I_A[diagind(implied.I_A)]) .+= 1 if is_gradient_required(targets) || is_hessian_required(targets) - imply.I_A⁻¹ = LinearAlgebra.inv!(factorize(imply.I_A)) - mul!(imply.F⨉I_A⁻¹, imply.F, imply.I_A⁻¹) + implied.I_A⁻¹ = LinearAlgebra.inv!(factorize(implied.I_A)) + mul!(implied.F⨉I_A⁻¹, implied.F, implied.I_A⁻¹) else - copyto!(imply.F⨉I_A⁻¹, imply.F) - rdiv!(imply.F⨉I_A⁻¹, factorize(imply.I_A)) + copyto!(implied.F⨉I_A⁻¹, implied.F) + rdiv!(implied.F⨉I_A⁻¹, factorize(implied.I_A)) end - mul!(imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹, imply.S) - mul!(imply.Σ, imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹') + mul!(implied.F⨉I_A⁻¹S, implied.F⨉I_A⁻¹, implied.S) + mul!(parent(implied.Σ), implied.F⨉I_A⁻¹S, implied.F⨉I_A⁻¹') - if MeanStruct(imply) === HasMeanStruct - mul!(imply.μ, imply.F⨉I_A⁻¹, imply.M) + if MeanStruct(implied) === HasMeanStruct + mul!(implied.μ, implied.F⨉I_A⁻¹, implied.M) end end @@ -215,9 +199,9 @@ end ### Recommended methods ############################################################################################ -function update_observed(imply::RAM, observed::SemObserved; kwargs...) - if nobserved_vars(observed) == size(imply.Σ, 1) - return imply +function update_observed(implied::RAM, observed::SemObserved; kwargs...) + if nobserved_vars(observed) == size(implied.Σ, 1) + return implied else return RAM(; observed = observed, kwargs...) end diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 32ffcc068..07acef019 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -2,7 +2,7 @@ ### Types ############################################################################################ @doc raw""" -Subtype of `SemImply` that implements the RAM notation with symbolic precomputation. +Subtype of `SemImplied` that implements the RAM notation with symbolic precomputation. # Constructor @@ -26,7 +26,7 @@ Subtype of `SemImply` that implements the RAM notation with symbolic precomputat # Extended help ## Implementation -Subtype of `SemImply`. +Subtype of `SemImplied`. ## Interfaces - `params(::RAMSymbolic) `-> vector of parameter ids @@ -63,7 +63,7 @@ and for models with a meanstructure, the model implied means are computed as ``` """ struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5} <: - SemImplySymbolic + SemImpliedSymbolic meanstruct::MS hessianeval::ExactHessian Σ_function::F1 @@ -201,19 +201,19 @@ end function update!( targets::EvaluationTargets, - imply::RAMSymbolic, + implied::RAMSymbolic, model::AbstractSemSingle, par, ) - imply.Σ_function(imply.Σ, par) - if MeanStruct(imply) === HasMeanStruct - imply.μ_function(imply.μ, par) + implied.Σ_function(implied.Σ, par) + if MeanStruct(implied) === HasMeanStruct + implied.μ_function(implied.μ, par) end if is_gradient_required(targets) || is_hessian_required(targets) - imply.∇Σ_function(imply.∇Σ, par) - if MeanStruct(imply) === HasMeanStruct - imply.∇μ_function(imply.∇μ, par) + implied.∇Σ_function(implied.∇Σ, par) + if MeanStruct(implied) === HasMeanStruct + implied.∇μ_function(implied.∇μ, par) end end end @@ -222,9 +222,9 @@ end ### Recommended methods ############################################################################################ -function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) - if nobserved_vars(observed) == size(imply.Σ, 1) - return imply +function update_observed(implied::RAMSymbolic, observed::SemObserved; kwargs...) + if nobserved_vars(observed) == size(implied.Σ, 1) + return implied else return RAMSymbolic(; observed = observed, kwargs...) end diff --git a/src/imply/abstract.jl b/src/imply/abstract.jl index 37834415d..05b0e2449 100644 --- a/src/imply/abstract.jl +++ b/src/imply/abstract.jl @@ -1,15 +1,15 @@ -# vars and params API methods for SemImply -vars(imply::SemImply) = vars(imply.ram_matrices) -observed_vars(imply::SemImply) = observed_vars(imply.ram_matrices) -latent_vars(imply::SemImply) = latent_vars(imply.ram_matrices) +# vars and params API methods for SemImplied +vars(implied::SemImplied) = vars(implied.ram_matrices) +observed_vars(implied::SemImplied) = observed_vars(implied.ram_matrices) +latent_vars(implied::SemImplied) = latent_vars(implied.ram_matrices) -nvars(imply::SemImply) = nvars(imply.ram_matrices) -nobserved_vars(imply::SemImply) = nobserved_vars(imply.ram_matrices) -nlatent_vars(imply::SemImply) = nlatent_vars(imply.ram_matrices) +nvars(implied::SemImplied) = nvars(implied.ram_matrices) +nobserved_vars(implied::SemImplied) = nobserved_vars(implied.ram_matrices) +nlatent_vars(implied::SemImplied) = nlatent_vars(implied.ram_matrices) -params(imply::SemImply) = params(imply.ram_matrices) -nparams(imply::SemImply) = nparams(imply.ram_matrices) +params(implied::SemImplied) = params(implied.ram_matrices) +nparams(implied::SemImplied) = nparams(implied.ram_matrices) # checks if the A matrix is acyclic # wraps A in LowerTriangular/UpperTriangular if it is triangular diff --git a/src/imply/empty.jl b/src/imply/empty.jl index 66373bc1b..e87dc72d1 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -2,19 +2,19 @@ ### Types ############################################################################################ """ -Empty placeholder for models that don't need an imply part. +Empty placeholder for models that don't need an implied part. (For example, models that only regularize parameters.) # Constructor - ImplyEmpty(;specification, kwargs...) + ImpliedEmpty(;specification, kwargs...) # Arguments - `specification`: either a `RAMMatrices` or `ParameterTable` object # Examples A multigroup model with ridge regularization could be specified as a `SemEnsemble` with one -model per group and an additional model with `ImplyEmpty` and `SemRidge` for the regularization part. +model per group and an additional model with `ImpliedEmpty` and `SemRidge` for the regularization part. # Extended help @@ -23,9 +23,9 @@ model per group and an additional model with `ImplyEmpty` and `SemRidge` for the - `nparams(::RAMSymbolic)` -> Number of parameters ## Implementation -Subtype of `SemImply`. +Subtype of `SemImplied`. """ -struct ImplyEmpty{V2} <: SemImply +struct ImpliedEmpty{V2} <: SemImplied hessianeval::ExactHessian meanstruct::NoMeanStruct ram_matrices::V2 @@ -35,18 +35,18 @@ end ### Constructors ############################################################################################ -function ImplyEmpty(; specification, kwargs...) - return ImplyEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) +function ImpliedEmpty(; specification, kwargs...) + return ImpliedEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) end ############################################################################################ ### methods ############################################################################################ -update!(targets::EvaluationTargets, imply::ImplyEmpty, par, model) = nothing +update!(targets::EvaluationTargets, implied::ImpliedEmpty, par, model) = nothing ############################################################################################ ### Recommended methods ############################################################################################ -update_observed(imply::ImplyEmpty, observed::SemObserved; kwargs...) = imply +update_observed(implied::ImpliedEmpty, observed::SemObserved; kwargs...) = implied diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index bf020d561..2c398090a 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -93,7 +93,7 @@ function evaluate!( gradient, hessian, semfiml::SemFIML, - implied::SemImply, + implied::SemImplied, model::AbstractSemSingle, params, ) @@ -148,20 +148,20 @@ function ∇F_one_pattern(μ_diff, Σ⁻¹, S, obs_mask, ∇ind, N, Jμ, JΣ, mo end end -function ∇F_fiml_outer!(G, JΣ, Jμ, imply::SemImplySymbolic, model, semfiml) - mul!(G, imply.∇Σ', JΣ) # should be transposed - mul!(G, imply.∇μ', Jμ, -1, 1) +function ∇F_fiml_outer!(G, JΣ, Jμ, implied::SemImpliedSymbolic, model, semfiml) + mul!(G, implied.∇Σ', JΣ) # should be transposed + mul!(G, implied.∇μ', Jμ, -1, 1) end -function ∇F_fiml_outer!(G, JΣ, Jμ, imply, model, semfiml) - Iₙ = sparse(1.0I, size(imply.A)...) - P = kron(imply.F⨉I_A⁻¹, imply.F⨉I_A⁻¹) - Q = kron(imply.S * imply.I_A⁻¹', Iₙ) +function ∇F_fiml_outer!(G, JΣ, Jμ, implied, model, semfiml) + Iₙ = sparse(1.0I, size(implied.A)...) + P = kron(implied.F⨉I_A⁻¹, implied.F⨉I_A⁻¹) + Q = kron(implied.S * implied.I_A⁻¹', Iₙ) Q .+= semfiml.commutator * Q - ∇Σ = P * (imply.∇S + Q * imply.∇A) + ∇Σ = P * (implied.∇S + Q * implied.∇A) - ∇μ = imply.F⨉I_A⁻¹ * imply.∇M + kron((imply.I_A⁻¹ * imply.M)', imply.F⨉I_A⁻¹) * imply.∇A + ∇μ = implied.F⨉I_A⁻¹ * implied.∇M + kron((implied.I_A⁻¹ * implied.M)', implied.F⨉I_A⁻¹) * implied.∇A mul!(G, ∇Σ', JΣ) # actually transposed mul!(G, ∇μ', Jμ, -1, 1) @@ -198,7 +198,7 @@ function ∇F_FIML!(G, observed::SemObservedMissing, semfiml, model) model, ) end - return ∇F_fiml_outer!(G, JΣ, Jμ, imply(model), model, semfiml) + return ∇F_fiml_outer!(G, JΣ, Jμ, implied(model), model, semfiml) end function prepare_SemFIML!(semfiml, model) @@ -212,8 +212,8 @@ function prepare_SemFIML!(semfiml, model) end function copy_per_pattern!(fiml::SemFIML, model::AbstractSem) - Σ = imply(model).Σ - μ = imply(model).μ + Σ = implied(model).Σ + μ = implied(model).μ data = observed(model) @inbounds @views for (i, pat) in enumerate(data.patterns) fiml.inverses[i] .= Σ[pat.measured_mask, pat.measured_mask] @@ -230,7 +230,7 @@ function batch_cholesky!(semfiml, model) end function check_fiml(semfiml, model) - copyto!(semfiml.imp_inv, imply(model).Σ) + copyto!(semfiml.imp_inv, implied(model).Σ) a = cholesky!(Symmetric(semfiml.imp_inv); check = false) return isposdef(a) end diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index e81d27de7..d14af648c 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -58,14 +58,14 @@ end ############################################################################################ ############################################################################################ -### Symbolic Imply Types +### Symbolic Implied Types function evaluate!( objective, gradient, hessian, semml::SemML, - implied::SemImplySymbolic, + implied::SemImpliedSymbolic, model::AbstractSemSingle, par, ) @@ -132,7 +132,7 @@ function evaluate!( end ############################################################################################ -### Non-Symbolic Imply Types +### Non-Symbolic Implied Types function evaluate!( objective, @@ -144,7 +144,7 @@ function evaluate!( par, ) if !isnothing(hessian) - error("hessian of ML + non-symbolic imply type is not available") + error("hessian of ML + non-symbolic implied type is not available") end Σ = implied.Σ diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 9702a9cf4..0fe2c9b3c 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -104,7 +104,7 @@ function evaluate!( gradient, hessian, semwls::SemWLS, - implied::SemImplySymbolic, + implied::SemImpliedSymbolic, model::AbstractSemSingle, par, ) diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index 6ec59ec39..02f637270 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -8,18 +8,18 @@ Ridge regularization. # Constructor - SemRidge(;α_ridge, which_ridge, nparams, parameter_type = Float64, imply = nothing, kwargs...) + SemRidge(;α_ridge, which_ridge, nparams, parameter_type = Float64, implied = nothing, kwargs...) # Arguments - `α_ridge`: hyperparameter for penalty term - `which_ridge::Vector`: Vector of parameter labels (Symbols) or indices that indicate which parameters should be regularized. - `nparams::Int`: number of parameters of the model -- `imply::SemImply`: imply part of the model +- `implied::SemImplied`: implied part of the model - `parameter_type`: type of the parameters # Examples ```julia -my_ridge = SemRidge(;α_ridge = 0.02, which_ridge = [:λ₁, :λ₂, :ω₂₃], nparams = 30, imply = my_imply) +my_ridge = SemRidge(;α_ridge = 0.02, which_ridge = [:λ₁, :λ₂, :ω₂₃], nparams = 30, implied = my_implied) ``` # Interfaces @@ -48,18 +48,18 @@ function SemRidge(; which_ridge, nparams, parameter_type = Float64, - imply = nothing, + implied = nothing, kwargs..., ) if eltype(which_ridge) <: Symbol - if isnothing(imply) + if isnothing(implied) throw( ArgumentError( - "When referring to parameters by label, `imply = ...` has to be specified", + "When referring to parameters by label, `implied = ...` has to be specified", ), ) else - par2ind = Dict(par => ind for (ind, par) in enumerate(params(imply))) + par2ind = Dict(par => ind for (ind, par) in enumerate(params(implied))) which_ridge = getindex.(Ref(par2ind), which_ridge) end end diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index f07b572aa..5b430e29e 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -23,28 +23,28 @@ is_hessian_required(::EvaluationTargets{<:Any, <:Any, H}) where {H} = H (targets::EvaluationTargets)(arg_tuple::Tuple) = targets(arg_tuple...) -# dispatch on SemImply +# dispatch on SemImplied evaluate!(objective, gradient, hessian, loss::SemLossFunction, model::AbstractSem, params) = - evaluate!(objective, gradient, hessian, loss, imply(model), model, params) + evaluate!(objective, gradient, hessian, loss, implied(model), model, params) # fallback method -function evaluate!(obj, grad, hess, loss::SemLossFunction, imply::SemImply, model, params) - isnothing(obj) || (obj = objective(loss, imply, model, params)) - isnothing(grad) || copyto!(grad, gradient(loss, imply, model, params)) - isnothing(hess) || copyto!(hess, hessian(loss, imply, model, params)) +function evaluate!(obj, grad, hess, loss::SemLossFunction, implied::SemImplied, model, params) + isnothing(obj) || (obj = objective(loss, implied, model, params)) + isnothing(grad) || copyto!(grad, gradient(loss, implied, model, params)) + isnothing(hess) || copyto!(hess, hessian(loss, implied, model, params)) return obj end # fallback methods -objective(f::SemLossFunction, imply::SemImply, model, params) = objective(f, model, params) -gradient(f::SemLossFunction, imply::SemImply, model, params) = gradient(f, model, params) -hessian(f::SemLossFunction, imply::SemImply, model, params) = hessian(f, model, params) - -# fallback method for SemImply that calls update_xxx!() methods -function update!(targets::EvaluationTargets, imply::SemImply, model, params) - is_objective_required(targets) && update_objective!(imply, model, params) - is_gradient_required(targets) && update_gradient!(imply, model, params) - is_hessian_required(targets) && update_hessian!(imply, model, params) +objective(f::SemLossFunction, implied::SemImplied, model, params) = objective(f, model, params) +gradient(f::SemLossFunction, implied::SemImplied, model, params) = gradient(f, model, params) +hessian(f::SemLossFunction, implied::SemImplied, model, params) = hessian(f, model, params) + +# fallback method for SemImplied that calls update_xxx!() methods +function update!(targets::EvaluationTargets, implied::SemImplied, model, params) + is_objective_required(targets) && update_objective!(implied, model, params) + is_gradient_required(targets) && update_gradient!(implied, model, params) + is_hessian_required(targets) && update_hessian!(implied, model, params) end # guess objective type @@ -72,8 +72,8 @@ objective_zero(objective, gradient, hessian) = function evaluate!(objective, gradient, hessian, model::AbstractSemSingle, params) targets = EvaluationTargets(objective, gradient, hessian) - # update imply state, its gradient and hessian (if required) - update!(targets, imply(model), model, params) + # update implied state, its gradient and hessian (if required) + update!(targets, implied(model), model, params) return evaluate!( !isnothing(objective) ? zero(objective) : nothing, gradient, @@ -90,8 +90,8 @@ end function evaluate!(objective, gradient, hessian, model::SemFiniteDiff, params) function obj(p) - # recalculate imply state for p - update!(EvaluationTargets{true, false, false}(), imply(model), model, p) + # recalculate implied state for p + update!(EvaluationTargets{true, false, false}(), implied(model), model, p) evaluate!( objective_zero(objective, gradient, hessian), nothing, @@ -165,7 +165,7 @@ Returns the objective value at `params`. The model object can be modified. # Implementation -To implement a new `SemImply` or `SemLossFunction` subtype, you need to add a method for +To implement a new `SemImplied` or `SemLossFunction` subtype, you need to add a method for objective!(newtype::MyNewType, params, model::AbstractSemSingle) To implement a new `AbstractSem` subtype, you need to add a method for @@ -179,7 +179,7 @@ function objective! end Writes the gradient value at `params` to `gradient`. # Implementation -To implement a new `SemImply` or `SemLossFunction` type, you can add a method for +To implement a new `SemImplied` or `SemLossFunction` type, you can add a method for gradient!(newtype::MyNewType, params, model::AbstractSemSingle) To implement a new `AbstractSem` subtype, you can add a method for @@ -193,7 +193,7 @@ function gradient! end Writes the hessian value at `params` to `hessian`. # Implementation -To implement a new `SemImply` or `SemLossFunction` type, you can add a method for +To implement a new `SemImplied` or `SemLossFunction` type, you can add a method for hessian!(newtype::MyNewType, params, model::AbstractSemSingle) To implement a new `AbstractSem` subtype, you can add a method for diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index c6669aa12..68bcc04ad 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -110,7 +110,7 @@ function prepare_param_bounds( default::Number, variance_default::Number, ) where {BOUND} - varparams = Set(variance_params(model.imply.ram_matrices)) + varparams = Set(variance_params(model.implied.ram_matrices)) res = [ begin def = in(p, varparams) ? variance_default : default diff --git a/src/types.jl b/src/types.jl index cfe916d9e..e802e057a 100644 --- a/src/types.jl +++ b/src/types.jl @@ -4,17 +4,17 @@ "Most abstract supertype for all SEMs" abstract type AbstractSem end -"Supertype for all single SEMs, e.g. SEMs that have at least the fields `observed`, `imply`, `loss`" +"Supertype for all single SEMs, e.g. SEMs that have at least the fields `observed`, `implied`, `loss`" abstract type AbstractSemSingle{O, I, L} <: AbstractSem end "Supertype for all collections of multiple SEMs" abstract type AbstractSemCollection <: AbstractSem end -"Meanstructure trait for `SemImply` subtypes" +"Meanstructure trait for `SemImplied` subtypes" abstract type MeanStruct end -"Indicates that `SemImply` subtype supports mean structure" +"Indicates that `SemImplied` subtype supports mean structure" struct HasMeanStruct <: MeanStruct end -"Indicates that `SemImply` subtype does not support mean structure" +"Indicates that `SemImplied` subtype does not support mean structure" struct NoMeanStruct <: MeanStruct end # default implementation @@ -24,7 +24,7 @@ MeanStruct(::Type{T}) where {T} = MeanStruct(semobj) = MeanStruct(typeof(semobj)) -"Hessian Evaluation trait for `SemImply` and `SemLossFunction` subtypes" +"Hessian Evaluation trait for `SemImplied` and `SemLossFunction` subtypes" abstract type HessianEval end struct ApproxHessian <: HessianEval end struct ExactHessian <: HessianEval end @@ -105,36 +105,36 @@ If you have a special kind of data, e.g. ordinal data, you should implement a su abstract type SemObserved end """ -Supertype of all objects that can serve as the imply field of a SEM. +Supertype of all objects that can serve as the implied field of a SEM. Computed model-implied values that should be compared with the observed data to find parameter estimates, e. g. the model implied covariance or mean. -If you would like to implement a different notation, e.g. LISREL, you should implement a subtype of SemImply. +If you would like to implement a different notation, e.g. LISREL, you should implement a subtype of SemImplied. """ -abstract type SemImply end +abstract type SemImplied end -"Subtype of SemImply for all objects that can serve as the imply field of a SEM and use some form of symbolic precomputation." -abstract type SemImplySymbolic <: SemImply end +"Subtype of SemImplied for all objects that can serve as the implied field of a SEM and use some form of symbolic precomputation." +abstract type SemImpliedSymbolic <: SemImplied end """ - Sem(;observed = SemObservedData, imply = RAM, loss = SemML, kwargs...) + Sem(;observed = SemObservedData, implied = RAM, loss = SemML, kwargs...) Constructor for the basic `Sem` type. -All additional kwargs are passed down to the constructors for the observed, imply, and loss fields. +All additional kwargs are passed down to the constructors for the observed, implied, and loss fields. # Arguments - `observed`: object of subtype `SemObserved` or a constructor. -- `imply`: object of subtype `SemImply` or a constructor. +- `implied`: object of subtype `SemImplied` or a constructor. - `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such. Returns a Sem with fields - `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref). -- `imply::SemImply`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImply`](@ref). +- `implied::SemImplied`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImplied`](@ref). - `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref). """ -mutable struct Sem{O <: SemObserved, I <: SemImply, L <: SemLoss} <: +mutable struct Sem{O <: SemObserved, I <: SemImplied, L <: SemLoss} <: AbstractSemSingle{O, I, L} observed::O - imply::I + implied::I loss::L end @@ -142,25 +142,25 @@ end # automatic differentiation ############################################################################################ """ - SemFiniteDiff(;observed = SemObservedData, imply = RAM, loss = SemML, kwargs...) + SemFiniteDiff(;observed = SemObservedData, implied = RAM, loss = SemML, kwargs...) A wrapper around [`Sem`](@ref) that substitutes dedicated evaluation of gradient and hessian with finite difference approximation. # Arguments - `observed`: object of subtype `SemObserved` or a constructor. -- `imply`: object of subtype `SemImply` or a constructor. +- `implied`: object of subtype `SemImplied` or a constructor. - `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such. Returns a Sem with fields - `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref). -- `imply::SemImply`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImply`](@ref). +- `implied::SemImplied`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImplied`](@ref). - `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref). """ -struct SemFiniteDiff{O <: SemObserved, I <: SemImply, L <: SemLoss} <: +struct SemFiniteDiff{O <: SemObserved, I <: SemImplied, L <: SemLoss} <: AbstractSemSingle{O, I, L} observed::O - imply::I + implied::I loss::L end diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 6991dd479..c97c9fb8e 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -4,11 +4,11 @@ const SEM = StructuralEquationModels # ML estimation ############################################################################################ -model_g1 = Sem(specification = specification_g1, data = dat_g1, imply = RAMSymbolic) +model_g1 = Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic) -model_g2 = Sem(specification = specification_g2, data = dat_g2, imply = RAM) +model_g2 = Sem(specification = specification_g2, data = dat_g2, implied = RAM) -@test SEM.params(model_g1.imply.ram_matrices) == SEM.params(model_g2.imply.ram_matrices) +@test SEM.params(model_g1.implied.ram_matrices) == SEM.params(model_g2.implied.ram_matrices) # test the different constructors model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) @@ -94,9 +94,9 @@ specification_s = convert(Dict{Symbol, RAMMatrices}, partable_s) specification_g1_s = specification_s[:Pasteur] specification_g2_s = specification_s[:Grant_White] -model_g1 = Sem(specification = specification_g1_s, data = dat_g1, imply = RAMSymbolic) +model_g1 = Sem(specification = specification_g1_s, data = dat_g1, implied = RAMSymbolic) -model_g2 = Sem(specification = specification_g2_s, data = dat_g2, imply = RAM) +model_g2 = Sem(specification = specification_g2_s, data = dat_g2, implied = RAM) model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) @@ -145,7 +145,7 @@ end end @testset "sorted | LowerTriangular A" begin - @test imply(model_ml_multigroup.sems[2]).A isa LowerTriangular + @test implied(model_ml_multigroup.sems[2]).A isa LowerTriangular end ############################################################################################ @@ -165,7 +165,7 @@ end using LinearAlgebra: isposdef, logdet, tr, inv function SEM.objective(ml::UserSemML, model::AbstractSem, params) - Σ = imply(model).Σ + Σ = implied(model).Σ Σₒ = SEM.obs_cov(observed(model)) if !isposdef(Σ) return Inf @@ -175,12 +175,12 @@ function SEM.objective(ml::UserSemML, model::AbstractSem, params) end # models -model_g1 = Sem(specification = specification_g1, data = dat_g1, imply = RAMSymbolic) +model_g1 = Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic) model_g2 = SemFiniteDiff( specification = specification_g2, data = dat_g2, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = UserSemML(), ) @@ -207,10 +207,10 @@ end ############################################################################################ model_ls_g1 = - Sem(specification = specification_g1, data = dat_g1, imply = RAMSymbolic, loss = SemWLS) + Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic, loss = SemWLS) model_ls_g2 = - Sem(specification = specification_g2, data = dat_g2, imply = RAMSymbolic, loss = SemWLS) + Sem(specification = specification_g2, data = dat_g2, implied = RAMSymbolic, loss = SemWLS) model_ls_multigroup = SemEnsemble(model_ls_g1, model_ls_g2; optimizer = semoptimizer) @@ -260,7 +260,7 @@ if !isnothing(specification_miss_g1) observed = SemObservedMissing, loss = SemFIML, data = dat_miss_g1, - imply = RAM, + implied = RAM, optimizer = SemOptimizerEmpty(), meanstructure = true, ) @@ -270,7 +270,7 @@ if !isnothing(specification_miss_g1) observed = SemObservedMissing, loss = SemFIML, data = dat_miss_g2, - imply = RAM, + implied = RAM, optimizer = SemOptimizerEmpty(), meanstructure = true, ) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 5e5244f91..c99115032 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -5,10 +5,10 @@ # observed --------------------------------------------------------------------------------- observed = SemObservedData(specification = spec, data = dat) -# imply -imply_ram = RAM(specification = spec) +# implied +implied_ram = RAM(specification = spec) -imply_ram_sym = RAMSymbolic(specification = spec) +implied_ram_sym = RAMSymbolic(specification = spec) # loss functions --------------------------------------------------------------------------- ml = SemML(observed = observed) @@ -29,18 +29,18 @@ optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- -model_ml = Sem(observed, imply_ram, loss_ml) +model_ml = Sem(observed, implied_ram, loss_ml) model_ls_sym = Sem(observed, RAMSymbolic(specification = spec, vech = true), loss_wls) -model_ml_sym = Sem(observed, imply_ram_sym, loss_ml) +model_ml_sym = Sem(observed, implied_ram_sym, loss_ml) -model_ridge = Sem(observed, imply_ram, SemLoss(ml, ridge)) +model_ridge = Sem(observed, implied_ram, SemLoss(ml, ridge)) -model_constant = Sem(observed, imply_ram, SemLoss(ml, constant)) +model_constant = Sem(observed, implied_ram, SemLoss(ml, constant)) model_ml_weighted = - Sem(observed, imply_ram, SemLoss(ml; loss_weights = [nsamples(model_ml)])) + Sem(observed, implied_ram, SemLoss(ml; loss_weights = [nsamples(model_ml)])) ############################################################################################ ### test gradients @@ -158,13 +158,13 @@ if opt_engine == :Optim ), ) - imply_sym_hessian_vech = RAMSymbolic(specification = spec, vech = true, hessian = true) + implied_sym_hessian_vech = RAMSymbolic(specification = spec, vech = true, hessian = true) - imply_sym_hessian = RAMSymbolic(specification = spec, hessian = true) + implied_sym_hessian = RAMSymbolic(specification = spec, hessian = true) - model_ls = Sem(observed, imply_sym_hessian_vech, loss_wls) + model_ls = Sem(observed, implied_sym_hessian_vech, loss_wls) - model_ml = Sem(observed, imply_sym_hessian, loss_ml) + model_ml = Sem(observed, implied_sym_hessian, loss_ml) @testset "ml_hessians" begin test_hessian(model_ml, start_test; atol = 1e-4) @@ -199,10 +199,10 @@ end # observed --------------------------------------------------------------------------------- observed = SemObservedData(specification = spec_mean, data = dat, meanstructure = true) -# imply -imply_ram = RAM(specification = spec_mean, meanstructure = true) +# implied +implied_ram = RAM(specification = spec_mean, meanstructure = true) -imply_ram_sym = RAMSymbolic(specification = spec_mean, meanstructure = true) +implied_ram_sym = RAMSymbolic(specification = spec_mean, meanstructure = true) # loss functions --------------------------------------------------------------------------- ml = SemML(observed = observed, meanstructure = true) @@ -218,7 +218,7 @@ loss_wls = SemLoss(wls) optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- -model_ml = Sem(observed, imply_ram, loss_ml) +model_ml = Sem(observed, implied_ram, loss_ml) model_ls = Sem( observed, @@ -226,7 +226,7 @@ model_ls = Sem( loss_wls, ) -model_ml_sym = Sem(observed, imply_ram_sym, loss_ml) +model_ml_sym = Sem(observed, implied_ram_sym, loss_ml) ############################################################################################ ### test gradients @@ -314,9 +314,9 @@ fiml = SemFIML(observed = observed, specification = spec_mean) loss_fiml = SemLoss(fiml) -model_ml = Sem(observed, imply_ram, loss_fiml) +model_ml = Sem(observed, implied_ram, loss_fiml) -model_ml_sym = Sem(observed, imply_ram_sym, loss_fiml) +model_ml_sym = Sem(observed, implied_ram_sym, loss_fiml) ############################################################################################ ### test gradients diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index cba86aef0..1d18ffed4 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -8,7 +8,7 @@ using Random, NLopt semoptimizer = SemOptimizer(engine = opt_engine) model_ml = Sem(specification = spec, data = dat) -@test SEM.params(model_ml.imply.ram_matrices) == SEM.params(spec) +@test SEM.params(model_ml.implied.ram_matrices) == SEM.params(spec) model_ml_cov = Sem( specification = spec, @@ -18,9 +18,9 @@ model_ml_cov = Sem( nsamples = 75, ) -model_ls_sym = Sem(specification = spec, data = dat, imply = RAMSymbolic, loss = SemWLS) +model_ls_sym = Sem(specification = spec, data = dat, implied = RAMSymbolic, loss = SemWLS) -model_ml_sym = Sem(specification = spec, data = dat, imply = RAMSymbolic) +model_ml_sym = Sem(specification = spec, data = dat, implied = RAMSymbolic) model_ridge = Sem( specification = spec, @@ -199,7 +199,7 @@ if opt_engine == :Optim model_ls = Sem( specification = spec, data = dat, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = SemWLS, hessian = true, algorithm = Newton(; @@ -211,7 +211,7 @@ if opt_engine == :Optim model_ml = Sem( specification = spec, data = dat, - imply = RAMSymbolic, + implied = RAMSymbolic, hessian = true, algorithm = Newton(), ) @@ -251,7 +251,7 @@ end model_ls = Sem( specification = spec_mean, data = dat, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = SemWLS, meanstructure = true, ) @@ -269,7 +269,7 @@ model_ml_cov = Sem( ) model_ml_sym = - Sem(specification = spec_mean, data = dat, imply = RAMSymbolic, meanstructure = true) + Sem(specification = spec_mean, data = dat, implied = RAMSymbolic, meanstructure = true) ############################################################################################ ### test gradients @@ -405,7 +405,7 @@ model_ml_sym = Sem( specification = spec_mean, data = dat_missing, observed = SemObservedMissing, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = SemFIML, meanstructure = true, ) diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 4b968bc49..6899fe7a7 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -53,11 +53,11 @@ start = [ repeat([0.5], 4) ] -imply_ml = RAMSymbolic(; specification = ram_matrices, start_val = start) +implied_ml = RAMSymbolic(; specification = ram_matrices, start_val = start) -imply_ml.Σ_function(imply_ml.Σ, true_val) +implied_ml.Σ_function(implied_ml.Σ, true_val) -true_dist = MultivariateNormal(imply_ml.Σ) +true_dist = MultivariateNormal(implied_ml.Σ) Random.seed!(1234) x = transpose(rand(true_dist, 100_000)) @@ -65,7 +65,7 @@ semobserved = SemObservedData(data = x, specification = nothing) loss_ml = SemLoss(SemML(; observed = semobserved, nparams = length(start))) -model_ml = Sem(semobserved, imply_ml, loss_ml) +model_ml = Sem(semobserved, implied_ml, loss_ml) objective!(model_ml, true_val) optimizer = SemOptimizerOptim( diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index bf44091d2..7ed190c22 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -46,25 +46,25 @@ function test_params_api(semobj, spec::SemSpecification) @test @inferred(params(semobj)) == params(spec) end -@testset "Sem(imply=$implytype, loss=$losstype)" for implytype in (RAM, RAMSymbolic), +@testset "Sem(implied=$impliedtype, loss=$losstype)" for impliedtype in (RAM, RAMSymbolic), losstype in (SemML, SemWLS) model = Sem( specification = ram_matrices, observed = obs, - imply = implytype, + implied = impliedtype, loss = losstype, ) @test model isa Sem - @test @inferred(imply(model)) isa implytype + @test @inferred(implied(model)) isa impliedtype @test @inferred(observed(model)) isa SemObserved test_vars_api(model, ram_matrices) test_params_api(model, ram_matrices) - test_vars_api(imply(model), ram_matrices) - test_params_api(imply(model), ram_matrices) + test_vars_api(implied(model), ram_matrices) + test_params_api(implied(model), ram_matrices) @test @inferred(loss(model)) isa SemLoss semloss = loss(model).functions[1] diff --git a/test/unit_tests/sorting.jl b/test/unit_tests/sorting.jl index f5bc38ae0..0908a6497 100644 --- a/test/unit_tests/sorting.jl +++ b/test/unit_tests/sorting.jl @@ -7,7 +7,7 @@ sort_vars!(partable) model_ml_sorted = Sem(specification = partable, data = dat) @testset "graph sorting" begin - @test model_ml_sorted.imply.I_A isa LowerTriangular + @test model_ml_sorted.implied.I_A isa LowerTriangular end @testset "ml_solution_sorted" begin From 39aee3d4d87d14a0763db4dcd922bbe599bc83ef Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 22 Dec 2024 19:45:32 -0800 Subject: [PATCH 170/364] imply -> implied: file renames --- docs/make.jl | 2 +- docs/src/developer/{imply.md => implied.md} | 0 src/StructuralEquationModels.jl | 8 ++++---- src/{imply => implied}/RAM/generic.jl | 0 src/{imply => implied}/RAM/symbolic.jl | 0 src/{imply => implied}/abstract.jl | 0 src/{imply => implied}/empty.jl | 0 7 files changed, 5 insertions(+), 5 deletions(-) rename docs/src/developer/{imply.md => implied.md} (100%) rename src/{imply => implied}/RAM/generic.jl (100%) rename src/{imply => implied}/RAM/symbolic.jl (100%) rename src/{imply => implied}/abstract.jl (100%) rename src/{imply => implied}/empty.jl (100%) diff --git a/docs/make.jl b/docs/make.jl index 4a55d55ce..4542cf48f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -32,7 +32,7 @@ makedocs( "Developer documentation" => [ "Extending the package" => "developer/extending.md", "Custom loss functions" => "developer/loss.md", - "Custom imply types" => "developer/imply.md", + "Custom implied types" => "developer/implied.md", "Custom optimizer types" => "developer/optimizer.md", "Custom observed types" => "developer/observed.md", "Custom model types" => "developer/sem.md", diff --git a/docs/src/developer/imply.md b/docs/src/developer/implied.md similarity index 100% rename from docs/src/developer/imply.md rename to docs/src/developer/implied.md diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a9a5af0d7..8bcf7a78d 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -50,10 +50,10 @@ include("observed/EM.jl") include("frontend/specification/Sem.jl") include("frontend/specification/documentation.jl") # implied -include("imply/abstract.jl") -include("imply/RAM/symbolic.jl") -include("imply/RAM/generic.jl") -include("imply/empty.jl") +include("implied/abstract.jl") +include("implied/RAM/symbolic.jl") +include("implied/RAM/generic.jl") +include("implied/empty.jl") # loss include("loss/ML/ML.jl") include("loss/ML/FIML.jl") diff --git a/src/imply/RAM/generic.jl b/src/implied/RAM/generic.jl similarity index 100% rename from src/imply/RAM/generic.jl rename to src/implied/RAM/generic.jl diff --git a/src/imply/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl similarity index 100% rename from src/imply/RAM/symbolic.jl rename to src/implied/RAM/symbolic.jl diff --git a/src/imply/abstract.jl b/src/implied/abstract.jl similarity index 100% rename from src/imply/abstract.jl rename to src/implied/abstract.jl diff --git a/src/imply/empty.jl b/src/implied/empty.jl similarity index 100% rename from src/imply/empty.jl rename to src/implied/empty.jl From 1fe165be02fc810197359cb2dff99e234464a7b7 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 2 Feb 2025 12:09:04 +0100 Subject: [PATCH 171/364] close #158 --- docs/src/tutorials/collection/multigroup.md | 2 +- docs/src/tutorials/constraints/constraints.md | 4 ++-- docs/src/tutorials/first_model.md | 6 +++--- docs/src/tutorials/inspection/inspection.md | 12 ++++++------ .../src/tutorials/regularization/regularization.md | 4 ++-- src/StructuralEquationModels.jl | 2 +- src/frontend/fit/summary.jl | 14 +++++++------- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/src/tutorials/collection/multigroup.md b/docs/src/tutorials/collection/multigroup.md index 5ee88e936..d0fc71796 100644 --- a/docs/src/tutorials/collection/multigroup.md +++ b/docs/src/tutorials/collection/multigroup.md @@ -83,7 +83,7 @@ We now fit the model and inspect the parameter estimates: ```@example mg; ansicolor = true solution = sem_fit(model_ml_multigroup) update_estimate!(partable, solution) -sem_summary(partable) +details(partable) ``` Other things you can query about your fitted model (fit measures, standard errors, etc.) are described in the section [Model inspection](@ref) and work the same way for multigroup models. \ No newline at end of file diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index a67ad7372..ffd83d4e0 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -52,7 +52,7 @@ model_fit = sem_fit(model) update_estimate!(partable, model_fit) -sem_summary(partable) +details(partable) ``` ### Define the constraints @@ -165,7 +165,7 @@ update_partable!( solution(model_fit_constrained), ) -sem_summary(partable) +details(partable) ``` As we can see, the constrained solution is very close to the original solution (compare the columns estimate and estimate_constr), with the difference that the constrained parameters fulfill their constraints. diff --git a/docs/src/tutorials/first_model.md b/docs/src/tutorials/first_model.md index 7568a5917..a285e29df 100644 --- a/docs/src/tutorials/first_model.md +++ b/docs/src/tutorials/first_model.md @@ -119,10 +119,10 @@ and compute fit measures as fit_measures(model_fit) ``` -We can also get a bit more information about the fitted model via the `sem_summary()` function: +We can also get a bit more information about the fitted model via the `details()` function: ```@example high_level; ansicolor = true -sem_summary(model_fit) +details(model_fit) ``` To investigate the parameter estimates, we can update our `partable` object to contain the new estimates: @@ -134,7 +134,7 @@ update_estimate!(partable, model_fit) and investigate the solution with ```@example high_level; ansicolor = true -sem_summary(partable) +details(partable) ``` Congratulations, you fitted and inspected your very first model! diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md index 88caf5812..faab8f8ed 100644 --- a/docs/src/tutorials/inspection/inspection.md +++ b/docs/src/tutorials/inspection/inspection.md @@ -53,10 +53,10 @@ model_fit = sem_fit(model) you end up with an object of type [`SemFit`](@ref). -You can get some more information about it by using the `sem_summary` function: +You can get some more information about it by using the `details` function: ```@example colored; ansicolor = true -sem_summary(model_fit) +details(model_fit) ``` To compute fit measures, we use @@ -73,12 +73,12 @@ AIC(model_fit) A list of available [Fit measures](@ref) is at the end of this page. -To inspect the parameter estimates, we can update a `ParameterTable` object and call `sem_summary` on it: +To inspect the parameter estimates, we can update a `ParameterTable` object and call `details` on it: ```@example colored; ansicolor = true; output = false update_estimate!(partable, model_fit) -sem_summary(partable) +details(partable) ``` We can also update the `ParameterTable` object with other information via [`update_partable!`](@ref). For example, if we want to compare hessian-based and bootstrap-based standard errors, we may write @@ -90,7 +90,7 @@ se_he = se_hessian(model_fit) update_partable!(partable, :se_hessian, params(model_fit), se_he) update_partable!(partable, :se_bootstrap, params(model_fit), se_bs) -sem_summary(partable) +details(partable) ``` ## Export results @@ -106,7 +106,7 @@ parameters_df = DataFrame(partable) # API - model inspection ```@docs -sem_summary +details update_estimate! update_partable! ``` diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index 4aaff1d0a..02d3b3bac 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -148,7 +148,7 @@ update_estimate!(partable, fit) update_partable!(partable, :estimate_lasso, params(fit_lasso), solution(fit_lasso)) -sem_summary(partable) +details(partable) ``` ## Second example - mixed l1 and l0 regularization @@ -182,5 +182,5 @@ Let's again compare the different results: ```@example reg update_partable!(partable, :estimate_mixed, params(fit_mixed), solution(fit_mixed)) -sem_summary(partable) +details(partable) ``` \ No newline at end of file diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a6677a4ed..b0ca407ff 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -131,7 +131,7 @@ export AbstractSem, SemFit, minimum, solution, - sem_summary, + details, objective!, gradient!, hessian!, diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index e6026e5f4..d9b137a58 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -1,4 +1,4 @@ -function sem_summary( +function details( sem_fit::SemFit; show_fitmeasures = false, color = :light_cyan, @@ -45,7 +45,7 @@ function sem_summary( print("\n") end -function sem_summary( +function details( partable::ParameterTable; color = :light_cyan, secondary_color = :light_yellow, @@ -250,7 +250,7 @@ function sem_summary( end -function sem_summary( +function details( partable::EnsembleParameterTable; color = :light_cyan, secondary_color = :light_yellow, @@ -291,7 +291,7 @@ function sem_summary( print("\n") printstyled(rpad(" Group: $k", 78), reverse = true) print("\n") - sem_summary( + details( partable.tables[k]; color = color, secondary_color = secondary_color, @@ -333,9 +333,9 @@ function Base.findall(fun::Function, partable::ParameterTable) end """ - (1) sem_summary(sem_fit::SemFit; show_fitmeasures = false) + (1) details(sem_fit::SemFit; show_fitmeasures = false) - (2) sem_summary(partable::AbstractParameterTable; ...) + (2) details(partable::AbstractParameterTable; ...) Print information about (1) a fitted SEM or (2) a parameter table to stdout. @@ -347,4 +347,4 @@ Print information about (1) a fitted SEM or (2) a parameter table to stdout. - `show_variables = true` - `show_columns = nothing`: columns names to include in the output e.g.`[:from, :to, :estimate]`) """ -function sem_summary end +function details end From e051d714fb631531b0ae7359ab5f76ad9871a766 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 2 Feb 2025 12:20:31 +0100 Subject: [PATCH 172/364] close #232 --- docs/src/developer/loss.md | 2 +- docs/src/performance/simulation.md | 12 +++++------ src/StructuralEquationModels.jl | 2 +- src/additional_functions/simulation.jl | 20 +++++++++---------- src/frontend/fit/standard_errors/bootstrap.jl | 4 ++-- .../political_democracy/constructor.jl | 8 ++++---- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index e1137dbf1..96d1ba566 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -195,7 +195,7 @@ end ### Update observed data If you are planing a simulation study where you have to fit the **same model** to many **different datasets**, it is computationally beneficial to not build the whole model completely new everytime you change your data. -Therefore, we provide a function to update the data of your model, `swap_observed(model(semfit); data = new_data)`. However, we can not know beforehand in what way your loss function depends on the specific datasets. The solution is to provide a method for `update_observed`. Since `Ridge` does not depend on the data at all, this is quite easy: +Therefore, we provide a function to update the data of your model, `replace_observed(model(semfit); data = new_data)`. However, we can not know beforehand in what way your loss function depends on the specific datasets. The solution is to provide a method for `update_observed`. Since `Ridge` does not depend on the data at all, this is quite easy: ```julia import StructuralEquationModels: update_observed diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index b8a5081fe..e46be64a7 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -7,12 +7,12 @@ ## Swap observed data In simulation studies, a common task is fitting the same model to many different datasets. It would be a waste of resources to reconstruct the complete model for each dataset. -We therefore provide the function `swap_observed` to change the `observed` part of a model, +We therefore provide the function `replace_observed` to change the `observed` part of a model, without necessarily reconstructing the other parts. For the [A first model](@ref), you would use it as -```@setup swap_observed +```@setup replace_observed using StructuralEquationModels observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] @@ -49,7 +49,7 @@ partable = ParameterTable( ) ``` -```@example swap_observed +```@example replace_observed data = example_data("political_democracy") data_1 = data[1:30, :] @@ -61,7 +61,7 @@ model = Sem( data = data_1 ) -model_updated = swap_observed(model; data = data_2, specification = partable) +model_updated = replace_observed(model; data = data_2, specification = partable) ``` !!! danger "Thread safety" @@ -76,7 +76,7 @@ model_updated = swap_observed(model; data = data_2, specification = partable) If you are building your models by parts, you can also update each part seperately with the function `update_observed`. For example, -```@example swap_observed +```@example replace_observed new_observed = SemObservedData(;data = data_2, specification = partable) @@ -88,6 +88,6 @@ new_optimizer = update_observed(my_optimizer, new_observed) ## API ```@docs -swap_observed +replace_observed update_observed ``` \ No newline at end of file diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a6677a4ed..ed49704f4 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -177,7 +177,7 @@ export AbstractSem, se_hessian, se_bootstrap, example_data, - swap_observed, + replace_observed, update_observed, @StenoGraph, →, diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 0b2626b15..e33b4f2fe 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -1,7 +1,7 @@ """ - (1) swap_observed(model::AbstractSemSingle; kwargs...) + (1) replace_observed(model::AbstractSemSingle; kwargs...) - (2) swap_observed(model::AbstractSemSingle, observed; kwargs...) + (2) replace_observed(model::AbstractSemSingle, observed; kwargs...) Return a new model with swaped observed part. @@ -13,7 +13,7 @@ Return a new model with swaped observed part. # Examples See the online documentation on [Swap observed data](@ref). """ -function swap_observed end +function replace_observed end """ update_observed(to_update, observed::SemObserved; kwargs...) @@ -34,15 +34,15 @@ function update_observed end ############################################################################################ # use the same observed type as before -swap_observed(model::AbstractSemSingle; kwargs...) = - swap_observed(model, typeof(observed(model)).name.wrapper; kwargs...) +replace_observed(model::AbstractSemSingle; kwargs...) = + replace_observed(model, typeof(observed(model)).name.wrapper; kwargs...) # construct a new observed type -swap_observed(model::AbstractSemSingle, observed_type; kwargs...) = - swap_observed(model, observed_type(; kwargs...); kwargs...) +replace_observed(model::AbstractSemSingle, observed_type; kwargs...) = + replace_observed(model, observed_type(; kwargs...); kwargs...) -swap_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = - swap_observed( +replace_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = + replace_observed( model, observed(model), imply(model), @@ -51,7 +51,7 @@ swap_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = kwargs..., ) -function swap_observed( +function replace_observed( model::AbstractSemSingle, old_observed, imply, diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index 9695e4cb3..e8d840d0c 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -7,7 +7,7 @@ Only works for single models. # Arguments - `n_boot`: number of boostrap samples - `data`: data to sample from. Only needed if different than the data from `sem_fit` -- `kwargs...`: passed down to `swap_observed` +- `kwargs...`: passed down to `replace_observed` """ function se_bootstrap( semfit::SemFit; @@ -42,7 +42,7 @@ function se_bootstrap( for _ in 1:n_boot sample_data = bootstrap_sample(data) - new_model = swap_observed( + new_model = replace_observed( model(semfit); data = sample_data, specification = specification, diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index cba86aef0..acab3b8f4 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -169,13 +169,13 @@ end Random.seed!(83472834) colnames = Symbol.(names(example_data("political_democracy"))) # simulate data - model_ml_new = swap_observed( + model_ml_new = replace_observed( model_ml, data = rand(model_ml, params, 1_000_000), specification = spec, obs_colnames = colnames, ) - model_ml_sym_new = swap_observed( + model_ml_sym_new = replace_observed( model_ml_sym, data = rand(model_ml_sym, params, 1_000_000), specification = spec, @@ -366,14 +366,14 @@ end Random.seed!(83472834) colnames = Symbol.(names(example_data("political_democracy"))) # simulate data - model_ml_new = swap_observed( + model_ml_new = replace_observed( model_ml, data = rand(model_ml, params, 1_000_000), specification = spec, obs_colnames = colnames, meanstructure = true, ) - model_ml_sym_new = swap_observed( + model_ml_sym_new = replace_observed( model_ml_sym, data = rand(model_ml_sym, params, 1_000_000), specification = spec, From 8b0f8805328bb89e71a56a9f31b00747f1a2a055 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Sun, 2 Feb 2025 13:37:59 +0100 Subject: [PATCH 173/364] Update ext/SEMProximalOptExt/ProximalAlgorithms.jl --- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index f82c2b005..94fcad247 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -61,7 +61,6 @@ function ProximalAlgorithms.value_and_gradient(model::AbstractSem, params) return obj, grad end -#ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) mutable struct ProximalResult result::Any From 8c703d6bd6add573cf6ebe3f2471909415c06de6 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 2 Feb 2025 15:00:13 +0100 Subject: [PATCH 174/364] suppress uninformative warnings during package testing --- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 1 - src/frontend/fit/summary.jl | 19 +++++----- .../specification/EnsembleParameterTable.jl | 8 ++--- src/frontend/specification/ParameterTable.jl | 2 +- src/frontend/specification/RAMMatrices.jl | 32 ++++++++++------- src/implied/abstract.jl | 5 +-- src/loss/ML/FIML.jl | 4 ++- src/objective_gradient_hessian.jl | 16 +++++++-- test/Project.toml | 1 + test/examples/multigroup/build_models.jl | 23 +++++++----- test/examples/multigroup/multigroup.jl | 8 ++--- test/examples/political_democracy/by_parts.jl | 7 ++-- .../political_democracy/constructor.jl | 4 +-- .../political_democracy.jl | 36 ++----------------- test/examples/proximal/ridge.jl | 4 +-- test/unit_tests/data_input_formats.jl | 6 ++-- 16 files changed, 83 insertions(+), 93 deletions(-) diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index 94fcad247..eceff0dc3 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -61,7 +61,6 @@ function ProximalAlgorithms.value_and_gradient(model::AbstractSem, params) return obj, grad end - mutable struct ProximalResult result::Any end diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index d9b137a58..70bf6816c 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -1,9 +1,4 @@ -function details( - sem_fit::SemFit; - show_fitmeasures = false, - color = :light_cyan, - digits = 2, -) +function details(sem_fit::SemFit; show_fitmeasures = false, color = :light_cyan, digits = 2) print("\n") println("Fitted Structural Equation Model") print("\n") @@ -51,7 +46,7 @@ function details( secondary_color = :light_yellow, digits = 2, show_variables = true, - show_columns = nothing + show_columns = nothing, ) if show_variables print("\n") @@ -150,7 +145,8 @@ function details( check_round(partable.columns[c][regression_indices]; digits = digits) for c in regression_columns ) - regression_columns[2] = regression_columns[2] == :relation ? Symbol("") : regression_columns[2] + regression_columns[2] = + regression_columns[2] == :relation ? Symbol("") : regression_columns[2] print("\n") pretty_table( @@ -222,7 +218,8 @@ function details( printstyled("Means: \n"; color = color) if isnothing(show_columns) - sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = + [:from, :relation, :to, :estimate, :param, :value_fixed, :start] mean_columns = sort_partially(sorted_columns, columns) else mean_columns = copy(show_columns) @@ -256,7 +253,7 @@ function details( secondary_color = :light_yellow, digits = 2, show_variables = true, - show_columns = nothing + show_columns = nothing, ) if show_variables print("\n") @@ -297,7 +294,7 @@ function details( secondary_color = secondary_color, digits = digits, show_variables = false, - show_columns = show_columns + show_columns = show_columns, ) end diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index b1c8fb8e6..d5ac7e51b 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -19,7 +19,7 @@ EnsembleParameterTable(::Nothing; params::Union{Nothing, Vector{Symbol}} = nothi ) # convert pairs to dict -EnsembleParameterTable(ps::Pair{K, V}...; params = nothing) where {K, V} = +EnsembleParameterTable(ps::Pair{K, V}...; params = nothing) where {K, V} = EnsembleParameterTable(Dict(ps...); params = params) # dictionary of SEM specifications @@ -148,8 +148,6 @@ end ############################################################################################ function Base.:(==)(p1::EnsembleParameterTable, p2::EnsembleParameterTable) - out = - (p1.tables == p2.tables) && - (p1.params == p2.params) + out = (p1.tables == p2.tables) && (p1.params == p2.params) return out -end \ No newline at end of file +end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 07c24e46e..c5ad010b3 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -128,7 +128,7 @@ end # Equality -------------------------------------------------------------------------------- function Base.:(==)(p1::ParameterTable, p2::ParameterTable) - out = + out = (p1.columns == p2.columns) && (p1.observed_vars == p2.observed_vars) && (p1.latent_vars == p2.latent_vars) && diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 0c5722f57..43fd87945 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -133,8 +133,10 @@ function RAMMatrices( @assert length(partable.sorted_vars) == nvars(partable) vars_sorted = copy(partable.sorted_vars) else - vars_sorted = [partable.observed_vars - partable.latent_vars] + vars_sorted = [ + partable.observed_vars + partable.latent_vars + ] end # indices of the vars (A/S/M rows or columns) @@ -216,13 +218,20 @@ function RAMMatrices( sort!(M_consts, by = first) end - return RAMMatrices(ParamsMatrix{T}(A_inds, A_consts, (n_vars, n_vars)), - ParamsMatrix{T}(S_inds, S_consts, (n_vars, n_vars)), - sparse(1:n_observed, - [vars_index[var] for var in partable.observed_vars], - ones(T, n_observed), n_observed, n_vars), - !isnothing(M_inds) ? ParamsVector{T}(M_inds, M_consts, (n_vars,)) : nothing, - params, vars_sorted) + return RAMMatrices( + ParamsMatrix{T}(A_inds, A_consts, (n_vars, n_vars)), + ParamsMatrix{T}(S_inds, S_consts, (n_vars, n_vars)), + sparse( + 1:n_observed, + [vars_index[var] for var in partable.observed_vars], + ones(T, n_observed), + n_observed, + n_vars, + ), + !isnothing(M_inds) ? ParamsVector{T}(M_inds, M_consts, (n_vars,)) : nothing, + params, + vars_sorted, + ) end Base.convert( @@ -360,10 +369,7 @@ function append_rows!( arr_ix = arr_ixs[arr.linear_indices[j]] skip_symmetric && (arr_ix ∈ visited_indices) && continue - push!( - partable, - partable_row(par, arr_ix, arr_name, varnames, free = true), - ) + push!(partable, partable_row(par, arr_ix, arr_name, varnames, free = true)) if skip_symmetric # mark index and its symmetric as visited push!(visited_indices, arr_ix) diff --git a/src/implied/abstract.jl b/src/implied/abstract.jl index 05b0e2449..99bb4d68d 100644 --- a/src/implied/abstract.jl +++ b/src/implied/abstract.jl @@ -25,8 +25,9 @@ function check_acyclic(A::AbstractMatrix; verbose::Bool = false) # check if non-triangular matrix is acyclic acyclic = isone(det(I - A)) if acyclic - verbose && @info "The matrix is acyclic. Reordering variables in the model to make the A matrix either Upper or Lower Triangular can significantly improve performance.\n" maxlog = - 1 + verbose && + @info "The matrix is acyclic. Reordering variables in the model to make the A matrix either Upper or Lower Triangular can significantly improve performance.\n" maxlog = + 1 end return A end diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 2c398090a..0ef542f70 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -161,7 +161,9 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, implied, model, semfiml) ∇Σ = P * (implied.∇S + Q * implied.∇A) - ∇μ = implied.F⨉I_A⁻¹ * implied.∇M + kron((implied.I_A⁻¹ * implied.M)', implied.F⨉I_A⁻¹) * implied.∇A + ∇μ = + implied.F⨉I_A⁻¹ * implied.∇M + + kron((implied.I_A⁻¹ * implied.M)', implied.F⨉I_A⁻¹) * implied.∇A mul!(G, ∇Σ', JΣ) # actually transposed mul!(G, ∇μ', Jμ, -1, 1) diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index 5b430e29e..4aafe4235 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -28,7 +28,15 @@ evaluate!(objective, gradient, hessian, loss::SemLossFunction, model::AbstractSe evaluate!(objective, gradient, hessian, loss, implied(model), model, params) # fallback method -function evaluate!(obj, grad, hess, loss::SemLossFunction, implied::SemImplied, model, params) +function evaluate!( + obj, + grad, + hess, + loss::SemLossFunction, + implied::SemImplied, + model, + params, +) isnothing(obj) || (obj = objective(loss, implied, model, params)) isnothing(grad) || copyto!(grad, gradient(loss, implied, model, params)) isnothing(hess) || copyto!(hess, hessian(loss, implied, model, params)) @@ -36,8 +44,10 @@ function evaluate!(obj, grad, hess, loss::SemLossFunction, implied::SemImplied, end # fallback methods -objective(f::SemLossFunction, implied::SemImplied, model, params) = objective(f, model, params) -gradient(f::SemLossFunction, implied::SemImplied, model, params) = gradient(f, model, params) +objective(f::SemLossFunction, implied::SemImplied, model, params) = + objective(f, model, params) +gradient(f::SemLossFunction, implied::SemImplied, model, params) = + gradient(f, model, params) hessian(f::SemLossFunction, implied::SemImplied, model, params) = hessian(f, model, params) # fallback method for SemImplied that calls update_xxx!() methods diff --git a/test/Project.toml b/test/Project.toml index 59db0b155..3cf1e50e3 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -15,4 +15,5 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index c97c9fb8e..2f5135176 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -17,10 +17,9 @@ model_ml_multigroup2 = SemEnsemble( data = dat, column = :school, groups = [:Pasteur, :Grant_White], - loss = SemML + loss = SemML, ) - # gradients @testset "ml_gradients_multigroup" begin test_gradient(model_ml_multigroup, start_test; atol = 1e-9) @@ -206,11 +205,19 @@ end # GLS estimation ############################################################################################ -model_ls_g1 = - Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic, loss = SemWLS) +model_ls_g1 = Sem( + specification = specification_g1, + data = dat_g1, + implied = RAMSymbolic, + loss = SemWLS, +) -model_ls_g2 = - Sem(specification = specification_g2, data = dat_g2, implied = RAMSymbolic, loss = SemWLS) +model_ls_g2 = Sem( + specification = specification_g2, + data = dat_g2, + implied = RAMSymbolic, + loss = SemWLS, +) model_ls_multigroup = SemEnsemble(model_ls_g1, model_ls_g2; optimizer = semoptimizer) @@ -239,7 +246,7 @@ end atol = 1e-5, ) - update_se_hessian!(partable, solution_ls) + @suppress update_se_hessian!(partable, solution_ls) test_estimates( partable, solution_lav[:parameter_estimates_ls]; @@ -283,7 +290,7 @@ if !isnothing(specification_miss_g1) groups = [:Pasteur, :Grant_White], loss = SemFIML, observed = SemObservedMissing, - meanstructure = true + meanstructure = true, ) ############################################################################################ diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index caaa5c3f7..7dd871ac2 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, FiniteDiff +using StructuralEquationModels, Test, FiniteDiff, Suppressor using LinearAlgebra: diagind, LowerTriangular const SEM = StructuralEquationModels @@ -71,10 +71,8 @@ specification_g2 = RAMMatrices(; vars = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], ) -partable = EnsembleParameterTable( - :Pasteur => specification_g1, - :Grant_White => specification_g2 -) +partable = + EnsembleParameterTable(:Pasteur => specification_g1, :Grant_White => specification_g2) specification_miss_g1 = nothing specification_miss_g2 = nothing diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index c99115032..88f98ded2 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -133,7 +133,7 @@ end ) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) - update_se_hessian!(partable, solution_ls) + @suppress update_se_hessian!(partable, solution_ls) test_estimates( partable, solution_lav[:parameter_estimates_ls]; @@ -158,7 +158,8 @@ if opt_engine == :Optim ), ) - implied_sym_hessian_vech = RAMSymbolic(specification = spec, vech = true, hessian = true) + implied_sym_hessian_vech = + RAMSymbolic(specification = spec, vech = true, hessian = true) implied_sym_hessian = RAMSymbolic(specification = spec, hessian = true) @@ -294,7 +295,7 @@ end ) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) - update_se_hessian!(partable_mean, solution_ls) + @suppress update_se_hessian!(partable_mean, solution_ls) test_estimates( partable_mean, solution_lav[:parameter_estimates_ls_mean]; diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 67538afa7..bbeb0c648 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -141,7 +141,7 @@ end ) @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) - update_se_hessian!(partable, solution_ls) + @suppress update_se_hessian!(partable, solution_ls) test_estimates( partable, solution_lav[:parameter_estimates_ls]; @@ -337,7 +337,7 @@ end ) @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) - update_se_hessian!(partable_mean, solution_ls) + @suppress update_se_hessian!(partable_mean, solution_ls) test_estimates( partable_mean, solution_lav[:parameter_estimates_ls_mean]; diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index a2e5089bb..9d026fb28 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, FiniteDiff +using StructuralEquationModels, Test, Suppressor, FiniteDiff SEM = StructuralEquationModels @@ -78,22 +78,7 @@ spec = RAMMatrices(; S = S, F = F, params = x, - vars = [ - :x1, - :x2, - :x3, - :y1, - :y2, - :y3, - :y4, - :y5, - :y6, - :y7, - :y8, - :ind60, - :dem60, - :dem65, - ], + vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65], ) partable = ParameterTable(spec) @@ -110,22 +95,7 @@ spec_mean = RAMMatrices(; F = F, M = M, params = [SEM.params(spec); Symbol.("x", string.(32:38))], - vars = [ - :x1, - :x2, - :x3, - :y1, - :y2, - :y3, - :y4, - :y5, - :y6, - :y7, - :y8, - :ind60, - :dem60, - :dem65, - ], + vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65], ) partable_mean = ParameterTable(spec_mean) diff --git a/test/examples/proximal/ridge.jl b/test/examples/proximal/ridge.jl index 16a318a12..8c0a1df7a 100644 --- a/test/examples/proximal/ridge.jl +++ b/test/examples/proximal/ridge.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators +using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators, Suppressor # load data dat = example_data("political_democracy") @@ -54,7 +54,7 @@ solution_ridge = sem_fit(model_ridge) model_prox = Sem(specification = partable, data = dat, loss = SemML) -solution_prox = sem_fit(model_prox, engine = :Proximal, operator_g = SqrNormL2(λ)) +solution_prox = @suppress sem_fit(model_prox, engine = :Proximal, operator_g = SqrNormL2(λ)) @testset "ridge_solution" begin @test isapprox(solution_prox.solution, solution_ridge.solution; rtol = 1e-4) diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index cc72673a6..183b067f5 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, Statistics +using StructuralEquationModels, Test, Statistics, Suppressor ### model specification -------------------------------------------------------------------- @@ -189,7 +189,7 @@ end ) # spec takes precedence in obs_vars order - observed_spec = SemObservedData( + observed_spec = @suppress SemObservedData( specification = spec, data = shuffle_dat, observed_vars = shuffle_names, @@ -451,7 +451,7 @@ end # SemObservedCovariance ) # spec takes precedence in obs_vars order - observed_spec = SemObservedMissing( + observed_spec = @suppress SemObservedMissing( specification = spec, observed_vars = shuffle_names, data = shuffle_dat_missing, From ba9d2c92571e20ee5b03f43a27dbc7828271848d Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 3 Feb 2025 17:48:33 +0100 Subject: [PATCH 175/364] turn simplification of symbolic terms by default off --- src/implied/RAM/symbolic.jl | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index 07acef019..44ad4949d 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -93,6 +93,7 @@ function RAMSymbolic(; specification::SemSpecification, loss_types = nothing, vech = false, + simplify_symbolics = false, gradient = true, hessian = false, meanstructure = false, @@ -116,7 +117,7 @@ function RAMSymbolic(; I_A⁻¹ = neumann_series(A) # Σ - Σ_symbolic = eval_Σ_symbolic(S, I_A⁻¹, F; vech = vech) + Σ_symbolic = eval_Σ_symbolic(S, I_A⁻¹, F; vech = vech, simplify = simplify_symbolics) #print(Symbolics.build_function(Σ_symbolic)[2]) Σ_function = Symbolics.build_function(Σ_symbolic, par, expression = Val{false})[2] Σ = zeros(size(Σ_symbolic)) @@ -157,7 +158,7 @@ function RAMSymbolic(; # μ if meanstructure MS = HasMeanStruct - μ_symbolic = eval_μ_symbolic(M, I_A⁻¹, F) + μ_symbolic = eval_μ_symbolic(M, I_A⁻¹, F; simplify = simplify_symbolics) μ_function = Symbolics.build_function(μ_symbolic, par, expression = Val{false})[2] μ = zeros(size(μ_symbolic)) if gradient @@ -235,23 +236,26 @@ end ############################################################################################ # expected covariations of observed vars -function eval_Σ_symbolic(S, I_A⁻¹, F; vech = false) +function eval_Σ_symbolic(S, I_A⁻¹, F; vech = false, simplify = false) Σ = F * I_A⁻¹ * S * permutedims(I_A⁻¹) * permutedims(F) Σ = Array(Σ) vech && (Σ = Σ[tril(trues(size(F, 1), size(F, 1)))]) - # Σ = Symbolics.simplify.(Σ) - Threads.@threads for i in eachindex(Σ) - Σ[i] = Symbolics.simplify(Σ[i]) + if simplify + Threads.@threads for i in eachindex(Σ) + Σ[i] = Symbolics.simplify(Σ[i]) + end end return Σ end # expected means of observed vars -function eval_μ_symbolic(M, I_A⁻¹, F) +function eval_μ_symbolic(M, I_A⁻¹, F; simplify = false) μ = F * I_A⁻¹ * M μ = Array(μ) - Threads.@threads for i in eachindex(μ) - μ[i] = Symbolics.simplify(μ[i]) + if simplify + Threads.@threads for i in eachindex(μ) + μ[i] = Symbolics.simplify(μ[i]) + end end return μ end From cd6413b60bdd81341a687268e4847bf4555453d7 Mon Sep 17 00:00:00 2001 From: Aaron Peikert Date: Mon, 3 Feb 2025 19:30:22 +0100 Subject: [PATCH 176/364] new version of StenoGraph results in fewer deprication notices --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 5937930d3..d55346aca 100644 --- a/Project.toml +++ b/Project.toml @@ -25,7 +25,7 @@ SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b" [compat] julia = "1.9, 1.10" -StenoGraphs = "0.2, 0.3" +StenoGraphs = "0.2 - 0.3, 0.4.1 - 0.5" DataFrames = "1" Distributions = "0.25" FiniteDiff = "2" From f0df6538f0220f964cbf51772698c317a0b4cf86 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 10:43:31 +0100 Subject: [PATCH 177/364] fix exporting structs from package extensions --- ext/SEMNLOptExt/NLopt.jl | 67 --------------------- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 25 -------- src/StructuralEquationModels.jl | 8 ++- src/package_extensions/SEMNLOptExt.jl | 64 ++++++++++++++++++++ src/package_extensions/SEMProximalOptExt.jl | 21 +++++++ 5 files changed, 92 insertions(+), 93 deletions(-) create mode 100644 src/package_extensions/SEMNLOptExt.jl create mode 100644 src/package_extensions/SEMProximalOptExt.jl diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index 959380292..ff868afc2 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -1,70 +1,3 @@ -############################################################################################ -### Types -############################################################################################ -""" -Connects to `NLopt.jl` as the optimization backend. - -# Constructor - - SemOptimizerNLopt(; - algorithm = :LD_LBFGS, - options = Dict{Symbol, Any}(), - local_algorithm = nothing, - local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), - kwargs...) - -# Arguments -- `algorithm`: optimization algorithm. -- `options::Dict{Symbol, Any}`: options for the optimization algorithm -- `local_algorithm`: local optimization algorithm -- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm -- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints -- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints - -# Example -```julia -my_optimizer = SemOptimizerNLopt() - -# constrained optimization with augmented lagrangian -my_constrained_optimizer = SemOptimizerNLopt(; - algorithm = :AUGLAG, - local_algorithm = :LD_LBFGS, - local_options = Dict(:ftol_rel => 1e-6), - inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0), -) -``` - -# Usage -All algorithms and options from the NLopt library are available, for more information see -the NLopt.jl package and the NLopt online documentation. -For information on how to use inequality and equality constraints, -see [Constrained optimization](@ref) in our online documentation. - -# Extended help - -## Interfaces -- `algorithm(::SemOptimizerNLopt)` -- `local_algorithm(::SemOptimizerNLopt)` -- `options(::SemOptimizerNLopt)` -- `local_options(::SemOptimizerNLopt)` -- `equality_constraints(::SemOptimizerNLopt)` -- `inequality_constraints(::SemOptimizerNLopt)` - -## Implementation - -Subtype of `SemOptimizer`. -""" -struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} - algorithm::A - local_algorithm::A2 - options::B - local_options::B2 - equality_constraints::C - inequality_constraints::C -end - Base.@kwdef struct NLoptConstraint f::Any tol = 0.0 diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index eceff0dc3..2f1775e85 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -1,28 +1,3 @@ -############################################################################################ -### Types -############################################################################################ -""" -Connects to `ProximalAlgorithms.jl` as the optimization backend. - -# Constructor - - SemOptimizerProximal(; - algorithm = ProximalAlgorithms.PANOC(), - operator_g, - operator_h = nothing, - kwargs..., - -# Arguments -- `algorithm`: optimization algorithm. -- `operator_g`: gradient of the objective function -- `operator_h`: optional hessian of the objective function -""" -mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} - algorithm::A - operator_g::B - operator_h::C -end - SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) SemOptimizerProximal(; diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 7c8923dc8..af2cd4bfe 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -82,6 +82,10 @@ include("frontend/fit/fitmeasures/fit_measures.jl") # standard errors include("frontend/fit/standard_errors/hessian.jl") include("frontend/fit/standard_errors/bootstrap.jl") +# extensions +include("package_extensions/SEMNLOptExt.jl") +include("package_extensions/SEMProximalOptExt.jl") + export AbstractSem, AbstractSemSingle, @@ -183,5 +187,7 @@ export AbstractSem, →, ←, ↔, - ⇔ + ⇔, + SemOptimizerNLopt, + SemOptimizerProximal end diff --git a/src/package_extensions/SEMNLOptExt.jl b/src/package_extensions/SEMNLOptExt.jl new file mode 100644 index 000000000..5d6d090c4 --- /dev/null +++ b/src/package_extensions/SEMNLOptExt.jl @@ -0,0 +1,64 @@ +""" +Connects to `NLopt.jl` as the optimization backend. +Only usable if `NLopt.jl` is loaded in the current Julia session! + +# Constructor + + SemOptimizerNLopt(; + algorithm = :LD_LBFGS, + options = Dict{Symbol, Any}(), + local_algorithm = nothing, + local_options = Dict{Symbol, Any}(), + equality_constraints = Vector{NLoptConstraint}(), + inequality_constraints = Vector{NLoptConstraint}(), + kwargs...) + +# Arguments +- `algorithm`: optimization algorithm. +- `options::Dict{Symbol, Any}`: options for the optimization algorithm +- `local_algorithm`: local optimization algorithm +- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm +- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints +- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints + +# Example +```julia +my_optimizer = SemOptimizerNLopt() + +# constrained optimization with augmented lagrangian +my_constrained_optimizer = SemOptimizerNLopt(; + algorithm = :AUGLAG, + local_algorithm = :LD_LBFGS, + local_options = Dict(:ftol_rel => 1e-6), + inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0), +) +``` + +# Usage +All algorithms and options from the NLopt library are available, for more information see +the NLopt.jl package and the NLopt online documentation. +For information on how to use inequality and equality constraints, +see [Constrained optimization](@ref) in our online documentation. + +# Extended help + +## Interfaces +- `algorithm(::SemOptimizerNLopt)` +- `local_algorithm(::SemOptimizerNLopt)` +- `options(::SemOptimizerNLopt)` +- `local_options(::SemOptimizerNLopt)` +- `equality_constraints(::SemOptimizerNLopt)` +- `inequality_constraints(::SemOptimizerNLopt)` + +## Implementation + +Subtype of `SemOptimizer`. +""" +struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} + algorithm::A + local_algorithm::A2 + options::B + local_options::B2 + equality_constraints::C + inequality_constraints::C +end \ No newline at end of file diff --git a/src/package_extensions/SEMProximalOptExt.jl b/src/package_extensions/SEMProximalOptExt.jl new file mode 100644 index 000000000..e8b256704 --- /dev/null +++ b/src/package_extensions/SEMProximalOptExt.jl @@ -0,0 +1,21 @@ +""" +Connects to `ProximalAlgorithms.jl` as the optimization backend. + +# Constructor + + SemOptimizerProximal(; + algorithm = ProximalAlgorithms.PANOC(), + operator_g, + operator_h = nothing, + kwargs..., + +# Arguments +- `algorithm`: optimization algorithm. +- `operator_g`: gradient of the objective function +- `operator_h`: optional hessian of the objective function +""" +mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} + algorithm::A + operator_g::B + operator_h::C +end \ No newline at end of file From 81a4bd9839df01e9f487b9aa13e3df107856114a Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 10:50:16 +0100 Subject: [PATCH 178/364] fix NLopt extension --- ext/SEMNLOptExt/NLopt.jl | 5 ----- ext/SEMNLOptExt/SEMNLOptExt.jl | 3 +-- src/StructuralEquationModels.jl | 1 + src/package_extensions/SEMNLOptExt.jl | 5 +++++ 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index ff868afc2..a614c501b 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -1,8 +1,3 @@ -Base.@kwdef struct NLoptConstraint - f::Any - tol = 0.0 -end - Base.convert( ::Type{NLoptConstraint}, tuple::NamedTuple{(:f, :tol), Tuple{F, T}}, diff --git a/ext/SEMNLOptExt/SEMNLOptExt.jl b/ext/SEMNLOptExt/SEMNLOptExt.jl index a159f6dc8..c79fc2b86 100644 --- a/ext/SEMNLOptExt/SEMNLOptExt.jl +++ b/ext/SEMNLOptExt/SEMNLOptExt.jl @@ -1,11 +1,10 @@ module SEMNLOptExt using StructuralEquationModels, NLopt +using StructuralEquationModels: SemOptimizerNLopt, NLoptConstraint SEM = StructuralEquationModels -export SemOptimizerNLopt, NLoptConstraint - include("NLopt.jl") end diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index af2cd4bfe..5d6b23ef4 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -189,5 +189,6 @@ export AbstractSem, ↔, ⇔, SemOptimizerNLopt, + NLoptConstraint, SemOptimizerProximal end diff --git a/src/package_extensions/SEMNLOptExt.jl b/src/package_extensions/SEMNLOptExt.jl index 5d6d090c4..7eae2f268 100644 --- a/src/package_extensions/SEMNLOptExt.jl +++ b/src/package_extensions/SEMNLOptExt.jl @@ -61,4 +61,9 @@ struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} local_options::B2 equality_constraints::C inequality_constraints::C +end + +Base.@kwdef struct NLoptConstraint + f::Any + tol = 0.0 end \ No newline at end of file From 9729819b86f375e4663de1fe9ec9c38d4932f580 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 10:54:12 +0100 Subject: [PATCH 179/364] fix Proximal extension --- ext/SEMProximalOptExt/SEMProximalOptExt.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/SEMProximalOptExt/SEMProximalOptExt.jl b/ext/SEMProximalOptExt/SEMProximalOptExt.jl index 156311367..0db21462d 100644 --- a/ext/SEMProximalOptExt/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt/SEMProximalOptExt.jl @@ -2,8 +2,7 @@ module SEMProximalOptExt using StructuralEquationModels using ProximalAlgorithms - -export SemOptimizerProximal +using StructuralEquationModels: SemOptimizerProximal SEM = StructuralEquationModels From 127da26bd7e24007d2ab136429d4d024364d0329 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 11:13:33 +0100 Subject: [PATCH 180/364] fix printing --- .../regularization/regularization.md | 33 +++++-------------- ext/SEMProximalOptExt/SEMProximalOptExt.jl | 2 +- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index 02d3b3bac..f9d19b176 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -5,40 +5,23 @@ For ridge regularization, you can simply use `SemRidge` as an additional loss function (for example, a model with the loss functions `SemML` and `SemRidge` corresponds to ridge-regularized maximum likelihood estimation). -For lasso, elastic net and (far) beyond, we provide the `ProximalSEM` package. You can install it and load it alongside `StructuralEquationModels`: +For lasso, elastic net and (far) beyond, you can load the `ProximalAlgorithms.jl` and `ProximalOperators.jl` packages alongside `StructuralEquationModels`: ```@setup reg -import Pkg -Pkg.add(url = "https://github.com/StructuralEquationModels/ProximalSEM.jl") - -using StructuralEquationModels, ProximalSEM -``` - -```julia -import Pkg -Pkg.add(url = "https://github.com/StructuralEquationModels/ProximalSEM.jl") - -using StructuralEquationModels, ProximalSEM -``` - -!!! warning "ProximalSEM is still WIP" - The ProximalSEM package does not have any releases yet, and is not well tested - until the first release, use at your own risk and expect interfaces to change without prior notice. - -Additionally, you need to install and load `ProximalOperators.jl`: - -```@setup reg -using ProximalOperators +using StructuralEquationModels, ProximalAlgorithms, ProximalOperators ``` ```julia +using Pkg +Pkg.add("ProximalAlgorithms") Pkg.add("ProximalOperators") -using ProximalOperators +using StructuralEquationModels, ProximalAlgorithms, ProximalOperators ``` ## `SemOptimizerProximal` -`ProximalSEM` provides a new "building block" for the optimizer part of a model, called `SemOptimizerProximal`. +To estimate regularized models, we provide a "building block" for the optimizer part, called `SemOptimizerProximal`. It connects our package to the [`ProximalAlgorithms.jl`](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) optimization backend, providing so-called proximal optimization algorithms. Those can handle, amongst other things, various forms of regularization. @@ -102,7 +85,9 @@ model = Sem( We labeled the covariances between the items because we want to regularize those: ```@example reg -ind = param_indices([:cov_15, :cov_24, :cov_26, :cov_37, :cov_48, :cov_68], model) +ind = getindex.( + [param_indices(model)], + [:cov_15, :cov_24, :cov_26, :cov_37, :cov_48, :cov_68]) ``` In the following, we fit the same model with lasso regularization of those covariances. diff --git a/ext/SEMProximalOptExt/SEMProximalOptExt.jl b/ext/SEMProximalOptExt/SEMProximalOptExt.jl index 0db21462d..192944fef 100644 --- a/ext/SEMProximalOptExt/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt/SEMProximalOptExt.jl @@ -2,7 +2,7 @@ module SEMProximalOptExt using StructuralEquationModels using ProximalAlgorithms -using StructuralEquationModels: SemOptimizerProximal +using StructuralEquationModels: SemOptimizerProximal, print_type_name, print_field_types SEM = StructuralEquationModels From f67d48cb76a822b5bbcd6b48a71ae2a9f1fab420 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 11:28:29 +0100 Subject: [PATCH 181/364] fix regularization docs --- .../regularization/regularization.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index f9d19b176..37e42975a 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -112,8 +112,7 @@ optimizer_lasso = SemOptimizerProximal( model_lasso = Sem( specification = partable, - data = data, - optimizer = optimizer_lasso + data = data ) ``` @@ -121,7 +120,7 @@ Let's fit the regularized model ```@example reg -fit_lasso = sem_fit(model_lasso) +fit_lasso = sem_fit(optimizer_lasso, model_lasso) ``` and compare the solution to unregularizted estimates: @@ -136,6 +135,12 @@ update_partable!(partable, :estimate_lasso, params(fit_lasso), solution(fit_lass details(partable) ``` +Instead of explicitely defining a `SemOptimizerProximal` object, you can also pass `engine = :Proximal` and additional keyword arguments to `sem_fit`: + +```@example reg +fit = sem_fit(model; engine = :Proximal, operator_g = NormL1(λ)) +``` + ## Second example - mixed l1 and l0 regularization You can choose to penalize different parameters with different types of regularization functions. @@ -150,16 +155,14 @@ To define a sup of separable proximal operators (i.e. no parameter is penalized we can use [`SlicedSeparableSum`](https://juliafirstorder.github.io/ProximalOperators.jl/stable/calculus/#ProximalOperators.SlicedSeparableSum) from the `ProximalOperators` package: ```@example reg -prox_operator = SlicedSeparableSum((NormL1(0.02), NormL0(20.0), NormL0(0.0)), ([ind], [12:22], [vcat(1:11, 23:25)])) +prox_operator = SlicedSeparableSum((NormL0(20.0), NormL1(0.02), NormL0(0.0)), ([ind], [9:11], [vcat(1:8, 12:25)])) model_mixed = Sem( specification = partable, - data = data, - optimizer = SemOptimizerProximal, - operator_g = prox_operator + data = data, ) -fit_mixed = sem_fit(model_mixed) +fit_mixed = sem_fit(model_mixed; engine = :Proximal, operator_g = prox_operator) ``` Let's again compare the different results: From e9dbb62a24dec7e5eeb2a014f88474d141fba646 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 10:27:41 +0100 Subject: [PATCH 182/364] start reworking docs --- docs/Project.toml | 3 + docs/src/assets/concept.svg | 155 +++++++---------- docs/src/assets/concept_typed.svg | 156 +++++++----------- docs/src/index.md | 2 +- docs/src/tutorials/backends/nlopt.md | 3 + docs/src/tutorials/concept.md | 12 +- .../tutorials/construction/build_by_parts.md | 18 +- .../construction/outer_constructor.md | 25 +-- docs/src/tutorials/first_model.md | 20 +-- .../specification/graph_interface.md | 22 +-- .../tutorials/specification/ram_matrices.md | 4 +- .../tutorials/specification/specification.md | 8 +- 12 files changed, 191 insertions(+), 237 deletions(-) diff --git a/docs/Project.toml b/docs/Project.toml index 9da7f0ab4..2daded98f 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,4 +1,7 @@ [deps] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" +ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" +ProximalSEM = "3652f839-8142-48b2-a17c-985bd14407c5" diff --git a/docs/src/assets/concept.svg b/docs/src/assets/concept.svg index 2a7a0b42a..138463b67 100644 --- a/docs/src/assets/concept.svg +++ b/docs/src/assets/concept.svg @@ -1,197 +1,166 @@ + id="defs23" /> - - - + inkscape:deskcolor="#d1d1d1"> + + + id="path3" /> + id="path4" /> + id="path5" /> + id="path6" /> + id="path7" /> + id="path8" /> + id="path9" /> + id="path10" /> + id="path11" /> + id="path12" /> + id="path13" /> + id="path14" /> - - - + id="path15" /> + id="path16" /> + id="path17" /> + id="path18" /> + id="path19" /> + id="path20" /> + id="path21" /> + id="path22" /> + id="path23" /> diff --git a/docs/src/assets/concept_typed.svg b/docs/src/assets/concept_typed.svg index 8281300f8..032adc9b8 100644 --- a/docs/src/assets/concept_typed.svg +++ b/docs/src/assets/concept_typed.svg @@ -1,197 +1,169 @@ + id="defs23" /> - - - + inkscape:deskcolor="#d1d1d1"> + + + id="path3" /> + id="path4" /> + id="path5" /> + id="path6" /> + id="path7" /> + id="path8" /> + id="path9" /> + id="path10" /> + id="path11" /> + id="path12" /> + id="path13" /> + id="path14" /> - - - + id="path15" /> + id="path16" /> + id="path17" /> + id="path18" /> + id="path19" /> + id="path20" /> + id="path21" /> + id="path22" /> + id="path23" /> diff --git a/docs/src/index.md b/docs/src/index.md index 8b2d6999e..add69459e 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -32,7 +32,7 @@ For examples of how to use the package, see the Tutorials. Models you can fit out of the box include - Linear SEM that can be specified in RAM notation - ML, GLS and FIML estimation -- Ridge Regularization +- Ridge/Lasso/... Regularization - Multigroup SEM - Sums of arbitrary loss functions (everything the optimizer can handle) diff --git a/docs/src/tutorials/backends/nlopt.md b/docs/src/tutorials/backends/nlopt.md index d4c5fdf8f..f861e174e 100644 --- a/docs/src/tutorials/backends/nlopt.md +++ b/docs/src/tutorials/backends/nlopt.md @@ -1,6 +1,7 @@ # Using NLopt.jl [`SemOptimizerNLopt`](@ref) implements the connection to `NLopt.jl`. +It is only available if the `NLopt` package is loaded alongside `StructuralEquationModel.jl` in the running Julia session. It takes a bunch of arguments: ```julia @@ -22,6 +23,8 @@ The defaults are LBFGS as the optimization algorithm and the standard options fr We can choose something different: ```julia +using NLopt + my_optimizer = SemOptimizerNLopt(; algorithm = :AUGLAG, options = Dict(:maxeval => 200), diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index b8d094abc..d663d3c2c 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -1,12 +1,13 @@ # Our Concept of a Structural Equation Model -In our package, every Structural Equation Model (`Sem`) consists of four parts: +In our package, every Structural Equation Model (`Sem`) consists of three parts (four, if you count the optimizer): ![SEM concept](../assets/concept.svg) Those parts are interchangable building blocks (like 'Legos'), i.e. there are different pieces available you can choose as the `observed` slot of the model, and stick them together with other pieces that can serve as the `implied` part. -The `observed` part is for observed data, the `implied` part is what the model implies about your data (e.g. the model implied covariance matrix), the loss part compares the observed data and implied properties (e.g. weighted least squares difference between the observed and implied covariance matrix) and the optimizer part connects to the optimization backend (e.g. the type of optimization algorithm used). +The `observed` part is for observed data, the `implied` part is what the model implies about your data (e.g. the model implied covariance matrix), and the loss part compares the observed data and implied properties (e.g. weighted least squares difference between the observed and implied covariance matrix). +The optimizer part is not part of the model itself, but it is needed to fit the model as it connects to the optimization backend (e.g. the type of optimization algorithm used). For example, to build a model for maximum likelihood estimation with the NLopt optimization suite as a backend you would choose `SemML` as a loss function and `SemOptimizerNLopt` as the optimizer. @@ -51,12 +52,12 @@ Available loss functions are ## The optimizer part aka `SemOptimizer` The optimizer part of a model connects to the numerical optimization backend used to fit the model. It can be used to control options like the optimization algorithm, linesearch, stopping criteria, etc. -There are currently two available backends, [`SemOptimizerOptim`](@ref) connecting to the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) backend, and [`SemOptimizerNLopt`](@ref) connecting to the [NLopt.jl](https://github.com/JuliaOpt/NLopt.jl) backend. -For more information about the available options see also the tutorials about [Using Optim.jl](@ref) and [Using NLopt.jl](@ref), as well as [Constrained optimization](@ref). +There are currently three available backends, [`SemOptimizerOptim`](@ref) connecting to the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) backend, [`SemOptimizerNLopt`](@ref) connecting to the [NLopt.jl](https://github.com/JuliaOpt/NLopt.jl) backend and [`SemOptimizerProximal`](@ref) connecting to [ProximalAlgorithms.jl](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl). +For more information about the available options see also the tutorials about [Using Optim.jl](@ref) and [Using NLopt.jl](@ref), as well as [Constrained optimization](@ref) and [Regularization](@ref) . # What to do next -You now have an understanding about our representation of structural equation models. +You now have an understanding of our representation of structural equation models. To learn more about how to use the package, you may visit the remaining tutorials. @@ -100,4 +101,5 @@ SemConstant SemOptimizer SemOptimizerOptim SemOptimizerNLopt +SemOptimizerProximal ``` \ No newline at end of file diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 071750a8c..27604d2a1 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -11,8 +11,8 @@ using StructuralEquationModels data = example_data("political_democracy") -observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] -latent_vars = [:ind60, :dem60, :dem65] +obs_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +lat_vars = [:ind60, :dem60, :dem65] graph = @StenoGraph begin @@ -27,8 +27,8 @@ graph = @StenoGraph begin ind60 → dem65 # variances - _(observed_vars) ↔ _(observed_vars) - _(latent_vars) ↔ _(latent_vars) + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ↔ _(lat_vars) # covariances y1 ↔ y5 @@ -40,8 +40,8 @@ end partable = ParameterTable( graph, - latent_vars = latent_vars, - observed_vars = observed_vars) + latent_vars = lat_vars, + observed_vars = obs_vars) ``` Now, we construct the different parts: @@ -59,9 +59,11 @@ ml = SemML(observed = observed) loss_ml = SemLoss(ml) # optimizer ------------------------------------------------------------------------------------- -optimizer = SemOptimizerOptim() +optimizer = SemOptimizerOptim(algorithm = BFGS()) # model ------------------------------------------------------------------------------------ -model_ml = Sem(observed, implied_ram, loss_ml, optimizer) +model_ml = Sem(observed, implied_ram, loss_ml) + +sem_fit(optimizer, model_ml) ``` \ No newline at end of file diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md index 0979f684a..7de3d9e61 100644 --- a/docs/src/tutorials/construction/outer_constructor.md +++ b/docs/src/tutorials/construction/outer_constructor.md @@ -35,7 +35,7 @@ model = Sem( ) ``` -For example, to construct a model for weighted least squares estimation that uses symbolic precomputation and the NLopt backend, write +For example, to construct a model for weighted least squares estimation that uses symbolic precomputation and the Optim backend, write ```julia model = Sem( @@ -43,7 +43,7 @@ model = Sem( data = data, implied = RAMSymbolic, loss = SemWLS, - optimizer = SemOptimizerNLopt + optimizer = SemOptimizerOptim ) ``` @@ -92,25 +92,29 @@ help>SemObservedMissing For observed data with missing values. Constructor - ≡≡≡≡≡≡≡≡≡≡≡≡≡ + ≡≡≡≡≡≡≡≡≡≡≡ SemObservedMissing(; - specification, data, - obs_colnames = nothing, + observed_vars = nothing, + specification = nothing, kwargs...) Arguments - ≡≡≡≡≡≡≡≡≡≡≡ + ≡≡≡≡≡≡≡≡≡ - • specification: either a RAMMatrices or ParameterTable object (1) + • specification: optional SEM model specification + (SemSpecification) • data: observed data - • obs_colnames::Vector{Symbol}: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) + • observed_vars::Vector{Symbol}: column names of the data (if + the object passed as data does not have column names, i.e. is + not a data frame) + + ──────────────────────────────────────────────────────────────────────── - ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── -Extended help is available with `??` +Extended help is available with `??SemObservedMissing` ``` ## Optimize loss functions without analytic gradient @@ -118,7 +122,6 @@ Extended help is available with `??` For loss functions without analytic gradients, it is possible to use finite difference approximation or automatic differentiation. All loss functions provided in the package do have analytic gradients (and some even hessians or approximations thereof), so there is no need do use this feature if you are only working with them. However, if you implement your own loss function, you do not have to provide analytic gradients. -This page is a about finite difference approximation. For information about how to use automatic differentiation, see the documentation of the [AutoDiffSEM](https://github.com/StructuralEquationModels/AutoDiffSEM) package. To use finite difference approximation, you may construct your model just as before, but swap the `Sem` constructor for `SemFiniteDiff`. For example diff --git a/docs/src/tutorials/first_model.md b/docs/src/tutorials/first_model.md index a285e29df..5b7284649 100644 --- a/docs/src/tutorials/first_model.md +++ b/docs/src/tutorials/first_model.md @@ -15,8 +15,8 @@ using StructuralEquationModels We then first define the graph of our model in a syntax which is similar to the R-package `lavaan`: ```@setup high_level -observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] -latent_vars = [:ind60, :dem60, :dem65] +obs_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +lat_vars = [:ind60, :dem60, :dem65] graph = @StenoGraph begin @@ -31,8 +31,8 @@ graph = @StenoGraph begin ind60 → dem65 # variances - _(observed_vars) ↔ _(observed_vars) - _(latent_vars) ↔ _(latent_vars) + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ↔ _(lat_vars) # covariances y1 ↔ y5 @@ -44,8 +44,8 @@ end ``` ```julia -observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] -latent_vars = [:ind60, :dem60, :dem65] +obs_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +lat_vars = [:ind60, :dem60, :dem65] graph = @StenoGraph begin @@ -60,8 +60,8 @@ graph = @StenoGraph begin ind60 → dem65 # variances - _(observed_vars) ↔ _(observed_vars) - _(latent_vars) ↔ _(latent_vars) + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ↔ _(lat_vars) # covariances y1 ↔ y5 @@ -84,8 +84,8 @@ We then use this graph to define a `ParameterTable` object ```@example high_level; ansicolor = true partable = ParameterTable( graph, - latent_vars = latent_vars, - observed_vars = observed_vars) + latent_vars = lat_vars, + observed_vars = obs_vars) ``` load the example data diff --git a/docs/src/tutorials/specification/graph_interface.md b/docs/src/tutorials/specification/graph_interface.md index 609c844c3..75e1d1b6d 100644 --- a/docs/src/tutorials/specification/graph_interface.md +++ b/docs/src/tutorials/specification/graph_interface.md @@ -12,13 +12,13 @@ end and convert it to a ParameterTable to construct your models: ```julia -observed_vars = ... -latent_vars = ... +obs_vars = ... +lat_vars = ... partable = ParameterTable( graph, - latent_vars = latent_vars, - observed_vars = observed_vars) + latent_vars = lat_vars, + observed_vars = obs_vars) model = Sem( specification = partable, @@ -66,23 +66,23 @@ As you saw above and in the [A first model](@ref) example, the graph object need ```julia partable = ParameterTable( graph, - latent_vars = latent_vars, - observed_vars = observed_vars) + latent_vars = lat_vars, + observed_vars = obs_vars) ``` The `ParameterTable` constructor also needs you to specify a vector of observed and latent variables, in the example above this would correspond to ```julia -observed_vars = [:x1 :x2 :x3 :x4 :x5 :x6 :x7 :x8 :x9] -latent_vars = [:ξ₁ :ξ₂ :ξ₃] +obs_vars = [:x1 :x2 :x3 :x4 :x5 :x6 :x7 :x8 :x9] +lat_vars = [:ξ₁ :ξ₂ :ξ₃] ``` The variable names (`:x1`) have to be symbols, the syntax `:something` creates an object of type `Symbol`. But you can also use vectors of symbols inside the graph specification, escaping them with `_(...)`. For example, this graph specification ```julia @StenoGraph begin - _(observed_vars) ↔ _(observed_vars) - _(latent_vars) ⇔ _(latent_vars) + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ⇔ _(lat_vars) end ``` creates undirected effects coresponding to @@ -95,7 +95,7 @@ Mean parameters are specified as a directed effect from `1` to the respective va ```julia @StenoGraph begin - Symbol("1") → _(observed_vars) + Symbol(1) → _(obs_vars) end ``` diff --git a/docs/src/tutorials/specification/ram_matrices.md b/docs/src/tutorials/specification/ram_matrices.md index 5f0757238..6e01eb38b 100644 --- a/docs/src/tutorials/specification/ram_matrices.md +++ b/docs/src/tutorials/specification/ram_matrices.md @@ -60,7 +60,7 @@ spec = RAMMatrices(; S = S, F = F, params = θ, - colnames = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] + vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) model = Sem( @@ -91,7 +91,7 @@ spec = RAMMatrices(; S = S, F = F, params = θ, - colnames = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] + vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) ``` diff --git a/docs/src/tutorials/specification/specification.md b/docs/src/tutorials/specification/specification.md index c426443f4..85bb37c00 100644 --- a/docs/src/tutorials/specification/specification.md +++ b/docs/src/tutorials/specification/specification.md @@ -10,8 +10,8 @@ This leads to the following chart: You can enter model specification at each point, but in general (and especially if you come from `lavaan`), it is the easiest to follow the red arrows: specify a graph object, convert it to a prameter table, and use this parameter table to construct your models ( just like we did in [A first model](@ref)): ```julia -observed_vars = ... -latent_vars = ... +obs_vars = ... +lat_vars = ... graph = @StenoGraph begin ... @@ -19,8 +19,8 @@ end partable = ParameterTable( graph, - latent_vars = latent_vars, - observed_vars = observed_vars) + latent_vars = lat_vars, + observed_vars = obs_vars) model = Sem( specification = partable, From cca249660907e7a264e68014be2aa2accca5c238 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 13:48:21 +0100 Subject: [PATCH 183/364] finish rewriting docs --- docs/src/developer/extending.md | 2 +- docs/src/developer/implied.md | 95 +++++++++++-------- docs/src/developer/loss.md | 80 ++++++---------- docs/src/developer/optimizer.md | 87 ++++++++--------- docs/src/developer/sem.md | 25 +---- docs/src/internals/files.md | 7 +- docs/src/internals/types.md | 4 +- docs/src/performance/simulation.md | 35 +++++-- docs/src/tutorials/backends/nlopt.md | 2 + docs/src/tutorials/backends/optim.md | 4 +- docs/src/tutorials/collection/collection.md | 3 +- docs/src/tutorials/collection/multigroup.md | 4 +- docs/src/tutorials/constraints/constraints.md | 5 +- .../construction/outer_constructor.md | 5 +- docs/src/tutorials/fitting/fitting.md | 24 ++++- docs/src/tutorials/meanstructure.md | 6 +- src/additional_functions/simulation.jl | 4 +- src/frontend/fit/summary.jl | 2 +- src/frontend/specification/ParameterTable.jl | 4 +- src/frontend/specification/RAMMatrices.jl | 10 +- src/implied/empty.jl | 10 +- test/examples/multigroup/build_models.jl | 18 ++-- test/examples/multigroup/multigroup.jl | 6 +- .../political_democracy.jl | 4 +- 24 files changed, 223 insertions(+), 223 deletions(-) diff --git a/docs/src/developer/extending.md b/docs/src/developer/extending.md index 074a8b710..5c3183da4 100644 --- a/docs/src/developer/extending.md +++ b/docs/src/developer/extending.md @@ -1,6 +1,6 @@ # Extending the package -As discussed in the section on [Model Construction](@ref), every Structural Equation Model (`Sem`) consists of four parts: +As discussed in the section on [Model Construction](@ref), every Structural Equation Model (`Sem`) consists of three (four with the optimizer) parts: ![SEM concept typed](../assets/concept_typed.svg) diff --git a/docs/src/developer/implied.md b/docs/src/developer/implied.md index 403ecfa84..bea824a94 100644 --- a/docs/src/developer/implied.md +++ b/docs/src/developer/implied.md @@ -10,78 +10,89 @@ struct MyImplied <: SemImplied end ``` -and at least a method to compute the objective +and a method to update!: ```julia import StructuralEquationModels: objective! -function objective!(implied::MyImplied, par, model::AbstractSemSingle) - ... - return nothing -end -``` +function update!(targets::EvaluationTargets, implied::MyImplied, model::AbstractSemSingle, params) -This method should compute and store things you want to make available to the loss functions, and returns `nothing`. For example, as we have seen in [Second example - maximum likelihood](@ref), the `RAM` implied type computes the model-implied covariance matrix and makes it available via `Σ(implied)`. -To make stored computations available to loss functions, simply write a function - for example, for the `RAM` implied type we defined + if is_objective_required(targets) + ... + end -```julia -Σ(implied::RAM) = implied.Σ + if is_gradient_required(targets) + ... + end + if is_hessian_required(targets) + ... + end + +end ``` -Additionally, you can specify methods for `gradient` and `hessian` as well as the combinations described in [Custom loss functions](@ref). +As you can see, `update` gets passed as a first argument `targets`, which is telling us whether the objective value, gradient, and/or hessian are needed. +We can then use the functions `is_..._required` and conditional on what the optimizer needs, we can compute and store things we want to make available to the loss functions. For example, as we have seen in [Second example - maximum likelihood](@ref), the `RAM` implied type computes the model-implied covariance matrix and makes it available via `implied.Σ`. -The last thing nedded to make it work is a method for `nparams` that takes your implied type and returns the number of parameters of the model: -```julia -nparams(implied::MyImplied) = ... -``` Just as described in [Custom loss functions](@ref), you may define a constructor. Typically, this will depend on the `specification = ...` argument that can be a `ParameterTable` or a `RAMMatrices` object. We implement an `ImpliedEmpty` type in our package that does nothing but serving as an `implied` field in case you are using a loss function that does not need any implied type at all. You may use it as a template for defining your own implied type, as it also shows how to handle the specification objects: ```julia -############################################################################ +############################################################################################ ### Types -############################################################################ +############################################################################################ +""" +Empty placeholder for models that don't need an implied part. +(For example, models that only regularize parameters.) -struct ImpliedEmpty{V, V2} <: SemImplied - identifier::V2 - n_par::V -end +# Constructor -############################################################################ -### Constructors -############################################################################ + ImpliedEmpty(;specification, kwargs...) + +# Arguments +- `specification`: either a `RAMMatrices` or `ParameterTable` object + +# Examples +A multigroup model with ridge regularization could be specified as a `SemEnsemble` with one +model per group and an additional model with `ImpliedEmpty` and `SemRidge` for the regularization part. -function ImpliedEmpty(; - specification, - kwargs...) +# Extended help - ram_matrices = RAMMatrices(specification) - identifier = StructuralEquationModels.identifier(ram_matrices) +## Interfaces +- `params(::RAMSymbolic) `-> Vector of parameter labels +- `nparams(::RAMSymbolic)` -> Number of parameters - n_par = length(ram_matrices.parameters) +## Implementation +Subtype of `SemImplied`. +""" +struct ImpliedEmpty{A, B, C} <: SemImplied + hessianeval::A + meanstruct::B + ram_matrices::C +end + +############################################################################################ +### Constructors +############################################################################################ - return ImpliedEmpty(identifier, n_par) +function ImpliedEmpty(;specification, meanstruct = NoMeanStruct(), hessianeval = ExactHessian(), kwargs...) + return ImpliedEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) end -############################################################################ +############################################################################################ ### methods -############################################################################ +############################################################################################ -objective!(implied::ImpliedEmpty, par, model) = nothing -gradient!(implied::ImpliedEmpty, par, model) = nothing -hessian!(implied::ImpliedEmpty, par, model) = nothing +update!(targets::EvaluationTargets, implied::ImpliedEmpty, par, model) = nothing -############################################################################ +############################################################################################ ### Recommended methods -############################################################################ - -identifier(implied::ImpliedEmpty) = implied.identifier -n_par(implied::ImpliedEmpty) = implied.n_par +############################################################################################ update_observed(implied::ImpliedEmpty, observed::SemObserved; kwargs...) = implied ``` -As you see, similar to [Custom loss functions](@ref) we implement a method for `update_observed`. Additionally, you should store the `identifier` from the specification object and write a method for `identifier`, as this will make it possible to access parameter indices by label. \ No newline at end of file +As you see, similar to [Custom loss functions](@ref) we implement a method for `update_observed`. \ No newline at end of file diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index 6d709b3be..57a7b485d 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -20,17 +20,22 @@ end ``` We store the hyperparameter α and the indices I of the parameters we want to regularize. -Additionaly, we need to define a *method* to compute the objective: +Additionaly, we need to define a *method* of the function `evaluate!` to compute the objective: ```@example loss -import StructuralEquationModels: objective! +import StructuralEquationModels: evaluate! -objective!(ridge::Ridge, par, model::AbstractSemSingle) = ridge.α*sum(par[ridge.I].^2) +evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, ridge::Ridge, model::AbstractSem, par) = + ridge.α * sum(i -> par[i]^2, ridge.I) ``` +The function `evaluate!` recognizes by the types of the arguments `objective`, `gradient` and `hessian` whether it should compute the objective value, gradient or hessian of the model w.r.t. the parameters. +In this case, `gradient` and `hessian` are of type `Nothing`, signifying that they should not be computed, but only the objective value. + That's all we need to make it work! For example, we can now fit [A first model](@ref) with ridge regularization: We first give some parameters labels to be able to identify them as targets for the regularization: + ```@example loss observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] latent_vars = [:ind60, :dem60, :dem65] @@ -65,7 +70,7 @@ partable = ParameterTable( observed_vars = observed_vars ) -parameter_indices = param_indices([:a, :b, :c], partable) +parameter_indices = getindex.([param_indices(partable)], [:a, :b, :c]) myridge = Ridge(0.01, parameter_indices) model = SemFiniteDiff( @@ -86,15 +91,23 @@ Note that the last argument to the `objective!` method is the whole model. There By far the biggest improvements in performance will result from specifying analytical gradients. We can do this for our example: ```@example loss -import StructuralEquationModels: gradient! - -function gradient!(ridge::Ridge, par, model::AbstractSemSingle) - gradient = zero(par) - gradient[ridge.I] .= 2*ridge.α*par[ridge.I] - return gradient +function evaluate!(objective, gradient, hessian::Nothing, ridge::Ridge, model::AbstractSem, par) + # compute gradient + if !isnothing(gradient) + fill!(gradient, 0) + gradient[ridge.I] .= 2 * ridge.α * par[ridge.I] + end + # compute objective + if !isnothing(objective) + return ridge.α * sum(i -> par[i]^2, ridge.I) + end end ``` +As you can see, in this method definition, both `objective` and `gradient` can be different from `nothing`. +We then check whether to compute the objective value and/or the gradient with `isnothing(objective)`/`isnothing(gradient)`. +This syntax makes it possible to compute objective value and gradient at the same time, which is beneficial when the the objective and gradient share common computations. + Now, instead of specifying a `SemFiniteDiff`, we can use the normal `Sem` constructor: ```@example loss @@ -119,46 +132,7 @@ using BenchmarkTools The exact results of those benchmarks are of course highly depended an your system (processor, RAM, etc.), but you should see that the median computation time with analytical gradients drops to about 5% of the computation without analytical gradients. -Additionally, you may provide analytic hessians by writing a method of the form - -```julia -function hessian!(ridge::Ridge, par, model::AbstractSemSingle) - ... - return hessian -end -``` - -however, this will only matter if you use an optimization algorithm that makes use of the hessians. Our default algorithmn `LBFGS` from the package `Optim.jl` does not use hessians (for example, the `Newton` algorithmn from the same package does). - -To improve performance even more, you can write a method of the form - -```julia -function objective_gradient!(ridge::Ridge, par, model::AbstractSemSingle) - ... - return objective, gradient -end -``` - -This is beneficial when the computation of the objective and gradient share common computations. For example, in maximum likelihood estimation, the model implied covariance matrix has to be inverted to both compute the objective and gradient. Whenever the optimization algorithmn asks for the objective value and gradient at the same point, we call `objective_gradient!` and only have to do the shared computations - in this case the matrix inversion - once. - -If you want to do hessian-based optimization, there are also the following methods: - -```julia -function objective_hessian!(ridge::Ridge, par, model::AbstractSemSingle) - ... - return objective, hessian -end - -function gradient_hessian!(ridge::Ridge, par, model::AbstractSemSingle) - ... - return gradient, hessian -end - -function objective_gradient_hessian!(ridge::Ridge, par, model::AbstractSemSingle) - ... - return objective, gradient, hessian -end -``` +Additionally, you may provide analytic hessians by writing a respective method for `evaluate!`. However, this will only matter if you use an optimization algorithm that makes use of the hessians. Our default algorithmn `LBFGS` from the package `Optim.jl` does not use hessians (for example, the `Newton` algorithmn from the same package does). ## Convenient @@ -241,11 +215,11 @@ With this information, we write can implement maximum likelihood optimization as struct MaximumLikelihood <: SemLossFunction end using LinearAlgebra -import StructuralEquationModels: Σ, obs_cov, objective! +import StructuralEquationModels: obs_cov, evaluate! -function objective!(semml::MaximumLikelihood, parameters, model::AbstractSem) +function evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, semml::MaximumLikelihood, model::AbstractSem, par) # access the model implied and observed covariance matrices - Σᵢ = Σ(implied(model)) + Σᵢ = implied(model).Σ Σₒ = obs_cov(observed(model)) # compute the objective if isposdef(Symmetric(Σᵢ)) # is the model implied covariance matrix positive definite? diff --git a/docs/src/developer/optimizer.md b/docs/src/developer/optimizer.md index 7480a9d91..82ec594d8 100644 --- a/docs/src/developer/optimizer.md +++ b/docs/src/developer/optimizer.md @@ -1,83 +1,70 @@ # Custom optimizer types The optimizer part of a model connects it to the optimization backend. -The first part of the implementation is very similar to loss functions, so we just show the implementation of `SemOptimizerOptim` here as a reference: +Let's say we want to implement a new optimizer as `SemOptimizerName`. The first part of the implementation is very similar to loss functions, so we just show the implementation of `SemOptimizerOptim` here as a reference: ```julia -############################################################################ +############################################################################################ ### Types and Constructor -############################################################################ - -mutable struct SemOptimizerOptim{A, B} <: SemOptimizer +############################################################################################ +mutable struct SemOptimizerName{A, B} <: SemOptimizer{:Name} algorithm::A options::B end -function SemOptimizerOptim(; - algorithm = LBFGS(), - options = Optim.Options(;f_tol = 1e-10, x_tol = 1.5e-8), - kwargs...) - return SemOptimizerOptim(algorithm, options) -end +SemOptimizer{:Name}(args...; kwargs...) = SemOptimizerName(args...; kwargs...) + +SemOptimizerName(; + algorithm = LBFGS(), + options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), + kwargs..., +) = SemOptimizerName(algorithm, options) -############################################################################ +############################################################################################ ### Recommended methods -############################################################################ +############################################################################################ -update_observed(optimizer::SemOptimizerOptim, observed::SemObserved; kwargs...) = optimizer +update_observed(optimizer::SemOptimizerName, observed::SemObserved; kwargs...) = optimizer -############################################################################ +############################################################################################ ### additional methods -############################################################################ +############################################################################################ -algorithm(optimizer::SemOptimizerOptim) = optimizer.algorithm -options(optimizer::SemOptimizerOptim) = optimizer.options +algorithm(optimizer::SemOptimizerName) = optimizer.algorithm +options(optimizer::SemOptimizerName) = optimizer.options ``` -Now comes a part that is a little bit more complicated: We need to write methods for `sem_fit`: - -```julia -function sem_fit( - model::AbstractSemSingle{O, I, L, D}; - start_val = start_val, - kwargs...) where {O, I, L, D <: SemOptimizerOptim} - - if !isa(start_val, Vector) - start_val = start_val(model; kwargs...) - end - - optimization_result = ... - - ... - - return SemFit(minimum, minimizer, start_val, model, optimization_result) -end -``` +Note that your optimizer is a subtype of `SemOptimizer{:Name}`, where you can choose a `:Name` that can later be used as a keyword argument to `sem_fit(engine = :Name)`. +Similarly, `SemOptimizer{:Name}(args...; kwargs...) = SemOptimizerName(args...; kwargs...)` should be defined as well as a constructor that uses only keyword arguments: -The method has to return a `SemFit` object that consists of the minimum of the objective at the solution, the minimizer (aka parameter estimates), the starting values, the model and the optimization result (which may be anything you desire for your specific backend). +´´´julia +SemOptimizerName(; + algorithm = LBFGS(), + options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), + kwargs..., +) = SemOptimizerName(algorithm, options) +´´´ +A method for `update_observed` and additional methods might be usefull, but are not necessary. -If we want our type to also work with `SemEnsemble` models, we also have to provide a method for that: +Now comes the substantive part: We need to provide a method for `sem_fit`: ```julia function sem_fit( - model::SemEnsemble{N, T , V, D, S}; - start_val = start_val, - kwargs...) where {N, T, V, D <: SemOptimizerOptim, S} - - if !isa(start_val, Vector) - start_val = start_val(model; kwargs...) - end - - + optim::SemOptimizerName, + model::AbstractSem, + start_params::AbstractVector; + kwargs..., +) optimization_result = ... ... - return SemFit(minimum, minimizer, start_val, model, optimization_result) - + return SemFit(minimum, minimizer, start_params, model, optimization_result) end ``` +The method has to return a `SemFit` object that consists of the minimum of the objective at the solution, the minimizer (aka parameter estimates), the starting values, the model and the optimization result (which may be anything you desire for your specific backend). + In addition, you might want to provide methods to access properties of your optimization result: ```julia diff --git a/docs/src/developer/sem.md b/docs/src/developer/sem.md index 0063a85cf..c54ff26af 100644 --- a/docs/src/developer/sem.md +++ b/docs/src/developer/sem.md @@ -1,37 +1,22 @@ # Custom model types -The abstract supertype for all models is `AbstractSem`, which has two subtypes, `AbstractSemSingle{O, I, L, D}` and `AbstractSemCollection`. Currently, there are 2 subtypes of `AbstractSemSingle`: `Sem`, `SemFiniteDiff`. All subtypes of `AbstractSemSingle` should have at least observed, implied, loss and optimizer fields, and share their types (`{O, I, L, D}`) with the parametric abstract supertype. For example, the `SemFiniteDiff` type is implemented as +The abstract supertype for all models is `AbstractSem`, which has two subtypes, `AbstractSemSingle{O, I, L}` and `AbstractSemCollection`. Currently, there are 2 subtypes of `AbstractSemSingle`: `Sem`, `SemFiniteDiff`. All subtypes of `AbstractSemSingle` should have at least observed, implied, loss and optimizer fields, and share their types (`{O, I, L}`) with the parametric abstract supertype. For example, the `SemFiniteDiff` type is implemented as ```julia -struct SemFiniteDiff{ - O <: SemObserved, - I <: SemImplied, - L <: SemLoss, - D <: SemOptimizer} <: AbstractSemSingle{O, I, L, D} +struct SemFiniteDiff{O <: SemObserved, I <: SemImplied, L <: SemLoss} <: + AbstractSemSingle{O, I, L} observed::O implied::I loss::L - optimizer::D end ``` -Additionally, we need to define a method to compute at least the objective value, and if you want to use gradient based optimizers (which you most probably will), we need also to define a method to compute the gradient. For example, the respective fallback methods for all `AbstractSemSingle` models are defined as +Additionally, you can change how objective/gradient/hessian values are computed by providing methods for `evaluate!`, e.g. from `SemFiniteDiff`'s implementation: ```julia -function objective!(model::AbstractSemSingle, parameters) - objective!(implied(model), parameters, model) - return objective!(loss(model), parameters, model) -end - -function gradient!(gradient, model::AbstractSemSingle, parameters) - fill!(gradient, zero(eltype(gradient))) - gradient!(implied(model), parameters, model) - gradient!(gradient, loss(model), parameters, model) -end +evaluate!(objective, gradient, hessian, model::SemFiniteDiff, params) = ... ``` -Note that the `gradient!` method takes a pre-allocated array that should be filled with the gradient values. - Additionally, we can define constructors like the one in `"src/frontend/specification/Sem.jl"`. It is also possible to add new subtypes for `AbstractSemCollection`. \ No newline at end of file diff --git a/docs/src/internals/files.md b/docs/src/internals/files.md index 9cf455fdc..0872c2b02 100644 --- a/docs/src/internals/files.md +++ b/docs/src/internals/files.md @@ -4,7 +4,7 @@ We briefly describe the file and folder structure of the package. ## Source code -All source code is in the `"src"` folder: +Source code is in the `"src"` folder: `"src"` - `"StructuralEquationModels.jl"` defines the module and the exported objects @@ -13,12 +13,15 @@ All source code is in the `"src"` folder: - The four folders `"observed"`, `"implied"`, `"loss"` and `"diff"` contain implementations of specific subtypes (for example, the `"loss"` folder contains a file `"ML.jl"` that implements the `SemML` loss function). - `"optimizer"` contains connections to different optimization backends (aka methods for `sem_fit`) - `"optim.jl"`: connection to the `Optim.jl` package - - `"NLopt.jl"`: connection to the `NLopt.jl` package - `"frontend"` contains user-facing functions - `"specification"` contains functionality for model specification - `"fit"` contains functionality for model assessment, like fit measures and standard errors - `"additional_functions"` contains helper functions for simulations, loading artifacts (example data) and various other things +Code for the package extentions can be found in the `"ext"` folder: +- `"SEMNLOptExt"` for connection to `NLopt.jl`. +- `"SEMProximalOptExt"` for connection to `ProximalAlgorithms.jl`. + ## Tests and Documentation Tests are in the `"test"` folder, documentation in the `"docs"` folder. \ No newline at end of file diff --git a/docs/src/internals/types.md b/docs/src/internals/types.md index 980d0f42f..e70a52ca4 100644 --- a/docs/src/internals/types.md +++ b/docs/src/internals/types.md @@ -3,11 +3,11 @@ The type hierarchy is implemented in `"src/types.jl"`. `AbstractSem`: the most abstract type in our package -- `AbstractSemSingle{O, I, L, D} <: AbstractSem` is an abstract parametric type that is a supertype of all single models +- `AbstractSemSingle{O, I, L} <: AbstractSem` is an abstract parametric type that is a supertype of all single models - `Sem`: models that do not need automatic differentiation or finite difference approximation - `SemFiniteDiff`: models whose gradients and/or hessians should be computed via finite difference approximation - `AbstractSemCollection <: AbstractSem` is an abstract supertype of all models that contain multiple `AbstractSem` submodels -Every `AbstractSemSingle` has to have `SemObserved`, `SemImplied`, `SemLoss` and `SemOptimizer` fields (and can have additional fields). +Every `AbstractSemSingle` has to have `SemObserved`, `SemImplied`, and `SemLoss` fields (and can have additional fields). `SemLoss` is a container for multiple `SemLossFunctions`. \ No newline at end of file diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index e46be64a7..881da6222 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -4,7 +4,7 @@ We are currently working on an interface for simulation studies. Until we are finished with this, this page is just a collection of tips. -## Swap observed data +## Replace observed data In simulation studies, a common task is fitting the same model to many different datasets. It would be a waste of resources to reconstruct the complete model for each dataset. We therefore provide the function `replace_observed` to change the `observed` part of a model, @@ -64,25 +64,44 @@ model = Sem( model_updated = replace_observed(model; data = data_2, specification = partable) ``` +If you are building your models by parts, you can also update each part seperately with the function `update_observed`. +For example, + +```@example replace_observed + +new_observed = SemObservedData(;data = data_2, specification = partable) + +my_optimizer = SemOptimizerOptim() + +new_optimizer = update_observed(my_optimizer, new_observed) +``` + +## Multithreading !!! danger "Thread safety" *This is only relevant when you are planning to fit updated models in parallel* - Models generated this way may share the same objects in memory (e.g. some parts of + Models generated by `replace_observed` may share the same objects in memory (e.g. some parts of `model` and `model_updated` are the same objects in memory.) Therefore, fitting both of these models in parallel will lead to **race conditions**, possibly crashing your computer. To avoid these problems, you should copy `model` before updating it. -If you are building your models by parts, you can also update each part seperately with the function `update_observed`. -For example, +Taking into account the warning above, fitting multiple models in parallel becomes as easy as: -```@example replace_observed +```julia +model1 = Sem( + specification = partable, + data = data_1 +) -new_observed = SemObservedData(;data = data_2, specification = partable) +model2 = deepcopy(replace_observed(model; data = data_2, specification = partable)) -my_optimizer = SemOptimizerOptim() +models = [model1, model2] +fits = Vector{SemFit}(undef, 2) -new_optimizer = update_observed(my_optimizer, new_observed) +Threads.@threads for i in 1:2 + fits[i] = sem_fit(models[i]) +end ``` ## API diff --git a/docs/src/tutorials/backends/nlopt.md b/docs/src/tutorials/backends/nlopt.md index f861e174e..2afa5e547 100644 --- a/docs/src/tutorials/backends/nlopt.md +++ b/docs/src/tutorials/backends/nlopt.md @@ -35,6 +35,8 @@ my_optimizer = SemOptimizerNLopt(; This uses an augmented lagrangian method with LBFGS as the local optimization algorithm, stops at a maximum of 200 evaluations and uses a relative tolerance of the objective value of `1e-6` as the stopping criterion for the local algorithm. +To see how to use the optimizer to actually fit a model now, check out the [Model fitting](@ref) section. + In the NLopt docs, you can find explanations about the different [algorithms](https://nlopt.readthedocs.io/en/latest/NLopt_Algorithms/) and a [tutorial](https://nlopt.readthedocs.io/en/latest/NLopt_Introduction/) that also explains the different options. To choose an algorithm, just pass its name without the 'NLOPT\_' prefix (for example, 'NLOPT\_LD\_SLSQP' can be used by passing `algorithm = :LD_SLSQP`). diff --git a/docs/src/tutorials/backends/optim.md b/docs/src/tutorials/backends/optim.md index aaaf4ac9b..cf287e773 100644 --- a/docs/src/tutorials/backends/optim.md +++ b/docs/src/tutorials/backends/optim.md @@ -17,6 +17,8 @@ my_optimizer = SemOptimizerOptim( ) ``` -A model with this optimizer object will use BFGS (!not L-BFGS) with a back tracking linesearch and a certain initial step length guess. Also, the trace of the optimization will be printed to the console. +This optimizer will use BFGS (!not L-BFGS) with a back tracking linesearch and a certain initial step length guess. Also, the trace of the optimization will be printed to the console. + +To see how to use the optimizer to actually fit a model now, check out the [Model fitting](@ref) section. For a list of all available algorithms and options, we refer to [this page](https://julianlsolvers.github.io/Optim.jl/stable/#user/config/) of the `Optim.jl` manual. \ No newline at end of file diff --git a/docs/src/tutorials/collection/collection.md b/docs/src/tutorials/collection/collection.md index 84fa00500..f60b7312c 100644 --- a/docs/src/tutorials/collection/collection.md +++ b/docs/src/tutorials/collection/collection.md @@ -15,11 +15,10 @@ model_2 = SemFiniteDiff(...) model_3 = Sem(...) -model_ensemble = SemEnsemble(model_1, model_2, model_3; optimizer = ...) +model_ensemble = SemEnsemble(model_1, model_2, model_3) ``` So you just construct the individual models (however you like) and pass them to `SemEnsemble`. -One important thing to note is that the individual optimizer entries of each model do not matter (as you can optimize your ensemble model only with one algorithmn from one optimization suite). Instead, `SemEnsemble` has its own optimizer part that specifies the backend for the whole ensemble model. You may also pass a vector of weigths to `SemEnsemble`. By default, those are set to ``N_{model}/N_{total}``, i.e. each model is weighted by the number of observations in it's data (which matches the formula for multigroup models). Multigroup models can also be specified via the graph interface; for an example, see [Multigroup models](@ref). diff --git a/docs/src/tutorials/collection/multigroup.md b/docs/src/tutorials/collection/multigroup.md index d0fc71796..23c13b950 100644 --- a/docs/src/tutorials/collection/multigroup.md +++ b/docs/src/tutorials/collection/multigroup.md @@ -81,8 +81,8 @@ model_ml_multigroup = SemEnsemble( We now fit the model and inspect the parameter estimates: ```@example mg; ansicolor = true -solution = sem_fit(model_ml_multigroup) -update_estimate!(partable, solution) +fit = sem_fit(model_ml_multigroup) +update_estimate!(partable, fit) details(partable) ``` diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index ffd83d4e0..b1fff82b8 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -148,11 +148,10 @@ In this example, we set both tolerances to `1e-8`. ```@example constraints model_constrained = Sem( specification = partable, - data = data, - optimizer = constrained_optimizer + data = data ) -model_fit_constrained = sem_fit(model_constrained) +model_fit_constrained = sem_fit(constrained_optimizer, model_constrained) ``` As you can see, the optimizer converged (`:XTOL_REACHED`) and investigating the solution yields diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md index 7de3d9e61..6a3cd2cef 100644 --- a/docs/src/tutorials/construction/outer_constructor.md +++ b/docs/src/tutorials/construction/outer_constructor.md @@ -21,7 +21,7 @@ Structural Equation Model The output of this call tells you exactly what model you just constructed (i.e. what the loss functions, observed, implied and optimizer parts are). -As you can see, by default, we use maximum likelihood estimation, the RAM implied type and the `Optim.jl` optimization backend. +As you can see, by default, we use maximum likelihood estimation abd the RAM implied type. To choose something different, you can provide it as a keyword argument: ```julia @@ -31,11 +31,10 @@ model = Sem( observed = ..., implied = ..., loss = ..., - optimizer = ... ) ``` -For example, to construct a model for weighted least squares estimation that uses symbolic precomputation and the Optim backend, write +For example, to construct a model for weighted least squares estimation that uses symbolic precomputation, write ```julia model = Sem( diff --git a/docs/src/tutorials/fitting/fitting.md b/docs/src/tutorials/fitting/fitting.md index b534ad754..a3e4b9b91 100644 --- a/docs/src/tutorials/fitting/fitting.md +++ b/docs/src/tutorials/fitting/fitting.md @@ -43,7 +43,29 @@ Structural Equation Model ∇f(x) calls: 524 ``` -You may optionally specify [Starting values](@ref). +## Choosing an optimizer + +To choose a different optimizer, you can call `sem_fit` with the keyword argument `engine = ...`, and pass additional keyword arguments: + +```julia +using Optim + +model_fit = sem_fit(model; engine = :Optim, algorithm = BFGS()) +``` + +Available options for engine are `:Optim`, `:NLopt` and `:Proximal`, where `:NLopt` and `:Proximal` are only available if the `NLopt.jl` and `ProximalAlgorithms.jl` packages are loaded respectively. + +The available keyword arguments are listed in the sections [Using Optim.jl](@ref), [Using NLopt.jl](@ref) and [Regularization](@ref). + +Alternative, you can also explicitely define a `SemOptimizer` and pass it as the first argument to `sem_fit`: + +```julia +my_optimizer = SemOptimizerOptim(algorithm = BFGS()) + +sem_fit(my_optimizer, model) +``` + +You may also optionally specify [Starting values](@ref). # API - model fitting diff --git a/docs/src/tutorials/meanstructure.md b/docs/src/tutorials/meanstructure.md index 692f6cebc..dd5a7f171 100644 --- a/docs/src/tutorials/meanstructure.md +++ b/docs/src/tutorials/meanstructure.md @@ -35,7 +35,7 @@ graph = @StenoGraph begin y8 ↔ y4 + y6 # means - Symbol("1") → _(observed_vars) + Symbol(1) → _(observed_vars) end partable = ParameterTable( @@ -73,7 +73,7 @@ graph = @StenoGraph begin y8 ↔ y4 + y6 # means - Symbol("1") → _(observed_vars) + Symbol(1) → _(observed_vars) end partable = ParameterTable( @@ -99,7 +99,7 @@ model = Sem( sem_fit(model) ``` -If we build the model by parts, we have to pass the `meanstructure = true` argument to every part that requires it (when in doubt, simply comsult the documentation for the respective part). +If we build the model by parts, we have to pass the `meanstructure = true` argument to every part that requires it (when in doubt, simply consult the documentation for the respective part). For our example, diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 89fb6d151..27d58f93f 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -11,7 +11,7 @@ Return a new model with swaped observed part. - `observed`: Either an object of subtype of `SemObserved` or a subtype of `SemObserved` # Examples -See the online documentation on [Swap observed data](@ref). +See the online documentation on [Replace observed data](@ref). """ function replace_observed end @@ -21,7 +21,7 @@ function replace_observed end Update a `SemImplied`, `SemLossFunction` or `SemOptimizer` object to use a `SemObserved` object. # Examples -See the online documentation on [Swap observed data](@ref). +See the online documentation on [Replace observed data](@ref). # Implementation You can provide a method for this function when defining a new type, for more information diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 70bf6816c..8ee134a9c 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -212,7 +212,7 @@ function details( ) print("\n") - mean_indices = findall(r -> (r.relation == :→) && (r.from == Symbol("1")), partable) + mean_indices = findall(r -> (r.relation == :→) && (r.from == Symbol(1)), partable) if length(mean_indices) > 0 printstyled("Means: \n"; color = color) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index c5ad010b3..74c963ccb 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -197,7 +197,7 @@ function sort_vars!(partable::ParameterTable) partable.columns[:relation], partable.columns[:from], partable.columns[:to], - ) if (rel == :→) && (from != Symbol("1")) + ) if (rel == :→) && (from != Symbol(1)) ] sort!(edges, by = last) # sort edges by target @@ -492,7 +492,7 @@ function lavaan_param_values!( ) lav_ind = nothing - if from == Symbol("1") + if from == Symbol(1) lav_ind = findallrows( r -> r[:lhs] == String(to) && diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 43fd87945..4ebea95fb 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -154,16 +154,16 @@ function RAMMatrices( S_consts = Vector{Pair{Int, T}}() # is there a meanstructure? M_inds = - any(==(Symbol("1")), partable.columns[:from]) ? + any(==(Symbol(1)), partable.columns[:from]) ? [Vector{Int64}() for _ in 1:length(params)] : nothing M_consts = !isnothing(M_inds) ? Vector{Pair{Int, T}}() : nothing for r in partable row_ind = vars_index[r.to] - col_ind = r.from != Symbol("1") ? vars_index[r.from] : nothing + col_ind = r.from != Symbol(1) ? vars_index[r.from] : nothing if !r.free - if (r.relation == :→) && (r.from == Symbol("1")) + if (r.relation == :→) && (r.from == Symbol(1)) push!(M_consts, row_ind => r.value_fixed) elseif r.relation == :→ push!( @@ -186,7 +186,7 @@ function RAMMatrices( end else par_ind = params_index[r.param] - if (r.relation == :→) && (r.from == Symbol("1")) + if (r.relation == :→) && (r.from == Symbol(1)) push!(M_inds[par_ind], row_ind) elseif r.relation == :→ push!(A_inds[par_ind], A_lin_ixs[CartesianIndex(row_ind, col_ind)]) @@ -328,7 +328,7 @@ function partable_row( # variable names if matrix == :M - from = Symbol("1") + from = Symbol(1) to = varnames[index] else from = varnames[index[2]] diff --git a/src/implied/empty.jl b/src/implied/empty.jl index e87dc72d1..11cc579a4 100644 --- a/src/implied/empty.jl +++ b/src/implied/empty.jl @@ -25,17 +25,17 @@ model per group and an additional model with `ImpliedEmpty` and `SemRidge` for t ## Implementation Subtype of `SemImplied`. """ -struct ImpliedEmpty{V2} <: SemImplied - hessianeval::ExactHessian - meanstruct::NoMeanStruct - ram_matrices::V2 +struct ImpliedEmpty{A, B, C} <: SemImplied + hessianeval::A + meanstruct::B + ram_matrices::C end ############################################################################################ ### Constructors ############################################################################################ -function ImpliedEmpty(; specification, kwargs...) +function ImpliedEmpty(;specification, meanstruct = NoMeanStruct(), hessianeval = ExactHessian(), kwargs...) return ImpliedEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) end diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 2f5135176..1e97617fc 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -11,7 +11,7 @@ model_g2 = Sem(specification = specification_g2, data = dat_g2, implied = RAM) @test SEM.params(model_g1.implied.ram_matrices) == SEM.params(model_g2.implied.ram_matrices) # test the different constructors -model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) +model_ml_multigroup = SemEnsemble(model_g1, model_g2) model_ml_multigroup2 = SemEnsemble( specification = partable, data = dat, @@ -28,7 +28,7 @@ end # fit @testset "ml_solution_multigroup" begin - solution = sem_fit(model_ml_multigroup) + solution = sem_fit(semoptimizer, model_ml_multigroup) update_estimate!(partable, solution) test_estimates( partable, @@ -36,7 +36,7 @@ end atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = sem_fit(model_ml_multigroup2) + solution = sem_fit(semoptimizer, model_ml_multigroup2) update_estimate!(partable, solution) test_estimates( partable, @@ -268,7 +268,6 @@ if !isnothing(specification_miss_g1) loss = SemFIML, data = dat_miss_g1, implied = RAM, - optimizer = SemOptimizerEmpty(), meanstructure = true, ) @@ -278,11 +277,10 @@ if !isnothing(specification_miss_g1) loss = SemFIML, data = dat_miss_g2, implied = RAM, - optimizer = SemOptimizerEmpty(), meanstructure = true, ) - model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) + model_ml_multigroup = SemEnsemble(model_g1, model_g2) model_ml_multigroup2 = SemEnsemble( specification = partable_miss, data = dat_missing, @@ -323,7 +321,7 @@ if !isnothing(specification_miss_g1) end @testset "fiml_solution_multigroup" begin - solution = sem_fit(model_ml_multigroup) + solution = sem_fit(semoptimizer, model_ml_multigroup) update_estimate!(partable_miss, solution) test_estimates( partable_miss, @@ -331,7 +329,7 @@ if !isnothing(specification_miss_g1) atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = sem_fit(model_ml_multigroup2) + solution = sem_fit(semoptimizer, model_ml_multigroup2) update_estimate!(partable_miss, solution) test_estimates( partable_miss, @@ -342,7 +340,7 @@ if !isnothing(specification_miss_g1) end @testset "fitmeasures/se_fiml" begin - solution = sem_fit(model_ml_multigroup) + solution = sem_fit(semoptimizer, model_ml_multigroup) test_fitmeasures( fit_measures(solution), solution_lav[:fitmeasures_fiml]; @@ -359,7 +357,7 @@ if !isnothing(specification_miss_g1) lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = sem_fit(model_ml_multigroup2) + solution = sem_fit(semoptimizer, model_ml_multigroup2) test_fitmeasures( fit_measures(solution), solution_lav[:fitmeasures_fiml]; diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 7dd871ac2..eac2b38dd 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -86,7 +86,7 @@ start_test = [ fill(0.05, 3) fill(0.01, 3) ] -semoptimizer = SemOptimizerOptim +semoptimizer = SemOptimizerOptim() @testset "RAMMatrices | constructor | Optim" begin include("build_models.jl") @@ -137,7 +137,7 @@ graph = @StenoGraph begin _(observed_vars) ↔ _(observed_vars) _(latent_vars) ⇔ _(latent_vars) - Symbol("1") → _(observed_vars) + Symbol(1) → _(observed_vars) end partable_miss = EnsembleParameterTable( @@ -169,7 +169,7 @@ start_test = [ 0.01 0.05 ] -semoptimizer = SemOptimizerOptim +semoptimizer = SemOptimizerOptim() @testset "Graph → Partable → RAMMatrices | constructor | Optim" begin include("build_models.jl") diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 9d026fb28..7394175b7 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -216,8 +216,8 @@ graph = @StenoGraph begin y3 ↔ y7 y8 ↔ y4 + y6 # means - Symbol("1") → _(mean_labels) .* _(observed_vars) - Symbol("1") → fixed(0) * ind60 + Symbol(1) → _(mean_labels) .* _(observed_vars) + Symbol(1) → fixed(0) * ind60 end spec_mean = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) From b91f25a0e88355b5880118ad8400667a5588a613 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 13:50:56 +0100 Subject: [PATCH 184/364] rm ProximalSEM from docs deps --- docs/Project.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/Project.toml b/docs/Project.toml index 2daded98f..42f6718a9 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -4,4 +4,3 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" -ProximalSEM = "3652f839-8142-48b2-a17c-985bd14407c5" From 56650e792cdef3056aae13e45fab038ec9f8517b Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 14:09:13 +0100 Subject: [PATCH 185/364] fix docs --- docs/src/tutorials/concept.md | 2 ++ docs/src/tutorials/constraints/constraints.md | 2 ++ docs/src/tutorials/construction/build_by_parts.md | 2 +- docs/src/tutorials/inspection/inspection.md | 1 + docs/src/tutorials/meanstructure.md | 2 +- 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index d663d3c2c..e4d116877 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -72,6 +72,8 @@ SemObserved SemObservedData SemObservedCovariance SemObservedMissing +samples +SemSpecification ``` ## implied diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index b1fff82b8..cdd9111a2 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -122,6 +122,8 @@ In NLopt, vector-valued constraints are also possible, but we refer to the docum We now have everything together to specify and fit our model. First, we specify our optimizer backend as ```@example constraints +using NLopt + constrained_optimizer = SemOptimizerNLopt( algorithm = :AUGLAG, options = Dict(:upper_bounds => upper_bounds, :xtol_abs => 1e-4), diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 27604d2a1..606a6576e 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -59,7 +59,7 @@ ml = SemML(observed = observed) loss_ml = SemLoss(ml) # optimizer ------------------------------------------------------------------------------------- -optimizer = SemOptimizerOptim(algorithm = BFGS()) +optimizer = SemOptimizerOptim() # model ------------------------------------------------------------------------------------ diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md index faab8f8ed..2b6d3191f 100644 --- a/docs/src/tutorials/inspection/inspection.md +++ b/docs/src/tutorials/inspection/inspection.md @@ -130,6 +130,7 @@ df minus2ll nobserved_vars nsamples +params nparams p_value RMSEA diff --git a/docs/src/tutorials/meanstructure.md b/docs/src/tutorials/meanstructure.md index dd5a7f171..60578224a 100644 --- a/docs/src/tutorials/meanstructure.md +++ b/docs/src/tutorials/meanstructure.md @@ -110,7 +110,7 @@ implied_ram = RAM(specification = partable, meanstructure = true) ml = SemML(observed = observed, meanstructure = true) -model = Sem(observed, implied_ram, SemLoss(ml), SemOptimizerOptim()) +model = Sem(observed, implied_ram, SemLoss(ml)) sem_fit(model) ``` \ No newline at end of file From 0f09635652770f882434ccffab18fe026e8959d7 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 14:17:44 +0100 Subject: [PATCH 186/364] fix docs --- docs/src/tutorials/concept.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index e4d116877..035144d62 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -73,6 +73,7 @@ SemObservedData SemObservedCovariance SemObservedMissing samples +observed_vars SemSpecification ``` From 5cc61bb0eca593746a51450cd0c8f638545f64d0 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 15:11:32 +0100 Subject: [PATCH 187/364] try to fix svgs for docs --- docs/src/assets/concept.svg | 17 +++++++++++++++++ docs/src/assets/concept_typed.svg | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/src/assets/concept.svg b/docs/src/assets/concept.svg index 138463b67..c5a3c6bb6 100644 --- a/docs/src/assets/concept.svg +++ b/docs/src/assets/concept.svg @@ -6,6 +6,7 @@ stroke="none" stroke-linecap="square" stroke-miterlimit="10" +<<<<<<< Updated upstream id="svg23" sodipodi:docname="Unbenannte Präsentation (2).svg" width="921600" @@ -34,6 +35,22 @@ margin="0" bleed="0" /> +======= + id="svg57" + width="610.56537" + height="300.26614" + xmlns="http://www.w3.org/2000/svg" + xmlns:svg="http://www.w3.org/2000/svg"> + + + + +>>>>>>> Stashed changes +======= + id="svg57" + width="610.56537" + height="300.26614" + xmlns="http://www.w3.org/2000/svg" + xmlns:svg="http://www.w3.org/2000/svg"> + + + + +>>>>>>> Stashed changes Date: Tue, 4 Feb 2025 15:13:04 +0100 Subject: [PATCH 188/364] try to fix svgs for docs --- docs/src/assets/concept.svg | 136 +++++++++++++---------------- docs/src/assets/concept_typed.svg | 139 +++++++++++++----------------- 2 files changed, 124 insertions(+), 151 deletions(-) diff --git a/docs/src/assets/concept.svg b/docs/src/assets/concept.svg index c5a3c6bb6..fa222a0d9 100644 --- a/docs/src/assets/concept.svg +++ b/docs/src/assets/concept.svg @@ -1,41 +1,11 @@ - - - - -======= id="svg57" width="610.56537" height="300.26614" @@ -50,134 +20,152 @@ clip-rule="nonzero" id="path2" /> ->>>>>>> Stashed changes + id="path7" /> + id="path9" /> + id="path11" /> + id="path13" /> + id="path15" /> + id="path17" /> + id="path19" /> + id="path21" /> + id="path23" /> + id="path25" /> + id="path27" /> + id="path29" /> + id="path31" /> + id="path33" /> + id="path35" /> + id="path37" /> + id="path39" /> + id="path41" /> + id="path43" /> + id="path45" /> + id="path47" /> + + + diff --git a/docs/src/assets/concept_typed.svg b/docs/src/assets/concept_typed.svg index 9b2d72305..88a0d8566 100644 --- a/docs/src/assets/concept_typed.svg +++ b/docs/src/assets/concept_typed.svg @@ -1,44 +1,11 @@ - - - - -======= id="svg57" width="610.56537" height="300.26614" @@ -53,134 +20,152 @@ clip-rule="nonzero" id="path2" /> ->>>>>>> Stashed changes + id="path7" /> + id="path9" /> + id="path11" /> + id="path13" /> + id="path15" /> + id="path17" /> + id="path19" /> + id="path21" /> + id="path23" /> + id="path25" /> + id="path27" /> + id="path29" /> + id="path31" /> + id="path33" /> + id="path35" /> + id="path37" /> + id="path39" /> + id="path41" /> + id="path43" /> + id="path45" /> + id="path47" /> + + + From b32701263b91959c1882dc07df40ab3dab7f64ce Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 15:33:26 +0100 Subject: [PATCH 189/364] update README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3eeafd332..79c11da21 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ It is still *in development*. Models you can fit include - Linear SEM that can be specified in RAM (or LISREL) notation - ML, GLS and FIML estimation -- Regularization +- Regularized SEM (Ridge, Lasso, L0, ...) - Multigroup SEM - Sums of arbitrary loss functions (everything the optimizer can handle). @@ -35,6 +35,7 @@ The package makes use of - Symbolics.jl for symbolically precomputing parts of the objective and gradients to generate fast, specialized functions. - SparseArrays.jl to speed up symbolic computations. - Optim.jl and NLopt.jl to provide a range of different Optimizers/Linesearches. +- ProximalAlgorithms.jl for regularization. - FiniteDiff.jl and ForwardDiff.jl to provide gradients for user-defined loss functions. # At the moment, we are still working on: From 4091804cca8c43d6eac6784fe489b61649ace1b3 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 15:38:04 +0100 Subject: [PATCH 190/364] bump version --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index d55346aca..94ab214e8 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "StructuralEquationModels" uuid = "383ca8c5-e4ff-4104-b0a9-f7b279deed53" authors = ["Maximilian Ernst", "Aaron Peikert"] -version = "0.2.4" +version = "0.3.0" [deps] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" @@ -24,7 +24,7 @@ Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b" [compat] -julia = "1.9, 1.10" +julia = "1.9, 1.10, 1.11" StenoGraphs = "0.2 - 0.3, 0.4.1 - 0.5" DataFrames = "1" Distributions = "0.25" From 955a18129d8f3ace49714aab7fde66b2293bfece Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 17:23:00 +0100 Subject: [PATCH 191/364] give macos some slack and format --- src/StructuralEquationModels.jl | 1 - src/implied/empty.jl | 7 ++++++- src/package_extensions/SEMNLOptExt.jl | 2 +- src/package_extensions/SEMProximalOptExt.jl | 2 +- test/examples/political_democracy/by_parts.jl | 2 +- test/examples/political_democracy/constructor.jl | 2 +- test/examples/proximal/ridge.jl | 2 +- 7 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 5d6b23ef4..6e1a934f3 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -86,7 +86,6 @@ include("frontend/fit/standard_errors/bootstrap.jl") include("package_extensions/SEMNLOptExt.jl") include("package_extensions/SEMProximalOptExt.jl") - export AbstractSem, AbstractSemSingle, AbstractSemCollection, diff --git a/src/implied/empty.jl b/src/implied/empty.jl index 11cc579a4..a80f8c185 100644 --- a/src/implied/empty.jl +++ b/src/implied/empty.jl @@ -35,7 +35,12 @@ end ### Constructors ############################################################################################ -function ImpliedEmpty(;specification, meanstruct = NoMeanStruct(), hessianeval = ExactHessian(), kwargs...) +function ImpliedEmpty(; + specification, + meanstruct = NoMeanStruct(), + hessianeval = ExactHessian(), + kwargs..., +) return ImpliedEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) end diff --git a/src/package_extensions/SEMNLOptExt.jl b/src/package_extensions/SEMNLOptExt.jl index 7eae2f268..69721ac94 100644 --- a/src/package_extensions/SEMNLOptExt.jl +++ b/src/package_extensions/SEMNLOptExt.jl @@ -66,4 +66,4 @@ end Base.@kwdef struct NLoptConstraint f::Any tol = 0.0 -end \ No newline at end of file +end diff --git a/src/package_extensions/SEMProximalOptExt.jl b/src/package_extensions/SEMProximalOptExt.jl index e8b256704..5d4007504 100644 --- a/src/package_extensions/SEMProximalOptExt.jl +++ b/src/package_extensions/SEMProximalOptExt.jl @@ -18,4 +18,4 @@ mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} algorithm::A operator_g::B operator_h::C -end \ No newline at end of file +end diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 88f98ded2..ddbbfc3fa 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -178,7 +178,7 @@ if opt_engine == :Optim @testset "ml_solution_hessian" begin solution = sem_fit(optimizer_obj, model_ml) update_estimate!(partable, solution) - test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) + test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-2) end @testset "ls_solution_hessian" begin diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index bbeb0c648..3f226b4c8 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -227,7 +227,7 @@ if opt_engine == :Optim @testset "ml_solution_hessian" begin solution = sem_fit(semoptimizer, model_ml) update_estimate!(partable, solution) - test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) + test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-2) end @testset "ls_solution_hessian" begin diff --git a/test/examples/proximal/ridge.jl b/test/examples/proximal/ridge.jl index 8c0a1df7a..3d116dcd4 100644 --- a/test/examples/proximal/ridge.jl +++ b/test/examples/proximal/ridge.jl @@ -57,5 +57,5 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML) solution_prox = @suppress sem_fit(model_prox, engine = :Proximal, operator_g = SqrNormL2(λ)) @testset "ridge_solution" begin - @test isapprox(solution_prox.solution, solution_ridge.solution; rtol = 1e-4) + @test isapprox(solution_prox.solution, solution_ridge.solution; rtol = 1e-3) end From cbb666b63f0ed21829e75b6717187bc776761f6a Mon Sep 17 00:00:00 2001 From: Aaron Peikert Date: Mon, 17 Mar 2025 20:14:17 +0100 Subject: [PATCH 192/364] Rename params (#253) * first sweep of renaming * fix destroyed types * parameter table column renamed to label * param and param_labels, params!, seem to work * allow partial execution of unit tests * remove non existing tests * fix model unittests * remove unnessary test layer * finish replacing * all unit tests passed * rename param_values -> params * add StatsAPI as dep * add coef and coefnames * rename df => dof (#254) * rename df => dof * import dof from StatsAPI * rename dof file * rename sem_fit => fit * typo * add nobs and fix testsw * add coeftable * fix proximal tests * fix exports and StatsAPI docstrings * fix tests * fix tests * thx evie for the typo :) * fix coeftable --------- Co-authored-by: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- Project.toml | 1 + README.md | 2 +- docs/src/developer/loss.md | 10 +- docs/src/developer/optimizer.md | 6 +- docs/src/internals/files.md | 2 +- docs/src/performance/mixed_differentiation.md | 6 +- docs/src/performance/mkl.md | 4 +- docs/src/performance/simulation.md | 2 +- docs/src/performance/starting_values.md | 4 +- docs/src/tutorials/collection/multigroup.md | 2 +- docs/src/tutorials/constraints/constraints.md | 6 +- .../tutorials/construction/build_by_parts.md | 2 +- .../construction/outer_constructor.md | 2 +- docs/src/tutorials/first_model.md | 2 +- docs/src/tutorials/fitting/fitting.md | 12 +- docs/src/tutorials/inspection/inspection.md | 12 +- docs/src/tutorials/meanstructure.md | 4 +- .../regularization/regularization.md | 14 +- .../tutorials/specification/ram_matrices.md | 4 +- ext/SEMNLOptExt/NLopt.jl | 4 +- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 2 +- src/StructuralEquationModels.jl | 16 +- src/additional_functions/params_array.jl | 44 +++--- src/frontend/StatsAPI.jl | 78 ++++++++++ src/frontend/common.jl | 4 +- src/frontend/fit/SemFit.jl | 2 +- src/frontend/fit/fitmeasures/RMSEA.jl | 8 +- .../fit/fitmeasures/{df.jl => dof.jl} | 10 +- src/frontend/fit/fitmeasures/fit_measures.jl | 2 +- src/frontend/fit/fitmeasures/p.jl | 2 +- src/frontend/fit/standard_errors/bootstrap.jl | 4 +- .../specification/EnsembleParameterTable.jl | 38 ++--- src/frontend/specification/ParameterTable.jl | 146 ++++++------------ src/frontend/specification/RAMMatrices.jl | 64 ++++---- src/frontend/specification/Sem.jl | 7 +- src/frontend/specification/StenoGraphs.jl | 12 +- src/frontend/specification/checks.jl | 14 +- src/frontend/specification/documentation.jl | 10 +- src/implied/RAM/generic.jl | 10 +- src/implied/RAM/symbolic.jl | 2 +- src/implied/abstract.jl | 2 +- src/implied/empty.jl | 2 +- src/loss/ML/FIML.jl | 10 +- src/loss/regularization/ridge.jl | 2 +- src/optimizer/abstract.jl | 14 +- src/optimizer/optim.jl | 2 +- src/types.jl | 12 +- test/examples/helper.jl | 12 +- test/examples/multigroup/build_models.jl | 28 ++-- test/examples/multigroup/multigroup.jl | 4 +- test/examples/political_democracy/by_parts.jl | 32 ++-- .../political_democracy/constraints.jl | 4 +- .../political_democracy/constructor.jl | 42 ++--- .../political_democracy.jl | 10 +- test/examples/proximal/l0.jl | 10 +- test/examples/proximal/lasso.jl | 12 +- test/examples/proximal/ridge.jl | 6 +- .../recover_parameters_twofact.jl | 4 +- test/unit_tests/StatsAPI.jl | 29 ++++ test/unit_tests/bootstrap.jl | 2 +- test/unit_tests/model.jl | 3 +- test/unit_tests/sorting.jl | 2 +- test/unit_tests/specification.jl | 10 +- test/unit_tests/unit_tests.jl | 46 ++++-- test/unit_tests/unit_tests_interactive.jl | 10 ++ 65 files changed, 488 insertions(+), 398 deletions(-) create mode 100644 src/frontend/StatsAPI.jl rename src/frontend/fit/fitmeasures/{df.jl => dof.jl} (62%) create mode 100644 test/unit_tests/StatsAPI.jl create mode 100644 test/unit_tests/unit_tests_interactive.jl diff --git a/Project.toml b/Project.toml index 94ab214e8..2b0075e39 100644 --- a/Project.toml +++ b/Project.toml @@ -18,6 +18,7 @@ PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +StatsAPI = "82ae8749-77ed-4fe6-ae5f-f523153014b0" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" StenoGraphs = "78862bba-adae-4a83-bb4d-33c106177f81" Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" diff --git a/README.md b/README.md index 79c11da21..9754a8c20 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Models you can fit include - Multigroup SEM - Sums of arbitrary loss functions (everything the optimizer can handle). -# What are the merrits? +# What are the merits? We provide fast objective functions, gradients, and for some cases hessians as well as approximations thereof. As a user, you can easily define custom loss functions. diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index 57a7b485d..931c2d0e5 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -79,7 +79,7 @@ model = SemFiniteDiff( loss = (SemML, myridge) ) -model_fit = sem_fit(model) +model_fit = fit(model) ``` This is one way of specifying the model - we now have **one model** with **multiple loss functions**. Because we did not provide a gradient for `Ridge`, we have to specify a `SemFiniteDiff` model that computes numerical gradients with finite difference approximation. @@ -117,7 +117,7 @@ model_new = Sem( loss = (SemML, myridge) ) -model_fit = sem_fit(model_new) +model_fit = fit(model_new) ``` The results are the same, but we can verify that the computational costs are way lower (for this, the julia package `BenchmarkTools` has to be installed): @@ -125,9 +125,9 @@ The results are the same, but we can verify that the computational costs are way ```julia using BenchmarkTools -@benchmark sem_fit(model) +@benchmark fit(model) -@benchmark sem_fit(model_new) +@benchmark fit(model_new) ``` The exact results of those benchmarks are of course highly depended an your system (processor, RAM, etc.), but you should see that the median computation time with analytical gradients drops to about 5% of the computation without analytical gradients. @@ -241,7 +241,7 @@ model_ml = SemFiniteDiff( loss = MaximumLikelihood() ) -model_fit = sem_fit(model_ml) +model_fit = fit(model_ml) ``` If you want to differentiate your own loss functions via automatic differentiation, check out the [AutoDiffSEM](https://github.com/StructuralEquationModels/AutoDiffSEM) package. diff --git a/docs/src/developer/optimizer.md b/docs/src/developer/optimizer.md index 82ec594d8..a651ec636 100644 --- a/docs/src/developer/optimizer.md +++ b/docs/src/developer/optimizer.md @@ -34,7 +34,7 @@ algorithm(optimizer::SemOptimizerName) = optimizer.algorithm options(optimizer::SemOptimizerName) = optimizer.options ``` -Note that your optimizer is a subtype of `SemOptimizer{:Name}`, where you can choose a `:Name` that can later be used as a keyword argument to `sem_fit(engine = :Name)`. +Note that your optimizer is a subtype of `SemOptimizer{:Name}`, where you can choose a `:Name` that can later be used as a keyword argument to `fit(engine = :Name)`. Similarly, `SemOptimizer{:Name}(args...; kwargs...) = SemOptimizerName(args...; kwargs...)` should be defined as well as a constructor that uses only keyword arguments: ´´´julia @@ -46,10 +46,10 @@ SemOptimizerName(; ´´´ A method for `update_observed` and additional methods might be usefull, but are not necessary. -Now comes the substantive part: We need to provide a method for `sem_fit`: +Now comes the substantive part: We need to provide a method for `fit`: ```julia -function sem_fit( +function fit( optim::SemOptimizerName, model::AbstractSem, start_params::AbstractVector; diff --git a/docs/src/internals/files.md b/docs/src/internals/files.md index 0872c2b02..90ceceaaf 100644 --- a/docs/src/internals/files.md +++ b/docs/src/internals/files.md @@ -11,7 +11,7 @@ Source code is in the `"src"` folder: - `"types.jl"` defines all abstract types and the basic type hierarchy - `"objective_gradient_hessian.jl"` contains methods for computing objective, gradient and hessian values for different model types as well as generic fallback methods - The four folders `"observed"`, `"implied"`, `"loss"` and `"diff"` contain implementations of specific subtypes (for example, the `"loss"` folder contains a file `"ML.jl"` that implements the `SemML` loss function). -- `"optimizer"` contains connections to different optimization backends (aka methods for `sem_fit`) +- `"optimizer"` contains connections to different optimization backends (aka methods for `fit`) - `"optim.jl"`: connection to the `Optim.jl` package - `"frontend"` contains user-facing functions - `"specification"` contains functionality for model specification diff --git a/docs/src/performance/mixed_differentiation.md b/docs/src/performance/mixed_differentiation.md index 2ac937077..b7ae333b5 100644 --- a/docs/src/performance/mixed_differentiation.md +++ b/docs/src/performance/mixed_differentiation.md @@ -19,7 +19,7 @@ model_ridge = SemFiniteDiff( model_ml_ridge = SemEnsemble(model_ml, model_ridge) -model_ml_ridge_fit = sem_fit(model_ml_ridge) +model_ml_ridge_fit = fit(model_ml_ridge) ``` The results of both methods will be the same, but we can verify that the computation costs differ (the package `BenchmarkTools` has to be installed for this): @@ -27,7 +27,7 @@ The results of both methods will be the same, but we can verify that the computa ```julia using BenchmarkTools -@benchmark sem_fit(model) +@benchmark fit(model) -@benchmark sem_fit(model_ml_ridge) +@benchmark fit(model_ml_ridge) ``` \ No newline at end of file diff --git a/docs/src/performance/mkl.md b/docs/src/performance/mkl.md index 0d5467658..4361ab445 100644 --- a/docs/src/performance/mkl.md +++ b/docs/src/performance/mkl.md @@ -27,9 +27,9 @@ To check the performance implications for fitting a SEM, you can use the [`Bench ```julia using BenchmarkTools -@benchmark sem_fit($your_model) +@benchmark fit($your_model) using MKL -@benchmark sem_fit($your_model) +@benchmark fit($your_model) ``` \ No newline at end of file diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index 881da6222..0cb2ea25d 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -100,7 +100,7 @@ models = [model1, model2] fits = Vector{SemFit}(undef, 2) Threads.@threads for i in 1:2 - fits[i] = sem_fit(models[i]) + fits[i] = fit(models[i]) end ``` diff --git a/docs/src/performance/starting_values.md b/docs/src/performance/starting_values.md index ba7b4f41d..2df8d94d4 100644 --- a/docs/src/performance/starting_values.md +++ b/docs/src/performance/starting_values.md @@ -1,9 +1,9 @@ # Starting values -The `sem_fit` function has a keyword argument that takes either a vector of starting values or a function that takes a model as input to compute starting values. Current options are `start_fabin3` for fabin 3 starting values [^Hägglund82] or `start_simple` for simple starting values. Additional keyword arguments to `sem_fit` are passed to the starting value function. For example, +The `fit` function has a keyword argument that takes either a vector of starting values or a function that takes a model as input to compute starting values. Current options are `start_fabin3` for fabin 3 starting values [^Hägglund82] or `start_simple` for simple starting values. Additional keyword arguments to `fit` are passed to the starting value function. For example, ```julia - sem_fit( + fit( model; start_val = start_simple, start_covariances_latent = 0.5 diff --git a/docs/src/tutorials/collection/multigroup.md b/docs/src/tutorials/collection/multigroup.md index 23c13b950..1007f4563 100644 --- a/docs/src/tutorials/collection/multigroup.md +++ b/docs/src/tutorials/collection/multigroup.md @@ -81,7 +81,7 @@ model_ml_multigroup = SemEnsemble( We now fit the model and inspect the parameter estimates: ```@example mg; ansicolor = true -fit = sem_fit(model_ml_multigroup) +fit = fit(model_ml_multigroup) update_estimate!(partable, fit) details(partable) ``` diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index cdd9111a2..338803cb3 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -48,7 +48,7 @@ model = Sem( data = data ) -model_fit = sem_fit(model) +model_fit = fit(model) update_estimate!(partable, model_fit) @@ -153,7 +153,7 @@ model_constrained = Sem( data = data ) -model_fit_constrained = sem_fit(constrained_optimizer, model_constrained) +model_fit_constrained = fit(constrained_optimizer, model_constrained) ``` As you can see, the optimizer converged (`:XTOL_REACHED`) and investigating the solution yields @@ -162,7 +162,7 @@ As you can see, the optimizer converged (`:XTOL_REACHED`) and investigating the update_partable!( partable, :estimate_constr, - params(model_fit_constrained), + param_labels(model_fit_constrained), solution(model_fit_constrained), ) diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 606a6576e..45d2a2ea1 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -65,5 +65,5 @@ optimizer = SemOptimizerOptim() model_ml = Sem(observed, implied_ram, loss_ml) -sem_fit(optimizer, model_ml) +fit(optimizer, model_ml) ``` \ No newline at end of file diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md index 6a3cd2cef..a1c0b8ad3 100644 --- a/docs/src/tutorials/construction/outer_constructor.md +++ b/docs/src/tutorials/construction/outer_constructor.md @@ -131,4 +131,4 @@ model = SemFiniteDiff( ) ``` -constructs a model that will use finite difference approximation if you estimate the parameters via `sem_fit(model)`. \ No newline at end of file +constructs a model that will use finite difference approximation if you estimate the parameters via `fit(model)`. \ No newline at end of file diff --git a/docs/src/tutorials/first_model.md b/docs/src/tutorials/first_model.md index 5b7284649..e8048966c 100644 --- a/docs/src/tutorials/first_model.md +++ b/docs/src/tutorials/first_model.md @@ -110,7 +110,7 @@ model = Sem( We can now fit the model via ```@example high_level; ansicolor = true -model_fit = sem_fit(model) +model_fit = fit(model) ``` and compute fit measures as diff --git a/docs/src/tutorials/fitting/fitting.md b/docs/src/tutorials/fitting/fitting.md index a3e4b9b91..fff06abaa 100644 --- a/docs/src/tutorials/fitting/fitting.md +++ b/docs/src/tutorials/fitting/fitting.md @@ -3,7 +3,7 @@ As we saw in [A first model](@ref), after you have build a model, you can fit it via ```julia -model_fit = sem_fit(model) +model_fit = fit(model) # output @@ -45,24 +45,24 @@ Structural Equation Model ## Choosing an optimizer -To choose a different optimizer, you can call `sem_fit` with the keyword argument `engine = ...`, and pass additional keyword arguments: +To choose a different optimizer, you can call `fit` with the keyword argument `engine = ...`, and pass additional keyword arguments: ```julia using Optim -model_fit = sem_fit(model; engine = :Optim, algorithm = BFGS()) +model_fit = fit(model; engine = :Optim, algorithm = BFGS()) ``` Available options for engine are `:Optim`, `:NLopt` and `:Proximal`, where `:NLopt` and `:Proximal` are only available if the `NLopt.jl` and `ProximalAlgorithms.jl` packages are loaded respectively. The available keyword arguments are listed in the sections [Using Optim.jl](@ref), [Using NLopt.jl](@ref) and [Regularization](@ref). -Alternative, you can also explicitely define a `SemOptimizer` and pass it as the first argument to `sem_fit`: +Alternative, you can also explicitely define a `SemOptimizer` and pass it as the first argument to `fit`: ```julia my_optimizer = SemOptimizerOptim(algorithm = BFGS()) -sem_fit(my_optimizer, model) +fit(my_optimizer, model) ``` You may also optionally specify [Starting values](@ref). @@ -70,5 +70,5 @@ You may also optionally specify [Starting values](@ref). # API - model fitting ```@docs -sem_fit +fit ``` \ No newline at end of file diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md index 2b6d3191f..abd416c1c 100644 --- a/docs/src/tutorials/inspection/inspection.md +++ b/docs/src/tutorials/inspection/inspection.md @@ -42,13 +42,13 @@ model = Sem( data = data ) -model_fit = sem_fit(model) +model_fit = fit(model) ``` After you fitted a model, ```julia -model_fit = sem_fit(model) +model_fit = fit(model) ``` you end up with an object of type [`SemFit`](@ref). @@ -87,8 +87,8 @@ We can also update the `ParameterTable` object with other information via [`upda se_bs = se_bootstrap(model_fit; n_boot = 20) se_he = se_hessian(model_fit) -update_partable!(partable, :se_hessian, params(model_fit), se_he) -update_partable!(partable, :se_bootstrap, params(model_fit), se_bs) +update_partable!(partable, :se_hessian, param_labels(model_fit), se_he) +update_partable!(partable, :se_bootstrap, param_labels(model_fit), se_bs) details(partable) ``` @@ -126,11 +126,11 @@ fit_measures AIC BIC χ² -df +dof minus2ll nobserved_vars nsamples -params +param_labels nparams p_value RMSEA diff --git a/docs/src/tutorials/meanstructure.md b/docs/src/tutorials/meanstructure.md index 60578224a..b2da5029a 100644 --- a/docs/src/tutorials/meanstructure.md +++ b/docs/src/tutorials/meanstructure.md @@ -96,7 +96,7 @@ model = Sem( meanstructure = true ) -sem_fit(model) +fit(model) ``` If we build the model by parts, we have to pass the `meanstructure = true` argument to every part that requires it (when in doubt, simply consult the documentation for the respective part). @@ -112,5 +112,5 @@ ml = SemML(observed = observed, meanstructure = true) model = Sem(observed, implied_ram, SemLoss(ml)) -sem_fit(model) +fit(model) ``` \ No newline at end of file diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index 37e42975a..3d82fcfba 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -120,25 +120,25 @@ Let's fit the regularized model ```@example reg -fit_lasso = sem_fit(optimizer_lasso, model_lasso) +fit_lasso = fit(optimizer_lasso, model_lasso) ``` and compare the solution to unregularizted estimates: ```@example reg -fit = sem_fit(model) +fit = fit(model) update_estimate!(partable, fit) -update_partable!(partable, :estimate_lasso, params(fit_lasso), solution(fit_lasso)) +update_partable!(partable, :estimate_lasso, param_labels(fit_lasso), solution(fit_lasso)) details(partable) ``` -Instead of explicitely defining a `SemOptimizerProximal` object, you can also pass `engine = :Proximal` and additional keyword arguments to `sem_fit`: +Instead of explicitely defining a `SemOptimizerProximal` object, you can also pass `engine = :Proximal` and additional keyword arguments to `fit`: ```@example reg -fit = sem_fit(model; engine = :Proximal, operator_g = NormL1(λ)) +fit = fit(model; engine = :Proximal, operator_g = NormL1(λ)) ``` ## Second example - mixed l1 and l0 regularization @@ -162,13 +162,13 @@ model_mixed = Sem( data = data, ) -fit_mixed = sem_fit(model_mixed; engine = :Proximal, operator_g = prox_operator) +fit_mixed = fit(model_mixed; engine = :Proximal, operator_g = prox_operator) ``` Let's again compare the different results: ```@example reg -update_partable!(partable, :estimate_mixed, params(fit_mixed), solution(fit_mixed)) +update_partable!(partable, :estimate_mixed, param_labels(fit_mixed), solution(fit_mixed)) details(partable) ``` \ No newline at end of file diff --git a/docs/src/tutorials/specification/ram_matrices.md b/docs/src/tutorials/specification/ram_matrices.md index 6e01eb38b..abe76ea6f 100644 --- a/docs/src/tutorials/specification/ram_matrices.md +++ b/docs/src/tutorials/specification/ram_matrices.md @@ -59,7 +59,7 @@ spec = RAMMatrices(; A = A, S = S, F = F, - params = θ, + param_labels = θ, vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) @@ -90,7 +90,7 @@ spec = RAMMatrices(; A = A, S = S, F = F, - params = θ, + param_labels = θ, vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) ``` diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index a614c501b..c5e0ad6cb 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -71,8 +71,8 @@ function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, opt) ) end -# sem_fit method -function SEM.sem_fit( +# fit method +function SEM.fit( optim::SemOptimizerNLopt, model::AbstractSem, start_params::AbstractVector; diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index 2f1775e85..0d4748e3a 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -40,7 +40,7 @@ mutable struct ProximalResult result::Any end -function SEM.sem_fit( +function SEM.fit( optim::SemOptimizerProximal, model::AbstractSem, start_params::AbstractVector; diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 6e1a934f3..f6068dc50 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -4,6 +4,7 @@ using LinearAlgebra, Optim, NLSolversBase, Statistics, + StatsAPI, StatsBase, SparseArrays, Symbolics, @@ -15,6 +16,8 @@ using LinearAlgebra, DelimitedFiles, DataFrames +import StatsAPI: params, coef, coefnames, dof, fit, nobs, coeftable + export StenoGraphs, @StenoGraph, meld const SEM = StructuralEquationModels @@ -37,6 +40,7 @@ include("frontend/specification/RAMMatrices.jl") include("frontend/specification/EnsembleParameterTable.jl") include("frontend/specification/StenoGraphs.jl") include("frontend/fit/summary.jl") +include("frontend/StatsAPI.jl") # pretty printing include("frontend/pretty_printing.jl") # observed @@ -74,7 +78,7 @@ include("additional_functions/simulation.jl") include("frontend/fit/fitmeasures/AIC.jl") include("frontend/fit/fitmeasures/BIC.jl") include("frontend/fit/fitmeasures/chi2.jl") -include("frontend/fit/fitmeasures/df.jl") +include("frontend/fit/fitmeasures/dof.jl") include("frontend/fit/fitmeasures/minus2ll.jl") include("frontend/fit/fitmeasures/p.jl") include("frontend/fit/fitmeasures/RMSEA.jl") @@ -89,6 +93,9 @@ include("package_extensions/SEMProximalOptExt.jl") export AbstractSem, AbstractSemSingle, AbstractSemCollection, + coef, + coefnames, + coeftable, Sem, SemFiniteDiff, SemEnsemble, @@ -129,8 +136,9 @@ export AbstractSem, obs_cov, obs_mean, nsamples, + nobs, samples, - sem_fit, + fit, SemFit, minimum, solution, @@ -165,13 +173,15 @@ export AbstractSem, sort_vars!, sort_vars, params, + params!, nparams, param_indices, + param_labels, fit_measures, AIC, BIC, χ², - df, + dof, fit_measures, minus2ll, p_value, diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index 3a58171aa..1031e349e 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -102,17 +102,17 @@ param_occurences(arr::ParamsArray, i::Integer) = """ materialize!(dest::AbstractArray{<:Any, N}, src::ParamsArray{<:Any, N}, - param_values::AbstractVector; + params::AbstractVector; set_constants::Bool = true, set_zeros::Bool = false) Materialize the parameterized array `src` into `dest` by substituting the parameter -references with the parameter values from `param_values`. +references with the parameter values from `params`. """ function materialize!( dest::AbstractArray{<:Any, N}, src::ParamsArray{<:Any, N}, - param_values::AbstractVector; + params::AbstractVector; set_constants::Bool = true, set_zeros::Bool = false, ) where {N} @@ -121,9 +121,9 @@ function materialize!( "Parameters ($(size(params_arr))) and destination ($(size(dest))) array sizes don't match", ), ) - nparams(src) == length(param_values) || throw( + nparams(src) == length(params) || throw( DimensionMismatch( - "Number of values ($(length(param_values))) does not match the number of parameters ($(nparams(src)))", + "Number of values ($(length(params))) does not match the number of parameters ($(nparams(src)))", ), ) Z = eltype(dest) <: Number ? eltype(dest) : eltype(src) @@ -133,7 +133,7 @@ function materialize!( dest[i] = val end end - @inbounds for (i, val) in enumerate(param_values) + @inbounds for (i, val) in enumerate(params) for j in param_occurences_range(src, i) dest[src.linear_indices[j]] = val end @@ -144,7 +144,7 @@ end function materialize!( dest::SparseMatrixCSC, src::ParamsMatrix, - param_values::AbstractVector; + params::AbstractVector; set_constants::Bool = true, set_zeros::Bool = false, ) @@ -154,9 +154,9 @@ function materialize!( "Parameters ($(size(params_arr))) and destination ($(size(dest))) array sizes don't match", ), ) - nparams(src) == length(param_values) || throw( + nparams(src) == length(params) || throw( DimensionMismatch( - "Number of values ($(length(param_values))) does not match the number of parameters ($(nparams(src)))", + "Number of values ($(length(params))) does not match the number of parameters ($(nparams(src)))", ), ) @@ -170,7 +170,7 @@ function materialize!( dest.nzval[j] = val end end - @inbounds for (i, val) in enumerate(param_values) + @inbounds for (i, val) in enumerate(params) for j in param_occurences_range(src, i) dest.nzval[src.nz_indices[j]] = val end @@ -180,33 +180,33 @@ end """ materialize([T], src::ParamsArray{<:Any, N}, - param_values::AbstractVector{T}) where T + params::AbstractVector{T}) where T Materialize the parameterized array `src` into a new array of type `T` -by substituting the parameter references with the parameter values from `param_values`. +by substituting the parameter references with the parameter values from `params`. """ -materialize(::Type{T}, arr::ParamsArray, param_values::AbstractVector) where {T} = - materialize!(similar(arr, T), arr, param_values, set_constants = true, set_zeros = true) +materialize(::Type{T}, arr::ParamsArray, params::AbstractVector) where {T} = + materialize!(similar(arr, T), arr, params, set_constants = true, set_zeros = true) -materialize(arr::ParamsArray, param_values::AbstractVector{T}) where {T} = - materialize(Union{T, eltype(arr)}, arr, param_values) +materialize(arr::ParamsArray, params::AbstractVector{T}) where {T} = + materialize(Union{T, eltype(arr)}, arr, params) # the hack to update the structured matrix (should be fine since the structure is imposed by ParamsMatrix) materialize!( dest::Union{Symmetric, LowerTriangular, UpperTriangular}, src::ParamsMatrix{<:Any}, - param_values::AbstractVector; + params::AbstractVector; kwargs..., -) = materialize!(parent(dest), src, param_values; kwargs...) +) = materialize!(parent(dest), src, params; kwargs...) function sparse_materialize( ::Type{T}, arr::ParamsMatrix, - param_values::AbstractVector, + params::AbstractVector, ) where {T} - nparams(arr) == length(param_values) || throw( + nparams(arr) == length(params) || throw( DimensionMismatch( - "Number of values ($(length(param_values))) does not match the number of parameter ($(nparams(arr)))", + "Number of values ($(length(params))) does not match the number of parameter ($(nparams(arr)))", ), ) @@ -218,7 +218,7 @@ function sparse_materialize( nz_lininds[nz_ind] = lin_ind end # fill parameters - @inbounds for (i, val) in enumerate(param_values) + @inbounds for (i, val) in enumerate(params) for j in param_occurences_range(arr, i) nz_ind = arr.nz_indices[j] nz_vals[nz_ind] = val diff --git a/src/frontend/StatsAPI.jl b/src/frontend/StatsAPI.jl new file mode 100644 index 000000000..edd677e34 --- /dev/null +++ b/src/frontend/StatsAPI.jl @@ -0,0 +1,78 @@ +""" + params!(out::AbstractVector, partable::ParameterTable, + col::Symbol = :estimate) + +Extract parameter values from the `col` column of `partable` +into the `out` vector. + +The `out` vector should be of `nparams(partable)` length. +The *i*-th element of the `out` vector will contain the +value of the *i*-th parameter from `params_labels(partable)`. + +Note that the function combines the duplicate occurences of the +same parameter in `partable` and will raise an error if the +values do not match. +""" +function params!( + out::AbstractVector, + partable::ParameterTable, + col::Symbol = :estimate, +) + (length(out) == nparams(partable)) || throw( + DimensionMismatch( + "The length of parameter values vector ($(length(out))) does not match the number of parameters ($(nparams(partable)))", + ), + ) + param_index = param_indices(partable) + params_col = partable.columns[col] + for (i, label) in enumerate(partable.columns[:label]) + (label == :const) && continue + param_ind = get(param_index, label, nothing) + @assert !isnothing(param_ind) "Parameter table contains unregistered parameter :$param at row #$i" + param = params_col[i] + if !isnan(out[param_ind]) + @assert isequal(out[param_ind], param) "Parameter :$label value at row #$i ($param) differs from the earlier encountered value ($(out[param_ind]))" + else + out[param_ind] = param + end + end + return out +end + +""" + params(x::ParameterTable, col::Symbol = :estimate) + +Extract parameter values from the `col` column of `partable`. + +Returns the values vector. The *i*-th element corresponds to +the value of *i*-th parameter from `params_label(partable)`. + +Note that the function combines the duplicate occurences of the +same parameter in `partable` and will raise an error if the +values do not match. +""" +params(partable::ParameterTable, col::Symbol = :estimate) = + params!(fill(NaN, nparams(partable)), partable, col) + +""" + coef(x::ParameterTable) + +For a `ParameterTable`, this function is synonymous to [`params`](@ref). +""" +coef(x::ParameterTable) = params(x) + +""" + coefnames(x::ParameterTable) + +Synonymous to [`param_labels`](@ref param_labels). +""" +coefnames(x::ParameterTable) = param_labels(x) + +""" + nobs(model::AbstractSem) -> Int + +Synonymous to [`nsamples`](@ref). +""" +nobs(model::AbstractSem) = nsamples(model) + +coeftable(model::AbstractSem; level::Real=0.95) = throw(ArgumentError("StructuralEquationModels does not support the `CoefTable` interface; see [`ParameterTable`](@ref) instead.")) \ No newline at end of file diff --git a/src/frontend/common.jl b/src/frontend/common.jl index 41d03effb..e89a6cf8b 100644 --- a/src/frontend/common.jl +++ b/src/frontend/common.jl @@ -14,7 +14,7 @@ Return the number of parameters in a SEM model associated with `semobj`. See also [`params`](@ref). """ -nparams(semobj) = length(params(semobj)) +nparams(semobj) = length(param_labels(semobj)) """ nvars(semobj) @@ -52,7 +52,7 @@ parind[:param_name] See also [`params`](@ref). """ -param_indices(semobj) = Dict(par => i for (i, par) in enumerate(params(semobj))) +param_indices(semobj) = Dict(par => i for (i, par) in enumerate(param_labels(semobj))) """ nsamples(semobj) diff --git a/src/frontend/fit/SemFit.jl b/src/frontend/fit/SemFit.jl index 84d2f502c..438da4da6 100644 --- a/src/frontend/fit/SemFit.jl +++ b/src/frontend/fit/SemFit.jl @@ -46,7 +46,7 @@ end # additional methods ############################################################################################ -params(fit::SemFit) = params(fit.model) +param_labels(fit::SemFit) = param_labels(fit.model) nparams(fit::SemFit) = nparams(fit.model) nsamples(fit::SemFit) = nsamples(fit.model) diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index b91e81d3e..b9fff648e 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -6,13 +6,13 @@ Return the RMSEA. function RMSEA end RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) = - RMSEA(df(sem_fit), χ²(sem_fit), nsamples(sem_fit)) + RMSEA(dof(sem_fit), χ²(sem_fit), nsamples(sem_fit)) RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}) = - sqrt(length(sem_fit.model.sems)) * RMSEA(df(sem_fit), χ²(sem_fit), nsamples(sem_fit)) + sqrt(length(sem_fit.model.sems)) * RMSEA(dof(sem_fit), χ²(sem_fit), nsamples(sem_fit)) -function RMSEA(df, chi2, nsamples) - rmsea = (chi2 - df) / (nsamples * df) +function RMSEA(dof, chi2, nsamples) + rmsea = (chi2 - dof) / (nsamples * dof) rmsea > 0 ? nothing : rmsea = 0 return sqrt(rmsea) end diff --git a/src/frontend/fit/fitmeasures/df.jl b/src/frontend/fit/fitmeasures/dof.jl similarity index 62% rename from src/frontend/fit/fitmeasures/df.jl rename to src/frontend/fit/fitmeasures/dof.jl index 4d9025601..3df49d89d 100644 --- a/src/frontend/fit/fitmeasures/df.jl +++ b/src/frontend/fit/fitmeasures/dof.jl @@ -1,14 +1,14 @@ """ - df(sem_fit::SemFit) - df(model::AbstractSem) + dof(sem_fit::SemFit) + dof(model::AbstractSem) Return the degrees of freedom. """ -function df end +function dof end -df(sem_fit::SemFit) = df(sem_fit.model) +dof(sem_fit::SemFit) = dof(sem_fit.model) -df(model::AbstractSem) = n_dp(model) - nparams(model) +dof(model::AbstractSem) = n_dp(model) - nparams(model) function n_dp(model::AbstractSemSingle) nvars = nobserved_vars(model) diff --git a/src/frontend/fit/fitmeasures/fit_measures.jl b/src/frontend/fit/fitmeasures/fit_measures.jl index 40e3caae0..2fc4dfba0 100644 --- a/src/frontend/fit/fitmeasures/fit_measures.jl +++ b/src/frontend/fit/fitmeasures/fit_measures.jl @@ -1,5 +1,5 @@ fit_measures(sem_fit) = - fit_measures(sem_fit, nparams, df, AIC, BIC, RMSEA, χ², p_value, minus2ll) + fit_measures(sem_fit, nparams, dof, AIC, BIC, RMSEA, χ², p_value, minus2ll) function fit_measures(sem_fit, args...) measures = Dict{Symbol, Union{Float64, Missing}}() diff --git a/src/frontend/fit/fitmeasures/p.jl b/src/frontend/fit/fitmeasures/p.jl index 3d4275f95..8c69d5ec2 100644 --- a/src/frontend/fit/fitmeasures/p.jl +++ b/src/frontend/fit/fitmeasures/p.jl @@ -3,4 +3,4 @@ Return the p value computed from the χ² test statistic. """ -p_value(sem_fit::SemFit) = 1 - cdf(Chisq(df(sem_fit)), χ²(sem_fit)) +p_value(sem_fit::SemFit) = 1 - cdf(Chisq(dof(sem_fit)), χ²(sem_fit)) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index e8d840d0c..4589dc020 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -1,5 +1,5 @@ """ - se_bootstrap(semfit::SemFit; n_boot = 3000, data = nothing, kwargs...) + se_bootstrap(sem_fit::SemFit; n_boot = 3000, data = nothing, kwargs...) Return boorstrap standard errors. Only works for single models. @@ -52,7 +52,7 @@ function se_bootstrap( new_solution .= 0.0 try - new_solution = solution(sem_fit(new_model; start_val = start)) + new_solution = solution(fit(new_model; start_val = start)) catch n_failed += 1 end diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index d5ac7e51b..14169dd94 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -4,7 +4,7 @@ struct EnsembleParameterTable <: AbstractParameterTable tables::Dict{Symbol, ParameterTable} - params::Vector{Symbol} + param_labels::Vector{Symbol} end ############################################################################################ @@ -12,35 +12,35 @@ end ############################################################################################ # constuct an empty table -EnsembleParameterTable(::Nothing; params::Union{Nothing, Vector{Symbol}} = nothing) = +EnsembleParameterTable(::Nothing; param_labels::Union{Nothing, Vector{Symbol}} = nothing) = EnsembleParameterTable( Dict{Symbol, ParameterTable}(), - isnothing(params) ? Symbol[] : copy(params), + isnothing(param_labels) ? Symbol[] : copy(param_labels), ) # convert pairs to dict -EnsembleParameterTable(ps::Pair{K, V}...; params = nothing) where {K, V} = - EnsembleParameterTable(Dict(ps...); params = params) +EnsembleParameterTable(ps::Pair{K, V}...; param_labels = nothing) where {K, V} = + EnsembleParameterTable(Dict(ps...); param_labels = param_labels) # dictionary of SEM specifications function EnsembleParameterTable( spec_ensemble::AbstractDict{K, V}; - params::Union{Nothing, Vector{Symbol}} = nothing, + param_labels::Union{Nothing, Vector{Symbol}} = nothing, ) where {K, V <: SemSpecification} - params = if isnothing(params) + param_labels = if isnothing(param_labels) # collect all SEM parameters in ensemble if not specified # and apply the set to all partables - unique(mapreduce(SEM.params, vcat, values(spec_ensemble), init = Vector{Symbol}())) + unique(mapreduce(SEM.param_labels, vcat, values(spec_ensemble), init = Vector{Symbol}())) else - copy(params) + copy(param_labels) end # convert each model specification to ParameterTable partables = Dict{Symbol, ParameterTable}( - Symbol(group) => convert(ParameterTable, spec; params) for + Symbol(group) => convert(ParameterTable, spec; param_labels) for (group, spec) in pairs(spec_ensemble) ) - return EnsembleParameterTable(partables, params) + return EnsembleParameterTable(partables, param_labels) end ############################################################################################ @@ -54,12 +54,12 @@ end function Base.convert( ::Type{Dict{K, RAMMatrices}}, partables::EnsembleParameterTable; - params::Union{AbstractVector{Symbol}, Nothing} = nothing, + param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing, ) where {K} - isnothing(params) || (params = SEM.params(partables)) + isnothing(param_labels) || (param_labels = SEM.param_labels(partables)) return Dict{K, RAMMatrices}( - K(key) => RAMMatrices(partable; params = params) for + K(key) => RAMMatrices(partable; param_labels = param_labels) for (key, partable) in pairs(partables.tables) ) end @@ -124,11 +124,11 @@ Base.getindex(partable::EnsembleParameterTable, group) = partable.tables[group] function update_partable!( partables::EnsembleParameterTable, column::Symbol, - param_values::AbstractDict{Symbol}, + params::AbstractDict{Symbol}, default::Any = nothing, ) for partable in values(partables.tables) - update_partable!(partable, column, param_values, default) + update_partable!(partable, column, params, default) end return partables end @@ -136,11 +136,11 @@ end function update_partable!( partables::EnsembleParameterTable, column::Symbol, - params::AbstractVector{Symbol}, + param_labels::AbstractVector{Symbol}, values::AbstractVector, default::Any = nothing, ) - return update_partable!(partables, column, Dict(zip(params, values)), default) + return update_partable!(partables, column, Dict(zip(param_labels, values)), default) end ############################################################################################ @@ -148,6 +148,6 @@ end ############################################################################################ function Base.:(==)(p1::EnsembleParameterTable, p2::EnsembleParameterTable) - out = (p1.tables == p2.tables) && (p1.params == p2.params) + out = (p1.tables == p2.tables) && (p1.param_labels == p2.param_labels) return out end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 74c963ccb..2af269372 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -7,7 +7,7 @@ struct ParameterTable{C} <: AbstractParameterTable observed_vars::Vector{Symbol} latent_vars::Vector{Symbol} sorted_vars::Vector{Symbol} - params::Vector{Symbol} + param_labels::Vector{Symbol} end ############################################################################################ @@ -24,7 +24,7 @@ empty_partable_columns(nrows::Integer = 0) = Dict{Symbol, Vector}( :value_fixed => fill(NaN, nrows), :start => fill(NaN, nrows), :estimate => fill(NaN, nrows), - :param => fill(Symbol(), nrows), + :label => fill(Symbol(), nrows), ) # construct using the provided columns data or create an empty table @@ -32,31 +32,31 @@ function ParameterTable( columns::Dict{Symbol, Vector}; observed_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, latent_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, - params::Union{AbstractVector{Symbol}, Nothing} = nothing, + param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing, ) - params = isnothing(params) ? unique!(filter(!=(:const), columns[:param])) : copy(params) - check_params(params, columns[:param]) + param_labels = isnothing(param_labels) ? unique!(filter(!=(:const), columns[:label])) : copy(param_labels) + check_param_labels(param_labels, columns[:label]) return ParameterTable( columns, !isnothing(observed_vars) ? copy(observed_vars) : Vector{Symbol}(), !isnothing(latent_vars) ? copy(latent_vars) : Vector{Symbol}(), Vector{Symbol}(), - params, + param_labels, ) end # new parameter table with different parameters order function ParameterTable( partable::ParameterTable; - params::Union{AbstractVector{Symbol}, Nothing} = nothing, + param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing, ) - isnothing(params) || check_params(params, partable.columns[:param]) + isnothing(param_labels) || check_param_labels(param_labels, partable.columns[:label]) return ParameterTable( Dict(col => copy(values) for (col, values) in pairs(partable.columns)), observed_vars = copy(observed_vars(partable)), latent_vars = copy(latent_vars(partable)), - params = params, + param_labels = param_labels, ) end @@ -80,10 +80,10 @@ end function Base.convert( ::Type{ParameterTable}, partable::ParameterTable; - params::Union{AbstractVector{Symbol}, Nothing} = nothing, + param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing, ) - return isnothing(params) || partable.params == params ? partable : - ParameterTable(partable; params) + return isnothing(param_labels) || partable.param_labels == param_labels ? partable : + ParameterTable(partable; param_labels) end function DataFrames.DataFrame( @@ -102,7 +102,7 @@ end function Base.show(io::IO, partable::ParameterTable) relevant_columns = - [:from, :relation, :to, :free, :value_fixed, :start, :estimate, :se, :param] + [:from, :relation, :to, :free, :value_fixed, :start, :estimate, :se, :label] shown_columns = filter!( col -> haskey(partable.columns, col) && length(partable.columns[col]) > 0, relevant_columns, @@ -133,7 +133,7 @@ function Base.:(==)(p1::ParameterTable, p2::ParameterTable) (p1.observed_vars == p2.observed_vars) && (p1.latent_vars == p2.latent_vars) && (p1.sorted_vars == p2.sorted_vars) && - (p1.params == p2.params) + (p1.param_labels == p2.param_labels) return out end @@ -153,18 +153,17 @@ Base.getindex(partable::ParameterTable, i::Integer) = ( to = partable.columns[:to][i], free = partable.columns[:free][i], value_fixed = partable.columns[:value_fixed][i], - param = partable.columns[:param][i], + param = partable.columns[:label][i], ) -Base.length(partable::ParameterTable) = length(partable.columns[:param]) +Base.length(partable::ParameterTable) = length(partable.columns[:label]) Base.eachindex(partable::ParameterTable) = Base.OneTo(length(partable)) Base.eltype(::Type{<:ParameterTable}) = ParameterTableRow Base.iterate(partable::ParameterTable, i::Integer = 1) = i > length(partable) ? nothing : (partable[i], i + 1) -params(partable::ParameterTable) = partable.params -nparams(partable::ParameterTable) = length(params(partable)) +nparams(partable::ParameterTable) = length(param_labels(partable)) # Sorting ---------------------------------------------------------------------------------- @@ -264,18 +263,18 @@ end function update_partable!( partable::ParameterTable, column::Symbol, - param_values::AbstractDict{Symbol, T}, + params::AbstractDict{Symbol, T}, default::Any = nothing, ) where {T} coldata = get!(() -> Vector{T}(undef, length(partable)), partable.columns, column) isvec_def = (default isa AbstractVector) && (length(default) == length(partable)) - for (i, par) in enumerate(partable.columns[:param]) + for (i, par) in enumerate(partable.columns[:label]) if par == :const coldata[i] = !isnothing(default) ? (isvec_def ? default[i] : default) : zero(T) - elseif haskey(param_values, par) - coldata[i] = param_values[par] + elseif haskey(params, par) + coldata[i] = params[par] else if isnothing(default) throw(KeyError(par)) @@ -289,31 +288,29 @@ function update_partable!( end """ - update_partable!(partable::AbstractParameterTable, params::Vector{Symbol}, values, column) + update_partable!(partable::AbstractParameterTable, param_labels::Vector{Symbol}, params, column) Write parameter `values` into `column` of `partable`. -The `params` and `values` vectors define the pairs of value +The `param_labels` and `params` vectors define the pairs of parameters, which are being matched to the `:param` column of the `partable`. """ function update_partable!( partable::ParameterTable, column::Symbol, - params::AbstractVector{Symbol}, - values::AbstractVector, + param_labels::AbstractVector{Symbol}, + params::AbstractVector, default::Any = nothing, ) - length(params) == length(values) || throw( + length(param_labels) == length(params) || throw( ArgumentError( - "The length of `params` ($(length(params))) and their `values` ($(length(values))) must be the same", + "The length of `param_labels` ($(length(param_labels))) and their `params` ($(length(param_labels))) must be the same", ), ) - dup_params = nonunique(params) - isempty(dup_params) || - throw(ArgumentError("Duplicate parameters detected: $(join(dup_params, ", "))")) - param_values = Dict(zip(params, values)) - update_partable!(partable, column, param_values, default) + check_param_labels(param_labels, nothing) + params = Dict(zip(param_labels, params)) + update_partable!(partable, column, params, default) end # update estimates ------------------------------------------------------------------------- @@ -327,14 +324,14 @@ Write parameter estimates from `fit` to the `:estimate` column of `partable` update_estimate!(partable::ParameterTable, fit::SemFit) = update_partable!( partable, :estimate, - params(fit), + param_labels(fit), fit.solution, partable.columns[:value_fixed], ) # fallback method for ensemble update_estimate!(partable::AbstractParameterTable, fit::SemFit) = - update_partable!(partable, :estimate, params(fit), fit.solution) + update_partable!(partable, :estimate, param_labels(fit), fit.solution) # update starting values ------------------------------------------------------------------- """ @@ -351,7 +348,7 @@ Write starting values from `fit` or `start_val` to the `:start` column of `parta update_start!(partable::AbstractParameterTable, fit::SemFit) = update_partable!( partable, :start, - params(fit), + param_labels(fit), fit.start_val, partable.columns[:value_fixed], ) @@ -365,7 +362,7 @@ function update_start!( if !(start_val isa Vector) start_val = start_val(model; kwargs...) end - return update_partable!(partable, :start, params(model), start_val) + return update_partable!(partable, :start, param_labels(model), start_val) end # update partable standard errors ---------------------------------------------------------- @@ -389,67 +386,12 @@ function update_se_hessian!( method = :finitediff, ) se = se_hessian(fit; method) - return update_partable!(partable, :se, params(fit), se) + return update_partable!(partable, :se, param_labels(fit), se) end -""" - param_values!(out::AbstractVector, partable::ParameterTable, - col::Symbol = :estimate) - -Extract parameter values from the `col` column of `partable` -into the `out` vector. - -The `out` vector should be of `nparams(partable)` length. -The *i*-th element of the `out` vector will contain the -value of the *i*-th parameter from `params(partable)`. - -Note that the function combines the duplicate occurences of the -same parameter in `partable` and will raise an error if the -values do not match. -""" -function param_values!( - out::AbstractVector, - partable::ParameterTable, - col::Symbol = :estimate, -) - (length(out) == nparams(partable)) || throw( - DimensionMismatch( - "The length of parameter values vector ($(length(out))) does not match the number of parameters ($(nparams(partable)))", - ), - ) - param_index = Dict(param => i for (i, param) in enumerate(params(partable))) - param_values_col = partable.columns[col] - for (i, param) in enumerate(partable.columns[:param]) - (param == :const) && continue - param_ind = get(param_index, param, nothing) - @assert !isnothing(param_ind) "Parameter table contains unregistered parameter :$param at row #$i" - val = param_values_col[i] - if !isnan(out[param_ind]) - @assert isequal(out[param_ind], val) "Parameter :$param value at row #$i ($val) differs from the earlier encountered value ($(out[param_ind]))" - else - out[param_ind] = val - end - end - return out -end - -""" - param_values(out::AbstractVector, col::Symbol = :estimate) - -Extract parameter values from the `col` column of `partable`. - -Returns the values vector. The *i*-th element corresponds to -the value of *i*-th parameter from `params(partable)`. - -Note that the function combines the duplicate occurences of the -same parameter in `partable` and will raise an error if the -values do not match. -""" -param_values(partable::ParameterTable, col::Symbol = :estimate) = - param_values!(fill(NaN, nparams(partable)), partable, col) """ - lavaan_param_values!(out::AbstractVector, partable_lav, + lavaan_params!(out::AbstractVector, partable_lav, partable::ParameterTable, lav_col::Symbol = :est, lav_group = nothing) @@ -457,14 +399,14 @@ Extract parameter values from the `partable_lav` lavaan model that match the parameters of `partable` into the `out` vector. The method sets the *i*-th element of the `out` vector to -the value of *i*-th parameter from `params(partable)`. +the value of *i*-th parameter from `param_labels(partable)`. Note that the lavaan and `partable` models are matched by the the names of variables in the tables (`from` and `to` columns) as well as the type of their relationship (`relation` column), and not by the names of the model parameters. """ -function lavaan_param_values!( +function lavaan_params!( out::AbstractVector, partable_lav, partable::ParameterTable, @@ -481,13 +423,13 @@ function lavaan_param_values!( ), ) partable_mask = findall(partable.columns[:free]) - param_index = Dict(param => i for (i, param) in enumerate(params(partable))) + param_index = param_indices(partable) lav_values = partable_lav[:, lav_col] for (from, to, type, id) in zip( [ view(partable.columns[k], partable_mask) for - k in [:from, :to, :relation, :param] + k in [:from, :to, :relation, :label] ]..., ) lav_ind = nothing @@ -562,7 +504,7 @@ function lavaan_param_values!( end """ - lavaan_param_values(partable_lav, partable::ParameterTable, + lavaan_params(partable_lav, partable::ParameterTable, lav_col::Symbol = :est, lav_group = nothing) Extract parameter values from the `partable_lav` lavaan model that @@ -570,19 +512,19 @@ match the parameters of `partable`. The `out` vector should be of `nparams(partable)` length. The *i*-th element of the `out` vector will contain the -value of the *i*-th parameter from `params(partable)`. +value of the *i*-th parameter from `param_labels(partable)`. Note that the lavaan and `partable` models are matched by the the names of variables in the tables (`from` and `to` columns), and the type of their relationship (`relation` column), but not by the ids of the model parameters. """ -lavaan_param_values( +lavaan_params( partable_lav, partable::ParameterTable, lav_col::Symbol = :est, lav_group = nothing, -) = lavaan_param_values!( +) = lavaan_params!( fill(NaN, nparams(partable)), partable_lav, partable, diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 4ebea95fb..75175a87d 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -8,7 +8,7 @@ struct RAMMatrices <: SemSpecification S::ParamsMatrix{Float64} F::SparseMatrixCSC{Float64} M::Union{ParamsVector{Float64}, Nothing} - params::Vector{Symbol} + param_labels::Vector{Symbol} vars::Union{Vector{Symbol}, Nothing} # better call it "variables": it's a mixture of observed and latent (and it gets confusing with get_vars()) end @@ -71,7 +71,7 @@ function RAMMatrices(; S::AbstractMatrix, F::AbstractMatrix, M::Union{AbstractVector, Nothing} = nothing, - params::AbstractVector{Symbol}, + param_labels::AbstractVector{Symbol}, vars::Union{AbstractVector{Symbol}, Nothing} = nothing, ) ncols = size(A, 2) @@ -101,16 +101,16 @@ function RAMMatrices(; ), ) end - check_params(params, nothing) + check_param_labels(param_labels, nothing) - A = ParamsMatrix{Float64}(A, params) - S = ParamsMatrix{Float64}(S, params) - M = !isnothing(M) ? ParamsVector{Float64}(M, params) : nothing + A = ParamsMatrix{Float64}(A, param_labels) + S = ParamsMatrix{Float64}(S, param_labels) + M = !isnothing(M) ? ParamsVector{Float64}(M, param_labels) : nothing spF = sparse(F) if any(!isone, spF.nzval) throw(ArgumentError("F should contain only 0s and 1s")) end - return RAMMatrices(A, S, F, M, copy(params), vars) + return RAMMatrices(A, S, F, M, copy(param_labels), vars) end ############################################################################################ @@ -119,11 +119,11 @@ end function RAMMatrices( partable::ParameterTable; - params::Union{AbstractVector{Symbol}, Nothing} = nothing, + param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing, ) - params = copy(isnothing(params) ? SEM.params(partable) : params) - check_params(params, partable.columns[:param]) - params_index = Dict(param => i for (i, param) in enumerate(params)) + param_labels = copy(isnothing(param_labels) ? SEM.param_labels(partable) : param_labels) + check_param_labels(param_labels, partable.columns[:label]) + param_labels_index = param_indices(partable) n_observed = length(partable.observed_vars) n_latent = length(partable.latent_vars) @@ -146,16 +146,16 @@ function RAMMatrices( # known_labels = Dict{Symbol, Int64}() T = nonmissingtype(eltype(partable.columns[:value_fixed])) - A_inds = [Vector{Int64}() for _ in 1:length(params)] + A_inds = [Vector{Int64}() for _ in 1:length(param_labels)] A_lin_ixs = LinearIndices((n_vars, n_vars)) - S_inds = [Vector{Int64}() for _ in 1:length(params)] + S_inds = [Vector{Int64}() for _ in 1:length(param_labels)] S_lin_ixs = LinearIndices((n_vars, n_vars)) A_consts = Vector{Pair{Int, T}}() S_consts = Vector{Pair{Int, T}}() # is there a meanstructure? M_inds = any(==(Symbol(1)), partable.columns[:from]) ? - [Vector{Int64}() for _ in 1:length(params)] : nothing + [Vector{Int64}() for _ in 1:length(param_labels)] : nothing M_consts = !isnothing(M_inds) ? Vector{Pair{Int, T}}() : nothing for r in partable @@ -185,7 +185,7 @@ function RAMMatrices( error("Unsupported relation: $(r.relation)") end else - par_ind = params_index[r.param] + par_ind = param_labels_index[r.param] if (r.relation == :→) && (r.from == Symbol(1)) push!(M_inds[par_ind], row_ind) elseif r.relation == :→ @@ -229,7 +229,7 @@ function RAMMatrices( n_vars, ), !isnothing(M_inds) ? ParamsVector{T}(M_inds, M_consts, (n_vars,)) : nothing, - params, + param_labels, vars_sorted, ) end @@ -237,8 +237,8 @@ end Base.convert( ::Type{RAMMatrices}, partable::ParameterTable; - params::Union{AbstractVector{Symbol}, Nothing} = nothing, -) = RAMMatrices(partable; params) + param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing, +) = RAMMatrices(partable; param_labels) ############################################################################################ ### get parameter table from RAMMatrices @@ -246,7 +246,7 @@ Base.convert( function ParameterTable( ram::RAMMatrices; - params::Union{AbstractVector{Symbol}, Nothing} = nothing, + param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing, observed_var_prefix::Symbol = :obs, latent_var_prefix::Symbol = :var, ) @@ -266,17 +266,17 @@ function ParameterTable( partable = ParameterTable( observed_vars = observed_vars, latent_vars = latent_vars, - params = isnothing(params) ? SEM.params(ram) : params, + param_labels = isnothing(param_labels) ? SEM.param_labels(ram) : param_labels, ) # fill the table - append_rows!(partable, ram.S, :S, ram.params, vars, skip_symmetric = true) - append_rows!(partable, ram.A, :A, ram.params, vars) + append_rows!(partable, ram.S, :S, ram.param_labels, vars, skip_symmetric = true) + append_rows!(partable, ram.A, :A, ram.param_labels, vars) if !isnothing(ram.M) - append_rows!(partable, ram.M, :M, ram.params, vars) + append_rows!(partable, ram.M, :M, ram.param_labels, vars) end - check_params(SEM.params(partable), partable.columns[:param]) + check_param_labels(SEM.param_labels(partable), partable.columns[:label]) return partable end @@ -284,8 +284,8 @@ end Base.convert( ::Type{<:ParameterTable}, ram::RAMMatrices; - params::Union{AbstractVector{Symbol}, Nothing} = nothing, -) = ParameterTable(ram; params) + param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing, +) = ParameterTable(ram; param_labels) ############################################################################################ ### Pretty Printing @@ -343,7 +343,7 @@ function partable_row( value_fixed = free ? 0.0 : val, start = 0.0, estimate = 0.0, - param = free ? val : :const, + label = free ? val : :const, ) end @@ -351,20 +351,20 @@ function append_rows!( partable::ParameterTable, arr::ParamsArray, arr_name::Symbol, - params::AbstractVector, + param_labels::AbstractVector, varnames::AbstractVector{Symbol}; skip_symmetric::Bool = false, ) - nparams(arr) == length(params) || throw( + nparams(arr) == length(param_labels) || throw( ArgumentError( - "Length of parameters vector ($(length(params))) does not match the number of parameters in the matrix ($(nparams(arr)))", + "Length of parameters vector ($(length(param_labels))) does not match the number of parameters in the matrix ($(nparams(arr)))", ), ) arr_ixs = eachindex(arr) # add parameters visited_indices = Set{eltype(arr_ixs)}() - for (i, par) in enumerate(params) + for (i, par) in enumerate(param_labels) for j in param_occurences_range(arr, i) arr_ix = arr_ixs[arr.linear_indices[j]] skip_symmetric && (arr_ix ∈ visited_indices) && continue @@ -399,7 +399,7 @@ function Base.:(==)(mat1::RAMMatrices, mat2::RAMMatrices) (mat1.S == mat2.S) && (mat1.F == mat2.F) && (mat1.M == mat2.M) && - (mat1.params == mat2.params) && + (mat1.param_labels == mat2.param_labels) && (mat1.vars == mat2.vars) ) return res diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 33440e257..7ba8f7fb7 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -35,7 +35,7 @@ vars(model::AbstractSemSingle) = vars(implied(model)) observed_vars(model::AbstractSemSingle) = observed_vars(implied(model)) latent_vars(model::AbstractSemSingle) = latent_vars(implied(model)) -params(model::AbstractSemSingle) = params(implied(model)) +param_labels(model::AbstractSemSingle) = param_labels(implied(model)) nparams(model::AbstractSemSingle) = nparams(implied(model)) """ @@ -45,6 +45,11 @@ Returns the [*observed*](@ref SemObserved) part of a model. """ observed(model::AbstractSemSingle) = model.observed +""" + nsamples(model::AbstractSem) -> Int + +Returns the number of samples from the [*observed*](@ref SemObserved) part of a model. +""" nsamples(model::AbstractSemSingle) = nsamples(observed(model)) """ diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 65bace302..314abcc35 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -40,7 +40,7 @@ function ParameterTable( graph::AbstractStenoGraph; observed_vars::AbstractVector{Symbol}, latent_vars::AbstractVector{Symbol}, - params::Union{AbstractVector{Symbol}, Nothing} = nothing, + param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing, group::Union{Integer, Nothing} = nothing, param_prefix::Symbol = :θ, ) @@ -54,7 +54,7 @@ function ParameterTable( free = columns[:free] value_fixed = columns[:value_fixed] start = columns[:start] - param_refs = columns[:param] + param_refs = columns[:label] # group = Vector{Symbol}(undef, n) for (i, element) in enumerate(graph) @@ -126,7 +126,7 @@ function ParameterTable( end end - return ParameterTable(columns; latent_vars, observed_vars, params) + return ParameterTable(columns; latent_vars, observed_vars, param_labels) end ############################################################################################ @@ -148,7 +148,7 @@ function EnsembleParameterTable( graph::AbstractStenoGraph; observed_vars::AbstractVector{Symbol}, latent_vars::AbstractVector{Symbol}, - params::Union{AbstractVector{Symbol}, Nothing} = nothing, + param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing, groups, ) graph = unique(graph) @@ -158,11 +158,11 @@ function EnsembleParameterTable( graph; observed_vars, latent_vars, - params, + param_labels, group = i, param_prefix = Symbol(:g, group), ) for (i, group) in enumerate(groups) ) - return EnsembleParameterTable(partables; params) + return EnsembleParameterTable(partables; param_labels) end diff --git a/src/frontend/specification/checks.jl b/src/frontend/specification/checks.jl index 5326e535f..5ef41c59d 100644 --- a/src/frontend/specification/checks.jl +++ b/src/frontend/specification/checks.jl @@ -1,18 +1,18 @@ # check if params vector correctly matches the parameter references (from the ParameterTable) -function check_params( - params::AbstractVector{Symbol}, +function check_param_labels( + param_labels::AbstractVector{Symbol}, param_refs::Union{AbstractVector{Symbol}, Nothing}, ) - dup_params = nonunique(params) - isempty(dup_params) || - throw(ArgumentError("Duplicate parameters detected: $(join(dup_params, ", "))")) - any(==(:const), params) && + dup_param_labels = nonunique(param_labels) + isempty(dup_param_labels) || + throw(ArgumentError("Duplicate parameter labels detected: $(join(dup_param_labels, ", "))")) + any(==(:const), param_labels) && throw(ArgumentError("Parameters constain reserved :const name")) if !isnothing(param_refs) # check if all references parameters are present all_refs = Set(id for id in param_refs if id != :const) - undecl_params = setdiff(all_refs, params) + undecl_params = setdiff(all_refs, param_labels) if !isempty(undecl_params) throw( ArgumentError( diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl index 72d95c6b4..54f43fa9c 100644 --- a/src/frontend/specification/documentation.jl +++ b/src/frontend/specification/documentation.jl @@ -1,4 +1,4 @@ -params(spec::SemSpecification) = spec.params +param_labels(spec::SemSpecification) = spec.param_labels """ vars(semobj) -> Vector{Symbol} @@ -65,7 +65,7 @@ function ParameterTable end (1) EnsembleParameterTable(;graph, observed_vars, latent_vars, groups) - (2) EnsembleParameterTable(ps::Pair...; params = nothing) + (2) EnsembleParameterTable(ps::Pair...; param_labels = nothing) Return an `EnsembleParameterTable` constructed from (1) a graph or (2) multiple specifications. @@ -73,7 +73,7 @@ Return an `EnsembleParameterTable` constructed from (1) a graph or (2) multiple - `graph`: graph defined via `@StenoGraph` - `observed_vars::Vector{Symbol}`: observed variable names - `latent_vars::Vector{Symbol}`: latent variable names -- `params::Vector{Symbol}`: (optional) a vector of parameter names +- `param_labels::Vector{Symbol}`: (optional) a vector of parameter names - `ps::Pair...`: `:group_name => specification`, where `specification` is either a `ParameterTable` or `RAMMatrices` # Examples @@ -88,7 +88,7 @@ function EnsembleParameterTable end (1) RAMMatrices(partable::ParameterTable) - (2) RAMMatrices(;A, S, F, M = nothing, params, vars) + (2) RAMMatrices(;A, S, F, M = nothing, param_labels, vars) (3) RAMMatrices(partable::EnsembleParameterTable) @@ -102,7 +102,7 @@ Return `RAMMatrices` constructed from (1) a parameter table or (2) individual ma - `S`: matrix of undirected effects - `F`: filter matrix - `M`: vector of mean effects -- `params::Vector{Symbol}`: parameter labels +- `param_labels::Vector{Symbol}`: parameter labels - `vars::Vector{Symbol}`: variable names corresponding to the A, S and F matrix columns # Examples diff --git a/src/implied/RAM/generic.jl b/src/implied/RAM/generic.jl index 30bd29bf4..301c455e9 100644 --- a/src/implied/RAM/generic.jl +++ b/src/implied/RAM/generic.jl @@ -34,7 +34,7 @@ and for models with a meanstructure, the model implied means are computed as ``` ## Interfaces -- `params(::RAM) `-> vector of parameter labels +- `param_labels(::RAM) `-> vector of parameter labels - `nparams(::RAM)` -> number of parameters - `Σ(::RAM)` -> model implied covariance matrix @@ -169,11 +169,11 @@ end ### methods ############################################################################################ -function update!(targets::EvaluationTargets, implied::RAM, model::AbstractSemSingle, params) - materialize!(implied.A, implied.ram_matrices.A, params) - materialize!(implied.S, implied.ram_matrices.S, params) +function update!(targets::EvaluationTargets, implied::RAM, model::AbstractSemSingle, param_labels) + materialize!(implied.A, implied.ram_matrices.A, param_labels) + materialize!(implied.S, implied.ram_matrices.S, param_labels) if !isnothing(implied.M) - materialize!(implied.M, implied.ram_matrices.M, params) + materialize!(implied.M, implied.ram_matrices.M, param_labels) end parent(implied.I_A) .= .-implied.A diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index 44ad4949d..eff193c17 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -29,7 +29,7 @@ Subtype of `SemImplied` that implements the RAM notation with symbolic precomput Subtype of `SemImplied`. ## Interfaces -- `params(::RAMSymbolic) `-> vector of parameter ids +- `param_labels(::RAMSymbolic) `-> vector of parameter ids - `nparams(::RAMSymbolic)` -> number of parameters - `Σ(::RAMSymbolic)` -> model implied covariance matrix diff --git a/src/implied/abstract.jl b/src/implied/abstract.jl index 99bb4d68d..af51440c6 100644 --- a/src/implied/abstract.jl +++ b/src/implied/abstract.jl @@ -8,7 +8,7 @@ nvars(implied::SemImplied) = nvars(implied.ram_matrices) nobserved_vars(implied::SemImplied) = nobserved_vars(implied.ram_matrices) nlatent_vars(implied::SemImplied) = nlatent_vars(implied.ram_matrices) -params(implied::SemImplied) = params(implied.ram_matrices) +param_labels(implied::SemImplied) = param_labels(implied.ram_matrices) nparams(implied::SemImplied) = nparams(implied.ram_matrices) # checks if the A matrix is acyclic diff --git a/src/implied/empty.jl b/src/implied/empty.jl index a80f8c185..3b0292e73 100644 --- a/src/implied/empty.jl +++ b/src/implied/empty.jl @@ -19,7 +19,7 @@ model per group and an additional model with `ImpliedEmpty` and `SemRidge` for t # Extended help ## Interfaces -- `params(::RAMSymbolic) `-> Vector of parameter labels +- `param_labels(::RAMSymbolic) `-> Vector of parameter labels - `nparams(::RAMSymbolic)` -> Number of parameters ## Implementation diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 0ef542f70..ca23ded97 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -95,12 +95,12 @@ function evaluate!( semfiml::SemFIML, implied::SemImplied, model::AbstractSemSingle, - params, + param_labels, ) isnothing(hessian) || error("Hessian not implemented for FIML") if !check_fiml(semfiml, model) - isnothing(objective) || (objective = non_posdef_return(params)) + isnothing(objective) || (objective = non_posdef_return(param_labels)) isnothing(gradient) || fill!(gradient, 1) return objective end @@ -109,7 +109,7 @@ function evaluate!( scale = inv(nsamples(observed(model))) isnothing(objective) || - (objective = scale * F_FIML(observed(model), semfiml, model, params)) + (objective = scale * F_FIML(observed(model), semfiml, model, param_labels)) isnothing(gradient) || (∇F_FIML!(gradient, observed(model), semfiml, model); gradient .*= scale) @@ -169,8 +169,8 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, implied, model, semfiml) mul!(G, ∇μ', Jμ, -1, 1) end -function F_FIML(observed::SemObservedMissing, semfiml, model, params) - F = zero(eltype(params)) +function F_FIML(observed::SemObservedMissing, semfiml, model, param_labels) + F = zero(eltype(param_labels)) for (i, pat) in enumerate(observed.patterns) F += F_one_pattern( semfiml.meandiff[i], diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index 02f637270..aee521624 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -59,7 +59,7 @@ function SemRidge(; ), ) else - par2ind = Dict(par => ind for (ind, par) in enumerate(params(implied))) + par2ind = param_indices(implied) which_ridge = getindex.(Ref(par2ind), which_ridge) end end diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index 68bcc04ad..2487b7c52 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -1,5 +1,5 @@ """ - sem_fit([optim::SemOptimizer], model::AbstractSem; + fit([optim::SemOptimizer], model::AbstractSem; [engine::Symbol], start_val = start_val, kwargs...) Return the fitted `model`. @@ -20,25 +20,25 @@ the online documentation on [Starting values](@ref). # Examples ```julia -sem_fit( +fit( my_model; start_val = start_simple, start_covariances_latent = 0.5) ``` """ -function sem_fit(optim::SemOptimizer, model::AbstractSem; start_val = nothing, kwargs...) +function fit(optim::SemOptimizer, model::AbstractSem; start_val = nothing, kwargs...) start_params = prepare_start_params(start_val, model; kwargs...) @assert start_params isa AbstractVector @assert length(start_params) == nparams(model) - sem_fit(optim, model, start_params; kwargs...) + fit(optim, model, start_params; kwargs...) end -sem_fit(model::AbstractSem; engine::Symbol = :Optim, start_val = nothing, kwargs...) = - sem_fit(SemOptimizer(; engine, kwargs...), model; start_val, kwargs...) +fit(model::AbstractSem; engine::Symbol = :Optim, start_val = nothing, kwargs...) = +fit(SemOptimizer(; engine, kwargs...), model; start_val, kwargs...) # fallback method -sem_fit(optim::SemOptimizer, model::AbstractSem, start_params; kwargs...) = +fit(optim::SemOptimizer, model::AbstractSem, start_params; kwargs...) = error("Optimizer $(optim) support not implemented.") # FABIN3 is the default method for single models diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index cec37a77a..8f5404bc2 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -102,7 +102,7 @@ optimizer(res::Optim.MultivariateOptimizationResults) = Optim.summary(res) n_iterations(res::Optim.MultivariateOptimizationResults) = Optim.iterations(res) convergence(res::Optim.MultivariateOptimizationResults) = Optim.converged(res) -function sem_fit( +function fit( optim::SemOptimizerOptim, model::AbstractSem, start_params::AbstractVector; diff --git a/src/types.jl b/src/types.jl index e802e057a..64a4acbac 100644 --- a/src/types.jl +++ b/src/types.jl @@ -188,13 +188,13 @@ Returns a SemEnsemble with fields - `n::Int`: Number of models. - `sems::Tuple`: `AbstractSem`s. - `weights::Vector`: Weights for each model. -- `params::Vector`: Stores parameter labels and their position. +- `param_labels::Vector`: Stores parameter labels and their position. """ struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, I} <: AbstractSemCollection n::N sems::T weights::V - params::I + param_labels::I end # constructor from multiple models @@ -209,16 +209,16 @@ function SemEnsemble(models...; weights = nothing, kwargs...) end # check parameters equality - params = SEM.params(models[1]) + param_labels = SEM.param_labels(models[1]) for model in models - if params != SEM.params(model) + if param_labels != SEM.param_labels(model) throw(ErrorException("The parameters of your models do not match. \n Maybe you tried to specify models of an ensemble via ParameterTables. \n In that case, you may use RAMMatrices instead.")) end end - return SemEnsemble(n, models, weights, params) + return SemEnsemble(n, models, weights, param_labels) end # constructor from EnsembleParameterTable and data set @@ -239,7 +239,7 @@ function SemEnsemble(; specification, data, groups, column = :group, kwargs...) return SemEnsemble(models...; weights = nothing, kwargs...) end -params(ensemble::SemEnsemble) = ensemble.params +param_labels(ensemble::SemEnsemble) = ensemble.param_labels """ n_models(ensemble::SemEnsemble) -> Integer diff --git a/test/examples/helper.jl b/test/examples/helper.jl index f35d2cac6..4ff9bd507 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -51,7 +51,7 @@ end fitmeasure_names_ml = Dict( :AIC => "aic", :BIC => "bic", - :df => "df", + :dof => "df", :χ² => "chisq", :p_value => "pvalue", :nparams => "npar", @@ -59,7 +59,7 @@ fitmeasure_names_ml = Dict( ) fitmeasure_names_ls = Dict( - :df => "df", + :dof => "df", :χ² => "chisq", :p_value => "pvalue", :nparams => "npar", @@ -89,8 +89,8 @@ function test_estimates( lav_group = nothing, skip::Bool = false, ) - actual = StructuralEquationModels.param_values(partable, col) - expected = StructuralEquationModels.lavaan_param_values( + actual = StructuralEquationModels.params(partable, col) + expected = StructuralEquationModels.lavaan_params( partable_lav, partable, lav_col, @@ -120,8 +120,8 @@ function test_estimates( actual = fill(NaN, nparams(ens_partable)) expected = fill(NaN, nparams(ens_partable)) for (key, partable) in pairs(ens_partable.tables) - StructuralEquationModels.param_values!(actual, partable, col) - StructuralEquationModels.lavaan_param_values!( + StructuralEquationModels.params!(actual, partable, col) + StructuralEquationModels.lavaan_params!( expected, partable_lav, partable, diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 1e97617fc..f6a7a230d 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -8,7 +8,7 @@ model_g1 = Sem(specification = specification_g1, data = dat_g1, implied = RAMSym model_g2 = Sem(specification = specification_g2, data = dat_g2, implied = RAM) -@test SEM.params(model_g1.implied.ram_matrices) == SEM.params(model_g2.implied.ram_matrices) +@test SEM.param_labels(model_g1.implied.ram_matrices) == SEM.param_labels(model_g2.implied.ram_matrices) # test the different constructors model_ml_multigroup = SemEnsemble(model_g1, model_g2) @@ -28,7 +28,7 @@ end # fit @testset "ml_solution_multigroup" begin - solution = sem_fit(semoptimizer, model_ml_multigroup) + solution = fit(semoptimizer, model_ml_multigroup) update_estimate!(partable, solution) test_estimates( partable, @@ -36,7 +36,7 @@ end atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = sem_fit(semoptimizer, model_ml_multigroup2) + solution = fit(semoptimizer, model_ml_multigroup2) update_estimate!(partable, solution) test_estimates( partable, @@ -47,7 +47,7 @@ end end @testset "fitmeasures/se_ml" begin - solution_ml = sem_fit(model_ml_multigroup) + solution_ml = fit(model_ml_multigroup) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; @@ -64,7 +64,7 @@ end lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution_ml = sem_fit(model_ml_multigroup2) + solution_ml = fit(model_ml_multigroup2) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; @@ -113,7 +113,7 @@ grad_fd = FiniteDiff.finite_difference_gradient( # fit @testset "ml_solution_multigroup | sorted" begin - solution = sem_fit(model_ml_multigroup) + solution = fit(model_ml_multigroup) update_estimate!(partable_s, solution) test_estimates( partable_s, @@ -124,7 +124,7 @@ grad_fd = FiniteDiff.finite_difference_gradient( end @testset "fitmeasures/se_ml | sorted" begin - solution_ml = sem_fit(model_ml_multigroup) + solution_ml = fit(model_ml_multigroup) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; @@ -191,7 +191,7 @@ end # fit @testset "solution_user_defined_loss" begin - solution = sem_fit(model_ml_multigroup) + solution = fit(model_ml_multigroup) update_estimate!(partable, solution) test_estimates( partable, @@ -226,7 +226,7 @@ model_ls_multigroup = SemEnsemble(model_ls_g1, model_ls_g2; optimizer = semoptim end @testset "ls_solution_multigroup" begin - solution = sem_fit(model_ls_multigroup) + solution = fit(model_ls_multigroup) update_estimate!(partable, solution) test_estimates( partable, @@ -237,7 +237,7 @@ end end @testset "fitmeasures/se_ls" begin - solution_ls = sem_fit(model_ls_multigroup) + solution_ls = fit(model_ls_multigroup) test_fitmeasures( fit_measures(solution_ls), solution_lav[:fitmeasures_ls]; @@ -321,7 +321,7 @@ if !isnothing(specification_miss_g1) end @testset "fiml_solution_multigroup" begin - solution = sem_fit(semoptimizer, model_ml_multigroup) + solution = fit(semoptimizer, model_ml_multigroup) update_estimate!(partable_miss, solution) test_estimates( partable_miss, @@ -329,7 +329,7 @@ if !isnothing(specification_miss_g1) atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = sem_fit(semoptimizer, model_ml_multigroup2) + solution = fit(semoptimizer, model_ml_multigroup2) update_estimate!(partable_miss, solution) test_estimates( partable_miss, @@ -340,7 +340,7 @@ if !isnothing(specification_miss_g1) end @testset "fitmeasures/se_fiml" begin - solution = sem_fit(semoptimizer, model_ml_multigroup) + solution = fit(semoptimizer, model_ml_multigroup) test_fitmeasures( fit_measures(solution), solution_lav[:fitmeasures_fiml]; @@ -357,7 +357,7 @@ if !isnothing(specification_miss_g1) lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = sem_fit(semoptimizer, model_ml_multigroup2) + solution = fit(semoptimizer, model_ml_multigroup2) test_fitmeasures( fit_measures(solution), solution_lav[:fitmeasures_fiml]; diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index eac2b38dd..239bf713c 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -59,7 +59,7 @@ specification_g1 = RAMMatrices(; A = A, S = S1, F = F, - params = x, + param_labels = x, vars = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], ) @@ -67,7 +67,7 @@ specification_g2 = RAMMatrices(; A = A, S = S2, F = F, - params = x, + param_labels = x, vars = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], ) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index ddbbfc3fa..3397b5f0a 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -70,7 +70,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ls", "ml", "ml"]) for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution" begin - solution = sem_fit(optimizer_obj, model) + solution = fit(optimizer_obj, model) update_estimate!(partable, solution) test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) end @@ -79,9 +79,9 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) end @testset "ridge_solution" begin - solution_ridge = sem_fit(optimizer_obj, model_ridge) - solution_ml = sem_fit(optimizer_obj, model_ml) - # solution_ridge_id = sem_fit(optimizer_obj, model_ridge_id) + solution_ridge = fit(optimizer_obj, model_ridge) + solution_ml = fit(optimizer_obj, model_ml) + # solution_ridge_id = fit(optimizer_obj, model_ridge_id) @test solution_ridge.minimum < solution_ml.minimum + 1 end @@ -97,8 +97,8 @@ end end @testset "ml_solution_weighted" begin - solution_ml = sem_fit(optimizer_obj, model_ml) - solution_ml_weighted = sem_fit(optimizer_obj, model_ml_weighted) + solution_ml = fit(optimizer_obj, model_ml) + solution_ml_weighted = fit(optimizer_obj, model_ml_weighted) @test solution(solution_ml) ≈ solution(solution_ml_weighted) rtol = 1e-3 @test nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml) ≈ StructuralEquationModels.minimum(solution_ml_weighted) rtol = 1e-6 @@ -109,7 +109,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml" begin - solution_ml = sem_fit(optimizer_obj, model_ml) + solution_ml = fit(optimizer_obj, model_ml) test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) update_se_hessian!(partable, solution_ml) @@ -123,7 +123,7 @@ end end @testset "fitmeasures/se_ls" begin - solution_ls = sem_fit(optimizer_obj, model_ls_sym) + solution_ls = fit(optimizer_obj, model_ls_sym) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -176,13 +176,13 @@ if opt_engine == :Optim end @testset "ml_solution_hessian" begin - solution = sem_fit(optimizer_obj, model_ml) + solution = fit(optimizer_obj, model_ml) update_estimate!(partable, solution) test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-2) end @testset "ls_solution_hessian" begin - solution = sem_fit(optimizer_obj, model_ls) + solution = fit(optimizer_obj, model_ls) update_estimate!(partable, solution) test_estimates( partable, @@ -254,7 +254,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ls", "ml"] .* "_mean" for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution_mean" begin - solution = sem_fit(optimizer_obj, model) + solution = fit(optimizer_obj, model) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) end @@ -267,7 +267,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml_mean" begin - solution_ml = sem_fit(optimizer_obj, model_ml) + solution_ml = fit(optimizer_obj, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; @@ -285,7 +285,7 @@ end end @testset "fitmeasures/se_ls_mean" begin - solution_ls = sem_fit(optimizer_obj, model_ls) + solution_ls = fit(optimizer_obj, model_ls) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -336,13 +336,13 @@ end ############################################################################################ @testset "fiml_solution" begin - solution = sem_fit(optimizer_obj, model_ml) + solution = fit(optimizer_obj, model_ml) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @testset "fiml_solution_symbolic" begin - solution = sem_fit(optimizer_obj, model_ml_sym) + solution = fit(optimizer_obj, model_ml_sym) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @@ -352,7 +352,7 @@ end ############################################################################################ @testset "fitmeasures/se_fiml" begin - solution_ml = sem_fit(optimizer_obj, model_ml) + solution_ml = fit(optimizer_obj, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; diff --git a/test/examples/political_democracy/constraints.jl b/test/examples/political_democracy/constraints.jl index fb2116023..cc1b0874d 100644 --- a/test/examples/political_democracy/constraints.jl +++ b/test/examples/political_democracy/constraints.jl @@ -39,14 +39,14 @@ constrained_optimizer = SemOptimizer(; ############################################################################################ @testset "ml_solution_maxeval" begin - solution_maxeval = sem_fit(model_ml, engine = :NLopt, options = Dict(:maxeval => 10)) + solution_maxeval = fit(model_ml, engine = :NLopt, options = Dict(:maxeval => 10)) @test solution_maxeval.optimization_result.problem.numevals == 10 @test solution_maxeval.optimization_result.result[3] == :MAXEVAL_REACHED end @testset "ml_solution_constrained" begin - solution_constrained = sem_fit(constrained_optimizer, model_ml) + solution_constrained = fit(constrained_optimizer, model_ml) @test solution_constrained.solution[31] * solution_constrained.solution[30] >= (0.6 - 1e-8) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 3f226b4c8..7a8adc72e 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -8,7 +8,7 @@ using Random, NLopt semoptimizer = SemOptimizer(engine = opt_engine) model_ml = Sem(specification = spec, data = dat) -@test SEM.params(model_ml.implied.ram_matrices) == SEM.params(spec) +@test SEM.param_labels(model_ml.implied.ram_matrices) == SEM.param_labels(spec) model_ml_cov = Sem( specification = spec, @@ -75,7 +75,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ml", "ls", "ml", "ml" for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution" begin - solution = sem_fit(semoptimizer, model) + solution = fit(semoptimizer, model) update_estimate!(partable, solution) test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) end @@ -84,9 +84,9 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) end @testset "ridge_solution" begin - solution_ridge = sem_fit(semoptimizer, model_ridge) - solution_ml = sem_fit(semoptimizer, model_ml) - # solution_ridge_id = sem_fit(semoptimizer, model_ridge_id) + solution_ridge = fit(semoptimizer, model_ridge) + solution_ml = fit(semoptimizer, model_ml) + # solution_ridge_id = fit(semoptimizer, model_ridge_id) @test abs(solution_ridge.minimum - solution_ml.minimum) < 1 end @@ -102,8 +102,8 @@ end end @testset "ml_solution_weighted" begin - solution_ml = sem_fit(semoptimizer, model_ml) - solution_ml_weighted = sem_fit(semoptimizer, model_ml_weighted) + solution_ml = fit(semoptimizer, model_ml) + solution_ml_weighted = fit(semoptimizer, model_ml_weighted) @test isapprox(solution(solution_ml), solution(solution_ml_weighted), rtol = 1e-3) @test isapprox( nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml), @@ -117,7 +117,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml" begin - solution_ml = sem_fit(semoptimizer, model_ml) + solution_ml = fit(semoptimizer, model_ml) test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) update_se_hessian!(partable, solution_ml) @@ -131,7 +131,7 @@ end end @testset "fitmeasures/se_ls" begin - solution_ls = sem_fit(semoptimizer, model_ls_sym) + solution_ls = fit(semoptimizer, model_ls_sym) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -182,8 +182,8 @@ end obs_colnames = colnames, ) # fit models - sol_ml = solution(sem_fit(semoptimizer, model_ml_new)) - sol_ml_sym = solution(sem_fit(semoptimizer, model_ml_sym_new)) + sol_ml = solution(fit(semoptimizer, model_ml_new)) + sol_ml_sym = solution(fit(semoptimizer, model_ml_sym_new)) # check solution @test maximum(abs.(sol_ml - params)) < 0.01 @test maximum(abs.(sol_ml_sym - params)) < 0.01 @@ -225,13 +225,13 @@ if opt_engine == :Optim end @testset "ml_solution_hessian" begin - solution = sem_fit(semoptimizer, model_ml) + solution = fit(semoptimizer, model_ml) update_estimate!(partable, solution) test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-2) end @testset "ls_solution_hessian" begin - solution = sem_fit(semoptimizer, model_ls) + solution = fit(semoptimizer, model_ls) update_estimate!(partable, solution) test_estimates( partable, @@ -296,7 +296,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ml", "ls", "ml"] .* " for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution_mean" begin - solution = sem_fit(semoptimizer, model) + solution = fit(semoptimizer, model) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) end @@ -309,7 +309,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml_mean" begin - solution_ml = sem_fit(semoptimizer, model_ml) + solution_ml = fit(semoptimizer, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; @@ -327,7 +327,7 @@ end end @testset "fitmeasures/se_ls_mean" begin - solution_ls = sem_fit(semoptimizer, model_ls) + solution_ls = fit(semoptimizer, model_ls) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -381,8 +381,8 @@ end meanstructure = true, ) # fit models - sol_ml = solution(sem_fit(semoptimizer, model_ml_new)) - sol_ml_sym = solution(sem_fit(semoptimizer, model_ml_sym_new)) + sol_ml = solution(fit(semoptimizer, model_ml_new)) + sol_ml_sym = solution(fit(semoptimizer, model_ml_sym_new)) # check solution @test maximum(abs.(sol_ml - params)) < 0.01 @test maximum(abs.(sol_ml_sym - params)) < 0.01 @@ -427,13 +427,13 @@ end ############################################################################################ @testset "fiml_solution" begin - solution = sem_fit(semoptimizer, model_ml) + solution = fit(semoptimizer, model_ml) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @testset "fiml_solution_symbolic" begin - solution = sem_fit(semoptimizer, model_ml_sym) + solution = fit(semoptimizer, model_ml_sym) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @@ -443,7 +443,7 @@ end ############################################################################################ @testset "fitmeasures/se_fiml" begin - solution_ml = sem_fit(semoptimizer, model_ml) + solution_ml = fit(semoptimizer, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 7394175b7..ad06e0fcd 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -77,13 +77,13 @@ spec = RAMMatrices(; A = A, S = S, F = F, - params = x, + param_labels = x, vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65], ) partable = ParameterTable(spec) -@test SEM.params(spec) == SEM.params(partable) +@test SEM.param_labels(spec) == SEM.param_labels(partable) # w. meanstructure ------------------------------------------------------------------------- @@ -94,13 +94,13 @@ spec_mean = RAMMatrices(; S = S, F = F, M = M, - params = [SEM.params(spec); Symbol.("x", string.(32:38))], + param_labels = [SEM.param_labels(spec); Symbol.("x", string.(32:38))], vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65], ) partable_mean = ParameterTable(spec_mean) -@test SEM.params(partable_mean) == SEM.params(spec_mean) +@test SEM.param_labels(partable_mean) == SEM.param_labels(spec_mean) start_test = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3)] start_test_mean = @@ -138,7 +138,7 @@ end spec = ParameterTable(spec) spec_mean = ParameterTable(spec_mean) -@test SEM.params(spec) == SEM.params(partable) +@test SEM.param_labels(spec) == SEM.param_labels(partable) partable = spec partable_mean = spec_mean diff --git a/test/examples/proximal/l0.jl b/test/examples/proximal/l0.jl index da20f3901..374f8e58a 100644 --- a/test/examples/proximal/l0.jl +++ b/test/examples/proximal/l0.jl @@ -35,7 +35,7 @@ ram_mat = RAMMatrices(partable) model = Sem(specification = partable, data = dat, loss = SemML) -fit = sem_fit(model) +sem_fit = fit(model) # use l0 from ProximalSEM # regularized @@ -44,11 +44,11 @@ prox_operator = model_prox = Sem(specification = partable, data = dat, loss = SemML) -fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = prox_operator) +fit_prox = fit(model_prox, engine = :Proximal, operator_g = prox_operator) @testset "l0 | solution_unregularized" begin @test fit_prox.optimization_result.result[:iterations] < 1000 - @test maximum(abs.(solution(fit) - solution(fit_prox))) < 0.002 + @test maximum(abs.(solution(sem_fit) - solution(fit_prox))) < 0.002 end # regularized @@ -56,12 +56,12 @@ prox_operator = SlicedSeparableSum((NormL0(0.0), NormL0(100.0)), ([1:30], [31])) model_prox = Sem(specification = partable, data = dat, loss = SemML) -fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = prox_operator) +fit_prox = fit(model_prox, engine = :Proximal, operator_g = prox_operator) @testset "l0 | solution_regularized" begin @test fit_prox.optimization_result.result[:iterations] < 1000 @test solution(fit_prox)[31] == 0.0 @test abs( - StructuralEquationModels.minimum(fit_prox) - StructuralEquationModels.minimum(fit), + StructuralEquationModels.minimum(fit_prox) - StructuralEquationModels.minimum(sem_fit), ) < 1.0 end diff --git a/test/examples/proximal/lasso.jl b/test/examples/proximal/lasso.jl index 314453df4..beb5cf529 100644 --- a/test/examples/proximal/lasso.jl +++ b/test/examples/proximal/lasso.jl @@ -35,18 +35,18 @@ ram_mat = RAMMatrices(partable) model = Sem(specification = partable, data = dat, loss = SemML) -fit = sem_fit(model) +sem_fit = fit(model) # use lasso from ProximalSEM λ = zeros(31) model_prox = Sem(specification = partable, data = dat, loss = SemML) -fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = NormL1(λ)) +fit_prox = fit(model_prox, engine = :Proximal, operator_g = NormL1(λ)) @testset "lasso | solution_unregularized" begin @test fit_prox.optimization_result.result[:iterations] < 1000 - @test maximum(abs.(solution(fit) - solution(fit_prox))) < 0.002 + @test maximum(abs.(solution(sem_fit) - solution(fit_prox))) < 0.002 end λ = zeros(31); @@ -54,11 +54,11 @@ end model_prox = Sem(specification = partable, data = dat, loss = SemML) -fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = NormL1(λ)) +fit_prox = fit(model_prox, engine = :Proximal, operator_g = NormL1(λ)) @testset "lasso | solution_regularized" begin @test fit_prox.optimization_result.result[:iterations] < 1000 - @test all(solution(fit_prox)[16:20] .< solution(fit)[16:20]) + @test all(solution(fit_prox)[16:20] .< solution(sem_fit)[16:20]) @test StructuralEquationModels.minimum(fit_prox) - - StructuralEquationModels.minimum(fit) < 0.03 + StructuralEquationModels.minimum(sem_fit) < 0.03 end diff --git a/test/examples/proximal/ridge.jl b/test/examples/proximal/ridge.jl index 3d116dcd4..fd7ae113d 100644 --- a/test/examples/proximal/ridge.jl +++ b/test/examples/proximal/ridge.jl @@ -35,7 +35,7 @@ ram_mat = RAMMatrices(partable) model = Sem(specification = partable, data = dat, loss = SemML) -fit = sem_fit(model) +sem_fit = fit(model) # use ridge from StructuralEquationModels model_ridge = Sem( @@ -46,7 +46,7 @@ model_ridge = Sem( which_ridge = 16:20, ) -solution_ridge = sem_fit(model_ridge) +solution_ridge = fit(model_ridge) # use ridge from ProximalSEM; SqrNormL2 uses λ/2 as penalty λ = zeros(31); @@ -54,7 +54,7 @@ solution_ridge = sem_fit(model_ridge) model_prox = Sem(specification = partable, data = dat, loss = SemML) -solution_prox = @suppress sem_fit(model_prox, engine = :Proximal, operator_g = SqrNormL2(λ)) +solution_prox = @suppress fit(model_prox, engine = :Proximal, operator_g = SqrNormL2(λ)) @testset "ridge_solution" begin @test isapprox(solution_prox.solution, solution_ridge.solution; rtol = 1e-3) diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 6899fe7a7..a3e426cbc 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -40,7 +40,7 @@ A = [ 0 0 0 0 0 0 0 0 ] -ram_matrices = RAMMatrices(; A = A, S = S, F = F, params = x, vars = nothing) +ram_matrices = RAMMatrices(; A = A, S = S, F = F, param_labels = x, vars = nothing) true_val = [ repeat([1], 8) @@ -73,6 +73,6 @@ optimizer = SemOptimizerOptim( Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), ) -solution_ml = sem_fit(optimizer, model_ml) +solution_ml = fit(optimizer, model_ml) @test true_val ≈ solution(solution_ml) atol = 0.05 diff --git a/test/unit_tests/StatsAPI.jl b/test/unit_tests/StatsAPI.jl new file mode 100644 index 000000000..8648fc363 --- /dev/null +++ b/test/unit_tests/StatsAPI.jl @@ -0,0 +1,29 @@ +using StructuralEquationModels +graph = @StenoGraph begin + a → b +end +partable = ParameterTable(graph, observed_vars = [:a, :b], latent_vars = Symbol[]) +update_partable!(partable, :estimate, param_labels(partable), [3.1415]) +data = randn(100, 2) +model = Sem( + specification = partable, + data = data +) +model_fit = fit(model) + +@testset "params" begin + out = [NaN] + StructuralEquationModels.params!(out, partable) + @test params(partable) == out == [3.1415] == coef(partable) +end +@testset "param_labels" begin + @test param_labels(partable) == [:θ_1] == coefnames(partable) +end + +@testset "nobs" begin + @test nobs(model) == nsamples(model) +end + +@testset "coeftable" begin + @test_throws "StructuralEquationModels does not support" coeftable(model) +end \ No newline at end of file diff --git a/test/unit_tests/bootstrap.jl b/test/unit_tests/bootstrap.jl index f30092865..a2d5b6832 100644 --- a/test/unit_tests/bootstrap.jl +++ b/test/unit_tests/bootstrap.jl @@ -1,4 +1,4 @@ -solution_ml = sem_fit(model_ml) +solution_ml = fit(model_ml) bs = se_bootstrap(solution_ml; n_boot = 20) update_se_hessian!(partable, solution_ml) diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index 7ed190c22..2bf5dedaf 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -25,6 +25,7 @@ graph = @StenoGraph begin y8 ↔ y4 + y6 end + ram_matrices = RAMMatrices(ParameterTable(graph, observed_vars = obs_vars, latent_vars = lat_vars)) @@ -43,7 +44,7 @@ end function test_params_api(semobj, spec::SemSpecification) @test @inferred(nparams(semobj)) == nparams(spec) - @test @inferred(params(semobj)) == params(spec) + @test @inferred(param_labels(semobj)) == param_labels(spec) end @testset "Sem(implied=$impliedtype, loss=$losstype)" for impliedtype in (RAM, RAMSymbolic), diff --git a/test/unit_tests/sorting.jl b/test/unit_tests/sorting.jl index 0908a6497..3c61e13c4 100644 --- a/test/unit_tests/sorting.jl +++ b/test/unit_tests/sorting.jl @@ -11,7 +11,7 @@ model_ml_sorted = Sem(specification = partable, data = dat) end @testset "ml_solution_sorted" begin - solution_ml_sorted = sem_fit(model_ml_sorted) + solution_ml_sorted = fit(model_ml_sorted) update_estimate!(partable, solution_ml_sorted) @test test_estimates(par_ml, partable, 0.01) end diff --git a/test/unit_tests/specification.jl b/test/unit_tests/specification.jl index ef9fc73a1..b69230d7f 100644 --- a/test/unit_tests/specification.jl +++ b/test/unit_tests/specification.jl @@ -58,8 +58,8 @@ end @test nvars(partable) == length(obs_vars) + length(lat_vars) @test issetequal(vars(partable), [obs_vars; lat_vars]) - # params API - @test params(partable) == [[:θ_1, :a₁, :λ₉]; Symbol.("θ_", 2:16)] + # param_labels API + @test param_labels(partable) == [[:θ_1, :a₁, :λ₉]; Symbol.("θ_", 2:16)] @test nparams(partable) == 18 # don't allow constructing ParameterTable from a graph for an ensemble @@ -116,7 +116,7 @@ end @test nparams(enspartable) == 36 @test issetequal( - params(enspartable), + param_labels(enspartable), [Symbol.("gPasteur_", 1:16); Symbol.("gGrant_White_", 1:17); [:a₁, :a₂, :λ₉]], ) end @@ -135,7 +135,7 @@ end @test nvars(ram_matrices) == length(obs_vars) + length(lat_vars) @test issetequal(vars(ram_matrices), [obs_vars; lat_vars]) - # params API + # param_labels API @test nparams(ram_matrices) == nparams(partable) - @test params(ram_matrices) == params(partable) + @test param_labels(ram_matrices) == param_labels(partable) end diff --git a/test/unit_tests/unit_tests.jl b/test/unit_tests/unit_tests.jl index a638b991d..7189addd4 100644 --- a/test/unit_tests/unit_tests.jl +++ b/test/unit_tests/unit_tests.jl @@ -1,21 +1,35 @@ using Test, SafeTestsets -@safetestset "Multithreading" begin - include("multithreading.jl") -end - -@safetestset "Matrix algebra helper functions" begin - include("matrix_helpers.jl") -end +# Define available test sets +available_tests = Dict( + "multithreading" => "Multithreading", + "matrix_helpers" => "Matrix algebra helper functions", + "data_input_formats" => "SemObserved", + "specification" => "SemSpecification", + "model" => "Sem model", + "StatsAPI" => "StatsAPI" +) -@safetestset "SemObserved" begin - include("data_input_formats.jl") -end - -@safetestset "SemSpecification" begin - include("specification.jl") -end +# Determine which tests to run based on command-line arguments +selected_tests = isempty(ARGS) ? collect(keys(available_tests)) : ARGS -@safetestset "Sem model" begin - include("model.jl") +@testset "All Tests" begin + for file in selected_tests + if haskey(available_tests, file) + let file_ = file, test_name = available_tests[file] + # Compute the literal values + test_sym = Symbol(file_) + file_jl = file_ * ".jl" + # Build the expression with no free variables: + ex = quote + @safetestset $(Symbol(test_sym)) = $test_name begin + include($file_jl) + end + end + eval(ex) + end + else + @warn "Test file '$file' not found in available tests. Skipping." + end + end end diff --git a/test/unit_tests/unit_tests_interactive.jl b/test/unit_tests/unit_tests_interactive.jl new file mode 100644 index 000000000..cf082fa60 --- /dev/null +++ b/test/unit_tests/unit_tests_interactive.jl @@ -0,0 +1,10 @@ +# requires: TestEnv to be installed globally, and the StructuralEquationModels package `]dev`ed +# example: julia test/unit_tests/unit_tests_interactive.jl matrix_helpers + +try + import TestEnv + TestEnv.activate("StructuralEquationModels") +catch e + @warn "Error initializing Test Env" exception=(e, catch_backtrace()) +end +include("unit_tests.jl") \ No newline at end of file From d4fcf2a205c5d4b9ac935d3fcb96e1c09095f5df Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Sun, 11 May 2025 15:56:24 +0200 Subject: [PATCH 193/364] main into devel (#265) * Documentation/fix docstrings (#260) Clean code, fix docstrings, fix docs * Hotfix/optim (#262) * version number * Update README.md --- Project.toml | 2 +- README.md | 4 +- docs/make.jl | 1 - docs/src/developer/implied.md | 14 ++-- docs/src/developer/loss.md | 2 +- docs/src/developer/observed.md | 4 +- docs/src/developer/optimizer.md | 8 +-- docs/src/internals/files.md | 2 +- docs/src/internals/internals.md | 2 +- docs/src/performance/simulation.md | 4 -- docs/src/tutorials/backends/nlopt.md | 2 +- .../tutorials/construction/build_by_parts.md | 10 +-- .../construction/outer_constructor.md | 5 +- .../regularization/regularization.md | 3 +- src/additional_functions/helper.jl | 67 ++----------------- src/additional_functions/simulation.jl | 2 +- .../start_val/start_fabin3.jl | 21 +----- .../start_val/start_simple.jl | 1 - src/frontend/common.jl | 4 +- src/frontend/fit/fitmeasures/fit_measures.jl | 2 +- src/frontend/fit/fitmeasures/minus2ll.jl | 2 +- src/frontend/fit/summary.jl | 12 +--- src/frontend/pretty_printing.jl | 6 +- src/frontend/specification/RAMMatrices.jl | 2 +- src/frontend/specification/Sem.jl | 10 --- src/frontend/specification/StenoGraphs.jl | 1 - src/frontend/specification/documentation.jl | 27 ++++---- src/implied/RAM/generic.jl | 40 +++++------ src/implied/RAM/symbolic.jl | 30 ++++----- src/implied/abstract.jl | 1 - src/implied/empty.jl | 7 +- src/loss/ML/FIML.jl | 4 -- src/loss/ML/ML.jl | 7 +- src/loss/WLS/WLS.jl | 9 +-- src/loss/constant/constant.jl | 4 -- src/loss/regularization/ridge.jl | 4 -- src/objective_gradient_hessian.jl | 11 +-- src/observed/EM.jl | 54 +++++++-------- src/observed/abstract.jl | 2 +- src/observed/covariance.jl | 12 ++-- src/observed/data.jl | 13 +--- src/observed/missing.jl | 16 ++--- src/optimizer/Empty.jl | 6 -- src/optimizer/abstract.jl | 14 +++- src/optimizer/optim.jl | 8 +-- src/package_extensions/SEMNLOptExt.jl | 4 -- src/package_extensions/SEMProximalOptExt.jl | 10 ++- src/types.jl | 6 +- .../recover_parameters_twofact.jl | 2 +- test/unit_tests/bootstrap.jl | 5 -- test/unit_tests/partable_dataframe.jl | 3 - test/unit_tests/ridge_id.jl | 8 --- test/unit_tests/sorting.jl | 17 ----- test/unit_tests/start_val.jl | 1 - test/unit_tests/unit_tests.jl | 8 --- 55 files changed, 173 insertions(+), 353 deletions(-) delete mode 100644 test/unit_tests/bootstrap.jl delete mode 100644 test/unit_tests/partable_dataframe.jl delete mode 100644 test/unit_tests/ridge_id.jl delete mode 100644 test/unit_tests/sorting.jl delete mode 100644 test/unit_tests/start_val.jl diff --git a/Project.toml b/Project.toml index 9a5bc0916..376347083 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "StructuralEquationModels" uuid = "383ca8c5-e4ff-4104-b0a9-f7b279deed53" authors = ["Maximilian Ernst", "Aaron Peikert"] -version = "0.4.0" +version = "0.4.2" [deps] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" diff --git a/README.md b/README.md index 7fc19d0f8..68b7bcb71 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ | [![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://structuralequationmodels.github.io/StructuralEquationModels.jl/) [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://structuralequationmodels.github.io/StructuralEquationModels.jl/dev/) | [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) [![Github Action CI](https://github.com/StructuralEquationModels/StructuralEquationModels.jl/workflows/CI_extended/badge.svg)](https://github.com/StructuralEquationModels/StructuralEquationModels.jl/actions/) [![codecov](https://codecov.io/gh/StructuralEquationModels/StructuralEquationModels.jl/branch/main/graph/badge.svg?token=P2kjzpvM4V)](https://codecov.io/gh/StructuralEquationModels/StructuralEquationModels.jl) | [![DOI](https://zenodo.org/badge/228649704.svg)](https://zenodo.org/badge/latestdoi/228649704) | > [!NOTE] -> Check out our [preprint](https://formal-methods-mpi.github.io/pkgmanuscript/manuscript.pdf) on the package! +> Check out our [preprint](https://doi.org/10.31234/osf.io/zwe8g_v1) on the package! # What is this Package for? @@ -39,7 +39,7 @@ The package makes use of - SparseArrays.jl to speed up symbolic computations. - Optim.jl and NLopt.jl to provide a range of different Optimizers/Linesearches. - ProximalAlgorithms.jl for regularization. -- FiniteDiff.jl and ForwardDiff.jl to provide gradients for user-defined loss functions. +- FiniteDiff.jl and to provide gradient approximations for user-defined loss functions. # At the moment, we are still working on: - optimizing performance for big models (with hundreds of parameters) diff --git a/docs/make.jl b/docs/make.jl index 4542cf48f..1bb68c4da 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -52,7 +52,6 @@ makedocs( "files" => "internals/files.md", "types" => "internals/types.md", ], - "Complementary material" => ["Mathematical appendix" => "complementary/maths.md"], ], format = Documenter.HTML( prettyurls = get(ENV, "CI", nothing) == "true", diff --git a/docs/src/developer/implied.md b/docs/src/developer/implied.md index bea824a94..056cd6638 100644 --- a/docs/src/developer/implied.md +++ b/docs/src/developer/implied.md @@ -62,11 +62,8 @@ model per group and an additional model with `ImpliedEmpty` and `SemRidge` for t # Extended help ## Interfaces -- `params(::RAMSymbolic) `-> Vector of parameter labels -- `nparams(::RAMSymbolic)` -> Number of parameters - -## Implementation -Subtype of `SemImplied`. +- `param_labels(::ImpliedEmpty) `-> Vector of parameter labels +- `nparams(::ImpliedEmpty)` -> Number of parameters """ struct ImpliedEmpty{A, B, C} <: SemImplied hessianeval::A @@ -78,7 +75,12 @@ end ### Constructors ############################################################################################ -function ImpliedEmpty(;specification, meanstruct = NoMeanStruct(), hessianeval = ExactHessian(), kwargs...) +function ImpliedEmpty(; + specification, + meanstruct = NoMeanStruct(), + hessianeval = ExactHessian(), + kwargs..., +) return ImpliedEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) end diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index 931c2d0e5..d6949842b 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -204,7 +204,7 @@ julia>? help?> RAM -help?> SemObservedCommon +help?> SemObservedData ``` We see that the model implied covariance matrix can be assessed as `Σ(implied)` and the observed covariance matrix as `obs_cov(observed)`. diff --git a/docs/src/developer/observed.md b/docs/src/developer/observed.md index 240c1c34f..ee795cd3e 100644 --- a/docs/src/developer/observed.md +++ b/docs/src/developer/observed.md @@ -28,8 +28,8 @@ nsamples(observed::MyObserved) = ... nobserved_vars(observed::MyObserved) = ... ``` -As always, you can add additional methods for properties that implied types and loss function want to access, for example (from the `SemObservedCommon` implementation): +As always, you can add additional methods for properties that implied types and loss function want to access, for example (from the `SemObservedData` implementation): ```julia -obs_cov(observed::SemObservedCommon) = observed.obs_cov +obs_cov(observed::SemObservedData) = observed.obs_cov ``` \ No newline at end of file diff --git a/docs/src/developer/optimizer.md b/docs/src/developer/optimizer.md index a651ec636..9e01ac87c 100644 --- a/docs/src/developer/optimizer.md +++ b/docs/src/developer/optimizer.md @@ -16,7 +16,7 @@ SemOptimizer{:Name}(args...; kwargs...) = SemOptimizerName(args...; kwargs...) SemOptimizerName(; algorithm = LBFGS(), - options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), + options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), kwargs..., ) = SemOptimizerName(algorithm, options) @@ -37,13 +37,13 @@ options(optimizer::SemOptimizerName) = optimizer.options Note that your optimizer is a subtype of `SemOptimizer{:Name}`, where you can choose a `:Name` that can later be used as a keyword argument to `fit(engine = :Name)`. Similarly, `SemOptimizer{:Name}(args...; kwargs...) = SemOptimizerName(args...; kwargs...)` should be defined as well as a constructor that uses only keyword arguments: -´´´julia +```julia SemOptimizerName(; algorithm = LBFGS(), - options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), + options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), kwargs..., ) = SemOptimizerName(algorithm, options) -´´´ +``` A method for `update_observed` and additional methods might be usefull, but are not necessary. Now comes the substantive part: We need to provide a method for `fit`: diff --git a/docs/src/internals/files.md b/docs/src/internals/files.md index 90ceceaaf..4c2338393 100644 --- a/docs/src/internals/files.md +++ b/docs/src/internals/files.md @@ -10,7 +10,7 @@ Source code is in the `"src"` folder: - `"StructuralEquationModels.jl"` defines the module and the exported objects - `"types.jl"` defines all abstract types and the basic type hierarchy - `"objective_gradient_hessian.jl"` contains methods for computing objective, gradient and hessian values for different model types as well as generic fallback methods -- The four folders `"observed"`, `"implied"`, `"loss"` and `"diff"` contain implementations of specific subtypes (for example, the `"loss"` folder contains a file `"ML.jl"` that implements the `SemML` loss function). +- The folders `"observed"`, `"implied"`, and `"loss"` contain implementations of specific subtypes (for example, the `"loss"` folder contains a file `"ML.jl"` that implements the `SemML` loss function). - `"optimizer"` contains connections to different optimization backends (aka methods for `fit`) - `"optim.jl"`: connection to the `Optim.jl` package - `"frontend"` contains user-facing functions diff --git a/docs/src/internals/internals.md b/docs/src/internals/internals.md index 77d1085c8..18f82dbca 100644 --- a/docs/src/internals/internals.md +++ b/docs/src/internals/internals.md @@ -1,3 +1,3 @@ # Internals and Design -On the following pages, we document the internals and design of the package. Those informations are no prerequisite for extending the package (as decribed in the developer documentation)!, but they may be useful and hopefully interesting. \ No newline at end of file +On the following pages, we document some technical information about the package. Those informations are no prerequisite for extending the package (as decribed in the developer documentation)!, but they may be useful. \ No newline at end of file diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index 0cb2ea25d..d268853f6 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -1,9 +1,5 @@ # Simulation studies -!!! note "Simulation study interface" - We are currently working on an interface for simulation studies. - Until we are finished with this, this page is just a collection of tips. - ## Replace observed data In simulation studies, a common task is fitting the same model to many different datasets. It would be a waste of resources to reconstruct the complete model for each dataset. diff --git a/docs/src/tutorials/backends/nlopt.md b/docs/src/tutorials/backends/nlopt.md index 2afa5e547..feb5c8f48 100644 --- a/docs/src/tutorials/backends/nlopt.md +++ b/docs/src/tutorials/backends/nlopt.md @@ -1,7 +1,7 @@ # Using NLopt.jl [`SemOptimizerNLopt`](@ref) implements the connection to `NLopt.jl`. -It is only available if the `NLopt` package is loaded alongside `StructuralEquationModel.jl` in the running Julia session. +It is only available if the `NLopt` package is loaded alongside `StructuralEquationModels.jl` in the running Julia session. It takes a bunch of arguments: ```julia diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 45d2a2ea1..6b6b59ac9 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -47,21 +47,21 @@ partable = ParameterTable( Now, we construct the different parts: ```@example build -# observed --------------------------------------------------------------------------------- +# observed ----------------------------------------------------------------------------- observed = SemObservedData(specification = partable, data = data) -# implied ------------------------------------------------------------------------------------ +# implied ------------------------------------------------------------------------------ implied_ram = RAM(specification = partable) -# loss ------------------------------------------------------------------------------------- +# loss --------------------------------------------------------------------------------- ml = SemML(observed = observed) loss_ml = SemLoss(ml) -# optimizer ------------------------------------------------------------------------------------- +# optimizer ---------------------------------------------------------------------------- optimizer = SemOptimizerOptim() -# model ------------------------------------------------------------------------------------ +# model -------------------------------------------------------------------------------- model_ml = Sem(observed, implied_ram, loss_ml) diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md index a1c0b8ad3..e27724307 100644 --- a/docs/src/tutorials/construction/outer_constructor.md +++ b/docs/src/tutorials/construction/outer_constructor.md @@ -14,9 +14,8 @@ Structural Equation Model - Loss Functions SemML - Fields - observed: SemObservedCommon - implied: RAM - optimizer: SemOptimizerOptim + observed: SemObservedData + implied: RAM ``` The output of this call tells you exactly what model you just constructed (i.e. what the loss functions, observed, implied and optimizer parts are). diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index b1fc5c157..f743ac79c 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -30,13 +30,12 @@ It can be used as ```julia SemOptimizerProximal( algorithm = ProximalAlgorithms.PANOC(), - options = Dict{Symbol, Any}(), operator_g, operator_h = nothing ) ``` -The proximal operator (aka the regularization function) can be passed as `operator_g`, available options are listed [here](https://juliafirstorder.github.io/ProximalOperators.jl/stable/functions/). +The proximal operator (aka the regularization function) can be passed as `operator_g`. The available Algorithms are listed [here](https://juliafirstorder.github.io/ProximalAlgorithms.jl/stable/guide/implemented_algorithms/). ## First example - lasso diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 5559034e0..6cbcb0573 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -14,84 +14,29 @@ function neumann_series(mat::SparseMatrixCSC; maxiter::Integer = size(mat, 1)) return inverse end -#= -function make_onelement_array(A) - isa(A, Array) ? nothing : (A = [A]) - return A -end - =# - -function semvec(observed, implied, loss, optimizer) - observed = make_onelement_array(observed) - implied = make_onelement_array(implied) - loss = make_onelement_array(loss) - optimizer = make_onelement_array(optimizer) - - #sem_vec = Array{AbstractSem}(undef, maximum(length.([observed, implied, loss, optimizer]))) - sem_vec = Sem.(observed, implied, loss, optimizer) - - return sem_vec -end - -skipmissing_mean(mat::AbstractMatrix) = - [mean(skipmissing(coldata)) for coldata in eachcol(mat)] - -function F_one_person(imp_mean, meandiff, inverse, data, logdet) - F = logdet - @. meandiff = data - imp_mean - F += dot(meandiff, inverse, meandiff) - return F -end - -function remove_all_missing(data::AbstractMatrix) - keep = Vector{Int64}() - for (i, coldata) in zip(axes(data, 1), eachrow(data)) - if any(!ismissing, coldata) - push!(keep, i) - end - end - return data[keep, :], keep -end - function batch_inv!(fun, model) for i in 1:size(fun.inverses, 1) fun.inverses[i] .= LinearAlgebra.inv!(fun.choleskys[i]) end end -#= -function batch_sym_inv_update!(fun::Union{LossFunction, DiffFunction}, model) - M_inv = inv(fun.choleskys[1]) - for i = 1:size(fun.inverses, 1) - if size(model.observed.patterns_not[i]) == 0 - fun.inverses[i] .= M_inv - else - ind_not = model.observed.patterns_not[i] - ind = model.observed.patterns[i] - - A = M_inv[ind_not, ind] - H = cholesky(M_inv[ind_not, ind_not]) - D = H \ A - out = M_inv[ind, ind] - LinearAlgebra.BLAS.gemm('T', 'N', 1.0, A, D) - fun.inverses[i] .= out - end - end -end =# - -function sparse_outer_mul!(C, A, B, ind) #computes A*S*B -> C, where ind gives the entries of S that are 1 +# computes A*S*B -> C, where ind gives the entries of S that are 1 +function sparse_outer_mul!(C, A, B, ind) fill!(C, 0.0) for i in 1:length(ind) BLAS.ger!(1.0, A[:, ind[i][1]], B[ind[i][2], :], C) end end -function sparse_outer_mul!(C, A, ind) #computes A*∇m, where ∇m ind gives the entries of ∇m that are 1 +# computes A*∇m, where ∇m ind gives the entries of ∇m that are 1 +function sparse_outer_mul!(C, A, ind) fill!(C, 0.0) @views C .= sum(A[:, ind], dims = 2) return C end -function sparse_outer_mul!(C, A, B::Vector, ind) #computes A*S*B -> C, where ind gives the entries of S that are 1 +# computes A*S*B -> C, where ind gives the entries of S that are 1 +function sparse_outer_mul!(C, A, B::Vector, ind) fill!(C, 0.0) @views @inbounds for i in 1:length(ind) C .+= B[ind[i][2]] .* A[:, ind[i][1]] diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 27d58f93f..da3e6a906 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -7,7 +7,7 @@ Return a new model with swaped observed part. # Arguments - `model::AbstractSemSingle`: model to swap the observed part of. -- `kwargs`: additional keyword arguments; typically includes `data = ...` +- `kwargs`: additional keyword arguments; typically includes `data` and `specification` - `observed`: Either an object of subtype of `SemObserved` or a subtype of `SemObserved` # Examples diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index bd55f21d7..ab79d9ada 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -18,7 +18,7 @@ end # SemObservedMissing function start_fabin3(observed::SemObservedMissing, implied, args...; kwargs...) if !observed.em_model.fitted - em_mvn(observed; kwargs...) + em_mvn!(observed; kwargs...) end return start_fabin3(implied.ram_matrices, observed.em_model.Σ, observed.em_model.μ) @@ -45,24 +45,6 @@ function start_fabin3( ) @assert length(F_var2obs) == size(F, 1) - # check in which matrix each parameter appears - - #= in_S = length.(S_ind) .!= 0 - in_A = length.(A_ind) .!= 0 - A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind] - in_Λ = [any(ind[2] .∈ F_ind) for ind in A_ind_c] - - if !isnothing(M) - in_M = length.(M_ind) .!= 0 - in_any = in_A .| in_S .| in_M - else - in_any = in_A .| in_S - end - - if !all(in_any) - @warn "Could not determine fabin3 starting values for some parameters, default to 0." - end =# - # set undirected parameters in S S_indices = CartesianIndices(S) for j in 1:nparams(S) @@ -79,7 +61,6 @@ function start_fabin3( # set loadings A_indices = CartesianIndices(A) - # ind_Λ = findall([is_in_Λ(ind_vec, F_ind) for ind_vec in A_ind_c]) # collect latent variable indicators in A # maps latent parameter to the vector of dependent vars diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index ad5148e3f..4fbc8719c 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -63,7 +63,6 @@ function start_simple( nparams(ram_matrices) start_val = zeros(n_par) - n_obs = nobserved_vars(ram_matrices) n_var = nvars(ram_matrices) C_indices = CartesianIndices((n_var, n_var)) diff --git a/src/frontend/common.jl b/src/frontend/common.jl index e89a6cf8b..2734e8f2b 100644 --- a/src/frontend/common.jl +++ b/src/frontend/common.jl @@ -1,7 +1,7 @@ # API methods supported by multiple SEM.jl types """ - params(semobj) -> Vector{Symbol} + params(partable::ParameterTable) -> Vector{Symbol} Return the vector of SEM model parameter identifiers. """ @@ -42,7 +42,7 @@ nlatent_vars(semobj) = length(latent_vars(semobj)) """ param_indices(semobj) -Returns a dict of parameter names and their indices in `semobj`. +Returns a dict of parameter labels and their indices in `semobj`. # Examples ```julia diff --git a/src/frontend/fit/fitmeasures/fit_measures.jl b/src/frontend/fit/fitmeasures/fit_measures.jl index 2fc4dfba0..afdde173b 100644 --- a/src/frontend/fit/fitmeasures/fit_measures.jl +++ b/src/frontend/fit/fitmeasures/fit_measures.jl @@ -14,6 +14,6 @@ end """ fit_measures(sem_fit, args...) -Return a default set of fit measures or the fit measures passed as `arg...`. +Return a default set of fit measures or the fit measures passed as `args...`. """ function fit_measures end diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 2cb87d79c..ab4d24e53 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -41,7 +41,7 @@ end function minus2ll(observed::SemObservedMissing) # fit EM-based mean and cov if not yet fitted # FIXME EM could be very computationally expensive - observed.em_model.fitted || em_mvn(observed) + observed.em_model.fitted || em_mvn!(observed) Σ = observed.em_model.Σ μ = observed.em_model.μ diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 8ee134a9c..3071d5653 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -15,7 +15,7 @@ function details(sem_fit::SemFit; show_fitmeasures = false, color = :light_cyan, println("Number of data samples: $(nsamples(sem_fit))") print("\n") printstyled( - "----------------------------------- Model ----------------------------------- \n"; + "----------------------------------- Model ------------------------------------ \n"; color = color, ) print("\n") @@ -23,7 +23,7 @@ function details(sem_fit::SemFit; show_fitmeasures = false, color = :light_cyan, print("\n") if show_fitmeasures printstyled( - "--------------------------------- Fitmeasures --------------------------------- \n"; + "-------------------------------- Fitmeasures --------------------------------- \n"; color = color, ) print("\n") @@ -51,7 +51,7 @@ function details( if show_variables print("\n") printstyled( - "--------------------------------- Variables --------------------------------- \n"; + "---------------------------------- Variables --------------------------------- \n"; color = color, ) print("\n") @@ -242,9 +242,6 @@ function details( ) print("\n") end - - #printstyled("""No need to copy and paste results, you can use CSV.write(DataFrame(my_partable), "myfile.csv")"""; hidden = true) - end function details( @@ -297,9 +294,6 @@ function details( show_columns = show_columns, ) end - - # printstyled("""No need to copy and paste results, you can use CSV.write(DataFrame(my_partable), "myfile.csv")"""; hidden = true) - end function check_round(vec; digits) diff --git a/src/frontend/pretty_printing.jl b/src/frontend/pretty_printing.jl index c1cd72c2f..2fa970f24 100644 --- a/src/frontend/pretty_printing.jl +++ b/src/frontend/pretty_printing.jl @@ -1,3 +1,7 @@ +############################################################## +# Some helpers to implement show methods for SEM.jl objects +############################################################## + function print_field_types(io::IO, struct_instance) fields = fieldnames(typeof(struct_instance)) types = [typeof(getproperty(struct_instance, field)) for field in fields] @@ -25,7 +29,7 @@ function print_type(io::IO, struct_instance) end ############################################################## -# Loss Functions, Implied, +# Loss Function, Implied, Observed, Optimizer ############################################################## function Base.show(io::IO, struct_inst::SemLossFunction) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 75175a87d..d430e9c01 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -9,7 +9,7 @@ struct RAMMatrices <: SemSpecification F::SparseMatrixCSC{Float64} M::Union{ParamsVector{Float64}, Nothing} param_labels::Vector{Symbol} - vars::Union{Vector{Symbol}, Nothing} # better call it "variables": it's a mixture of observed and latent (and it gets confusing with get_vars()) + vars::Union{Vector{Symbol}, Nothing} end nparams(ram::RAMMatrices) = nparams(ram.A) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 7ba8f7fb7..53858abd4 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -45,11 +45,6 @@ Returns the [*observed*](@ref SemObserved) part of a model. """ observed(model::AbstractSemSingle) = model.observed -""" - nsamples(model::AbstractSem) -> Int - -Returns the number of samples from the [*observed*](@ref SemObserved) part of a model. -""" nsamples(model::AbstractSemSingle) = nsamples(observed(model)) """ @@ -156,11 +151,6 @@ end # pretty printing ############################################################## -#= function Base.show(io::IO, sem::Sem{O, I, L, D}) where {O, I, L, D} - lossfuntypes = @. nameof(typeof(sem.loss.functions)) - print(io, "Sem{$(nameof(O)), $(nameof(I)), $lossfuntypes, $(nameof(D))}") -end =# - function Base.show(io::IO, sem::Sem{O, I, L}) where {O, I, L} lossfuntypes = @. string(nameof(typeof(sem.loss.functions))) lossfuntypes = " " .* lossfuntypes .* ("\n") diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 314abcc35..79e17f719 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -55,7 +55,6 @@ function ParameterTable( value_fixed = columns[:value_fixed] start = columns[:start] param_refs = columns[:label] - # group = Vector{Symbol}(undef, n) for (i, element) in enumerate(graph) edge = element isa ModifiedEdge ? element.edge : element diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl index c61a0142a..4de1d9061 100644 --- a/src/frontend/specification/documentation.jl +++ b/src/frontend/specification/documentation.jl @@ -32,13 +32,6 @@ function latent_vars end latent_vars(spec::SemSpecification) = error("latent_vars(spec::$(typeof(spec))) is not implemented") -""" - param_labels(semobj) -> Vector{Symbol} - -Return the vector of parameter labels (in the same order as [`params`](@ref)). -""" -param_labels(spec::SemSpecification) = spec.param_labels - """ vars(semobj) -> Vector{Symbol} @@ -71,14 +64,22 @@ function latent_vars end latent_vars(spec::SemSpecification) = error("latent_vars(spec::$(typeof(spec))) is not implemented") +""" + param_labels(semobj) -> Vector{Symbol} + +Return the vector of parameter labels (in the same order as [`params`](@ref)). +""" +param_labels(spec::SemSpecification) = spec.param_labels + + """ `ParameterTable`s contain the specification of a structural equation model. # Constructor - (1) ParameterTable(;graph, observed_vars, latent_vars, ...) + (1) ParameterTable(graph; observed_vars, latent_vars, ...) - (2) ParameterTable(ram_matrices) + (2) ParameterTable(ram_matrices; ...) Return a `ParameterTable` constructed from (1) a graph or (2) RAM matrices. @@ -89,7 +90,7 @@ Return a `ParameterTable` constructed from (1) a graph or (2) RAM matrices. - `ram_matrices::RAMMatrices`: a `RAMMatrices` object # Examples -See the online documentation on [Model specification](@ref) and the [ParameterTable interface](@ref). +See the online documentation on [Model specification](@ref) and the [Graph interface](@ref). # Extended help ## Additional keyword arguments @@ -102,7 +103,7 @@ function ParameterTable end # Constructor - (1) EnsembleParameterTable(;graph, observed_vars, latent_vars, groups) + (1) EnsembleParameterTable(graph; observed_vars, latent_vars, groups) (2) EnsembleParameterTable(ps::Pair...; param_labels = nothing) @@ -125,9 +126,9 @@ function EnsembleParameterTable end # Constructor - (1) RAMMatrices(partable::ParameterTable) + (1) RAMMatrices(partable::ParameterTable; param_labels = nothing) - (2) RAMMatrices(;A, S, F, M = nothing, param_labels, vars) + (2) RAMMatrices(;A, S, F, M = nothing, param_labels, vars = nothing) (3) RAMMatrices(partable::EnsembleParameterTable) diff --git a/src/implied/RAM/generic.jl b/src/implied/RAM/generic.jl index 301c455e9..fd2ef61b5 100644 --- a/src/implied/RAM/generic.jl +++ b/src/implied/RAM/generic.jl @@ -6,8 +6,7 @@ Model implied covariance and means via RAM notation. # Constructor - RAM(; - specification, + RAM(;specification, meanstructure = false, gradient = true, kwargs...) @@ -19,9 +18,6 @@ Model implied covariance and means via RAM notation. # Extended help -## Implementation -Subtype of `SemImplied`. - ## RAM notation The model implied covariance matrix is computed as @@ -37,33 +33,32 @@ and for models with a meanstructure, the model implied means are computed as - `param_labels(::RAM) `-> vector of parameter labels - `nparams(::RAM)` -> number of parameters -- `Σ(::RAM)` -> model implied covariance matrix -- `μ(::RAM)` -> model implied mean vector +- `ram.Σ` -> model implied covariance matrix +- `ram.μ` -> model implied mean vector RAM matrices for the current parameter values: -- `A(::RAM)` -- `S(::RAM)` -- `F(::RAM)` -- `M(::RAM)` +- `ram.A` +- `ram.S` +- `ram.F` +- `ram.M` Jacobians of RAM matrices w.r.t to the parameter vector `θ` -- `∇A(::RAM)` -> ``∂vec(A)/∂θᵀ`` -- `∇S(::RAM)` -> ``∂vec(S)/∂θᵀ`` -- `∇M(::RAM)` = ``∂M/∂θᵀ`` +- `ram.∇A` -> ``∂vec(A)/∂θᵀ`` +- `ram.∇S` -> ``∂vec(S)/∂θᵀ`` +- `ram.∇M` = ``∂M/∂θᵀ`` Vector of indices of each parameter in the respective RAM matrix: -- `A_indices(::RAM)` -- `S_indices(::RAM)` -- `M_indices(::RAM)` +- `ram.A_indices` +- `ram.S_indices` +- `ram.M_indices` Additional interfaces -- `F⨉I_A⁻¹(::RAM)` -> ``F(I-A)^{-1}`` -- `F⨉I_A⁻¹S(::RAM)` -> ``F(I-A)^{-1}S`` -- `I_A(::RAM)` -> ``I-A`` -- `has_meanstructure(::RAM)` -> `Val{Bool}` does the model have a meanstructure? +- `ram.F⨉I_A⁻¹` -> ``F(I-A)^{-1}`` +- `ram.F⨉I_A⁻¹S` -> ``F(I-A)^{-1}S`` +- `ram.I_A` -> ``I-A`` Only available in gradient! calls: -- `I_A⁻¹(::RAM)` -> ``(I-A)^{-1}`` +- `ram.I_A⁻¹` -> ``(I-A)^{-1}`` """ mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, M1, M2, M3, M4, S1, S2, S3} <: SemImplied meanstruct::MS @@ -97,7 +92,6 @@ end function RAM(; specification::SemSpecification, - #vech = false, gradient_required = true, meanstructure = false, kwargs..., diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index eff193c17..9634bfa89 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -6,7 +6,8 @@ Subtype of `SemImplied` that implements the RAM notation with symbolic precomput # Constructor - RAMSymbolic(;specification, + RAMSymbolic(; + specification, vech = false, gradient = true, hessian = false, @@ -25,32 +26,25 @@ Subtype of `SemImplied` that implements the RAM notation with symbolic precomput # Extended help -## Implementation -Subtype of `SemImplied`. - ## Interfaces - `param_labels(::RAMSymbolic) `-> vector of parameter ids - `nparams(::RAMSymbolic)` -> number of parameters -- `Σ(::RAMSymbolic)` -> model implied covariance matrix -- `μ(::RAMSymbolic)` -> model implied mean vector +- `ram.Σ` -> model implied covariance matrix +- `ram.μ` -> model implied mean vector Jacobians (only available in gradient! calls) -- `∇Σ(::RAMSymbolic)` -> ``∂vec(Σ)/∂θᵀ`` -- `∇μ(::RAMSymbolic)` -> ``∂μ/∂θᵀ`` +- `ram.∇Σ` -> ``∂vec(Σ)/∂θᵀ`` +- `ram.∇μ` -> ``∂μ/∂θᵀ`` -- `∇Σ_function(::RAMSymbolic)` -> function to overwrite `∇Σ` in place, - i.e. `∇Σ_function(∇Σ, θ)`. Normally, you do not want to use this but simply - query `∇Σ(::RAMSymbolic)`. +- `ram.∇Σ_function` -> function to overwrite `∇Σ` in place, + i.e. `∇Σ_function(∇Σ, θ)`. Typically, you do not want to use this but simply + query `ram.∇Σ`. Hessians -The computation of hessians is more involved, and uses the "chain rule for -hessian matrices". -Therefore, we desribe it at length in the mathematical appendix of the online documentation, -and the relevant interfaces are omitted here. - -Additional interfaces -- `has_meanstructure(::RAMSymbolic)` -> `Val{Bool}` does the model have a meanstructure? +The computation of hessians is more involved. +Therefore, we desribe it in the online documentation, +and the respective interfaces are omitted here. ## RAM notation The model implied covariance matrix is computed as diff --git a/src/implied/abstract.jl b/src/implied/abstract.jl index af51440c6..d4868d746 100644 --- a/src/implied/abstract.jl +++ b/src/implied/abstract.jl @@ -1,4 +1,3 @@ - # vars and params API methods for SemImplied vars(implied::SemImplied) = vars(implied.ram_matrices) observed_vars(implied::SemImplied) = observed_vars(implied.ram_matrices) diff --git a/src/implied/empty.jl b/src/implied/empty.jl index 3b0292e73..82a6c9469 100644 --- a/src/implied/empty.jl +++ b/src/implied/empty.jl @@ -19,11 +19,8 @@ model per group and an additional model with `ImpliedEmpty` and `SemRidge` for t # Extended help ## Interfaces -- `param_labels(::RAMSymbolic) `-> Vector of parameter labels -- `nparams(::RAMSymbolic)` -> Number of parameters - -## Implementation -Subtype of `SemImplied`. +- `param_labels(::ImpliedEmpty) `-> Vector of parameter labels +- `nparams(::ImpliedEmpty)` -> Number of parameters """ struct ImpliedEmpty{A, B, C} <: SemImplied hessianeval::A diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index ca23ded97..ea8da6b37 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -19,10 +19,6 @@ my_fiml = SemFIML(observed = my_observed, specification = my_parameter_table) # Interfaces Analytic gradients are available. - -# Extended help -## Implementation -Subtype of `SemLossFunction`. """ mutable struct SemFIML{INV, C, L, O, M, IM, I, T, W} <: SemLossFunction hessianeval::ExactHessian diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index d14af648c..ec5eb997c 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -21,11 +21,8 @@ my_ml = SemML(observed = my_observed) ``` # Interfaces -Analytic gradients are available, and for models without a meanstructure, also analytic hessians. - -# Extended help -## Implementation -Subtype of `SemLossFunction`. +Analytic gradients are available, and for models without a meanstructure +and RAMSymbolic implied type, also analytic hessians. """ struct SemML{HE <: HessianEval, INV, M, M2} <: SemLossFunction hessianeval::HE diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 0fe2c9b3c..dd5be4874 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -5,6 +5,7 @@ ############################################################################################ """ Weighted least squares estimation. +At the moment only available with the `RAMSymbolic` implied type. # Constructor @@ -32,11 +33,7 @@ my_wls = SemWLS(observed = my_observed) ``` # Interfaces -Analytic gradients are available, and for models without a meanstructure, also analytic hessians. - -# Extended help -## Implementation -Subtype of `SemLossFunction`. +Analytic gradients are available, and for models without a meanstructure also analytic hessians. """ struct SemWLS{HE <: HessianEval, Vt, St, C} <: SemLossFunction hessianeval::HE @@ -124,7 +121,7 @@ function evaluate!( if issparse(∇σ) gradient .= (σ₋' * V * ∇σ)' else # save one allocation - mul!(gradient, σ₋' * V, ∇σ) # actually transposed, but should be fine for vectors + mul!(gradient, σ₋' * V, ∇σ) end gradient .*= -2 end diff --git a/src/loss/constant/constant.jl b/src/loss/constant/constant.jl index cb5157346..3aed5e27c 100644 --- a/src/loss/constant/constant.jl +++ b/src/loss/constant/constant.jl @@ -20,10 +20,6 @@ Constant loss term. Can be used for comparability to other packages. # Interfaces Analytic gradients and hessians are available. - -# Extended help -## Implementation -Subtype of `SemLossFunction`. """ struct SemConstant{C} <: SemLossFunction hessianeval::ExactHessian diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index aee521624..90cbcc231 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -24,10 +24,6 @@ my_ridge = SemRidge(;α_ridge = 0.02, which_ridge = [:λ₁, :λ₂, :ω₂₃], # Interfaces Analytic gradients and hessians are available. - -# Extended help -## Implementation -Subtype of `SemLossFunction`. """ struct SemRidge{P, W1, W2, GT, HT} <: SemLossFunction hessianeval::ExactHessian diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index 4aafe4235..06f39329f 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -95,7 +95,8 @@ function evaluate!(objective, gradient, hessian, model::AbstractSemSingle, param end ############################################################################################ -# methods for SemFiniteDiff (approximate gradient and hessian with finite differences of objective) +# methods for SemFiniteDiff +# (approximate gradient and hessian with finite differences of objective) ############################################################################################ function evaluate!(objective, gradient, hessian, model::SemFiniteDiff, params) @@ -157,14 +158,6 @@ function evaluate!(objective, gradient, hessian, ensemble::SemEnsemble, params) return objective end -# throw an error by default if gradient! and hessian! are not implemented - -#= gradient!(lossfun::SemLossFunction, par, model) = - throw(ArgumentError("gradient for $(typeof(lossfun).name.wrapper) is not available")) - -hessian!(lossfun::SemLossFunction, par, model) = - throw(ArgumentError("hessian for $(typeof(lossfun).name.wrapper) is not available")) =# - ############################################################################################ # Documentation ############################################################################################ diff --git a/src/observed/EM.jl b/src/observed/EM.jl index beac45ca8..46d0622be 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -3,16 +3,32 @@ ############################################################################################ # An EM Algorithm for MVN-distributed Data with missing values -# Adapted from supplementary Material to the book Machine Learning: A Probabilistic Perspective -# Copyright (2010) Kevin Murphy and Matt Dunham -# found at https://github.com/probml/pmtk3/blob/master/toolbox/BasicModels/gauss/sub/gaussMissingFitEm.m -# and at https://github.com/probml/pmtk3/blob/master/toolbox/Algorithms/optimization/emAlgo.m +# Adapted from https://github.com/probml/pmtk3, licensed as +#= The MIT License -# what about random restarts? +Copyright (2010) Kevin Murphy and Matt Dunham + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. =# # outer function --------------------------------------------------------------------------- """ - em_mvn(; + em_mvn!(; observed::SemObservedMissing, start_em = start_em_observed, max_iter_em = 100, @@ -22,7 +38,7 @@ Estimates the covariance matrix and mean vector of the normal distribution via expectation maximization for `observed`. Overwrites the statistics stored in `observed`. """ -function em_mvn( +function em_mvn!( observed::SemObservedMissing; start_em = start_em_observed, max_iter_em = 100, @@ -32,7 +48,7 @@ function em_mvn( nvars = nobserved_vars(observed) nsamps = nsamples(observed) - # preallocate stuff? + # preallocate stuff 𝔼x_pre = zeros(nvars) 𝔼xxᵀ_pre = zeros(nvars, nvars) @@ -45,9 +61,6 @@ function em_mvn( end end - # ess = 𝔼x, 𝔼xxᵀ, ismissing, missingRows, nsamps - # estepFn = (em_model, data) -> estep(em_model, data, EXsum, EXXsum, ismissing, missingRows, nsamps) - # initialize em_model = start_em(observed; kwargs...) em_model_prev = EmMVNModel(zeros(nvars, nvars), zeros(nvars), false) @@ -65,13 +78,11 @@ function em_mvn( @warn "EM Algorithm for MVN missing data did not converge. Likelihood for FIML is not interpretable. Maybe try passing different starting values via 'start_em = ...' " elseif iter > 1 - # done = isapprox(ll, ll_prev; rtol = rtol) done = isapprox(em_model_prev.μ, em_model.μ; rtol = rtol_em) & isapprox(em_model_prev.Σ, em_model.Σ; rtol = rtol_em) end - # print("$iter \n") iter = iter + 1 em_model_prev.μ, em_model_prev.Σ = em_model.μ, em_model.Σ end @@ -135,23 +146,7 @@ end function em_mvn_Mstep!(em_model, nsamples, 𝔼x, 𝔼xxᵀ) em_model.μ = 𝔼x / nsamples Σ = Symmetric(𝔼xxᵀ / nsamples - em_model.μ * em_model.μ') - - # ridge Σ - # while !isposdef(Σ) - # Σ += 0.5I - # end - em_model.Σ = Σ - - # diagonalization - #if !isposdef(Σ) - # print("Matrix not positive definite") - # em_model.Σ .= 0 - # em_model.Σ[diagind(em_model.Σ)] .= diag(Σ) - #else - # em_model.Σ = Σ - #end - return nothing end @@ -178,7 +173,6 @@ function start_em_simple(observed::SemObservedMissing; kwargs...) μ = zeros(nvars) Σ = rand(nvars, nvars) Σ = Σ * Σ' - # Σ = Matrix(1.0I, nvars, nvars) return EmMVNModel(Σ, μ, false) end diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index bb92ea12e..cf5000e4f 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -48,7 +48,7 @@ end # function to prepare input data shared by SemObserved implementations # returns tuple of -# 1) the matrix of data +# 1) the data matrix # 2) the observed variable symbols that match matrix columns # 3) the permutation of the original observed_vars (nothing if no reordering) # If observed_vars is not specified, the vars order is taken from the specification. diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 221ef5ca3..f81fe8e57 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -9,10 +9,9 @@ const SemObservedCovariance{S} = SemObservedData{Nothing, S} SemObservedCovariance(; specification, obs_cov, - obs_colnames = nothing, - meanstructure = false, + nsamples, obs_mean = nothing, - nsamples::Integer, + observed_vars = nothing, kwargs...) Construct [`SemObserved`](@ref) without providing the observations data, @@ -21,12 +20,13 @@ but with the covariations (`obs_cov`) and the means (`obs_means`) of the observe Returns [`SemObservedCovariance`](@ref) object. # Arguments -- `obs_cov`: pre-computed covariations of the observed variables +- `obs_cov`: pre-computed covariance matrix of the observed variables +- `nsamples::Integer`: number of samples (observed data points) used to compute `obs_cov` and `obs_means`, + used for calculating fit statistics - `obs_mean`: optional pre-computed means of the observed variables - `observed_vars::AbstractVector`: IDs of the observed variables (rows and columns of the `obs_cov` matrix) - `specification`: optional SEM specification ([`SemSpecification`](@ref)) -- `nsamples::Number`: number of samples (observed data points) used to compute `obs_cov` and `obs_means` - necessary for calculating fit statistics + """ function SemObservedCovariance(; obs_cov::AbstractMatrix, diff --git a/src/observed/data.jl b/src/observed/data.jl index b6ddaa43d..7ba38edf5 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -10,9 +10,9 @@ For observed data without missings. kwargs...) # Arguments -- `specification`: optional SEM specification ([`SemSpecification`](@ref)) - `data`: observed data -- *DataFrame* or *Matrix* - `observed_vars::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) +- `specification`: optional SEM specification ([`SemSpecification`](@ref)) # Extended help ## Interfaces @@ -20,11 +20,8 @@ For observed data without missings. - `nobserved_vars(::SemObservedData)` -> number of observed (manifested) variables - `samples(::SemObservedData)` -> observed data -- `obs_cov(::SemObservedData)` -> observed.obs_cov -- `obs_mean(::SemObservedData)` -> observed.obs_mean - -## Implementation -Subtype of `SemObserved` +- `obs_cov(::SemObservedData)` -> observed covariance matrix +- `obs_mean(::SemObservedData)` -> observed mean vector """ struct SemObservedData{D <: Union{Nothing, AbstractMatrix}, S <: Number} <: SemObserved data::D @@ -48,10 +45,6 @@ function SemObservedData(; return SemObservedData(data, obs_vars, obs_cov, vec(obs_mean), size(data, 1)) end -############################################################################################ -### Recommended methods -############################################################################################ - ############################################################################################ ### additional methods ############################################################################################ diff --git a/src/observed/missing.jl b/src/observed/missing.jl index cf699252e..ac8b7ea5f 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -25,9 +25,9 @@ For observed data with missing values. kwargs...) # Arguments -- `specification`: optional SEM model specification ([`SemSpecification`](@ref)) - `data`: observed data - `observed_vars::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) +- `specification`: optional SEM model specification ([`SemSpecification`](@ref)) # Extended help ## Interfaces @@ -35,10 +35,14 @@ For observed data with missing values. - `nobserved_vars(::SemObservedMissing)` -> number of observed variables - `samples(::SemObservedMissing)` -> data matrix (contains both measured and missing values) -- `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via expectation maximization -## Implementation -Subtype of `SemObserved` +## Expectation maximization +`em_mvn!(::SemObservedMissing)` can be called to fit a covariance matrix and mean vector to the data +using an expectation maximization (EM) algorithm under the assumption of multivariate normality. +After, the following methods are available: +- `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via EM +- `obs_cov(::SemObservedData)` -> EM covariance matrix +- `obs_mean(::SemObservedData)` -> EM mean vector """ struct SemObservedMissing{T <: Real, S <: Real, E <: EmMVNModel} <: SemObserved data::Matrix{Union{T, Missing}} @@ -91,10 +95,6 @@ function SemObservedMissing(; ) end -############################################################################################ -### Recommended methods -############################################################################################ - ############################################################################################ ### Additional methods ############################################################################################ diff --git a/src/optimizer/Empty.jl b/src/optimizer/Empty.jl index 45a20db55..1bf0c30ac 100644 --- a/src/optimizer/Empty.jl +++ b/src/optimizer/Empty.jl @@ -8,12 +8,6 @@ an optimizer part. # Constructor SemOptimizerEmpty() - -# Extended help - -## Implementation - -Subtype of `SemOptimizer`. """ struct SemOptimizerEmpty <: SemOptimizer{:Empty} end diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index 2487b7c52..3b1e98842 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -20,11 +20,17 @@ the online documentation on [Starting values](@ref). # Examples ```julia -fit( - my_model; +fit(my_model; start_val = start_simple, start_covariances_latent = 0.5) ``` + +```julia +using Optim + +fit(my_model; + algorithm = BFGS()) +``` """ function fit(optim::SemOptimizer, model::AbstractSem; start_val = nothing, kwargs...) start_params = prepare_start_params(start_val, model; kwargs...) @@ -49,6 +55,10 @@ prepare_start_params(start_val::Nothing, model::AbstractSemSingle; kwargs...) = prepare_start_params(start_val::Nothing, model::AbstractSem; kwargs...) = start_simple(model; kwargs...) +# first argument is a function +prepare_start_params(start_val, model::AbstractSem; kwargs...) = + start_val(model; kwargs...) + function prepare_start_params(start_val::AbstractVector, model::AbstractSem; kwargs...) (length(start_val) == nparams(model)) || throw( DimensionMismatch( diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 8f5404bc2..bd57942d8 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -12,11 +12,11 @@ Connects to `Optim.jl` as the optimization backend. SemOptimizerOptim(; algorithm = LBFGS(), - options = Optim.Options(;f_tol = 1e-10, x_tol = 1.5e-8), + options = Optim.Options(;f_reltol = 1e-10, x_abstol = 1.5e-8), kwargs...) # Arguments -- `algorithm`: optimization algorithm. +- `algorithm`: optimization algorithm from `Optim.jl` - `options::Optim.Options`: options for the optimization algorithm # Usage @@ -67,7 +67,7 @@ SemOptimizer{:Optim}(args...; kwargs...) = SemOptimizerOptim(args...; kwargs...) SemOptimizerOptim(; algorithm = LBFGS(), - options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), + options = Optim.Options(;f_reltol = 1e-10, x_abstol = 1.5e-8), kwargs..., ) = SemOptimizerOptim(algorithm, options) @@ -110,7 +110,7 @@ function fit( upper_bounds::Union{AbstractVector, AbstractDict, Nothing} = nothing, lower_bound = -Inf, upper_bound = Inf, - variance_lower_bound::Number = 0.0, + variance_lower_bound::Number = -Inf, variance_upper_bound::Number = Inf, kwargs..., ) diff --git a/src/package_extensions/SEMNLOptExt.jl b/src/package_extensions/SEMNLOptExt.jl index 69721ac94..64c4cff04 100644 --- a/src/package_extensions/SEMNLOptExt.jl +++ b/src/package_extensions/SEMNLOptExt.jl @@ -49,10 +49,6 @@ see [Constrained optimization](@ref) in our online documentation. - `local_options(::SemOptimizerNLopt)` - `equality_constraints(::SemOptimizerNLopt)` - `inequality_constraints(::SemOptimizerNLopt)` - -## Implementation - -Subtype of `SemOptimizer`. """ struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} algorithm::A diff --git a/src/package_extensions/SEMProximalOptExt.jl b/src/package_extensions/SEMProximalOptExt.jl index 5d4007504..ad4c2da2a 100644 --- a/src/package_extensions/SEMProximalOptExt.jl +++ b/src/package_extensions/SEMProximalOptExt.jl @@ -1,5 +1,6 @@ """ Connects to `ProximalAlgorithms.jl` as the optimization backend. +Can be used for regularized SEM, for a tutorial see the online docs on [Regularization](@ref). # Constructor @@ -11,8 +12,13 @@ Connects to `ProximalAlgorithms.jl` as the optimization backend. # Arguments - `algorithm`: optimization algorithm. -- `operator_g`: gradient of the objective function -- `operator_h`: optional hessian of the objective function +- `operator_g`: proximal operator (e.g., regularization penalty) +- `operator_h`: optional second proximal operator + +# Usage +All algorithms and operators from `ProximalAlgorithms.jl` are available, +for more information see the online docs on [Regularization](@ref) and +the documentation of `ProximalAlgorithms.jl` / `ProximalOperators.jl`. """ mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} algorithm::A diff --git a/src/types.jl b/src/types.jl index 64a4acbac..44d472ebf 100644 --- a/src/types.jl +++ b/src/types.jl @@ -106,7 +106,7 @@ abstract type SemObserved end """ Supertype of all objects that can serve as the implied field of a SEM. -Computed model-implied values that should be compared with the observed data to find parameter estimates, +Computes model-implied values that should be compared with the observed data to find parameter estimates, e. g. the model implied covariance or mean. If you would like to implement a different notation, e.g. LISREL, you should implement a subtype of SemImplied. """ @@ -168,7 +168,7 @@ end # ensemble models ############################################################################################ """ - (1) SemEnsemble(models..., weights = nothing, kwargs...) + (1) SemEnsemble(models...; weights = nothing, kwargs...) (2) SemEnsemble(;specification, data, groups, column = :group, kwargs...) @@ -189,6 +189,8 @@ Returns a SemEnsemble with fields - `sems::Tuple`: `AbstractSem`s. - `weights::Vector`: Weights for each model. - `param_labels::Vector`: Stores parameter labels and their position. + +For instructions on multigroup models, see the online documentation. """ struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, I} <: AbstractSemCollection n::N diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index a3e426cbc..a7b4cec9a 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -70,7 +70,7 @@ objective!(model_ml, true_val) optimizer = SemOptimizerOptim( BFGS(; linesearch = BackTracking(order = 3), alphaguess = InitialHagerZhang()),# m = 100), - Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), + Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), ) solution_ml = fit(optimizer, model_ml) diff --git a/test/unit_tests/bootstrap.jl b/test/unit_tests/bootstrap.jl deleted file mode 100644 index a2d5b6832..000000000 --- a/test/unit_tests/bootstrap.jl +++ /dev/null @@ -1,5 +0,0 @@ -solution_ml = fit(model_ml) -bs = se_bootstrap(solution_ml; n_boot = 20) - -update_se_hessian!(partable, solution_ml) -update_partable!(partable, solution_ml, bs, :se_boot) diff --git a/test/unit_tests/partable_dataframe.jl b/test/unit_tests/partable_dataframe.jl deleted file mode 100644 index 7786e2ec1..000000000 --- a/test/unit_tests/partable_dataframe.jl +++ /dev/null @@ -1,3 +0,0 @@ -# Convert PramaterTable to DataFrame - -# Convert EnsembleParameterTable to DataFrame diff --git a/test/unit_tests/ridge_id.jl b/test/unit_tests/ridge_id.jl deleted file mode 100644 index 13366bf01..000000000 --- a/test/unit_tests/ridge_id.jl +++ /dev/null @@ -1,8 +0,0 @@ -model_ridge_id = Sem( - specification = spec, - data = dat, - loss = (SemML, SemRidge), - α_ridge = 0.001, - which_ridge = [:x16, :x17, :x18, :x19, :x20], - optimizer = semoptimizer, -) diff --git a/test/unit_tests/sorting.jl b/test/unit_tests/sorting.jl deleted file mode 100644 index 3c61e13c4..000000000 --- a/test/unit_tests/sorting.jl +++ /dev/null @@ -1,17 +0,0 @@ -############################################################################ -### test variables sorting -############################################################################ - -sort_vars!(partable) - -model_ml_sorted = Sem(specification = partable, data = dat) - -@testset "graph sorting" begin - @test model_ml_sorted.implied.I_A isa LowerTriangular -end - -@testset "ml_solution_sorted" begin - solution_ml_sorted = fit(model_ml_sorted) - update_estimate!(partable, solution_ml_sorted) - @test test_estimates(par_ml, partable, 0.01) -end diff --git a/test/unit_tests/start_val.jl b/test/unit_tests/start_val.jl deleted file mode 100644 index 8b1378917..000000000 --- a/test/unit_tests/start_val.jl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/test/unit_tests/unit_tests.jl b/test/unit_tests/unit_tests.jl index 883098707..7189addd4 100644 --- a/test/unit_tests/unit_tests.jl +++ b/test/unit_tests/unit_tests.jl @@ -33,11 +33,3 @@ selected_tests = isempty(ARGS) ? collect(keys(available_tests)) : ARGS end end end - -@safetestset "SemSpecification" begin - include("specification.jl") -end - -@safetestset "Sem model" begin - include("model.jl") -end From 56f05671fda1bd02be339131e343215f85e0567b Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:01:50 +0200 Subject: [PATCH 194/364] Documentation/fix docstrings (#260) Clean code, fix docstrings, fix docs --- src/frontend/specification/documentation.jl | 34 --------------------- 1 file changed, 34 deletions(-) diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl index 4de1d9061..e46620fbc 100644 --- a/src/frontend/specification/documentation.jl +++ b/src/frontend/specification/documentation.jl @@ -1,37 +1,3 @@ -param_labels(spec::SemSpecification) = spec.param_labels - -""" - vars(semobj) -> Vector{Symbol} - -Return the vector of SEM model variables (both observed and latent) -in the order specified by the model. -""" -function vars end - -vars(spec::SemSpecification) = error("vars(spec::$(typeof(spec))) is not implemented") - -""" - observed_vars(semobj) -> Vector{Symbol} - -Return the vector of SEM model observed variable in the order specified by the -model, which also should match the order of variables in [`SemObserved`](@ref). -""" -function observed_vars end - -observed_vars(spec::SemSpecification) = - error("observed_vars(spec::$(typeof(spec))) is not implemented") - -""" - latent_vars(semobj) -> Vector{Symbol} - -Return the vector of SEM model latent variable in the order specified by the -model. -""" -function latent_vars end - -latent_vars(spec::SemSpecification) = - error("latent_vars(spec::$(typeof(spec))) is not implemented") - """ vars(semobj) -> Vector{Symbol} From f595ceb6bfa21e51dbd87cbd13deed47e52414bc Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Mon, 12 May 2025 15:09:44 +0200 Subject: [PATCH 195/364] Replace observed collections (#266) * add groups field to SemEnsemble, streamline replace_observed code, add replace_observed for ensemble models, add bootstrap se for ensemble models * fix mg tests --- src/additional_functions/simulation.jl | 88 ++++++++++++------- src/frontend/fit/standard_errors/bootstrap.jl | 68 +++++++++++--- src/types.jl | 16 ++-- test/examples/multigroup/build_models.jl | 13 +++ 4 files changed, 133 insertions(+), 52 deletions(-) diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index da3e6a906..a787516b2 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -3,6 +3,8 @@ (2) replace_observed(model::AbstractSemSingle, observed; kwargs...) + (3) replace_observed(model::SemEnsemble; column = :group, weights = nothing, kwargs...) + Return a new model with swaped observed part. # Arguments @@ -10,6 +12,14 @@ Return a new model with swaped observed part. - `kwargs`: additional keyword arguments; typically includes `data` and `specification` - `observed`: Either an object of subtype of `SemObserved` or a subtype of `SemObserved` +# For SemEnsemble models: +- `column`: if a DataFrame is passed as `data = ...`, which column signifies the group? +- `weights`: how to weight the different sub-models, + defaults to number of samples per group in the new data +- `kwargs`: has to be a dict with keys equal to the group names. + For `data` can also be a DataFrame with `column` containing the group information, + and for `specification` can also be an `EnsembleParameterTable`. + # Examples See the online documentation on [Replace observed data](@ref). """ @@ -37,51 +47,28 @@ function update_observed end replace_observed(model::AbstractSemSingle; kwargs...) = replace_observed(model, typeof(observed(model)).name.wrapper; kwargs...) -# construct a new observed type -replace_observed(model::AbstractSemSingle, observed_type; kwargs...) = - replace_observed(model, observed_type(; kwargs...); kwargs...) - -replace_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = - replace_observed( - model, - observed(model), - implied(model), - loss(model), - new_observed; - kwargs..., - ) - -function replace_observed( - model::AbstractSemSingle, - old_observed, - implied, - loss, - new_observed::SemObserved; - kwargs..., -) +function replace_observed(model::AbstractSemSingle, observed_type; kwargs...) + new_observed = observed_type(;kwargs...) kwargs = Dict{Symbol, Any}(kwargs...) # get field types kwargs[:observed_type] = typeof(new_observed) - kwargs[:old_observed_type] = typeof(old_observed) - kwargs[:implied_type] = typeof(implied) - kwargs[:loss_types] = [typeof(lossfun) for lossfun in loss.functions] + kwargs[:old_observed_type] = typeof(model.observed) + kwargs[:implied_type] = typeof(model.implied) + kwargs[:loss_types] = [typeof(lossfun) for lossfun in model.loss.functions] # update implied - implied = update_observed(implied, new_observed; kwargs...) - kwargs[:implied] = implied - kwargs[:nparams] = nparams(implied) + new_implied = update_observed(model.implied, new_observed; kwargs...) + kwargs[:implied] = new_implied + kwargs[:nparams] = nparams(new_implied) # update loss - loss = update_observed(loss, new_observed; kwargs...) - kwargs[:loss] = loss - - #new_implied = update_observed(model.implied, new_observed; kwargs...) + new_loss = update_observed(model.loss, new_observed; kwargs...) return Sem( new_observed, - update_observed(model.implied, new_observed; kwargs...), - update_observed(model.loss, new_observed; kwargs...), + new_implied, + new_loss ) end @@ -92,6 +79,39 @@ function update_observed(loss::SemLoss, new_observed; kwargs...) return SemLoss(new_functions, loss.weights) end + +function replace_observed( + emodel::SemEnsemble; + column = :group, + weights = nothing, + kwargs..., +) + kwargs = Dict{Symbol, Any}(kwargs...) + # allow for EnsembleParameterTable to be passed as specification + if haskey(kwargs, :specification) && isa(kwargs[:specification], EnsembleParameterTable) + kwargs[:specification] = convert(Dict{Symbol, RAMMatrices}, kwargs[:specification]) + end + # allow for DataFrame with group variable "column" to be passed as new data + if haskey(kwargs, :data) && isa(kwargs[:data], DataFrame) + kwargs[:data] = Dict( + group => select( + filter( + r -> r[column] == group, + kwargs[:data]), + Not(column)) for group in emodel.groups) + end + # update each model for new data + models = emodel.sems + new_models = Tuple( + replace_observed(m; group_kwargs(g, kwargs)...) for (m, g) in zip(models, emodel.groups) + ) + return SemEnsemble(new_models...; weights = weights, groups = emodel.groups) +end + +function group_kwargs(g, kwargs) + return Dict(k => kwargs[k][g] for k in keys(kwargs)) +end + ############################################################################################ # simulate data ############################################################################################ diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index 4589dc020..4b3e302bb 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -2,7 +2,6 @@ se_bootstrap(sem_fit::SemFit; n_boot = 3000, data = nothing, kwargs...) Return boorstrap standard errors. -Only works for single models. # Arguments - `n_boot`: number of boostrap samples @@ -10,19 +9,12 @@ Only works for single models. - `kwargs...`: passed down to `replace_observed` """ function se_bootstrap( - semfit::SemFit; + semfit::SemFit{Mi, So, St, Mo, O}; n_boot = 3000, data = nothing, specification = nothing, kwargs..., -) - if model(semfit) isa AbstractSemCollection - throw( - ArgumentError( - "bootstrap standard errors for ensemble models are not available yet", - ), - ) - end + ) where {Mi, So, St, Mo <: AbstractSemSingle, O} if isnothing(data) data = samples(observed(model(semfit))) @@ -69,6 +61,62 @@ function se_bootstrap( return sd end +function se_bootstrap( + semfit::SemFit{Mi, So, St, Mo, O}; + n_boot = 3000, + data = nothing, + specification = nothing, + kwargs..., + ) where {Mi, So, St, Mo <: SemEnsemble, O} + + models = semfit.model.sems + groups = semfit.model.groups + + if isnothing(data) + data = Dict(g => samples(observed(m)) for (g, m) in zip(groups, models)) + end + + data = Dict(k => prepare_data_bootstrap(data[k]) for k in keys(data)) + + start = solution(semfit) + + new_solution = zero(start) + sum = zero(start) + squared_sum = zero(start) + + n_failed = 0.0 + + converged = true + + for _ in 1:n_boot + sample_data = Dict(k => bootstrap_sample(data[k]) for k in keys(data)) + new_model = replace_observed( + semfit.model; + data = sample_data, + specification = specification, + kwargs..., + ) + + new_solution .= 0.0 + + try + new_solution = solution(fit(new_model; start_val = start)) + catch + n_failed += 1 + end + + @. sum += new_solution + @. squared_sum += new_solution^2 + + converged = true + end + + n_conv = n_boot - n_failed + sd = sqrt.(squared_sum / n_conv - (sum / n_conv) .^ 2) + print("Number of nonconverged models: ", n_failed, "\n") + return sd +end + function prepare_data_bootstrap(data) return Matrix(data) end diff --git a/src/types.jl b/src/types.jl index 44d472ebf..660c1c43f 100644 --- a/src/types.jl +++ b/src/types.jl @@ -168,7 +168,7 @@ end # ensemble models ############################################################################################ """ - (1) SemEnsemble(models...; weights = nothing, kwargs...) + (1) SemEnsemble(models...; weights = nothing, groups = nothing, kwargs...) (2) SemEnsemble(;specification, data, groups, column = :group, kwargs...) @@ -192,24 +192,24 @@ Returns a SemEnsemble with fields For instructions on multigroup models, see the online documentation. """ -struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, I} <: AbstractSemCollection +struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, I, G <: Vector{Symbol}} <: AbstractSemCollection n::N sems::T weights::V param_labels::I + groups::G end # constructor from multiple models -function SemEnsemble(models...; weights = nothing, kwargs...) +function SemEnsemble(models...; weights = nothing, groups = nothing, kwargs...) n = length(models) - # default weights - if isnothing(weights) nsamples_total = sum(nsamples, models) weights = [nsamples(model) / nsamples_total for model in models] end - + # default group labels + groups = isnothing(groups) ? Symbol.(:g, 1:n) : groups # check parameters equality param_labels = SEM.param_labels(models[1]) for model in models @@ -220,7 +220,7 @@ function SemEnsemble(models...; weights = nothing, kwargs...) end end - return SemEnsemble(n, models, weights, param_labels) + return SemEnsemble(n, models, weights, param_labels, groups) end # constructor from EnsembleParameterTable and data set @@ -238,7 +238,7 @@ function SemEnsemble(; specification, data, groups, column = :group, kwargs...) model = Sem(; specification = ram_matrices, data = data_group, kwargs...) push!(models, model) end - return SemEnsemble(models...; weights = nothing, kwargs...) + return SemEnsemble(models...; weights = nothing, groups = groups, kwargs...) end param_labels(ensemble::SemEnsemble) = ensemble.param_labels diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index f6a7a230d..f5ea0b5d7 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -20,6 +20,13 @@ model_ml_multigroup2 = SemEnsemble( loss = SemML, ) +model_ml_multigroup3 = replace_observed( + model_ml_multigroup2, + column = :school, + specification = partable, + data = dat, +) + # gradients @testset "ml_gradients_multigroup" begin test_gradient(model_ml_multigroup, start_test; atol = 1e-9) @@ -46,6 +53,12 @@ end ) end +@testset "replace_observed_multigroup" begin + sem_fit_1 = fit(semoptimizer, model_ml_multigroup) + sem_fit_2 = fit(semoptimizer, model_ml_multigroup3) + @test sem_fit_1.solution ≈ sem_fit_2.solution +end + @testset "fitmeasures/se_ml" begin solution_ml = fit(model_ml_multigroup) test_fitmeasures( From a0197a38687dce48a1818aec9aa21f1f39ec73df Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Tue, 30 Dec 2025 18:24:47 +0100 Subject: [PATCH 196/364] explicitely import types for extensions (#291) --- ext/SEMNLOptExt/SEMNLOptExt.jl | 2 +- ext/SEMProximalOptExt/SEMProximalOptExt.jl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ext/SEMNLOptExt/SEMNLOptExt.jl b/ext/SEMNLOptExt/SEMNLOptExt.jl index c79fc2b86..bf905e3ac 100644 --- a/ext/SEMNLOptExt/SEMNLOptExt.jl +++ b/ext/SEMNLOptExt/SEMNLOptExt.jl @@ -1,7 +1,7 @@ module SEMNLOptExt using StructuralEquationModels, NLopt -using StructuralEquationModels: SemOptimizerNLopt, NLoptConstraint +import StructuralEquationModels: SemOptimizerNLopt, NLoptConstraint SEM = StructuralEquationModels diff --git a/ext/SEMProximalOptExt/SEMProximalOptExt.jl b/ext/SEMProximalOptExt/SEMProximalOptExt.jl index 192944fef..04be35cb7 100644 --- a/ext/SEMProximalOptExt/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt/SEMProximalOptExt.jl @@ -1,8 +1,9 @@ module SEMProximalOptExt using StructuralEquationModels +using StructuralEquationModels: print_type_name, print_field_types using ProximalAlgorithms -using StructuralEquationModels: SemOptimizerProximal, print_type_name, print_field_types +import StructuralEquationModels: SemOptimizerProximal SEM = StructuralEquationModels From e4b78b55d77e40750d3de265b5503d09063382b6 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 15 Jan 2026 15:47:26 +0100 Subject: [PATCH 197/364] workaround for StenoGraph import --- src/StructuralEquationModels.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index f6068dc50..46692bd5b 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -18,7 +18,7 @@ using LinearAlgebra, import StatsAPI: params, coef, coefnames, dof, fit, nobs, coeftable -export StenoGraphs, @StenoGraph, meld +export StenoGraphs, @StenoGraph, meld, SimpleNode const SEM = StructuralEquationModels From 27bdbce756707b4fd77a3800d95e1c2de29ee239 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 15 Jan 2026 22:19:45 +0100 Subject: [PATCH 198/364] throw sensible errors when missings or meanstructure are used incorrectly --- src/implied/RAM/generic.jl | 7 ++----- src/implied/RAM/symbolic.jl | 2 ++ src/implied/abstract.jl | 14 ++++++++++++++ src/loss/ML/FIML.jl | 9 ++++++++- src/loss/ML/ML.jl | 15 +++++++++++++++ src/loss/WLS/WLS.jl | 21 +++++++++++++++++++++ src/observed/data.jl | 14 ++++++++++++++ 7 files changed, 76 insertions(+), 6 deletions(-) diff --git a/src/implied/RAM/generic.jl b/src/implied/RAM/generic.jl index fd2ef61b5..4ec58b4eb 100644 --- a/src/implied/RAM/generic.jl +++ b/src/implied/RAM/generic.jl @@ -98,6 +98,8 @@ function RAM(; ) ram_matrices = convert(RAMMatrices, specification) + check_meanstructure_specification(meanstructure, ram_matrices) + # get dimensions of the model n_par = nparams(ram_matrices) n_obs = nobserved_vars(ram_matrices) @@ -126,11 +128,6 @@ function RAM(; # μ if meanstructure MS = HasMeanStruct - !isnothing(ram_matrices.M) || throw( - ArgumentError( - "You set `meanstructure = true`, but your model specification contains no mean parameters.", - ), - ) M_pre = materialize(ram_matrices.M, rand_params) ∇M = gradient_required ? sparse_gradient(ram_matrices.M) : nothing μ = zeros(n_obs) diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index 9634bfa89..7066be1a4 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -96,6 +96,8 @@ function RAMSymbolic(; ) ram_matrices = convert(RAMMatrices, specification) + check_meanstructure_specification(meanstructure, ram_matrices) + n_par = nparams(ram_matrices) par = (Symbolics.@variables θ[1:n_par])[1] diff --git a/src/implied/abstract.jl b/src/implied/abstract.jl index d4868d746..cff4d58a4 100644 --- a/src/implied/abstract.jl +++ b/src/implied/abstract.jl @@ -31,3 +31,17 @@ function check_acyclic(A::AbstractMatrix; verbose::Bool = false) return A end end + +# Verify that the `meanstructure` argument aligns with the model specification. +function check_meanstructure_specification(meanstructure, ram_matrices) + if meanstructure & isnothing(ram_matrices.M) + throw(ArgumentError( + "You set `meanstructure = true`, but your model specification contains no mean parameters." + )) + end + if !meanstructure & !isnothing(ram_matrices.M) + throw(ArgumentError( + "If your model specification contains mean parameters, you have to set `meanstructure = true`." + )) + end +end \ No newline at end of file diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index ea8da6b37..6731b1a16 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -43,7 +43,14 @@ end ### Constructors ############################################################################################ -function SemFIML(; observed::SemObservedMissing, specification, kwargs...) +function SemFIML(; observed::SemObservedMissing, implied, specification, kwargs...) + + if implied.meanstruct isa NoMeanStruct + throw(ArgumentError( + "Full information maximum likelihood (FIML) can only be used with a meanstructure. + Did you forget to set `Sem(..., meanstructure = true)`?")) + end + inverses = [zeros(nmeasured_vars(pat), nmeasured_vars(pat)) for pat in observed.patterns] choleskys = Array{Cholesky{Float64, Array{Float64, 2}}, 1}(undef, length(inverses)) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index ec5eb997c..4c216f9c1 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -39,6 +39,21 @@ end ############################################################################################ function SemML(; observed::SemObserved, approximate_hessian::Bool = false, kwargs...) + + if observed isa SemObservedMissing + throw(ArgumentError( + "Normal maximum likelihood estimation can't be used with `SemObservedMissing`. + Use full information maximum likelihood (FIML) estimation or remove missing + values in your data. + A FIML model can be constructed with + Sem( + ..., + observed = SemObservedMissing, + loss = SemFIML, + meanstructure = true + )")) + end + obsmean = obs_mean(observed) obscov = obs_cov(observed) meandiff = isnothing(obsmean) ? nothing : copy(obsmean) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index dd5be4874..87be97282 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -51,12 +51,33 @@ SemWLS{HE}(args...) where {HE <: HessianEval} = function SemWLS(; observed, + implied, wls_weight_matrix = nothing, wls_weight_matrix_mean = nothing, approximate_hessian = false, meanstructure = false, kwargs..., ) + + if observed isa SemObservedMissing + throw(ArgumentError( + "WLS estimation can't be used with `SemObservedMissing`. + Use full information maximum likelihood (FIML) estimation or remove missing + values in your data. + A FIML model can be constructed with + Sem( + ..., + observed = SemObservedMissing, + loss = SemFIML, + meanstructure = true + )")) + end + + if !(implied isa RAMSymbolic) + throw(ArgumentError( + "WLS estimation is only available with the implied type RAMSymbolic at the moment.")) + end + nobs_vars = nobserved_vars(observed) tril_ind = filter(x -> (x[1] >= x[2]), CartesianIndices(obs_cov(observed))) s = obs_cov(observed)[tril_ind] diff --git a/src/observed/data.jl b/src/observed/data.jl index 7ba38edf5..fffeb36bd 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -38,10 +38,24 @@ function SemObservedData(; observed_var_prefix::Union{Symbol, AbstractString} = :obs, kwargs..., ) + data, obs_vars, _ = prepare_data(data, observed_vars, specification; observed_var_prefix) obs_mean, obs_cov = mean_and_cov(data, 1) + if any(ismissing.(data)) + throw(ArgumentError( + "Your dataset contains missing values. + Remove missing values or use full information maximum likelihood (FIML) estimation. + A FIML model can be constructed with + Sem( + ..., + observed = SemObservedMissing, + loss = SemFIML, + meanstructure = true + )")) + end + return SemObservedData(data, obs_vars, obs_cov, vec(obs_mean), size(data, 1)) end From db4e86e9c4efd7e44a967f835815c6a00465d9f1 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 15 Jan 2026 22:34:24 +0100 Subject: [PATCH 199/364] fix model construction unit tests --- test/unit_tests/model.jl | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index 2bf5dedaf..af4440585 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -47,14 +47,13 @@ function test_params_api(semobj, spec::SemSpecification) @test @inferred(param_labels(semobj)) == param_labels(spec) end -@testset "Sem(implied=$impliedtype, loss=$losstype)" for impliedtype in (RAM, RAMSymbolic), - losstype in (SemML, SemWLS) +@testset "Sem(implied=$impliedtype, loss=SemML)" for impliedtype in (RAM, RAMSymbolic) model = Sem( specification = ram_matrices, observed = obs, implied = impliedtype, - loss = losstype, + loss = SemML, ) @test model isa Sem @@ -69,7 +68,33 @@ end @test @inferred(loss(model)) isa SemLoss semloss = loss(model).functions[1] - @test semloss isa losstype + @test semloss isa SemML @test @inferred(nsamples(model)) == nsamples(obs) end + +@testset "Sem(implied=RAMSymbolic, loss=SemWLS)" begin + + model = Sem( + specification = ram_matrices, + observed = obs, + implied = RAMSymbolic, + loss = SemWLS, + ) + + @test model isa Sem + @test @inferred(implied(model)) isa RAMSymbolic + @test @inferred(observed(model)) isa SemObserved + + test_vars_api(model, ram_matrices) + test_params_api(model, ram_matrices) + + test_vars_api(implied(model), ram_matrices) + test_params_api(implied(model), ram_matrices) + + @test @inferred(loss(model)) isa SemLoss + semloss = loss(model).functions[1] + @test semloss isa SemWLS + + @test @inferred(nsamples(model)) == nsamples(obs) +end \ No newline at end of file From 998b04b28fe34e17412e66707029f7cfb015588c Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 30 Dec 2025 19:03:32 +0100 Subject: [PATCH 200/364] simplify update_partable! syntax --- docs/src/tutorials/constraints/constraints.md | 2 +- docs/src/tutorials/inspection/inspection.md | 4 +-- .../regularization/regularization.md | 4 +-- src/frontend/specification/ParameterTable.jl | 25 +++++++++++++------ 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index 338803cb3..3d9b3d1a1 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -162,7 +162,7 @@ As you can see, the optimizer converged (`:XTOL_REACHED`) and investigating the update_partable!( partable, :estimate_constr, - param_labels(model_fit_constrained), + model_fit_constrained, solution(model_fit_constrained), ) diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md index 170e4c075..ff572eb58 100644 --- a/docs/src/tutorials/inspection/inspection.md +++ b/docs/src/tutorials/inspection/inspection.md @@ -87,8 +87,8 @@ We can also update the `ParameterTable` object with other information via [`upda se_bs = se_bootstrap(model_fit; n_boot = 20) se_he = se_hessian(model_fit) -update_partable!(partable, :se_hessian, param_labels(model_fit), se_he) -update_partable!(partable, :se_bootstrap, param_labels(model_fit), se_bs) +update_partable!(partable, :se_hessian, model_fit, se_he) +update_partable!(partable, :se_bootstrap, model_fit, se_bs) details(partable) ``` diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index f743ac79c..bcb9b7747 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -129,7 +129,7 @@ sem_fit = fit(model) update_estimate!(partable, sem_fit) -update_partable!(partable, :estimate_lasso, param_labels(fit_lasso), solution(fit_lasso)) +update_partable!(partable, :estimate_lasso, fit_lasso, solution(fit_lasso)) details(partable) ``` @@ -167,7 +167,7 @@ fit_mixed = fit(model_mixed; engine = :Proximal, operator_g = prox_operator) Let's again compare the different results: ```@example reg -update_partable!(partable, :estimate_mixed, param_labels(fit_mixed), solution(fit_mixed)) +update_partable!(partable, :estimate_mixed, fit_mixed, solution(fit_mixed)) details(partable) ``` \ No newline at end of file diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 2af269372..5f8aff1eb 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -288,14 +288,25 @@ function update_partable!( end """ - update_partable!(partable::AbstractParameterTable, param_labels::Vector{Symbol}, params, column) - -Write parameter `values` into `column` of `partable`. - -The `param_labels` and `params` vectors define the pairs of -parameters, which are being matched to the `:param` column -of the `partable`. + (1) update_partable!(partable::AbstractParameterTable, column, fitted:SemFit, params, default = nothing) + + (2) update_partable!(partable::AbstractParameterTable, column, param_labels::Vector{Symbol}, params, default = nothing) + +Add a new column to a parameter table. +`column` is the name of the column, `params` contains the values of the new column, +and `fitted` or `param_labels` is used to match the values to the correct parameter labels. +The `default` value is used if a parameter in `partable` does not occur in `param_labels`. """ +function update_partable!( + partable::AbstractParameterTable, + column::Symbol, + fitted::SemFit, + params::AbstractVector, + default::Any = nothing, +) + update_partable!(partable, column, param_labels(fitted), params, default) +end + function update_partable!( partable::ParameterTable, column::Symbol, From 8f443e241e6b24c5f891f5015511a25b12c532ca Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 20 Jan 2026 15:35:40 -0800 Subject: [PATCH 201/364] gh/FormatCheck: update to julia-format@v4 --- .github/workflows/FormatCheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/FormatCheck.yml b/.github/workflows/FormatCheck.yml index e02e1c04c..0049fc31e 100644 --- a/.github/workflows/FormatCheck.yml +++ b/.github/workflows/FormatCheck.yml @@ -7,7 +7,7 @@ jobs: code-style: runs-on: ubuntu-latest steps: - - uses: julia-actions/julia-format@v3 + - uses: julia-actions/julia-format@v4 with: version: '1' suggestion-label: 'format-suggest' From 5d14c4e8afaa670bf743d55710b95b09823750ff Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 20 Jan 2026 15:41:59 -0800 Subject: [PATCH 202/364] gh/FormatCheck: update JuliaFormatter to v2 --- .github/workflows/FormatCheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/FormatCheck.yml b/.github/workflows/FormatCheck.yml index 0049fc31e..4be9f54d9 100644 --- a/.github/workflows/FormatCheck.yml +++ b/.github/workflows/FormatCheck.yml @@ -9,5 +9,5 @@ jobs: steps: - uses: julia-actions/julia-format@v4 with: - version: '1' + version: '2' suggestion-label: 'format-suggest' From 326c77ccaa7e7c21cc338f3ac5f3346d2f26399e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 20 Jan 2026 15:56:26 -0800 Subject: [PATCH 203/364] gh/FormatCheck: always show suggestions --- .github/workflows/FormatCheck.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/FormatCheck.yml b/.github/workflows/FormatCheck.yml index 4be9f54d9..d005ca942 100644 --- a/.github/workflows/FormatCheck.yml +++ b/.github/workflows/FormatCheck.yml @@ -10,4 +10,3 @@ jobs: - uses: julia-actions/julia-format@v4 with: version: '2' - suggestion-label: 'format-suggest' From 9f41f920548fb1f33c07b73676c85c040e2f0230 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 22 Jan 2026 14:28:06 +0100 Subject: [PATCH 204/364] update concept figures --- docs/src/assets/concept.svg | 172 +----------------------------- docs/src/assets/concept_typed.svg | 172 +----------------------------- 2 files changed, 2 insertions(+), 342 deletions(-) diff --git a/docs/src/assets/concept.svg b/docs/src/assets/concept.svg index fa222a0d9..f82e47943 100644 --- a/docs/src/assets/concept.svg +++ b/docs/src/assets/concept.svg @@ -1,171 +1 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/docs/src/assets/concept_typed.svg b/docs/src/assets/concept_typed.svg index 88a0d8566..e046819ae 100644 --- a/docs/src/assets/concept_typed.svg +++ b/docs/src/assets/concept_typed.svg @@ -1,171 +1 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + \ No newline at end of file From 8f2a10191f5529d3db0d8713a4616499ffae7794 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 22 Jan 2026 14:32:14 +0100 Subject: [PATCH 205/364] add SemOptimizerProximal to table --- docs/src/tutorials/concept.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index 035144d62..128117a09 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -21,13 +21,13 @@ So everything that can be used as the 'observed' part has to be of type `SemObse Here is an overview on the available building blocks: -|[`SemObserved`](@ref) | [`SemImplied`](@ref) | [`SemLossFunction`](@ref) | [`SemOptimizer`](@ref) | -|---------------------------------|-----------------------|---------------------------|-------------------------------| -| [`SemObservedData`](@ref) | [`RAM`](@ref) | [`SemML`](@ref) | [`SemOptimizerOptim`](@ref) | -| [`SemObservedCovariance`](@ref) | [`RAMSymbolic`](@ref) | [`SemWLS`](@ref) | [`SemOptimizerNLopt`](@ref) | -| [`SemObservedMissing`](@ref) | [`ImpliedEmpty`](@ref)| [`SemFIML`](@ref) | | -| | | [`SemRidge`](@ref) | | -| | | [`SemConstant`](@ref) | | +|[`SemObserved`](@ref) | [`SemImplied`](@ref) | [`SemLossFunction`](@ref) | [`SemOptimizer`](@ref) | +|---------------------------------|-----------------------|---------------------------|--------------------------------| +| [`SemObservedData`](@ref) | [`RAM`](@ref) | [`SemML`](@ref) | [`SemOptimizerOptim`](@ref) | +| [`SemObservedCovariance`](@ref) | [`RAMSymbolic`](@ref) | [`SemWLS`](@ref) | [`SemOptimizerNLopt`](@ref) | +| [`SemObservedMissing`](@ref) | [`ImpliedEmpty`](@ref)| [`SemFIML`](@ref) | [`SemOptimizerProximal`](@ref) | +| | | [`SemRidge`](@ref) | | +| | | [`SemConstant`](@ref) | | The rest of this page explains the building blocks for each part. First, we explain every part and give an overview on the different options that are available. After that, the [API - model parts](@ref) section serves as a reference for detailed explanations about the different options. (How to stick them together to a final model is explained in the section on [Model Construction](@ref).) From 9da45e4aa2f35c6367f2b9682c950de5f59dc883 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 27 Jan 2026 11:43:37 -0800 Subject: [PATCH 206/364] No-op code & docs cleanups (#245) * minus2ll(): cleanup method signatures * fix chi2 * fix RMSEA * p_values(): use ccdf() * RAM: don't need to copy (I-A) * EM: move code refs to docstring * fix batch_sym_inv_updates() ws * RAMSymbolic: rename _func to _eval! also remove unused _symbolic fields * md ws fixes * test/Proximal: move usings to the central file * tests: move usings in the top file * params_array: fix formatting * simulation: fix formatting * checks: fix formatting * bootstrap: fix formatting * types: fix formatting * StatsAPI: fix formatting * documentation.jl: fix formatting * EnsParTable: fix formatting * ParTable: fix formatting * generic: fix formatting * missing: fix formatting * abstract: fix formatting * optim: fix ws * test/model: fix ws * test/proximal/l0: fix ws * test/multigroup: fix ws * test/StatsAPI: fix ws * tests/unit_test_iactive: fix ws * tests/unit_tests: fix formatting * fix formattng * NLopt: fixup formatting --------- Co-authored-by: Alexey Stukalov --- docs/src/tutorials/concept.md | 4 +- docs/src/tutorials/constraints/constraints.md | 18 ++-- .../tutorials/construction/build_by_parts.md | 2 +- docs/src/tutorials/fitting/fitting.md | 20 ++-- docs/src/tutorials/meanstructure.md | 4 +- .../regularization/regularization.md | 14 +-- .../specification/graph_interface.md | 12 +-- .../specification/parameter_table.md | 2 +- .../tutorials/specification/ram_matrices.md | 26 ++--- ext/SEMNLOptExt/NLopt.jl | 2 +- src/additional_functions/helper.jl | 6 +- src/additional_functions/params_array.jl | 10 +- src/additional_functions/simulation.jl | 21 ++-- src/frontend/StatsAPI.jl | 12 +-- src/frontend/fit/fitmeasures/RMSEA.jl | 11 ++- src/frontend/fit/fitmeasures/chi2.jl | 97 ++++++++---------- src/frontend/fit/fitmeasures/minus2ll.jl | 49 ++++------ src/frontend/fit/fitmeasures/p.jl | 2 +- src/frontend/fit/standard_errors/bootstrap.jl | 8 +- .../specification/EnsembleParameterTable.jl | 2 +- src/frontend/specification/ParameterTable.jl | 33 +++---- src/frontend/specification/checks.jl | 7 +- src/frontend/specification/documentation.jl | 1 - src/implied/RAM/generic.jl | 9 +- src/implied/RAM/symbolic.jl | 98 +++++++++---------- src/loss/ML/ML.jl | 3 +- src/loss/WLS/WLS.jl | 3 +- src/observed/EM.jl | 7 +- src/optimizer/abstract.jl | 5 +- src/optimizer/optim.jl | 2 +- src/types.jl | 3 +- test/examples/helper.jl | 8 +- test/examples/multigroup/build_models.jl | 3 +- test/examples/multigroup/multigroup.jl | 8 +- .../political_democracy/constructor.jl | 3 - .../political_democracy.jl | 2 + test/examples/proximal/l0.jl | 5 +- test/examples/proximal/lasso.jl | 2 - test/examples/proximal/proximal.jl | 2 + test/examples/proximal/ridge.jl | 2 - .../recover_parameters_twofact.jl | 2 +- test/unit_tests/StatsAPI.jl | 9 +- test/unit_tests/model.jl | 1 - test/unit_tests/unit_tests.jl | 2 +- test/unit_tests/unit_tests_interactive.jl | 2 +- 45 files changed, 244 insertions(+), 300 deletions(-) diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index 035144d62..2b453925a 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -50,8 +50,8 @@ Available loss functions are - [`SemRidge`](@ref): ridge regularization ## The optimizer part aka `SemOptimizer` -The optimizer part of a model connects to the numerical optimization backend used to fit the model. -It can be used to control options like the optimization algorithm, linesearch, stopping criteria, etc. +The optimizer part of a model connects to the numerical optimization backend used to fit the model. +It can be used to control options like the optimization algorithm, linesearch, stopping criteria, etc. There are currently three available backends, [`SemOptimizerOptim`](@ref) connecting to the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) backend, [`SemOptimizerNLopt`](@ref) connecting to the [NLopt.jl](https://github.com/JuliaOpt/NLopt.jl) backend and [`SemOptimizerProximal`](@ref) connecting to [ProximalAlgorithms.jl](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl). For more information about the available options see also the tutorials about [Using Optim.jl](@ref) and [Using NLopt.jl](@ref), as well as [Constrained optimization](@ref) and [Regularization](@ref) . diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index 3d9b3d1a1..c433240a9 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -38,7 +38,7 @@ end partable = ParameterTable( graph, - latent_vars = latent_vars, + latent_vars = latent_vars, observed_vars = observed_vars) data = example_data("political_democracy") @@ -64,7 +64,7 @@ Let's introduce some constraints: (Of course those constaints only serve an illustratory purpose.) -We first need to get the indices of the respective parameters that are invoved in the constraints. +We first need to get the indices of the respective parameters that are invoved in the constraints. We can look up their labels in the output above, and retrieve their indices as ```@example constraints @@ -112,7 +112,7 @@ end ``` If the algorithm needs gradients at an iteration, it will pass the vector `gradient` that is of the same size as the parameters. -With `if length(gradient) > 0` we check if the algorithm needs gradients, and if it does, we fill the `gradient` vector with the gradients +With `if length(gradient) > 0` we check if the algorithm needs gradients, and if it does, we fill the `gradient` vector with the gradients of the constraint w.r.t. the parameters. In NLopt, vector-valued constraints are also possible, but we refer to the documentation for that. @@ -134,7 +134,7 @@ constrained_optimizer = SemOptimizerNLopt( ``` As you see, the equality constraints and inequality constraints are passed as keyword arguments, and the bounds are passed as options for the (outer) optimization algorithm. -Additionally, for equality and inequality constraints, a feasibility tolerance can be specified that controls if a solution can be accepted, even if it violates the constraints by a small amount. +Additionally, for equality and inequality constraints, a feasibility tolerance can be specified that controls if a solution can be accepted, even if it violates the constraints by a small amount. Especially for equality constraints, it is recommended to allow for a small positive tolerance. In this example, we set both tolerances to `1e-8`. @@ -142,7 +142,7 @@ In this example, we set both tolerances to `1e-8`. We have often observed that the default convergence criteria in NLopt lead to non-convergence flags. Indeed, this example does not convergence with default criteria. As you see above, we used a realively liberal absolute tolerance in the optimization parameters of 1e-4. - This should not be a problem in most cases, as the sampling variance in (almost all) structural equation models + This should not be a problem in most cases, as the sampling variance in (almost all) structural equation models should lead to uncertainty in the parameter estimates that are orders of magnitude larger. We nontheless recommend choosing a convergence criterion with care (i.e. w.r.t. the scale of your parameters), inspecting the solutions for plausibility, and comparing them to unconstrained solutions. @@ -162,14 +162,14 @@ As you can see, the optimizer converged (`:XTOL_REACHED`) and investigating the update_partable!( partable, :estimate_constr, - model_fit_constrained, - solution(model_fit_constrained), - ) + model_fit_constrained, + solution(model_fit_constrained), +) details(partable) ``` -As we can see, the constrained solution is very close to the original solution (compare the columns estimate and estimate_constr), with the difference that the constrained parameters fulfill their constraints. +As we can see, the constrained solution is very close to the original solution (compare the columns estimate and estimate_constr), with the difference that the constrained parameters fulfill their constraints. As all parameters are estimated simultaneously, it is expexted that some unconstrained parameters are also affected (e.g., the constraint on `dem60 → y2` leads to a higher estimate of the residual variance `y2 ↔ y2`). ## Using the Optim.jl backend diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 6b6b59ac9..680e28804 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -40,7 +40,7 @@ end partable = ParameterTable( graph, - latent_vars = lat_vars, + latent_vars = lat_vars, observed_vars = obs_vars) ``` diff --git a/docs/src/tutorials/fitting/fitting.md b/docs/src/tutorials/fitting/fitting.md index fff06abaa..d7353c9f9 100644 --- a/docs/src/tutorials/fitting/fitting.md +++ b/docs/src/tutorials/fitting/fitting.md @@ -7,19 +7,19 @@ model_fit = fit(model) # output -Fitted Structural Equation Model -=============================================== ---------------------- Model ------------------- +Fitted Structural Equation Model +=============================================== +--------------------- Model ------------------- -Structural Equation Model -- Loss Functions +Structural Equation Model +- Loss Functions SemML -- Fields - observed: SemObservedData - implied: RAM - optimizer: SemOptimizerOptim +- Fields + observed: SemObservedData + implied: RAM + optimizer: SemOptimizerOptim -------------- Optimization result ------------- +------------- Optimization result ------------- * Status: success diff --git a/docs/src/tutorials/meanstructure.md b/docs/src/tutorials/meanstructure.md index b2da5029a..4e6d2a36a 100644 --- a/docs/src/tutorials/meanstructure.md +++ b/docs/src/tutorials/meanstructure.md @@ -40,7 +40,7 @@ end partable = ParameterTable( graph, - latent_vars = latent_vars, + latent_vars = latent_vars, observed_vars = observed_vars) ``` @@ -78,7 +78,7 @@ end partable = ParameterTable( graph, - latent_vars = latent_vars, + latent_vars = latent_vars, observed_vars = observed_vars) ``` diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index bcb9b7747..2b2c6df30 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -2,7 +2,7 @@ ## Setup -For ridge regularization, you can simply use `SemRidge` as an additional loss function +For ridge regularization, you can simply use `SemRidge` as an additional loss function (for example, a model with the loss functions `SemML` and `SemRidge` corresponds to ridge-regularized maximum likelihood estimation). For lasso, elastic net and (far) beyond, you can load the `ProximalAlgorithms.jl` and `ProximalOperators.jl` packages alongside `StructuralEquationModels`: @@ -22,7 +22,7 @@ using StructuralEquationModels, ProximalAlgorithms, ProximalOperators ## `SemOptimizerProximal` To estimate regularized models, we provide a "building block" for the optimizer part, called `SemOptimizerProximal`. -It connects our package to the [`ProximalAlgorithms.jl`](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) optimization backend, providing so-called proximal optimization algorithms. +It connects our package to the [`ProximalAlgorithms.jl`](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) optimization backend, providing so-called proximal optimization algorithms. Those can handle, amongst other things, various forms of regularization. It can be used as @@ -32,7 +32,7 @@ SemOptimizerProximal( algorithm = ProximalAlgorithms.PANOC(), operator_g, operator_h = nothing - ) +) ``` The proximal operator (aka the regularization function) can be passed as `operator_g`. @@ -69,7 +69,7 @@ end partable = ParameterTable( graph, - latent_vars = latent_vars, + latent_vars = latent_vars, observed_vars = observed_vars ) @@ -85,7 +85,7 @@ We labeled the covariances between the items because we want to regularize those ```@example reg ind = getindex.( - [param_indices(model)], + Ref(param_indices(model)), [:cov_15, :cov_24, :cov_26, :cov_37, :cov_48, :cov_68]) ``` @@ -107,7 +107,7 @@ and use `SemOptimizerProximal`. ```@example reg optimizer_lasso = SemOptimizerProximal( operator_g = NormL1(λ) - ) +) model_lasso = Sem( specification = partable, @@ -158,7 +158,7 @@ prox_operator = SlicedSeparableSum((NormL0(20.0), NormL1(0.02), NormL0(0.0)), ([ model_mixed = Sem( specification = partable, - data = data, + data = data, ) fit_mixed = fit(model_mixed; engine = :Proximal, operator_g = prox_operator) diff --git a/docs/src/tutorials/specification/graph_interface.md b/docs/src/tutorials/specification/graph_interface.md index 75e1d1b6d..62eeef00b 100644 --- a/docs/src/tutorials/specification/graph_interface.md +++ b/docs/src/tutorials/specification/graph_interface.md @@ -1,6 +1,6 @@ # Graph interface -## Workflow +## Workflow As discussed before, when using the graph interface, you can specify your model as a graph ```julia @@ -17,7 +17,7 @@ lat_vars = ... partable = ParameterTable( graph, - latent_vars = lat_vars, + latent_vars = lat_vars, observed_vars = obs_vars) model = Sem( @@ -32,7 +32,7 @@ In general, there are two different types of parameters: **directed** and **indi We allow multiple variables on both sides of an arrow, for example `x → [y z]` or `[a b] → [c d]`. The later specifies element wise edges; that is its the same as `a → c; b → d`. If you want edges corresponding to the cross-product, we have the double lined arrow `[a b] ⇒ [c d]`, corresponding to `a → c; a → d; b → c; b → d`. The undirected arrows ↔ (element-wise) and ⇔ (crossproduct) behave the same way. !!! note "Unicode symbols in julia" - The `→` symbol is a unicode symbol allowed in julia (among many others; see this [list](https://docs.julialang.org/en/v1/manual/unicode-input/)). You can enter it in the julia REPL or the vscode IDE by typing `\to` followed by hitting `tab`. Similarly, + The `→` symbol is a unicode symbol allowed in julia (among many others; see this [list](https://docs.julialang.org/en/v1/manual/unicode-input/)). You can enter it in the julia REPL or the vscode IDE by typing `\to` followed by hitting `tab`. Similarly, - `←` = `\leftarrow`, - `↔` = `\leftrightarrow`, - `⇒` = `\Rightarrow`, @@ -54,7 +54,7 @@ graph = @StenoGraph begin ξ₃ ↔ fixed(1.0)*ξ₃ end ``` -would +would - fix the directed effects from `ξ₁` to `x1` and from `ξ₂` to `x2` to `1` - leave the directed effect from `ξ₃` to `x7` free but instead restrict the variance of `ξ₃` to `1` - give the effect from `ξ₁` to `x3` the label `:a` (which can be convenient later if you want to retrieve information from your model about that specific parameter) @@ -66,7 +66,7 @@ As you saw above and in the [A first model](@ref) example, the graph object need ```julia partable = ParameterTable( graph, - latent_vars = lat_vars, + latent_vars = lat_vars, observed_vars = obs_vars) ``` @@ -85,7 +85,7 @@ The variable names (`:x1`) have to be symbols, the syntax `:something` creates a _(lat_vars) ⇔ _(lat_vars) end ``` -creates undirected effects coresponding to +creates undirected effects coresponding to 1. the variances of all observed variables and 2. the variances plus covariances of all latent variables So if you want to work with a subset of variables, simply specify a vector of symbols `somevars = [...]`, and inside the graph specification, refer to them as `_(somevars)`. diff --git a/docs/src/tutorials/specification/parameter_table.md b/docs/src/tutorials/specification/parameter_table.md index c328a3b1a..62c45c9a4 100644 --- a/docs/src/tutorials/specification/parameter_table.md +++ b/docs/src/tutorials/specification/parameter_table.md @@ -5,5 +5,5 @@ As lavaan also uses parameter tables to store model specifications, we are worki ## Convert from and to RAMMatrices -To convert a RAMMatrices object to a ParameterTable, simply use `partable = ParameterTable(rammatrices)`. +To convert a RAMMatrices object to a ParameterTable, simply use `partable = ParameterTable(rammatrices)`. To convert an object of type `ParameterTable` to RAMMatrices, you can use `ram_matrices = RAMMatrices(partable)`. \ No newline at end of file diff --git a/docs/src/tutorials/specification/ram_matrices.md b/docs/src/tutorials/specification/ram_matrices.md index abe76ea6f..2730ff4bf 100644 --- a/docs/src/tutorials/specification/ram_matrices.md +++ b/docs/src/tutorials/specification/ram_matrices.md @@ -1,6 +1,6 @@ # RAMMatrices interface -Models can also be specified by an object of type `RAMMatrices`. +Models can also be specified by an object of type `RAMMatrices`. The RAM (reticular action model) specification corresponds to three matrices; the `A` matrix containing all directed parameters, the `S` matrix containing all undirected parameters, and the `F` matrix filtering out latent variables from the model implied covariance. The model implied covariance matrix for the observed variables of a SEM is then computed as @@ -56,9 +56,9 @@ A =[0 0 0 0 0 0 0 0 0 0 0 1.0 0 0 θ = Symbol.(:θ, 1:31) spec = RAMMatrices(; - A = A, - S = S, - F = F, + A = A, + S = S, + F = F, param_labels = θ, vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) @@ -71,9 +71,9 @@ model = Sem( Let's look at this step by step: -First, we specify the `A`, `S` and `F`-Matrices. -For a free parameter, we write a `Symbol` like `:θ1` (or any other symbol we like) to the corresponding place in the respective matrix, to constrain parameters to be equal we just use the same `Symbol` in the respective entries. -To fix a parameter (as in the `A`-Matrix above), we just write down the number we want to fix it to. +First, we specify the `A`, `S` and `F`-Matrices. +For a free parameter, we write a `Symbol` like `:θ1` (or any other symbol we like) to the corresponding place in the respective matrix, to constrain parameters to be equal we just use the same `Symbol` in the respective entries. +To fix a parameter (as in the `A`-Matrix above), we just write down the number we want to fix it to. All other entries are 0. Second, we specify a vector of symbols containing our parameters: @@ -82,14 +82,14 @@ Second, we specify a vector of symbols containing our parameters: θ = Symbol.(:θ, 1:31) ``` -Third, we construct an object of type `RAMMatrices`, passing our matrices and parameters, as well as the column names of our matrices. +Third, we construct an object of type `RAMMatrices`, passing our matrices and parameters, as well as the column names of our matrices. Those are quite important, as they will be used to rearrange your data to match it to your `RAMMatrices` specification. ```julia spec = RAMMatrices(; - A = A, - S = S, - F = F, + A = A, + S = S, + F = F, param_labels = θ, vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) @@ -109,7 +109,7 @@ According to the RAM, model implied mean values of the observed variables are co ```math \mu = F(I-A)^{-1}M ``` -where `M` is a vector of mean parameters. To estimate the means of the observed variables in our example (and set the latent means to `0`), we would specify the model just as before but add +where `M` is a vector of mean parameters. To estimate the means of the observed variables in our example (and set the latent means to `0`), we would specify the model just as before but add ```julia ... @@ -128,5 +128,5 @@ spec = RAMMatrices(; ## Convert from and to ParameterTables -To convert a RAMMatrices object to a ParameterTable, simply use `partable = ParameterTable(ram_matrices)`. +To convert a RAMMatrices object to a ParameterTable, simply use `partable = ParameterTable(ram_matrices)`. To convert an object of type `ParameterTable` to RAMMatrices, you can use `ram_matrices = RAMMatrices(partable)`. \ No newline at end of file diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index c5e0ad6cb..27bc30039 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -19,7 +19,7 @@ function SemOptimizerNLopt(; applicable(iterate, equality_constraints) && !isa(equality_constraints, NamedTuple) || (equality_constraints = [equality_constraints]) applicable(iterate, inequality_constraints) && - !isa(inequality_constraints, NamedTuple) || + !isa(inequality_constraints, NamedTuple) || (inequality_constraints = [inequality_constraints]) return SemOptimizerNLopt( algorithm, diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 6cbcb0573..d6a1fc6c8 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -21,7 +21,7 @@ function batch_inv!(fun, model) end # computes A*S*B -> C, where ind gives the entries of S that are 1 -function sparse_outer_mul!(C, A, B, ind) +function sparse_outer_mul!(C, A, B, ind) fill!(C, 0.0) for i in 1:length(ind) BLAS.ger!(1.0, A[:, ind[i][1]], B[ind[i][2], :], C) @@ -29,14 +29,14 @@ function sparse_outer_mul!(C, A, B, ind) end # computes A*∇m, where ∇m ind gives the entries of ∇m that are 1 -function sparse_outer_mul!(C, A, ind) +function sparse_outer_mul!(C, A, ind) fill!(C, 0.0) @views C .= sum(A[:, ind], dims = 2) return C end # computes A*S*B -> C, where ind gives the entries of S that are 1 -function sparse_outer_mul!(C, A, B::Vector, ind) +function sparse_outer_mul!(C, A, B::Vector, ind) fill!(C, 0.0) @views @inbounds for i in 1:length(ind) C .+= B[ind[i][2]] .* A[:, ind[i][1]] diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index 1031e349e..8cd89032e 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -199,11 +199,7 @@ materialize!( kwargs..., ) = materialize!(parent(dest), src, params; kwargs...) -function sparse_materialize( - ::Type{T}, - arr::ParamsMatrix, - params::AbstractVector, -) where {T} +function sparse_materialize(::Type{T}, arr::ParamsMatrix, params::AbstractVector) where {T} nparams(arr) == length(params) || throw( DimensionMismatch( "Number of values ($(length(params))) does not match the number of parameter ($(nparams(arr)))", @@ -251,8 +247,8 @@ sparse_gradient(arr::ParamsArray{T}) where {T} = sparse_gradient(T, arr) # range of parameters that are referenced in the matrix function params_range(arr::ParamsArray; allow_gaps::Bool = false) - first_i = findfirst(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:nparams(arr)-1) - last_i = findlast(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:nparams(arr)-1) + first_i = findfirst(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:(nparams(arr)-1)) + last_i = findlast(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:(nparams(arr)-1)) if !allow_gaps && !isnothing(first_i) && !isnothing(last_i) for i in first_i:last_i diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index a787516b2..4839bc272 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -48,7 +48,7 @@ replace_observed(model::AbstractSemSingle; kwargs...) = replace_observed(model, typeof(observed(model)).name.wrapper; kwargs...) function replace_observed(model::AbstractSemSingle, observed_type; kwargs...) - new_observed = observed_type(;kwargs...) + new_observed = observed_type(; kwargs...) kwargs = Dict{Symbol, Any}(kwargs...) # get field types @@ -65,11 +65,7 @@ function replace_observed(model::AbstractSemSingle, observed_type; kwargs...) # update loss new_loss = update_observed(model.loss, new_observed; kwargs...) - return Sem( - new_observed, - new_implied, - new_loss - ) + return Sem(new_observed, new_implied, new_loss) end function update_observed(loss::SemLoss, new_observed; kwargs...) @@ -79,7 +75,6 @@ function update_observed(loss::SemLoss, new_observed; kwargs...) return SemLoss(new_functions, loss.weights) end - function replace_observed( emodel::SemEnsemble; column = :group, @@ -94,16 +89,16 @@ function replace_observed( # allow for DataFrame with group variable "column" to be passed as new data if haskey(kwargs, :data) && isa(kwargs[:data], DataFrame) kwargs[:data] = Dict( - group => select( - filter( - r -> r[column] == group, - kwargs[:data]), - Not(column)) for group in emodel.groups) + group => + select(filter(r -> r[column] == group, kwargs[:data]), Not(column)) for + group in emodel.groups + ) end # update each model for new data models = emodel.sems new_models = Tuple( - replace_observed(m; group_kwargs(g, kwargs)...) for (m, g) in zip(models, emodel.groups) + replace_observed(m; group_kwargs(g, kwargs)...) for + (m, g) in zip(models, emodel.groups) ) return SemEnsemble(new_models...; weights = weights, groups = emodel.groups) end diff --git a/src/frontend/StatsAPI.jl b/src/frontend/StatsAPI.jl index edd677e34..b00c451af 100644 --- a/src/frontend/StatsAPI.jl +++ b/src/frontend/StatsAPI.jl @@ -13,11 +13,7 @@ Note that the function combines the duplicate occurences of the same parameter in `partable` and will raise an error if the values do not match. """ -function params!( - out::AbstractVector, - partable::ParameterTable, - col::Symbol = :estimate, -) +function params!(out::AbstractVector, partable::ParameterTable, col::Symbol = :estimate) (length(out) == nparams(partable)) || throw( DimensionMismatch( "The length of parameter values vector ($(length(out))) does not match the number of parameters ($(nparams(partable)))", @@ -75,4 +71,8 @@ Synonymous to [`nsamples`](@ref). """ nobs(model::AbstractSem) = nsamples(model) -coeftable(model::AbstractSem; level::Real=0.95) = throw(ArgumentError("StructuralEquationModels does not support the `CoefTable` interface; see [`ParameterTable`](@ref) instead.")) \ No newline at end of file +coeftable(model::AbstractSem; level::Real = 0.95) = throw( + ArgumentError( + "StructuralEquationModels does not support the `CoefTable` interface; see [`ParameterTable`](@ref) instead.", + ), +) diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index b9fff648e..f9dae84ed 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -1,15 +1,16 @@ """ - RMSEA(sem_fit::SemFit) + RMSEA(fit::SemFit) Return the RMSEA. """ function RMSEA end -RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) = - RMSEA(dof(sem_fit), χ²(sem_fit), nsamples(sem_fit)) +RMSEA(fit::SemFit) = RMSEA(fit, fit.model) -RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}) = - sqrt(length(sem_fit.model.sems)) * RMSEA(dof(sem_fit), χ²(sem_fit), nsamples(sem_fit)) +RMSEA(fit::SemFit, model::AbstractSemSingle) = RMSEA(dof(fit), χ²(fit), nsamples(fit)) + +RMSEA(fit::SemFit, model::SemEnsemble) = + sqrt(length(model.sems)) * RMSEA(dof(fit), χ²(fit), nsamples(fit)) function RMSEA(dof, chi2, nsamples) rmsea = (chi2 - dof) / (nsamples * dof) diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index 333783f95..dc19467fc 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -1,89 +1,70 @@ """ - χ²(sem_fit::SemFit) + χ²(fit::SemFit) Return the χ² value. """ -function χ² end +χ²(fit::SemFit) = χ²(fit, fit.model) ############################################################################################ # Single Models ############################################################################################ -# SemFit splices loss functions ------------------------------------------------------------ -χ²(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) = χ²( - sem_fit, - sem_fit.model.observed, - sem_fit.model.implied, - sem_fit.model.loss.functions..., -) +χ²(fit::SemFit, model::AbstractSemSingle) = + sum(loss -> χ²(loss, fit, model), model.loss.functions) # RAM + SemML -χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemML) = - (nsamples(sem_fit) - 1) * - (sem_fit.minimum - logdet(observed.obs_cov) - nobserved_vars(observed)) +χ²(lossfun::SemML, fit::SemFit, model::AbstractSemSingle) = + (nsamples(fit) - 1) * + (fit.minimum - logdet(obs_cov(observed(model))) - nobserved_vars(observed(model))) # bollen, p. 115, only correct for GLS weight matrix -χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemWLS) = - (nsamples(sem_fit) - 1) * sem_fit.minimum +χ²(lossfun::SemWLS, fit::SemFit, model::AbstractSemSingle) = + (nsamples(fit) - 1) * fit.minimum # FIML -function χ²(sem_fit::SemFit, observed::SemObservedMissing, imp, loss_ml::SemFIML) - ll_H0 = minus2ll(sem_fit) - ll_H1 = minus2ll(observed) - chi2 = ll_H0 - ll_H1 - return chi2 +function χ²(lossfun::SemFIML, fit::SemFit, model::AbstractSemSingle) + ll_H0 = minus2ll(fit) + ll_H1 = minus2ll(observed(model)) + return ll_H0 - ll_H1 end ############################################################################################ # Collections ############################################################################################ -# SemFit splices loss functions ------------------------------------------------------------ -χ²(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}) = - χ²(sem_fit, sem_fit.model, sem_fit.model.sems[1].loss.functions[1]) +function χ²(fit::SemFit, models::SemEnsemble) + isempty(models.sems) && return 0.0 -function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemWLS} - check_ensemble_length(model) - check_lossfun_types(model, L) - return (nsamples(model) - 1) * sem_fit.minimum -end + lossfun = models.sems[1].loss.functions[1] + # check that all models use the same single loss function + L = typeof(lossfun) + for (i, sem) in enumerate(models.sems) + if length(sem.loss.functions) > 1 + @error "Model for group #$i has $(length(sem.loss.functions)) loss functions. Only the single one is supported" + end + cur_lossfun = sem.loss.functions[1] + if !isa(cur_lossfun, L) + @error "Loss function for group #$i model is $(typeof(cur_lossfun)), expected $L. Heterogeneous loss functions are not supported" + end + end -function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemML} - check_ensemble_length(model) - check_lossfun_types(model, L) - F_G = sem_fit.minimum - F_G -= sum([ - w * (logdet(m.observed.obs_cov) + nobserved_vars(m.observed)) for - (w, m) in zip(model.weights, model.sems) - ]) - return (nsamples(model) - 1) * F_G + return χ²(lossfun, fit, models) end -function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemFIML} - check_ensemble_length(model) - check_lossfun_types(model, L) - - ll_H0 = minus2ll(sem_fit) - ll_H1 = sum(minus2ll.(observed.(models(model)))) - chi2 = ll_H0 - ll_H1 - - return chi2 +function χ²(lossfun::SemWLS, fit::SemFit, models::SemEnsemble) + return (nsamples(models) - 1) * fit.minimum end -function check_ensemble_length(model) - for sem in model.sems - if length(sem.loss.functions) > 1 - @error "A model for one of the groups contains multiple loss functions." - end +function χ²(lossfun::SemML, fit::SemFit, models::SemEnsemble) + G = sum(zip(models.weights, models.sems)) do (w, model) + data = observed(model) + w * (logdet(obs_cov(data)) + nobserved_vars(data)) end + return (nsamples(models) - 1) * (fit.minimum - G) end -function check_lossfun_types(model, type) - for sem in model.sems - for lossfun in sem.loss.functions - if !isa(lossfun, type) - @error "Your model(s) contain multiple lossfunctions with differing types." - end - end - end +function χ²(lossfun::SemFIML, fit::SemFit, models::SemEnsemble) + ll_H0 = minus2ll(fit) + ll_H1 = sum(minus2ll ∘ observed, models.sems) + return ll_H0 - ll_H1 end diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index ab4d24e53..9b211fb44 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -9,30 +9,31 @@ function minus2ll end # Single Models ############################################################################################ -# SemFit splices loss functions ------------------------------------------------------------ -minus2ll( - sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}, -) = minus2ll( - sem_fit, - sem_fit.model.observed, - sem_fit.model.implied, - sem_fit.model.loss.functions..., -) - -minus2ll(sem_fit::SemFit, obs, imp, args...) = minus2ll(sem_fit.minimum, obs, imp, args...) +minus2ll(fit::SemFit) = minus2ll(fit, fit.model) + +function minus2ll(fit::SemFit, model::AbstractSemSingle) + minimum = objective(model, fit.solution) + return minus2ll(minimum, model) +end + +minus2ll(minimum::Number, model::AbstractSemSingle) = + sum(lossfun -> minus2ll(lossfun, minimum, model), model.loss.functions) # SemML ------------------------------------------------------------------------------------ -minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, loss_ml::SemML) = - nsamples(obs) * (minimum + log(2π) * nobserved_vars(obs)) +function minus2ll(lossfun::SemML, minimum::Number, model::AbstractSemSingle) + obs = observed(model) + return nsamples(obs) * (minimum + log(2π) * nobserved_vars(obs)) +end # WLS -------------------------------------------------------------------------------------- -minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, loss_ml::SemWLS) = missing +minus2ll(lossfun::SemWLS, minimum::Number, model::AbstractSemSingle) = missing # compute likelihood for missing data - H0 ------------------------------------------------- # -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n -function minus2ll(minimum::Number, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemFIML) - F = minimum * nsamples(observed) - F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), observed.patterns) +function minus2ll(lossfun::SemFIML, minimum::Number, model::AbstractSemSingle) + obs = observed(model)::SemObservedMissing + F = minimum * nsamples(obs) + F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), obs.patterns) return F end @@ -66,16 +67,4 @@ end # Collection ############################################################################################ -minus2ll(minimum, model::AbstractSemSingle) = - minus2ll(minimum, model.observed, model.implied, model.loss.functions...) - -function minus2ll( - sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}, -) - m2ll = 0.0 - for sem in sem_fit.model.sems - minimum = objective!(sem, sem_fit.solution) - m2ll += minus2ll(minimum, sem) - end - return m2ll -end +minus2ll(fit::SemFit, model::SemEnsemble) = sum(Base.Fix1(minus2ll, fit), model.sems) diff --git a/src/frontend/fit/fitmeasures/p.jl b/src/frontend/fit/fitmeasures/p.jl index 8c69d5ec2..da9bedaf6 100644 --- a/src/frontend/fit/fitmeasures/p.jl +++ b/src/frontend/fit/fitmeasures/p.jl @@ -3,4 +3,4 @@ Return the p value computed from the χ² test statistic. """ -p_value(sem_fit::SemFit) = 1 - cdf(Chisq(dof(sem_fit)), χ²(sem_fit)) +p_value(sem_fit::SemFit) = ccdf(Chisq(dof(sem_fit)), χ²(sem_fit)) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index 4b3e302bb..bebad2935 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -14,8 +14,7 @@ function se_bootstrap( data = nothing, specification = nothing, kwargs..., - ) where {Mi, So, St, Mo <: AbstractSemSingle, O} - +) where {Mi, So, St, Mo <: AbstractSemSingle, O} if isnothing(data) data = samples(observed(model(semfit))) end @@ -67,11 +66,10 @@ function se_bootstrap( data = nothing, specification = nothing, kwargs..., - ) where {Mi, So, St, Mo <: SemEnsemble, O} - +) where {Mi, So, St, Mo <: SemEnsemble, O} models = semfit.model.sems groups = semfit.model.groups - + if isnothing(data) data = Dict(g => samples(observed(m)) for (g, m) in zip(groups, models)) end diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 14169dd94..227e902ff 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -30,7 +30,7 @@ function EnsembleParameterTable( param_labels = if isnothing(param_labels) # collect all SEM parameters in ensemble if not specified # and apply the set to all partables - unique(mapreduce(SEM.param_labels, vcat, values(spec_ensemble), init = Vector{Symbol}())) + mapreduce(SEM.param_labels, vcat, values(spec_ensemble), init = Symbol[]) |> unique else copy(param_labels) end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 5f8aff1eb..600c9058c 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -34,7 +34,9 @@ function ParameterTable( latent_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing, ) - param_labels = isnothing(param_labels) ? unique!(filter(!=(:const), columns[:label])) : copy(param_labels) + param_labels = + isnothing(param_labels) ? unique!(filter(!=(:const), columns[:label])) : + copy(param_labels) check_param_labels(param_labels, columns[:label]) return ParameterTable( columns, @@ -400,7 +402,6 @@ function update_se_hessian!( return update_partable!(partable, :se, param_labels(fit), se) end - """ lavaan_params!(out::AbstractVector, partable_lav, partable::ParameterTable, @@ -449,8 +450,8 @@ function lavaan_params!( lav_ind = findallrows( r -> r[:lhs] == String(to) && - r[:op] == "~1" && - (isnothing(lav_group) || r[:group] == lav_group), + r[:op] == "~1" && + (isnothing(lav_group) || r[:group] == lav_group), partable_lav, ) else @@ -469,20 +470,20 @@ function lavaan_params!( lav_ind = findallrows( r -> ( - (r[:lhs] == String(from) && r[:rhs] == String(to)) || - (r[:lhs] == String(to) && r[:rhs] == String(from)) - ) && - r[:op] == lav_type && - (isnothing(lav_group) || r[:group] == lav_group), + (r[:lhs] == String(from) && r[:rhs] == String(to)) || + (r[:lhs] == String(to) && r[:rhs] == String(from)) + ) && + r[:op] == lav_type && + (isnothing(lav_group) || r[:group] == lav_group), partable_lav, ) else lav_ind = findallrows( r -> r[:lhs] == String(from) && - r[:rhs] == String(to) && - r[:op] == lav_type && - (isnothing(lav_group) || r[:group] == lav_group), + r[:rhs] == String(to) && + r[:op] == lav_type && + (isnothing(lav_group) || r[:group] == lav_group), partable_lav, ) end @@ -535,10 +536,4 @@ lavaan_params( partable::ParameterTable, lav_col::Symbol = :est, lav_group = nothing, -) = lavaan_params!( - fill(NaN, nparams(partable)), - partable_lav, - partable, - lav_col, - lav_group, -) +) = lavaan_params!(fill(NaN, nparams(partable)), partable_lav, partable, lav_col, lav_group) diff --git a/src/frontend/specification/checks.jl b/src/frontend/specification/checks.jl index 5ef41c59d..2d00be26d 100644 --- a/src/frontend/specification/checks.jl +++ b/src/frontend/specification/checks.jl @@ -4,8 +4,11 @@ function check_param_labels( param_refs::Union{AbstractVector{Symbol}, Nothing}, ) dup_param_labels = nonunique(param_labels) - isempty(dup_param_labels) || - throw(ArgumentError("Duplicate parameter labels detected: $(join(dup_param_labels, ", "))")) + isempty(dup_param_labels) || throw( + ArgumentError( + "Duplicate parameter labels detected: $(join(dup_param_labels, ", "))", + ), + ) any(==(:const), param_labels) && throw(ArgumentError("Parameters constain reserved :const name")) diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl index e46620fbc..a3a8d2659 100644 --- a/src/frontend/specification/documentation.jl +++ b/src/frontend/specification/documentation.jl @@ -37,7 +37,6 @@ Return the vector of parameter labels (in the same order as [`params`](@ref)). """ param_labels(spec::SemSpecification) = spec.param_labels - """ `ParameterTable`s contain the specification of a structural equation model. diff --git a/src/implied/RAM/generic.jl b/src/implied/RAM/generic.jl index fd2ef61b5..6d4a18194 100644 --- a/src/implied/RAM/generic.jl +++ b/src/implied/RAM/generic.jl @@ -152,7 +152,7 @@ function RAM(; F⨉I_A⁻¹, F⨉I_A⁻¹S, I_A, - copy(I_A), + similar(I_A), ∇A, ∇S, ∇M, @@ -163,7 +163,12 @@ end ### methods ############################################################################################ -function update!(targets::EvaluationTargets, implied::RAM, model::AbstractSemSingle, param_labels) +function update!( + targets::EvaluationTargets, + implied::RAM, + model::AbstractSemSingle, + param_labels, +) materialize!(implied.A, implied.ram_matrices.A, param_labels) materialize!(implied.S, implied.ram_matrices.S, param_labels) if !isnothing(implied.M) diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index 9634bfa89..39efe453b 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -37,13 +37,13 @@ Jacobians (only available in gradient! calls) - `ram.∇Σ` -> ``∂vec(Σ)/∂θᵀ`` - `ram.∇μ` -> ``∂μ/∂θᵀ`` -- `ram.∇Σ_function` -> function to overwrite `∇Σ` in place, - i.e. `∇Σ_function(∇Σ, θ)`. Typically, you do not want to use this but simply +- `∇Σ_eval!(::RAMSymbolic)` -> function to evaluate `∇Σ` in place, + i.e. `∇Σ_eval!(∇Σ, θ)`. Typically, you do not want to use this but simply query `ram.∇Σ`. Hessians The computation of hessians is more involved. -Therefore, we desribe it in the online documentation, +Therefore, we desribe it in the online documentation, and the respective interfaces are omitted here. ## RAM notation @@ -56,23 +56,19 @@ and for models with a meanstructure, the model implied means are computed as \mu = F(I-A)^{-1}M ``` """ -struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5} <: - SemImpliedSymbolic +struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, V2, F4, A4, F5, A5} <: SemImpliedSymbolic meanstruct::MS hessianeval::ExactHessian - Σ_function::F1 - ∇Σ_function::F2 - ∇²Σ_function::F3 + Σ_eval!::F1 + ∇Σ_eval!::F2 + ∇²Σ_eval!::F3 Σ::A1 ∇Σ::A2 ∇²Σ::A3 - Σ_symbolic::S1 - ∇Σ_symbolic::S2 - ∇²Σ_symbolic::S3 ram_matrices::V2 - μ_function::F4 + μ_eval!::F4 μ::A4 - ∇μ_function::F5 + ∇μ_eval!::F5 ∇μ::A5 RAMSymbolic{MS}(args...) where {MS <: MeanStruct} = @@ -111,81 +107,75 @@ function RAMSymbolic(; I_A⁻¹ = neumann_series(A) # Σ - Σ_symbolic = eval_Σ_symbolic(S, I_A⁻¹, F; vech = vech, simplify = simplify_symbolics) - #print(Symbolics.build_function(Σ_symbolic)[2]) - Σ_function = Symbolics.build_function(Σ_symbolic, par, expression = Val{false})[2] - Σ = zeros(size(Σ_symbolic)) - precompile(Σ_function, (typeof(Σ), Vector{Float64})) + Σ_sym = eval_Σ_symbolic(S, I_A⁻¹, F; vech, simplify = simplify_symbolics) + #print(Symbolics.build_function(Σ_sym)[2]) + Σ_eval! = Symbolics.build_function(Σ_sym, par, expression = Val{false})[2] + Σ = zeros(size(Σ_sym)) + precompile(Σ_eval!, (typeof(Σ), Vector{Float64})) # ∇Σ if gradient - ∇Σ_symbolic = Symbolics.sparsejacobian(vec(Σ_symbolic), [par...]) - ∇Σ_function = Symbolics.build_function(∇Σ_symbolic, par, expression = Val{false})[2] - constr = findnz(∇Σ_symbolic) - ∇Σ = sparse(constr[1], constr[2], fill(1.0, nnz(∇Σ_symbolic)), size(∇Σ_symbolic)...) - precompile(∇Σ_function, (typeof(∇Σ), Vector{Float64})) + ∇Σ_sym = Symbolics.sparsejacobian(vec(Σ_sym), [par...]) + ∇Σ_eval! = Symbolics.build_function(∇Σ_sym, par, expression = Val{false})[2] + constr = findnz(∇Σ_sym) + ∇Σ = sparse(constr[1], constr[2], fill(1.0, nnz(∇Σ_sym)), size(∇Σ_sym)...) + precompile(∇Σ_eval!, (typeof(∇Σ), Vector{Float64})) else - ∇Σ_symbolic = nothing - ∇Σ_function = nothing + ∇Σ_eval! = nothing ∇Σ = nothing end if hessian && !approximate_hessian - n_sig = length(Σ_symbolic) - ∇²Σ_symbolic_vec = [Symbolics.sparsehessian(σᵢ, [par...]) for σᵢ in vec(Σ_symbolic)] + n_sig = length(Σ_sym) + ∇²Σ_sym_vec = [Symbolics.sparsehessian(σᵢ, [par...]) for σᵢ in vec(Σ_sym)] @variables J[1:n_sig] - ∇²Σ_symbolic = zeros(Num, n_par, n_par) + ∇²Σ_sym = zeros(Num, n_par, n_par) for i in 1:n_sig - ∇²Σ_symbolic += J[i] * ∇²Σ_symbolic_vec[i] + ∇²Σ_sym += J[i] * ∇²Σ_sym_vec[i] end - ∇²Σ_function = - Symbolics.build_function(∇²Σ_symbolic, J, par, expression = Val{false})[2] + ∇²Σ_eval! = Symbolics.build_function(∇²Σ_sym, J, par, expression = Val{false})[2] ∇²Σ = zeros(n_par, n_par) else - ∇²Σ_symbolic = nothing - ∇²Σ_function = nothing + ∇²Σ_sym = nothing + ∇²Σ_eval! = nothing ∇²Σ = nothing end # μ if meanstructure MS = HasMeanStruct - μ_symbolic = eval_μ_symbolic(M, I_A⁻¹, F; simplify = simplify_symbolics) - μ_function = Symbolics.build_function(μ_symbolic, par, expression = Val{false})[2] - μ = zeros(size(μ_symbolic)) + μ_sym = eval_μ_symbolic(M, I_A⁻¹, F; simplify = simplify_symbolics) + μ_eval! = Symbolics.build_function(μ_sym, par, expression = Val{false})[2] + μ = zeros(size(μ_sym)) if gradient - ∇μ_symbolic = Symbolics.jacobian(μ_symbolic, [par...]) - ∇μ_function = - Symbolics.build_function(∇μ_symbolic, par, expression = Val{false})[2] + ∇μ_sym = Symbolics.jacobian(μ_sym, [par...]) + ∇μ_eval! = Symbolics.build_function(∇μ_sym, par, expression = Val{false})[2] ∇μ = zeros(size(F, 1), size(par, 1)) else - ∇μ_function = nothing + ∇μ_eval! = nothing ∇μ = nothing end else MS = NoMeanStruct - μ_function = nothing + μ_eval! = nothing μ = nothing - ∇μ_function = nothing + ∇μ_eval! = nothing ∇μ = nothing end return RAMSymbolic{MS}( - Σ_function, - ∇Σ_function, - ∇²Σ_function, + Σ_eval!, + ∇Σ_eval!, + ∇²Σ_eval!, Σ, ∇Σ, ∇²Σ, - Σ_symbolic, - ∇Σ_symbolic, - ∇²Σ_symbolic, ram_matrices, - μ_function, + μ_eval!, μ, - ∇μ_function, + ∇μ_eval!, ∇μ, ) end @@ -200,15 +190,15 @@ function update!( model::AbstractSemSingle, par, ) - implied.Σ_function(implied.Σ, par) + implied.Σ_eval!(implied.Σ, par) if MeanStruct(implied) === HasMeanStruct - implied.μ_function(implied.μ, par) + implied.μ_eval!(implied.μ, par) end if is_gradient_required(targets) || is_hessian_required(targets) - implied.∇Σ_function(implied.∇Σ, par) + implied.∇Σ_eval!(implied.∇Σ, par) if MeanStruct(implied) === HasMeanStruct - implied.∇μ_function(implied.∇μ, par) + implied.∇μ_eval!(implied.∇μ, par) end end end diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index ec5eb997c..f5a42e3f4 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -114,10 +114,9 @@ function evaluate!( if HessianEval(semml) === ApproxHessian mul!(hessian, ∇Σ' * kron(Σ⁻¹, Σ⁻¹), ∇Σ, 2, 0) else - ∇²Σ_function! = implied.∇²Σ_function ∇²Σ = implied.∇²Σ # inner - ∇²Σ_function!(∇²Σ, J, par) + implied.∇²Σ_eval!(∇²Σ, J, par) # outer H_outer = kron(2Σ⁻¹ΣₒΣ⁻¹ - Σ⁻¹, Σ⁻¹) mul!(hessian, ∇Σ' * H_outer, ∇Σ) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index dd5be4874..59508ce16 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -127,10 +127,9 @@ function evaluate!( end isnothing(hessian) || (mul!(hessian, ∇σ' * V, ∇σ, 2, 0)) if !isnothing(hessian) && (HessianEval(semwls) === ExactHessian) - ∇²Σ_function! = implied.∇²Σ_function ∇²Σ = implied.∇²Σ J = -2 * (σ₋' * semwls.V)' - ∇²Σ_function!(∇²Σ, J, par) + implied.∇²Σ_eval!(∇²Σ, J, par) hessian .+= ∇²Σ end if MeanStruct(implied) === HasMeanStruct diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 46d0622be..288082ccc 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -26,7 +26,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. =# -# outer function --------------------------------------------------------------------------- """ em_mvn!(; observed::SemObservedMissing, @@ -37,6 +36,12 @@ THE SOFTWARE. =# Estimates the covariance matrix and mean vector of the normal distribution via expectation maximization for `observed`. Overwrites the statistics stored in `observed`. + +Uses the EM algorithm for MVN-distributed data with missing values +adapted from the supplementary material to the book *Machine Learning: A Probabilistic Perspective*, +copyright (2010) Kevin Murphy and Matt Dunham: see +[*gaussMissingFitEm.m*](https://github.com/probml/pmtk3/blob/master/toolbox/BasicModels/gauss/sub/gaussMissingFitEm.m) and +[*emAlgo.m*](https://github.com/probml/pmtk3/blob/master/toolbox/Algorithms/optimization/emAlgo.m) scripts. """ function em_mvn!( observed::SemObservedMissing; diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index 3b1e98842..c1ad72592 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -41,7 +41,7 @@ function fit(optim::SemOptimizer, model::AbstractSem; start_val = nothing, kwarg end fit(model::AbstractSem; engine::Symbol = :Optim, start_val = nothing, kwargs...) = -fit(SemOptimizer(; engine, kwargs...), model; start_val, kwargs...) + fit(SemOptimizer(; engine, kwargs...), model; start_val, kwargs...) # fallback method fit(optim::SemOptimizer, model::AbstractSem, start_params; kwargs...) = @@ -56,8 +56,7 @@ prepare_start_params(start_val::Nothing, model::AbstractSem; kwargs...) = start_simple(model; kwargs...) # first argument is a function -prepare_start_params(start_val, model::AbstractSem; kwargs...) = - start_val(model; kwargs...) +prepare_start_params(start_val, model::AbstractSem; kwargs...) = start_val(model; kwargs...) function prepare_start_params(start_val::AbstractVector, model::AbstractSem; kwargs...) (length(start_val) == nparams(model)) || throw( diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index bd57942d8..2d782473a 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -67,7 +67,7 @@ SemOptimizer{:Optim}(args...; kwargs...) = SemOptimizerOptim(args...; kwargs...) SemOptimizerOptim(; algorithm = LBFGS(), - options = Optim.Options(;f_reltol = 1e-10, x_abstol = 1.5e-8), + options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), kwargs..., ) = SemOptimizerOptim(algorithm, options) diff --git a/src/types.jl b/src/types.jl index 660c1c43f..0e279e5b2 100644 --- a/src/types.jl +++ b/src/types.jl @@ -192,7 +192,8 @@ Returns a SemEnsemble with fields For instructions on multigroup models, see the online documentation. """ -struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, I, G <: Vector{Symbol}} <: AbstractSemCollection +struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, I, G <: Vector{Symbol}} <: + AbstractSemCollection n::N sems::T weights::V diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 4ff9bd507..acc3ccd08 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -90,12 +90,8 @@ function test_estimates( skip::Bool = false, ) actual = StructuralEquationModels.params(partable, col) - expected = StructuralEquationModels.lavaan_params( - partable_lav, - partable, - lav_col, - lav_group, - ) + expected = + StructuralEquationModels.lavaan_params(partable_lav, partable, lav_col, lav_group) @test !any(isnan, actual) @test !any(isnan, expected) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index f5ea0b5d7..e12d1c7c0 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -8,7 +8,8 @@ model_g1 = Sem(specification = specification_g1, data = dat_g1, implied = RAMSym model_g2 = Sem(specification = specification_g2, data = dat_g2, implied = RAM) -@test SEM.param_labels(model_g1.implied.ram_matrices) == SEM.param_labels(model_g2.implied.ram_matrices) +@test SEM.param_labels(model_g1.implied.ram_matrices) == + SEM.param_labels(model_g2.implied.ram_matrices) # test the different constructors model_ml_multigroup = SemEnsemble(model_g1, model_g2) diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 239bf713c..43de554ce 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -9,11 +9,11 @@ dat = example_data("holzinger_swineford") dat_missing = example_data("holzinger_swineford_missing") solution_lav = example_data("holzinger_swineford_solution") -dat_g1 = dat[dat.school.=="Pasteur", :] -dat_g2 = dat[dat.school.=="Grant-White", :] +dat_g1 = dat[dat.school .== "Pasteur", :] +dat_g2 = dat[dat.school .== "Grant-White", :] -dat_miss_g1 = dat_missing[dat_missing.school.=="Pasteur", :] -dat_miss_g2 = dat_missing[dat_missing.school.=="Grant-White", :] +dat_miss_g1 = dat_missing[dat_missing.school .== "Pasteur", :] +dat_miss_g2 = dat_missing[dat_missing.school .== "Grant-White", :] dat.school = ifelse.(dat.school .== "Pasteur", :Pasteur, :Grant_White) dat_missing.school = ifelse.(dat_missing.school .== "Pasteur", :Pasteur, :Grant_White) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 7a8adc72e..4045141ce 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -1,6 +1,3 @@ -using Statistics: cov, mean -using Random, NLopt - ############################################################################################ ### models w.o. meanstructure ############################################################################################ diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index ad06e0fcd..cbdf7ea74 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -1,4 +1,6 @@ using StructuralEquationModels, Test, Suppressor, FiniteDiff +using Statistics: cov, mean +using Random, NLopt SEM = StructuralEquationModels diff --git a/test/examples/proximal/l0.jl b/test/examples/proximal/l0.jl index 374f8e58a..f74dfb2d1 100644 --- a/test/examples/proximal/l0.jl +++ b/test/examples/proximal/l0.jl @@ -1,5 +1,3 @@ -using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators - # load data dat = example_data("political_democracy") @@ -62,6 +60,7 @@ fit_prox = fit(model_prox, engine = :Proximal, operator_g = prox_operator) @test fit_prox.optimization_result.result[:iterations] < 1000 @test solution(fit_prox)[31] == 0.0 @test abs( - StructuralEquationModels.minimum(fit_prox) - StructuralEquationModels.minimum(sem_fit), + StructuralEquationModels.minimum(fit_prox) - + StructuralEquationModels.minimum(sem_fit), ) < 1.0 end diff --git a/test/examples/proximal/lasso.jl b/test/examples/proximal/lasso.jl index beb5cf529..356ac6188 100644 --- a/test/examples/proximal/lasso.jl +++ b/test/examples/proximal/lasso.jl @@ -1,5 +1,3 @@ -using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators - # load data dat = example_data("political_democracy") diff --git a/test/examples/proximal/proximal.jl b/test/examples/proximal/proximal.jl index 40e72a1ef..84a9162cb 100644 --- a/test/examples/proximal/proximal.jl +++ b/test/examples/proximal/proximal.jl @@ -1,3 +1,5 @@ +using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators, Suppressor + @testset "Ridge" begin include("ridge.jl") end diff --git a/test/examples/proximal/ridge.jl b/test/examples/proximal/ridge.jl index fd7ae113d..61b7fa12a 100644 --- a/test/examples/proximal/ridge.jl +++ b/test/examples/proximal/ridge.jl @@ -1,5 +1,3 @@ -using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators, Suppressor - # load data dat = example_data("political_democracy") diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index a7b4cec9a..ce7dc61ff 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -55,7 +55,7 @@ start = [ implied_ml = RAMSymbolic(; specification = ram_matrices, start_val = start) -implied_ml.Σ_function(implied_ml.Σ, true_val) +implied_ml.Σ_eval!(implied_ml.Σ, true_val) true_dist = MultivariateNormal(implied_ml.Σ) diff --git a/test/unit_tests/StatsAPI.jl b/test/unit_tests/StatsAPI.jl index 8648fc363..5907ee7b5 100644 --- a/test/unit_tests/StatsAPI.jl +++ b/test/unit_tests/StatsAPI.jl @@ -5,10 +5,7 @@ end partable = ParameterTable(graph, observed_vars = [:a, :b], latent_vars = Symbol[]) update_partable!(partable, :estimate, param_labels(partable), [3.1415]) data = randn(100, 2) -model = Sem( - specification = partable, - data = data -) +model = Sem(specification = partable, data = data) model_fit = fit(model) @testset "params" begin @@ -25,5 +22,5 @@ end end @testset "coeftable" begin - @test_throws "StructuralEquationModels does not support" coeftable(model) -end \ No newline at end of file + @test_throws "StructuralEquationModels does not support" coeftable(model) +end diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index 2bf5dedaf..d2c1b7a02 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -25,7 +25,6 @@ graph = @StenoGraph begin y8 ↔ y4 + y6 end - ram_matrices = RAMMatrices(ParameterTable(graph, observed_vars = obs_vars, latent_vars = lat_vars)) diff --git a/test/unit_tests/unit_tests.jl b/test/unit_tests/unit_tests.jl index 7189addd4..4d7dad7cf 100644 --- a/test/unit_tests/unit_tests.jl +++ b/test/unit_tests/unit_tests.jl @@ -7,7 +7,7 @@ available_tests = Dict( "data_input_formats" => "SemObserved", "specification" => "SemSpecification", "model" => "Sem model", - "StatsAPI" => "StatsAPI" + "StatsAPI" => "StatsAPI", ) # Determine which tests to run based on command-line arguments diff --git a/test/unit_tests/unit_tests_interactive.jl b/test/unit_tests/unit_tests_interactive.jl index cf082fa60..10a384403 100644 --- a/test/unit_tests/unit_tests_interactive.jl +++ b/test/unit_tests/unit_tests_interactive.jl @@ -7,4 +7,4 @@ try catch e @warn "Error initializing Test Env" exception=(e, catch_backtrace()) end -include("unit_tests.jl") \ No newline at end of file +include("unit_tests.jl") From 453fcc9ff81e1e81e3b746ea4695b5f1976e960b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 21 Mar 2025 21:00:35 -0700 Subject: [PATCH 207/364] Revert "fix Proximal extension" This reverts commit 9729819b86f375e4663de1fe9ec9c38d4932f580. --- ext/SEMProximalOptExt/SEMProximalOptExt.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/SEMProximalOptExt/SEMProximalOptExt.jl b/ext/SEMProximalOptExt/SEMProximalOptExt.jl index 04be35cb7..bedf1920e 100644 --- a/ext/SEMProximalOptExt/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt/SEMProximalOptExt.jl @@ -3,7 +3,8 @@ module SEMProximalOptExt using StructuralEquationModels using StructuralEquationModels: print_type_name, print_field_types using ProximalAlgorithms -import StructuralEquationModels: SemOptimizerProximal + +export SemOptimizerProximal SEM = StructuralEquationModels From abc2847899447bf6e10ea76f105d6913c88a2442 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 21 Mar 2025 21:01:59 -0700 Subject: [PATCH 208/364] Revert "fix NLopt extension" This reverts commit 81a4bd9839df01e9f487b9aa13e3df107856114a. --- ext/SEMNLOptExt/NLopt.jl | 5 +++++ ext/SEMNLOptExt/SEMNLOptExt.jl | 3 ++- src/StructuralEquationModels.jl | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index 27bc30039..ac282ad6d 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -1,3 +1,8 @@ +Base.@kwdef struct NLoptConstraint + f::Any + tol = 0.0 +end + Base.convert( ::Type{NLoptConstraint}, tuple::NamedTuple{(:f, :tol), Tuple{F, T}}, diff --git a/ext/SEMNLOptExt/SEMNLOptExt.jl b/ext/SEMNLOptExt/SEMNLOptExt.jl index bf905e3ac..a159f6dc8 100644 --- a/ext/SEMNLOptExt/SEMNLOptExt.jl +++ b/ext/SEMNLOptExt/SEMNLOptExt.jl @@ -1,10 +1,11 @@ module SEMNLOptExt using StructuralEquationModels, NLopt -import StructuralEquationModels: SemOptimizerNLopt, NLoptConstraint SEM = StructuralEquationModels +export SemOptimizerNLopt, NLoptConstraint + include("NLopt.jl") end diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 46692bd5b..e0cd1e7a4 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -198,6 +198,5 @@ export AbstractSem, ↔, ⇔, SemOptimizerNLopt, - NLoptConstraint, SemOptimizerProximal end From 56cdef1f26c482828b1466bc120fad31e6ed8c18 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 21 Mar 2025 21:03:35 -0700 Subject: [PATCH 209/364] Revert "fix exporting structs from package extensions" This reverts commit f0df6538f0220f964cbf51772698c317a0b4cf86. --- ext/SEMNLOptExt/NLopt.jl | 68 +++++++++++++++++++++ ext/SEMProximalOptExt/ProximalAlgorithms.jl | 32 ++++++++++ src/StructuralEquationModels.jl | 7 +-- src/package_extensions/SEMNLOptExt.jl | 65 -------------------- src/package_extensions/SEMProximalOptExt.jl | 27 -------- 5 files changed, 101 insertions(+), 98 deletions(-) delete mode 100644 src/package_extensions/SEMNLOptExt.jl delete mode 100644 src/package_extensions/SEMProximalOptExt.jl diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index ac282ad6d..694247cac 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -1,3 +1,71 @@ +############################################################################################ +### Types +############################################################################################ +""" +Connects to `NLopt.jl` as the optimization backend. +Only usable if `NLopt.jl` is loaded in the current Julia session! + +# Constructor + + SemOptimizerNLopt(; + algorithm = :LD_LBFGS, + options = Dict{Symbol, Any}(), + local_algorithm = nothing, + local_options = Dict{Symbol, Any}(), + equality_constraints = Vector{NLoptConstraint}(), + inequality_constraints = Vector{NLoptConstraint}(), + kwargs...) + +# Arguments +- `algorithm`: optimization algorithm. +- `options::Dict{Symbol, Any}`: options for the optimization algorithm +- `local_algorithm`: local optimization algorithm +- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm +- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints +- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints + +# Example +```julia +my_optimizer = SemOptimizerNLopt() + +# constrained optimization with augmented lagrangian +my_constrained_optimizer = SemOptimizerNLopt(; + algorithm = :AUGLAG, + local_algorithm = :LD_LBFGS, + local_options = Dict(:ftol_rel => 1e-6), + inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0), +) +``` + +# Usage +All algorithms and options from the NLopt library are available, for more information see +the NLopt.jl package and the NLopt online documentation. +For information on how to use inequality and equality constraints, +see [Constrained optimization](@ref) in our online documentation. + +# Extended help + +## Interfaces +- `algorithm(::SemOptimizerNLopt)` +- `local_algorithm(::SemOptimizerNLopt)` +- `options(::SemOptimizerNLopt)` +- `local_options(::SemOptimizerNLopt)` +- `equality_constraints(::SemOptimizerNLopt)` +- `inequality_constraints(::SemOptimizerNLopt)` + +## Implementation + +Subtype of `SemOptimizer`. +""" +struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} + algorithm::A + local_algorithm::A2 + options::B + local_options::B2 + equality_constraints::C + inequality_constraints::C +end + Base.@kwdef struct NLoptConstraint f::Any tol = 0.0 diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index 0d4748e3a..aec61e57e 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -1,3 +1,35 @@ +############################################################################################ +### Types +############################################################################################ +""" +Connects to `ProximalAlgorithms.jl` as the optimization backend. + +Can be used for regularized SEM, for a tutorial see the online docs on [Regularization](@ref). + +# Constructor + + SemOptimizerProximal(; + algorithm = ProximalAlgorithms.PANOC(), + operator_g, + operator_h = nothing, + kwargs..., + +# Arguments +- `algorithm`: optimization algorithm. +- `operator_g`: proximal operator (e.g., regularization penalty) +- `operator_h`: optional second proximal operator + +# Usage +All algorithms and operators from `ProximalAlgorithms.jl` are available, +for more information see the online docs on [Regularization](@ref) and +the documentation of `ProximalAlgorithms.jl` / `ProximalOperators.jl`. +""" +mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} + algorithm::A + operator_g::B + operator_h::C +end + SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) SemOptimizerProximal(; diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index e0cd1e7a4..a306eccfa 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -86,9 +86,6 @@ include("frontend/fit/fitmeasures/fit_measures.jl") # standard errors include("frontend/fit/standard_errors/hessian.jl") include("frontend/fit/standard_errors/bootstrap.jl") -# extensions -include("package_extensions/SEMNLOptExt.jl") -include("package_extensions/SEMProximalOptExt.jl") export AbstractSem, AbstractSemSingle, @@ -196,7 +193,5 @@ export AbstractSem, →, ←, ↔, - ⇔, - SemOptimizerNLopt, - SemOptimizerProximal + ⇔ end diff --git a/src/package_extensions/SEMNLOptExt.jl b/src/package_extensions/SEMNLOptExt.jl deleted file mode 100644 index 64c4cff04..000000000 --- a/src/package_extensions/SEMNLOptExt.jl +++ /dev/null @@ -1,65 +0,0 @@ -""" -Connects to `NLopt.jl` as the optimization backend. -Only usable if `NLopt.jl` is loaded in the current Julia session! - -# Constructor - - SemOptimizerNLopt(; - algorithm = :LD_LBFGS, - options = Dict{Symbol, Any}(), - local_algorithm = nothing, - local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), - kwargs...) - -# Arguments -- `algorithm`: optimization algorithm. -- `options::Dict{Symbol, Any}`: options for the optimization algorithm -- `local_algorithm`: local optimization algorithm -- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm -- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints -- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints - -# Example -```julia -my_optimizer = SemOptimizerNLopt() - -# constrained optimization with augmented lagrangian -my_constrained_optimizer = SemOptimizerNLopt(; - algorithm = :AUGLAG, - local_algorithm = :LD_LBFGS, - local_options = Dict(:ftol_rel => 1e-6), - inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0), -) -``` - -# Usage -All algorithms and options from the NLopt library are available, for more information see -the NLopt.jl package and the NLopt online documentation. -For information on how to use inequality and equality constraints, -see [Constrained optimization](@ref) in our online documentation. - -# Extended help - -## Interfaces -- `algorithm(::SemOptimizerNLopt)` -- `local_algorithm(::SemOptimizerNLopt)` -- `options(::SemOptimizerNLopt)` -- `local_options(::SemOptimizerNLopt)` -- `equality_constraints(::SemOptimizerNLopt)` -- `inequality_constraints(::SemOptimizerNLopt)` -""" -struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} - algorithm::A - local_algorithm::A2 - options::B - local_options::B2 - equality_constraints::C - inequality_constraints::C -end - -Base.@kwdef struct NLoptConstraint - f::Any - tol = 0.0 -end diff --git a/src/package_extensions/SEMProximalOptExt.jl b/src/package_extensions/SEMProximalOptExt.jl deleted file mode 100644 index ad4c2da2a..000000000 --- a/src/package_extensions/SEMProximalOptExt.jl +++ /dev/null @@ -1,27 +0,0 @@ -""" -Connects to `ProximalAlgorithms.jl` as the optimization backend. -Can be used for regularized SEM, for a tutorial see the online docs on [Regularization](@ref). - -# Constructor - - SemOptimizerProximal(; - algorithm = ProximalAlgorithms.PANOC(), - operator_g, - operator_h = nothing, - kwargs..., - -# Arguments -- `algorithm`: optimization algorithm. -- `operator_g`: proximal operator (e.g., regularization penalty) -- `operator_h`: optional second proximal operator - -# Usage -All algorithms and operators from `ProximalAlgorithms.jl` are available, -for more information see the online docs on [Regularization](@ref) and -the documentation of `ProximalAlgorithms.jl` / `ProximalOperators.jl`. -""" -mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} - algorithm::A - operator_g::B - operator_h::C -end From 421927ef79a466c399494bb6b83a764a37467b7d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 21 Mar 2025 21:18:02 -0700 Subject: [PATCH 210/364] types.jl: move SemOptimizer API into abstract.jl --- src/optimizer/abstract.jl | 17 +++++++++++++++++ src/types.jl | 11 ----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index c1ad72592..f00e50552 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -1,3 +1,20 @@ +engine(::Type{SemOptimizer{E}}) where {E} = E +engine(optimizer::SemOptimizer) = engine(typeof(optimizer)) + +SemOptimizer(args...; engine::Symbol = :Optim, kwargs...) = + SemOptimizer{engine}(args...; kwargs...) + +# fallback optimizer constructor +function SemOptimizer{E}(args...; kwargs...) where {E} + if E == :NLOpt + error("$E optimizer requires \"using NLopt\".") + elseif E == :Proximal + error("$E optimizer requires \"using ProximalAlgorithms\".") + else + error("$E optimizer is not supported.") + end +end + """ fit([optim::SemOptimizer], model::AbstractSem; [engine::Symbol], start_val = start_val, kwargs...) diff --git a/src/types.jl b/src/types.jl index 0e279e5b2..73a650e6f 100644 --- a/src/types.jl +++ b/src/types.jl @@ -86,17 +86,6 @@ If you want to connect the SEM package to a new optimization backend, you should """ abstract type SemOptimizer{E} end -engine(::Type{SemOptimizer{E}}) where {E} = E -engine(optimizer::SemOptimizer) = engine(typeof(optimizer)) - -SemOptimizer(args...; engine::Symbol = :Optim, kwargs...) = - SemOptimizer{engine}(args...; kwargs...) - -# fallback optimizer constructor -function SemOptimizer{E}(args...; kwargs...) where {E} - throw(ErrorException("$E optimizer is not supported.")) -end - """ Supertype of all objects that can serve as the observed field of a SEM. Pre-processes data and computes sufficient statistics for example. From 84bd7bdbb0ea9e30b519a9d1e8aaf372e8d3f5f8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 21 Mar 2025 23:25:57 -0700 Subject: [PATCH 211/364] NLoptResult should not be mutable --- ext/SEMNLOptExt/NLopt.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index 694247cac..a51a3d065 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -124,7 +124,7 @@ local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints -mutable struct NLoptResult +struct NLoptResult result::Any problem::Any end From 930e0e5f005546aab1afee605c027a3738b2dc4a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 21 Mar 2025 23:32:15 -0700 Subject: [PATCH 212/364] SemNLOpt: use f or f => tol pair for constraints It is a simple and intuitive syntax and avoids declaring new types. Also allow specifying default constraint tolerance as `constraint_tol`. --- docs/src/tutorials/constraints/constraints.md | 6 +- ext/SEMNLOptExt/NLopt.jl | 108 +++++++++--------- ext/SEMNLOptExt/SEMNLOptExt.jl | 2 +- .../political_democracy/constraints.jl | 4 +- 4 files changed, 61 insertions(+), 59 deletions(-) diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index c433240a9..938a2bb9e 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -1,6 +1,6 @@ # Constrained optimization -## Using the NLopt backend +## Using the NLopt engine ### Define an example model @@ -128,8 +128,8 @@ constrained_optimizer = SemOptimizerNLopt( algorithm = :AUGLAG, options = Dict(:upper_bounds => upper_bounds, :xtol_abs => 1e-4), local_algorithm = :LD_LBFGS, - equality_constraints = NLoptConstraint(;f = eq_constraint, tol = 1e-8), - inequality_constraints = NLoptConstraint(;f = ineq_constraint, tol = 1e-8), + equality_constraints = (eq_constraint => 1e-8), + inequality_constraints = (ineq_constraint => 1e-8), ) ``` diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index a51a3d065..fe052b135 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -1,6 +1,9 @@ ############################################################################################ ### Types ############################################################################################ + +const NLoptConstraint = Pair{Any, Number} + """ Connects to `NLopt.jl` as the optimization backend. Only usable if `NLopt.jl` is loaded in the current Julia session! @@ -12,8 +15,9 @@ Only usable if `NLopt.jl` is loaded in the current Julia session! options = Dict{Symbol, Any}(), local_algorithm = nothing, local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), + equality_constraints = nothing, + inequality_constraints = nothing, + constraint_tol::Number = 0.0, kwargs...) # Arguments @@ -21,19 +25,32 @@ Only usable if `NLopt.jl` is loaded in the current Julia session! - `options::Dict{Symbol, Any}`: options for the optimization algorithm - `local_algorithm`: local optimization algorithm - `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm -- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints -- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints +- `equality_constraints: optional equality constraints +- `inequality_constraints:: optional inequality constraints +- `constraint_tol::Number`: default tolerance for constraints + +## Constraints specification + +Equality and inequality constraints arguments could be a single constraint or any +iterable constraints container (e.g. vector or tuple). +Each constraint could be a function or any other callable object that +takes the two input arguments: + - the vector of the model parameters; + - the array for the in-place calculation of the constraint gradient. +To override the default tolerance, the constraint could be specified +as a pair of the function and its tolerance: `constraint_func => tol`. # Example ```julia -my_optimizer = SemOptimizerNLopt() +my_optimizer = SemOptimizer(engine = :NLopt) # constrained optimization with augmented lagrangian -my_constrained_optimizer = SemOptimizerNLopt(; +my_constrained_optimizer = SemOptimizer(; + engine = :NLopt, algorithm = :AUGLAG, local_algorithm = :LD_LBFGS, local_options = Dict(:ftol_rel => 1e-6), - inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0), + inequality_constraints = (my_constraint => tol), ) ``` @@ -57,25 +74,15 @@ see [Constrained optimization](@ref) in our online documentation. Subtype of `SemOptimizer`. """ -struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} - algorithm::A - local_algorithm::A2 - options::B - local_options::B2 - equality_constraints::C - inequality_constraints::C +struct SemOptimizerNLopt <: SemOptimizer{:NLopt} + algorithm::Symbol + local_algorithm::Union{Symbol, Nothing} + options::Dict{Symbol, Any} + local_options::Dict{Symbol, Any} + equality_constraints::Vector{NLoptConstraint} + inequality_constraints::Vector{NLoptConstraint} end -Base.@kwdef struct NLoptConstraint - f::Any - tol = 0.0 -end - -Base.convert( - ::Type{NLoptConstraint}, - tuple::NamedTuple{(:f, :tol), Tuple{F, T}}, -) where {F, T} = NLoptConstraint(tuple.f, tuple.tol) - ############################################################################################ ### Constructor ############################################################################################ @@ -85,22 +92,26 @@ function SemOptimizerNLopt(; local_algorithm = nothing, options = Dict{Symbol, Any}(), local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), - kwargs..., + equality_constraints = nothing, + inequality_constraints = nothing, + constraint_tol::Number = 0.0, + kwargs..., # FIXME remove the sink for unused kwargs ) - applicable(iterate, equality_constraints) && !isa(equality_constraints, NamedTuple) || - (equality_constraints = [equality_constraints]) - applicable(iterate, inequality_constraints) && - !isa(inequality_constraints, NamedTuple) || - (inequality_constraints = [inequality_constraints]) + constraint(f::Any) = f => constraint_tol + constraint(f_and_tol::Pair) = f_and_tol + + constraints(::Nothing) = Vector{NLoptConstraint}() + constraints(constraints) = + applicable(iterate, constraints) && !isa(constraints, Pair) ? + [constraint(constr) for constr in constraints] : [constraint(constraints)] + return SemOptimizerNLopt( algorithm, local_algorithm, options, local_options, - convert.(NLoptConstraint, equality_constraints), - convert.(NLoptConstraint, inequality_constraints), + constraints(equality_constraints), + constraints(inequality_constraints), ) end @@ -151,10 +162,7 @@ function SEM.fit( start_params::AbstractVector; kwargs..., ) - - # construct the NLopt problem - opt = construct_NLopt_problem(optim.algorithm, optim.options, length(start_params)) - set_NLopt_constraints!(opt, optim) + opt = construct_NLopt(optim.algorithm, optim.options, nparams(model)) opt.min_objective = (par, G) -> SEM.evaluate!( zero(eltype(par)), @@ -163,13 +171,16 @@ function SEM.fit( model, par, ) + for (f, tol) in optim.inequality_constraints + inequality_constraint!(opt, f, tol) + end + for (f, tol) in optim.equality_constraints + equality_constraint!(opt, f, tol) + end if !isnothing(optim.local_algorithm) - opt_local = construct_NLopt_problem( - optim.local_algorithm, - optim.local_options, - length(start_params), - ) + opt_local = + construct_NLopt(optim.local_algorithm, optim.local_options, nparams(model)) opt.local_optimizer = opt_local end @@ -183,7 +194,7 @@ end ### additional functions ############################################################################################ -function construct_NLopt_problem(algorithm, options, npar) +function construct_NLopt(algorithm, options, npar) opt = Opt(algorithm, npar) for (key, val) in pairs(options) @@ -193,15 +204,6 @@ function construct_NLopt_problem(algorithm, options, npar) return opt end -function set_NLopt_constraints!(opt::Opt, optimizer::SemOptimizerNLopt) - for con in optimizer.inequality_constraints - inequality_constraint!(opt, con.f, con.tol) - end - for con in optimizer.equality_constraints - equality_constraint!(opt, con.f, con.tol) - end -end - ############################################################################################ # pretty printing ############################################################################################ diff --git a/ext/SEMNLOptExt/SEMNLOptExt.jl b/ext/SEMNLOptExt/SEMNLOptExt.jl index a159f6dc8..61c41338b 100644 --- a/ext/SEMNLOptExt/SEMNLOptExt.jl +++ b/ext/SEMNLOptExt/SEMNLOptExt.jl @@ -4,7 +4,7 @@ using StructuralEquationModels, NLopt SEM = StructuralEquationModels -export SemOptimizerNLopt, NLoptConstraint +export SemOptimizerNLopt include("NLopt.jl") diff --git a/test/examples/political_democracy/constraints.jl b/test/examples/political_democracy/constraints.jl index cc1b0874d..7a6670fa3 100644 --- a/test/examples/political_democracy/constraints.jl +++ b/test/examples/political_democracy/constraints.jl @@ -26,8 +26,8 @@ constrained_optimizer = SemOptimizer(; algorithm = :AUGLAG, local_algorithm = :LD_LBFGS, options = Dict(:xtol_rel => 1e-4), - # equality_constraints = (f = eq_constraint, tol = 1e-14), - inequality_constraints = (f = ineq_constraint, tol = 0.0), + # equality_constraints = (eq_constraint => 1e-14), + inequality_constraints = (ineq_constraint => 0.0), ) @test constrained_optimizer isa SemOptimizer{:NLopt} From 230af39bdf54c78acb680d10a9b84ea6b76323c6 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 4 Feb 2026 18:32:07 -0800 Subject: [PATCH 213/364] NLopt: cleanup docstring --- ext/SEMNLOptExt/NLopt.jl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index fe052b135..d18c35e23 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -5,12 +5,13 @@ const NLoptConstraint = Pair{Any, Number} """ -Connects to `NLopt.jl` as the optimization backend. -Only usable if `NLopt.jl` is loaded in the current Julia session! +Uses *NLopt.jl* as the optimization engine. +Only available if *NLopt.jl* is loaded in the current Julia session! # Constructor - SemOptimizerNLopt(; + SemOptimizer(; + engine = :NLopt, algorithm = :LD_LBFGS, options = Dict{Symbol, Any}(), local_algorithm = nothing, @@ -55,8 +56,9 @@ my_constrained_optimizer = SemOptimizer(; ``` # Usage -All algorithms and options from the NLopt library are available, for more information see -the NLopt.jl package and the NLopt online documentation. +All algorithms and options from the *NLopt* library are available, for more information see +the [*NLopt.jl*](https://github.com/JuliaOpt/NLopt.jl) package and the +[NLopt docs](https://nlopt.readthedocs.io/en/latest/). For information on how to use inequality and equality constraints, see [Constrained optimization](@ref) in our online documentation. From d1355a0562385b685e8f4dc6014a83f927255f80 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 4 Feb 2026 18:33:05 -0800 Subject: [PATCH 214/364] NLopt: update/simplify docs use SemOptimizer(engine = :NLopt) instead of SemOptimizerNLopt() as this is a more universal scheme --- docs/src/tutorials/backends/nlopt.md | 39 +++++------- docs/src/tutorials/constraints/constraints.md | 60 +++++++++---------- 2 files changed, 44 insertions(+), 55 deletions(-) diff --git a/docs/src/tutorials/backends/nlopt.md b/docs/src/tutorials/backends/nlopt.md index feb5c8f48..3ad4bf497 100644 --- a/docs/src/tutorials/backends/nlopt.md +++ b/docs/src/tutorials/backends/nlopt.md @@ -1,31 +1,21 @@ # Using NLopt.jl -[`SemOptimizerNLopt`](@ref) implements the connection to `NLopt.jl`. -It is only available if the `NLopt` package is loaded alongside `StructuralEquationModels.jl` in the running Julia session. -It takes a bunch of arguments: +When [`NLopt.jl`](https://github.com/jump-dev/NLopt.jl) is loaded in the running Julia session, +it could be used by the [`SemOptimizer`](@ref) by specifying `engine = :NLopt` +(see [NLopt-specific options](@ref `SemOptimizerNLopt`)). +Among other things, `NLopt` enables constrained optimization of the SEM models, which is +explained in the [Constrained optimization](@ref) section. -```julia - • algorithm: optimization algorithm - - • options::Dict{Symbol, Any}: options for the optimization algorithm - - • local_algorithm: local optimization algorithm - - • local_options::Dict{Symbol, Any}: options for the local optimization algorithm - - • equality_constraints::Vector{NLoptConstraint}: vector of equality constraints - - • inequality_constraints::Vector{NLoptConstraint}: vector of inequality constraints -``` -Constraints are explained in the section on [Constrained optimization](@ref). - -The defaults are LBFGS as the optimization algorithm and the standard options from `NLopt.jl`. -We can choose something different: +We can override the default *NLopt* algorithm (LFBGS) and instead use +the *augmented lagrangian* method with LBFGS as the *local* optimization algorithm, +stop at a maximum of 200 evaluations and use a relative tolerance of +the objective value of `1e-6` as the stopping criterion for the local algorithm: ```julia using NLopt -my_optimizer = SemOptimizerNLopt(; +my_optimizer = SemOptimizer(; + engine = :NLopt, algorithm = :AUGLAG, options = Dict(:maxeval => 200), local_algorithm = :LD_LBFGS, @@ -33,15 +23,14 @@ my_optimizer = SemOptimizerNLopt(; ) ``` -This uses an augmented lagrangian method with LBFGS as the local optimization algorithm, stops at a maximum of 200 evaluations and uses a relative tolerance of the objective value of `1e-6` as the stopping criterion for the local algorithm. - To see how to use the optimizer to actually fit a model now, check out the [Model fitting](@ref) section. -In the NLopt docs, you can find explanations about the different [algorithms](https://nlopt.readthedocs.io/en/latest/NLopt_Algorithms/) and a [tutorial](https://nlopt.readthedocs.io/en/latest/NLopt_Introduction/) that also explains the different options. +In the *NLopt* docs, you can find details about the [optimization algorithms](https://nlopt.readthedocs.io/en/latest/NLopt_Algorithms/), +and the [tutorial](https://nlopt.readthedocs.io/en/latest/NLopt_Introduction/) that demonstrates how to tweak their behavior. To choose an algorithm, just pass its name without the 'NLOPT\_' prefix (for example, 'NLOPT\_LD\_SLSQP' can be used by passing `algorithm = :LD_SLSQP`). -The README of the [julia package](https://github.com/JuliaOpt/NLopt.jl) may also be helpful, and provides a list of options: +The README of the [*NLopt.jl*](https://github.com/JuliaOpt/NLopt.jl) may also be helpful, and provides a list of options: - `algorithm` - `stopval` diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index 938a2bb9e..32bb6a529 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -1,10 +1,15 @@ # Constrained optimization +*SEM.jl* allows to fit models with additional constraints imposed on the parameters. + ## Using the NLopt engine +*NLopt.jl* is one of *SEM.jl* optimization engines that supports constrained optimization. +In the example below we show how to specify constraints for the *SEM* model when using *NLopt*. + ### Define an example model -Let's revisit our model from [A first model](@ref): +Let's revisit our model from [A first model](@ref) and fit it first without constraints: ```@example constraints using StructuralEquationModels @@ -57,39 +62,40 @@ details(partable) ### Define the constraints -Let's introduce some constraints: +Let's introduce some constraints (they are not based on any real properties of the underlying study and serve only as an example): 1. **Equality constraint**: The covariances `y3 ↔ y7` and `y8 ↔ y4` should sum up to `1`. 2. **Inequality constraint**: The difference between the loadings `dem60 → y2` and `dem60 → y3` should be smaller than `0.1` 3. **Bound constraint**: The directed effect from `ind60 → dem65` should be smaller than `0.5` -(Of course those constaints only serve an illustratory purpose.) - -We first need to get the indices of the respective parameters that are invoved in the constraints. -We can look up their labels in the output above, and retrieve their indices as +Since *NLopt* does not have access to the SEM parameter names, its constaints are defined on the vector of all SEM parameters. +We have to look up the indices of the parameters involved in the constraints to construct the respective functions. ```@example constraints parind = param_indices(model) parind[:y3y7] # 29 ``` -The bound constraint is easy to specify: Just give a vector of upper or lower bounds that contains the bound for each parameter. In our example, only the parameter labeled `:λₗ` has an upper bound, and the number of total parameters is `n_par(model) = 31`, so we define +The bound constraint is easy to specify: just give a vector of upper or lower bounds for each parameter. +In our example, only the parameter labeled `:λₗ` has an upper bound, and the number of total parameters is `n_par(model) = 31`, so ```@example constraints upper_bounds = fill(Inf, 31) upper_bounds[parind[:λₗ]] = 0.5 ``` -The equailty and inequality constraints have to be reformulated to be of the form `x = 0` or `x ≤ 0`: -1. `y3 ↔ y7 + y8 ↔ y4 - 1 = 0` -2. `dem60 → y2 - dem60 → y3 - 0.1 ≤ 0` +The equailty and inequality constraints have to be reformulated in the `f(θ) = 0` or `f(θ) ≤ 0` form, +where `θ` is the vector of SEM parameters: +1. `f(θ) = 0`, where `f(θ) = y3 ↔ y7 + y8 ↔ y4 - 1` +2. `g(θ) ≤ 0`, where `g(θ) = dem60 → y2 - dem60 → y3 - 0.1` -Now they can be defined as functions of the parameter vector: +If the optimization algorithm needs gradients, it will pass the `gradient` vector that is of the same size as the parameters, +and the constraint function has to calculate the gradient in-place. ```@example constraints parind[:y3y7] # 29 parind[:y8y4] # 30 # θ[29] + θ[30] - 1 = 0.0 -function eq_constraint(θ, gradient) +function f(θ, gradient) if length(gradient) > 0 gradient .= 0.0 gradient[29] = 1.0 @@ -101,7 +107,7 @@ end parind[:λ₂] # 3 parind[:λ₃] # 4 # θ[3] - θ[4] - 0.1 ≤ 0 -function ineq_constraint(θ, gradient) +function g(θ, gradient) if length(gradient) > 0 gradient .= 0.0 gradient[3] = 1.0 @@ -111,29 +117,26 @@ function ineq_constraint(θ, gradient) end ``` -If the algorithm needs gradients at an iteration, it will pass the vector `gradient` that is of the same size as the parameters. -With `if length(gradient) > 0` we check if the algorithm needs gradients, and if it does, we fill the `gradient` vector with the gradients -of the constraint w.r.t. the parameters. - -In NLopt, vector-valued constraints are also possible, but we refer to the documentation for that. +In *NLopt*, vector-valued constraints are also possible, but we refer to the documentation for that. ### Fit the model -We now have everything together to specify and fit our model. First, we specify our optimizer backend as +Now we can construct the *SemOptimizer* that will use the *NLopt* engine for constrained optimization. ```@example constraints using NLopt -constrained_optimizer = SemOptimizerNLopt( +constrained_optimizer = SemOptimizer( + engine = :NLopt, algorithm = :AUGLAG, options = Dict(:upper_bounds => upper_bounds, :xtol_abs => 1e-4), local_algorithm = :LD_LBFGS, - equality_constraints = (eq_constraint => 1e-8), - inequality_constraints = (ineq_constraint => 1e-8), + equality_constraints = (f => 1e-8), + inequality_constraints = (g => 1e-8), ) ``` -As you see, the equality constraints and inequality constraints are passed as keyword arguments, and the bounds are passed as options for the (outer) optimization algorithm. +As you see, the equality and inequality constraints are passed as keyword arguments, and the bounds are passed as options for the (outer) optimization algorithm. Additionally, for equality and inequality constraints, a feasibility tolerance can be specified that controls if a solution can be accepted, even if it violates the constraints by a small amount. Especially for equality constraints, it is recommended to allow for a small positive tolerance. In this example, we set both tolerances to `1e-8`. @@ -141,19 +144,16 @@ In this example, we set both tolerances to `1e-8`. !!! warning "Convergence criteria" We have often observed that the default convergence criteria in NLopt lead to non-convergence flags. Indeed, this example does not convergence with default criteria. - As you see above, we used a realively liberal absolute tolerance in the optimization parameters of 1e-4. + As you see above, we used a relatively liberal absolute tolerance in the optimization parameters of 1e-4. This should not be a problem in most cases, as the sampling variance in (almost all) structural equation models should lead to uncertainty in the parameter estimates that are orders of magnitude larger. We nontheless recommend choosing a convergence criterion with care (i.e. w.r.t. the scale of your parameters), inspecting the solutions for plausibility, and comparing them to unconstrained solutions. -```@example constraints -model_constrained = Sem( - specification = partable, - data = data -) +We now have everything to fit our model under constraints: -model_fit_constrained = fit(constrained_optimizer, model_constrained) +```@example constraints +model_fit_constrained = fit(constrained_optimizer, model) ``` As you can see, the optimizer converged (`:XTOL_REACHED`) and investigating the solution yields From 6f3ccd537ef794e8df776a00ddcc157da94e3168 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 21 Mar 2025 23:35:11 -0700 Subject: [PATCH 215/364] Optim.md: SemOptimizerOptim => SemOptimizer --- docs/src/tutorials/backends/optim.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/src/tutorials/backends/optim.md b/docs/src/tutorials/backends/optim.md index cf287e773..a16537ec4 100644 --- a/docs/src/tutorials/backends/optim.md +++ b/docs/src/tutorials/backends/optim.md @@ -1,23 +1,23 @@ # Using Optim.jl -[`SemOptimizerOptim`](@ref) implements the connection to `Optim.jl`. -It takes two arguments, `algorithm` and `options`. -The defaults are LBFGS as the optimization algorithm and the standard options from `Optim.jl`. -We can load the `Optim` and `LineSearches` packages to choose something different: +[Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) is the default optimization engine of *SEM.jl*, +see [`SemOptimizerOptim`](@ref) for a full list of its parameters. +It defaults to the LBFGS optimization, but we can load the `Optim` and `LineSearches` packages +and specify BFGS (!not L-BFGS) with a back-tracking linesearch and Hager-Zhang initial step length guess: ```julia using Optim, LineSearches -my_optimizer = SemOptimizerOptim( +my_optimizer = SemOptimizer( algorithm = BFGS( - linesearch = BackTracking(order=3), + linesearch = BackTracking(order=3), alphaguess = InitialHagerZhang() - ), - options = Optim.Options(show_trace = true) - ) + ), + options = Optim.Options(show_trace = true) +) ``` -This optimizer will use BFGS (!not L-BFGS) with a back tracking linesearch and a certain initial step length guess. Also, the trace of the optimization will be printed to the console. +Note that we used `options` to print the optimization progress to the console. To see how to use the optimizer to actually fit a model now, check out the [Model fitting](@ref) section. From 242c6021a8f2ed28f64154f154c4e65cae0c726d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 21 Mar 2025 23:37:20 -0700 Subject: [PATCH 216/364] regulariz.md: SemOptimProx => SemOptimizer --- .../regularization/regularization.md | 67 ++++++++----------- 1 file changed, 29 insertions(+), 38 deletions(-) diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index 2b2c6df30..17add030a 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -5,7 +5,9 @@ For ridge regularization, you can simply use `SemRidge` as an additional loss function (for example, a model with the loss functions `SemML` and `SemRidge` corresponds to ridge-regularized maximum likelihood estimation). -For lasso, elastic net and (far) beyond, you can load the `ProximalAlgorithms.jl` and `ProximalOperators.jl` packages alongside `StructuralEquationModels`: +You can define lasso, elastic net and other forms of regularization using [`ProximalOperators.jl`](https://github.com/JuliaFirstOrder/ProximalOperators.jl) +and optimize the SEM model with [`ProximalAlgorithms.jl`](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) +that provides so-called *proximal optimization* algorithms. ```@setup reg using StructuralEquationModels, ProximalAlgorithms, ProximalOperators @@ -19,24 +21,22 @@ Pkg.add("ProximalOperators") using StructuralEquationModels, ProximalAlgorithms, ProximalOperators ``` -## `SemOptimizerProximal` +## Proximal optimization -To estimate regularized models, we provide a "building block" for the optimizer part, called `SemOptimizerProximal`. -It connects our package to the [`ProximalAlgorithms.jl`](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) optimization backend, providing so-called proximal optimization algorithms. -Those can handle, amongst other things, various forms of regularization. - -It can be used as +With *ProximalAlgorithms* package loaded, it is now possible to use `:Proximal` optimization engine +in `SemOptimizer` for estimating regularized models. ```julia -SemOptimizerProximal( +SemOptimizer(; + engine = :Proximal, algorithm = ProximalAlgorithms.PANOC(), operator_g, operator_h = nothing ) ``` -The proximal operator (aka the regularization function) can be passed as `operator_g`. -The available Algorithms are listed [here](https://juliafirstorder.github.io/ProximalAlgorithms.jl/stable/guide/implemented_algorithms/). +The *proximal operator* (aka the *regularization function*) is passed as `operator_g`, see [available operators](https://juliafirstorder.github.io/ProximalOperators.jl/stable/functions/). +The `algorithm` is chosen from one of the [available algorithms](https://juliafirstorder.github.io/ProximalAlgorithms.jl/stable/guide/implemented_algorithms/). ## First example - lasso @@ -84,7 +84,7 @@ model = Sem( We labeled the covariances between the items because we want to regularize those: ```@example reg -ind = getindex.( +cov_inds = getindex.( Ref(param_indices(model)), [:cov_15, :cov_24, :cov_26, :cov_37, :cov_48, :cov_68]) ``` @@ -96,30 +96,24 @@ The lasso penalty is defined as \sum \lambda_i \lvert \theta_i \rvert ``` -From the previously linked [documentation](https://juliafirstorder.github.io/ProximalOperators.jl/stable/functions/#ProximalOperators.NormL1), we find that lasso regularization is named `NormL1` in the `ProximalOperators` package, and that we can pass an array of hyperparameters (`λ`) to control the amount of regularization for each parameter. To regularize only the observed item covariances, we define `λ` as +In `ProximalOperators.jl`, lasso regularization is represented by the [`NormL1`](https://juliafirstorder.github.io/ProximalOperators.jl/stable/functions/#ProximalOperators.NormL1) operator. It allows controlling the amount of +regularization individually for each SEM model parameter via the vector of hyperparameters (`λ`). +To regularize only the observed item covariances, we define `λ` as ```@example reg -λ = zeros(31); λ[ind] .= 0.02 -``` - -and use `SemOptimizerProximal`. +λ = zeros(31); λ[cov_inds] .= 0.02 -```@example reg -optimizer_lasso = SemOptimizerProximal( +optimizer_lasso = SemOptimizer( + engine = :Proximal, operator_g = NormL1(λ) ) - -model_lasso = Sem( - specification = partable, - data = data -) ``` Let's fit the regularized model ```@example reg -fit_lasso = fit(optimizer_lasso, model_lasso) +fit_lasso = fit(optimizer_lasso, model) ``` and compare the solution to unregularizted estimates: @@ -134,34 +128,31 @@ update_partable!(partable, :estimate_lasso, fit_lasso, solution(fit_lasso)) details(partable) ``` -Instead of explicitely defining a `SemOptimizerProximal` object, you can also pass `engine = :Proximal` and additional keyword arguments to `fit`: +Instead of explicitly defining a `SemOptimizer` object, you can also pass `engine = :Proximal` +and additional keyword arguments directly to the `fit` function: ```@example reg -sem_fit = fit(model; engine = :Proximal, operator_g = NormL1(λ)) +fit_lasso2 = fit(model; engine = :Proximal, operator_g = NormL1(λ)) ``` ## Second example - mixed l1 and l0 regularization You can choose to penalize different parameters with different types of regularization functions. -Let's use the lasso again on the covariances, but additionally penalyze the error variances of the observed items via l0 regularization. +Let's use the *lasso* (*l1*) again on the covariances, but additionally penalize the error variances of the observed items via *l0* regularization. -The l0 penalty is defined as +The *l0* penalty is defined as ```math -\lambda \mathrm{nnz}(\theta) +l_0 = \lambda \mathrm{nnz}(\theta) ``` -To define a sup of separable proximal operators (i.e. no parameter is penalized twice), -we can use [`SlicedSeparableSum`](https://juliafirstorder.github.io/ProximalOperators.jl/stable/calculus/#ProximalOperators.SlicedSeparableSum) from the `ProximalOperators` package: +Since we apply *l1* and *l0* to the disjoint sets of parameters, this regularization could be represented as +as sum of *separable proximal operators* (i.e. no parameter is penalized twice) +implemented by the [`SlicedSeparableSum`](https://juliafirstorder.github.io/ProximalOperators.jl/stable/calculus/#ProximalOperators.SlicedSeparableSum) operator: ```@example reg -prox_operator = SlicedSeparableSum((NormL0(20.0), NormL1(0.02), NormL0(0.0)), ([ind], [9:11], [vcat(1:8, 12:25)])) - -model_mixed = Sem( - specification = partable, - data = data, -) +l0_and_l1_reg = SlicedSeparableSum((NormL0(20.0), NormL1(0.02), NormL0(0.0)), ([cov_inds], [9:11], [vcat(1:8, 12:25)])) -fit_mixed = fit(model_mixed; engine = :Proximal, operator_g = prox_operator) +fit_mixed = fit(model; engine = :Proximal, operator_g = l0_and_l1_reg) ``` Let's again compare the different results: From 3e5c9ac14af3c252c36fcb1ab6e7ce8a88e2c985 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 27 Jan 2026 09:36:13 -0800 Subject: [PATCH 217/364] optimizer_engine(): rename and fix signature optimizer_engine(): add docstring --- src/optimizer/abstract.jl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index f00e50552..5bdf65b26 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -1,6 +1,3 @@ -engine(::Type{SemOptimizer{E}}) where {E} = E -engine(optimizer::SemOptimizer) = engine(typeof(optimizer)) - SemOptimizer(args...; engine::Symbol = :Optim, kwargs...) = SemOptimizer{engine}(args...; kwargs...) @@ -15,6 +12,15 @@ function SemOptimizer{E}(args...; kwargs...) where {E} end end +""" + optimizer_engine(::Type{<:SemOptimizer}) + optimizer_engine(::SemOptimizer) + +Returns the engine name (`Symbol`) for a [`SemOptimizer`](@ref) instance or subtype. +""" +optimizer_engine(::Type{<:SemOptimizer{E}}) where {E} = E +optimizer_engine(optim::SemOptimizer) = optimizer_engine(typeof(optim)) + """ fit([optim::SemOptimizer], model::AbstractSem; [engine::Symbol], start_val = start_val, kwargs...) From 9471fbe91efdda573cc0f90025975f38cdded0df Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 27 Jan 2026 09:39:33 -0800 Subject: [PATCH 218/364] optimizer_engines(): new method --- Project.toml | 2 ++ src/StructuralEquationModels.jl | 2 ++ src/optimizer/abstract.jl | 14 +++++++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 376347083..0cc596b94 100644 --- a/Project.toml +++ b/Project.toml @@ -8,6 +8,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" @@ -30,6 +31,7 @@ StenoGraphs = "0.2 - 0.3, 0.4.1 - 0.5" DataFrames = "1" Distributions = "0.25" FiniteDiff = "2" +InteractiveUtils = "1.11.0" LineSearches = "7" NLSolversBase = "7" NLopt = "0.6, 1" diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a306eccfa..f537dafb3 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -18,6 +18,8 @@ using LinearAlgebra, import StatsAPI: params, coef, coefnames, dof, fit, nobs, coeftable +using InteractiveUtils: subtypes + export StenoGraphs, @StenoGraph, meld, SimpleNode const SEM = StructuralEquationModels diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index 5bdf65b26..d0da444fe 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -21,9 +21,21 @@ Returns the engine name (`Symbol`) for a [`SemOptimizer`](@ref) instance or subt optimizer_engine(::Type{<:SemOptimizer{E}}) where {E} = E optimizer_engine(optim::SemOptimizer) = optimizer_engine(typeof(optim)) +""" + optimizer_engines() + +Returns a vector of optimizer engines supported by the `engine` keyword argument of +the [`SemOptimizer`](@ref) constructor. + +The list of engines depends on the Julia packages loaded (with the `using` directive) +into the current session. +""" +optimizer_engines() = + Symbol[optimizer_engine(opt_type) for opt_type in subtypes(SemOptimizer)] + """ fit([optim::SemOptimizer], model::AbstractSem; - [engine::Symbol], start_val = start_val, kwargs...) + [engine::Symbol], start_val = start_val, kwargs...) Return the fitted `model`. From f4f92808bc757ba9cebde91db2df7fca8acf37df Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 2 Feb 2026 18:58:28 -0800 Subject: [PATCH 219/364] export optmizer_engine() --- src/StructuralEquationModels.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index f537dafb3..b2e5edd8d 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -125,6 +125,9 @@ export AbstractSem, SemOptimizerEmpty, SemOptimizerOptim, optimizer, + optimizer_engine, + optimizer_engine_doc, + optimizer_engines, n_iterations, convergence, SemObserved, From 41c130879edbcb4309ccb5f1d3430988800cb2c6 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 2 Feb 2026 19:04:05 -0800 Subject: [PATCH 220/364] sem_optimizer_subtype(engine) internal method returning the type that implements SemOptimizer{engine} --- docs/src/developer/optimizer.md | 4 +-- ext/SEMNLOptExt/NLopt.jl | 4 +-- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 4 +-- src/optimizer/Empty.jl | 6 +---- src/optimizer/abstract.jl | 29 ++++++++++++++++----- src/optimizer/optim.jl | 4 +-- 6 files changed, 31 insertions(+), 20 deletions(-) diff --git a/docs/src/developer/optimizer.md b/docs/src/developer/optimizer.md index 9e01ac87c..7e2b27b51 100644 --- a/docs/src/developer/optimizer.md +++ b/docs/src/developer/optimizer.md @@ -1,6 +1,6 @@ # Custom optimizer types -The optimizer part of a model connects it to the optimization backend. +The optimizer part of a model connects it to the optimization backend. Let's say we want to implement a new optimizer as `SemOptimizerName`. The first part of the implementation is very similar to loss functions, so we just show the implementation of `SemOptimizerOptim` here as a reference: ```julia @@ -12,7 +12,7 @@ mutable struct SemOptimizerName{A, B} <: SemOptimizer{:Name} options::B end -SemOptimizer{:Name}(args...; kwargs...) = SemOptimizerName(args...; kwargs...) +SEM.sem_optimizer_subtype(::Val{:Name}) = SemOptimizerName SemOptimizerName(; algorithm = LBFGS(), diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index d18c35e23..29006b8d0 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -85,6 +85,8 @@ struct SemOptimizerNLopt <: SemOptimizer{:NLopt} inequality_constraints::Vector{NLoptConstraint} end +SEM.sem_optimizer_subtype(::Val{:NLopt}) = SemOptimizerNLopt + ############################################################################################ ### Constructor ############################################################################################ @@ -117,8 +119,6 @@ function SemOptimizerNLopt(; ) end -SEM.SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) - ############################################################################################ ### Recommended methods ############################################################################################ diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index aec61e57e..cd7664537 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -30,8 +30,6 @@ mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} operator_h::C end -SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) - SemOptimizerProximal(; algorithm = ProximalAlgorithms.PANOC(), operator_g, @@ -39,6 +37,8 @@ SemOptimizerProximal(; kwargs..., ) = SemOptimizerProximal(algorithm, operator_g, operator_h) +SEM.sem_optimizer_subtype(::Val{:Proximal}) = SemOptimizerProximal + ############################################################################################ ### Recommended methods ############################################################################################ diff --git a/src/optimizer/Empty.jl b/src/optimizer/Empty.jl index 1bf0c30ac..51ab5f840 100644 --- a/src/optimizer/Empty.jl +++ b/src/optimizer/Empty.jl @@ -11,11 +11,7 @@ an optimizer part. """ struct SemOptimizerEmpty <: SemOptimizer{:Empty} end -############################################################################################ -### Constructor -############################################################################################ - -SemOptimizer{:Empty}() = SemOptimizerEmpty() +sem_optimizer_subtype(::Val{:Empty}) = SemOptimizerEmpty ############################################################################################ ### Recommended methods diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index d0da444fe..2abe95fa3 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -1,17 +1,32 @@ -SemOptimizer(args...; engine::Symbol = :Optim, kwargs...) = - SemOptimizer{engine}(args...; kwargs...) - -# fallback optimizer constructor -function SemOptimizer{E}(args...; kwargs...) where {E} - if E == :NLOpt +# throw unsupported engine error +function throw_engine_error(E) + if typeof(E) !== Symbol + throw(ArgumentError("engine argument must be a Symbol.")) + elseif E == :NLopt error("$E optimizer requires \"using NLopt\".") elseif E == :Proximal error("$E optimizer requires \"using ProximalAlgorithms\".") else - error("$E optimizer is not supported.") + error("$E optimizer engine is not supported.") end end +# return the type implementing SemOptimizer{engine} +# the method should be overridden in the extension +sem_optimizer_subtype(engine::Symbol) = sem_optimizer_subtype(Val(engine)) + +# fallback method for unsupported engines +sem_optimizer_subtype(::Val{E}) where {E} = throw_engine_error(E) + +# default constructor that dispatches to the engine-specific type +SemOptimizer(::Val{E}, args...; kwargs...) where {E} = + sem_optimizer_subtype(E)(args...; kwargs...) + +SemOptimizer{E}(args...; kwargs...) where {E} = SemOptimizer(Val(E), args...; kwargs...) + +SemOptimizer(args...; engine::Symbol = :Optim, kwargs...) = + SemOptimizer(Val(engine), args...; kwargs...) + """ optimizer_engine(::Type{<:SemOptimizer}) optimizer_engine(::SemOptimizer) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 2d782473a..9a5a5139f 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -63,14 +63,14 @@ mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} options::B end -SemOptimizer{:Optim}(args...; kwargs...) = SemOptimizerOptim(args...; kwargs...) - SemOptimizerOptim(; algorithm = LBFGS(), options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), kwargs..., ) = SemOptimizerOptim(algorithm, options) +sem_optimizer_subtype(::Val{:Optim}) = SemOptimizerOptim + ############################################################################################ ### Recommended methods ############################################################################################ From a1cf400abef948fe4ec67101032d3da38241412d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 4 Feb 2026 17:21:33 -0800 Subject: [PATCH 221/364] streamline engine error throwing --- src/optimizer/abstract.jl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index 2abe95fa3..53b49909e 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -1,13 +1,16 @@ +const optimizer_engine_dependencies = + Dict(:NLopt => ["NLopt"], :Proximal => ["ProximalAlgorithms"]) + # throw unsupported engine error function throw_engine_error(E) if typeof(E) !== Symbol throw(ArgumentError("engine argument must be a Symbol.")) - elseif E == :NLopt - error("$E optimizer requires \"using NLopt\".") - elseif E == :Proximal - error("$E optimizer requires \"using ProximalAlgorithms\".") + elseif haskey(optimizer_engine_dependencies, E) + error( + "optimizer \":$E\" requires \"using $(join(optimizer_engine_dependencies[E], ", "))\".", + ) else - error("$E optimizer engine is not supported.") + error("optimizer engine \":$E\" is not supported.") end end From efd49110a23bb4d43420bce7d9f2c0d82907283d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 4 Feb 2026 17:09:21 -0800 Subject: [PATCH 222/364] SemOptimizer{E}: remove docstring --- src/types.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/types.jl b/src/types.jl index 73a650e6f..92ca2c284 100644 --- a/src/types.jl +++ b/src/types.jl @@ -79,11 +79,6 @@ end Base.:*(x::SemWeight{Nothing}, y) = y Base.:*(x::SemWeight, y) = x.w * y -""" -Supertype of all objects that can serve as the `optimizer` field of a SEM. -Connects the SEM to its optimization backend and controls options like the optimization algorithm. -If you want to connect the SEM package to a new optimization backend, you should implement a subtype of SemOptimizer. -""" abstract type SemOptimizer{E} end """ From 341085336f128cf152838be71febabfdcc011cea Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 27 Jan 2026 09:48:44 -0800 Subject: [PATCH 223/364] SemOptimizer: cleanup docstrings --- ext/SEMNLOptExt/NLopt.jl | 56 ++++++++------------- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 31 +++++------- src/optimizer/Empty.jl | 8 ++- src/optimizer/abstract.jl | 24 +++++++++ src/optimizer/optim.jl | 48 +++++++----------- 5 files changed, 79 insertions(+), 88 deletions(-) diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index 29006b8d0..1c6b97120 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -4,12 +4,22 @@ const NLoptConstraint = Pair{Any, Number} -""" -Uses *NLopt.jl* as the optimization engine. -Only available if *NLopt.jl* is loaded in the current Julia session! +struct SemOptimizerNLopt <: SemOptimizer{:NLopt} + algorithm::Symbol + local_algorithm::Union{Symbol, Nothing} + options::Dict{Symbol, Any} + local_options::Dict{Symbol, Any} + equality_constraints::Vector{NLoptConstraint} + inequality_constraints::Vector{NLoptConstraint} +end + +SEM.sem_optimizer_subtype(::Val{:NLopt}) = SemOptimizerNLopt -# Constructor +############################################################################################ +### Constructor +############################################################################################ +""" SemOptimizer(; engine = :NLopt, algorithm = :LD_LBFGS, @@ -21,6 +31,10 @@ Only available if *NLopt.jl* is loaded in the current Julia session! constraint_tol::Number = 0.0, kwargs...) +Uses *NLopt.jl* as the optimization engine. For more information on the available algorithms +and options, see the [*NLopt.jl*](https://github.com/JuliaOpt/NLopt.jl) package and +the [NLopt docs](https://nlopt.readthedocs.io/en/latest/). + # Arguments - `algorithm`: optimization algorithm. - `options::Dict{Symbol, Any}`: options for the optimization algorithm @@ -38,8 +52,10 @@ Each constraint could be a function or any other callable object that takes the two input arguments: - the vector of the model parameters; - the array for the in-place calculation of the constraint gradient. -To override the default tolerance, the constraint could be specified +To override the default tolerance, the constraint can be specified as a pair of the function and its tolerance: `constraint_func => tol`. +For information on how to use inequality and equality constraints, +see [Constrained optimization](@ref) in our online documentation. # Example ```julia @@ -55,42 +71,14 @@ my_constrained_optimizer = SemOptimizer(; ) ``` -# Usage -All algorithms and options from the *NLopt* library are available, for more information see -the [*NLopt.jl*](https://github.com/JuliaOpt/NLopt.jl) package and the -[NLopt docs](https://nlopt.readthedocs.io/en/latest/). -For information on how to use inequality and equality constraints, -see [Constrained optimization](@ref) in our online documentation. - -# Extended help - -## Interfaces +# Interfaces - `algorithm(::SemOptimizerNLopt)` - `local_algorithm(::SemOptimizerNLopt)` - `options(::SemOptimizerNLopt)` - `local_options(::SemOptimizerNLopt)` - `equality_constraints(::SemOptimizerNLopt)` - `inequality_constraints(::SemOptimizerNLopt)` - -## Implementation - -Subtype of `SemOptimizer`. """ -struct SemOptimizerNLopt <: SemOptimizer{:NLopt} - algorithm::Symbol - local_algorithm::Union{Symbol, Nothing} - options::Dict{Symbol, Any} - local_options::Dict{Symbol, Any} - equality_constraints::Vector{NLoptConstraint} - inequality_constraints::Vector{NLoptConstraint} -end - -SEM.sem_optimizer_subtype(::Val{:NLopt}) = SemOptimizerNLopt - -############################################################################################ -### Constructor -############################################################################################ - function SemOptimizerNLopt(; algorithm = :LD_LBFGS, local_algorithm = nothing, diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index cd7664537..e3bb98245 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -1,35 +1,30 @@ ############################################################################################ ### Types ############################################################################################ -""" -Connects to `ProximalAlgorithms.jl` as the optimization backend. - -Can be used for regularized SEM, for a tutorial see the online docs on [Regularization](@ref). - -# Constructor +mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} + algorithm::A + operator_g::B + operator_h::C +end +""" SemOptimizerProximal(; algorithm = ProximalAlgorithms.PANOC(), operator_g, operator_h = nothing, kwargs..., + ) + +Connects to `ProximalAlgorithms.jl` as the optimization backend. For more information on +the available algorithms and options, see the online docs on [Regularization](@ref) and +the documentation of [*ProximalAlgorithms.jl*](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) / +[ProximalOperators.jl](https://github.com/JuliaFirstOrder/ProximalOperators.jl). # Arguments -- `algorithm`: optimization algorithm. +- `algorithm`: proximal optimization algorithm. - `operator_g`: proximal operator (e.g., regularization penalty) - `operator_h`: optional second proximal operator - -# Usage -All algorithms and operators from `ProximalAlgorithms.jl` are available, -for more information see the online docs on [Regularization](@ref) and -the documentation of `ProximalAlgorithms.jl` / `ProximalOperators.jl`. """ -mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} - algorithm::A - operator_g::B - operator_h::C -end - SemOptimizerProximal(; algorithm = ProximalAlgorithms.PANOC(), operator_g, diff --git a/src/optimizer/Empty.jl b/src/optimizer/Empty.jl index 51ab5f840..f95c067ce 100644 --- a/src/optimizer/Empty.jl +++ b/src/optimizer/Empty.jl @@ -1,13 +1,11 @@ ############################################################################################ ### Types ############################################################################################ -""" -Empty placeholder for models that don't need -an optimizer part. -# Constructor +""" + SemOptimizer(engine = :Empty) - SemOptimizerEmpty() +Constructs a dummy placeholder optimizer for models that don't need it. """ struct SemOptimizerEmpty <: SemOptimizer{:Empty} end diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index 53b49909e..c4eadc70a 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -21,6 +21,30 @@ sem_optimizer_subtype(engine::Symbol) = sem_optimizer_subtype(Val(engine)) # fallback method for unsupported engines sem_optimizer_subtype(::Val{E}) where {E} = throw_engine_error(E) +""" + SemOptimizer(args...; engine::Symbol = :Optim, kwargs...) + +Constructs a `SemOptimizer` object that can be passed to [`fit`](@ref) for specifying aspects +of the numerical optimization involved in fitting a SEM. + +The keyword `engine` controlls which Julia package is used, with `:Optim` being the default. +- [`optimizer_engines()`](@ref optimizer_engines) prints a list of currently available engines. +- [`optimizer_engine_doc(EngineName)`](@ref optimizer_engine_doc) prints information on the usage of a specific engine. + +More engines become available if specific packages are loaded, for example +[*NLopt.jl*](https://github.com/JuliaOpt/NLopt.jl) (also see [Constrained optimization](@ref) +in the online documentation) or +[*ProximalAlgorithms.jl*](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) +(also see [Regularization](@ref) in the online documentation). + +The arguments `args...` and `kwargs...` are engine-specific and control further +aspects of the optimization process, such as the algorithm, convergence criteria or constraints. +Information on those can be accessed with [`optimizer_engine_doc`](@ref). + +[Custom optimizer types](@ref) shows how to connect the *SEM.jl* package to a completely new optimization engine. +""" +SemOptimizer + # default constructor that dispatches to the engine-specific type SemOptimizer(::Val{E}, args...; kwargs...) where {E} = sem_optimizer_subtype(E)(args...; kwargs...) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 9a5a5139f..9ab477067 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -3,34 +3,36 @@ ############################################################################################ ### Types and Constructor ############################################################################################ -""" - SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} - -Connects to `Optim.jl` as the optimization backend. -# Constructor +# SemOptimizer for the Optim.jl +mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} + algorithm::A + options::B +end - SemOptimizerOptim(; +""" + SemOptimizer(; + engine = :Optim, algorithm = LBFGS(), options = Optim.Options(;f_reltol = 1e-10, x_abstol = 1.5e-8), kwargs...) +Connects to *Optim.jl* as the optimization engine. + +For more information on the available algorithms and options, +see the [*Optim.jl* docs](https://julianlsolvers.github.io/Optim.jl/stable/). + # Arguments -- `algorithm`: optimization algorithm from `Optim.jl` +- `algorithm`: optimization algorithm from *Optim.jl* - `options::Optim.Options`: options for the optimization algorithm -# Usage -All algorithms and options from the Optim.jl library are available, for more information see -the Optim.jl online documentation. - # Examples ```julia -my_optimizer = SemOptimizerOptim() - # hessian based optimization with backtracking linesearch and modified initial step size using Optim, LineSearches -my_newton_optimizer = SemOptimizerOptim( +my_newton_optimizer = SemOptimizer( + engine = :Optim, algorithm = Newton( ;linesearch = BackTracking(order=3), alphaguess = InitialHagerZhang() @@ -38,10 +40,7 @@ my_newton_optimizer = SemOptimizerOptim( ) ``` -# Extended help - -## Constrained optimization - +# Constrained optimization When using the `Fminbox` or `SAMIN` constrained optimization algorithms, the vector or dictionary of lower and upper bounds for each model parameter can be specified via `lower_bounds` and `upper_bounds` keyword arguments. @@ -49,20 +48,7 @@ Alternatively, the `lower_bound` and `upper_bound` keyword arguments can be used the default bound for all non-variance model parameters, and the `variance_lower_bound` and `variance_upper_bound` keyword -- for the variance parameters (the diagonal of the *S* matrix). - -## Interfaces -- `algorithm(::SemOptimizerOptim)` -- `options(::SemOptimizerOptim)` - -## Implementation - -Subtype of `SemOptimizer`. """ -mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} - algorithm::A - options::B -end - SemOptimizerOptim(; algorithm = LBFGS(), options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), From 6ba91f476276ec08fd5f91226c6037b15e2651b8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 4 Feb 2026 17:01:24 -0800 Subject: [PATCH 224/364] optimizer_engine_doc() --- src/optimizer/abstract.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index c4eadc70a..d9e52fdc8 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -75,6 +75,15 @@ into the current session. optimizer_engines() = Symbol[optimizer_engine(opt_type) for opt_type in subtypes(SemOptimizer)] +""" + optimizer_engine_doc(engine::Symbol) + +Shows documentation for the optimizer engine. + +For a list of available engines, call [`optimizer_engines`](@ref). +""" +optimizer_engine_doc(engine) = Base.Docs.doc(sem_optimizer_subtype(engine)) + """ fit([optim::SemOptimizer], model::AbstractSem; [engine::Symbol], start_val = start_val, kwargs...) From 2e5c9b3ccbbe2964e188e50a48dae1761317f834 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 30 Jan 2026 10:39:13 +0100 Subject: [PATCH 225/364] fix proximal extension --- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 26 ++++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index e3bb98245..bf0fd2a2a 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -48,14 +48,9 @@ SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwar SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm ############################################################################ -### Pretty Printing +### Model fitting ############################################################################ -function Base.show(io::IO, struct_inst::SemOptimizerProximal) - print_type_name(io, struct_inst) - print_field_types(io, struct_inst) -end - ## connect to ProximalAlgorithms.jl function ProximalAlgorithms.value_and_gradient(model::AbstractSem, params) grad = similar(params) @@ -106,10 +101,29 @@ function SEM.fit( ) end +############################################################################################ +### additional methods +############################################################################################ + +SEM.algorithm_name(res::ProximalResult) = SEM.algorithm_name(res.optimizer.algorithm) +SEM.algorithm_name( + ::ProximalAlgorithms.IterativeAlgorithm{I, H, S, D, K}, +) where {I, H, S, D, K} = nameof(I) + +SEM.convergence( + ::ProximalResult, +) = "No standard convergence criteria for proximal \n algorithms available." +SEM.n_iterations(res::ProximalResult) = res.n_iterations + ############################################################################################ # pretty printing ############################################################################################ +function Base.show(io::IO, struct_inst::SemOptimizerProximal) + print_type_name(io, struct_inst) + print_field_types(io, struct_inst) +end + function Base.show(io::IO, result::ProximalResult) print(io, "Minimum: $(round(result.result[:minimum]; digits = 2)) \n") print(io, "No. evaluations: $(result.result[:iterations]) \n") From e47a94f5bde2c2c8ed3312afe6fefd93dbab3514 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 4 Feb 2026 17:02:09 -0800 Subject: [PATCH 226/364] don't export SemOptimizerOptim update docs to use SemOptimizer --- docs/src/performance/simulation.md | 2 +- docs/src/tutorials/construction/build_by_parts.md | 2 +- docs/src/tutorials/construction/outer_constructor.md | 2 +- docs/src/tutorials/fitting/fitting.md | 3 +-- src/StructuralEquationModels.jl | 2 -- test/examples/multigroup/multigroup.jl | 4 ++-- .../examples/recover_parameters/recover_parameters_twofact.jl | 2 +- 7 files changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index d268853f6..85a0c0a0c 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -67,7 +67,7 @@ For example, new_observed = SemObservedData(;data = data_2, specification = partable) -my_optimizer = SemOptimizerOptim() +my_optimizer = SemOptimizer() new_optimizer = update_observed(my_optimizer, new_observed) ``` diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 680e28804..52e12f30b 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -59,7 +59,7 @@ ml = SemML(observed = observed) loss_ml = SemLoss(ml) # optimizer ---------------------------------------------------------------------------- -optimizer = SemOptimizerOptim() +optimizer = SemOptimizer() # model -------------------------------------------------------------------------------- diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md index e27724307..e0c69ef3c 100644 --- a/docs/src/tutorials/construction/outer_constructor.md +++ b/docs/src/tutorials/construction/outer_constructor.md @@ -41,7 +41,7 @@ model = Sem( data = data, implied = RAMSymbolic, loss = SemWLS, - optimizer = SemOptimizerOptim + optimizer = SemOptimizer ) ``` diff --git a/docs/src/tutorials/fitting/fitting.md b/docs/src/tutorials/fitting/fitting.md index d7353c9f9..1af03ce8e 100644 --- a/docs/src/tutorials/fitting/fitting.md +++ b/docs/src/tutorials/fitting/fitting.md @@ -17,7 +17,6 @@ Structural Equation Model - Fields observed: SemObservedData implied: RAM - optimizer: SemOptimizerOptim ------------- Optimization result ------------- @@ -60,7 +59,7 @@ The available keyword arguments are listed in the sections [Using Optim.jl](@ref Alternative, you can also explicitely define a `SemOptimizer` and pass it as the first argument to `fit`: ```julia -my_optimizer = SemOptimizerOptim(algorithm = BFGS()) +my_optimizer = SemOptimizer(algorithm = BFGS()) fit(my_optimizer, model) ``` diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index b2e5edd8d..71cedf1c4 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -122,8 +122,6 @@ export AbstractSem, SemWLS, loss, SemOptimizer, - SemOptimizerEmpty, - SemOptimizerOptim, optimizer, optimizer_engine, optimizer_engine_doc, diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 43de554ce..2d43c3d2a 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -86,7 +86,7 @@ start_test = [ fill(0.05, 3) fill(0.01, 3) ] -semoptimizer = SemOptimizerOptim() +semoptimizer = SemOptimizer() @testset "RAMMatrices | constructor | Optim" begin include("build_models.jl") @@ -169,7 +169,7 @@ start_test = [ 0.01 0.05 ] -semoptimizer = SemOptimizerOptim() +semoptimizer = SemOptimizer() @testset "Graph → Partable → RAMMatrices | constructor | Optim" begin include("build_models.jl") diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index ce7dc61ff..9f9503af8 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -68,7 +68,7 @@ loss_ml = SemLoss(SemML(; observed = semobserved, nparams = length(start))) model_ml = Sem(semobserved, implied_ml, loss_ml) objective!(model_ml, true_val) -optimizer = SemOptimizerOptim( +optimizer = SemOptimizer( BFGS(; linesearch = BackTracking(order = 3), alphaguess = InitialHagerZhang()),# m = 100), Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), ) From 309c5789c9aebf0094774265bb345176087403c5 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 4 Feb 2026 17:07:04 -0800 Subject: [PATCH 227/364] SemFit: add opt_engine() to the output --- src/frontend/fit/SemFit.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/frontend/fit/SemFit.jl b/src/frontend/fit/SemFit.jl index 438da4da6..4074f3da3 100644 --- a/src/frontend/fit/SemFit.jl +++ b/src/frontend/fit/SemFit.jl @@ -39,6 +39,10 @@ function Base.show(io::IO, semfit::SemFit) #print(io, "Objective value: $(round(semfit.minimum, digits = 4)) \n") print(io, "------------- Optimization result ------------- \n") print(io, "\n") + print(io, "engine: ") + print(io, optimizer_engine(semfit)) + print(io, "\n") + print(io, "\n") print(io, semfit.optimization_result) end @@ -58,6 +62,7 @@ model(sem_fit::SemFit) = sem_fit.model optimization_result(sem_fit::SemFit) = sem_fit.optimization_result # optimizer properties +optimizer_engine(sem_fit::SemFit) = optimizer_engine(optimization_result(sem_fit)) optimizer(sem_fit::SemFit) = optimizer(optimization_result(sem_fit)) n_iterations(sem_fit::SemFit) = n_iterations(optimization_result(sem_fit)) convergence(sem_fit::SemFit) = convergence(optimization_result(sem_fit)) From 38939b709a942d3166ec5f5624a4115092738854 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 4 Feb 2026 16:52:56 -0800 Subject: [PATCH 228/364] SemOptimizerResult: streamline optim results --- docs/src/developer/optimizer.md | 3 +- ext/SEMNLOptExt/NLopt.jl | 62 +++++++++------------ ext/SEMProximalOptExt/ProximalAlgorithms.jl | 48 +++++----------- src/frontend/fit/SemFit.jl | 4 +- src/frontend/fit/summary.jl | 3 +- src/optimizer/abstract.jl | 4 ++ src/optimizer/optim.jl | 31 +++++------ src/types.jl | 3 + test/examples/proximal/l0.jl | 4 +- test/examples/proximal/lasso.jl | 4 +- 10 files changed, 71 insertions(+), 95 deletions(-) diff --git a/docs/src/developer/optimizer.md b/docs/src/developer/optimizer.md index 7e2b27b51..3e7cadaf8 100644 --- a/docs/src/developer/optimizer.md +++ b/docs/src/developer/optimizer.md @@ -30,7 +30,6 @@ update_observed(optimizer::SemOptimizerName, observed::SemObserved; kwargs...) = ### additional methods ############################################################################################ -algorithm(optimizer::SemOptimizerName) = optimizer.algorithm options(optimizer::SemOptimizerName) = optimizer.options ``` @@ -68,7 +67,7 @@ The method has to return a `SemFit` object that consists of the minimum of the o In addition, you might want to provide methods to access properties of your optimization result: ```julia -optimizer(res::MyOptimizationResult) = ... +algorithm_name(res::MyOptimizationResult) = ... n_iterations(res::MyOptimizationResult) = ... convergence(res::MyOptimizationResult) = ... ``` \ No newline at end of file diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index 1c6b97120..909dbbfc1 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -118,31 +118,32 @@ SEM.update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs. ### additional methods ############################################################################################ -SEM.algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm local_algorithm(optimizer::SemOptimizerNLopt) = optimizer.local_algorithm SEM.options(optimizer::SemOptimizerNLopt) = optimizer.options local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints -struct NLoptResult +# wrapper for the NLopt optimization result +struct NLoptResult <: SEM.SemOptimizerResult{SemOptimizerNLopt} + optimizer::SemOptimizerNLopt result::Any problem::Any end -SEM.optimizer(res::NLoptResult) = res.problem.algorithm +SEM.algorithm_name(res::NLoptResult) = res.problem.algorithm SEM.n_iterations(res::NLoptResult) = res.problem.numevals SEM.convergence(res::NLoptResult) = res.result[3] -# construct SemFit from fitted NLopt object -function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, opt) - return SemFit( - optimization_result[1], - optimization_result[2], - start_val, - model, - NLoptResult(optimization_result, opt), - ) +# construct NLopt.jl problem +function NLopt_problem(algorithm, options, npar) + problem = Opt(algorithm, npar) + + for (key, val) in pairs(options) + setproperty!(problem, key, val) + end + + return problem end # fit method @@ -152,8 +153,8 @@ function SEM.fit( start_params::AbstractVector; kwargs..., ) - opt = construct_NLopt(optim.algorithm, optim.options, nparams(model)) - opt.min_objective = + problem = NLopt_problem(optim.algorithm, optim.options, nparams(model)) + problem.min_objective = (par, G) -> SEM.evaluate!( zero(eltype(par)), !isnothing(G) && !isempty(G) ? G : nothing, @@ -162,36 +163,27 @@ function SEM.fit( par, ) for (f, tol) in optim.inequality_constraints - inequality_constraint!(opt, f, tol) + inequality_constraint!(problem, f, tol) end for (f, tol) in optim.equality_constraints - equality_constraint!(opt, f, tol) + equality_constraint!(problem, f, tol) end if !isnothing(optim.local_algorithm) - opt_local = - construct_NLopt(optim.local_algorithm, optim.local_options, nparams(model)) - opt.local_optimizer = opt_local + problem.local_optimizer = + NLopt_problem(optim.local_algorithm, optim.local_options, nparams(model)) end # fit - result = NLopt.optimize(opt, start_params) + result = NLopt.optimize(problem, start_params) - return SemFit_NLopt(result, model, start_params, opt) -end - -############################################################################################ -### additional functions -############################################################################################ - -function construct_NLopt(algorithm, options, npar) - opt = Opt(algorithm, npar) - - for (key, val) in pairs(options) - setproperty!(opt, key, val) - end - - return opt + return SemFit( + result[1], # minimum + result[2], # optimal params + start_val, + model, + NLoptResult(optim, result, problem), + ) end ############################################################################################ diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index bf0fd2a2a..1d7f83632 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -41,12 +41,6 @@ SEM.sem_optimizer_subtype(::Val{:Proximal}) = SemOptimizerProximal SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwargs...) = optimizer -############################################################################################ -### additional methods -############################################################################################ - -SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm - ############################################################################ ### Model fitting ############################################################################ @@ -58,8 +52,11 @@ function ProximalAlgorithms.value_and_gradient(model::AbstractSem, params) return obj, grad end -mutable struct ProximalResult - result::Any +# wrapper for the Proximal optimization result +struct ProximalResult{O <: SemOptimizer{:Proximal}} <: SEM.SemOptimizerResult{O} + optimizer::O + minimum::Float64 + n_iterations::Int end function SEM.fit( @@ -69,10 +66,10 @@ function SEM.fit( kwargs..., ) if isnothing(optim.operator_h) - solution, iterations = + solution, niterations = optim.algorithm(x0 = start_params, f = model, g = optim.operator_g) else - solution, iterations = optim.algorithm( + solution, niterations = optim.algorithm( x0 = start_params, f = model, g = optim.operator_g, @@ -80,25 +77,9 @@ function SEM.fit( ) end - minimum = objective!(model, solution) + optim_res = ProximalResult(optim, objective!(model, solution), niterations) - optimization_result = Dict( - :minimum => minimum, - :iterations => iterations, - :algorithm => optim.algorithm, - :operator_g => optim.operator_g, - ) - - isnothing(optim.operator_h) || - push!(optimization_result, :operator_h => optim.operator_h) - - return SemFit( - minimum, - solution, - start_params, - model, - ProximalResult(optimization_result), - ) + return SemFit(optim_res.minimum, solution, start_params, model, optim_res) end ############################################################################################ @@ -125,10 +106,9 @@ function Base.show(io::IO, struct_inst::SemOptimizerProximal) end function Base.show(io::IO, result::ProximalResult) - print(io, "Minimum: $(round(result.result[:minimum]; digits = 2)) \n") - print(io, "No. evaluations: $(result.result[:iterations]) \n") - print(io, "Operator: $(nameof(typeof(result.result[:operator_g]))) \n") - if haskey(result.result, :operator_h) - print(io, "Second Operator: $(nameof(typeof(result.result[:operator_h]))) \n") - end + print(io, "Minimum: $(round(result.minimum; digits = 2)) \n") + print(io, "No. evaluations: $(result.n_iterations) \n") + print(io, "Operator: $(nameof(typeof(result.optimizer.operator_g))) \n") + op_h = result.optimizer.operator_h + isnothing(op_h) || print(io, "Second Operator: $(nameof(typeof(op_h))) \n") end diff --git a/src/frontend/fit/SemFit.jl b/src/frontend/fit/SemFit.jl index 4074f3da3..84db81e3a 100644 --- a/src/frontend/fit/SemFit.jl +++ b/src/frontend/fit/SemFit.jl @@ -13,7 +13,7 @@ Fitted structural equation model. - `model(::SemFit)` - `optimization_result(::SemFit)` -- `optimizer(::SemFit)` -> optimization algorithm +- `algorithm_name(::SemFit)` -> optimization algorithm - `n_iterations(::SemFit)` -> number of iterations - `convergence(::SemFit)` -> convergence properties """ @@ -63,6 +63,6 @@ optimization_result(sem_fit::SemFit) = sem_fit.optimization_result # optimizer properties optimizer_engine(sem_fit::SemFit) = optimizer_engine(optimization_result(sem_fit)) -optimizer(sem_fit::SemFit) = optimizer(optimization_result(sem_fit)) +algorithm_name(sem_fit::SemFit) = algorithm_name(optimization_result(sem_fit)) n_iterations(sem_fit::SemFit) = n_iterations(optimization_result(sem_fit)) convergence(sem_fit::SemFit) = convergence(optimization_result(sem_fit)) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 3071d5653..435b17470 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -7,7 +7,8 @@ function details(sem_fit::SemFit; show_fitmeasures = false, color = :light_cyan, color = color, ) print("\n") - println("Optimization algorithm: $(optimizer(sem_fit))") + println("Optimization engine: $(optimizer_engine(sem_fit))") + println("Optimization algorithm: $(algorithm_name(sem_fit))") println("Convergence: $(convergence(sem_fit))") println("No. iterations/evaluations: $(n_iterations(sem_fit))") print("\n") diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index d9e52fdc8..e9a8c47ba 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -84,6 +84,10 @@ For a list of available engines, call [`optimizer_engines`](@ref). """ optimizer_engine_doc(engine) = Base.Docs.doc(sem_optimizer_subtype(engine)) +optimizer(result::SemOptimizerResult) = result.optimizer + +optimizer_engine(result::SemOptimizerResult) = optimizer_engine(result.optimizer) + """ fit([optim::SemOptimizer], model::AbstractSem; [engine::Symbol], start_val = start_val, kwargs...) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 9ab477067..52b27bc88 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -67,26 +67,17 @@ update_observed(optimizer::SemOptimizerOptim, observed::SemObserved; kwargs...) ### additional methods ############################################################################################ -algorithm(optimizer::SemOptimizerOptim) = optimizer.algorithm options(optimizer::SemOptimizerOptim) = optimizer.options -function SemFit( - optimization_result::Optim.MultivariateOptimizationResults, - model::AbstractSem, - start_val, -) - return SemFit( - optimization_result.minimum, - optimization_result.minimizer, - start_val, - model, - optimization_result, - ) +# wrapper for the Optim.jl result +struct SemOptimResult{O <: SemOptimizerOptim} <: SemOptimizerResult{O} + optimizer::O + result::Optim.MultivariateOptimizationResults end -optimizer(res::Optim.MultivariateOptimizationResults) = Optim.summary(res) -n_iterations(res::Optim.MultivariateOptimizationResults) = Optim.iterations(res) -convergence(res::Optim.MultivariateOptimizationResults) = Optim.converged(res) +algorithm_name(res::SemOptimResult) = Optim.summary(res.result) +n_iterations(res::SemOptimResult) = Optim.iterations(res.result) +convergence(res::SemOptimResult) = Optim.converged(res.result) function fit( optim::SemOptimizerOptim, @@ -133,5 +124,11 @@ function fit( optim.options, ) end - return SemFit(result, model, start_params) + return SemFit( + result.minimum, + result.minimizer, + start_params, + model, + SemOptimResult(optim, result), + ) end diff --git a/src/types.jl b/src/types.jl index 92ca2c284..777165f37 100644 --- a/src/types.jl +++ b/src/types.jl @@ -81,6 +81,9 @@ Base.:*(x::SemWeight, y) = x.w * y abstract type SemOptimizer{E} end +# wrapper around optimization result +abstract type SemOptimizerResult{O <: SemOptimizer} end + """ Supertype of all objects that can serve as the observed field of a SEM. Pre-processes data and computes sufficient statistics for example. diff --git a/test/examples/proximal/l0.jl b/test/examples/proximal/l0.jl index f74dfb2d1..8542ac458 100644 --- a/test/examples/proximal/l0.jl +++ b/test/examples/proximal/l0.jl @@ -45,7 +45,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML) fit_prox = fit(model_prox, engine = :Proximal, operator_g = prox_operator) @testset "l0 | solution_unregularized" begin - @test fit_prox.optimization_result.result[:iterations] < 1000 + @test n_iterations(fit_prox.optimization_result) < 1000 @test maximum(abs.(solution(sem_fit) - solution(fit_prox))) < 0.002 end @@ -57,7 +57,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML) fit_prox = fit(model_prox, engine = :Proximal, operator_g = prox_operator) @testset "l0 | solution_regularized" begin - @test fit_prox.optimization_result.result[:iterations] < 1000 + @test n_iterations(fit_prox.optimization_result) < 1000 @test solution(fit_prox)[31] == 0.0 @test abs( StructuralEquationModels.minimum(fit_prox) - diff --git a/test/examples/proximal/lasso.jl b/test/examples/proximal/lasso.jl index 356ac6188..9138f6884 100644 --- a/test/examples/proximal/lasso.jl +++ b/test/examples/proximal/lasso.jl @@ -43,7 +43,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML) fit_prox = fit(model_prox, engine = :Proximal, operator_g = NormL1(λ)) @testset "lasso | solution_unregularized" begin - @test fit_prox.optimization_result.result[:iterations] < 1000 + @test n_iterations(fit_prox.optimization_result) < 1000 @test maximum(abs.(solution(sem_fit) - solution(fit_prox))) < 0.002 end @@ -55,7 +55,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML) fit_prox = fit(model_prox, engine = :Proximal, operator_g = NormL1(λ)) @testset "lasso | solution_regularized" begin - @test fit_prox.optimization_result.result[:iterations] < 1000 + @test n_iterations(fit_prox.optimization_result) < 1000 @test all(solution(fit_prox)[16:20] .< solution(sem_fit)[16:20]) @test StructuralEquationModels.minimum(fit_prox) - StructuralEquationModels.minimum(sem_fit) < 0.03 From ccbf55f508375e4f07ef0e33c1845e733cee92f5 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 27 Jan 2026 14:03:37 -0800 Subject: [PATCH 229/364] docs: fix optimizer engine docs - enable docstrings from extensions - fix references to ext. docstrings --- docs/make.jl | 7 ++++++- docs/src/tutorials/backends/nlopt.md | 2 +- docs/src/tutorials/backends/optim.md | 4 ++-- docs/src/tutorials/concept.md | 25 ++++++++++++++----------- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 1bb68c4da..042b83587 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,7 +1,12 @@ -using Documenter, StructuralEquationModels +using Documenter, StructuralEquationModels, NLopt, ProximalAlgorithms, ProximalOperators + +SEM = StructuralEquationModels +SEMNLOptExt = Base.get_extension(StructuralEquationModels, :SEMNLOptExt) +SEMProximalOptExt = Base.get_extension(StructuralEquationModels, :SEMProximalOptExt) makedocs( sitename = "StructuralEquationModels.jl", + modules = [SEM, SEMNLOptExt, SEMProximalOptExt], pages = [ "index.md", "Tutorials" => [ diff --git a/docs/src/tutorials/backends/nlopt.md b/docs/src/tutorials/backends/nlopt.md index 3ad4bf497..692628fd1 100644 --- a/docs/src/tutorials/backends/nlopt.md +++ b/docs/src/tutorials/backends/nlopt.md @@ -2,7 +2,7 @@ When [`NLopt.jl`](https://github.com/jump-dev/NLopt.jl) is loaded in the running Julia session, it could be used by the [`SemOptimizer`](@ref) by specifying `engine = :NLopt` -(see [NLopt-specific options](@ref `SemOptimizerNLopt`)). +(see [NLopt-specific options](@ref SEMNLOptExt.SemOptimizerNLopt)). Among other things, `NLopt` enables constrained optimization of the SEM models, which is explained in the [Constrained optimization](@ref) section. diff --git a/docs/src/tutorials/backends/optim.md b/docs/src/tutorials/backends/optim.md index a16537ec4..b1b07885e 100644 --- a/docs/src/tutorials/backends/optim.md +++ b/docs/src/tutorials/backends/optim.md @@ -1,7 +1,7 @@ # Using Optim.jl -[Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) is the default optimization engine of *SEM.jl*, -see [`SemOptimizerOptim`](@ref) for a full list of its parameters. +[*Optim.jl*](https://github.com/JuliaNLSolvers/Optim.jl) is the default optimization engine of *SEM.jl*, +see [`SEM.SemOptimizerOptim`](@ref) for a full list of its parameters. It defaults to the LBFGS optimization, but we can load the `Optim` and `LineSearches` packages and specify BFGS (!not L-BFGS) with a back-tracking linesearch and Hager-Zhang initial step length guess: diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index 2b453925a..49f0d404f 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -21,13 +21,13 @@ So everything that can be used as the 'observed' part has to be of type `SemObse Here is an overview on the available building blocks: -|[`SemObserved`](@ref) | [`SemImplied`](@ref) | [`SemLossFunction`](@ref) | [`SemOptimizer`](@ref) | -|---------------------------------|-----------------------|---------------------------|-------------------------------| -| [`SemObservedData`](@ref) | [`RAM`](@ref) | [`SemML`](@ref) | [`SemOptimizerOptim`](@ref) | -| [`SemObservedCovariance`](@ref) | [`RAMSymbolic`](@ref) | [`SemWLS`](@ref) | [`SemOptimizerNLopt`](@ref) | -| [`SemObservedMissing`](@ref) | [`ImpliedEmpty`](@ref)| [`SemFIML`](@ref) | | -| | | [`SemRidge`](@ref) | | -| | | [`SemConstant`](@ref) | | +|[`SemObserved`](@ref) | [`SemImplied`](@ref) | [`SemLossFunction`](@ref) | [`SemOptimizer`](@ref) | +|---------------------------------|-----------------------|---------------------------|----------------------------| +| [`SemObservedData`](@ref) | [`RAM`](@ref) | [`SemML`](@ref) | [:Optim](@ref StructuralEquationModels.SemOptimizerOptim) | +| [`SemObservedCovariance`](@ref) | [`RAMSymbolic`](@ref) | [`SemWLS`](@ref) | [:NLopt](@ref SEMNLOptExt.SemOptimizerNLopt) | +| [`SemObservedMissing`](@ref) | [`ImpliedEmpty`](@ref)| [`SemFIML`](@ref) | [:Proximal](@ref SEMProximalOptExt.SemOptimizerProximal) | +| | | [`SemRidge`](@ref) | | +| | | [`SemConstant`](@ref) | | The rest of this page explains the building blocks for each part. First, we explain every part and give an overview on the different options that are available. After that, the [API - model parts](@ref) section serves as a reference for detailed explanations about the different options. (How to stick them together to a final model is explained in the section on [Model Construction](@ref).) @@ -52,7 +52,7 @@ Available loss functions are ## The optimizer part aka `SemOptimizer` The optimizer part of a model connects to the numerical optimization backend used to fit the model. It can be used to control options like the optimization algorithm, linesearch, stopping criteria, etc. -There are currently three available backends, [`SemOptimizerOptim`](@ref) connecting to the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) backend, [`SemOptimizerNLopt`](@ref) connecting to the [NLopt.jl](https://github.com/JuliaOpt/NLopt.jl) backend and [`SemOptimizerProximal`](@ref) connecting to [ProximalAlgorithms.jl](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl). +There are currently three available engines (i.e., backends used to carry out the numerical optimization), [`:Optim`](@ref StructuralEquationModels.SemOptimizerOptim) connecting to the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) backend, [`:NLopt`](@ref SEMNLOptExt.SemOptimizerNLopt) connecting to the [NLopt.jl](https://github.com/JuliaOpt/NLopt.jl) backend and [`:Proximal`](@ref SEMProximalOptExt.SemOptimizerProximal) connecting to [ProximalAlgorithms.jl](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl). For more information about the available options see also the tutorials about [Using Optim.jl](@ref) and [Using NLopt.jl](@ref), as well as [Constrained optimization](@ref) and [Regularization](@ref) . # What to do next @@ -101,8 +101,11 @@ SemConstant ## optimizer ```@docs +optimizer_engines +optimizer_engine +optimizer_engine_doc SemOptimizer -SemOptimizerOptim -SemOptimizerNLopt -SemOptimizerProximal +SEM.SemOptimizerOptim +SEMNLOptExt.SemOptimizerNLopt +SEMProximalOptExt.SemOptimizerProximal ``` \ No newline at end of file From 13272362d4cf6cb3389cb2759aec3dffe4257b02 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 4 Feb 2026 13:14:44 -0800 Subject: [PATCH 230/364] docs/make.jl: disable doctest temporary until the links are fixed Co-authored-by: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- docs/make.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/make.jl b/docs/make.jl index 042b83587..f3824dd79 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -65,6 +65,7 @@ makedocs( collapselevel = 1, ), doctest = false, + checkdocs = :none, ) # doctest(StructuralEquationModels, fix=true) From 2f2293c5205026178639c3646cacd49f57d3c38d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 2 Feb 2026 19:09:30 -0800 Subject: [PATCH 231/364] optimizer.md: rename to SemOptimizerMyopt --- docs/src/developer/optimizer.md | 43 ++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/docs/src/developer/optimizer.md b/docs/src/developer/optimizer.md index 3e7cadaf8..164fa4f42 100644 --- a/docs/src/developer/optimizer.md +++ b/docs/src/developer/optimizer.md @@ -1,47 +1,56 @@ # Custom optimizer types The optimizer part of a model connects it to the optimization backend. -Let's say we want to implement a new optimizer as `SemOptimizerName`. The first part of the implementation is very similar to loss functions, so we just show the implementation of `SemOptimizerOptim` here as a reference: +Let's say we want to implement a new optimizer as `SemOptimizerMyopt`. +The first part of the implementation is very similar to loss functions, +so we just show the implementation of `SemOptimizerOptim` here as a reference: ```julia ############################################################################################ ### Types and Constructor ############################################################################################ -mutable struct SemOptimizerName{A, B} <: SemOptimizer{:Name} +struct SemOptimizerMyopt{A, B} <: SemOptimizer{:Myopt} algorithm::A options::B end -SEM.sem_optimizer_subtype(::Val{:Name}) = SemOptimizerName +SEM.sem_optimizer_subtype(::Val{:Myopt}) = SemOptimizerMyopt -SemOptimizerName(; +SemOptimizerMyopt(; algorithm = LBFGS(), options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), kwargs..., -) = SemOptimizerName(algorithm, options) +) = SemOptimizerMyopt(algorithm, options) + +struct MyOptResult{O <: SemOptimizerMyopt} <: SEM.SemOptimizerResult{O} + optimizer::O + ... +end ############################################################################################ ### Recommended methods ############################################################################################ -update_observed(optimizer::SemOptimizerName, observed::SemObserved; kwargs...) = optimizer +update_observed(optimizer::SemOptimizerMyopt, observed::SemObserved; kwargs...) = optimizer ############################################################################################ ### additional methods ############################################################################################ -options(optimizer::SemOptimizerName) = optimizer.options +options(optimizer::SemOptimizerMyopt) = optimizer.options ``` -Note that your optimizer is a subtype of `SemOptimizer{:Name}`, where you can choose a `:Name` that can later be used as a keyword argument to `fit(engine = :Name)`. -Similarly, `SemOptimizer{:Name}(args...; kwargs...) = SemOptimizerName(args...; kwargs...)` should be defined as well as a constructor that uses only keyword arguments: +Note that your optimizer is a subtype of `SemOptimizer{:Myopt}`, +where you can choose a `:Myopt` that can later be used as a keyword argument to `fit(engine = :Myopt)`. +Similarly, `SemOptimizer{:Myopt}(args...; kwargs...) = SemOptimizerMyopt(args...; kwargs...)` +should be defined as well as a constructor that uses only keyword arguments: ```julia -SemOptimizerName(; +SemOptimizerMyopt(; algorithm = LBFGS(), options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), kwargs..., -) = SemOptimizerName(algorithm, options) +) = SemOptimizerMyopt(algorithm, options) ``` A method for `update_observed` and additional methods might be usefull, but are not necessary. @@ -49,15 +58,15 @@ Now comes the substantive part: We need to provide a method for `fit`: ```julia function fit( - optim::SemOptimizerName, + optim::SemOptimizerMyopt, model::AbstractSem, start_params::AbstractVector; kwargs..., ) - optimization_result = ... - ... + optimization_result = MyoptResult(optim, ...) + return SemFit(minimum, minimizer, start_params, model, optimization_result) end ``` @@ -67,7 +76,7 @@ The method has to return a `SemFit` object that consists of the minimum of the o In addition, you might want to provide methods to access properties of your optimization result: ```julia -algorithm_name(res::MyOptimizationResult) = ... -n_iterations(res::MyOptimizationResult) = ... -convergence(res::MyOptimizationResult) = ... +algorithm_name(res::MyOptResult) = ... +n_iterations(res::MyOptResult) = ... +convergence(res::MyOptResult) = ... ``` \ No newline at end of file From 5cdcbb1204fc531c64d3a8c596ea9d102fcc7e38 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 2 Feb 2026 19:23:58 -0800 Subject: [PATCH 232/364] docs: apply suggestions --- docs/src/tutorials/backends/nlopt.md | 8 ++++---- .../src/tutorials/regularization/regularization.md | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/src/tutorials/backends/nlopt.md b/docs/src/tutorials/backends/nlopt.md index 692628fd1..8e6b83bee 100644 --- a/docs/src/tutorials/backends/nlopt.md +++ b/docs/src/tutorials/backends/nlopt.md @@ -1,9 +1,9 @@ # Using NLopt.jl When [`NLopt.jl`](https://github.com/jump-dev/NLopt.jl) is loaded in the running Julia session, -it could be used by the [`SemOptimizer`](@ref) by specifying `engine = :NLopt` +it can be used by the [`SemOptimizer`](@ref) by specifying `engine = :NLopt` (see [NLopt-specific options](@ref SEMNLOptExt.SemOptimizerNLopt)). -Among other things, `NLopt` enables constrained optimization of the SEM models, which is +Among other things, `NLopt` enables constrained optimization of SEMs, which is explained in the [Constrained optimization](@ref) section. We can override the default *NLopt* algorithm (LFBGS) and instead use @@ -28,9 +28,9 @@ To see how to use the optimizer to actually fit a model now, check out the [Mode In the *NLopt* docs, you can find details about the [optimization algorithms](https://nlopt.readthedocs.io/en/latest/NLopt_Algorithms/), and the [tutorial](https://nlopt.readthedocs.io/en/latest/NLopt_Introduction/) that demonstrates how to tweak their behavior. -To choose an algorithm, just pass its name without the 'NLOPT\_' prefix (for example, 'NLOPT\_LD\_SLSQP' can be used by passing `algorithm = :LD_SLSQP`). +To choose an algorithm, just pass its name without the `NLOPT_` prefix (for example, `NLOPT_LD_SLSQP` can be used by passing `algorithm = :LD_SLSQP`). -The README of the [*NLopt.jl*](https://github.com/JuliaOpt/NLopt.jl) may also be helpful, and provides a list of options: +The *README* of [*NLopt.jl*](https://github.com/JuliaOpt/NLopt.jl) may also be helpful, and provides a list of options: - `algorithm` - `stopval` diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index 17add030a..79e301c21 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -6,8 +6,7 @@ For ridge regularization, you can simply use `SemRidge` as an additional loss fu (for example, a model with the loss functions `SemML` and `SemRidge` corresponds to ridge-regularized maximum likelihood estimation). You can define lasso, elastic net and other forms of regularization using [`ProximalOperators.jl`](https://github.com/JuliaFirstOrder/ProximalOperators.jl) -and optimize the SEM model with [`ProximalAlgorithms.jl`](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) -that provides so-called *proximal optimization* algorithms. +and optimize the SEM with so-called *proximal optimization* algorithms from [`ProximalAlgorithms.jl`](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl). ```@setup reg using StructuralEquationModels, ProximalAlgorithms, ProximalOperators @@ -23,8 +22,8 @@ using StructuralEquationModels, ProximalAlgorithms, ProximalOperators ## Proximal optimization -With *ProximalAlgorithms* package loaded, it is now possible to use `:Proximal` optimization engine -in `SemOptimizer` for estimating regularized models. +With the *ProximalAlgorithms* package loaded, it is now possible to use the `:Proximal` +optimization engine in `SemOptimizer` for estimating regularized models. ```julia SemOptimizer(; @@ -138,15 +137,16 @@ fit_lasso2 = fit(model; engine = :Proximal, operator_g = NormL1(λ)) ## Second example - mixed l1 and l0 regularization You can choose to penalize different parameters with different types of regularization functions. -Let's use the *lasso* (*l1*) again on the covariances, but additionally penalize the error variances of the observed items via *l0* regularization. +Let's use the *lasso* (*l1*) again on the covariances, but additionally penalize the error variances of +the observed items via *l0* regularization. The *l0* penalty is defined as ```math l_0 = \lambda \mathrm{nnz}(\theta) ``` -Since we apply *l1* and *l0* to the disjoint sets of parameters, this regularization could be represented as -as sum of *separable proximal operators* (i.e. no parameter is penalized twice) +Since we apply *l1* and *l0* to the disjoint sets of parameters, this regularization can be +represented as a sum of *separable proximal operators* (i.e. no parameter is penalized twice) implemented by the [`SlicedSeparableSum`](https://juliafirstorder.github.io/ProximalOperators.jl/stable/calculus/#ProximalOperators.SlicedSeparableSum) operator: ```@example reg From f1f453cf5ce8f7175ea8a045cf750e3716514509 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 5 Feb 2026 15:53:22 -0800 Subject: [PATCH 233/364] dovs/optimizer.md: more updates for the new API --- docs/src/developer/optimizer.md | 55 +++++++++++++++++---------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/docs/src/developer/optimizer.md b/docs/src/developer/optimizer.md index 164fa4f42..b5c9a6e09 100644 --- a/docs/src/developer/optimizer.md +++ b/docs/src/developer/optimizer.md @@ -1,9 +1,7 @@ # Custom optimizer types -The optimizer part of a model connects it to the optimization backend. +The optimizer part of a model connects it to the optimization engine. Let's say we want to implement a new optimizer as `SemOptimizerMyopt`. -The first part of the implementation is very similar to loss functions, -so we just show the implementation of `SemOptimizerOptim` here as a reference: ```julia ############################################################################################ @@ -17,12 +15,12 @@ end SEM.sem_optimizer_subtype(::Val{:Myopt}) = SemOptimizerMyopt SemOptimizerMyopt(; - algorithm = LBFGS(), - options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), + algorithm = ..., + options = ..., kwargs..., ) = SemOptimizerMyopt(algorithm, options) -struct MyOptResult{O <: SemOptimizerMyopt} <: SEM.SemOptimizerResult{O} +struct MyoptResult{O <: SemOptimizerMyopt} <: SEM.SemOptimizerResult{O} optimizer::O ... end @@ -40,21 +38,15 @@ update_observed(optimizer::SemOptimizerMyopt, observed::SemObserved; kwargs...) options(optimizer::SemOptimizerMyopt) = optimizer.options ``` -Note that your optimizer is a subtype of `SemOptimizer{:Myopt}`, -where you can choose a `:Myopt` that can later be used as a keyword argument to `fit(engine = :Myopt)`. -Similarly, `SemOptimizer{:Myopt}(args...; kwargs...) = SemOptimizerMyopt(args...; kwargs...)` -should be defined as well as a constructor that uses only keyword arguments: +Note that `SemOptimizerMyopt` is defined as a subtype of [`SemOptimizer{:Myopt}`](@ref SEM.SemOptimizer)`, +and `SEM.sem_optimizer_subtype(::Val{:Myopt})` returns `SemOptimizerMyopt`. +This instructs *SEM.jl* to use `SemOptimizerMyopt` when `:Myopt` is specified as the engine for +model fitting: `fit(..., engine = :Myopt)`. -```julia -SemOptimizerMyopt(; - algorithm = LBFGS(), - options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), - kwargs..., -) = SemOptimizerMyopt(algorithm, options) -``` A method for `update_observed` and additional methods might be usefull, but are not necessary. -Now comes the substantive part: We need to provide a method for `fit`: +Now comes the essential part: we need to provide the [`fit`](@ref) method with `SemOptimizerMyopt` +as the first positional argument. ```julia function fit( @@ -63,20 +55,29 @@ function fit( start_params::AbstractVector; kwargs..., ) - ... + # ... prepare the Myopt optimization problem - optimization_result = MyoptResult(optim, ...) + myopt_res = ... # fit the problem with the Myopt engine + minimum = ... # extract the minimum from myopt_res + minimizer = ... # extract the solution (parameter estimates) + optim_result = MyoptResult(optim, myopt_res, ...) # store the original Myopt result and params - return SemFit(minimum, minimizer, start_params, model, optimization_result) + return SemFit(minimum, minimizer, start_params, model, optim_result) end ``` -The method has to return a `SemFit` object that consists of the minimum of the objective at the solution, the minimizer (aka parameter estimates), the starting values, the model and the optimization result (which may be anything you desire for your specific backend). +This method is responsible for converting the SEM into the format required by your optimization engine, +running the optimization, extracting the solution and returning the `SemFit` object, which should package: +* the minimum of the objective at the solution +* the minimizer (the vector of the SEM parameter estimates) +* the starting values +* the SEM model +* `MyoptResult` object with any relevant engine-specific details you want to preserve -In addition, you might want to provide methods to access properties of your optimization result: +In addition, you might want to provide methods to access engine-specific properties stored in `MyoptResult`: ```julia -algorithm_name(res::MyOptResult) = ... -n_iterations(res::MyOptResult) = ... -convergence(res::MyOptResult) = ... -``` \ No newline at end of file +algorithm_name(res::MyoptResult) = ... +n_iterations(res::MyoptResult) = ... +convergence(res::MyoptResult) = ... +``` From 1f8d2a9a91fa31143c68054f34d047307dbf4695 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 5 Feb 2026 16:41:08 -0800 Subject: [PATCH 234/364] .gh/FormatCheck: run on pull_request Fixes the format checking, because when run in pull_request_target, it does not check out the correct commit --- .github/workflows/FormatCheck.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/FormatCheck.yml b/.github/workflows/FormatCheck.yml index d005ca942..5612cd8d3 100644 --- a/.github/workflows/FormatCheck.yml +++ b/.github/workflows/FormatCheck.yml @@ -1,8 +1,7 @@ name: Format suggestions on: - pull_request_target: - # this argument is not required if you don't use the `suggestion-label` input - types: [ opened, reopened, synchronize, labeled, unlabeled ] + push: # Runs on all pushes to any branch + pull_request: # Runs on all PR events (open, sync, reopen) jobs: code-style: runs-on: ubuntu-latest From 811ae0f1d195509e061ae34037809b94874dc883 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 13 Feb 2026 10:50:57 +0100 Subject: [PATCH 235/364] improve meanstructure error message --- src/implied/abstract.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/implied/abstract.jl b/src/implied/abstract.jl index cff4d58a4..6d298f65c 100644 --- a/src/implied/abstract.jl +++ b/src/implied/abstract.jl @@ -41,7 +41,7 @@ function check_meanstructure_specification(meanstructure, ram_matrices) end if !meanstructure & !isnothing(ram_matrices.M) throw(ArgumentError( - "If your model specification contains mean parameters, you have to set `meanstructure = true`." + "If your model specification contains mean parameters, you have to set `Sem(..., meanstructure = true)`." )) end end \ No newline at end of file From fc2bc1664ecb1085197511876b72f250a2630693 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 13 Feb 2026 11:10:44 +0100 Subject: [PATCH 236/364] add docstring for --- src/frontend/fit/SemFit.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/frontend/fit/SemFit.jl b/src/frontend/fit/SemFit.jl index 438da4da6..eeaf8be9f 100644 --- a/src/frontend/fit/SemFit.jl +++ b/src/frontend/fit/SemFit.jl @@ -52,6 +52,11 @@ nsamples(fit::SemFit) = nsamples(fit.model) # access fields minimum(sem_fit::SemFit) = sem_fit.minimum +""" + solution(sem_fit::SemFit) + +Returns the vector of parameter estimates from a fitted SEM. +""" solution(sem_fit::SemFit) = sem_fit.solution start_val(sem_fit::SemFit) = sem_fit.start_val model(sem_fit::SemFit) = sem_fit.model From 61acab6cf6659eeb9651dcc5ac89d677ff06a249 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 13 Feb 2026 11:51:17 +0100 Subject: [PATCH 237/364] add docstring for --- src/objective_gradient_hessian.jl | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index 06f39329f..69915ffa9 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -23,6 +23,31 @@ is_hessian_required(::EvaluationTargets{<:Any, <:Any, H}) where {H} = H (targets::EvaluationTargets)(arg_tuple::Tuple) = targets(arg_tuple...) +""" + evaluate!(objective, gradient, hessian [, lossfun], model, params) + +Evaluates the objective, gradient, and/or Hessian at the given parameter vector. +If a loss function is passed, only this specific loss function is evaluated, otherwise, +the sum of all loss functions in the model is evaluated. + +If objective, gradient or hessian are `nothing`, they are not evaluated. +For example, since many numerical optimization algorithms don't require a Hessian, +the computation will be turned off by setting `hessian` to `nothing`. + +# Arguments +- `objective`: a Number if the objective should be evaluated, otherwise `nothing` +- `gradient`: a pre-allocated vector the gradient should be written to, otherwise `nothing` +- `hessian`: a pre-allocated matrix the Hessian should be written to, otherwise `nothing` +- `lossfun::SemLossFunction`: loss function to evaluate +- `model::AbstractSem`: model to evaluate +- `params`: vector of parameters + +# Implementing a new loss function +To implement a new loss function, a new method for `evaluate!` has to be defined. +This is explained in the online documentation on [Custom loss functions](@ref). +""" +function evaluate! end + # dispatch on SemImplied evaluate!(objective, gradient, hessian, loss::SemLossFunction, model::AbstractSem, params) = evaluate!(objective, gradient, hessian, loss, implied(model), model, params) From 90c6cf32cff31bf7d5275f553d8727fe005d0e2f Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sat, 14 Feb 2026 15:35:09 +0100 Subject: [PATCH 238/364] fix printing of Optim solution --- src/optimizer/optim.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 52b27bc88..83ebbe5e1 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -132,3 +132,9 @@ function fit( SemOptimResult(optim, result), ) end + +############################################################################################ +# pretty printing +############################################################################################ + +Base.show(io::IO, result::SemOptimResult) = Base.show(io, result.result) From 57ec89bbc04a34360fba1b83dc6413f438050932 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:13:15 +0100 Subject: [PATCH 239/364] add helper functions for heterogenous lossfuns and scaling corrections --- src/additional_functions/helper.jl | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index d6a1fc6c8..b6de2e34c 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -89,3 +89,37 @@ function nonunique(values::AbstractVector) end return res end + +# check that a model only has a single lossfun +function check_single_lossfun(model::AbstractSemSingle; throw_error) + if (length(model.loss.functions) > 1) & throw_error + @error "The model has $(length(sem.loss.functions)) loss functions. + Only a single loss function is supported." + end + return isone(length(model.loss.functions)) +end + +# check that all models use the same single loss function +function check_single_lossfun(models::AbstractSemSingle...; throw_error) + uniform = true + lossfun = models[1].loss.functions[1] + L = typeof(lossfun) + for (i, model) in enumerate(models) + uniform &= check_single_lossfun(model; throw_error = throw_error) + cur_lossfun = model.loss.functions[1] + if !isa(cur_lossfun, L) & throw_error + @error "Loss function for group #$i model is $(typeof(cur_lossfun)), expected $L. + Heterogeneous loss functions are not supported." + end + uniform &= isa(cur_lossfun, L) + end + return uniform +end + +check_single_lossfun(model::SemEnsemble; throw_error) = + check_single_lossfun(model.sems...; throw_error) + +# sclaing corrections for fit measures and multigroup models +dof_correction(::SemFIML) = 0 +dof_correction(::SemML) = -1 +dof_correction(::SemWLS) = -1 \ No newline at end of file From 1db3106cd2c501b95846a9d59b42450f25f13c71 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:14:09 +0100 Subject: [PATCH 240/364] adapt default multigroup weights and give info about defaults used --- src/types.jl | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/types.jl b/src/types.jl index 777165f37..5dc7c524c 100644 --- a/src/types.jl +++ b/src/types.jl @@ -192,10 +192,7 @@ end function SemEnsemble(models...; weights = nothing, groups = nothing, kwargs...) n = length(models) # default weights - if isnothing(weights) - nsamples_total = sum(nsamples, models) - weights = [nsamples(model) / nsamples_total for model in models] - end + weights = isnothing(weights) ? multigroup_weights(models, n) : weights # default group labels groups = isnothing(groups) ? Symbol.(:g, 1:n) : groups # check parameters equality @@ -226,7 +223,25 @@ function SemEnsemble(; specification, data, groups, column = :group, kwargs...) model = Sem(; specification = ram_matrices, data = data_group, kwargs...) push!(models, model) end - return SemEnsemble(models...; weights = nothing, groups = groups, kwargs...) + return SemEnsemble(models...; groups = groups, kwargs...) +end + +function multigroup_weights(models, n) + nsamples_total = sum(nsamples, models) + uniform_lossfun = check_single_lossfun(models...; throw_error = false) + if !uniform_lossfun + @info "Your ensemble model contains heterogeneous loss functions. + Default weights of (#samples per group/#total samples) will be used". + return [(nsamples(model)) / (nsamples_total) for model in models] + end + lossfun = models[1].loss.functions[1] + if !applicable(dof_correction, lossfun) + @info "We don't know how to choose group weights for the specified loss function. + Default weights of (#samples per group/#total samples) will be used". + return [(nsamples(model)) / (nsamples_total) for model in models] + end + dc = dof_correction(lossfun) + return [(nsamples(model)-dc) / (nsamples_total-n*dc) for model in models] end param_labels(ensemble::SemEnsemble) = ensemble.param_labels From 8b5de2ea5f2ae8584da2b89a04a887c28cda9fe2 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:20:02 +0100 Subject: [PATCH 241/364] refactor minus2ll --- src/frontend/fit/fitmeasures/minus2ll.jl | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 9b211fb44..888993817 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -3,36 +3,31 @@ Return the negative 2* log likelihood. """ -function minus2ll end +minus2ll(fit::SemFit) = minus2ll(fit, fit.model) ############################################################################################ # Single Models ############################################################################################ -minus2ll(fit::SemFit) = minus2ll(fit, fit.model) - function minus2ll(fit::SemFit, model::AbstractSemSingle) - minimum = objective(model, fit.solution) - return minus2ll(minimum, model) + check_single_lossfun(model; throw_error = true) + return minus2ll(model.loss.functions[1], fit, model) end -minus2ll(minimum::Number, model::AbstractSemSingle) = - sum(lossfun -> minus2ll(lossfun, minimum, model), model.loss.functions) - # SemML ------------------------------------------------------------------------------------ -function minus2ll(lossfun::SemML, minimum::Number, model::AbstractSemSingle) +function minus2ll(::SemML, fit::SemFit, model::AbstractSemSingle) obs = observed(model) - return nsamples(obs) * (minimum + log(2π) * nobserved_vars(obs)) + return nsamples(obs) * (fit.minimum + log(2π) * nobserved_vars(obs)) end # WLS -------------------------------------------------------------------------------------- -minus2ll(lossfun::SemWLS, minimum::Number, model::AbstractSemSingle) = missing +minus2ll(::SemWLS, ::SemFit, ::AbstractSemSingle) = missing # compute likelihood for missing data - H0 ------------------------------------------------- -# -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n -function minus2ll(lossfun::SemFIML, minimum::Number, model::AbstractSemSingle) +# -2ll = (∑ log(2π)*(nᵢ*mᵢ)) + F*n +function minus2ll(::SemFIML, fit::SemFit, model::AbstractSemSingle) obs = observed(model)::SemObservedMissing - F = minimum * nsamples(obs) + F = fit.minimum * nsamples(obs) F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), obs.patterns) return F end From f13704ed8f7057f763279478f2ea0dd9088d1ade Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:32:45 +0100 Subject: [PATCH 242/364] refactor minus2ll --- src/frontend/fit/fitmeasures/minus2ll.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 888993817..4547738b8 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -62,4 +62,7 @@ end # Collection ############################################################################################ -minus2ll(fit::SemFit, model::SemEnsemble) = sum(Base.Fix1(minus2ll, fit), model.sems) +function minus2ll(fit::SemFit, model::SemEnsemble) + check_single_lossfun(model; throw_error = true) + return sum(Base.Fix1(minus2ll, fit), model.sems) +end From 618e2df755cdf18fdee4131661c2ae76c5e52482 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:32:59 +0100 Subject: [PATCH 243/364] refactor chi2 --- src/frontend/fit/fitmeasures/chi2.jl | 44 ++++++++++------------------ 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index dc19467fc..bd76b6ce1 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -9,20 +9,21 @@ Return the χ² value. # Single Models ############################################################################################ -χ²(fit::SemFit, model::AbstractSemSingle) = - sum(loss -> χ²(loss, fit, model), model.loss.functions) +function χ²(fit::SemFit, model::AbstractSemSingle) + check_single_lossfun(model; throw_error = true) + return χ²(model.loss.functions[1], fit::SemFit, model::AbstractSemSingle) +end -# RAM + SemML -χ²(lossfun::SemML, fit::SemFit, model::AbstractSemSingle) = +χ²(::SemML, fit::SemFit, model::AbstractSemSingle) = (nsamples(fit) - 1) * (fit.minimum - logdet(obs_cov(observed(model))) - nobserved_vars(observed(model))) # bollen, p. 115, only correct for GLS weight matrix -χ²(lossfun::SemWLS, fit::SemFit, model::AbstractSemSingle) = +χ²(::SemWLS, fit::SemFit, model::AbstractSemSingle) = (nsamples(fit) - 1) * fit.minimum # FIML -function χ²(lossfun::SemFIML, fit::SemFit, model::AbstractSemSingle) +function χ²(::SemFIML, fit::SemFit, model::AbstractSemSingle) ll_H0 = minus2ll(fit) ll_H1 = minus2ll(observed(model)) return ll_H0 - ll_H1 @@ -32,38 +33,25 @@ end # Collections ############################################################################################ -function χ²(fit::SemFit, models::SemEnsemble) - isempty(models.sems) && return 0.0 - - lossfun = models.sems[1].loss.functions[1] - # check that all models use the same single loss function - L = typeof(lossfun) - for (i, sem) in enumerate(models.sems) - if length(sem.loss.functions) > 1 - @error "Model for group #$i has $(length(sem.loss.functions)) loss functions. Only the single one is supported" - end - cur_lossfun = sem.loss.functions[1] - if !isa(cur_lossfun, L) - @error "Loss function for group #$i model is $(typeof(cur_lossfun)), expected $L. Heterogeneous loss functions are not supported" - end - end - - return χ²(lossfun, fit, models) +function χ²(fit::SemFit, model::SemEnsemble) + check_single_lossfun(model; throw_error = true) + lossfun = model.sems[1].loss.functions[1] + return χ²(lossfun, fit, model) end -function χ²(lossfun::SemWLS, fit::SemFit, models::SemEnsemble) - return (nsamples(models) - 1) * fit.minimum +function χ²(::SemWLS, fit::SemFit, models::SemEnsemble) + return (nsamples(models) - models.n) * fit.minimum end -function χ²(lossfun::SemML, fit::SemFit, models::SemEnsemble) +function χ²(::SemML, fit::SemFit, models::SemEnsemble) G = sum(zip(models.weights, models.sems)) do (w, model) data = observed(model) w * (logdet(obs_cov(data)) + nobserved_vars(data)) end - return (nsamples(models) - 1) * (fit.minimum - G) + return (nsamples(models) - models.n) * (fit.minimum - G) end -function χ²(lossfun::SemFIML, fit::SemFit, models::SemEnsemble) +function χ²(::SemFIML, fit::SemFit, models::SemEnsemble) ll_H0 = minus2ll(fit) ll_H1 = sum(minus2ll ∘ observed, models.sems) return ll_H0 - ll_H1 From c63ca4c4e4a8babe75bcafc06897afb8c579310f Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:33:16 +0100 Subject: [PATCH 244/364] refactor RMSEA --- src/frontend/fit/fitmeasures/RMSEA.jl | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index f9dae84ed..553bdca85 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -7,13 +7,20 @@ function RMSEA end RMSEA(fit::SemFit) = RMSEA(fit, fit.model) -RMSEA(fit::SemFit, model::AbstractSemSingle) = RMSEA(dof(fit), χ²(fit), nsamples(fit)) +function RMSEA(fit::SemFit, model::AbstractSemSingle) + check_uniform_lossfun(model) + return RMSEA(dof(fit), χ²(fit), nsamples(fit)-dof_correction(model.loss.functions[1])) +end -RMSEA(fit::SemFit, model::SemEnsemble) = - sqrt(length(model.sems)) * RMSEA(dof(fit), χ²(fit), nsamples(fit)) +function RMSEA(fit::SemFit, model::SemEnsemble) + check_single_lossfun(model; throw_error = true) + n = nsamples(fit)-model.n*dof_correction(model.sems[1].loss.functions[1]) + return sqrt(length(model.sems)) * RMSEA(dof(fit), χ²(fit), n) +end -function RMSEA(dof, chi2, nsamples) - rmsea = (chi2 - dof) / (nsamples * dof) - rmsea > 0 ? nothing : rmsea = 0 +function RMSEA(dof, chi2, c) + rmsea = (chi2 - dof) / (c * dof) + rmsea = rmsea > 0 ? rmsea : 0 return sqrt(rmsea) end + From e8c867b7d7e1f300782b8cf13bdf88f2c3a82448 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 16 Feb 2026 15:00:26 +0100 Subject: [PATCH 245/364] adapt mg models and fitmeasures --- src/additional_functions/helper.jl | 8 ++++---- src/frontend/fit/fitmeasures/RMSEA.jl | 14 +++++++++----- src/frontend/fit/fitmeasures/chi2.jl | 13 ++++++++----- src/frontend/fit/fitmeasures/minus2ll.jl | 14 +++++++------- src/types.jl | 6 +++--- 5 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index b6de2e34c..0bd5b53f3 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -119,7 +119,7 @@ end check_single_lossfun(model::SemEnsemble; throw_error) = check_single_lossfun(model.sems...; throw_error) -# sclaing corrections for fit measures and multigroup models -dof_correction(::SemFIML) = 0 -dof_correction(::SemML) = -1 -dof_correction(::SemWLS) = -1 \ No newline at end of file +# scaling corrections for multigroup models +mg_correction(::SemFIML) = 0 +mg_correction(::SemML) = 0 +mg_correction(::SemWLS) = -1 \ No newline at end of file diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index 553bdca85..9059aa1db 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -8,19 +8,23 @@ function RMSEA end RMSEA(fit::SemFit) = RMSEA(fit, fit.model) function RMSEA(fit::SemFit, model::AbstractSemSingle) - check_uniform_lossfun(model) - return RMSEA(dof(fit), χ²(fit), nsamples(fit)-dof_correction(model.loss.functions[1])) + check_single_lossfun(model; throw_error = true) + return RMSEA(dof(fit), χ²(fit), nsamples(fit)+rmsea_correction(model.loss.functions[1])) end function RMSEA(fit::SemFit, model::SemEnsemble) check_single_lossfun(model; throw_error = true) - n = nsamples(fit)-model.n*dof_correction(model.sems[1].loss.functions[1]) + n = nsamples(fit)+model.n*rmsea_correction(model.sems[1].loss.functions[1]) return sqrt(length(model.sems)) * RMSEA(dof(fit), χ²(fit), n) end -function RMSEA(dof, chi2, c) - rmsea = (chi2 - dof) / (c * dof) +function RMSEA(dof, chi2, N⁻) + rmsea = (chi2 - dof) / (N⁻ * dof) rmsea = rmsea > 0 ? rmsea : 0 return sqrt(rmsea) end +# scaling corrections +rmsea_correction(::SemFIML) = 0 +rmsea_correction(::SemML) = -1 +rmsea_correction(::SemWLS) = -1 \ No newline at end of file diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index bd76b6ce1..9ebb06bd9 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -16,7 +16,7 @@ end χ²(::SemML, fit::SemFit, model::AbstractSemSingle) = (nsamples(fit) - 1) * - (fit.minimum - logdet(obs_cov(observed(model))) - nobserved_vars(observed(model))) + (fit.minimum - logdet(obs_cov(observed(model))) - nobserved_vars(model)) # bollen, p. 115, only correct for GLS weight matrix χ²(::SemWLS, fit::SemFit, model::AbstractSemSingle) = @@ -44,11 +44,14 @@ function χ²(::SemWLS, fit::SemFit, models::SemEnsemble) end function χ²(::SemML, fit::SemFit, models::SemEnsemble) - G = sum(zip(models.weights, models.sems)) do (w, model) - data = observed(model) - w * (logdet(obs_cov(data)) + nobserved_vars(data)) + F = 0 + for model in models.sems + Fᵢ = objective(model, fit.solution) + Fᵢ -= logdet(obs_cov(observed(model))) + nobserved_vars(model) + Fᵢ *= nsamples(model) - 1 + F += Fᵢ end - return (nsamples(models) - models.n) * (fit.minimum - G) + return F end function χ²(::SemFIML, fit::SemFit, models::SemEnsemble) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 4547738b8..961822ef5 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -11,23 +11,23 @@ minus2ll(fit::SemFit) = minus2ll(fit, fit.model) function minus2ll(fit::SemFit, model::AbstractSemSingle) check_single_lossfun(model; throw_error = true) - return minus2ll(model.loss.functions[1], fit, model) + F = objective(model, fit.solution) + return minus2ll(model.loss.functions[1], F, model) end # SemML ------------------------------------------------------------------------------------ -function minus2ll(::SemML, fit::SemFit, model::AbstractSemSingle) - obs = observed(model) - return nsamples(obs) * (fit.minimum + log(2π) * nobserved_vars(obs)) +function minus2ll(::SemML, F, model::AbstractSemSingle) + return nsamples(model) * (F + log(2π) * nobserved_vars(model)) end # WLS -------------------------------------------------------------------------------------- -minus2ll(::SemWLS, ::SemFit, ::AbstractSemSingle) = missing +minus2ll(::SemWLS, F, ::AbstractSemSingle) = missing # compute likelihood for missing data - H0 ------------------------------------------------- # -2ll = (∑ log(2π)*(nᵢ*mᵢ)) + F*n -function minus2ll(::SemFIML, fit::SemFit, model::AbstractSemSingle) +function minus2ll(::SemFIML, F, model::AbstractSemSingle) obs = observed(model)::SemObservedMissing - F = fit.minimum * nsamples(obs) + F *= nsamples(obs) F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), obs.patterns) return F end diff --git a/src/types.jl b/src/types.jl index 5dc7c524c..3f695bfa3 100644 --- a/src/types.jl +++ b/src/types.jl @@ -235,13 +235,13 @@ function multigroup_weights(models, n) return [(nsamples(model)) / (nsamples_total) for model in models] end lossfun = models[1].loss.functions[1] - if !applicable(dof_correction, lossfun) + if !applicable(mg_correction, lossfun) @info "We don't know how to choose group weights for the specified loss function. Default weights of (#samples per group/#total samples) will be used". return [(nsamples(model)) / (nsamples_total) for model in models] end - dc = dof_correction(lossfun) - return [(nsamples(model)-dc) / (nsamples_total-n*dc) for model in models] + c = mg_correction(lossfun) + return [(nsamples(model)+c) / (nsamples_total+n*c) for model in models] end param_labels(ensemble::SemEnsemble) = ensemble.param_labels From f4339244b07ad079743a0f27077cd0a245931fd7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 23:30:13 +0000 Subject: [PATCH 246/364] build(deps): bump actions/checkout from 4 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/CI.yml | 2 +- .github/workflows/CI_ecosystem.yml | 2 +- .github/workflows/CI_extended.yml | 2 +- .github/workflows/documentation.yml | 2 +- .github/workflows/preview-documentation.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index bddf77dd1..c84df96d3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -24,7 +24,7 @@ jobs: arch: - x64 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.version }} diff --git a/.github/workflows/CI_ecosystem.yml b/.github/workflows/CI_ecosystem.yml index 9c51db21e..dad19df84 100644 --- a/.github/workflows/CI_ecosystem.yml +++ b/.github/workflows/CI_ecosystem.yml @@ -27,7 +27,7 @@ jobs: arch: - x64 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.version }} diff --git a/.github/workflows/CI_extended.yml b/.github/workflows/CI_extended.yml index 200311ec0..c72af0364 100644 --- a/.github/workflows/CI_extended.yml +++ b/.github/workflows/CI_extended.yml @@ -27,7 +27,7 @@ jobs: arch: - x64 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.version }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index f0f2570a6..29d182f79 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -14,7 +14,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: julia-actions/setup-julia@latest with: version: '1' diff --git a/.github/workflows/preview-documentation.yml b/.github/workflows/preview-documentation.yml index b85e6154f..da5f72721 100644 --- a/.github/workflows/preview-documentation.yml +++ b/.github/workflows/preview-documentation.yml @@ -13,7 +13,7 @@ jobs: if: startsWith(github.head_ref, 'documentation/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: julia-actions/setup-julia@latest with: version: '1' From f1401bdb813299e7c69daf516632e4e7f2864818 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 23:09:28 +0000 Subject: [PATCH 247/364] build(deps): bump actions/github-script from 7 to 8 Bumps [actions/github-script](https://github.com/actions/github-script) from 7 to 8. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v7...v8) --- updated-dependencies: - dependency-name: actions/github-script dependency-version: '8' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/preview-documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-documentation.yml b/.github/workflows/preview-documentation.yml index da5f72721..9a9519d47 100644 --- a/.github/workflows/preview-documentation.yml +++ b/.github/workflows/preview-documentation.yml @@ -26,7 +26,7 @@ jobs: DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key run: julia --project=docs/ docs/make.jl push_preview=true - - uses: actions/github-script@v7 + - uses: actions/github-script@v8 if: steps.deploy.outcome == 'success' with: github-token: ${{secrets.GITHUB_TOKEN}} From 3cd1a6d8a6d2c0d96d6ecb75f51a0b71740bb5bc Mon Sep 17 00:00:00 2001 From: Aaron Peikert Date: Wed, 18 Feb 2026 08:56:30 +0100 Subject: [PATCH 248/364] require StenoGraphs v0.5; closes #289 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 0cc596b94..73691e6c2 100644 --- a/Project.toml +++ b/Project.toml @@ -27,7 +27,7 @@ SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b" [compat] julia = "1.9, 1.10, 1.11" -StenoGraphs = "0.2 - 0.3, 0.4.1 - 0.5" +StenoGraphs = "0.5" DataFrames = "1" Distributions = "0.25" FiniteDiff = "2" From c116954c92dbc5d4b79f571fd306a6fdfb323503 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Wed, 18 Feb 2026 13:12:22 +0100 Subject: [PATCH 249/364] remove unused Stenograph exports --- src/StructuralEquationModels.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 71cedf1c4..2affc4a73 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -20,7 +20,7 @@ import StatsAPI: params, coef, coefnames, dof, fit, nobs, coeftable using InteractiveUtils: subtypes -export StenoGraphs, @StenoGraph, meld, SimpleNode +export @StenoGraph, meld const SEM = StructuralEquationModels From 787638b01c46e08a3d7bcb7be2c34af570e6c76d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 3 Feb 2025 18:46:47 -0800 Subject: [PATCH 250/364] FIML: SemFIMLPattern * refactor FIML code to consolidate pattern methods under SemFIMLPattern * rename semfiml/lossfun to loss for consistency --- src/additional_functions/helper.jl | 6 - src/frontend/fit/fitmeasures/minus2ll.jl | 9 +- src/loss/ML/FIML.jl | 251 +++++++++++------------ src/observed/missing_pattern.jl | 8 +- 4 files changed, 129 insertions(+), 145 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index d6a1fc6c8..426277d07 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -14,12 +14,6 @@ function neumann_series(mat::SparseMatrixCSC; maxiter::Integer = size(mat, 1)) return inverse end -function batch_inv!(fun, model) - for i in 1:size(fun.inverses, 1) - fun.inverses[i] .= LinearAlgebra.inv!(fun.choleskys[i]) - end -end - # computes A*S*B -> C, where ind gives the entries of S that are 1 function sparse_outer_mul!(C, A, B, ind) fill!(C, 0.0) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 9b211fb44..64058c26c 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -47,15 +47,20 @@ function minus2ll(observed::SemObservedMissing) Σ = observed.em_model.Σ μ = observed.em_model.μ + # FIXME: this code is duplicate to objective(fiml, ...) F = sum(observed.patterns) do pat # implied covariance/mean Σᵢ = Σ[pat.measured_mask, pat.measured_mask] Σᵢ_chol = cholesky!(Σᵢ) ld = logdet(Σᵢ_chol) Σᵢ⁻¹ = LinearAlgebra.inv!(Σᵢ_chol) - meandiffᵢ = pat.measured_mean - μ[pat.measured_mask] + μ_diffᵢ = pat.measured_mean - μ[pat.measured_mask] - F_one_pattern(meandiffᵢ, Σᵢ⁻¹, pat.measured_cov, ld, nsamples(pat)) + F_pat = ld + dot(μ_diffᵢ, Σᵢ⁻¹, μ_diffᵢ) + if nsamples(pat) > 1 + F_pat += dot(pat.measured_cov, Σᵢ⁻¹) + end + F_pat * nsamples(pat) end F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), observed.patterns) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 6731b1a16..da5ccb7c6 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -1,16 +1,85 @@ ############################################################################################ ### Types ############################################################################################ + +# state of SemFIML for a specific missing pattern (`SemObservedMissingPattern` type) +struct SemFIMLPattern{T} + ∇ind::Vector{Int} # indices of co-observed variable pairs + Σ⁻¹::Matrix{T} # preallocated inverse of implied cov + logdet::Ref{T} # logdet of implied cov + μ_diff::Vector{T} # implied mean difference +end + +# allocate arrays for pattern FIML +function SemFIMLPattern(pat::SemObservedMissingPattern) + nobs_vars = nobserved_vars(pat) + nmeas_vars = nmeasured_vars(pat) + + # generate linear indicies of co-observed variable pairs for each pattern + Σ_linind = LinearIndices((nobs_vars, nobs_vars)) + pat_vars = findall(pat.measured_mask) + ∇ind = vec(Σ_linind[pat_vars, pat_vars]) + + return SemFIMLPattern(∇ind, zeros(nmeas_vars, nmeas_vars), Ref(NaN), zeros(nmeas_vars)) +end + +function prepare!( + patloss::SemFIMLPattern, + pat::SemObservedMissingPattern, + Σ::AbstractMatrix, + μ::AbstractVector, +) + @inbounds @. @views begin + patloss.Σ⁻¹ = Σ[pat.measured_mask, pat.measured_mask] + patloss.μ_diff = pat.measured_mean - μ[pat.measured_mask] + end + Σ_chol = cholesky!(Symmetric(patloss.Σ⁻¹)) + patloss.logdet[] = logdet(Σ_chol) + LinearAlgebra.inv!(Σ_chol) # updates loss.Σ⁻¹ + return patloss +end + +prepare!(patloss::SemFIMLPattern, pat::SemObservedMissingPattern, implied::SemImplied) = + prepare!(patloss, pat, implied.Σ, implied.μ) + +function objective(patloss::SemFIMLPattern{T}, pat::SemObservedMissingPattern) where {T} + F = patloss.logdet[] + dot(patloss.μ_diff, patloss.Σ⁻¹, patloss.μ_diff) + if nsamples(pat) > 1 + F += dot(pat.measured_cov, patloss.Σ⁻¹) + F *= nsamples(pat) + end + return F +end + +function gradient!(JΣ, Jμ, patloss::SemFIMLPattern, pat::SemObservedMissingPattern) + Σ⁻¹ = Symmetric(patloss.Σ⁻¹) + μ_diff⨉Σ⁻¹ = patloss.μ_diff' * Σ⁻¹ + if nsamples(pat) > 1 + JΣ_pat = Σ⁻¹ * (I - pat.measured_cov * Σ⁻¹ - patloss.μ_diff * μ_diff⨉Σ⁻¹) + JΣ_pat .*= nsamples(pat) + else + JΣ_pat = Σ⁻¹ * (I - patloss.μ_diff * μ_diff⨉Σ⁻¹) + end + @inbounds vec(JΣ)[patloss.∇ind] .+= vec(JΣ_pat) + + lmul!(2 * nsamples(pat), μ_diff⨉Σ⁻¹) + @inbounds Jμ[pat.measured_mask] .+= μ_diff⨉Σ⁻¹' + return nothing +end + """ -Full information maximum likelihood estimation. Can handle observed data with missings. + SemFIML{T, W} <: SemLossFunction + +Full information maximum likelihood (FIML) estimation. +Can handle observed data with missing values. # Constructor - SemFIML(;observed, specification, kwargs...) + SemFIML(; observed::SemObservedMissing, specification, kwargs...) # Arguments -- `observed::SemObservedMissing`: the observed part of the model -- `specification`: either a `RAMMatrices` or `ParameterTable` object +- `observed`: the observed data with missing values (see [`SemObservedMissing`](@ref)) +- `specification`: [`SemSpecification`](@ref) object # Examples ```julia @@ -20,19 +89,11 @@ my_fiml = SemFIML(observed = my_observed, specification = my_parameter_table) # Interfaces Analytic gradients are available. """ -mutable struct SemFIML{INV, C, L, O, M, IM, I, T, W} <: SemLossFunction +struct SemFIML{T, W} <: SemLossFunction hessianeval::ExactHessian - inverses::INV #preallocated inverses of imp_cov - choleskys::C #preallocated choleskys - logdets::L #logdets of implied covmats - - ∇ind::O - - imp_mean::IM - meandiff::M - imp_inv::I + patterns::Vector{SemFIMLPattern{T}} - mult::T + imp_inv::Matrix{T} # implied inverse commutator::CommutationMatrix @@ -44,44 +105,19 @@ end ############################################################################################ function SemFIML(; observed::SemObservedMissing, implied, specification, kwargs...) - - if implied.meanstruct isa NoMeanStruct - throw(ArgumentError( - "Full information maximum likelihood (FIML) can only be used with a meanstructure. - Did you forget to set `Sem(..., meanstructure = true)`?")) - end - - inverses = - [zeros(nmeasured_vars(pat), nmeasured_vars(pat)) for pat in observed.patterns] - choleskys = Array{Cholesky{Float64, Array{Float64, 2}}, 1}(undef, length(inverses)) - - n_patterns = length(observed.patterns) - logdets = zeros(n_patterns) - - imp_mean = [zeros(nmeasured_vars(pat)) for pat in observed.patterns] - meandiff = [zeros(nmeasured_vars(pat)) for pat in observed.patterns] - - nobs_vars = nobserved_vars(observed) - imp_inv = zeros(nobs_vars, nobs_vars) - mult = similar.(inverses) - - # generate linear indicies of co-observed variable pairs for each pattern - Σ_linind = LinearIndices((nobs_vars, nobs_vars)) - ∇ind = map(observed.patterns) do pat - pat_vars = findall(pat.measured_mask) - vec(Σ_linind[pat_vars, pat_vars]) + if MeanStruct(implied) === NoMeanStruct + """ + Full information maximum likelihood (FIML) can only be used with a meanstructure. + Did you forget to set `Sem(..., meanstructure = true)`? + """ |> + ArgumentError |> + throw end return SemFIML( ExactHessian(), - inverses, - choleskys, - logdets, - ∇ind, - imp_mean, - meandiff, - imp_inv, - mult, + [SemFIMLPattern(pat) for pat in observed.patterns], + zeros(nobserved_vars(observed), nobserved_vars(observed)), CommutationMatrix(nvars(specification)), nothing, ) @@ -95,26 +131,26 @@ function evaluate!( objective, gradient, hessian, - semfiml::SemFIML, + loss::SemFIML, implied::SemImplied, model::AbstractSemSingle, - param_labels, + params, ) isnothing(hessian) || error("Hessian not implemented for FIML") - if !check_fiml(semfiml, model) - isnothing(objective) || (objective = non_posdef_return(param_labels)) + if !check(loss, model) + isnothing(objective) || (objective = non_posdef_return(params)) isnothing(gradient) || fill!(gradient, 1) return objective end - prepare_SemFIML!(semfiml, model) + prepare!(loss, model) scale = inv(nsamples(observed(model))) isnothing(objective) || - (objective = scale * F_FIML(observed(model), semfiml, model, param_labels)) + (objective = scale * F_FIML(eltype(params), loss, observed(model), model)) isnothing(gradient) || - (∇F_FIML!(gradient, observed(model), semfiml, model); gradient .*= scale) + (∇F_FIML!(gradient, loss, observed(model), model); gradient .*= scale) return objective end @@ -123,44 +159,38 @@ end ### Recommended methods ############################################################################################ -update_observed(lossfun::SemFIML, observed::SemObserved; kwargs...) = +update_observed(loss::SemFIML, observed::SemObserved; kwargs...) = SemFIML(; observed = observed, kwargs...) ############################################################################################ ### additional functions ############################################################################################ -function F_one_pattern(meandiff, inverse, obs_cov, logdet, N) - F = logdet + dot(meandiff, inverse, meandiff) - if N > one(N) - F += dot(obs_cov, inverse) +function prepare!(loss::SemFIML, observed::SemObservedMissing, implied::SemImplied) + @inbounds for (patloss, pat) in zip(loss.patterns, observed.patterns) + prepare!(patloss, pat, implied.Σ, implied.μ) end - return F * N end -function ∇F_one_pattern(μ_diff, Σ⁻¹, S, obs_mask, ∇ind, N, Jμ, JΣ, model) - diff⨉inv = μ_diff' * Σ⁻¹ - - if N > one(N) - JΣ[∇ind] .+= N * vec(Σ⁻¹ * (I - S * Σ⁻¹ - μ_diff * diff⨉inv)) - @. Jμ[obs_mask] += (N * 2 * diff⨉inv)' +prepare!(loss::SemFIML, model::AbstractSemSingle) = + prepare!(loss, observed(model), implied(model)) - else - JΣ[∇ind] .+= vec(Σ⁻¹ * (I - μ_diff * diff⨉inv)) - @. Jμ[obs_mask] += (2 * diff⨉inv)' - end +function check(loss::SemFIML, model::AbstractSemSingle) + copyto!(loss.imp_inv, implied(model).Σ) + a = cholesky!(Symmetric(loss.imp_inv); check = false) + return isposdef(a) end -function ∇F_fiml_outer!(G, JΣ, Jμ, implied::SemImpliedSymbolic, model, semfiml) +function ∇F_fiml_outer!(G, JΣ, Jμ, loss::SemFIML, implied::SemImpliedSymbolic, model) mul!(G, implied.∇Σ', JΣ) # should be transposed mul!(G, implied.∇μ', Jμ, -1, 1) end -function ∇F_fiml_outer!(G, JΣ, Jμ, implied, model, semfiml) +function ∇F_fiml_outer!(G, JΣ, Jμ, loss::SemFIML, implied, model) Iₙ = sparse(1.0I, size(implied.A)...) P = kron(implied.F⨉I_A⁻¹, implied.F⨉I_A⁻¹) Q = kron(implied.S * implied.I_A⁻¹', Iₙ) - Q .+= semfiml.commutator * Q + Q .+= loss.commutator * Q ∇Σ = P * (implied.∇S + Q * implied.∇A) @@ -172,70 +202,25 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, implied, model, semfiml) mul!(G, ∇μ', Jμ, -1, 1) end -function F_FIML(observed::SemObservedMissing, semfiml, model, param_labels) - F = zero(eltype(param_labels)) - for (i, pat) in enumerate(observed.patterns) - F += F_one_pattern( - semfiml.meandiff[i], - semfiml.inverses[i], - pat.measured_cov, - semfiml.logdets[i], - nsamples(pat), - ) +function F_FIML( + ::Type{T}, + loss::SemFIML, + observed::SemObservedMissing, + model::AbstractSemSingle, +) where {T} + F = zero(T) + for (patloss, pat) in zip(loss.patterns, observed.patterns) + F += objective(patloss, pat) end return F end -function ∇F_FIML!(G, observed::SemObservedMissing, semfiml, model) +function ∇F_FIML!(G, loss::SemFIML, observed::SemObservedMissing, model::AbstractSemSingle) Jμ = zeros(nobserved_vars(model)) JΣ = zeros(nobserved_vars(model)^2) - for (i, pat) in enumerate(observed.patterns) - ∇F_one_pattern( - semfiml.meandiff[i], - semfiml.inverses[i], - pat.measured_cov, - pat.measured_mask, - semfiml.∇ind[i], - nsamples(pat), - Jμ, - JΣ, - model, - ) - end - return ∇F_fiml_outer!(G, JΣ, Jμ, implied(model), model, semfiml) -end - -function prepare_SemFIML!(semfiml, model) - copy_per_pattern!(semfiml, model) - batch_cholesky!(semfiml, model) - #batch_sym_inv_update!(semfiml, model) - batch_inv!(semfiml, model) - for (i, pat) in enumerate(observed(model).patterns) - semfiml.meandiff[i] .= pat.measured_mean .- semfiml.imp_mean[i] - end -end - -function copy_per_pattern!(fiml::SemFIML, model::AbstractSem) - Σ = implied(model).Σ - μ = implied(model).μ - data = observed(model) - @inbounds @views for (i, pat) in enumerate(data.patterns) - fiml.inverses[i] .= Σ[pat.measured_mask, pat.measured_mask] - fiml.imp_mean[i] .= μ[pat.measured_mask] + for (patloss, pat) in zip(loss.patterns, observed.patterns) + gradient!(JΣ, Jμ, patloss, pat) end -end - -function batch_cholesky!(semfiml, model) - for i in 1:size(semfiml.inverses, 1) - semfiml.choleskys[i] = cholesky!(Symmetric(semfiml.inverses[i])) - semfiml.logdets[i] = logdet(semfiml.choleskys[i]) - end - return true -end - -function check_fiml(semfiml, model) - copyto!(semfiml.imp_inv, implied(model).Σ) - a = cholesky!(Symmetric(semfiml.imp_inv); check = false) - return isposdef(a) + ∇F_fiml_outer!(G, JΣ, Jμ, loss, implied(model), model) end diff --git a/src/observed/missing_pattern.jl b/src/observed/missing_pattern.jl index 6ac6a360b..8d9a1eb93 100644 --- a/src/observed/missing_pattern.jl +++ b/src/observed/missing_pattern.jl @@ -5,10 +5,10 @@ struct SemObservedMissingPattern{T, S} measured_mask::BitVector # measured vars mask miss_mask::BitVector # missing vars mask rows::Vector{Int} # rows in original data - data::Matrix{T} # non-missing submatrix of data + data::Matrix{T} # non-missing submatrix of data (vars × observations) measured_mean::Vector{S} # means of measured vars - measured_cov::Matrix{S} # covariance of measured vars + measured_cov::Symmetric{S, Matrix{S}} # covariance of measured vars end function SemObservedMissingPattern( @@ -32,9 +32,9 @@ function SemObservedMissingPattern( measured_mask, .!measured_mask, rows, - pat_data, + permutedims(pat_data), dropdims(pat_mean, dims = 1), - pat_cov, + Symmetric(pat_cov), ) end From f599442f6d14a03fbbb115d5bc966ffb163a1ae1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 12:24:05 -0700 Subject: [PATCH 251/364] EM: optimizations --- src/observed/EM.jl | 142 ++++++++++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 59 deletions(-) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 288082ccc..ff0b7cb9a 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -50,33 +50,33 @@ function em_mvn!( rtol_em = 1e-4, kwargs..., ) - nvars = nobserved_vars(observed) - nsamps = nsamples(observed) - - # preallocate stuff - 𝔼x_pre = zeros(nvars) - 𝔼xxᵀ_pre = zeros(nvars, nvars) - - ### precompute for full cases - fullpat = observed.patterns[1] - if nmissed_vars(fullpat) == 0 - for row in eachrow(fullpat.data) - 𝔼x_pre += row - 𝔼xxᵀ_pre += row * row' + nobs_vars = nobserved_vars(observed) + + # precompute for full cases + 𝔼x_full = zeros(nobs_vars) + 𝔼xxᵀ_full = zeros(nobs_vars, nobs_vars) + nsamples_full = 0 + for pat in patterns + if nmissed_vars(pat) == 0 + 𝔼x_full .+= sum(pat.data, dims = 2) + mul!(𝔼xxᵀ_full, pat.data, pat.data', 1, 1) + nsamples_full += nsamples(pat) end end + if nsamples_full == 0 + @warn "No full cases in data" + end # initialize em_model = start_em(observed; kwargs...) - em_model_prev = EmMVNModel(zeros(nvars, nvars), zeros(nvars), false) + em_model_prev = EmMVNModel(zeros(nobs_vars, nobs_vars), zeros(nobs_vars), false) iter = 1 done = false - 𝔼x = zeros(nvars) - 𝔼xxᵀ = zeros(nvars, nvars) + 𝔼x = zeros(nobs_vars) + 𝔼xxᵀ = zeros(nobs_vars, nobs_vars) while !done - em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xxᵀ_pre) - em_mvn_Mstep!(em_model, nsamps, 𝔼x, 𝔼xxᵀ) + step!(em_model, observed, 𝔼x, 𝔼xxᵀ, 𝔼x_pre, 𝔼xxᵀ_pre) if iter > max_iter_em done = true @@ -84,12 +84,14 @@ function em_mvn!( Maybe try passing different starting values via 'start_em = ...' " elseif iter > 1 done = - isapprox(em_model_prev.μ, em_model.μ; rtol = rtol_em) & + isapprox(em_model_prev.μ, em_model.μ; rtol = rtol_em) && isapprox(em_model_prev.Σ, em_model.Σ; rtol = rtol_em) end - iter = iter + 1 - em_model_prev.μ, em_model_prev.Σ = em_model.μ, em_model.Σ + # print("$iter \n") + iter += 1 + copyto!(em_model_prev.μ, em_model.μ) + copyto!(em_model_prev.Σ, em_model.Σ) end # update EM Mode in observed @@ -100,17 +102,25 @@ function em_mvn!( return nothing end -# E and M step ----------------------------------------------------------------------------- - -function em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xxᵀ_pre) - 𝔼x .= 0.0 - 𝔼xxᵀ .= 0.0 - - 𝔼xᵢ = copy(𝔼x) - 𝔼xxᵀᵢ = copy(𝔼xxᵀ) - - μ = em_model.μ - Σ = em_model.Σ +# E and M steps combined +function em_step!( + Σ::AbstractMatrix, + μ::AbstractVector, + Σ₀::AbstractMatrix, + μ₀::AbstractVector, + patterns::AbstractVector{<:SemObservedMissingPattern}, + 𝔼xxᵀ_full::AbstractMatrix, + 𝔼x_full::AbstractVector, + nsamples_full::Integer; + max_nsamples_em::Union{Integer, Nothing} = nothing, + min_eigval::Union{Number, Nothing} = nothing, +) + # E step: update 𝔼x and 𝔼xxᵀ + copy!(μ, 𝔼x_full) + copy!(Σ, 𝔼xxᵀ_full) + nsamples_used = nsamples_full + mul!(Σ, μ₀, μ₀', -nsamples_used, 1) + axpy!(-nsamples_used, μ₀, μ) # Compute the expected sufficient statistics for pat in observed.patterns @@ -121,38 +131,52 @@ function em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xx o = pat.measured_mask # precompute for pattern - Σoo = Σ[o, o] + Σoo_chol = cholesky(Symmetric(Σ[o, o])) Σuo = Σ[u, o] μu = μ[u] μo = μ[o] - V = Σ[u, u] - Σuo * (Σoo \ Σ[o, u]) - - # loop trough data - for rowdata in eachrow(pat.data) - m = μu + Σuo * (Σoo \ (rowdata - μo)) + 𝔼xu = fill!(similar(μu), 0) + 𝔼xo = fill!(similar(μo), 0) + 𝔼xᵢu = similar(μu) + + 𝔼xxᵀuo = fill!(similar(Σuo), 0) + 𝔼xxᵀuu = n_obs(pat) * (Σ[u, u] - Σuo * (Σoo_chol \ Σuo')) + + # loop through observations + @inbounds for rowdata in eachcol(pat.data) + mul!(𝔼xᵢu, Σuo, Σoo_chol \ (rowdata - μo)) + 𝔼xᵢu .+= μu + mul!(𝔼xxᵀuu, 𝔼xᵢu, 𝔼xᵢu', 1, 1) + mul!(𝔼xxᵀuo, 𝔼xᵢu, rowdata', 1, 1) + 𝔼xu .+= 𝔼xᵢu + 𝔼xo .+= rowdata + end - 𝔼xᵢ[u] = m - 𝔼xᵢ[o] = rowdata - 𝔼xxᵀᵢ[u, u] = 𝔼xᵢ[u] * 𝔼xᵢ[u]' + V - 𝔼xxᵀᵢ[o, o] = 𝔼xᵢ[o] * 𝔼xᵢ[o]' - 𝔼xxᵀᵢ[o, u] = 𝔼xᵢ[o] * 𝔼xᵢ[u]' - 𝔼xxᵀᵢ[u, o] = 𝔼xᵢ[u] * 𝔼xᵢ[o]' + 𝔼xxᵀ[o, o] .+= pat.data' * pat.data + 𝔼xxᵀ[u, o] .+= 𝔼xxᵀuo + 𝔼xxᵀ[o, u] .+= 𝔼xxᵀuo' + 𝔼xxᵀ[u, u] .+= 𝔼xxᵀuu - 𝔼x .+= 𝔼xᵢ - 𝔼xxᵀ .+= 𝔼xxᵀᵢ - end + 𝔼x[o] .+= 𝔼xo + 𝔼x[u] .+= 𝔼xu end - 𝔼x .+= 𝔼x_pre - 𝔼xxᵀ .+= 𝔼xxᵀ_pre -end + # M step: update Σ and μ + lmul!(1 / nsamples_used, Σ) + lmul!(1 / nsamples_used, μ) + # at this point μ = μ - μ₀ + # and Σ = Σ + (μ - μ₀)×(μ - μ₀)' - μ₀×μ₀' + mul!(Σ, μ, μ₀', -1, 1) + mul!(Σ, μ₀, μ', -1, 1) + mul!(Σ, μ, μ', -1, 1) + μ .+= μ₀ -function em_mvn_Mstep!(em_model, nsamples, 𝔼x, 𝔼xxᵀ) - em_model.μ = 𝔼x / nsamples - Σ = Symmetric(𝔼xxᵀ / nsamples - em_model.μ * em_model.μ') - em_model.Σ = Σ - return nothing + em_model.μ .= 𝔼x ./ nsamples(observed) + em_model.Σ .= 𝔼xxᵀ ./ nsamples(observed) + mul!(em_model.Σ, em_model.μ, em_model.μ', -1, 1) + + return em_model end # generate starting values ----------------------------------------------------------------- @@ -160,13 +184,13 @@ end # use μ and Σ of full cases function start_em_observed(observed::SemObservedMissing; kwargs...) fullpat = observed.patterns[1] - if (nmissed_vars(fullpat) == 0) && (nobserved_vars(fullpat) > 1) + if (nmissed_vars(fullpat) == 0) && (nsamples(fullpat) > 1) μ = copy(fullpat.measured_mean) - Σ = copy(Symmetric(fullpat.measured_cov)) + Σ = copy(fullpat.measured_cov) if !isposdef(Σ) - Σ = Matrix(Diagonal(Σ)) + Σ = Diagonal(Σ) end - return EmMVNModel(Σ, μ, false) + return EmMVNModel(convert(Matrix, Σ), μ, false) else return start_em_simple(observed, kwargs...) end From 56834b5f0759b926cbee04672f880f3db8819b1b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 22 Dec 2024 12:22:24 -0800 Subject: [PATCH 252/364] EM MVN: decouple from SemObsMissing so EM MVN could be done when SemObsMissing is constructed --- .../start_val/start_fabin3.jl | 11 +- src/frontend/fit/fitmeasures/minus2ll.jl | 7 +- src/observed/EM.jl | 184 ++++++++++-------- src/observed/missing.jl | 68 ++++--- 4 files changed, 146 insertions(+), 124 deletions(-) diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index ab79d9ada..53d3442a7 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -11,19 +11,10 @@ function start_fabin3(model::AbstractSemSingle; kwargs...) return start_fabin3(model.observed, model.implied, model.loss.functions..., kwargs...) end -function start_fabin3(observed, implied, args...; kwargs...) +function start_fabin3(observed::SemObserved, implied::SemImplied, args...; kwargs...) return start_fabin3(implied.ram_matrices, obs_cov(observed), obs_mean(observed)) end -# SemObservedMissing -function start_fabin3(observed::SemObservedMissing, implied, args...; kwargs...) - if !observed.em_model.fitted - em_mvn!(observed; kwargs...) - end - - return start_fabin3(implied.ram_matrices, observed.em_model.Σ, observed.em_model.μ) -end - function start_fabin3( ram_matrices::RAMMatrices, Σ::AbstractMatrix, diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 64058c26c..2b5a39491 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -40,12 +40,7 @@ end # compute likelihood for missing data - H1 ------------------------------------------------- # -2ll = ∑ log(2π)*(nᵢ + mᵢ) + ln(Σᵢ) + (mᵢ - μᵢ)ᵀ Σᵢ⁻¹ (mᵢ - μᵢ)) + tr(SᵢΣᵢ) function minus2ll(observed::SemObservedMissing) - # fit EM-based mean and cov if not yet fitted - # FIXME EM could be very computationally expensive - observed.em_model.fitted || em_mvn!(observed) - - Σ = observed.em_model.Σ - μ = observed.em_model.μ + Σ, μ = obs_cov(observed), obs_mean(observed) # FIXME: this code is duplicate to objective(fiml, ...) F = sum(observed.patterns) do pat diff --git a/src/observed/EM.jl b/src/observed/EM.jl index ff0b7cb9a..d65e5d6ef 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -27,15 +27,34 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. =# """ - em_mvn!(; - observed::SemObservedMissing, - start_em = start_em_observed, - max_iter_em = 100, - rtol_em = 1e-4, - kwargs...) - -Estimates the covariance matrix and mean vector of the normal distribution via expectation maximization for `observed`. -Overwrites the statistics stored in `observed`. + em_mvn(patterns::AbstractVector{SemObservedMissingPattern}; + start_em = start_em_observed, + max_iter_em = 100, + rtol_em = 1e-4, + kwargs...) + +Estimate the covariance and the mean for data with missing values using +the expectation maximization (EM) algorithm. + +# Arguments +- `patterns`: the observed data with missing values, grouped by missingness pattern (see [` + SemObservedMissingPattern`](@ref)) +- `max_iter_em`: the maximum number of EM iterations +- `rtol_em`: the relative tolerance for convergence of the EM algorithm +- `max_nsamples_em`: the maximum number of samples to use for each pattern in each EM iteration, + by default all samples are used, but for large datasets it may be desirable to use a random + subset of the data for each pattern in each EM iteration to speed up the algorithm +- `min_eigval`: the minimum eigenvalue for the covariance matrix; + if not `nothing`, the covariance matrix is regularized in each EM iteration to ensure that + all eigenvalues are not smaller than `min_eigval`, which can help with convergence; +- `start_em`: the function to generate starting values for the EM algorithm, by default + `start_em_observed` which uses the mean and covariance of the full cases if available +- `start_kwargs...`: keyword arguments to pass to the `start_em` function + +Returns the tuple of the covariance matrix and the mean vector for the estimated +multivariate normal (MVN) distribution. + +# References Uses the EM algorithm for MVN-distributed data with missing values adapted from the supplementary material to the book *Machine Learning: A Probabilistic Perspective*, @@ -43,14 +62,14 @@ copyright (2010) Kevin Murphy and Matt Dunham: see [*gaussMissingFitEm.m*](https://github.com/probml/pmtk3/blob/master/toolbox/BasicModels/gauss/sub/gaussMissingFitEm.m) and [*emAlgo.m*](https://github.com/probml/pmtk3/blob/master/toolbox/Algorithms/optimization/emAlgo.m) scripts. """ -function em_mvn!( - observed::SemObservedMissing; +function em_mvn( + patterns::AbstractVector{<:SemObservedMissingPattern}; start_em = start_em_observed, - max_iter_em = 100, - rtol_em = 1e-4, + max_iter_em::Integer = 100, + rtol_em::Number = 1e-4, kwargs..., ) - nobs_vars = nobserved_vars(observed) + nobs_vars = nobserved_vars(patterns[1]) # precompute for full cases 𝔼x_full = zeros(nobs_vars) @@ -68,38 +87,42 @@ function em_mvn!( end # initialize - em_model = start_em(observed; kwargs...) - em_model_prev = EmMVNModel(zeros(nobs_vars, nobs_vars), zeros(nobs_vars), false) - iter = 1 - done = false - 𝔼x = zeros(nobs_vars) - 𝔼xxᵀ = zeros(nobs_vars, nobs_vars) - - while !done - step!(em_model, observed, 𝔼x, 𝔼xxᵀ, 𝔼x_pre, 𝔼xxᵀ_pre) - - if iter > max_iter_em - done = true - @warn "EM Algorithm for MVN missing data did not converge. Likelihood for FIML is not interpretable. - Maybe try passing different starting values via 'start_em = ...' " - elseif iter > 1 - done = - isapprox(em_model_prev.μ, em_model.μ; rtol = rtol_em) && - isapprox(em_model_prev.Σ, em_model.Σ; rtol = rtol_em) + Σ₀, μ = start_em(patterns; kwargs...) + Σ = convert(Matrix, Σ₀) + @assert all(isfinite, Σ) all(isfinite, μ) + Σ_prev, μ_prev = copy(Σ), copy(μ) + + iter = 0 + converged = false + while !converged && (iter < max_iter_em) + em_step!(Σ, μ, Σ_prev, μ_prev, patterns, 𝔼x_full, 𝔼xxᵀ_full) + + if iter > 0 + Δμ = norm(μ - μ_prev) + ΔΣ = norm(Σ - Σ_prev) + Δμ_rel = Δμ / max(norm(μ_prev), norm(μ)) + ΔΣ_rel = ΔΣ / max(norm(Σ_prev), norm(Σ)) + #@info "Iteration #$iter: ΔΣ=$(ΔΣ) ΔΣ/Σ=$(ΔΣ_rel) Δμ=$(Δμ) Δμ/μ=$(Δμ_rel)" + # converged = isapprox(ll, ll_prev; rtol = rtol) + converged = ΔΣ_rel <= rtol_em && Δμ_rel <= rtol_em + end + if !converged + Σ, Σ_prev = Σ_prev, Σ + μ, μ_prev = μ_prev, μ end - - # print("$iter \n") iter += 1 - copyto!(em_model_prev.μ, em_model.μ) - copyto!(em_model_prev.Σ, em_model.Σ) + #@info "$iter\n" end - # update EM Mode in observed - observed.em_model.Σ .= em_model.Σ - observed.em_model.μ .= em_model.μ - observed.em_model.fitted = true + if !converged + @warn "EM Algorithm for MVN missing data did not converge in $iter iterations.\n" * + "Likelihood for FIML is not interpretable.\n" * + "Maybe try passing different starting values via 'start_em = ...' " + else + @info "EM for MVN missing data converged in $iter iterations" + end - return nothing + return Σ, μ end # E and M steps combined @@ -109,21 +132,15 @@ function em_step!( Σ₀::AbstractMatrix, μ₀::AbstractVector, patterns::AbstractVector{<:SemObservedMissingPattern}, - 𝔼xxᵀ_full::AbstractMatrix, - 𝔼x_full::AbstractVector, - nsamples_full::Integer; - max_nsamples_em::Union{Integer, Nothing} = nothing, - min_eigval::Union{Number, Nothing} = nothing, + 𝔼x_full, + 𝔼xxᵀ_full, ) # E step: update 𝔼x and 𝔼xxᵀ copy!(μ, 𝔼x_full) copy!(Σ, 𝔼xxᵀ_full) - nsamples_used = nsamples_full - mul!(Σ, μ₀, μ₀', -nsamples_used, 1) - axpy!(-nsamples_used, μ₀, μ) # Compute the expected sufficient statistics - for pat in observed.patterns + for pat in patterns (nmissed_vars(pat) == 0) && continue # skip full cases # observed and unobserved vars @@ -131,17 +148,17 @@ function em_step!( o = pat.measured_mask # precompute for pattern - Σoo_chol = cholesky(Symmetric(Σ[o, o])) - Σuo = Σ[u, o] - μu = μ[u] - μo = μ[o] + Σoo_chol = cholesky(Symmetric(Σ₀[o, o])) + Σuo = Σ₀[u, o] + μu = μ₀[u] + μo = μ₀[o] 𝔼xu = fill!(similar(μu), 0) 𝔼xo = fill!(similar(μo), 0) 𝔼xᵢu = similar(μu) 𝔼xxᵀuo = fill!(similar(Σuo), 0) - 𝔼xxᵀuu = n_obs(pat) * (Σ[u, u] - Σuo * (Σoo_chol \ Σuo')) + 𝔼xxᵀuu = n_obs(pat) * (Σ₀[u, u] - Σuo * (Σoo_chol \ Σuo')) # loop through observations @inbounds for rowdata in eachcol(pat.data) @@ -153,59 +170,56 @@ function em_step!( 𝔼xo .+= rowdata end - 𝔼xxᵀ[o, o] .+= pat.data' * pat.data - 𝔼xxᵀ[u, o] .+= 𝔼xxᵀuo - 𝔼xxᵀ[o, u] .+= 𝔼xxᵀuo' - 𝔼xxᵀ[u, u] .+= 𝔼xxᵀuu + Σ[o, o] .+= pat.data' * pat.data + Σ[u, o] .+= 𝔼xxᵀuo + Σ[o, u] .+= 𝔼xxᵀuo' + Σ[u, u] .+= 𝔼xxᵀuu - 𝔼x[o] .+= 𝔼xo - 𝔼x[u] .+= 𝔼xu + μ[o] .+= 𝔼xo + μ[u] .+= 𝔼xu end # M step: update Σ and μ - lmul!(1 / nsamples_used, Σ) - lmul!(1 / nsamples_used, μ) - # at this point μ = μ - μ₀ - # and Σ = Σ + (μ - μ₀)×(μ - μ₀)' - μ₀×μ₀' - mul!(Σ, μ, μ₀', -1, 1) - mul!(Σ, μ₀, μ', -1, 1) + k = inv(sum(nsamples, patterns)) + lmul!(k, Σ) + lmul!(k, μ) mul!(Σ, μ, μ', -1, 1) - μ .+= μ₀ - - em_model.μ .= 𝔼x ./ nsamples(observed) - em_model.Σ .= 𝔼xxᵀ ./ nsamples(observed) - mul!(em_model.Σ, em_model.μ, em_model.μ', -1, 1) - return em_model + return Σ, μ end # generate starting values ----------------------------------------------------------------- # use μ and Σ of full cases -function start_em_observed(observed::SemObservedMissing; kwargs...) - fullpat = observed.patterns[1] +function start_em_observed(patterns::AbstractVector{<:SemObservedMissingPattern}; kwargs...) + fullpat = patterns[1] if (nmissed_vars(fullpat) == 0) && (nsamples(fullpat) > 1) μ = copy(fullpat.measured_mean) - Σ = copy(fullpat.measured_cov) + Σ = copy(parent(fullpat.measured_cov)) if !isposdef(Σ) Σ = Diagonal(Σ) end - return EmMVNModel(convert(Matrix, Σ), μ, false) + return Σ, μ else - return start_em_simple(observed, kwargs...) + return start_em_simple(patterns, kwargs...) end end # use μ = O and Σ = I -function start_em_simple(observed::SemObservedMissing; kwargs...) - nvars = nobserved_vars(observed) - μ = zeros(nvars) - Σ = rand(nvars, nvars) +function start_em_simple(patterns::AbstractVector{<:SemObservedMissingPattern}; kwargs...) + nobs_vars = nobserved_vars(first(patterns)) + μ = zeros(nobs_vars) + Σ = rand(nobs_vars, nobs_vars) Σ = Σ * Σ' - return EmMVNModel(Σ, μ, false) + return Σ, μ end # set to passed values -function start_em_set(observed::SemObservedMissing; model_em, kwargs...) - return em_model +function start_em_set( + patterns::AbstractVector{<:SemObservedMissingPattern}; + obs_cov::AbstractMatrix, + obs_mean::AbstractVector, + kwargs..., +) + return copy(obs_cov), copy(obs_mean) end diff --git a/src/observed/missing.jl b/src/observed/missing.jl index ac8b7ea5f..f089203ed 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -2,17 +2,6 @@ ### Types ############################################################################################ -# Type to store Expectation Maximization result -------------------------------------------- -mutable struct EmMVNModel{A, b, B} - Σ::A - μ::b - fitted::B -end - -# FIXME type unstable -obs_mean(em::EmMVNModel) = ifelse(em.fitted, em.μ, nothing) -obs_cov(em::EmMVNModel) = ifelse(em.fitted, em.Σ, nothing) - """ For observed data with missing values. @@ -44,13 +33,15 @@ After, the following methods are available: - `obs_cov(::SemObservedData)` -> EM covariance matrix - `obs_mean(::SemObservedData)` -> EM mean vector """ -struct SemObservedMissing{T <: Real, S <: Real, E <: EmMVNModel} <: SemObserved +struct SemObservedMissing{T <: Real, S <: Real} <: SemObserved data::Matrix{Union{T, Missing}} observed_vars::Vector{Symbol} nsamples::Int patterns::Vector{SemObservedMissingPattern{T, S}} - em_model::E + em_kwargs::AbstractDict + obs_cov::Ref{Matrix{S}} + obs_mean::Ref{Vector{S}} end ############################################################################################ @@ -62,7 +53,8 @@ function SemObservedMissing(; observed_vars::Union{AbstractVector, Nothing} = nothing, specification::Union{SemSpecification, Nothing} = nothing, observed_var_prefix::Union{Symbol, AbstractString} = :obs, - kwargs..., + lazy_cov::Bool = true, + em_kwargs..., ) data, obs_vars, _ = prepare_data(data, observed_vars, specification; observed_var_prefix) @@ -83,22 +75,52 @@ function SemObservedMissing(; ] sort!(patterns, by = nmissed_vars) - # allocate EM model (but don't fit) - em_model = EmMVNModel(zeros(nobs_vars, nobs_vars), zeros(nobs_vars), false) + S = isempty(patterns) ? Float64 : eltype(patterns[1].measured_mean) + if lazy_cov + # defer EM covariance and mean calculation until requested + em_cov_ref = Ref{Matrix{S}}() + em_mean_ref = Ref{Vector{S}}() + else + em_cov, em_mean = em_mvn(patterns; em_kwargs...) + em_cov_ref, em_mean_ref = Ref(em_cov), Ref(em_mean) + end return SemObservedMissing( convert(Matrix{Union{nonmissingtype(eltype(data)), Missing}}, data), obs_vars, nsamples, patterns, - em_model, + em_kwargs, # remember EM kwargs for calculate_cov! + em_cov_ref, + em_mean_ref, ) end -############################################################################################ -### Additional methods -############################################################################################ +""" + calculate_cov!(observed::SemObservedMissing; em_kwargs...) + +Force calculation of the observed mean and covariance using the EM algorithm. -em_model(observed::SemObservedMissing) = observed.em_model -obs_mean(observed::SemObservedMissing) = obs_mean(em_model(observed)) -obs_cov(observed::SemObservedMissing) = obs_cov(em_model(observed)) +# Arguments +- `observed`: the observed data with missing values (see [`SemObservedMissing`](@ref)) +- `em_kwargs...`: keyword arguments for the EM algorithm (see [`em_mvn`](@ref)), + the values provided here override the EM arguments passed to the + [`SemObservedMissing`](@ref) constructor +""" +function calculate_cov!(observed::SemObservedMissing; em_kwargs...) + em_kwargs = merge(observed.em_kwargs, em_kwargs) + em_cov, em_mean = em_mvn(observed.patterns; em_kwargs...) + observed.obs_cov[] = em_cov + observed.obs_mean[] = em_mean + return observed +end + +function obs_cov(observed::SemObservedMissing{<:Any, S}) where {S} + isassigned(observed.obs_cov) || calculate_cov!(observed) + return observed.obs_cov[]::Matrix{S} +end + +function obs_mean(observed::SemObservedMissing{<:Any, S}) where {S} + isassigned(observed.obs_mean) || calculate_cov!(observed) + return observed.obs_mean[]::Vector{S} +end From 79c8041cb6f47ad1da6d856e2bcd7e3de2a2c82c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 9 Feb 2026 13:37:33 -0800 Subject: [PATCH 253/364] ObsMissing: docstring update --- src/observed/missing.jl | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/observed/missing.jl b/src/observed/missing.jl index f089203ed..52115cf33 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -3,7 +3,9 @@ ############################################################################################ """ -For observed data with missing values. + SemObservedMissing{T <: Real, S <: Real} <: SemObserved + +[`SemObserved`](@ref) implementation for data with missing values. # Constructor @@ -11,27 +13,21 @@ For observed data with missing values. data, observed_vars = nothing, specification = nothing, - kwargs...) + lazy_cov = true, + em_kwargs...) # Arguments - `data`: observed data - `observed_vars::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) - `specification`: optional SEM model specification ([`SemSpecification`](@ref)) - -# Extended help -## Interfaces -- `nsamples(::SemObservedMissing)` -> number of samples (data points) -- `nobserved_vars(::SemObservedMissing)` -> number of observed variables - -- `samples(::SemObservedMissing)` -> data matrix (contains both measured and missing values) - -## Expectation maximization -`em_mvn!(::SemObservedMissing)` can be called to fit a covariance matrix and mean vector to the data -using an expectation maximization (EM) algorithm under the assumption of multivariate normality. -After, the following methods are available: -- `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via EM -- `obs_cov(::SemObservedData)` -> EM covariance matrix -- `obs_mean(::SemObservedData)` -> EM mean vector +- `lazy_cov::Bool`: whether to defer covariance and mean calculation until requested (default: `true`) +- `em_kwargs...`: keyword arguments to pass to the EM algorithm (see [`em_mvn`](@ref)) + +`SemObservedMissing` could be used in combination with [`SemFIML`](@ref) loss for the +*full information maximum likelihood* (FIML) to fit SEM with missing data. +It could also be used with other loss functions, e.g. [`SemML`](@ref); +in that case the approximated observed covariance and mean would be calculated using +the *EM* algorithm (see [`em_mvn`](@ref)). """ struct SemObservedMissing{T <: Real, S <: Real} <: SemObserved data::Matrix{Union{T, Missing}} From 249bcce8a22643a851588cc9452e7e439e6a36bb Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 17 Apr 2024 01:06:55 -0700 Subject: [PATCH 254/364] EM MVN: further optimizations - report rel_error if not converged - add max_nsamples_em opttion to limit number of samples used --- src/observed/EM.jl | 130 +++++++++++++++++++++++++++++++-------------- 1 file changed, 90 insertions(+), 40 deletions(-) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index d65e5d6ef..1b84ab575 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -28,10 +28,11 @@ THE SOFTWARE. =# """ em_mvn(patterns::AbstractVector{SemObservedMissingPattern}; - start_em = start_em_observed, max_iter_em = 100, rtol_em = 1e-4, - kwargs...) + max_nsamples_em = nothing, + start_em = start_em_observed, + start_kwargs...) Estimate the covariance and the mean for data with missing values using the expectation maximization (EM) algorithm. @@ -56,7 +57,7 @@ multivariate normal (MVN) distribution. # References -Uses the EM algorithm for MVN-distributed data with missing values +Based on the EM algorithm for MVN-distributed data with missing values adapted from the supplementary material to the book *Machine Learning: A Probabilistic Perspective*, copyright (2010) Kevin Murphy and Matt Dunham: see [*gaussMissingFitEm.m*](https://github.com/probml/pmtk3/blob/master/toolbox/BasicModels/gauss/sub/gaussMissingFitEm.m) and @@ -64,10 +65,11 @@ copyright (2010) Kevin Murphy and Matt Dunham: see """ function em_mvn( patterns::AbstractVector{<:SemObservedMissingPattern}; - start_em = start_em_observed, max_iter_em::Integer = 100, rtol_em::Number = 1e-4, - kwargs..., + max_nsamples_em::Union{Integer, Nothing} = nothing, + start_em = start_em_observed, + start_kwargs..., ) nobs_vars = nobserved_vars(patterns[1]) @@ -87,15 +89,27 @@ function em_mvn( end # initialize - Σ₀, μ = start_em(patterns; kwargs...) + Σ₀, μ = start_em(patterns; start_kwargs...) Σ = convert(Matrix, Σ₀) @assert all(isfinite, Σ) all(isfinite, μ) Σ_prev, μ_prev = copy(Σ), copy(μ) iter = 0 converged = false + Δμ_rel = NaN + ΔΣ_rel = NaN while !converged && (iter < max_iter_em) - em_step!(Σ, μ, Σ_prev, μ_prev, patterns, 𝔼x_full, 𝔼xxᵀ_full) + em_step!( + Σ, + μ, + Σ_prev, + μ_prev, + patterns, + 𝔼xxᵀ_full, + 𝔼x_full, + nsamples_full; + max_nsamples_em, + ) if iter > 0 Δμ = norm(μ - μ_prev) @@ -115,13 +129,16 @@ function em_mvn( end if !converged - @warn "EM Algorithm for MVN missing data did not converge in $iter iterations.\n" * + @warn "EM inference for MVN missing data did not converge in $iter iterations.\n" * + "Final tolerances: ΔΣ/Σ=$(ΔΣ_rel), Δμ/μ=$(Δμ_rel).\n" * "Likelihood for FIML is not interpretable.\n" * "Maybe try passing different starting values via 'start_em = ...' " else - @info "EM for MVN missing data converged in $iter iterations" + verbose && @info "EM for MVN missing data converged in $iter iterations: ΔΣ/Σ=$(ΔΣ_rel), Δμ/μ=$(Δμ_rel)." end + StatsBase._symmetrize!(Σ) + return Σ, μ end @@ -132,58 +149,91 @@ function em_step!( Σ₀::AbstractMatrix, μ₀::AbstractVector, patterns::AbstractVector{<:SemObservedMissingPattern}, - 𝔼x_full, - 𝔼xxᵀ_full, + 𝔼xxᵀ_full::AbstractMatrix, + 𝔼x_full::AbstractVector, + nsamples_full::Integer; + max_nsamples_em::Union{Integer, Nothing} = nothing, ) # E step: update 𝔼x and 𝔼xxᵀ copy!(μ, 𝔼x_full) copy!(Σ, 𝔼xxᵀ_full) + nsamples_used = nsamples_full + mul!(Σ, μ₀, μ₀', -nsamples_used, 1) + axpy!(-nsamples_used, μ₀, μ) # Compute the expected sufficient statistics for pat in patterns - (nmissed_vars(pat) == 0) && continue # skip full cases + (nmissed_vars(pat) == 0) && continue # full cases already accounted for # observed and unobserved vars u = pat.miss_mask o = pat.measured_mask - # precompute for pattern - Σoo_chol = cholesky(Symmetric(Σ₀[o, o])) - Σuo = Σ₀[u, o] - μu = μ₀[u] - μo = μ₀[o] - - 𝔼xu = fill!(similar(μu), 0) - 𝔼xo = fill!(similar(μo), 0) - 𝔼xᵢu = similar(μu) - - 𝔼xxᵀuo = fill!(similar(Σuo), 0) - 𝔼xxᵀuu = n_obs(pat) * (Σ₀[u, u] - Σuo * (Σoo_chol \ Σuo')) + # compute cholesky to speed-up ldiv!() + Σ₀oo_chol = cholesky(Symmetric(Σ₀[o, o])) + Σ₀uo = Σ₀[u, o] + μ₀u = μ₀[u] + μ₀o = μ₀[o] + + # get pattern observations + nsamples_pat = + !isnothing(max_nsamples_em) ? min(max_nsamples_em, nsamples(pat)) : + nsamples(pat) + zo = + nsamples_pat < nsamples(pat) ? + pat.data[:, sort!(sample(1:nsamples(pat), nsamples_pat, replace = false))] : + copy(pat.data) + zo .-= μ₀o # subtract current mean from observations + + 𝔼zo = sum(zo, dims = 2) + 𝔼zu = fill!(similar(μ₀u), 0) + + 𝔼zzᵀuo = fill!(similar(Σ₀uo), 0) + 𝔼zzᵀuu = nsamples_pat * Σ₀[u, u] + mul!(𝔼zzᵀuu, Σ₀uo, Σ₀oo_chol \ Σ₀uo', -nsamples_pat, 1) # loop through observations - @inbounds for rowdata in eachcol(pat.data) - mul!(𝔼xᵢu, Σuo, Σoo_chol \ (rowdata - μo)) - 𝔼xᵢu .+= μu - mul!(𝔼xxᵀuu, 𝔼xᵢu, 𝔼xᵢu', 1, 1) - mul!(𝔼xxᵀuo, 𝔼xᵢu, rowdata', 1, 1) - 𝔼xu .+= 𝔼xᵢu - 𝔼xo .+= rowdata + yᵢo = similar(μ₀o) + 𝔼zᵢu = similar(μ₀u) + @inbounds for zᵢo in eachcol(zo) + ldiv!(yᵢo, Σ₀oo_chol, zᵢo) + mul!(𝔼zᵢu, Σ₀uo, yᵢo) + mul!(𝔼zzᵀuu, 𝔼zᵢu, 𝔼zᵢu', 1, 1) + mul!(𝔼zzᵀuo, 𝔼zᵢu, zᵢo', 1, 1) + 𝔼zu .+= 𝔼zᵢu end + # correct 𝔼zzᵀ by adding back μ₀×𝔼z' + 𝔼z'×μ₀ + mul!(𝔼zzᵀuo, μ₀u, 𝔼zo', 1, 1) + mul!(𝔼zzᵀuo, 𝔼zu, μ₀o', 1, 1) + + mul!(𝔼zzᵀuu, μ₀u, 𝔼zu', 1, 1) + mul!(𝔼zzᵀuu, 𝔼zu, μ₀u', 1, 1) + + 𝔼zzᵀoo = zo * zo' + mul!(𝔼zzᵀoo, μ₀o, 𝔼zo', 1, 1) + mul!(𝔼zzᵀoo, 𝔼zo, μ₀o', 1, 1) + + # update Σ and μ + Σ[o, o] .+= 𝔼zzᵀoo + Σ[u, o] .+= 𝔼zzᵀuo + Σ[o, u] .+= 𝔼zzᵀuo' + Σ[u, u] .+= 𝔼zzᵀuu - Σ[o, o] .+= pat.data' * pat.data - Σ[u, o] .+= 𝔼xxᵀuo - Σ[o, u] .+= 𝔼xxᵀuo' - Σ[u, u] .+= 𝔼xxᵀuu + μ[o] .+= 𝔼zo + μ[u] .+= 𝔼zu - μ[o] .+= 𝔼xo - μ[u] .+= 𝔼xu + nsamples_used += nsamples_pat end # M step: update Σ and μ - k = inv(sum(nsamples, patterns)) - lmul!(k, Σ) - lmul!(k, μ) + lmul!(1 / nsamples_used, Σ) + lmul!(1 / nsamples_used, μ) + # at this point μ = μ - μ₀ + # and Σ = Σ + (μ - μ₀)×(μ - μ₀)' - μ₀×μ₀' + mul!(Σ, μ, μ₀', -1, 1) + mul!(Σ, μ₀, μ', -1, 1) mul!(Σ, μ, μ', -1, 1) + μ .+= μ₀ return Σ, μ end From 4bc93f8f4dabc90234deeb743907858deb20d936 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 9 Feb 2026 13:51:50 -0800 Subject: [PATCH 255/364] trunc_eigvals(): new helper func --- src/additional_functions/helper.jl | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 426277d07..5442357f0 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -69,6 +69,38 @@ function elimination_matrix(n::Integer) return L end +# truncate eigenvalues of a symmetric matrix and return the result +function trunc_eigvals( + mtx::AbstractMatrix{T}, + min_eigval::Number; + mtx_label::AbstractString = "matrix", + verbose::Bool = false, +) where {T} + size(mtx, 1) == size(mtx, 2) || + throw(ArgumentError("Matrix must be square, $(size(mtx)) given")) + issymmetric(mtx) || throw(ArgumentError("Matrix must be symmetric")) + + # eigen decomposition of the mtx + mtx_eig = eigen(convert(Matrix{T}, mtx)) + verbose && + @info "min(eigvals($mtx_label))=$(Base.minimum(mtx_eig.values)), N(eigvals < $min_eigval) = $(sum(<(min_eigval), mtx_eig.values))" + + eigmin = Base.minimum(mtx_eig.values) + if eigmin < min_eigval + # substitute small eigvals with min_eigval + eigvals_mtx = Diagonal(max.(mtx_eig.values, min_eigval)) + newmtx = mtx_eig.vectors * eigvals_mtx * mtx_eig.vectors' + StatsBase._symmetrize!(newmtx) + if verbose + Δmtx = newmtx .- mtx + @info "Δ($mtx_label, posdef)=$(norm(Δmtx, 2)), min,max(Δᵢ)=$(extrema(Δmtx))" + end + return newmtx + else + return mtx + end +end + # returns the vector of non-unique values in the order of appearance # each non-unique values is reported once function nonunique(values::AbstractVector) From 3e1049d273ae9c299b91b3db52312fb8453621db Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 9 Feb 2026 15:02:49 -0800 Subject: [PATCH 256/364] em_mvn(): min_eigval to enforce pos-def --- src/observed/EM.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 1b84ab575..cf74216c7 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -31,6 +31,7 @@ THE SOFTWARE. =# max_iter_em = 100, rtol_em = 1e-4, max_nsamples_em = nothing, + min_eigval = nothing, start_em = start_em_observed, start_kwargs...) @@ -68,6 +69,7 @@ function em_mvn( max_iter_em::Integer = 100, rtol_em::Number = 1e-4, max_nsamples_em::Union{Integer, Nothing} = nothing, + min_eigval::Union{Number, Nothing} = nothing, start_em = start_em_observed, start_kwargs..., ) @@ -109,6 +111,7 @@ function em_mvn( 𝔼x_full, nsamples_full; max_nsamples_em, + min_eigval, ) if iter > 0 @@ -153,6 +156,7 @@ function em_step!( 𝔼x_full::AbstractVector, nsamples_full::Integer; max_nsamples_em::Union{Integer, Nothing} = nothing, + min_eigval::Union{Number, Nothing} = nothing, ) # E step: update 𝔼x and 𝔼xxᵀ copy!(μ, 𝔼x_full) @@ -235,6 +239,9 @@ function em_step!( mul!(Σ, μ, μ', -1, 1) μ .+= μ₀ + # try to fix non-positive-definite Σ + isnothing(min_eigval) || copyto!(Σ, trunc_eigvals(Σ, min_eigval)) + return Σ, μ end From d02ff3a4f90ad8f56f567830e4ddde525fa7209e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 9 Feb 2026 14:02:14 -0800 Subject: [PATCH 257/364] add ProgressMeter dep --- Project.toml | 1 + src/StructuralEquationModels.jl | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 73691e6c2..073f2bd00 100644 --- a/Project.toml +++ b/Project.toml @@ -16,6 +16,7 @@ NLSolversBase = "d41bc354-129a-5804-8e4c-c37616107c6c" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 2affc4a73..88c9f932e 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -14,7 +14,8 @@ using LinearAlgebra, StenoGraphs, LazyArtifacts, DelimitedFiles, - DataFrames + DataFrames, + ProgressMeter import StatsAPI: params, coef, coefnames, dof, fit, nobs, coeftable From 64c7f4cd85b87f337e7a99eea9cd10933186f62d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 9 Feb 2026 13:50:33 -0800 Subject: [PATCH 258/364] em_mvn(): verbose arg and progress bar --- src/observed/EM.jl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index cf74216c7..465aa81d0 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -70,12 +70,13 @@ function em_mvn( rtol_em::Number = 1e-4, max_nsamples_em::Union{Integer, Nothing} = nothing, min_eigval::Union{Number, Nothing} = nothing, + verbose::Bool = false, start_em = start_em_observed, start_kwargs..., ) nobs_vars = nobserved_vars(patterns[1]) - # precompute for full cases + verbose && @info "Estimating N(μ, Σ) for complete observations..." 𝔼x_full = zeros(nobs_vars) 𝔼xxᵀ_full = zeros(nobs_vars, nobs_vars) nsamples_full = 0 @@ -90,7 +91,7 @@ function em_mvn( @warn "No full cases in data" end - # initialize + verbose && @info "Estimating initial μ and Σ..." Σ₀, μ = start_em(patterns; start_kwargs...) Σ = convert(Matrix, Σ₀) @assert all(isfinite, Σ) all(isfinite, μ) @@ -100,6 +101,12 @@ function em_mvn( converged = false Δμ_rel = NaN ΔΣ_rel = NaN + progress = Progress( + max_iter_em, + dt = 1.0, + showspeed = true, + desc = "EM inference of MVN(μ, Σ)", + ) while !converged && (iter < max_iter_em) em_step!( Σ, @@ -128,8 +135,9 @@ function em_mvn( μ, μ_prev = μ_prev, μ end iter += 1 - #@info "$iter\n" + next!(progress, step = 1, showvalues = [("ΔΣ/Σ", ΔΣ_rel), ("Δμ/μ", Δμ_rel)]) end + finish!(progress) if !converged @warn "EM inference for MVN missing data did not converge in $iter iterations.\n" * @@ -137,7 +145,8 @@ function em_mvn( "Likelihood for FIML is not interpretable.\n" * "Maybe try passing different starting values via 'start_em = ...' " else - verbose && @info "EM for MVN missing data converged in $iter iterations: ΔΣ/Σ=$(ΔΣ_rel), Δμ/μ=$(Δμ_rel)." + verbose && + @info "EM for MVN missing data converged in $iter iterations: ΔΣ/Σ=$(ΔΣ_rel), Δμ/μ=$(Δμ_rel)." end StatsBase._symmetrize!(Σ) From 405a64ca026a67656e031a9209786154a8b6c495 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 14 Apr 2024 16:22:46 -0700 Subject: [PATCH 259/364] test/fiml: set EM MVN rtol=1e-10 to make tests pass --- test/examples/political_democracy/by_parts.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 3397b5f0a..f9d73667c 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -309,7 +309,8 @@ end ### fiml ############################################################################################ -observed = SemObservedMissing(specification = spec_mean, data = dat_missing) +observed = + SemObservedMissing(specification = spec_mean, data = dat_missing, rtol_em = 1e-10) fiml = SemFIML(observed = observed, specification = spec_mean) From c0cda00e7abb522c6994a3fa583477b30c3c4b32 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 11 Aug 2024 12:09:29 -0700 Subject: [PATCH 260/364] SemObsMissing: fix obs_mean() test it's ok to return vector --- test/unit_tests/data_input_formats.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 183b067f5..7f024e110 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -93,8 +93,7 @@ function test_observed( @test @inferred(obs_mean(observed)) == dat_mean end else - # FIXME @inferred is broken for EM cov/mean since it may return nothing if EM was not run - @test @inferred(obs_mean(observed)) isa AbstractVector{Float64} broken = true # EM-based means + @test @inferred(obs_mean(observed)) isa AbstractVector{Float64} # EM-based means end else @test @inferred(obs_mean(observed)) === nothing skip = true From 9b05e61e731bf7ffb8015e6905458edf59ffab03 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 13 Feb 2026 14:31:50 -0800 Subject: [PATCH 261/364] em_step!(): enforce symmetry --- src/observed/EM.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 465aa81d0..88af6112a 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -248,8 +248,13 @@ function em_step!( mul!(Σ, μ, μ', -1, 1) μ .+= μ₀ - # try to fix non-positive-definite Σ - isnothing(min_eigval) || copyto!(Σ, trunc_eigvals(Σ, min_eigval)) + StatsBase._symmetrize!(Σ) # correct numerical errors + if !isnothing(min_eigval) + # try to fix non-positive-definite Σ + reg_Σ = trunc_eigvals(Σ, min_eigval) + # if no truncation was done, reg_Σ will be the same object + (reg_Σ === Σ) || copyto!(Σ, reg_Σ) + end return Σ, μ end From 4d8d40e4043f5a9616e5b4454b15ba20224e7586 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 15 Feb 2026 16:09:25 -0800 Subject: [PATCH 262/364] SemML&WLS: warn if SemObsMissing don't throw an exception --- src/loss/ML/ML.jl | 23 +++++++++++------------ src/loss/WLS/WLS.jl | 30 +++++++++++++++--------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index bf0c245ff..6461ba087 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -39,19 +39,18 @@ end ############################################################################################ function SemML(; observed::SemObserved, approximate_hessian::Bool = false, kwargs...) - if observed isa SemObservedMissing - throw(ArgumentError( - "Normal maximum likelihood estimation can't be used with `SemObservedMissing`. - Use full information maximum likelihood (FIML) estimation or remove missing - values in your data. - A FIML model can be constructed with - Sem( - ..., - observed = SemObservedMissing, - loss = SemFIML, - meanstructure = true - )")) + @warn """ + ML estimation with `SemObservedMissing` will use an approximate covariance and mean estimated with EM algorithm. + For more accurate results, consider using full information maximum likelihood (FIML) estimation or remove missing values in your data. + A FIML model can be constructed with + Sem( + ..., + observed = SemObservedMissing, + loss = SemFIML, + meanstructure = true + ) + """ end obsmean = obs_mean(observed) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 104c8affe..b2aed17c0 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -58,24 +58,24 @@ function SemWLS(; meanstructure = false, kwargs..., ) - if observed isa SemObservedMissing - throw(ArgumentError( - "WLS estimation can't be used with `SemObservedMissing`. - Use full information maximum likelihood (FIML) estimation or remove missing - values in your data. - A FIML model can be constructed with - Sem( - ..., - observed = SemObservedMissing, - loss = SemFIML, - meanstructure = true - )")) + @warn """ + WLS estimation with `SemObservedMissing` will use an approximate covariance and mean estimated with EM algorithm. + For more accurate results, consider using full information maximum likelihood (FIML) estimation or remove missing values in your data. + A FIML model can be constructed with + Sem( + ..., + observed = SemObservedMissing, + loss = SemFIML, + meanstructure = true + ) + """ end - if !(implied isa RAMSymbolic) - throw(ArgumentError( - "WLS estimation is only available with the implied type RAMSymbolic at the moment.")) + if !(implied isa RAMSymbolic) + "WLS estimation is only available with the implied type RAMSymbolic at the moment." |> + ArgumentError |> + throw end nobs_vars = nobserved_vars(observed) From 36b88e62253aa114ae17451e84121f3ab771ce23 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:41:08 -0700 Subject: [PATCH 263/364] SemObservedData: fix ws and msg indent --- src/observed/data.jl | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/observed/data.jl b/src/observed/data.jl index fffeb36bd..30d433e03 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -38,22 +38,24 @@ function SemObservedData(; observed_var_prefix::Union{Symbol, AbstractString} = :obs, kwargs..., ) - data, obs_vars, _ = prepare_data(data, observed_vars, specification; observed_var_prefix) obs_mean, obs_cov = mean_and_cov(data, 1) - if any(ismissing.(data)) - throw(ArgumentError( - "Your dataset contains missing values. - Remove missing values or use full information maximum likelihood (FIML) estimation. - A FIML model can be constructed with - Sem( - ..., - observed = SemObservedMissing, - loss = SemFIML, - meanstructure = true - )")) + if any(ismissing, data) + """ + Your dataset contains missing values. + Remove missing values or use full information maximum likelihood (FIML) estimation. + A FIML model can be constructed with + Sem( + ..., + observed = SemObservedMissing, + loss = SemFIML, + meanstructure = true + ) + """ |> + ArgumentError |> + throw end return SemObservedData(data, obs_vars, obs_cov, vec(obs_mean), size(data, 1)) From 1d51d9a78c9f238fc2223f50c5799aeb0e84d395 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 27 May 2024 16:28:52 -0700 Subject: [PATCH 264/364] fit_measures(): formatting tweaks --- src/frontend/fit/fitmeasures/fit_measures.jl | 32 ++++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/frontend/fit/fitmeasures/fit_measures.jl b/src/frontend/fit/fitmeasures/fit_measures.jl index afdde173b..185b348c0 100644 --- a/src/frontend/fit/fitmeasures/fit_measures.jl +++ b/src/frontend/fit/fitmeasures/fit_measures.jl @@ -1,19 +1,25 @@ -fit_measures(sem_fit) = - fit_measures(sem_fit, nparams, dof, AIC, BIC, RMSEA, χ², p_value, minus2ll) +fit_measures(fit) = fit_measures(fit, nparams, dof, AIC, BIC, RMSEA, χ², p_value, minus2ll) -function fit_measures(sem_fit, args...) - measures = Dict{Symbol, Union{Float64, Missing}}() +fit_measures(fit, measures...) = Dict(Symbol(fn) => fn(fit) for fn in measures) - for arg in args - push!(measures, Symbol(arg) => arg(sem_fit)) - end +""" + fit_measures(fit::SemFit, measures...) -> Dict{Symbol} - return measures -end +Calculate fit measures for the SEM solution. -""" - fit_measures(sem_fit, args...) +The `measures` are the functions that calculate fit measures for a given SEM solution +([`SemFit`](@ref) object). If no `measures` are specified, the default set of measures is used. + +Returns a dictionary of the fit measures, where the keys are the function names. + +# Examples + +```julia +fit_measures(semfit) +fit_measures(semfit, nparams, dof, p_value) +``` -Return a default set of fit measures or the fit measures passed as `args...`. +# See also +[`AIC`](@ref), [`BIC`](@ref), [`RMSEA`](@ref), [`χ²`](@ref), [`p_value`](@ref), [`minus2ll`](@ref) """ -function fit_measures end +fit_measures From 1d9bc286464670b4a649ba3461b147b7807ea087 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 31 Aug 2024 14:59:14 -0700 Subject: [PATCH 265/364] improve docstrings for fit measures --- src/frontend/fit/fitmeasures/AIC.jl | 9 ++++++--- src/frontend/fit/fitmeasures/BIC.jl | 9 ++++++--- src/frontend/fit/fitmeasures/RMSEA.jl | 18 +++++++++++++++++- src/frontend/fit/fitmeasures/chi2.jl | 9 ++++++++- src/frontend/fit/fitmeasures/dof.jl | 14 +++++++++++--- src/frontend/fit/fitmeasures/minus2ll.jl | 7 +++++-- src/frontend/fit/fitmeasures/p.jl | 9 ++++++--- 7 files changed, 59 insertions(+), 16 deletions(-) diff --git a/src/frontend/fit/fitmeasures/AIC.jl b/src/frontend/fit/fitmeasures/AIC.jl index f26f1f4dc..91bc09810 100644 --- a/src/frontend/fit/fitmeasures/AIC.jl +++ b/src/frontend/fit/fitmeasures/AIC.jl @@ -1,6 +1,9 @@ """ - AIC(sem_fit::SemFit) + AIC(fit::SemFit) -Return the akaike information criterion. +Calculate the *AIC* ([*Akaike information criterion*](https://en.wikipedia.org/wiki/Akaike_information_criterion)). + +# See also +[`fit_measures`](@ref) """ -AIC(sem_fit::SemFit) = minus2ll(sem_fit) + 2nparams(sem_fit) +AIC(fit::SemFit) = minus2ll(fit) + 2nparams(fit) diff --git a/src/frontend/fit/fitmeasures/BIC.jl b/src/frontend/fit/fitmeasures/BIC.jl index 20638f4e4..44190375a 100644 --- a/src/frontend/fit/fitmeasures/BIC.jl +++ b/src/frontend/fit/fitmeasures/BIC.jl @@ -1,6 +1,9 @@ """ - BIC(sem_fit::SemFit) + BIC(fit::SemFit) -Return the bayesian information criterion. +Calculate the *BIC* ([*Bayesian information criterion*](https://en.wikipedia.org/wiki/Bayesian_information_criterion)). + +# See also +[`fit_measures`](@ref) """ -BIC(sem_fit::SemFit) = minus2ll(sem_fit) + log(nsamples(sem_fit)) * nparams(sem_fit) +BIC(fit::SemFit) = minus2ll(fit) + log(nsamples(fit)) * nparams(fit) diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index f9dae84ed..890a7ed53 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -1,7 +1,23 @@ """ RMSEA(fit::SemFit) -Return the RMSEA. +Calculate the RMSEA ([*Root Mean Squared Error of Approximation*](https://meth.psychopen.eu/index.php/meth/article/download/2333/2333.html?inline=1#sec1)). + +Uses the formula +```math +\\mathrm{RMSEA} = \\sqrt{\\frac{\\chi^2 - N_{\\mathrm{df}}}{N_{\\mathrm{obs}} * N_{\\mathrm{df}}}}, +``` +where *χ²* is the chi-squared statistic, ``N_{\\mathrm{df}}`` is the degrees of freedom, +and ``N_{\\mathrm{obs}}`` is the (corrected) number of observations +for the SEM model. + +# See also +[`fit_measures`](@ref), [`χ²`](@ref), [`dof`](@ref) + +# Extended help + +For multigroup models, the correction proposed by J.H. Steiger is applied +(see [Steiger, J. H. (1998). *A note on multiple sample extensions of the RMSEA fit index*](https://doi.org/10.1080/10705519809540115)). """ function RMSEA end diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index dc19467fc..d1bf8f926 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -1,7 +1,14 @@ """ χ²(fit::SemFit) -Return the χ² value. +Calculate the *χ²* (*chi-square*) value for the `fit`. + +The *χ²* is a test statistic for the SEM goodness-of-fit. +It compares the *implied* covariance matrix of the SEM model +with the *observed* covariance matrix. + +# See also +[`fit_measures`](@ref) """ χ²(fit::SemFit) = χ²(fit, fit.model) diff --git a/src/frontend/fit/fitmeasures/dof.jl b/src/frontend/fit/fitmeasures/dof.jl index 3df49d89d..0e051d02a 100644 --- a/src/frontend/fit/fitmeasures/dof.jl +++ b/src/frontend/fit/fitmeasures/dof.jl @@ -1,12 +1,20 @@ """ - dof(sem_fit::SemFit) + dof(fit::SemFit) dof(model::AbstractSem) -Return the degrees of freedom. +Get the *degrees of freedom* for the SEM model. + +The *degrees of freedom* for the SEM with *N* observed variables is the difference +between the number of non-redundant elements in the observed covariance matrix +(*½N(N+1)*) and the number of model parameters, *q* ([`nparams(model)`](@ref nparams)). +If the SEM also models the observed means, the formula becomes *½N(N+1) + N - q*. + +# See also +[`fit_measures`](@ref) """ function dof end -dof(sem_fit::SemFit) = dof(sem_fit.model) +dof(fit::SemFit) = dof(fit.model) dof(model::AbstractSem) = n_dp(model) - nparams(model) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 2b5a39491..f87b6f179 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -1,7 +1,10 @@ """ - minus2ll(sem_fit::SemFit) + minus2ll(fit::SemFit) -Return the negative 2* log likelihood. +Calculate the *-2⋅log(likelihood(fit))*. + +# See also +[`fit_measures`](@ref) """ function minus2ll end diff --git a/src/frontend/fit/fitmeasures/p.jl b/src/frontend/fit/fitmeasures/p.jl index da9bedaf6..50cf7220c 100644 --- a/src/frontend/fit/fitmeasures/p.jl +++ b/src/frontend/fit/fitmeasures/p.jl @@ -1,6 +1,9 @@ """ - p(sem_fit::SemFit) + p_value(fit::SemFit) -Return the p value computed from the χ² test statistic. +Calculate the *p*-value for the *χ²* test statistic. + +# See also +[`fit_measures`](@ref), [`χ²`](@ref) """ -p_value(sem_fit::SemFit) = ccdf(Chisq(dof(sem_fit)), χ²(sem_fit)) +p_value(fit::SemFit) = ccdf(Chisq(dof(fit)), χ²(fit)) From f2654528e686a348f719b9cf0f402ad8c2268a8e Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:13:15 +0100 Subject: [PATCH 266/364] add helper functions for heterogenous lossfuns and scaling corrections --- src/additional_functions/helper.jl | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 5442357f0..e632a9723 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -115,3 +115,37 @@ function nonunique(values::AbstractVector) end return res end + +# check that a model only has a single lossfun +function check_single_lossfun(model::AbstractSemSingle; throw_error) + if (length(model.loss.functions) > 1) & throw_error + @error "The model has $(length(sem.loss.functions)) loss functions. + Only a single loss function is supported." + end + return isone(length(model.loss.functions)) +end + +# check that all models use the same single loss function +function check_single_lossfun(models::AbstractSemSingle...; throw_error) + uniform = true + lossfun = models[1].loss.functions[1] + L = typeof(lossfun) + for (i, model) in enumerate(models) + uniform &= check_single_lossfun(model; throw_error = throw_error) + cur_lossfun = model.loss.functions[1] + if !isa(cur_lossfun, L) & throw_error + @error "Loss function for group #$i model is $(typeof(cur_lossfun)), expected $L. + Heterogeneous loss functions are not supported." + end + uniform &= isa(cur_lossfun, L) + end + return uniform +end + +check_single_lossfun(model::SemEnsemble; throw_error) = + check_single_lossfun(model.sems...; throw_error) + +# sclaing corrections for fit measures and multigroup models +dof_correction(::SemFIML) = 0 +dof_correction(::SemML) = -1 +dof_correction(::SemWLS) = -1 \ No newline at end of file From b93db77e953454659fe338c57b015357d37efce6 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:14:09 +0100 Subject: [PATCH 267/364] adapt default multigroup weights and give info about defaults used --- src/types.jl | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/types.jl b/src/types.jl index 777165f37..5dc7c524c 100644 --- a/src/types.jl +++ b/src/types.jl @@ -192,10 +192,7 @@ end function SemEnsemble(models...; weights = nothing, groups = nothing, kwargs...) n = length(models) # default weights - if isnothing(weights) - nsamples_total = sum(nsamples, models) - weights = [nsamples(model) / nsamples_total for model in models] - end + weights = isnothing(weights) ? multigroup_weights(models, n) : weights # default group labels groups = isnothing(groups) ? Symbol.(:g, 1:n) : groups # check parameters equality @@ -226,7 +223,25 @@ function SemEnsemble(; specification, data, groups, column = :group, kwargs...) model = Sem(; specification = ram_matrices, data = data_group, kwargs...) push!(models, model) end - return SemEnsemble(models...; weights = nothing, groups = groups, kwargs...) + return SemEnsemble(models...; groups = groups, kwargs...) +end + +function multigroup_weights(models, n) + nsamples_total = sum(nsamples, models) + uniform_lossfun = check_single_lossfun(models...; throw_error = false) + if !uniform_lossfun + @info "Your ensemble model contains heterogeneous loss functions. + Default weights of (#samples per group/#total samples) will be used". + return [(nsamples(model)) / (nsamples_total) for model in models] + end + lossfun = models[1].loss.functions[1] + if !applicable(dof_correction, lossfun) + @info "We don't know how to choose group weights for the specified loss function. + Default weights of (#samples per group/#total samples) will be used". + return [(nsamples(model)) / (nsamples_total) for model in models] + end + dc = dof_correction(lossfun) + return [(nsamples(model)-dc) / (nsamples_total-n*dc) for model in models] end param_labels(ensemble::SemEnsemble) = ensemble.param_labels From 2c66951b6b224b71444d3c791f591679073aaa04 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:20:02 +0100 Subject: [PATCH 268/364] refactor minus2ll --- src/frontend/fit/fitmeasures/minus2ll.jl | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index f87b6f179..2b5672568 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -6,36 +6,31 @@ Calculate the *-2⋅log(likelihood(fit))*. # See also [`fit_measures`](@ref) """ -function minus2ll end +minus2ll(fit::SemFit) = minus2ll(fit, fit.model) ############################################################################################ # Single Models ############################################################################################ -minus2ll(fit::SemFit) = minus2ll(fit, fit.model) - function minus2ll(fit::SemFit, model::AbstractSemSingle) - minimum = objective(model, fit.solution) - return minus2ll(minimum, model) + check_single_lossfun(model; throw_error = true) + return minus2ll(model.loss.functions[1], fit, model) end -minus2ll(minimum::Number, model::AbstractSemSingle) = - sum(lossfun -> minus2ll(lossfun, minimum, model), model.loss.functions) - # SemML ------------------------------------------------------------------------------------ -function minus2ll(lossfun::SemML, minimum::Number, model::AbstractSemSingle) +function minus2ll(::SemML, fit::SemFit, model::AbstractSemSingle) obs = observed(model) - return nsamples(obs) * (minimum + log(2π) * nobserved_vars(obs)) + return nsamples(obs) * (fit.minimum + log(2π) * nobserved_vars(obs)) end # WLS -------------------------------------------------------------------------------------- -minus2ll(lossfun::SemWLS, minimum::Number, model::AbstractSemSingle) = missing +minus2ll(::SemWLS, ::SemFit, ::AbstractSemSingle) = missing # compute likelihood for missing data - H0 ------------------------------------------------- -# -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n -function minus2ll(lossfun::SemFIML, minimum::Number, model::AbstractSemSingle) +# -2ll = (∑ log(2π)*(nᵢ*mᵢ)) + F*n +function minus2ll(::SemFIML, fit::SemFit, model::AbstractSemSingle) obs = observed(model)::SemObservedMissing - F = minimum * nsamples(obs) + F = fit.minimum * nsamples(obs) F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), obs.patterns) return F end From 2039c11de207633b80257c407ec7360bf480493d Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:32:45 +0100 Subject: [PATCH 269/364] refactor minus2ll --- src/frontend/fit/fitmeasures/minus2ll.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 2b5672568..615f4aa66 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -65,4 +65,7 @@ end # Collection ############################################################################################ -minus2ll(fit::SemFit, model::SemEnsemble) = sum(Base.Fix1(minus2ll, fit), model.sems) +function minus2ll(fit::SemFit, model::SemEnsemble) + check_single_lossfun(model; throw_error = true) + return sum(Base.Fix1(minus2ll, fit), model.sems) +end From 8d65c8016cb3d68736745f9e490f3dec7dae7b9f Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:32:59 +0100 Subject: [PATCH 270/364] refactor chi2 --- src/frontend/fit/fitmeasures/chi2.jl | 44 ++++++++++------------------ 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index d1bf8f926..fa7372afe 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -16,20 +16,21 @@ with the *observed* covariance matrix. # Single Models ############################################################################################ -χ²(fit::SemFit, model::AbstractSemSingle) = - sum(loss -> χ²(loss, fit, model), model.loss.functions) +function χ²(fit::SemFit, model::AbstractSemSingle) + check_single_lossfun(model; throw_error = true) + return χ²(model.loss.functions[1], fit::SemFit, model::AbstractSemSingle) +end -# RAM + SemML -χ²(lossfun::SemML, fit::SemFit, model::AbstractSemSingle) = +χ²(::SemML, fit::SemFit, model::AbstractSemSingle) = (nsamples(fit) - 1) * (fit.minimum - logdet(obs_cov(observed(model))) - nobserved_vars(observed(model))) # bollen, p. 115, only correct for GLS weight matrix -χ²(lossfun::SemWLS, fit::SemFit, model::AbstractSemSingle) = +χ²(::SemWLS, fit::SemFit, model::AbstractSemSingle) = (nsamples(fit) - 1) * fit.minimum # FIML -function χ²(lossfun::SemFIML, fit::SemFit, model::AbstractSemSingle) +function χ²(::SemFIML, fit::SemFit, model::AbstractSemSingle) ll_H0 = minus2ll(fit) ll_H1 = minus2ll(observed(model)) return ll_H0 - ll_H1 @@ -39,38 +40,25 @@ end # Collections ############################################################################################ -function χ²(fit::SemFit, models::SemEnsemble) - isempty(models.sems) && return 0.0 - - lossfun = models.sems[1].loss.functions[1] - # check that all models use the same single loss function - L = typeof(lossfun) - for (i, sem) in enumerate(models.sems) - if length(sem.loss.functions) > 1 - @error "Model for group #$i has $(length(sem.loss.functions)) loss functions. Only the single one is supported" - end - cur_lossfun = sem.loss.functions[1] - if !isa(cur_lossfun, L) - @error "Loss function for group #$i model is $(typeof(cur_lossfun)), expected $L. Heterogeneous loss functions are not supported" - end - end - - return χ²(lossfun, fit, models) +function χ²(fit::SemFit, model::SemEnsemble) + check_single_lossfun(model; throw_error = true) + lossfun = model.sems[1].loss.functions[1] + return χ²(lossfun, fit, model) end -function χ²(lossfun::SemWLS, fit::SemFit, models::SemEnsemble) - return (nsamples(models) - 1) * fit.minimum +function χ²(::SemWLS, fit::SemFit, models::SemEnsemble) + return (nsamples(models) - models.n) * fit.minimum end -function χ²(lossfun::SemML, fit::SemFit, models::SemEnsemble) +function χ²(::SemML, fit::SemFit, models::SemEnsemble) G = sum(zip(models.weights, models.sems)) do (w, model) data = observed(model) w * (logdet(obs_cov(data)) + nobserved_vars(data)) end - return (nsamples(models) - 1) * (fit.minimum - G) + return (nsamples(models) - models.n) * (fit.minimum - G) end -function χ²(lossfun::SemFIML, fit::SemFit, models::SemEnsemble) +function χ²(::SemFIML, fit::SemFit, models::SemEnsemble) ll_H0 = minus2ll(fit) ll_H1 = sum(minus2ll ∘ observed, models.sems) return ll_H0 - ll_H1 From d84d41d2355dd359adf4ec739d3f1b8bf41f17d1 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 15 Feb 2026 18:33:16 +0100 Subject: [PATCH 271/364] refactor RMSEA --- src/frontend/fit/fitmeasures/RMSEA.jl | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index 890a7ed53..538d08012 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -23,13 +23,20 @@ function RMSEA end RMSEA(fit::SemFit) = RMSEA(fit, fit.model) -RMSEA(fit::SemFit, model::AbstractSemSingle) = RMSEA(dof(fit), χ²(fit), nsamples(fit)) +function RMSEA(fit::SemFit, model::AbstractSemSingle) + check_uniform_lossfun(model) + return RMSEA(dof(fit), χ²(fit), nsamples(fit)-dof_correction(model.loss.functions[1])) +end -RMSEA(fit::SemFit, model::SemEnsemble) = - sqrt(length(model.sems)) * RMSEA(dof(fit), χ²(fit), nsamples(fit)) +function RMSEA(fit::SemFit, model::SemEnsemble) + check_single_lossfun(model; throw_error = true) + n = nsamples(fit)-model.n*dof_correction(model.sems[1].loss.functions[1]) + return sqrt(length(model.sems)) * RMSEA(dof(fit), χ²(fit), n) +end -function RMSEA(dof, chi2, nsamples) - rmsea = (chi2 - dof) / (nsamples * dof) - rmsea > 0 ? nothing : rmsea = 0 +function RMSEA(dof, chi2, c) + rmsea = (chi2 - dof) / (c * dof) + rmsea = rmsea > 0 ? rmsea : 0 return sqrt(rmsea) end + From c62e0732695c8672b191811e0cc639ec23e3ad1e Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 16 Feb 2026 15:00:26 +0100 Subject: [PATCH 272/364] adapt mg models and fitmeasures --- src/additional_functions/helper.jl | 8 ++++---- src/frontend/fit/fitmeasures/RMSEA.jl | 14 +++++++++----- src/frontend/fit/fitmeasures/chi2.jl | 13 ++++++++----- src/frontend/fit/fitmeasures/minus2ll.jl | 14 +++++++------- src/types.jl | 6 +++--- 5 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index e632a9723..6e9f1f1be 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -145,7 +145,7 @@ end check_single_lossfun(model::SemEnsemble; throw_error) = check_single_lossfun(model.sems...; throw_error) -# sclaing corrections for fit measures and multigroup models -dof_correction(::SemFIML) = 0 -dof_correction(::SemML) = -1 -dof_correction(::SemWLS) = -1 \ No newline at end of file +# scaling corrections for multigroup models +mg_correction(::SemFIML) = 0 +mg_correction(::SemML) = 0 +mg_correction(::SemWLS) = -1 \ No newline at end of file diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index 538d08012..64cb1985e 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -24,19 +24,23 @@ function RMSEA end RMSEA(fit::SemFit) = RMSEA(fit, fit.model) function RMSEA(fit::SemFit, model::AbstractSemSingle) - check_uniform_lossfun(model) - return RMSEA(dof(fit), χ²(fit), nsamples(fit)-dof_correction(model.loss.functions[1])) + check_single_lossfun(model; throw_error = true) + return RMSEA(dof(fit), χ²(fit), nsamples(fit)+rmsea_correction(model.loss.functions[1])) end function RMSEA(fit::SemFit, model::SemEnsemble) check_single_lossfun(model; throw_error = true) - n = nsamples(fit)-model.n*dof_correction(model.sems[1].loss.functions[1]) + n = nsamples(fit)+model.n*rmsea_correction(model.sems[1].loss.functions[1]) return sqrt(length(model.sems)) * RMSEA(dof(fit), χ²(fit), n) end -function RMSEA(dof, chi2, c) - rmsea = (chi2 - dof) / (c * dof) +function RMSEA(dof, chi2, N⁻) + rmsea = (chi2 - dof) / (N⁻ * dof) rmsea = rmsea > 0 ? rmsea : 0 return sqrt(rmsea) end +# scaling corrections +rmsea_correction(::SemFIML) = 0 +rmsea_correction(::SemML) = -1 +rmsea_correction(::SemWLS) = -1 \ No newline at end of file diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index fa7372afe..8ce5f0798 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -23,7 +23,7 @@ end χ²(::SemML, fit::SemFit, model::AbstractSemSingle) = (nsamples(fit) - 1) * - (fit.minimum - logdet(obs_cov(observed(model))) - nobserved_vars(observed(model))) + (fit.minimum - logdet(obs_cov(observed(model))) - nobserved_vars(model)) # bollen, p. 115, only correct for GLS weight matrix χ²(::SemWLS, fit::SemFit, model::AbstractSemSingle) = @@ -51,11 +51,14 @@ function χ²(::SemWLS, fit::SemFit, models::SemEnsemble) end function χ²(::SemML, fit::SemFit, models::SemEnsemble) - G = sum(zip(models.weights, models.sems)) do (w, model) - data = observed(model) - w * (logdet(obs_cov(data)) + nobserved_vars(data)) + F = 0 + for model in models.sems + Fᵢ = objective(model, fit.solution) + Fᵢ -= logdet(obs_cov(observed(model))) + nobserved_vars(model) + Fᵢ *= nsamples(model) - 1 + F += Fᵢ end - return (nsamples(models) - models.n) * (fit.minimum - G) + return F end function χ²(::SemFIML, fit::SemFit, models::SemEnsemble) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 615f4aa66..c6a954efa 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -14,23 +14,23 @@ minus2ll(fit::SemFit) = minus2ll(fit, fit.model) function minus2ll(fit::SemFit, model::AbstractSemSingle) check_single_lossfun(model; throw_error = true) - return minus2ll(model.loss.functions[1], fit, model) + F = objective(model, fit.solution) + return minus2ll(model.loss.functions[1], F, model) end # SemML ------------------------------------------------------------------------------------ -function minus2ll(::SemML, fit::SemFit, model::AbstractSemSingle) - obs = observed(model) - return nsamples(obs) * (fit.minimum + log(2π) * nobserved_vars(obs)) +function minus2ll(::SemML, F, model::AbstractSemSingle) + return nsamples(model) * (F + log(2π) * nobserved_vars(model)) end # WLS -------------------------------------------------------------------------------------- -minus2ll(::SemWLS, ::SemFit, ::AbstractSemSingle) = missing +minus2ll(::SemWLS, F, ::AbstractSemSingle) = missing # compute likelihood for missing data - H0 ------------------------------------------------- # -2ll = (∑ log(2π)*(nᵢ*mᵢ)) + F*n -function minus2ll(::SemFIML, fit::SemFit, model::AbstractSemSingle) +function minus2ll(::SemFIML, F, model::AbstractSemSingle) obs = observed(model)::SemObservedMissing - F = fit.minimum * nsamples(obs) + F *= nsamples(obs) F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), obs.patterns) return F end diff --git a/src/types.jl b/src/types.jl index 5dc7c524c..3f695bfa3 100644 --- a/src/types.jl +++ b/src/types.jl @@ -235,13 +235,13 @@ function multigroup_weights(models, n) return [(nsamples(model)) / (nsamples_total) for model in models] end lossfun = models[1].loss.functions[1] - if !applicable(dof_correction, lossfun) + if !applicable(mg_correction, lossfun) @info "We don't know how to choose group weights for the specified loss function. Default weights of (#samples per group/#total samples) will be used". return [(nsamples(model)) / (nsamples_total) for model in models] end - dc = dof_correction(lossfun) - return [(nsamples(model)-dc) / (nsamples_total-n*dc) for model in models] + c = mg_correction(lossfun) + return [(nsamples(model)+c) / (nsamples_total+n*c) for model in models] end param_labels(ensemble::SemEnsemble) = ensemble.param_labels From d6015301e4f2c55868a78f8e8bfb8ce080bc9ec6 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Thu, 19 Feb 2026 11:32:19 +0100 Subject: [PATCH 273/364] Apply suggestion from @github-actions[bot] Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/additional_functions/helper.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 6e9f1f1be..268880b39 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -148,4 +148,4 @@ check_single_lossfun(model::SemEnsemble; throw_error) = # scaling corrections for multigroup models mg_correction(::SemFIML) = 0 mg_correction(::SemML) = 0 -mg_correction(::SemWLS) = -1 \ No newline at end of file +mg_correction(::SemWLS) = -1 From 7942c1f7e52cf99364e6d1c3ed70d8a8e4e7c766 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Thu, 19 Feb 2026 11:32:32 +0100 Subject: [PATCH 274/364] Apply suggestion from @github-actions[bot] Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/additional_functions/helper.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 268880b39..b3f5212b7 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -142,7 +142,7 @@ function check_single_lossfun(models::AbstractSemSingle...; throw_error) return uniform end -check_single_lossfun(model::SemEnsemble; throw_error) = +check_single_lossfun(model::SemEnsemble; throw_error) = check_single_lossfun(model.sems...; throw_error) # scaling corrections for multigroup models From efd0a4d8c67ecb929cca9dc1b736fbb7ff6241b9 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Thu, 19 Feb 2026 11:32:49 +0100 Subject: [PATCH 275/364] Apply suggestion from @github-actions[bot] Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/frontend/fit/fitmeasures/RMSEA.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index 64cb1985e..8539896f4 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -43,4 +43,4 @@ end # scaling corrections rmsea_correction(::SemFIML) = 0 rmsea_correction(::SemML) = -1 -rmsea_correction(::SemWLS) = -1 \ No newline at end of file +rmsea_correction(::SemWLS) = -1 From e628276780b0c04635868f63f8e3f0b99ad502e5 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 8 Mar 2026 18:11:22 +0100 Subject: [PATCH 276/364] add CFI --- src/StructuralEquationModels.jl | 2 + src/frontend/fit/fitmeasures/CFI.jl | 72 +++++++++++++++++++ test/examples/multigroup/build_models.jl | 44 ++++++++++++ test/examples/multigroup/multigroup.jl | 13 ++++ test/examples/political_democracy/by_parts.jl | 26 ++++++- .../political_democracy/constructor.jl | 45 +++++++++++- .../political_democracy.jl | 11 +++ 7 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 src/frontend/fit/fitmeasures/CFI.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 71cedf1c4..3d5625e05 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -84,6 +84,7 @@ include("frontend/fit/fitmeasures/dof.jl") include("frontend/fit/fitmeasures/minus2ll.jl") include("frontend/fit/fitmeasures/p.jl") include("frontend/fit/fitmeasures/RMSEA.jl") +include("frontend/fit/fitmeasures/CFI.jl") include("frontend/fit/fitmeasures/fit_measures.jl") # standard errors include("frontend/fit/standard_errors/hessian.jl") @@ -186,6 +187,7 @@ export AbstractSem, minus2ll, p_value, RMSEA, + CFI, EmMVNModel, se_hessian, se_bootstrap, diff --git a/src/frontend/fit/fitmeasures/CFI.jl b/src/frontend/fit/fitmeasures/CFI.jl new file mode 100644 index 000000000..8759e5d48 --- /dev/null +++ b/src/frontend/fit/fitmeasures/CFI.jl @@ -0,0 +1,72 @@ +""" + CFI(fit::SemFit) + +Return the CFI. +""" +function CFI end + + +# if the user provides a baseline model +CFI(fit::SemFit, fit_baseline::SemFit) = + CFI(χ²(fit), dof(fit), χ²(fit_baseline), dof(fit_baseline)) + +# no baseline -> variance only model +CFI(fit::SemFit) = CFI(fit, fit.model) + +function CFI(fit::SemFit, model::AbstractSem) + dof₀ = dof_varonly(model) + χ²₀ = χ²_varonly(model) + return CFI(χ²(fit), dof(fit), χ²₀, dof₀) +end + +# basic CFI function +function CFI(χ², dof, χ²₀, dof₀) + λ = χ² - dof + λ₀ = χ²₀ - dof₀ + return 1 - maximum([λ, 0])/maximum([λ, λ₀, 0]) +end + +### +function χ²_varonly(model::AbstractSemSingle) + check_single_lossfun(model; throw_error = true) + return χ²_varonly(model.loss.functions[1], model) +end + +function χ²_varonly(model::SemEnsemble) + check_single_lossfun(model; throw_error = true) + return sum(χ²_varonly, model.sems) +end + +function χ²_varonly(::SemML, model::AbstractSemSingle) + N⁻ = (nsamples(model) - 1) + S = obs_cov(observed(model)) + Σ₀ = Diagonal(S) + p = nobserved_vars(model) + return N⁻*(logdet(Σ₀) + tr(inv(Σ₀)*S) - logdet(S) - p) +end + +# for the optimal variance only model, we have to solve 1/2 tr((I-XS⁻¹)^2) with X diagonal +function χ²_varonly(::SemWLS, model) + N⁻ = (nsamples(model) - 1) + S⁻¹ = inv((obs_cov(observed(model)))) + Σ₀ = Diagonal(inv(S⁻¹ .* S⁻¹)*diag(S⁻¹)) + return N⁻*0.5*tr((I - Σ₀*S⁻¹)^2) +end + +# For FIML, an explicit bl model has to be passed +function χ²_varonly(::SemFIML, model) + """ + Computing the CFI with FIML requires explicitely passing a fitted baseline model as + CFI(fit::SemFit, fit_baseline::SemFit) + """ |> ArgumentError |> throw +end + +function dof_varonly(model::AbstractSemSingle) + nparams_varonly = nobserved_vars(model) + if MeanStruct(model.implied) === HasMeanStruct + nparams_varonly *= 2 + end + return n_dp(model) - nparams_varonly +end + +dof_varonly(model::SemEnsemble) = sum(dof_varonly, model.sems) \ No newline at end of file diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index e12d1c7c0..4e61d6807 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -68,6 +68,12 @@ end rtol = 1e-2, atol = 1e-7, ) + test_fitmeasures( + Dict(:CFI => CFI(solution_ml)), + solution_lav[:fitmeasures_ml]; + fitmeasure_names = Dict(:CFI => "cfi") + ) + update_se_hessian!(partable, solution_ml) test_estimates( partable, @@ -85,6 +91,12 @@ end rtol = 1e-2, atol = 1e-7, ) + test_fitmeasures( + Dict(:CFI => CFI(solution_ml)), + solution_lav[:fitmeasures_ml]; + fitmeasure_names = Dict(:CFI => "cfi") + ) + update_se_hessian!(partable, solution_ml) test_estimates( partable, @@ -145,6 +157,11 @@ end rtol = 1e-2, atol = 1e-7, ) + test_fitmeasures( + Dict(:CFI => CFI(solution_ml)), + solution_lav[:fitmeasures_ml]; + fitmeasure_names = Dict(:CFI => "cfi") + ) update_se_hessian!(partable_s, solution_ml) test_estimates( @@ -259,6 +276,11 @@ end rtol = 1e-2, atol = 1e-5, ) + test_fitmeasures( + Dict(:CFI => CFI(solution_ls)), + solution_lav[:fitmeasures_ls]; + fitmeasure_names = Dict(:CFI => "cfi") + ) @suppress update_se_hessian!(partable, solution_ls) test_estimates( @@ -305,6 +327,16 @@ if !isnothing(specification_miss_g1) meanstructure = true, ) + model_ml_varonly = SemEnsemble( + specification = partable_varonly, + data = dat_missing, + column = :school, + groups = [:Pasteur, :Grant_White], + loss = SemFIML, + observed = SemObservedMissing, + meanstructure = true, + ) + ############################################################################################ ### test gradients ############################################################################################ @@ -361,6 +393,12 @@ if !isnothing(specification_miss_g1) rtol = 1e-3, atol = 0, ) + solution_varonly = fit(semoptimizer, model_ml_varonly) + test_fitmeasures( + Dict(:CFI => CFI(solution, solution_varonly)), + solution_lav[:fitmeasures_fiml]; + fitmeasure_names = Dict(:CFI => "cfi") + ) update_se_hessian!(partable_miss, solution) test_estimates( partable_miss, @@ -378,6 +416,12 @@ if !isnothing(specification_miss_g1) rtol = 1e-3, atol = 0, ) + test_fitmeasures( + Dict(:CFI => CFI(solution, solution_varonly)), + solution_lav[:fitmeasures_fiml]; + fitmeasure_names = Dict(:CFI => "cfi") + ) + update_se_hessian!(partable_miss, solution) test_estimates( partable_miss, diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 2d43c3d2a..1f89714d8 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -152,6 +152,19 @@ specification_miss = convert(Dict{Symbol, RAMMatrices}, partable_miss) specification_miss_g1 = specification_miss[:Pasteur] specification_miss_g2 = specification_miss[:Grant_White] +# CFI baseline model +graph_varonly = @StenoGraph begin + _(observed_vars) ↔ _(observed_vars) + Symbol(1) → _(observed_vars) +end + +partable_varonly = EnsembleParameterTable( + graph_varonly; + observed_vars = observed_vars, + latent_vars = latent_vars, + groups = [:Pasteur, :Grant_White], +) + start_test = [ fill(0.5, 6) fill(1.0, 9) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 3397b5f0a..3f0c4393f 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -110,7 +110,16 @@ end @testset "fitmeasures/se_ml" begin solution_ml = fit(optimizer_obj, model_ml) - test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) + test_fitmeasures( + fit_measures(solution_ml), + solution_lav[:fitmeasures_ml]; + atol = 1e-3 + ) + test_fitmeasures( + Dict(:CFI => CFI(solution_ml)), + solution_lav[:fitmeasures_ml]; + fitmeasure_names = Dict(:CFI => "cfi") + ) update_se_hessian!(partable, solution_ml) test_estimates( @@ -131,6 +140,11 @@ end atol = 1e-3, fitmeasure_names = fitmeasure_names_ls, ) + test_fitmeasures( + Dict(:CFI => CFI(solution_ls)), + solution_lav[:fitmeasures_ls]; + fitmeasure_names = Dict(:CFI => "cfi") + ) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) @suppress update_se_hessian!(partable, solution_ls) @@ -273,6 +287,11 @@ end solution_lav[:fitmeasures_ml_mean]; atol = 1e-3, ) + test_fitmeasures( + Dict(:CFI => CFI(solution_ml)), + solution_lav[:fitmeasures_ml_mean]; + fitmeasure_names = Dict(:CFI => "cfi") + ) update_se_hessian!(partable_mean, solution_ml) test_estimates( @@ -293,6 +312,11 @@ end atol = 1e-3, fitmeasure_names = fitmeasure_names_ls, ) + test_fitmeasures( + Dict(:CFI => CFI(solution_ls)), + solution_lav[:fitmeasures_ls_mean]; + fitmeasure_names = Dict(:CFI => "cfi") + ) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) @suppress update_se_hessian!(partable_mean, solution_ls) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 4045141ce..ba5ecb908 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -115,7 +115,16 @@ end @testset "fitmeasures/se_ml" begin solution_ml = fit(semoptimizer, model_ml) - test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) + test_fitmeasures( + fit_measures(solution_ml), + solution_lav[:fitmeasures_ml]; + atol = 1e-3 + ) + test_fitmeasures( + Dict(:CFI => CFI(solution_ml)), + solution_lav[:fitmeasures_ml]; + fitmeasure_names = Dict(:CFI => "cfi") + ) update_se_hessian!(partable, solution_ml) test_estimates( @@ -136,6 +145,12 @@ end atol = 1e-3, fitmeasure_names = fitmeasure_names_ls, ) + test_fitmeasures( + Dict(:CFI => CFI(solution_ls)), + solution_lav[:fitmeasures_ls]; + fitmeasure_names = Dict(:CFI => "cfi") + ) + @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) @suppress update_se_hessian!(partable, solution_ls) @@ -312,6 +327,11 @@ end solution_lav[:fitmeasures_ml_mean]; atol = 0.002, ) + test_fitmeasures( + Dict(:CFI => CFI(solution_ml)), + solution_lav[:fitmeasures_ml_mean]; + fitmeasure_names = Dict(:CFI => "cfi") + ) update_se_hessian!(partable_mean, solution_ml) test_estimates( @@ -333,6 +353,11 @@ end fitmeasure_names = fitmeasure_names_ls, ) @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) + test_fitmeasures( + Dict(:CFI => CFI(solution_ls)), + solution_lav[:fitmeasures_ls_mean]; + fitmeasure_names = Dict(:CFI => "cfi") + ) @suppress update_se_hessian!(partable_mean, solution_ls) test_estimates( @@ -407,6 +432,16 @@ model_ml_sym = Sem( meanstructure = true, ) +if !ismissing(spec_varonly) + model_varonly = Sem( + specification = spec_varonly, + data = dat_missing, + observed = SemObservedMissing, + loss = SemFIML, + meanstructure = true, + ) +end + ############################################################################################ ### test gradients ############################################################################################ @@ -446,6 +481,14 @@ end solution_lav[:fitmeasures_fiml]; atol = 1e-3, ) + if !ismissing(spec_varonly) + solution_varonly = fit(semoptimizer, model_varonly) + test_fitmeasures( + Dict(:CFI => CFI(solution_ml, solution_varonly)), + solution_lav[:fitmeasures_fiml]; + fitmeasure_names = Dict(:CFI => "cfi") + ) + end update_se_hessian!(partable_mean, solution_ml) test_estimates( diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index cbdf7ea74..c42e2529c 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -104,6 +104,8 @@ partable_mean = ParameterTable(spec_mean) @test SEM.param_labels(partable_mean) == SEM.param_labels(spec_mean) +spec_varonly = missing + start_test = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3)] start_test_mean = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3); fill(0.1, 7)] @@ -144,6 +146,7 @@ spec_mean = ParameterTable(spec_mean) partable = spec partable_mean = spec_mean +spec_varonly = missing opt_engine = :Optim @testset "RAMMatrices → ParameterTable | constructor | Optim" begin @@ -228,6 +231,14 @@ sort_vars!(spec_mean) partable_mean = spec_mean +# varonly model for CFI +graph_varonly = @StenoGraph begin + _(observed_vars) ↔ _(observed_vars) + Symbol(1) → _(observed_vars) +end + +spec_varonly = ParameterTable(graph_varonly, latent_vars = latent_vars, observed_vars = observed_vars) + start_test = [fill(0.5, 8); fill(0.05, 3); fill(1.0, 11); fill(0.05, 9)] start_test_mean = [fill(0.5, 8); fill(0.05, 3); fill(1.0, 11); fill(0.05, 3); fill(0.05, 13)] From 0c447da0d4151ee20d23697a7c7d425735b0d6c3 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 8 Mar 2026 18:23:41 +0100 Subject: [PATCH 277/364] add docstring for CFI --- src/frontend/fit/fitmeasures/CFI.jl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/frontend/fit/fitmeasures/CFI.jl b/src/frontend/fit/fitmeasures/CFI.jl index 8759e5d48..f7691872a 100644 --- a/src/frontend/fit/fitmeasures/CFI.jl +++ b/src/frontend/fit/fitmeasures/CFI.jl @@ -1,7 +1,16 @@ """ - CFI(fit::SemFit) + (1) CFI(fit::SemFit, fit_baseline::SemFit) -Return the CFI. + (2) CFI(fit::SemFit) + +Calculate the Comparative Fit Index (CFI). + +The CFI ranges from 0-1 and measures how much better the model +fits the data compared to a baseline model. +If no baseline model is provided, a model with unconstrained +variances (and means) is compaired against. +For multigroup models, variances (and means) per group are free +without any equality constraints between groups. """ function CFI end From 7334a6bb43d033bfabbe7d2d51c662ee99512413 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Sun, 8 Mar 2026 18:29:21 +0100 Subject: [PATCH 278/364] Apply suggestions from formatter Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/additional_functions/helper.jl | 2 +- src/frontend/fit/fitmeasures/CFI.jl | 8 +++++--- test/examples/multigroup/build_models.jl | 12 ++++++------ test/examples/political_democracy/by_parts.jl | 14 +++++--------- test/examples/political_democracy/constructor.jl | 16 ++++++---------- .../political_democracy/political_democracy.jl | 3 ++- 6 files changed, 25 insertions(+), 30 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 268880b39..b3f5212b7 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -142,7 +142,7 @@ function check_single_lossfun(models::AbstractSemSingle...; throw_error) return uniform end -check_single_lossfun(model::SemEnsemble; throw_error) = +check_single_lossfun(model::SemEnsemble; throw_error) = check_single_lossfun(model.sems...; throw_error) # scaling corrections for multigroup models diff --git a/src/frontend/fit/fitmeasures/CFI.jl b/src/frontend/fit/fitmeasures/CFI.jl index f7691872a..53276b93d 100644 --- a/src/frontend/fit/fitmeasures/CFI.jl +++ b/src/frontend/fit/fitmeasures/CFI.jl @@ -16,7 +16,7 @@ function CFI end # if the user provides a baseline model -CFI(fit::SemFit, fit_baseline::SemFit) = +CFI(fit::SemFit, fit_baseline::SemFit) = CFI(χ²(fit), dof(fit), χ²(fit_baseline), dof(fit_baseline)) # no baseline -> variance only model @@ -67,7 +67,9 @@ function χ²_varonly(::SemFIML, model) """ Computing the CFI with FIML requires explicitely passing a fitted baseline model as CFI(fit::SemFit, fit_baseline::SemFit) - """ |> ArgumentError |> throw + """ |> + ArgumentError |> + throw end function dof_varonly(model::AbstractSemSingle) @@ -78,4 +80,4 @@ function dof_varonly(model::AbstractSemSingle) return n_dp(model) - nparams_varonly end -dof_varonly(model::SemEnsemble) = sum(dof_varonly, model.sems) \ No newline at end of file +dof_varonly(model::SemEnsemble) = sum(dof_varonly, model.sems) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 4e61d6807..e10a8a058 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -71,7 +71,7 @@ end test_fitmeasures( Dict(:CFI => CFI(solution_ml)), solution_lav[:fitmeasures_ml]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) update_se_hessian!(partable, solution_ml) @@ -94,7 +94,7 @@ end test_fitmeasures( Dict(:CFI => CFI(solution_ml)), solution_lav[:fitmeasures_ml]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) update_se_hessian!(partable, solution_ml) @@ -160,7 +160,7 @@ end test_fitmeasures( Dict(:CFI => CFI(solution_ml)), solution_lav[:fitmeasures_ml]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) update_se_hessian!(partable_s, solution_ml) @@ -279,7 +279,7 @@ end test_fitmeasures( Dict(:CFI => CFI(solution_ls)), solution_lav[:fitmeasures_ls]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) @suppress update_se_hessian!(partable, solution_ls) @@ -397,7 +397,7 @@ if !isnothing(specification_miss_g1) test_fitmeasures( Dict(:CFI => CFI(solution, solution_varonly)), solution_lav[:fitmeasures_fiml]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) update_se_hessian!(partable_miss, solution) test_estimates( @@ -419,7 +419,7 @@ if !isnothing(specification_miss_g1) test_fitmeasures( Dict(:CFI => CFI(solution, solution_varonly)), solution_lav[:fitmeasures_fiml]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) update_se_hessian!(partable_miss, solution) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 5325295e8..d2d468a98 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -110,15 +110,11 @@ end @testset "fitmeasures/se_ml" begin solution_ml = fit(optimizer_obj, model_ml) - test_fitmeasures( - fit_measures(solution_ml), - solution_lav[:fitmeasures_ml]; - atol = 1e-3 - ) + test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) test_fitmeasures( Dict(:CFI => CFI(solution_ml)), solution_lav[:fitmeasures_ml]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) update_se_hessian!(partable, solution_ml) @@ -143,7 +139,7 @@ end test_fitmeasures( Dict(:CFI => CFI(solution_ls)), solution_lav[:fitmeasures_ls]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) @@ -290,7 +286,7 @@ end test_fitmeasures( Dict(:CFI => CFI(solution_ml)), solution_lav[:fitmeasures_ml_mean]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) update_se_hessian!(partable_mean, solution_ml) @@ -315,7 +311,7 @@ end test_fitmeasures( Dict(:CFI => CFI(solution_ls)), solution_lav[:fitmeasures_ls_mean]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index ba5ecb908..45de75d13 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -115,15 +115,11 @@ end @testset "fitmeasures/se_ml" begin solution_ml = fit(semoptimizer, model_ml) - test_fitmeasures( - fit_measures(solution_ml), - solution_lav[:fitmeasures_ml]; - atol = 1e-3 - ) + test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) test_fitmeasures( Dict(:CFI => CFI(solution_ml)), solution_lav[:fitmeasures_ml]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) update_se_hessian!(partable, solution_ml) @@ -148,7 +144,7 @@ end test_fitmeasures( Dict(:CFI => CFI(solution_ls)), solution_lav[:fitmeasures_ls]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) @@ -330,7 +326,7 @@ end test_fitmeasures( Dict(:CFI => CFI(solution_ml)), solution_lav[:fitmeasures_ml_mean]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) update_se_hessian!(partable_mean, solution_ml) @@ -356,7 +352,7 @@ end test_fitmeasures( Dict(:CFI => CFI(solution_ls)), solution_lav[:fitmeasures_ls_mean]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) @suppress update_se_hessian!(partable_mean, solution_ls) @@ -486,7 +482,7 @@ end test_fitmeasures( Dict(:CFI => CFI(solution_ml, solution_varonly)), solution_lav[:fitmeasures_fiml]; - fitmeasure_names = Dict(:CFI => "cfi") + fitmeasure_names = Dict(:CFI => "cfi"), ) end diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index c42e2529c..9c8cc2a7b 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -237,7 +237,8 @@ graph_varonly = @StenoGraph begin Symbol(1) → _(observed_vars) end -spec_varonly = ParameterTable(graph_varonly, latent_vars = latent_vars, observed_vars = observed_vars) +spec_varonly = + ParameterTable(graph_varonly, latent_vars = latent_vars, observed_vars = observed_vars) start_test = [fill(0.5, 8); fill(0.05, 3); fill(1.0, 11); fill(0.05, 9)] start_test_mean = From cec86583e7f51d0ce48c333a43aeb9158bc02079 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Sun, 8 Mar 2026 18:29:46 +0100 Subject: [PATCH 279/364] Update src/frontend/fit/fitmeasures/CFI.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/frontend/fit/fitmeasures/CFI.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/frontend/fit/fitmeasures/CFI.jl b/src/frontend/fit/fitmeasures/CFI.jl index 53276b93d..9f3c5a2d9 100644 --- a/src/frontend/fit/fitmeasures/CFI.jl +++ b/src/frontend/fit/fitmeasures/CFI.jl @@ -14,7 +14,6 @@ without any equality constraints between groups. """ function CFI end - # if the user provides a baseline model CFI(fit::SemFit, fit_baseline::SemFit) = CFI(χ²(fit), dof(fit), χ²(fit_baseline), dof(fit_baseline)) From 17a8d0f0ccbbe51b61897510f41e5349c47dde97 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 9 Mar 2026 01:00:10 +0100 Subject: [PATCH 280/364] refactor bootstrap se and allow parallel fitting --- src/frontend/fit/standard_errors/bootstrap.jl | 193 +++++++++--------- 1 file changed, 92 insertions(+), 101 deletions(-) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index bebad2935..c781fb218 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -1,127 +1,118 @@ """ - se_bootstrap(sem_fit::SemFit; n_boot = 3000, data = nothing, kwargs...) + se_bootstrap( + sem_fit::SemFit; + n_boot = 3000, + data = nothing, + specification = nothing, + parallel = false, + kwargs...) -Return boorstrap standard errors. +Return bootstrap standard errors. # Arguments - `n_boot`: number of boostrap samples - `data`: data to sample from. Only needed if different than the data from `sem_fit` +- `specification`: a `ParameterTable` or `RAMMatrices` object passed down to `replace_observed`. + Necessary for FIML models. +- `parallel`: if `true`, run bootstrap samples in parallel on all available threads. + The number of threads is controlled by the `JULIA_NUM_THREADS` environment variable or + the `--threads` flag when starting Julia. - `kwargs...`: passed down to `replace_observed` """ function se_bootstrap( - semfit::SemFit{Mi, So, St, Mo, O}; + fitted::SemFit{Mi, So, St, Mo, O}; n_boot = 3000, data = nothing, specification = nothing, + parallel = false, kwargs..., -) where {Mi, So, St, Mo <: AbstractSemSingle, O} - if isnothing(data) - data = samples(observed(model(semfit))) - end - - data = prepare_data_bootstrap(data) - - start = solution(semfit) - - new_solution = zero(start) - sum = zero(start) - squared_sum = zero(start) - - n_failed = 0.0 - - converged = true - - for _ in 1:n_boot - sample_data = bootstrap_sample(data) - new_model = replace_observed( - model(semfit); - data = sample_data, - specification = specification, - kwargs..., - ) - - new_solution .= 0.0 - - try - new_solution = solution(fit(new_model; start_val = start)) - catch - n_failed += 1 +) where {Mi, So, St, Mo, O} + # access data and convert to matrix + data = prepare_data_bootstrap(data, fitted.model) + start = solution(fitted) + # pre-allocations + total_sum = zero(start) + total_squared_sum = zero(start) + n_failed = Ref(0) + # fit to bootstrap samples + if !parallel + for _ in 1:n_boot + sample_data = bootstrap_sample(data) + new_model = replace_observed( + fitted.model; + data = sample_data, specification = specification, kwargs...) + try + sol = solution(fit(new_model; start_val = start)) + @. total_sum += sol + @. total_squared_sum += sol^2 + catch + n_failed[] += 1 + end + end + else + n_threads = Threads.nthreads() + # Pre-create one independent model copy per thread via deepcopy. + model_pool = Channel(n_threads) + for _ in 1:n_threads + put!(model_pool, deepcopy(fitted.model)) + end + # fit models in parallel + lk = ReentrantLock() + Threads.@threads for _ in 1:n_boot + thread_model = take!(model_pool) + try + sample_data = bootstrap_sample(data) + new_model = replace_observed( + thread_model; + data = sample_data, specification = specification, kwargs...) + sol = solution(fit(new_model; start_val = start)) + lock(lk) do + @. total_sum += sol + @. total_squared_sum += sol^2 + end + catch + lock(lk) do + n_failed[] += 1 + end + finally + put!(model_pool, thread_model) + end end - - @. sum += new_solution - @. squared_sum += new_solution^2 - - converged = true end - - n_conv = n_boot - n_failed - sd = sqrt.(squared_sum / n_conv - (sum / n_conv) .^ 2) - print("Number of nonconverged models: ", n_failed, "\n") + # compute parameters + n_conv = n_boot - n_failed[] + sd = sqrt.(total_squared_sum / n_conv - (total_sum / n_conv) .^ 2) + if !iszero(n_failed[]) + @warn "During bootstrap sampling, "*string(n_failed[])*" models did not converge" + end return sd end -function se_bootstrap( - semfit::SemFit{Mi, So, St, Mo, O}; - n_boot = 3000, - data = nothing, - specification = nothing, - kwargs..., -) where {Mi, So, St, Mo <: SemEnsemble, O} - models = semfit.model.sems - groups = semfit.model.groups +function bootstrap_sample(data::Matrix) + nobs = size(data, 1) + index_new = rand(1:nobs, nobs) + data_new = data[index_new, :] + return data_new +end + +bootstrap_sample(data::Dict) = Dict(k => bootstrap_sample(data[k]) for k in keys(data)) +function prepare_data_bootstrap(data, model::AbstractSemSingle) if isnothing(data) - data = Dict(g => samples(observed(m)) for (g, m) in zip(groups, models)) + data = samples(observed(model)) end + data = Matrix(data) + return data +end - data = Dict(k => prepare_data_bootstrap(data[k]) for k in keys(data)) - - start = solution(semfit) - - new_solution = zero(start) - sum = zero(start) - squared_sum = zero(start) - - n_failed = 0.0 - - converged = true - - for _ in 1:n_boot - sample_data = Dict(k => bootstrap_sample(data[k]) for k in keys(data)) - new_model = replace_observed( - semfit.model; - data = sample_data, - specification = specification, - kwargs..., - ) - - new_solution .= 0.0 - - try - new_solution = solution(fit(new_model; start_val = start)) - catch - n_failed += 1 - end - - @. sum += new_solution - @. squared_sum += new_solution^2 - - converged = true +function prepare_data_bootstrap(data, model::SemEnsemble) + sems = model.sems + groups = model.groups + if isnothing(data) + data = Dict(g => samples(observed(m)) for (g, m) in zip(groups, sems)) end - - n_conv = n_boot - n_failed - sd = sqrt.(squared_sum / n_conv - (sum / n_conv) .^ 2) - print("Number of nonconverged models: ", n_failed, "\n") - return sd + data = Dict(k => Matrix(data[k]) for k in keys(data)) + return data end -function prepare_data_bootstrap(data) - return Matrix(data) -end -function bootstrap_sample(data) - nobs = size(data, 1) - index_new = rand(1:nobs, nobs) - data_new = data[index_new, :] - return data_new -end From ead681f2f6a1152317205ddf9b5185e0e9903b88 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Mon, 9 Mar 2026 01:05:31 +0100 Subject: [PATCH 281/364] Apply suggestions from formatter Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/frontend/fit/standard_errors/bootstrap.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index c781fb218..eb7aefa7b 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -40,7 +40,10 @@ function se_bootstrap( sample_data = bootstrap_sample(data) new_model = replace_observed( fitted.model; - data = sample_data, specification = specification, kwargs...) + data = sample_data, + specification = specification, + kwargs..., + ) try sol = solution(fit(new_model; start_val = start)) @. total_sum += sol @@ -64,7 +67,10 @@ function se_bootstrap( sample_data = bootstrap_sample(data) new_model = replace_observed( thread_model; - data = sample_data, specification = specification, kwargs...) + data = sample_data, + specification = specification, + kwargs..., + ) sol = solution(fit(new_model; start_val = start)) lock(lk) do @. total_sum += sol From b7398ede3926fc3455e371ed2bee2f6b81bac495 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 9 Mar 2026 11:18:18 +0100 Subject: [PATCH 282/364] update to PrettyTable v3 --- Project.toml | 2 +- src/frontend/fit/summary.jl | 30 ++++++++++---------- src/frontend/specification/ParameterTable.jl | 6 ++-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Project.toml b/Project.toml index 073f2bd00..b445f3db4 100644 --- a/Project.toml +++ b/Project.toml @@ -37,7 +37,7 @@ LineSearches = "7" NLSolversBase = "7" NLopt = "0.6, 1" Optim = "1" -PrettyTables = "2" +PrettyTables = "3" ProximalAlgorithms = "0.7" StatsBase = "0.33, 0.34" Symbolics = "4, 5, 6" diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 435b17470..c8495b799 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -114,10 +114,10 @@ function details( print("\n") pretty_table( loading_array; - header = header_cols, - tf = PrettyTables.tf_borderless, + column_labels = header_cols, + table_format = TextTableFormat(borders = text_table_borders__borderless), alignment = :l, - formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, + formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v], ) print("\n") end @@ -152,10 +152,10 @@ function details( print("\n") pretty_table( regression_array; - header = regression_columns, - tf = PrettyTables.tf_borderless, + column_labels = regression_columns, + table_format = TextTableFormat(borders = text_table_borders__borderless), alignment = :l, - formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, + formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v], ) print("\n") @@ -179,10 +179,10 @@ function details( print("\n") pretty_table( var_array; - header = var_columns, - tf = PrettyTables.tf_borderless, + column_labels = var_columns, + table_format = TextTableFormat(borders = text_table_borders__borderless), alignment = :l, - formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, + formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v], ) print("\n") @@ -206,10 +206,10 @@ function details( print("\n") pretty_table( covar_array; - header = covar_columns, - tf = PrettyTables.tf_borderless, + column_labels = covar_columns, + table_format = TextTableFormat(borders = text_table_borders__borderless), alignment = :l, - formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, + formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v], ) print("\n") @@ -236,10 +236,10 @@ function details( print("\n") pretty_table( mean_array; - header = mean_columns, - tf = PrettyTables.tf_borderless, + column_labels = mean_columns, + table_format = TextTableFormat(borders = text_table_borders__borderless), alignment = :l, - formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, + formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v], ) print("\n") end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 600c9058c..ecff7c4ec 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -114,10 +114,10 @@ function Base.show(io::IO, partable::ParameterTable) pretty_table( io, as_matrix, - header = (shown_columns, [eltype(partable.columns[col]) for col in shown_columns]), - tf = PrettyTables.tf_compact, + column_labels = [shown_columns, [eltype(partable.columns[col]) for col in shown_columns]], + table_format = TextTableFormat(borders = text_table_borders__compact), # TODO switch to `missing` as non-specified values and suppress printing of `missing` instead - formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v, + formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v], ) print(io, "Latent Variables: $(partable.latent_vars) \n") From ad8a8c1ec23a661f0f461918c8c863cc526af608 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 9 Mar 2026 23:05:34 +0100 Subject: [PATCH 283/364] add converged method --- ext/SEMNLOptExt/NLopt.jl | 4 ++++ ext/SEMProximalOptExt/ProximalAlgorithms.jl | 4 +++- src/frontend/fit/SemFit.jl | 4 +++- src/optimizer/optim.jl | 12 +++++++++++- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index 909dbbfc1..90004b907 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -134,6 +134,10 @@ end SEM.algorithm_name(res::NLoptResult) = res.problem.algorithm SEM.n_iterations(res::NLoptResult) = res.problem.numevals SEM.convergence(res::NLoptResult) = res.result[3] +function SEM.converged(res::NLoptResult) + flag = res.result[3] + return flag ∈ [:SUCCESS, :STOPVAL_REACHED, :FTOL_REACHED, :XTOL_REACHED] +end # construct NLopt.jl problem function NLopt_problem(algorithm, options, npar) diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index 1d7f83632..70c4f3963 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -90,10 +90,12 @@ SEM.algorithm_name(res::ProximalResult) = SEM.algorithm_name(res.optimizer.algor SEM.algorithm_name( ::ProximalAlgorithms.IterativeAlgorithm{I, H, S, D, K}, ) where {I, H, S, D, K} = nameof(I) - SEM.convergence( ::ProximalResult, ) = "No standard convergence criteria for proximal \n algorithms available." +SEM.converged( + ::ProximalResult, +) = missing SEM.n_iterations(res::ProximalResult) = res.n_iterations ############################################################################################ diff --git a/src/frontend/fit/SemFit.jl b/src/frontend/fit/SemFit.jl index 9c2d114e7..1d2e82a60 100644 --- a/src/frontend/fit/SemFit.jl +++ b/src/frontend/fit/SemFit.jl @@ -15,7 +15,8 @@ Fitted structural equation model. - `algorithm_name(::SemFit)` -> optimization algorithm - `n_iterations(::SemFit)` -> number of iterations -- `convergence(::SemFit)` -> convergence properties +- `convergence(::SemFit)` -> convergence flags +- `converged(::SemFit)` -> convergence success """ mutable struct SemFit{Mi, So, St, Mo, O} minimum::Mi @@ -71,3 +72,4 @@ optimizer_engine(sem_fit::SemFit) = optimizer_engine(optimization_result(sem_fit algorithm_name(sem_fit::SemFit) = algorithm_name(optimization_result(sem_fit)) n_iterations(sem_fit::SemFit) = n_iterations(optimization_result(sem_fit)) convergence(sem_fit::SemFit) = convergence(optimization_result(sem_fit)) +converged(sem_fit::SemFit) = converged(optimization_result(sem_fit)) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 83ebbe5e1..704131938 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -77,7 +77,17 @@ end algorithm_name(res::SemOptimResult) = Optim.summary(res.result) n_iterations(res::SemOptimResult) = Optim.iterations(res.result) -convergence(res::SemOptimResult) = Optim.converged(res.result) +function convergence(res::SemOptimResult) + flags = res.result.stopped_by + active_flags = Symbol[] + for key in keys(flags) + if flags[key] + push!(active_flags, key) + end + end + return active_flags +end +converged(res::SemOptimResult) = Optim.converged(res.result) function fit( optim::SemOptimizerOptim, From 8a9a81b7bcd0cf6bf070fdbe6218cc1058808960 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 9 Mar 2026 23:07:35 +0100 Subject: [PATCH 284/364] refactor bootstrap and add bootstrap for any statistic --- src/frontend/fit/standard_errors/bootstrap.jl | 210 ++++++++++++++++-- 1 file changed, 186 insertions(+), 24 deletions(-) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index eb7aefa7b..12828a09a 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -1,32 +1,178 @@ +""" + bootstrap( + fitted::SemFit; + statistic = solution, + n_boot = 3000, + data = nothing, + specification = nothing, + engine = :Optim, + parallel = false, + fit_kwargs = Dict(), + replace_kwargs = Dict()) + +Return bootstrap samples for `statistic`. + +# Arguments +- `fitted`: a fitted SEM. +- `statistic`: any function that can be called on a `SemFit` object. + The output will be returned as the bootstrap sample. +- `n_boot`: number of boostrap samples +- `data`: data to sample from. Only needed if different than the data from `sem_fit` +- `specification`: a `ParameterTable` or `RAMMatrices` object passed to `replace_observed`. + Necessary for FIML models. +- `engine`: optimizer engine, passed to `fit`. +- `parallel`: if `true`, run bootstrap samples in parallel on all available threads. + The number of threads is controlled by the `JULIA_NUM_THREADS` environment variable or + the `--threads` flag when starting Julia. +- `fit_kwargs` : a `Dict` controlling model fitting for each bootstrap sample, + passed to `fit` +- `replace_kwargs`: a `Dict` passed to `replace_observed` + +# Example +```julia +# 1000 boostrap samples of the minimum, fitted with :Optim +bootstrap( + fitted; + statistic = StructuralEquationModels.minimum, + n_boot = 1000, + engine = :Optim, +) +``` +""" +function bootstrap( + fitted::SemFit; + statistic = solution, + n_boot = 3000, + data = nothing, + specification = nothing, + engine = :Optim, + parallel = false, + fit_kwargs = Dict(), + replace_kwargs = Dict() +) + # access data and convert to matrix + data = prepare_data_bootstrap(data, fitted.model) + start = solution(fitted) + # pre-allocations + out = [] + conv = [] + n_failed = Ref(0) + # fit to bootstrap samples + if !parallel + for _ in 1:n_boot + try + sample_data = bootstrap_sample(data) + new_model = replace_observed( + fitted.model; + data = sample_data, + specification = specification, + replace_kwargs..., + ) + new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) + sample = statistic(new_fit) + c = converged(new_fit) + push!(out, sample) + push!(conv, c) + catch + n_failed[] += 1 + end + end + else + n_threads = Threads.nthreads() + # Pre-create one independent model copy per thread via deepcopy. + model_pool = Channel(n_threads) + for _ in 1:n_threads + put!(model_pool, deepcopy(fitted.model)) + end + # fit models in parallel + lk = ReentrantLock() + Threads.@threads for _ in 1:n_boot + thread_model = take!(model_pool) + try + sample_data = bootstrap_sample(data) + new_model = replace_observed( + thread_model; + data = sample_data, + specification = specification, + replace_kwargs..., + ) + new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) + sample = statistic(new_fit) + c = converged(new_fit) + lock(lk) do + push!(out, sample) + push!(conv, c) + end + catch + lock(lk) do + n_failed[] += 1 + end + finally + put!(model_pool, thread_model) + end + end + end + # compute parameters + if !iszero(n_failed[]) + @warn "During bootstrap sampling, "*string(n_failed[])*" samples errored." + end + return Dict( + :samples => out, + :n_boot => n_boot, + :n_converged => sum(conv), + :converged => conv, + :n_errored => n_failed[]) +end + """ se_bootstrap( - sem_fit::SemFit; + fitted::SemFit; n_boot = 3000, data = nothing, specification = nothing, + engine = :Optim, parallel = false, - kwargs...) + fit_kwargs = Dict(), + replace_kwargs = Dict()) Return bootstrap standard errors. # Arguments +- `fitted`: a fitted SEM. - `n_boot`: number of boostrap samples - `data`: data to sample from. Only needed if different than the data from `sem_fit` -- `specification`: a `ParameterTable` or `RAMMatrices` object passed down to `replace_observed`. +- `specification`: a `ParameterTable` or `RAMMatrices` object passed to `replace_observed`. Necessary for FIML models. +- `engine`: optimizer engine, passed to `fit`. - `parallel`: if `true`, run bootstrap samples in parallel on all available threads. The number of threads is controlled by the `JULIA_NUM_THREADS` environment variable or the `--threads` flag when starting Julia. -- `kwargs...`: passed down to `replace_observed` +- `fit_kwargs` : a `Dict` controlling model fitting for each bootstrap sample, + passed to `sem_fit` +- `replace_kwargs`: a `Dict` passed to `replace_observed` + +# Example +```julia +# 1000 boostrap samples, fitted with :NLopt +using NLopt + +se_bootstrap( + fitted; + n_boot = 1000, + engine = :NLopt, +) +``` """ function se_bootstrap( - fitted::SemFit{Mi, So, St, Mo, O}; + fitted::SemFit; n_boot = 3000, data = nothing, specification = nothing, + engine = :Optim, parallel = false, - kwargs..., -) where {Mi, So, St, Mo, O} + fit_kwargs = Dict(), + replace_kwargs = Dict() +) # access data and convert to matrix data = prepare_data_bootstrap(data, fitted.model) start = solution(fitted) @@ -34,20 +180,26 @@ function se_bootstrap( total_sum = zero(start) total_squared_sum = zero(start) n_failed = Ref(0) + n_conv = Ref(0) # fit to bootstrap samples if !parallel for _ in 1:n_boot - sample_data = bootstrap_sample(data) - new_model = replace_observed( - fitted.model; - data = sample_data, - specification = specification, - kwargs..., - ) try - sol = solution(fit(new_model; start_val = start)) - @. total_sum += sol - @. total_squared_sum += sol^2 + sample_data = bootstrap_sample(data) + new_model = replace_observed( + fitted.model; + data = sample_data, + specification = specification, + replace_kwargs..., + ) + new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) + sol = solution(new_fit) + conv = converged(new_fit) + if conv + n_conv[] += 1 + @. total_sum += sol + @. total_squared_sum += sol^2 + end catch n_failed[] += 1 end @@ -69,12 +221,17 @@ function se_bootstrap( thread_model; data = sample_data, specification = specification, - kwargs..., + replace_kwargs..., ) - sol = solution(fit(new_model; start_val = start)) - lock(lk) do - @. total_sum += sol - @. total_squared_sum += sol^2 + new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) + sol = solution(new_fit) + conv = converged(new_fit) + if conv + lock(lk) do + n_conv[] += 1 + @. total_sum += sol + @. total_squared_sum += sol^2 + end end catch lock(lk) do @@ -86,14 +243,19 @@ function se_bootstrap( end end # compute parameters - n_conv = n_boot - n_failed[] + n_conv = n_conv[] sd = sqrt.(total_squared_sum / n_conv - (total_sum / n_conv) .^ 2) if !iszero(n_failed[]) - @warn "During bootstrap sampling, "*string(n_failed[])*" models did not converge" + @warn "During bootstrap sampling, "*string(n_failed[])*" samples errored" end + @info string(n_conv)*" models converged" return sd end +############################################################################################ +### Helper Functions +############################################################################################ + function bootstrap_sample(data::Matrix) nobs = size(data, 1) index_new = rand(1:nobs, nobs) From 12821562f365321010217c9400ccd50f182b2ee7 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 9 Mar 2026 23:08:46 +0100 Subject: [PATCH 285/364] add CI and p values --- .../standard_errors/confidence_intervals.jl | 44 +++++++++++++++++++ src/frontend/fit/standard_errors/z_test.jl | 36 +++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 src/frontend/fit/standard_errors/confidence_intervals.jl create mode 100644 src/frontend/fit/standard_errors/z_test.jl diff --git a/src/frontend/fit/standard_errors/confidence_intervals.jl b/src/frontend/fit/standard_errors/confidence_intervals.jl new file mode 100644 index 000000000..432e60819 --- /dev/null +++ b/src/frontend/fit/standard_errors/confidence_intervals.jl @@ -0,0 +1,44 @@ +_doc_normal_CI = """ + (1) normal_CI(fitted, se; α = 0.05, name_lower = :ci_lower, name_upper = :ci_upper) + + (2) normal_CI!(partable, fitted, se; α = 0.05, name_lower = :ci_lower, name_upper = :ci_upper) + +Return normal-theory confidence intervals for all model parameters. +`normal_CI!` additionally writes the result into `partable`. + +# Arguments +- `fitted`: a fitted SEM. +- `se`: standard errors for each parameter, e.g. from [`se_hessian`](@ref) or + [`se_bootstrap`](@ref). +- `partable`: a [`ParameterTable`](@ref) to write confidence intervals to. +- `α`: significance level. Defaults to `0.05` (95% intervals). +- `name_lower`: column name for the lower bound in `partable`. Defaults to `:ci_lower`. +- `name_upper`: column name for the upper bound in `partable`. Defaults to `:ci_upper`. + +# Returns +- a `Dict` with keys `name_lower` and `name_upper`, each mapping to a vector of bounds + over all parameters. +""" + +@doc "$(_doc_normal_CI)" +function normal_CI( + fitted, se; α = 0.05, name_lower = :ci_lower, name_upper = :ci_upper) + qnt = quantile(Normal(0, 1), 1-α/2); + sol = solution(fitted) + return Dict(name_lower => sol - qnt*se, name_upper => sol + qnt*se) +end + +@doc "$(_doc_normal_CI)" +function normal_CI!( + partable, + fitted, + se; + α = 0.05, + name_lower = :ci_lower, + name_upper = :ci_upper) + cis = normal_CI( + fitted, se; α, name_lower, name_upper) + update_partable!(partable, name_lower, fitted, cis[name_lower]) + update_partable!(partable, name_upper, fitted, cis[name_upper]) + return cis +end diff --git a/src/frontend/fit/standard_errors/z_test.jl b/src/frontend/fit/standard_errors/z_test.jl new file mode 100644 index 000000000..9705de1be --- /dev/null +++ b/src/frontend/fit/standard_errors/z_test.jl @@ -0,0 +1,36 @@ +_doc_z_test = """ + (1) z_test(fitted, se) + + (2) z_test!(partable, fitted, se, name = :p_value) + +Return two-sided p-values from a z-test for each model parameter. + +Tests the null hypothesis that each parameter is zero using the test statistic +`z = estimate / se`, which is compared against a standard normal distribution. +`z_test!` additionally writes the result into `partable`. + +# Arguments +- `fitted`: a fitted SEM. +- `se`: standard errors for each parameter, e.g. from [`se_hessian`](@ref) or + [`se_bootstrap`](@ref). +- `partable`: a [`ParameterTable`](@ref) to write p-values to. +- `name`: column name for the p-values in `partable`. Defaults to `:p_value`. + +# Returns +- a vector of p-values. +""" + +@doc "$(_doc_z_test)" +function z_test(fitted, se) + dev = solution(fitted)./se + dist = Normal(0, 1) + p = 2*ccdf.(dist, abs.(dev)) + return p +end + +@doc "$(_doc_z_test)" +function z_test!(partable, fitted, se, name = :p_value) + p = z_test(fitted, se) + update_partable!(partable, name, fitted, p) + return p +end From 2a3ceb610ef28f3047b5616744115995b0d51029 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 9 Mar 2026 23:09:00 +0100 Subject: [PATCH 286/364] add exports --- src/StructuralEquationModels.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index f144c98bc..19dd6f43a 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -90,6 +90,8 @@ include("frontend/fit/fitmeasures/fit_measures.jl") # standard errors include("frontend/fit/standard_errors/hessian.jl") include("frontend/fit/standard_errors/bootstrap.jl") +include("frontend/fit/standard_errors/z_test.jl") +include("frontend/fit/standard_errors/confidence_intervals.jl") export AbstractSem, AbstractSemSingle, @@ -129,6 +131,7 @@ export AbstractSem, optimizer_engine_doc, optimizer_engines, n_iterations, + converged, convergence, SemObserved, SemObservedData, @@ -191,7 +194,12 @@ export AbstractSem, CFI, EmMVNModel, se_hessian, + bootstrap, se_bootstrap, + normal_CI, + normal_CI!, + z_test, + z_test!, example_data, replace_observed, update_observed, From 163eee7db26eb574494e42aeae20ef68537b3696 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 9 Mar 2026 23:09:35 +0100 Subject: [PATCH 287/364] add boostrap, CI and p-values tests --- test/examples/helper.jl | 19 +++++++++++++++++++ test/examples/multigroup/build_models.jl | 6 ++++++ .../political_democracy/constructor.jl | 15 +++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index acc3ccd08..3e66ef888 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -135,3 +135,22 @@ function test_estimates( @test actual ≈ expected rtol = rtol atol = atol norm = Base.Fix2(norm, Inf) end end + +function test_bootstrap(model_fit; n_boot = 500) + # hessian and bootstrap se are close + se_he = se_hessian(model_fit) + se_bs = se_bootstrap(model_fit; n_boot = n_boot) + @test isapprox(se_bs, se_he, rtol = 0.2) + # se_bootstrap and bootstrap |> se are close + bs_samples = bootstrap(model_fit; n_boot = n_boot) + @test bs_samples[:n_converged] > 990 + bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) + se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) + @test isapprox(se_bs_2, se_bs, rtol = 0.05) +end + +function smoketest_CI_z(model_fit, partable) + se_he = se_hessian(model_fit) + normal_CI!(partable, model_fit, se_he) + z_test!(partable, model_fit, se_he) +end \ No newline at end of file diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index e10a8a058..fa4df6dd9 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -83,6 +83,8 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) + test_bootstrap(solution_ml) + smoketest_CI_z(solution_ml, partable) solution_ml = fit(model_ml_multigroup2) test_fitmeasures( @@ -291,6 +293,8 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) + test_bootstrap(solution_ls) + smoketest_CI_z(solution_ls, partable) end ############################################################################################ @@ -408,6 +412,8 @@ if !isnothing(specification_miss_g1) lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) + test_bootstrap(solution; n_boot = 500) + smoketest_CI_z(solution, partable_miss) solution = fit(semoptimizer, model_ml_multigroup2) test_fitmeasures( diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 45de75d13..5ced819f6 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -130,6 +130,9 @@ end col = :se, lav_col = :se, ) + + test_bootstrap(solution_ml) + smoketest_CI_z(solution_ml, partable) end @testset "fitmeasures/se_ls" begin @@ -157,6 +160,9 @@ end col = :se, lav_col = :se, ) + + test_bootstrap(solution_ls) + smoketest_CI_z(solution_ls, partable) end ############################################################################################ @@ -337,6 +343,9 @@ end col = :se, lav_col = :se, ) + + test_bootstrap(solution_ml) + smoketest_CI_z(solution_ml, partable_mean) end @testset "fitmeasures/se_ls_mean" begin @@ -363,6 +372,9 @@ end col = :se, lav_col = :se, ) + + test_bootstrap(solution_ls) + smoketest_CI_z(solution_ls, partable_mean) end ############################################################################################ @@ -494,4 +506,7 @@ end col = :se, lav_col = :se, ) + + test_bootstrap(solution_ml) + smoketest_CI_z(solution_ml, partable_mean) end From 43bb31277f3c47426b8b56bd360104f44f1ac5a3 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Mon, 9 Mar 2026 23:20:34 +0100 Subject: [PATCH 288/364] Apply suggestions from formatter Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 4 +--- src/frontend/fit/standard_errors/bootstrap.jl | 11 ++++++----- .../standard_errors/confidence_intervals.jl | 19 +++++++++---------- src/frontend/fit/standard_errors/z_test.jl | 2 +- test/examples/helper.jl | 2 +- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index 70c4f3963..3ec324530 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -93,9 +93,7 @@ SEM.algorithm_name( SEM.convergence( ::ProximalResult, ) = "No standard convergence criteria for proximal \n algorithms available." -SEM.converged( - ::ProximalResult, -) = missing +SEM.converged(::ProximalResult) = missing SEM.n_iterations(res::ProximalResult) = res.n_iterations ############################################################################################ diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index 12828a09a..5b9f28aba 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -48,7 +48,7 @@ function bootstrap( engine = :Optim, parallel = false, fit_kwargs = Dict(), - replace_kwargs = Dict() + replace_kwargs = Dict(), ) # access data and convert to matrix data = prepare_data_bootstrap(data, fitted.model) @@ -121,7 +121,8 @@ function bootstrap( :n_boot => n_boot, :n_converged => sum(conv), :converged => conv, - :n_errored => n_failed[]) + :n_errored => n_failed[], + ) end """ @@ -171,7 +172,7 @@ function se_bootstrap( engine = :Optim, parallel = false, fit_kwargs = Dict(), - replace_kwargs = Dict() + replace_kwargs = Dict(), ) # access data and convert to matrix data = prepare_data_bootstrap(data, fitted.model) @@ -196,7 +197,7 @@ function se_bootstrap( sol = solution(new_fit) conv = converged(new_fit) if conv - n_conv[] += 1 + n_conv[] += 1 @. total_sum += sol @. total_squared_sum += sol^2 end @@ -228,7 +229,7 @@ function se_bootstrap( conv = converged(new_fit) if conv lock(lk) do - n_conv[] += 1 + n_conv[] += 1 @. total_sum += sol @. total_squared_sum += sol^2 end diff --git a/src/frontend/fit/standard_errors/confidence_intervals.jl b/src/frontend/fit/standard_errors/confidence_intervals.jl index 432e60819..20bf58a73 100644 --- a/src/frontend/fit/standard_errors/confidence_intervals.jl +++ b/src/frontend/fit/standard_errors/confidence_intervals.jl @@ -21,8 +21,7 @@ Return normal-theory confidence intervals for all model parameters. """ @doc "$(_doc_normal_CI)" -function normal_CI( - fitted, se; α = 0.05, name_lower = :ci_lower, name_upper = :ci_upper) +function normal_CI(fitted, se; α = 0.05, name_lower = :ci_lower, name_upper = :ci_upper) qnt = quantile(Normal(0, 1), 1-α/2); sol = solution(fitted) return Dict(name_lower => sol - qnt*se, name_upper => sol + qnt*se) @@ -30,14 +29,14 @@ end @doc "$(_doc_normal_CI)" function normal_CI!( - partable, - fitted, - se; - α = 0.05, - name_lower = :ci_lower, - name_upper = :ci_upper) - cis = normal_CI( - fitted, se; α, name_lower, name_upper) + partable, + fitted, + se; + α = 0.05, + name_lower = :ci_lower, + name_upper = :ci_upper, +) + cis = normal_CI(fitted, se; α, name_lower, name_upper) update_partable!(partable, name_lower, fitted, cis[name_lower]) update_partable!(partable, name_upper, fitted, cis[name_upper]) return cis diff --git a/src/frontend/fit/standard_errors/z_test.jl b/src/frontend/fit/standard_errors/z_test.jl index 9705de1be..27bebf147 100644 --- a/src/frontend/fit/standard_errors/z_test.jl +++ b/src/frontend/fit/standard_errors/z_test.jl @@ -22,7 +22,7 @@ Tests the null hypothesis that each parameter is zero using the test statistic @doc "$(_doc_z_test)" function z_test(fitted, se) - dev = solution(fitted)./se + dev = solution(fitted) ./ se dist = Normal(0, 1) p = 2*ccdf.(dist, abs.(dev)) return p diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 3e66ef888..4c05223f1 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -153,4 +153,4 @@ function smoketest_CI_z(model_fit, partable) se_he = se_hessian(model_fit) normal_CI!(partable, model_fit, se_he) z_test!(partable, model_fit, se_he) -end \ No newline at end of file +end From cf2f514466505f69c866c75ca27338918df28664 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 9 Mar 2026 23:39:28 +0100 Subject: [PATCH 289/364] fix bootstrap tests --- test/examples/helper.jl | 2 +- test/examples/multigroup/multigroup.jl | 1 + test/examples/political_democracy/political_democracy.jl | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 4c05223f1..dda6112b3 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -143,7 +143,7 @@ function test_bootstrap(model_fit; n_boot = 500) @test isapprox(se_bs, se_he, rtol = 0.2) # se_bootstrap and bootstrap |> se are close bs_samples = bootstrap(model_fit; n_boot = n_boot) - @test bs_samples[:n_converged] > 990 + @test bs_samples[:n_converged] > 0.95*n_boot bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) @test isapprox(se_bs_2, se_bs, rtol = 0.05) diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 1f89714d8..78af390c2 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -1,5 +1,6 @@ using StructuralEquationModels, Test, FiniteDiff, Suppressor using LinearAlgebra: diagind, LowerTriangular +using Statistics: var const SEM = StructuralEquationModels diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 9c8cc2a7b..b929ced68 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -1,5 +1,5 @@ using StructuralEquationModels, Test, Suppressor, FiniteDiff -using Statistics: cov, mean +using Statistics: cov, mean, var using Random, NLopt SEM = StructuralEquationModels From add4739430f87b3b176bd844473cb631059e69a6 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 10 Mar 2026 00:22:04 +0100 Subject: [PATCH 290/364] fix bootstrap tests --- src/frontend/fit/standard_errors/bootstrap.jl | 4 ++-- test/examples/helper.jl | 6 +++--- test/examples/multigroup/build_models.jl | 6 +++--- test/examples/political_democracy/constructor.jl | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index 5b9f28aba..45fe53454 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -19,7 +19,7 @@ Return bootstrap samples for `statistic`. - `n_boot`: number of boostrap samples - `data`: data to sample from. Only needed if different than the data from `sem_fit` - `specification`: a `ParameterTable` or `RAMMatrices` object passed to `replace_observed`. - Necessary for FIML models. + Necessary for FIML / WLS models. - `engine`: optimizer engine, passed to `fit`. - `parallel`: if `true`, run bootstrap samples in parallel on all available threads. The number of threads is controlled by the `JULIA_NUM_THREADS` environment variable or @@ -143,7 +143,7 @@ Return bootstrap standard errors. - `n_boot`: number of boostrap samples - `data`: data to sample from. Only needed if different than the data from `sem_fit` - `specification`: a `ParameterTable` or `RAMMatrices` object passed to `replace_observed`. - Necessary for FIML models. + Necessary for FIML / WLS models. - `engine`: optimizer engine, passed to `fit`. - `parallel`: if `true`, run bootstrap samples in parallel on all available threads. The number of threads is controlled by the `JULIA_NUM_THREADS` environment variable or diff --git a/test/examples/helper.jl b/test/examples/helper.jl index dda6112b3..5757bfcfc 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -136,13 +136,13 @@ function test_estimates( end end -function test_bootstrap(model_fit; n_boot = 500) +function test_bootstrap(model_fit, spec; n_boot = 500) # hessian and bootstrap se are close se_he = se_hessian(model_fit) - se_bs = se_bootstrap(model_fit; n_boot = n_boot) + se_bs = se_bootstrap(model_fit; specification = spec, n_boot = n_boot) @test isapprox(se_bs, se_he, rtol = 0.2) # se_bootstrap and bootstrap |> se are close - bs_samples = bootstrap(model_fit; n_boot = n_boot) + bs_samples = bootstrap(model_fit; specification = spec, n_boot = n_boot) @test bs_samples[:n_converged] > 0.95*n_boot bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index fa4df6dd9..7f2acbded 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -83,7 +83,7 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution_ml) + test_bootstrap(solution_ml, partable) smoketest_CI_z(solution_ml, partable) solution_ml = fit(model_ml_multigroup2) @@ -293,7 +293,7 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution_ls) + test_bootstrap(solution_ls, partable) smoketest_CI_z(solution_ls, partable) end @@ -412,7 +412,7 @@ if !isnothing(specification_miss_g1) lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution; n_boot = 500) + test_bootstrap(solution, partable_miss) smoketest_CI_z(solution, partable_miss) solution = fit(semoptimizer, model_ml_multigroup2) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 5ced819f6..7c7f84f37 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -131,7 +131,7 @@ end lav_col = :se, ) - test_bootstrap(solution_ml) + test_bootstrap(solution_ml, partable) smoketest_CI_z(solution_ml, partable) end @@ -161,7 +161,7 @@ end lav_col = :se, ) - test_bootstrap(solution_ls) + test_bootstrap(solution_ls, partable) smoketest_CI_z(solution_ls, partable) end @@ -344,7 +344,7 @@ end lav_col = :se, ) - test_bootstrap(solution_ml) + test_bootstrap(solution_ml, partable_mean) smoketest_CI_z(solution_ml, partable_mean) end @@ -373,7 +373,7 @@ end lav_col = :se, ) - test_bootstrap(solution_ls) + test_bootstrap(solution_ls, partable_mean) smoketest_CI_z(solution_ls, partable_mean) end @@ -507,6 +507,6 @@ end lav_col = :se, ) - test_bootstrap(solution_ml) + test_bootstrap(solution_ml, partable_mean) smoketest_CI_z(solution_ml, partable_mean) end From 83440587537bcf6113c55bbe68b287c75386ba35 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 12 Mar 2026 11:05:39 +0100 Subject: [PATCH 291/364] fix nobs var check in update_observed, keep previous args in update_observed, fix bs + tests --- src/frontend/fit/standard_errors/bootstrap.jl | 33 ++++++++++--------- src/implied/RAM/generic.jl | 8 +++-- src/implied/RAM/symbolic.jl | 11 +++++-- src/loss/ML/ML.jl | 5 ++- src/loss/WLS/WLS.jl | 6 ++-- test/examples/helper.jl | 29 +++++++++++----- test/examples/multigroup/build_models.jl | 2 +- test/examples/multigroup/multigroup.jl | 3 ++ .../political_democracy/constructor.jl | 8 +++-- .../political_democracy.jl | 2 ++ 10 files changed, 71 insertions(+), 36 deletions(-) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index 45fe53454..e13058978 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -1,10 +1,10 @@ """ bootstrap( - fitted::SemFit; + fitted::SemFit, + specification::SemSpecification; statistic = solution, n_boot = 3000, data = nothing, - specification = nothing, engine = :Optim, parallel = false, fit_kwargs = Dict(), @@ -14,12 +14,11 @@ Return bootstrap samples for `statistic`. # Arguments - `fitted`: a fitted SEM. +- `specification`: a `ParameterTable` or `RAMMatrices` object passed to `replace_observed`. - `statistic`: any function that can be called on a `SemFit` object. The output will be returned as the bootstrap sample. - `n_boot`: number of boostrap samples - `data`: data to sample from. Only needed if different than the data from `sem_fit` -- `specification`: a `ParameterTable` or `RAMMatrices` object passed to `replace_observed`. - Necessary for FIML / WLS models. - `engine`: optimizer engine, passed to `fit`. - `parallel`: if `true`, run bootstrap samples in parallel on all available threads. The number of threads is controlled by the `JULIA_NUM_THREADS` environment variable or @@ -40,11 +39,11 @@ bootstrap( ``` """ function bootstrap( - fitted::SemFit; + fitted::SemFit, + specification::SemSpecification; statistic = solution, n_boot = 3000, data = nothing, - specification = nothing, engine = :Optim, parallel = false, fit_kwargs = Dict(), @@ -56,6 +55,7 @@ function bootstrap( # pre-allocations out = [] conv = [] + errors = [] n_failed = Ref(0) # fit to bootstrap samples if !parallel @@ -73,8 +73,9 @@ function bootstrap( c = converged(new_fit) push!(out, sample) push!(conv, c) - catch + catch e n_failed[] += 1 + push!(errors, e) end end else @@ -103,9 +104,10 @@ function bootstrap( push!(out, sample) push!(conv, c) end - catch + catch e lock(lk) do n_failed[] += 1 + push!(errors, e) end finally put!(model_pool, thread_model) @@ -119,19 +121,19 @@ function bootstrap( return Dict( :samples => out, :n_boot => n_boot, - :n_converged => sum(conv), + :n_converged => isempty(conv) ? 0 : sum(conv), :converged => conv, :n_errored => n_failed[], + :errors => errors ) end """ se_bootstrap( - fitted::SemFit; + fitted::SemFit, + specification::SemSpecification; n_boot = 3000, data = nothing, - specification = nothing, - engine = :Optim, parallel = false, fit_kwargs = Dict(), replace_kwargs = Dict()) @@ -140,10 +142,9 @@ Return bootstrap standard errors. # Arguments - `fitted`: a fitted SEM. +- `specification`: a `ParameterTable` or `RAMMatrices` object passed to `replace_observed`. - `n_boot`: number of boostrap samples - `data`: data to sample from. Only needed if different than the data from `sem_fit` -- `specification`: a `ParameterTable` or `RAMMatrices` object passed to `replace_observed`. - Necessary for FIML / WLS models. - `engine`: optimizer engine, passed to `fit`. - `parallel`: if `true`, run bootstrap samples in parallel on all available threads. The number of threads is controlled by the `JULIA_NUM_THREADS` environment variable or @@ -165,10 +166,10 @@ se_bootstrap( ``` """ function se_bootstrap( - fitted::SemFit; + fitted::SemFit, + specification::SemSpecification; n_boot = 3000, data = nothing, - specification = nothing, engine = :Optim, parallel = false, fit_kwargs = Dict(), diff --git a/src/implied/RAM/generic.jl b/src/implied/RAM/generic.jl index 37591c232..3b8596874 100644 --- a/src/implied/RAM/generic.jl +++ b/src/implied/RAM/generic.jl @@ -196,9 +196,13 @@ end ############################################################################################ function update_observed(implied::RAM, observed::SemObserved; kwargs...) - if nobserved_vars(observed) == size(implied.Σ, 1) + if nobserved_vars(observed) == nobserved_vars(implied) return implied else - return RAM(; observed = observed, kwargs...) + return RAM(; + observed = observed, + gradient_required = !isnothing(implied.∇A), + meanstructure = MeanStruct(implied) == HasMeanStruct, + kwargs...) end end diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index 436f339b7..0d3ba5e11 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -210,10 +210,17 @@ end ############################################################################################ function update_observed(implied::RAMSymbolic, observed::SemObserved; kwargs...) - if nobserved_vars(observed) == size(implied.Σ, 1) + if nobserved_vars(observed) == nobserved_vars(implied) return implied else - return RAMSymbolic(; observed = observed, kwargs...) + return RAMSymbolic(; + observed = observed, + vech = implied.Σ isa Vector, + gradient = !isnothing(implied.∇Σ), + hessian = !isnothing(implied.∇²Σ), + meanstructure = MeanStruct(implied) == HasMeanStruct, + approximate_hessian = isnothing(implied.∇²Σ), + kwargs...) end end diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 6461ba087..67d4fe524 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -237,6 +237,9 @@ function update_observed(lossfun::SemML, observed::SemObserved; kwargs...) if size(lossfun.Σ⁻¹) == size(obs_cov(observed)) return lossfun else - return SemML(; observed = observed, kwargs...) + return SemML(; + observed = observed, + approximate_hessian = HessianEval(lossfun) == ApproxHessian, + kwargs...) end end diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index b2aed17c0..2b10d7b47 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -173,5 +173,7 @@ end ### Recommended methods ############################################################################################ -update_observed(lossfun::SemWLS, observed::SemObserved; kwargs...) = - SemWLS(; observed = observed, kwargs...) +update_observed(lossfun::SemWLS, observed::SemObserved; kwargs...) = SemWLS(; + observed = observed, + meanstructure = MeanStruct(kwargs[:implied]) == HasMeanStruct, + kwargs...) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 5757bfcfc..f4b37a72b 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -136,17 +136,28 @@ function test_estimates( end end -function test_bootstrap(model_fit, spec; n_boot = 500) +function test_bootstrap(model_fit, spec; compare_hessian = true, compare_bs = true, n_boot = 500) + se_bs = se_bootstrap(model_fit, spec; n_boot = n_boot) # hessian and bootstrap se are close - se_he = se_hessian(model_fit) - se_bs = se_bootstrap(model_fit; specification = spec, n_boot = n_boot) - @test isapprox(se_bs, se_he, rtol = 0.2) + if compare_hessian + se_he = se_hessian(model_fit) + @test isapprox(se_bs, se_he, rtol = 0.2) + end # se_bootstrap and bootstrap |> se are close - bs_samples = bootstrap(model_fit; specification = spec, n_boot = n_boot) - @test bs_samples[:n_converged] > 0.95*n_boot - bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) - se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) - @test isapprox(se_bs_2, se_bs, rtol = 0.05) + if compare_bs + bs_samples = bootstrap(model_fit, spec; n_boot = n_boot) + @test bs_samples[:n_converged] > 0.95*n_boot + bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) + se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) + @test isapprox(se_bs_2, se_bs, rtol = 0.05) + end +end + +function smoketest_bootstrap(model_fit, spec; n_boot = 5) + # hessian and bootstrap se are close + se_bs = se_bootstrap(model_fit, spec; n_boot = n_boot) + bs_samples = bootstrap(model_fit, spec; n_boot = n_boot) + return se_bs, bs_samples end function smoketest_CI_z(model_fit, partable) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 7f2acbded..f16860ef7 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -293,7 +293,7 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution_ls, partable) + test_bootstrap(solution_ls, partable; compare_bs = false) smoketest_CI_z(solution_ls, partable) end diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 78af390c2..dd654731d 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -1,6 +1,9 @@ using StructuralEquationModels, Test, FiniteDiff, Suppressor using LinearAlgebra: diagind, LowerTriangular using Statistics: var +using Random + +Random.seed!(948723) const SEM = StructuralEquationModels diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 7c7f84f37..1c1c42e54 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -161,7 +161,7 @@ end lav_col = :se, ) - test_bootstrap(solution_ls, partable) + test_bootstrap(solution_ls, partable; compare_bs = false) smoketest_CI_z(solution_ls, partable) end @@ -373,7 +373,8 @@ end lav_col = :se, ) - test_bootstrap(solution_ls, partable_mean) + test_bootstrap(solution_ls, partable_mean, compare_bs = false) + # smoketest_bootstrap(solution_ls, partable_mean) smoketest_CI_z(solution_ls, partable_mean) end @@ -507,6 +508,7 @@ end lav_col = :se, ) - test_bootstrap(solution_ml, partable_mean) + # test_bootstrap(solution_ml, partable_mean) # too much compute + smoketest_bootstrap(solution_ml, partable_mean) smoketest_CI_z(solution_ml, partable_mean) end diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index b929ced68..8c8d6c36e 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -2,6 +2,8 @@ using StructuralEquationModels, Test, Suppressor, FiniteDiff using Statistics: cov, mean, var using Random, NLopt +Random.seed!(464577) + SEM = StructuralEquationModels include( From 2c5549a453472d0d23be2dd2700147f52ceacce9 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 12 Mar 2026 11:58:20 +0100 Subject: [PATCH 292/364] increase tolerance for bootstrap test --- test/examples/helper.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index f4b37a72b..223d8f3d3 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -149,7 +149,7 @@ function test_bootstrap(model_fit, spec; compare_hessian = true, compare_bs = tr @test bs_samples[:n_converged] > 0.95*n_boot bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) - @test isapprox(se_bs_2, se_bs, rtol = 0.05) + @test isapprox(se_bs_2, se_bs, rtol = 0.1) end end From b4cc3a3decaaa68343b8568418770c523edc5efd Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 12 Mar 2026 12:06:26 +0100 Subject: [PATCH 293/364] increase tolerance for bootstrap test --- test/examples/helper.jl | 13 ++++++++++--- test/examples/multigroup/build_models.jl | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 223d8f3d3..faadfd71d 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -136,12 +136,19 @@ function test_estimates( end end -function test_bootstrap(model_fit, spec; compare_hessian = true, compare_bs = true, n_boot = 500) +function test_bootstrap( + model_fit, + spec; + compare_hessian = true, + rtol_hessian = 0.2, + compare_bs = true, + rtol_bs = 0.1, + n_boot = 500) se_bs = se_bootstrap(model_fit, spec; n_boot = n_boot) # hessian and bootstrap se are close if compare_hessian se_he = se_hessian(model_fit) - @test isapprox(se_bs, se_he, rtol = 0.2) + @test isapprox(se_bs, se_he, rtol = rtol_hessian) end # se_bootstrap and bootstrap |> se are close if compare_bs @@ -149,7 +156,7 @@ function test_bootstrap(model_fit, spec; compare_hessian = true, compare_bs = tr @test bs_samples[:n_converged] > 0.95*n_boot bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) - @test isapprox(se_bs_2, se_bs, rtol = 0.1) + @test isapprox(se_bs_2, se_bs, rtol = rtol_bs) end end diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index f16860ef7..6d7addc87 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -83,7 +83,7 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution_ml, partable) + test_bootstrap(solution_ml, partable, rtol_hessian = 0.3) smoketest_CI_z(solution_ml, partable) solution_ml = fit(model_ml_multigroup2) @@ -412,7 +412,7 @@ if !isnothing(specification_miss_g1) lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution, partable_miss) + test_bootstrap(solution, partable_miss, rtol_hessian = 0.3) smoketest_CI_z(solution, partable_miss) solution = fit(semoptimizer, model_ml_multigroup2) From 6f4ac26e0dadf1e08feffd7fa7cf856abb448810 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 12 Mar 2026 19:02:17 +0100 Subject: [PATCH 294/364] remove bootstrap try-catch and update tests --- src/frontend/fit/standard_errors/bootstrap.jl | 115 +++++++----------- test/examples/helper.jl | 2 +- test/examples/multigroup/build_models.jl | 2 +- 3 files changed, 43 insertions(+), 76 deletions(-) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index e13058978..a1225412b 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -55,28 +55,21 @@ function bootstrap( # pre-allocations out = [] conv = [] - errors = [] - n_failed = Ref(0) # fit to bootstrap samples if !parallel for _ in 1:n_boot - try - sample_data = bootstrap_sample(data) - new_model = replace_observed( - fitted.model; - data = sample_data, - specification = specification, - replace_kwargs..., - ) - new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) - sample = statistic(new_fit) - c = converged(new_fit) - push!(out, sample) - push!(conv, c) - catch e - n_failed[] += 1 - push!(errors, e) - end + sample_data = bootstrap_sample(data) + new_model = replace_observed( + fitted.model; + data = sample_data, + specification = specification, + replace_kwargs..., + ) + new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) + sample = statistic(new_fit) + c = converged(new_fit) + push!(out, sample) + push!(conv, c) end else n_threads = Threads.nthreads() @@ -89,42 +82,28 @@ function bootstrap( lk = ReentrantLock() Threads.@threads for _ in 1:n_boot thread_model = take!(model_pool) - try - sample_data = bootstrap_sample(data) - new_model = replace_observed( - thread_model; - data = sample_data, - specification = specification, - replace_kwargs..., - ) - new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) - sample = statistic(new_fit) - c = converged(new_fit) - lock(lk) do - push!(out, sample) - push!(conv, c) - end - catch e - lock(lk) do - n_failed[] += 1 - push!(errors, e) - end - finally - put!(model_pool, thread_model) + sample_data = bootstrap_sample(data) + new_model = replace_observed( + thread_model; + data = sample_data, + specification = specification, + replace_kwargs..., + ) + new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) + sample = statistic(new_fit) + c = converged(new_fit) + lock(lk) do + push!(out, sample) + push!(conv, c) end + put!(model_pool, thread_model) end end - # compute parameters - if !iszero(n_failed[]) - @warn "During bootstrap sampling, "*string(n_failed[])*" samples errored." - end return Dict( :samples => out, :n_boot => n_boot, :n_converged => isempty(conv) ? 0 : sum(conv), :converged => conv, - :n_errored => n_failed[], - :errors => errors ) end @@ -181,8 +160,6 @@ function se_bootstrap( # pre-allocations total_sum = zero(start) total_squared_sum = zero(start) - n_failed = Ref(0) - n_conv = Ref(0) # fit to bootstrap samples if !parallel for _ in 1:n_boot @@ -217,39 +194,29 @@ function se_bootstrap( lk = ReentrantLock() Threads.@threads for _ in 1:n_boot thread_model = take!(model_pool) - try - sample_data = bootstrap_sample(data) - new_model = replace_observed( - thread_model; - data = sample_data, - specification = specification, - replace_kwargs..., - ) - new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) - sol = solution(new_fit) - conv = converged(new_fit) - if conv - lock(lk) do - n_conv[] += 1 - @. total_sum += sol - @. total_squared_sum += sol^2 - end - end - catch + sample_data = bootstrap_sample(data) + new_model = replace_observed( + thread_model; + data = sample_data, + specification = specification, + replace_kwargs..., + ) + new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) + sol = solution(new_fit) + conv = converged(new_fit) + if conv lock(lk) do - n_failed[] += 1 + n_conv[] += 1 + @. total_sum += sol + @. total_squared_sum += sol^2 end - finally - put!(model_pool, thread_model) end + put!(model_pool, thread_model) end end # compute parameters n_conv = n_conv[] sd = sqrt.(total_squared_sum / n_conv - (total_sum / n_conv) .^ 2) - if !iszero(n_failed[]) - @warn "During bootstrap sampling, "*string(n_failed[])*" samples errored" - end @info string(n_conv)*" models converged" return sd end diff --git a/test/examples/helper.jl b/test/examples/helper.jl index faadfd71d..31d65679a 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -153,7 +153,7 @@ function test_bootstrap( # se_bootstrap and bootstrap |> se are close if compare_bs bs_samples = bootstrap(model_fit, spec; n_boot = n_boot) - @test bs_samples[:n_converged] > 0.95*n_boot + @test bs_samples[:n_converged] >= 0.95*n_boot bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) @test isapprox(se_bs_2, se_bs, rtol = rtol_bs) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 6d7addc87..905ff6594 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -12,7 +12,7 @@ model_g2 = Sem(specification = specification_g2, data = dat_g2, implied = RAM) SEM.param_labels(model_g2.implied.ram_matrices) # test the different constructors -model_ml_multigroup = SemEnsemble(model_g1, model_g2) +model_ml_multigroup = SemEnsemble(model_g1, model_g2; groups = [:Pasteur, :Grant_White]) model_ml_multigroup2 = SemEnsemble( specification = partable, data = dat, From ea2d876122aaf51428ed580818f5a5aaff07a027 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 12 Mar 2026 19:20:51 +0100 Subject: [PATCH 295/364] fix bootstrap --- src/frontend/fit/standard_errors/bootstrap.jl | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index a1225412b..b2a909e9b 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -160,27 +160,24 @@ function se_bootstrap( # pre-allocations total_sum = zero(start) total_squared_sum = zero(start) + n_conv = Ref(0) # fit to bootstrap samples if !parallel for _ in 1:n_boot - try - sample_data = bootstrap_sample(data) - new_model = replace_observed( - fitted.model; - data = sample_data, - specification = specification, - replace_kwargs..., - ) - new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) - sol = solution(new_fit) - conv = converged(new_fit) - if conv - n_conv[] += 1 - @. total_sum += sol - @. total_squared_sum += sol^2 - end - catch - n_failed[] += 1 + sample_data = bootstrap_sample(data) + new_model = replace_observed( + fitted.model; + data = sample_data, + specification = specification, + replace_kwargs..., + ) + new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) + sol = solution(new_fit) + conv = converged(new_fit) + if conv + n_conv[] += 1 + @. total_sum += sol + @. total_squared_sum += sol^2 end end else From 95e1776f2d21b121bd93c979754f16a799c096cd Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 12 Mar 2026 21:23:41 +0100 Subject: [PATCH 296/364] fix bootstrap --- test/examples/multigroup/build_models.jl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 905ff6594..e6ff2d182 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -83,7 +83,7 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution_ml, partable, rtol_hessian = 0.3) + test_bootstrap(solution_ml, partable, rtol_hessian = 0.3, rtol_bs = 0.2) smoketest_CI_z(solution_ml, partable) solution_ml = fit(model_ml_multigroup2) @@ -252,7 +252,7 @@ model_ls_g2 = Sem( loss = SemWLS, ) -model_ls_multigroup = SemEnsemble(model_ls_g1, model_ls_g2; optimizer = semoptimizer) +model_ls_multigroup = SemEnsemble(model_ls_g1, model_ls_g2; groups = [:Pasteur, :Grant_White], optimizer = semoptimizer) @testset "ls_gradients_multigroup" begin test_gradient(model_ls_multigroup, start_test; atol = 1e-9) @@ -412,8 +412,7 @@ if !isnothing(specification_miss_g1) lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution, partable_miss, rtol_hessian = 0.3) - smoketest_CI_z(solution, partable_miss) + solution = fit(semoptimizer, model_ml_multigroup2) test_fitmeasures( @@ -428,6 +427,9 @@ if !isnothing(specification_miss_g1) fitmeasure_names = Dict(:CFI => "cfi"), ) + test_bootstrap(solution, partable_miss, rtol_hessian = 0.3) + smoketest_CI_z(solution, partable_miss) + update_se_hessian!(partable_miss, solution) test_estimates( partable_miss, From d1ac265d82c96f18694131b6530f428e5002669e Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 12 Mar 2026 22:28:03 +0100 Subject: [PATCH 297/364] fix bootstrap --- test/examples/multigroup/build_models.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index e6ff2d182..5cbb345c7 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -83,7 +83,7 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution_ml, partable, rtol_hessian = 0.3, rtol_bs = 0.2) + test_bootstrap(solution_ml, partable; rtol_hessian = 0.3, rtol_bs = 0.2, n_boot = 1_000) smoketest_CI_z(solution_ml, partable) solution_ml = fit(model_ml_multigroup2) @@ -293,7 +293,7 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution_ls, partable; compare_bs = false) + test_bootstrap(solution_ls, partable; compare_bs = false, rtol_hessian = 0.3) smoketest_CI_z(solution_ls, partable) end @@ -427,7 +427,7 @@ if !isnothing(specification_miss_g1) fitmeasure_names = Dict(:CFI => "cfi"), ) - test_bootstrap(solution, partable_miss, rtol_hessian = 0.3) + test_bootstrap(solution, partable_miss; compare_bs = false, rtol_hessian = 0.3) smoketest_CI_z(solution, partable_miss) update_se_hessian!(partable_miss, solution) From b75adc5aeccbcd914555459671f7062ad5f85de7 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 13 Mar 2026 21:51:04 +0100 Subject: [PATCH 298/364] fix mg weights --- src/types.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types.jl b/src/types.jl index 3f695bfa3..3a6b5fdf1 100644 --- a/src/types.jl +++ b/src/types.jl @@ -231,13 +231,13 @@ function multigroup_weights(models, n) uniform_lossfun = check_single_lossfun(models...; throw_error = false) if !uniform_lossfun @info "Your ensemble model contains heterogeneous loss functions. - Default weights of (#samples per group/#total samples) will be used". + Default weights of (#samples per group/#total samples) will be used." return [(nsamples(model)) / (nsamples_total) for model in models] end lossfun = models[1].loss.functions[1] if !applicable(mg_correction, lossfun) @info "We don't know how to choose group weights for the specified loss function. - Default weights of (#samples per group/#total samples) will be used". + Default weights of (#samples per group/#total samples) will be used." return [(nsamples(model)) / (nsamples_total) for model in models] end c = mg_correction(lossfun) From 23ad54d1003810fc44d41757639601f7d4e9c3f0 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 20 Mar 2026 13:07:34 +0100 Subject: [PATCH 299/364] fix bootstrap tests --- test/examples/helper.jl | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 31d65679a..f2b9daf46 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,4 +1,5 @@ using LinearAlgebra: norm +using Suppressor function is_extended_tests() return lowercase(get(ENV, "JULIA_EXTENDED_TESTS", "false")) == "true" @@ -144,19 +145,23 @@ function test_bootstrap( compare_bs = true, rtol_bs = 0.1, n_boot = 500) - se_bs = se_bootstrap(model_fit, spec; n_boot = n_boot) - # hessian and bootstrap se are close - if compare_hessian - se_he = se_hessian(model_fit) - @test isapprox(se_bs, se_he, rtol = rtol_hessian) - end - # se_bootstrap and bootstrap |> se are close - if compare_bs - bs_samples = bootstrap(model_fit, spec; n_boot = n_boot) - @test bs_samples[:n_converged] >= 0.95*n_boot - bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) - se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) - @test isapprox(se_bs_2, se_bs, rtol = rtol_bs) + @testset rng = Random.seed!(32432) "bootstrap" begin + se_bs = @suppress se_bootstrap(model_fit, spec; n_boot = n_boot) + # hessian and bootstrap se are close + if compare_hessian + se_he = @suppress se_hessian(model_fit) + #println(maximum(abs.(se_he - se_bs))) + @test isapprox(se_bs, se_he, rtol = rtol_hessian) + end + # se_bootstrap and bootstrap |> se are close + if compare_bs + bs_samples = bootstrap(model_fit, spec; n_boot = n_boot) + @test bs_samples[:n_converged] >= 0.95*n_boot + bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) + se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) + #println(maximum(abs.(se_bs_2 - se_bs))) + @test isapprox(se_bs_2, se_bs, rtol = rtol_bs) + end end end @@ -168,7 +173,7 @@ function smoketest_bootstrap(model_fit, spec; n_boot = 5) end function smoketest_CI_z(model_fit, partable) - se_he = se_hessian(model_fit) + se_he = @suppress se_hessian(model_fit) normal_CI!(partable, model_fit, se_he) z_test!(partable, model_fit, se_he) end From 41e428b00706f27c88edaeb66c3b24b700efabd1 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 20 Mar 2026 13:08:14 +0100 Subject: [PATCH 300/364] fix bootstrap tests --- test/examples/multigroup/build_models.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 5cbb345c7..156ca1f1b 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -83,7 +83,8 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution_ml, partable; rtol_hessian = 0.3, rtol_bs = 0.2, n_boot = 1_000) + + test_bootstrap(solution_ml, partable; rtol_hessian = 0.3) smoketest_CI_z(solution_ml, partable) solution_ml = fit(model_ml_multigroup2) @@ -293,7 +294,7 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - test_bootstrap(solution_ls, partable; compare_bs = false, rtol_hessian = 0.3) + # test_bootstrap(solution_ls, partable; compare_bs = false, rtol_hessian = 0.3) smoketest_CI_z(solution_ls, partable) end @@ -427,7 +428,7 @@ if !isnothing(specification_miss_g1) fitmeasure_names = Dict(:CFI => "cfi"), ) - test_bootstrap(solution, partable_miss; compare_bs = false, rtol_hessian = 0.3) + test_bootstrap(solution, partable_miss; compare_bs = false, rtol_hessian = 0.5) smoketest_CI_z(solution, partable_miss) update_se_hessian!(partable_miss, solution) From 4e011b2706912578077f9c430b40f222e1d9c593 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 20 Mar 2026 13:08:43 +0100 Subject: [PATCH 301/364] add WLS option to fix weight matrix for updaeting observed data --- src/loss/WLS/WLS.jl | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 2b10d7b47..60a43fdf9 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -173,7 +173,19 @@ end ### Recommended methods ############################################################################################ -update_observed(lossfun::SemWLS, observed::SemObserved; kwargs...) = SemWLS(; - observed = observed, - meanstructure = MeanStruct(kwargs[:implied]) == HasMeanStruct, - kwargs...) +function update_observed(lossfun::SemWLS, observed::SemObserved; recompute_V = true, kwargs...) + if recompute_V + return SemWLS(; + observed = observed, + meanstructure = MeanStruct(kwargs[:implied]) == HasMeanStruct, + kwargs...) + else + return SemWLS(; + observed = observed, + wls_weight_matrix = lossfun.V, + wls_weight_matrix_mean = lossfun.V_μ, + meanstructure = MeanStruct(kwargs[:implied]) == HasMeanStruct, + kwargs...) + + end +end From a42d5d5b8e9673a7147879132adede514b85e517 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 20 Mar 2026 20:52:41 +0100 Subject: [PATCH 302/364] narrower output types and remove locks for bootstrap + code formatting --- src/frontend/fit/standard_errors/bootstrap.jl | 22 +++++++++---------- src/implied/RAM/generic.jl | 3 ++- src/implied/RAM/symbolic.jl | 15 +++++++------ src/loss/ML/ML.jl | 7 +++--- src/loss/WLS/WLS.jl | 6 +++-- test/examples/helper.jl | 6 +++-- test/examples/multigroup/build_models.jl | 7 +++++- 7 files changed, 38 insertions(+), 28 deletions(-) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index b2a909e9b..845a209ee 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -53,11 +53,11 @@ function bootstrap( data = prepare_data_bootstrap(data, fitted.model) start = solution(fitted) # pre-allocations - out = [] - conv = [] + out = Vector{Any}(nothing, n_boot) + conv = fill(false, n_boot) # fit to bootstrap samples if !parallel - for _ in 1:n_boot + for i in 1:n_boot sample_data = bootstrap_sample(data) new_model = replace_observed( fitted.model; @@ -68,8 +68,8 @@ function bootstrap( new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) sample = statistic(new_fit) c = converged(new_fit) - push!(out, sample) - push!(conv, c) + out[i] = sample + conv[i] = c end else n_threads = Threads.nthreads() @@ -80,7 +80,7 @@ function bootstrap( end # fit models in parallel lk = ReentrantLock() - Threads.@threads for _ in 1:n_boot + Threads.@threads for i in 1:n_boot thread_model = take!(model_pool) sample_data = bootstrap_sample(data) new_model = replace_observed( @@ -92,17 +92,15 @@ function bootstrap( new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) sample = statistic(new_fit) c = converged(new_fit) - lock(lk) do - push!(out, sample) - push!(conv, c) - end + out[i] = sample + conv[i] = c put!(model_pool, thread_model) end end return Dict( - :samples => out, + :samples => collect(a for a in out), :n_boot => n_boot, - :n_converged => isempty(conv) ? 0 : sum(conv), + :n_converged => sum(conv), :converged => conv, ) end diff --git a/src/implied/RAM/generic.jl b/src/implied/RAM/generic.jl index 3b8596874..4c1fa323c 100644 --- a/src/implied/RAM/generic.jl +++ b/src/implied/RAM/generic.jl @@ -203,6 +203,7 @@ function update_observed(implied::RAM, observed::SemObserved; kwargs...) observed = observed, gradient_required = !isnothing(implied.∇A), meanstructure = MeanStruct(implied) == HasMeanStruct, - kwargs...) + kwargs..., + ) end end diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index 0d3ba5e11..df7c497ad 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -214,13 +214,14 @@ function update_observed(implied::RAMSymbolic, observed::SemObserved; kwargs...) return implied else return RAMSymbolic(; - observed = observed, - vech = implied.Σ isa Vector, - gradient = !isnothing(implied.∇Σ), - hessian = !isnothing(implied.∇²Σ), - meanstructure = MeanStruct(implied) == HasMeanStruct, - approximate_hessian = isnothing(implied.∇²Σ), - kwargs...) + observed = observed, + vech = implied.Σ isa Vector, + gradient = !isnothing(implied.∇Σ), + hessian = !isnothing(implied.∇²Σ), + meanstructure = MeanStruct(implied) == HasMeanStruct, + approximate_hessian = isnothing(implied.∇²Σ), + kwargs..., + ) end end diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 67d4fe524..ce77ea9c3 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -238,8 +238,9 @@ function update_observed(lossfun::SemML, observed::SemObserved; kwargs...) return lossfun else return SemML(; - observed = observed, - approximate_hessian = HessianEval(lossfun) == ApproxHessian, - kwargs...) + observed = observed, + approximate_hessian = HessianEval(lossfun) == ApproxHessian, + kwargs..., + ) end end diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 60a43fdf9..655fcd95c 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -178,14 +178,16 @@ function update_observed(lossfun::SemWLS, observed::SemObserved; recompute_V = t return SemWLS(; observed = observed, meanstructure = MeanStruct(kwargs[:implied]) == HasMeanStruct, - kwargs...) + kwargs..., + ) else return SemWLS(; observed = observed, wls_weight_matrix = lossfun.V, wls_weight_matrix_mean = lossfun.V_μ, meanstructure = MeanStruct(kwargs[:implied]) == HasMeanStruct, - kwargs...) + kwargs..., + ) end end diff --git a/test/examples/helper.jl b/test/examples/helper.jl index f2b9daf46..0d3c7b7c1 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -144,7 +144,8 @@ function test_bootstrap( rtol_hessian = 0.2, compare_bs = true, rtol_bs = 0.1, - n_boot = 500) + n_boot = 500, + ) @testset rng = Random.seed!(32432) "bootstrap" begin se_bs = @suppress se_bootstrap(model_fit, spec; n_boot = n_boot) # hessian and bootstrap se are close @@ -157,7 +158,8 @@ function test_bootstrap( if compare_bs bs_samples = bootstrap(model_fit, spec; n_boot = n_boot) @test bs_samples[:n_converged] >= 0.95*n_boot - bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) + bs_samples = + cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) #println(maximum(abs.(se_bs_2 - se_bs))) @test isapprox(se_bs_2, se_bs, rtol = rtol_bs) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 156ca1f1b..f058a4f44 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -253,7 +253,12 @@ model_ls_g2 = Sem( loss = SemWLS, ) -model_ls_multigroup = SemEnsemble(model_ls_g1, model_ls_g2; groups = [:Pasteur, :Grant_White], optimizer = semoptimizer) +model_ls_multigroup = SemEnsemble( + model_ls_g1, + model_ls_g2; + groups = [:Pasteur, :Grant_White], + optimizer = semoptimizer, +) @testset "ls_gradients_multigroup" begin test_gradient(model_ls_multigroup, start_test; atol = 1e-9) From 039985523dd12676a8150387367d0c6db7fc35f4 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Fri, 20 Mar 2026 21:00:46 +0100 Subject: [PATCH 303/364] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/loss/WLS/WLS.jl | 8 ++++++-- test/examples/helper.jl | 18 +++++++++--------- test/examples/multigroup/build_models.jl | 1 - 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 655fcd95c..b7f66d558 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -173,7 +173,12 @@ end ### Recommended methods ############################################################################################ -function update_observed(lossfun::SemWLS, observed::SemObserved; recompute_V = true, kwargs...) +function update_observed( + lossfun::SemWLS, + observed::SemObserved; + recompute_V = true, + kwargs..., +) if recompute_V return SemWLS(; observed = observed, @@ -188,6 +193,5 @@ function update_observed(lossfun::SemWLS, observed::SemObserved; recompute_V = t meanstructure = MeanStruct(kwargs[:implied]) == HasMeanStruct, kwargs..., ) - end end diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 0d3c7b7c1..c4191fdb1 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -138,14 +138,14 @@ function test_estimates( end function test_bootstrap( - model_fit, - spec; - compare_hessian = true, - rtol_hessian = 0.2, - compare_bs = true, - rtol_bs = 0.1, - n_boot = 500, - ) + model_fit, + spec; + compare_hessian = true, + rtol_hessian = 0.2, + compare_bs = true, + rtol_bs = 0.1, + n_boot = 500, +) @testset rng = Random.seed!(32432) "bootstrap" begin se_bs = @suppress se_bootstrap(model_fit, spec; n_boot = n_boot) # hessian and bootstrap se are close @@ -158,7 +158,7 @@ function test_bootstrap( if compare_bs bs_samples = bootstrap(model_fit, spec; n_boot = n_boot) @test bs_samples[:n_converged] >= 0.95*n_boot - bs_samples = + bs_samples = cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) #println(maximum(abs.(se_bs_2 - se_bs))) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index f058a4f44..bb7db3b50 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -419,7 +419,6 @@ if !isnothing(specification_miss_g1) lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = fit(semoptimizer, model_ml_multigroup2) test_fitmeasures( fit_measures(solution), From d1a9bcd900efcc5ec11bcc3de9d9f1bd9642c420 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 11:03:51 -0700 Subject: [PATCH 304/364] Project.toml: support Symbolics v7 & Utils v4 --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index b445f3db4..eabc5b36f 100644 --- a/Project.toml +++ b/Project.toml @@ -40,8 +40,8 @@ Optim = "1" PrettyTables = "3" ProximalAlgorithms = "0.7" StatsBase = "0.33, 0.34" -Symbolics = "4, 5, 6" -SymbolicUtils = "1.4 - 1.5, 1.7, 2, 3" +Symbolics = "4, 5, 6, 7" +SymbolicUtils = "1.4 - 1.5, 1.7, 2, 3, 4" StatsAPI = "1" [extras] From 97728bec889e2b8f2a676d6869b55fb616c27188 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 11:03:51 -0700 Subject: [PATCH 305/364] prepare_start_params(): tighten type check --- src/optimizer/abstract.jl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index e9a8c47ba..0c7913c48 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -154,11 +154,24 @@ function prepare_start_params(start_val::AbstractVector, model::AbstractSem; kwa "The length of `start_val` vector ($(length(start_val))) does not match the number of model parameters ($(nparams(model))).", ), ) + (eltype(start_val) <: Number) || throw( + TypeError( + :prepare_start_params, + "start_val elements must be numeric", + Number, + eltype(start_val), + ), + ) return start_val end function prepare_start_params(start_val::AbstractDict, model::AbstractSem; kwargs...) - return [start_val[param] for param in params(model)] # convert to a vector + # convert to a vector + return prepare_start_params( + [start_val[param] for param in params(model)], + model; + kwargs..., + ) end # get from the ParameterTable (potentially from a different model with match param names) From f430ec25b57fd5e906c502d1289cf5d968d48292 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 11:03:51 -0700 Subject: [PATCH 306/364] SemImplied/SemLossFun: drop meanstructure kwarg - for SemImplied require spec::SemSpec as positional - for SemLossFunction require implied argument --- src/frontend/specification/Sem.jl | 10 +++++- src/implied/RAM/generic.jl | 24 +++++-------- src/implied/RAM/symbolic.jl | 32 ++++++----------- src/implied/abstract.jl | 14 -------- src/loss/ML/FIML.jl | 11 +++--- src/loss/ML/ML.jl | 3 +- src/loss/WLS/WLS.jl | 27 ++++++++------ .../recover_parameters_twofact.jl | 2 +- test/unit_tests/model.jl | 35 ++++--------------- 9 files changed, 59 insertions(+), 99 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 53858abd4..a47bad4bf 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -109,7 +109,15 @@ function get_fields!(kwargs, specification, observed, implied, loss) # implied if !isa(implied, SemImplied) - implied = implied(; specification, kwargs...) + # FIXME remove this implicit logic + # SemWLS only accepts vech-ed implied covariance + if isa(loss, Type) && (loss <: SemWLS) && !haskey(kwargs, :vech) + implied_kwargs = copy(kwargs) + implied_kwargs[:vech] = true + else + implied_kwargs = kwargs + end + implied = implied(specification; implied_kwargs...) end kwargs[:implied] = implied diff --git a/src/implied/RAM/generic.jl b/src/implied/RAM/generic.jl index 4c1fa323c..d57500a36 100644 --- a/src/implied/RAM/generic.jl +++ b/src/implied/RAM/generic.jl @@ -6,14 +6,10 @@ Model implied covariance and means via RAM notation. # Constructor - RAM(;specification, - meanstructure = false, - gradient = true, - kwargs...) + RAM(; specification, gradient = true, kwargs...) # Arguments - `specification`: either a `RAMMatrices` or `ParameterTable` object -- `meanstructure::Bool`: does the model have a meanstructure? - `gradient::Bool`: is gradient-based optimization used # Extended help @@ -53,9 +49,9 @@ Vector of indices of each parameter in the respective RAM matrix: - `ram.M_indices` Additional interfaces -- `ram.F⨉I_A⁻¹` -> ``F(I-A)^{-1}`` -- `ram.F⨉I_A⁻¹S` -> ``F(I-A)^{-1}S`` -- `ram.I_A` -> ``I-A`` +- `F⨉I_A⁻¹(::RAM)` -> ``F(I-A)^{-1}`` +- `F⨉I_A⁻¹S(::RAM)` -> ``F(I-A)^{-1}S`` +- `I_A(::RAM)` -> ``I-A`` Only available in gradient! calls: - `ram.I_A⁻¹` -> ``(I-A)^{-1}`` @@ -90,15 +86,13 @@ end ### Constructors ############################################################################################ -function RAM(; - specification::SemSpecification, +function RAM( + spec::SemSpecification; + #vech = false, gradient_required = true, - meanstructure = false, kwargs..., ) - ram_matrices = convert(RAMMatrices, specification) - - check_meanstructure_specification(meanstructure, ram_matrices) + ram_matrices = convert(RAMMatrices, spec) # get dimensions of the model n_par = nparams(ram_matrices) @@ -126,7 +120,7 @@ function RAM(; end # μ - if meanstructure + if !isnothing(ram_matrices.M) MS = HasMeanStruct M_pre = materialize(ram_matrices.M, rand_params) ∇M = gradient_required ? sparse_gradient(ram_matrices.M) : nothing diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index df7c497ad..0f7868dae 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -12,12 +12,10 @@ Subtype of `SemImplied` that implements the RAM notation with symbolic precomput gradient = true, hessian = false, approximate_hessian = false, - meanstructure = false, kwargs...) # Arguments - `specification`: either a `RAMMatrices` or `ParameterTable` object -- `meanstructure::Bool`: does the model have a meanstructure? - `gradient::Bool`: is gradient-based optimization used - `hessian::Bool`: is hessian-based optimization used - `approximate_hessian::Bool`: for hessian based optimization: should the hessian be approximated @@ -79,20 +77,16 @@ end ### Constructors ############################################################################################ -function RAMSymbolic(; - specification::SemSpecification, - loss_types = nothing, - vech = false, - simplify_symbolics = false, - gradient = true, - hessian = false, - meanstructure = false, - approximate_hessian = false, +function RAMSymbolic( + spec::SemSpecification; + vech::Bool = false, + simplify_symbolics::Bool = false, + gradient::Bool = true, + hessian::Bool = false, + approximate_hessian::Bool = false, kwargs..., ) - ram_matrices = convert(RAMMatrices, specification) - - check_meanstructure_specification(meanstructure, ram_matrices) + ram_matrices = convert(RAMMatrices, spec) n_par = nparams(ram_matrices) par = (Symbolics.@variables θ[1:n_par])[1] @@ -102,10 +96,6 @@ function RAMSymbolic(; M = !isnothing(ram_matrices.M) ? materialize(Num, ram_matrices.M, par) : nothing F = ram_matrices.F - if !isnothing(loss_types) && any(T -> T <: SemWLS, loss_types) - vech = true - end - I_A⁻¹ = neumann_series(A) # Σ @@ -146,7 +136,7 @@ function RAMSymbolic(; end # μ - if meanstructure + if !isnothing(ram_matrices.M) MS = HasMeanStruct μ_sym = eval_μ_symbolic(M, I_A⁻¹, F; simplify = simplify_symbolics) μ_eval! = Symbolics.build_function(μ_sym, par, expression = Val{false})[2] @@ -230,10 +220,10 @@ end ############################################################################################ # expected covariations of observed vars -function eval_Σ_symbolic(S, I_A⁻¹, F; vech = false, simplify = false) +function eval_Σ_symbolic(S, I_A⁻¹, F; vech::Bool = false, simplify::Bool = false) Σ = F * I_A⁻¹ * S * permutedims(I_A⁻¹) * permutedims(F) Σ = Array(Σ) - vech && (Σ = Σ[tril(trues(size(F, 1), size(F, 1)))]) + vech && (Σ = SEM.vech(Σ)) if simplify Threads.@threads for i in eachindex(Σ) Σ[i] = Symbolics.simplify(Σ[i]) diff --git a/src/implied/abstract.jl b/src/implied/abstract.jl index 6d298f65c..d4868d746 100644 --- a/src/implied/abstract.jl +++ b/src/implied/abstract.jl @@ -31,17 +31,3 @@ function check_acyclic(A::AbstractMatrix; verbose::Bool = false) return A end end - -# Verify that the `meanstructure` argument aligns with the model specification. -function check_meanstructure_specification(meanstructure, ram_matrices) - if meanstructure & isnothing(ram_matrices.M) - throw(ArgumentError( - "You set `meanstructure = true`, but your model specification contains no mean parameters." - )) - end - if !meanstructure & !isnothing(ram_matrices.M) - throw(ArgumentError( - "If your model specification contains mean parameters, you have to set `Sem(..., meanstructure = true)`." - )) - end -end \ No newline at end of file diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index da5ccb7c6..8572b15ac 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -75,15 +75,16 @@ Can handle observed data with missing values. # Constructor - SemFIML(; observed::SemObservedMissing, specification, kwargs...) + SemFIML(; observed::SemObservedMissing, implied::SemImplied, kwargs...) # Arguments -- `observed`: the observed data with missing values (see [`SemObservedMissing`](@ref)) -- `specification`: [`SemSpecification`](@ref) object +- `observed::SemObservedMissing`: the observed part of the model + (see [`SemObservedMissing`](@ref)) +- `implied::SemImplied`: the implied part of the model # Examples ```julia -my_fiml = SemFIML(observed = my_observed, specification = my_parameter_table) +my_fiml = SemFIML(observed = my_observed, implied = my_implied) ``` # Interfaces @@ -118,7 +119,7 @@ function SemFIML(; observed::SemObservedMissing, implied, specification, kwargs. ExactHessian(), [SemFIMLPattern(pat) for pat in observed.patterns], zeros(nobserved_vars(observed), nobserved_vars(observed)), - CommutationMatrix(nvars(specification)), + CommutationMatrix(nvars(implied)), nothing, ) end diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index ce77ea9c3..aae1dadae 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -8,11 +8,10 @@ Maximum likelihood estimation. # Constructor - SemML(;observed, meanstructure = false, approximate_hessian = false, kwargs...) + SemML(; observed, approximate_hessian = false, kwargs...) # Arguments - `observed::SemObserved`: the observed part of the model -- `meanstructure::Bool`: does the model have a meanstructure? - `approximate_hessian::Bool`: if hessian-based optimization is used, should the hessian be swapped for an approximation # Examples diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index b7f66d558..9de011f60 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -10,8 +10,7 @@ At the moment only available with the `RAMSymbolic` implied type. # Constructor SemWLS(; - observed, - meanstructure = false, + observed, implied, wls_weight_matrix = nothing, wls_weight_matrix_mean = nothing, approximate_hessian = false, @@ -19,7 +18,7 @@ At the moment only available with the `RAMSymbolic` implied type. # Arguments - `observed`: the `SemObserved` part of the model -- `meanstructure::Bool`: does the model have a meanstructure? +- `implied::SemImplied`: the implied part of the model - `approximate_hessian::Bool`: should the hessian be swapped for an approximation - `wls_weight_matrix`: the weight matrix for weighted least squares. Defaults to GLS estimation (``0.5*(D^T*kron(S,S)*D)`` where D is the duplication matrix @@ -29,7 +28,7 @@ At the moment only available with the `RAMSymbolic` implied type. # Examples ```julia -my_wls = SemWLS(observed = my_observed) +my_wls = SemWLS(observed = my_observed, implied = my_implied) ``` # Interfaces @@ -50,12 +49,11 @@ SemWLS{HE}(args...) where {HE <: HessianEval} = SemWLS{HE, map(typeof, args)...}(HE(), args...) function SemWLS(; - observed, - implied, - wls_weight_matrix = nothing, - wls_weight_matrix_mean = nothing, - approximate_hessian = false, - meanstructure = false, + observed::SemObserved, + implied::SemImplied, + wls_weight_matrix::Union{AbstractMatrix, Nothing} = nothing, + wls_weight_matrix_mean::Union{AbstractMatrix, Nothing} = nothing, + approximate_hessian::Bool = false, kwargs..., ) if observed isa SemObservedMissing @@ -81,6 +79,10 @@ function SemWLS(; nobs_vars = nobserved_vars(observed) tril_ind = filter(x -> (x[1] >= x[2]), CartesianIndices(obs_cov(observed))) s = obs_cov(observed)[tril_ind] + size(s) == size(implied.Σ) || + throw(DimensionMismatch("SemWLS requires implied covariance to be in vech-ed form " * + "(vectorized lower triangular part of Σ matrix): $(size(s)) expected, $(size(implied.Σ)) found.\n" * + "$(nameof(typeof(implied))) must be constructed with vech=true.")) # compute V here if isnothing(wls_weight_matrix) @@ -94,9 +96,12 @@ function SemWLS(; "wls_weight_matrix has to be of size $(length(tril_ind))×$(length(tril_ind))", ) end + size(wls_weight_matrix) == (length(s), length(s)) || + DimensionMismatch("wls_weight_matrix has to be of size $(length(s))×$(length(s))") - if meanstructure + if MeanStruct(implied) == HasMeanStruct if isnothing(wls_weight_matrix_mean) + @warn "Computing WLS weight matrix for the meanstructure using obs_cov()" wls_weight_matrix_mean = inv(obs_cov(observed)) else size(wls_weight_matrix_mean) == (nobs_vars, nobs_vars) || DimensionMismatch( diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 9f9503af8..a4bd7d5ff 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -53,7 +53,7 @@ start = [ repeat([0.5], 4) ] -implied_ml = RAMSymbolic(; specification = ram_matrices, start_val = start) +implied_ml = RAMSymbolic(ram_matrices; start_val = start) implied_ml.Σ_eval!(implied_ml.Σ, true_val) diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index d9f9254be..fbe2a937f 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -46,13 +46,16 @@ function test_params_api(semobj, spec::SemSpecification) @test @inferred(param_labels(semobj)) == param_labels(spec) end -@testset "Sem(implied=$impliedtype, loss=SemML)" for impliedtype in (RAM, RAMSymbolic) - +@testset "Sem(implied=$impliedtype, loss=$losstype)" for (impliedtype, losstype) in [ + (RAM, SemML), + (RAMSymbolic, SemML), + (RAMSymbolic, SemWLS), +] model = Sem( specification = ram_matrices, observed = obs, implied = impliedtype, - loss = SemML, + loss = losstype, ) @test model isa Sem @@ -71,29 +74,3 @@ end @test @inferred(nsamples(model)) == nsamples(obs) end - -@testset "Sem(implied=RAMSymbolic, loss=SemWLS)" begin - - model = Sem( - specification = ram_matrices, - observed = obs, - implied = RAMSymbolic, - loss = SemWLS, - ) - - @test model isa Sem - @test @inferred(implied(model)) isa RAMSymbolic - @test @inferred(observed(model)) isa SemObserved - - test_vars_api(model, ram_matrices) - test_params_api(model, ram_matrices) - - test_vars_api(implied(model), ram_matrices) - test_params_api(implied(model), ram_matrices) - - @test @inferred(loss(model)) isa SemLoss - semloss = loss(model).functions[1] - @test semloss isa SemWLS - - @test @inferred(nsamples(model)) == nsamples(obs) -end \ No newline at end of file From 6031dcc00a34159134139a39f5ce425e20dddac7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 11:17:30 -0700 Subject: [PATCH 307/364] refactor Sem, SemEnsemble, SemLoss --- src/StructuralEquationModels.jl | 17 +- src/additional_functions/simulation.jl | 87 ++-- src/additional_functions/start_val/common.jl | 17 + .../start_val/start_fabin3.jl | 16 +- .../start_val/start_simple.jl | 34 +- src/frontend/finite_diff.jl | 35 ++ src/frontend/fit/fitmeasures/RMSEA.jl | 32 +- src/frontend/fit/fitmeasures/chi2.jl | 73 ++- src/frontend/fit/fitmeasures/dof.jl | 11 +- src/frontend/fit/fitmeasures/minus2ll.jl | 34 +- src/frontend/fit/standard_errors/hessian.jl | 25 +- src/frontend/pretty_printing.jl | 8 +- src/frontend/specification/Sem.jl | 439 +++++++++++++----- src/implied/RAM/generic.jl | 13 +- src/implied/RAM/symbolic.jl | 12 +- src/implied/empty.jl | 6 +- src/loss/ML/FIML.jl | 86 ++-- src/loss/ML/ML.jl | 90 ++-- src/loss/WLS/WLS.jl | 94 ++-- src/loss/abstract.jl | 42 ++ src/loss/constant/constant.jl | 28 +- src/loss/regularization/ridge.jl | 9 +- src/objective_gradient_hessian.jl | 251 +++++----- src/optimizer/abstract.jl | 14 +- src/types.jl | 243 +++------- test/examples/multigroup/build_models.jl | 222 +++------ test/examples/political_democracy/by_parts.jl | 284 +++++------ .../political_democracy/constraints.jl | 4 +- .../political_democracy/constructor.jl | 169 +++---- .../recover_parameters_twofact.jl | 24 +- test/unit_tests/model.jl | 5 +- 31 files changed, 1203 insertions(+), 1221 deletions(-) create mode 100644 src/additional_functions/start_val/common.jl create mode 100644 src/frontend/finite_diff.jl create mode 100644 src/loss/abstract.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 19dd6f43a..d98e79258 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -44,6 +44,7 @@ include("frontend/specification/EnsembleParameterTable.jl") include("frontend/specification/StenoGraphs.jl") include("frontend/fit/summary.jl") include("frontend/StatsAPI.jl") +include("frontend/finite_diff.jl") # pretty printing include("frontend/pretty_printing.jl") # observed @@ -53,26 +54,28 @@ include("observed/covariance.jl") include("observed/missing_pattern.jl") include("observed/missing.jl") include("observed/EM.jl") -# constructor -include("frontend/specification/Sem.jl") -include("frontend/specification/documentation.jl") # implied include("implied/abstract.jl") include("implied/RAM/symbolic.jl") include("implied/RAM/generic.jl") include("implied/empty.jl") # loss +include("loss/abstract.jl") include("loss/ML/ML.jl") include("loss/ML/FIML.jl") include("loss/regularization/ridge.jl") include("loss/WLS/WLS.jl") include("loss/constant/constant.jl") +# constructor +include("frontend/specification/Sem.jl") +include("frontend/specification/documentation.jl") # optimizer include("optimizer/abstract.jl") include("optimizer/Empty.jl") include("optimizer/optim.jl") # helper functions include("additional_functions/helper.jl") +include("additional_functions/start_val/common.jl") include("additional_functions/start_val/start_fabin3.jl") include("additional_functions/start_val/start_simple.jl") include("additional_functions/artifacts.jl") @@ -94,14 +97,11 @@ include("frontend/fit/standard_errors/z_test.jl") include("frontend/fit/standard_errors/confidence_intervals.jl") export AbstractSem, - AbstractSemSingle, - AbstractSemCollection, coef, coefnames, coeftable, Sem, SemFiniteDiff, - SemEnsemble, MeanStruct, NoMeanStruct, HasMeanStruct, @@ -116,8 +116,8 @@ export AbstractSem, start_val, start_fabin3, start_simple, + AbstractLoss, SemLoss, - SemLossFunction, SemML, SemFIML, em_mvn, @@ -125,6 +125,9 @@ export AbstractSem, SemConstant, SemWLS, loss, + nsem_terms, + sem_terms, + sem_term, SemOptimizer, optimizer, optimizer_engine, diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 4839bc272..6d694c972 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -43,36 +43,42 @@ function update_observed end # change observed (data) without reconstructing the whole model ############################################################################################ +# don't change non-SEM terms +replace_observed(loss::AbstractLoss; kwargs...) = loss + # use the same observed type as before -replace_observed(model::AbstractSemSingle; kwargs...) = - replace_observed(model, typeof(observed(model)).name.wrapper; kwargs...) +replace_observed(loss::SemLoss; kwargs...) = + replace_observed(loss, typeof(SEM.observed(loss)).name.wrapper; kwargs...) + +# construct a new observed type +replace_observed(loss::SemLoss, observed_type; kwargs...) = + replace_observed(loss, observed_type(; kwargs...); kwargs...) -function replace_observed(model::AbstractSemSingle, observed_type; kwargs...) - new_observed = observed_type(; kwargs...) +function replace_observed(loss::SemLoss, new_observed::SemObserved; kwargs...) kwargs = Dict{Symbol, Any}(kwargs...) + old_observed = SEM.observed(loss) + implied = SEM.implied(loss) # get field types kwargs[:observed_type] = typeof(new_observed) - kwargs[:old_observed_type] = typeof(model.observed) - kwargs[:implied_type] = typeof(model.implied) - kwargs[:loss_types] = [typeof(lossfun) for lossfun in model.loss.functions] + kwargs[:old_observed_type] = typeof(old_observed) # update implied - new_implied = update_observed(model.implied, new_observed; kwargs...) + new_implied = update_observed(implied, new_observed; kwargs...) kwargs[:implied] = new_implied + kwargs[:implied_type] = typeof(new_implied) kwargs[:nparams] = nparams(new_implied) # update loss - new_loss = update_observed(model.loss, new_observed; kwargs...) - - return Sem(new_observed, new_implied, new_loss) + return update_observed(loss, new_observed; kwargs...) end -function update_observed(loss::SemLoss, new_observed; kwargs...) - new_functions = Tuple( - update_observed(lossfun, new_observed; kwargs...) for lossfun in loss.functions - ) - return SemLoss(new_functions, loss.weights) +replace_observed(loss::LossTerm; kwargs...) = + LossTerm(replace_observed(loss.loss; kwargs...), loss.id, loss.weight) + +function replace_observed(sem::Sem; kwargs...) + updated_terms = Tuple(replace_observed(term; kwargs...) for term in loss_terms(sem)) + return Sem(updated_terms...) end function replace_observed( @@ -111,39 +117,38 @@ end # simulate data ############################################################################################ """ - (1) rand(model::AbstractSemSingle, params, n) - - (2) rand(model::AbstractSemSingle, n) + rand(sem::Union{Sem, SemLoss, SemImplied}, [params], n) -Sample normally distributed data from the model-implied covariance matrix and mean vector. +Sample from the multivariate normal distribution implied by the SEM model. # Arguments -- `model::AbstractSemSingle`: model to simulate from. -- `params`: parameter values to simulate from. -- `n::Integer`: Number of samples. +- `sem`: SEM model to use. Ensemble models with multiple SEM terms are not supported. +- `params`: optional SEM model parameters to simulate from, otherwise uses the + current state of implied covariances and means. +- `n::Integer`: Number of samples to draw. # Examples ```julia rand(model, start_simple(model), 100) ``` """ -function Distributions.rand( - model::AbstractSemSingle{O, I, L}, - params, - n::Integer, -) where {O, I <: Union{RAM, RAMSymbolic}, L} - update!(EvaluationTargets{true, false, false}(), model.implied, model, params) - return rand(model, n) -end - -function Distributions.rand( - model::AbstractSemSingle{O, I, L}, - n::Integer, -) where {O, I <: Union{RAM, RAMSymbolic}, L} - if MeanStruct(model.implied) === NoMeanStruct - data = permutedims(rand(MvNormal(Symmetric(model.implied.Σ)), n)) - elseif MeanStruct(model.implied) === HasMeanStruct - data = permutedims(rand(MvNormal(model.implied.μ, Symmetric(model.implied.Σ)), n)) +function Distributions.rand(implied::SemImplied, params, n::Integer) + if !isnothing(params) + # update the implied covariances with the new model params + update!(EvaluationTargets{true, false, false}(), implied, params) + end + Σ = Symmetric(implied.Σ) + if MeanStruct(implied) === NoMeanStruct + return permutedims(rand(MvNormal(Σ), n)) + elseif MeanStruct(implied) === HasMeanStruct + return permutedims(rand(MvNormal(implied.μ, Σ), n)) end - return data end + +Distributions.rand(loss::SemLoss, params, n::Integer) = rand(SEM.implied(loss), params, n) + +Distributions.rand(model::Sem, params, n::Integer) = rand(sem_term(model), params, n) + +# rand() overloads without SEM params +Distributions.rand(implied::Union{SemImplied, SemLoss, Sem}, n::Integer) = + Distributions.rand(implied, nothing, n) diff --git a/src/additional_functions/start_val/common.jl b/src/additional_functions/start_val/common.jl new file mode 100644 index 000000000..92c85d6f5 --- /dev/null +++ b/src/additional_functions/start_val/common.jl @@ -0,0 +1,17 @@ + +# start values for SEM Models (including ensembles) +function start_values(f, model::AbstractSem; kwargs...) + start_vals = fill(0.0, nparams(model)) + + # initialize parameters using the SEM loss terms + # (first SEM loss term that sets given parameter to nonzero value) + for term in loss_terms(model) + issemloss(term) || continue + term_start_vals = f(loss(term); kwargs...) + for (i, val) in enumerate(term_start_vals) + iszero(val) || (start_vals[i] = val) + end + end + + return start_vals +end diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 53d3442a7..543370285 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -7,12 +7,17 @@ Not available for ensemble models. function start_fabin3 end # splice model and loss functions -function start_fabin3(model::AbstractSemSingle; kwargs...) - return start_fabin3(model.observed, model.implied, model.loss.functions..., kwargs...) +function start_fabin3(model::SemLoss; kwargs...) + return start_fabin3(model.observed, model.implied; kwargs...) end -function start_fabin3(observed::SemObserved, implied::SemImplied, args...; kwargs...) - return start_fabin3(implied.ram_matrices, obs_cov(observed), obs_mean(observed)) +function start_fabin3(observed::SemObserved, implied::SemImplied; kwargs...) + return start_fabin3( + implied.ram_matrices, + obs_cov(observed), + # ignore observed means if no meansturcture + !isnothing(implied.ram_matrices.M) ? obs_mean(observed) : nothing, + ) end function start_fabin3( @@ -161,3 +166,6 @@ end function is_in_Λ(ind_vec, F_ind) return any(ind -> !(ind[2] ∈ F_ind) && (ind[1] ∈ F_ind), ind_vec) end + +# ensembles +start_fabin3(model::AbstractSem; kwargs...) = start_values(start_fabin3, model; kwargs...) diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 4fbc8719c..afdbf92ee 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -15,34 +15,11 @@ Return a vector of simple starting values. """ function start_simple end -# Single Models ---------------------------------------------------------------------------- -function start_simple(model::AbstractSemSingle; kwargs...) - return start_simple(model.observed, model.implied, model.loss.functions...; kwargs...) -end - -function start_simple(observed, implied, args...; kwargs...) - return start_simple(implied.ram_matrices; kwargs...) -end - -# Ensemble Models -------------------------------------------------------------------------- -function start_simple(model::SemEnsemble; kwargs...) - start_vals = [] - - for sem in model.sems - push!(start_vals, start_simple(sem; kwargs...)) - end - - has_start_val = [.!iszero.(start_val) for start_val in start_vals] +start_simple(model::SemLoss; kwargs...) = + start_simple(observed(model), implied(model); kwargs...) - start_val = similar(start_vals[1]) - start_val .= 0.0 - - for (j, indices) in enumerate(has_start_val) - start_val[indices] .= start_vals[j][indices] - end - - return start_val -end +start_simple(observed::SemObserved, implied::SemImplied; kwargs...) = + start_simple(implied.ram_matrices; kwargs...) function start_simple( ram_matrices::RAMMatrices; @@ -103,3 +80,6 @@ function start_simple( end return start_val end + +# multigroup models +start_simple(model::AbstractSem; kwargs...) = start_values(start_simple, model; kwargs...) diff --git a/src/frontend/finite_diff.jl b/src/frontend/finite_diff.jl new file mode 100644 index 000000000..ee0a9bf96 --- /dev/null +++ b/src/frontend/finite_diff.jl @@ -0,0 +1,35 @@ +_unwrap(wrapper::SemFiniteDiff) = wrapper.model +params(wrapper::SemFiniteDiff) = params(wrapper.model) +loss_terms(wrapper::SemFiniteDiff) = loss_terms(wrapper.model) + +FiniteDiffLossWrappers = Union{LossFiniteDiff, SemLossFiniteDiff} + +_unwrap(term::AbstractLoss) = term +_unwrap(wrapper::FiniteDiffLossWrappers) = wrapper.loss +implied(wrapper::FiniteDiffLossWrappers) = implied(_unwrap(wrapper)) +observed(wrapper::FiniteDiffLossWrappers) = observed(_unwrap(wrapper)) + +FiniteDiffWrapper(model::AbstractSem) = SemFiniteDiff(model) +FiniteDiffWrapper(loss::AbstractLoss) = LossFiniteDiff(loss) +FiniteDiffWrapper(loss::SemLoss) = SemLossFiniteDiff(loss) + +function evaluate!( + objective, + gradient, + hessian, + sem::Union{SemFiniteDiff, FiniteDiffLossWrappers}, + params, +) + wrapped = _unwrap(sem) + obj(p) = _evaluate!( + objective_zero(objective, gradient, hessian), + nothing, + nothing, + wrapped, + p, + ) + isnothing(gradient) || FiniteDiff.finite_difference_gradient!(gradient, obj, params) + isnothing(hessian) || FiniteDiff.finite_difference_hessian!(hessian, obj, params) + # FIXME if objective is not calculated, SemLoss implied states may not correspond to params + return !isnothing(objective) ? obj(params) : nothing +end diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index 8539896f4..7406b74cd 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -19,28 +19,18 @@ for the SEM model. For multigroup models, the correction proposed by J.H. Steiger is applied (see [Steiger, J. H. (1998). *A note on multiple sample extensions of the RMSEA fit index*](https://doi.org/10.1080/10705519809540115)). """ -function RMSEA end - RMSEA(fit::SemFit) = RMSEA(fit, fit.model) -function RMSEA(fit::SemFit, model::AbstractSemSingle) - check_single_lossfun(model; throw_error = true) - return RMSEA(dof(fit), χ²(fit), nsamples(fit)+rmsea_correction(model.loss.functions[1])) -end - -function RMSEA(fit::SemFit, model::SemEnsemble) - check_single_lossfun(model; throw_error = true) - n = nsamples(fit)+model.n*rmsea_correction(model.sems[1].loss.functions[1]) - return sqrt(length(model.sems)) * RMSEA(dof(fit), χ²(fit), n) -end - -function RMSEA(dof, chi2, N⁻) - rmsea = (chi2 - dof) / (N⁻ * dof) - rmsea = rmsea > 0 ? rmsea : 0 - return sqrt(rmsea) +# scaling corrections +RMSEA_corr_scale(::Type{<:SemFIML}) = 0 +RMSEA_corr_scale(::Type{<:SemML}) = -1 +RMSEA_corr_scale(::Type{<:SemWLS}) = -1 + +function RMSEA(fit::SemFit, model::AbstractSem) + term_type = check_single_lossfun(model; throw_error = true) + n = nsamples(fit) + nsem_terms(model) * RMSEA_corr_scale(term_type) + sqrt(nsem_terms(model)) * RMSEA(dof(fit), χ²(fit), n) end -# scaling corrections -rmsea_correction(::SemFIML) = 0 -rmsea_correction(::SemML) = -1 -rmsea_correction(::SemWLS) = -1 +RMSEA(dof::Number, chi2::Number, nsamples::Number) = + sqrt(max((chi2 - dof) / (nsamples * dof), 0.0)) diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index 8ce5f0798..22d6c2e26 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -12,57 +12,42 @@ with the *observed* covariance matrix. """ χ²(fit::SemFit) = χ²(fit, fit.model) -############################################################################################ -# Single Models -############################################################################################ +function χ²(fit::SemFit, model::AbstractSem) + terms = sem_terms(model) + isempty(terms) && return 0.0 + + term1 = _unwrap(loss(terms[1])) + L = typeof(term1).name + + # check that all SemLoss terms are of the same class (ML, FIML, WLS etc), ignore typeparams + for (i, term) in enumerate(terms) + lossterm = _unwrap(loss(term)) + @assert lossterm isa SemLoss + if typeof(_unwrap(lossterm)).name != L + @error "SemLoss term #$i is $(typeof(_unwrap(lossterm)).name), expected $L. Heterogeneous loss functions are not supported" + end + end -function χ²(fit::SemFit, model::AbstractSemSingle) - check_single_lossfun(model; throw_error = true) - return χ²(model.loss.functions[1], fit::SemFit, model::AbstractSemSingle) + return χ²(typeof(term1), fit, model) end -χ²(::SemML, fit::SemFit, model::AbstractSemSingle) = - (nsamples(fit) - 1) * - (fit.minimum - logdet(obs_cov(observed(model))) - nobserved_vars(model)) - # bollen, p. 115, only correct for GLS weight matrix -χ²(::SemWLS, fit::SemFit, model::AbstractSemSingle) = - (nsamples(fit) - 1) * fit.minimum - -# FIML -function χ²(::SemFIML, fit::SemFit, model::AbstractSemSingle) - ll_H0 = minus2ll(fit) - ll_H1 = minus2ll(observed(model)) - return ll_H0 - ll_H1 -end - -############################################################################################ -# Collections -############################################################################################ - -function χ²(fit::SemFit, model::SemEnsemble) - check_single_lossfun(model; throw_error = true) - lossfun = model.sems[1].loss.functions[1] - return χ²(lossfun, fit, model) -end - -function χ²(::SemWLS, fit::SemFit, models::SemEnsemble) - return (nsamples(models) - models.n) * fit.minimum -end - -function χ²(::SemML, fit::SemFit, models::SemEnsemble) - F = 0 - for model in models.sems - Fᵢ = objective(model, fit.solution) - Fᵢ -= logdet(obs_cov(observed(model))) + nobserved_vars(model) - Fᵢ *= nsamples(model) - 1 - F += Fᵢ +χ²(::Type{<:SemWLS}, fit::SemFit, model::AbstractSem) = (nsamples(model) - 1) * fit.minimum + +function χ²(::Type{<:SemML}, fit::SemFit, model::AbstractSem) + G = sum(loss_terms(model)) do term + if issemloss(term) + data = observed(term) + something(weight(term), 1.0) * (logdet(obs_cov(data)) + nobserved_vars(data)) + else + return 0.0 + end end - return F + return (nsamples(model) - 1) * (fit.minimum - G) end -function χ²(::SemFIML, fit::SemFit, models::SemEnsemble) +function χ²(::Type{<:SemFIML}, fit::SemFit, model::AbstractSem) ll_H0 = minus2ll(fit) - ll_H1 = sum(minus2ll ∘ observed, models.sems) + ll_H1 = sum(minus2ll ∘ observed, sem_terms(model)) return ll_H0 - ll_H1 end diff --git a/src/frontend/fit/fitmeasures/dof.jl b/src/frontend/fit/fitmeasures/dof.jl index 0e051d02a..49b7febf1 100644 --- a/src/frontend/fit/fitmeasures/dof.jl +++ b/src/frontend/fit/fitmeasures/dof.jl @@ -18,13 +18,16 @@ dof(fit::SemFit) = dof(fit.model) dof(model::AbstractSem) = n_dp(model) - nparams(model) -function n_dp(model::AbstractSemSingle) - nvars = nobserved_vars(model) +# length of Σ and μ (if present) +function n_dp(implied::SemImplied) + nvars = nobserved_vars(implied) ndp = 0.5(nvars^2 + nvars) - if !isnothing(model.implied.μ) + if !isnothing(implied.μ) ndp += nvars end return ndp end -n_dp(model::SemEnsemble) = sum(n_dp.(model.sems)) +n_dp(term::SemLoss) = n_dp(implied(term)) + +n_dp(model::AbstractSem) = sum(n_dp ∘ loss, sem_terms(model)) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index c6a954efa..3b353f5c7 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -6,31 +6,27 @@ Calculate the *-2⋅log(likelihood(fit))*. # See also [`fit_measures`](@ref) """ -minus2ll(fit::SemFit) = minus2ll(fit, fit.model) +minus2ll(fit::SemFit) = minus2ll(fit.model, fit) ############################################################################################ -# Single Models +# Single SEM Terms Models ############################################################################################ -function minus2ll(fit::SemFit, model::AbstractSemSingle) - check_single_lossfun(model; throw_error = true) - F = objective(model, fit.solution) - return minus2ll(model.loss.functions[1], F, model) +function minus2ll(term::SemLoss, fit::SemFit) + minimum = objective(term, fit.solution) + return minus2ll(term, minimum) end -# SemML ------------------------------------------------------------------------------------ -function minus2ll(::SemML, F, model::AbstractSemSingle) - return nsamples(model) * (F + log(2π) * nobserved_vars(model)) -end +minus2ll(term::SemML, minimum::Number) = + nsamples(term) * (minimum + log(2π) * nobserved_vars(term)) -# WLS -------------------------------------------------------------------------------------- -minus2ll(::SemWLS, F, ::AbstractSemSingle) = missing +minus2ll(term::SemWLS, minimum::Number) = missing # compute likelihood for missing data - H0 ------------------------------------------------- -# -2ll = (∑ log(2π)*(nᵢ*mᵢ)) + F*n -function minus2ll(::SemFIML, F, model::AbstractSemSingle) - obs = observed(model)::SemObservedMissing - F *= nsamples(obs) +# -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n +function minus2ll(term::SemFIML, minimum::Number) + obs = observed(term)::SemObservedMissing + F = minimum * nsamples(obs) F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), obs.patterns) return F end @@ -62,10 +58,10 @@ function minus2ll(observed::SemObservedMissing) end ############################################################################################ -# Collection +# Multi-group ############################################################################################ -function minus2ll(fit::SemFit, model::SemEnsemble) +function minus2ll(model::AbstractSem, fit::SemFit) check_single_lossfun(model; throw_error = true) - return sum(Base.Fix1(minus2ll, fit), model.sems) + sum(Base.Fix2(minus2ll, fit) ∘ _unwrap ∘ loss, sem_terms(model)) end diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index 6ae53407f..80b96d337 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -35,20 +35,21 @@ function se_hessian(fit::SemFit; method = :finitediff) end # Addition functions ------------------------------------------------------------- -function H_scaling(model::AbstractSemSingle) - if length(model.loss.functions) > 1 - @warn "Hessian scaling for multiple loss functions is not implemented yet" - end - return H_scaling(model.loss.functions[1], model) -end - -H_scaling(lossfun::SemML, model::AbstractSemSingle) = 2 / (nsamples(model) - 1) +H_scaling(loss::SemML) = 2 / (nsamples(loss) - 1) -function H_scaling(lossfun::SemWLS, model::AbstractSemSingle) +function H_scaling(loss::SemWLS) @warn "Standard errors for WLS are only correct if a GLS weight matrix (the default) is used." - return 2 / (nsamples(model) - 1) + return 2 / (nsamples(loss) - 1) end -H_scaling(lossfun::SemFIML, model::AbstractSemSingle) = 2 / nsamples(model) +H_scaling(loss::SemFIML) = 2 / nsamples(loss) -H_scaling(model::SemEnsemble) = 2 / nsamples(model) +function H_scaling(model::AbstractSem) + semterms = SEM.sem_terms(model) + if length(semterms) > 1 + #@warn "Hessian scaling for multiple loss functions is not implemented yet" + return 2 / nsamples(model) + else + return length(semterms) >= 1 ? H_scaling(loss(semterms[1])) : 1.0 + end +end diff --git a/src/frontend/pretty_printing.jl b/src/frontend/pretty_printing.jl index 2fa970f24..7b6975f68 100644 --- a/src/frontend/pretty_printing.jl +++ b/src/frontend/pretty_printing.jl @@ -32,9 +32,11 @@ end # Loss Function, Implied, Observed, Optimizer ############################################################## -function Base.show(io::IO, struct_inst::SemLossFunction) - print_type_name(io, struct_inst) - print_field_types(io, struct_inst) +function Base.show(io::IO, sem::SemLoss) + println(io, "Structural Equation Model Loss ($(nameof(typeof(sem))))") + println(io, "- Observed: $(nameof(typeof(observed(sem)))) ($(nsamples(sem)) samples)") + println(io, "- Implied: $(nameof(typeof(implied(sem)))) ($(nparams(sem)) parameters)") + println(io, "- Variables: $(nobserved_vars(sem)) observed, $(nlatent_vars(sem)) latent") end function Base.show(io::IO, struct_inst::SemImplied) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index a47bad4bf..684cfa62b 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -1,7 +1,167 @@ +losstype(::Type{<:LossTerm{L, W, I}}) where {L, W, I} = L +losstype(term::LossTerm) = losstype(typeof(term)) +loss(term::LossTerm) = term.loss +weight(term::LossTerm) = term.weight +id(term::LossTerm) = term.id + +""" + issemloss(term::LossTerm) -> Bool + +Check if a SEM model term is a SEM loss function ([`SemLoss`](@ref)). +""" +issemloss(term::LossTerm) = isa(loss(term), SemLoss) + +for f in ( + :implied, + :observed, + :nsamples, + :observed_vars, + :nobserved_vars, + :vars, + :nvars, + :latent_vars, + :nlatent_vars, + :params, + :nparams, +) + @eval $f(term::LossTerm) = $f(loss(term)) +end + +function Base.show(io::IO, term::LossTerm) + if !isnothing(id(term)) + print(io, ":$(id(term)): ") + end + print(io, nameof(losstype(term))) + if issemloss(term) + print( + io, + " ($(nsamples(term)) samples, $(nobserved_vars(term)) observed, $(nlatent_vars(term)) latent variables)", + ) + end + if !isnothing(weight(term)) + print(io, " w=$(round(weight(term), digits=3))") + else + print(io, " w=1") + end +end + ############################################################################################ # constructor for Sem types ############################################################################################ +function multigroup_weights(models, n) + nsamples_total = sum(nsamples, models) + uniform_lossfun = check_single_lossfun(models...; throw_error = false) + if !uniform_lossfun + @info """ + Your ensemble model contains heterogeneous loss functions. + Default weights of (#samples per group/#total samples) will be used + """ + return [(nsamples(model)) / (nsamples_total) for model in models] + end + lossfun = models[1].loss.functions[1] + if !applicable(mg_correction, lossfun) + @info """ + We don't know how to choose group weights for the specified loss function. + Default weights of (#samples per group/#total samples) will be used + """ + return [(nsamples(model)) / (nsamples_total) for model in models] + end + c = mg_correction(lossfun) + return [(nsamples(model)+c) / (nsamples_total+n*c) for model in models] +end + +function Sem( + loss_terms...; + params::Union{Vector{Symbol}, Nothing} = nothing, + default_sem_weights = :nsamples, +) + default_sem_weights ∈ [:nsamples, :uniform, :one] || + throw(ArgumentError("Unsupported default_sem_weights=:$default_sem_weights")) + # assemble a list of weighted losses and check params equality + terms = Vector{LossTerm}() + params = !isnothing(params) ? copy(params) : params + has_sem_weights = false + nsems = 0 + for inp_term in loss_terms + if inp_term isa AbstractLoss # term + term = inp_term + term_w = nothing + term_id = nothing + elseif inp_term isa Pair + if inp_term[1] isa AbstractLoss # term => weight + term, term_w = inp_term + term_id = nothing + elseif inp_term[2] isa AbstractLoss # id => term + term_id, term = inp_term + term_w = nothing + elseif inp_term[2] isa Pair # id => term => weight + term_id, (term, term_w) = inp_term + isa(term, AbstractLoss) || throw( + ArgumentError( + "AbstractLoss expected as a second argument of a loss term double pair (id => loss => weight), $(nameof(typeof(term))) found", + ), + ) + end + elseif inp_term isa LossTerm + term_id = id(inp_term) + term = loss(inp_term) + term_w = weight(inp_term) + else + "[id =>] AbstractLoss [=> weight] expected as a loss term, $(nameof(typeof(inp_term))) found" |> + ArgumentError |> + throw + end + + if term isa SemLoss + nsems += 1 + has_sem_weights |= !isnothing(term_w) + # check integrity + if isnothing(params) + params = SEM.params(term) + elseif params != SEM.params(term) + # FIXME the suggestion might no longer be relevant, since ParTable also stores params order + """ + The parameters of your SEM models do not match. + Maybe you tried to specify models of an ensemble via ParameterTables. + In that case, you may use RAMMatrices instead. + """ |> error + end + check_observed_vars(term) + elseif !(term isa AbstractLoss) + "AbstractLoss term expected at $(length(terms)+1) position, $(nameof(typeof(term))) found" |> + ArgumentError |> + throw + end + push!(terms, LossTerm(term, term_id, term_w)) + end + isnothing(params) && throw(ErrorException("No SEM models provided.")) + + if !has_sem_weights && nsems > 1 + # set the weights of SEMs in the ensemble + if default_sem_weights == :nsamples + # weight SEM by the number of samples + nsamples_total = sum(nsamples(term) for term in terms if issemloss(term)) + for (i, term) in enumerate(terms) + if issemloss(term) + terms[i] = + LossTerm(loss(term), id(term), nsamples(term) / nsamples_total) + end + end + elseif default_sem_weights == :uniform # uniform weights + for (i, term) in enumerate(terms) + if issemloss(term) + terms[i] = LossTerm(loss(term), id(term), 1 / nsems) + end + end + elseif default_sem_weights == :one # do nothing + end + end + + terms_tuple = Tuple(terms) + return Sem{typeof(terms_tuple)}(terms_tuple, params) +end + function Sem(; specification = ParameterTable, observed::O = SemObservedData, @@ -13,99 +173,127 @@ function Sem(; set_field_type_kwargs!(kwdict, observed, implied, loss, O, I) - observed, implied, loss = get_fields!(kwdict, specification, observed, implied, loss) - - sem = Sem(observed, implied, loss) + loss = get_fields!(kwdict, specification, observed, implied, loss) - return sem + return Sem(loss...) end +############################################################################################ +# functions +############################################################################################ + +params(model::AbstractSem) = model.params + """ - implied(model::AbstractSemSingle) -> SemImplied + loss_terms(model::AbstractSem) -Returns the [*implied*](@ref SemImplied) part of a model. +Returns a tuple of all [`LossTerm`](@ref) weighted terms in the SEM model. + +See also [`sem_terms`](@ref), [`loss_term`](@ref). """ -implied(model::AbstractSemSingle) = model.implied +loss_terms(model::AbstractSem) = model.loss_terms +nloss_terms(model::AbstractSem) = length(loss_terms(model)) -nvars(model::AbstractSemSingle) = nvars(implied(model)) -nobserved_vars(model::AbstractSemSingle) = nobserved_vars(implied(model)) -nlatent_vars(model::AbstractSemSingle) = nlatent_vars(implied(model)) +""" + sem_terms(model::AbstractSem) -vars(model::AbstractSemSingle) = vars(implied(model)) -observed_vars(model::AbstractSemSingle) = observed_vars(implied(model)) -latent_vars(model::AbstractSemSingle) = latent_vars(implied(model)) +Returns a tuple of all weighted SEM terms in the SEM model. -param_labels(model::AbstractSemSingle) = param_labels(implied(model)) -nparams(model::AbstractSemSingle) = nparams(implied(model)) +In comparison to [`loss_terms`](@ref) that returns all model terms, including e.g. +regularization terms, this function returns only the [`SemLoss`] terms. +See also [`loss_terms`](@ref), [`sem_term`](@ref). """ - observed(model::AbstractSemSingle) -> SemObserved +sem_terms(model::AbstractSem) = Tuple(term for term in loss_terms(model) if issemloss(term)) +nsem_terms(model::AbstractSem) = sum(issemloss, loss_terms(model)) + +nsamples(model::AbstractSem) = + sum(term -> issemloss(term) ? nsamples(term) : 0, loss_terms(model)) -Returns the [*observed*](@ref SemObserved) part of a model. """ -observed(model::AbstractSemSingle) = model.observed + loss_term(model::AbstractSem, id::Any) -> AbstractLoss -nsamples(model::AbstractSemSingle) = nsamples(observed(model)) +Returns the loss term with the specified `id` from the `model`. +Throws an error if the model has no term with the specified `id`. +See also [`loss_terms`](@ref). """ - loss(model::AbstractSemSingle) -> SemLoss +function loss_term(model::AbstractSem, id::Any) + for term in loss_terms(model) + if SEM.id(term) == id + return loss(term) + end + end + error("No loss term with id=$id found") +end -Returns the [*loss*](@ref SemLoss) function of a model. """ -loss(model::AbstractSemSingle) = model.loss - -# sum of samples in all sub-models -nsamples(ensemble::SemEnsemble) = sum(nsamples, ensemble.sems) - -function SemFiniteDiff(; - specification = ParameterTable, - observed::O = SemObservedData, - implied::I = RAM, - loss::L = SemML, - kwargs..., -) where {O, I, L} - kwdict = Dict{Symbol, Any}(kwargs...) + sem_term(model::AbstractSem, [id]) -> SemLoss - set_field_type_kwargs!(kwdict, observed, implied, loss, O, I) +Returns the SEM loss term with the specified `id` from the `model`. +Throws an error if the model has no term with the specified `id` or +if it is not of a [`SemLoss`](@ref) type. - observed, implied, loss = get_fields!(kwdict, specification, observed, implied, loss) +If no `id` is specified and the model contains only one SEM term, the term is returned. +Throws an error if the model contains multiple SEM terms. - sem = SemFiniteDiff(observed, implied, loss) +See also [`loss_term`](@ref), [`sem_terms`](@ref). +""" +function sem_term(model::AbstractSem, id::Any) + term = loss_term(model, id) + issemloss(term) || error("Loss term with id=$id ($(typeof(term))) is not a SEM term") + return term +end - return sem +function sem_term(model::AbstractSem, _::Nothing = nothing) + if nsem_terms(model) != 1 + error( + "Model contains $(nsem_terms(model)) SEM terms, you have to specify a specific term", + ) + end + for term in loss_terms(model) + issemloss(term) && return loss(term) + end + error("Unreachable reached") end -############################################################################################ -# functions -############################################################################################ +# wrappers arounds a single SemLoss term +observed(model::AbstractSem, id::Nothing = nothing) = observed(sem_term(model, id)) +implied(model::AbstractSem, id::Nothing = nothing) = implied(sem_term(model, id)) +vars(model::AbstractSem, id::Nothing = nothing) = vars(implied(model, id)) +observed_vars(model::AbstractSem, id::Nothing = nothing) = observed_vars(implied(model, id)) +latent_vars(model::AbstractSem, id::Nothing = nothing) = latent_vars(implied(model, id)) function set_field_type_kwargs!(kwargs, observed, implied, loss, O, I) kwargs[:observed_type] = O <: Type ? observed : typeof(observed) kwargs[:implied_type] = I <: Type ? implied : typeof(implied) if loss isa SemLoss - kwargs[:loss_types] = [ - lossfun isa SemLossFunction ? typeof(lossfun) : lossfun for - lossfun in loss.functions - ] - elseif applicable(iterate, loss) kwargs[:loss_types] = - [lossfun isa SemLossFunction ? typeof(lossfun) : lossfun for lossfun in loss] + [aloss isa SemLoss ? typeof(aloss) : aloss for aloss in loss.functions] + elseif applicable(iterate, loss) + kwargs[:loss_types] = [aloss isa SemLoss ? typeof(aloss) : aloss for aloss in loss] else - kwargs[:loss_types] = [loss isa SemLossFunction ? typeof(loss) : loss] + kwargs[:loss_types] = [loss isa SemLoss ? typeof(loss) : loss] end end # construct Sem fields -function get_fields!(kwargs, specification, observed, implied, loss) - if !isa(specification, SemSpecification) - specification = specification(; kwargs...) +function get_fields!(kwargs, spec, observed, implied, loss) + if !isa(spec, SemSpecification) + spec = spec(; kwargs...) end # observed if !isa(observed, SemObserved) - observed = observed(; specification, kwargs...) + observed = if spec isa EnsembleParameterTable + Dict( + term_id => observed(; specification = term_spec, kwargs...) for + (term_id, term_spec) in pairs(spec.tables) + ) + else + observed(; specification = spec, kwargs...) + end end - kwargs[:observed] = observed # implied if !isa(implied, SemImplied) @@ -117,95 +305,98 @@ function get_fields!(kwargs, specification, observed, implied, loss) else implied_kwargs = kwargs end - implied = implied(specification; implied_kwargs...) + implied = if spec isa EnsembleParameterTable + Dict( + term_id => implied(term_spec; implied_kwargs...) for + (term_id, term_spec) in pairs(spec.tables) + ) + else + implied(spec; implied_kwargs...) + end end - kwargs[:implied] = implied - kwargs[:nparams] = nparams(implied) - # loss - loss = get_SemLoss(loss; specification, kwargs...) - kwargs[:loss] = loss + loss_kwargs = copy(kwargs) + loss_kwargs[:nparams] = nparams(spec) + loss = build_SemTerms(loss, observed, implied; loss_kwargs...) - return observed, implied, loss + return loss end # construct loss field -function get_SemLoss(loss; kwargs...) +function build_SemTerms(loss, observed, implied; kwargs...) + function build_SemLoss(aloss, observed, implied) + if loss isa AbstractLoss + return loss + elseif aloss <: SemLoss{O, I} where {O, I} + return aloss(observed, implied; kwargs...) + else + return aloss(; kwargs...) + end + end + if loss isa SemLoss - nothing + return loss elseif applicable(iterate, loss) - loss_out = [] - for lossfun in loss - if isa(lossfun, SemLossFunction) - push!(loss_out, lossfun) - else - lossfun = lossfun(; kwargs...) - push!(loss_out, lossfun) - end - end - loss = SemLoss(loss_out...; kwargs...) + return [build_SemLoss(aloss, observed, implied) for aloss in loss] else - if !isa(loss, SemLossFunction) - loss = SemLoss(loss(; kwargs...); kwargs...) + if isa(observed, AbstractDict) && isa(implied, AbstractDict) + observed_ids = Set(keys(observed)) + implied_ids = Set(keys(implied)) + if observed_ids != implied_ids + """" + The term ids of the observed and the implied data do not match. + Observed term ids: $(observed_ids), implied term ids: $(implied_ids) + """ |> + ArgumentError |> + throw + end + loss_out = [ + begin + term_implied = implied[term_id] + if observed_vars(term_observed) != observed_vars(term_implied) + "observed_vars differ between the observed and the implied for the term $term_id" |> + ArgumentError |> + throw + end + LossTerm( + build_SemLoss(loss, term_observed, term_implied), + term_id, + nothing, + ) + end for (term_id, term_observed) in pairs(observed) + ] + return loss_out else - loss = SemLoss(loss; kwargs...) + if observed_vars(observed) != observed_vars(implied) + "observed_vars differ between the observed and the implied" |> + ArgumentError |> + throw + end + return (build_SemLoss(loss, observed, implied),) end end - return loss +end + +function update_observed(sem::Sem, new_observed; kwargs...) + new_terms = Tuple( + update_observed(lossterm.loss, new_observed; kwargs...) for + lossterm in loss_terms(sem) + ) + return Sem(new_terms...) end ############################################################## # pretty printing ############################################################## -function Base.show(io::IO, sem::Sem{O, I, L}) where {O, I, L} - lossfuntypes = @. string(nameof(typeof(sem.loss.functions))) - lossfuntypes = " " .* lossfuntypes .* ("\n") - print(io, "Structural Equation Model \n") - print(io, "- Loss Functions \n") - print(io, lossfuntypes...) - print(io, "- Fields \n") - print(io, " observed: $(nameof(O)) \n") - print(io, " implied: $(nameof(I)) \n") -end - -function Base.show(io::IO, sem::SemFiniteDiff{O, I, L}) where {O, I, L} - lossfuntypes = @. string(nameof(typeof(sem.loss.functions))) - lossfuntypes = " " .* lossfuntypes .* ("\n") - print(io, "Structural Equation Model : Finite Diff Approximation\n") - print(io, "- Loss Functions \n") - print(io, lossfuntypes...) - print(io, "- Fields \n") - print(io, " observed: $(nameof(O)) \n") - print(io, " implied: $(nameof(I)) \n") -end - -function Base.show(io::IO, loss::SemLoss) - lossfuntypes = @. string(nameof(typeof(loss.functions))) - lossfuntypes = " " .* lossfuntypes .* ("\n") - print(io, "SemLoss \n") - print(io, "- Loss Functions \n") - print(io, lossfuntypes...) - print(io, "- Weights \n") - for weight in loss.weights - if isnothing(weight.w) - print(io, " one \n") - else - print(io, "$(round.(weight.w, digits = 2)) \n") - end - end -end - -function Base.show(io::IO, models::SemEnsemble) - print(io, "SemEnsemble \n") - print(io, "- Number of Models: $(models.n) \n") - print(io, "- Weights: $(round.(models.weights, digits = 2)) \n") - - print(io, "\n", "Models: \n") - print(io, "===============================================", "\n") - for (model, i) in zip(models.sems, 1:models.n) - print(io, "---------------------- ", i, " ----------------------", "\n") - print(io, model) +function Base.show(io::IO, sem::AbstractSem) + println(io, "Structural Equation Model ($(nameof(typeof(sem))))") + println(io, "- $(nparams(sem)) parameters") + println(io, "- Loss terms:") + for term in loss_terms(sem) + print(io, " - ") + print(io, term) + println(io) end end diff --git a/src/implied/RAM/generic.jl b/src/implied/RAM/generic.jl index d57500a36..f1c1e08d0 100644 --- a/src/implied/RAM/generic.jl +++ b/src/implied/RAM/generic.jl @@ -154,16 +154,11 @@ end ### methods ############################################################################################ -function update!( - targets::EvaluationTargets, - implied::RAM, - model::AbstractSemSingle, - param_labels, -) - materialize!(implied.A, implied.ram_matrices.A, param_labels) - materialize!(implied.S, implied.ram_matrices.S, param_labels) +function update!(targets::EvaluationTargets, implied::RAM, params) + materialize!(implied.A, implied.ram_matrices.A, params) + materialize!(implied.S, implied.ram_matrices.S, params) if !isnothing(implied.M) - materialize!(implied.M, implied.ram_matrices.M, param_labels) + materialize!(implied.M, implied.ram_matrices.M, params) end parent(implied.I_A) .= .-implied.A diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index 0f7868dae..4c9bda914 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -176,12 +176,7 @@ end ### objective, gradient, hessian ############################################################################################ -function update!( - targets::EvaluationTargets, - implied::RAMSymbolic, - model::AbstractSemSingle, - par, -) +function update!(targets::EvaluationTargets, implied::RAMSymbolic, par) implied.Σ_eval!(implied.Σ, par) if MeanStruct(implied) === HasMeanStruct implied.μ_eval!(implied.μ, par) @@ -223,7 +218,10 @@ end function eval_Σ_symbolic(S, I_A⁻¹, F; vech::Bool = false, simplify::Bool = false) Σ = F * I_A⁻¹ * S * permutedims(I_A⁻¹) * permutedims(F) Σ = Array(Σ) - vech && (Σ = SEM.vech(Σ)) + if vech + n = size(Σ, 1) + Σ = [Σ[i, j] for j in 1:n for i in j:n] + end if simplify Threads.@threads for i in eachindex(Σ) Σ[i] = Symbolics.simplify(Σ[i]) diff --git a/src/implied/empty.jl b/src/implied/empty.jl index 82a6c9469..a327ee134 100644 --- a/src/implied/empty.jl +++ b/src/implied/empty.jl @@ -13,8 +13,8 @@ Empty placeholder for models that don't need an implied part. - `specification`: either a `RAMMatrices` or `ParameterTable` object # Examples -A multigroup model with ridge regularization could be specified as a `SemEnsemble` with one -model per group and an additional model with `ImpliedEmpty` and `SemRidge` for the regularization part. +A multigroup model with ridge regularization could be specified as a `Sem` with one +SEM term (`SemLoss`) per group and an additional `SemRidge` regularization term. # Extended help @@ -45,7 +45,7 @@ end ### methods ############################################################################################ -update!(targets::EvaluationTargets, implied::ImpliedEmpty, par, model) = nothing +update!(targets::EvaluationTargets, implied::ImpliedEmpty, par) = nothing ############################################################################################ ### Recommended methods diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 8572b15ac..fdedf3984 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -75,23 +75,27 @@ Can handle observed data with missing values. # Constructor - SemFIML(; observed::SemObservedMissing, implied::SemImplied, kwargs...) + SemFIML(observed::SemObservedMissing, implied::SemImplied) # Arguments - `observed::SemObservedMissing`: the observed part of the model (see [`SemObservedMissing`](@ref)) - `implied::SemImplied`: the implied part of the model + (see [`SemImplied`](@ref)) # Examples ```julia -my_fiml = SemFIML(observed = my_observed, implied = my_implied) +my_fiml = SemFIML(my_observed, my_implied) ``` # Interfaces Analytic gradients are available. """ -struct SemFIML{T, W} <: SemLossFunction +struct SemFIML{O, I, T, W} <: SemLoss{O, I} hessianeval::ExactHessian + + observed::O + implied::I patterns::Vector{SemFIMLPattern{T}} imp_inv::Matrix{T} # implied inverse @@ -105,7 +109,7 @@ end ### Constructors ############################################################################################ -function SemFIML(; observed::SemObservedMissing, implied, specification, kwargs...) +function SemFIML(observed::SemObservedMissing, implied::SemImplied; kwargs...) if MeanStruct(implied) === NoMeanStruct """ Full information maximum likelihood (FIML) can only be used with a meanstructure. @@ -117,6 +121,8 @@ function SemFIML(; observed::SemObservedMissing, implied, specification, kwargs. return SemFIML( ExactHessian(), + observed, + implied, [SemFIMLPattern(pat) for pat in observed.patterns], zeros(nobserved_vars(observed), nobserved_vars(observed)), CommutationMatrix(nvars(implied)), @@ -128,30 +134,28 @@ end ### methods ############################################################################################ -function evaluate!( - objective, - gradient, - hessian, - loss::SemFIML, - implied::SemImplied, - model::AbstractSemSingle, - params, -) +function evaluate!(objective, gradient, hessian, loss::SemFIML, params) isnothing(hessian) || error("Hessian not implemented for FIML") - if !check(loss, model) + implied = SEM.implied(loss) + observed = SEM.observed(loss) + + copyto!(loss.imp_inv, implied.Σ) + Σ_chol = cholesky!(Symmetric(loss.imp_inv); check = false) + + if !isposdef(Σ_chol) isnothing(objective) || (objective = non_posdef_return(params)) isnothing(gradient) || fill!(gradient, 1) return objective end - prepare!(loss, model) + @inbounds for (patloss, pat) in zip(loss.patterns, observed.patterns) + prepare!(patloss, pat, implied) + end - scale = inv(nsamples(observed(model))) - isnothing(objective) || - (objective = scale * F_FIML(eltype(params), loss, observed(model), model)) - isnothing(gradient) || - (∇F_FIML!(gradient, loss, observed(model), model); gradient .*= scale) + scale = inv(nsamples(observed)) + isnothing(objective) || (objective = scale * F_FIML(eltype(params), loss)) + isnothing(gradient) || (∇F_FIML!(gradient, loss); gradient .*= scale) return objective end @@ -167,27 +171,14 @@ update_observed(loss::SemFIML, observed::SemObserved; kwargs...) = ### additional functions ############################################################################################ -function prepare!(loss::SemFIML, observed::SemObservedMissing, implied::SemImplied) - @inbounds for (patloss, pat) in zip(loss.patterns, observed.patterns) - prepare!(patloss, pat, implied.Σ, implied.μ) - end -end - -prepare!(loss::SemFIML, model::AbstractSemSingle) = - prepare!(loss, observed(model), implied(model)) - -function check(loss::SemFIML, model::AbstractSemSingle) - copyto!(loss.imp_inv, implied(model).Σ) - a = cholesky!(Symmetric(loss.imp_inv); check = false) - return isposdef(a) +function ∇F_fiml_outer!(G, JΣ, Jμ, loss::SemFIML{O, I}) where {O, I <: SemImpliedSymbolic} + mul!(G, loss.implied.∇Σ', JΣ) # should be transposed + mul!(G, loss.implied.∇μ', Jμ, -1, 1) end -function ∇F_fiml_outer!(G, JΣ, Jμ, loss::SemFIML, implied::SemImpliedSymbolic, model) - mul!(G, implied.∇Σ', JΣ) # should be transposed - mul!(G, implied.∇μ', Jμ, -1, 1) -end +function ∇F_fiml_outer!(G, JΣ, Jμ, loss::SemFIML) + implied = loss.implied -function ∇F_fiml_outer!(G, JΣ, Jμ, loss::SemFIML, implied, model) Iₙ = sparse(1.0I, size(implied.A)...) P = kron(implied.F⨉I_A⁻¹, implied.F⨉I_A⁻¹) Q = kron(implied.S * implied.I_A⁻¹', Iₙ) @@ -203,25 +194,20 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, loss::SemFIML, implied, model) mul!(G, ∇μ', Jμ, -1, 1) end -function F_FIML( - ::Type{T}, - loss::SemFIML, - observed::SemObservedMissing, - model::AbstractSemSingle, -) where {T} +function F_FIML(::Type{T}, loss::SemFIML) where {T} F = zero(T) - for (patloss, pat) in zip(loss.patterns, observed.patterns) + for (patloss, pat) in zip(loss.patterns, loss.observed.patterns) F += objective(patloss, pat) end return F end -function ∇F_FIML!(G, loss::SemFIML, observed::SemObservedMissing, model::AbstractSemSingle) - Jμ = zeros(nobserved_vars(model)) - JΣ = zeros(nobserved_vars(model)^2) +function ∇F_FIML!(G, loss::SemFIML) + Jμ = zeros(nobserved_vars(loss)) + JΣ = zeros(nobserved_vars(loss)^2) - for (patloss, pat) in zip(loss.patterns, observed.patterns) + for (patloss, pat) in zip(loss.patterns, loss.observed.patterns) gradient!(JΣ, Jμ, patloss, pat) end - ∇F_fiml_outer!(G, JΣ, Jμ, loss, implied(model), model) + ∇F_fiml_outer!(G, JΣ, Jμ, loss) end diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index aae1dadae..2d449d73c 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -8,36 +8,41 @@ Maximum likelihood estimation. # Constructor - SemML(; observed, approximate_hessian = false, kwargs...) + SemML(observed, implied; approximate_hessian = false) # Arguments - `observed::SemObserved`: the observed part of the model +- `implied::SemImplied`: [`SemImplied`](@ref) instance - `approximate_hessian::Bool`: if hessian-based optimization is used, should the hessian be swapped for an approximation # Examples ```julia -my_ml = SemML(observed = my_observed) +my_ml = SemML(my_observed, my_implied) ``` # Interfaces Analytic gradients are available, and for models without a meanstructure -and RAMSymbolic implied type, also analytic hessians. +and `RAMSymbolic` implied type, also analytic hessians. """ -struct SemML{HE <: HessianEval, INV, M, M2} <: SemLossFunction +struct SemML{O, I, HE <: HessianEval, INV, M, M2} <: SemLoss{O, I} + observed::O + implied::I hessianeval::HE Σ⁻¹::INV Σ⁻¹Σₒ::M meandiff::M2 - - SemML{HE}(args...) where {HE <: HessianEval} = - new{HE, map(typeof, args)...}(HE(), args...) end ############################################################################################ ### Constructors ############################################################################################ -function SemML(; observed::SemObserved, approximate_hessian::Bool = false, kwargs...) +function SemML( + observed::SemObserved, + implied::SemImplied; + approximate_hessian::Bool = false, + kwargs..., +) if observed isa SemObservedMissing @warn """ ML estimation with `SemObservedMissing` will use an approximate covariance and mean estimated with EM algorithm. @@ -51,12 +56,25 @@ function SemML(; observed::SemObserved, approximate_hessian::Bool = false, kwarg ) """ end + # check integrity + check_observed_vars(observed, implied) + he = approximate_hessian ? ApproxHessian() : ExactHessian() obsmean = obs_mean(observed) obscov = obs_cov(observed) meandiff = isnothing(obsmean) ? nothing : copy(obsmean) - return SemML{approximate_hessian ? ApproxHessian : ExactHessian}( + return SemML{ + typeof(observed), + typeof(implied), + typeof(he), + typeof(obscov), + typeof(obscov), + typeof(meandiff), + }( + observed, + implied, + he, similar(obscov), similar(obscov), meandiff, @@ -74,20 +92,20 @@ function evaluate!( objective, gradient, hessian, - semml::SemML, - implied::SemImpliedSymbolic, - model::AbstractSemSingle, + loss::SemML{<:Any, <:SemImpliedSymbolic}, par, ) + implied = SEM.implied(loss) + if !isnothing(hessian) (MeanStruct(implied) === HasMeanStruct) && throw(DomainError(H, "hessian of ML + meanstructure is not available")) end Σ = implied.Σ - Σₒ = obs_cov(observed(model)) - Σ⁻¹Σₒ = semml.Σ⁻¹Σₒ - Σ⁻¹ = semml.Σ⁻¹ + Σₒ = obs_cov(observed(loss)) + Σ⁻¹Σₒ = loss.Σ⁻¹Σₒ + Σ⁻¹ = loss.Σ⁻¹ copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) @@ -105,7 +123,7 @@ function evaluate!( if MeanStruct(implied) === HasMeanStruct μ = implied.μ - μₒ = obs_mean(observed(model)) + μₒ = obs_mean(observed(loss)) μ₋ = μₒ - μ isnothing(objective) || (objective += dot(μ₋, Σ⁻¹, μ₋)) @@ -124,7 +142,7 @@ function evaluate!( mul!(gradient, ∇Σ', J') end if !isnothing(hessian) - if HessianEval(semml) === ApproxHessian + if HessianEval(loss) === ApproxHessian mul!(hessian, ∇Σ' * kron(Σ⁻¹, Σ⁻¹), ∇Σ, 2, 0) else ∇²Σ = implied.∇²Σ @@ -143,24 +161,17 @@ end ############################################################################################ ### Non-Symbolic Implied Types -function evaluate!( - objective, - gradient, - hessian, - semml::SemML, - implied::RAM, - model::AbstractSemSingle, - par, -) +function evaluate!(objective, gradient, hessian, loss::SemML, par) if !isnothing(hessian) error("hessian of ML + non-symbolic implied type is not available") end - Σ = implied.Σ - Σₒ = obs_cov(observed(model)) - Σ⁻¹Σₒ = semml.Σ⁻¹Σₒ - Σ⁻¹ = semml.Σ⁻¹ + implied = SEM.implied(loss) + Σ = implied.Σ + Σₒ = obs_cov(observed(loss)) + Σ⁻¹Σₒ = loss.Σ⁻¹Σₒ + Σ⁻¹ = loss.Σ⁻¹ copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) if !isposdef(Σ_chol) @@ -179,7 +190,7 @@ function evaluate!( if MeanStruct(implied) === HasMeanStruct μ = implied.μ - μₒ = obs_mean(observed(model)) + μₒ = obs_mean(observed(loss)) μ₋ = μₒ - μ objective += dot(μ₋, Σ⁻¹, μ₋) end @@ -198,7 +209,7 @@ function evaluate!( if MeanStruct(implied) === HasMeanStruct μ = implied.μ - μₒ = obs_mean(observed(model)) + μₒ = obs_mean(observed(loss)) ∇M = implied.∇M M = implied.M μ₋ = μₒ - μ @@ -229,16 +240,17 @@ end ### recommended methods ############################################################################################ -update_observed(lossfun::SemML, observed::SemObservedMissing; kwargs...) = +update_observed(loss::SemML, observed::SemObservedMissing; kwargs...) = error("ML estimation does not work with missing data - use FIML instead") -function update_observed(lossfun::SemML, observed::SemObserved; kwargs...) - if size(lossfun.Σ⁻¹) == size(obs_cov(observed)) - return lossfun +function update_observed(loss::SemML, observed::SemObserved; kwargs...) + if (obs_cov(loss) == obs_cov(observed)) && (obs_mean(loss) == obs_mean(observed)) + return loss # no change else - return SemML(; - observed = observed, - approximate_hessian = HessianEval(lossfun) == ApproxHessian, + return SemML( + observed, + loss.implied; + approximate_hessian = HessianEval(loss) == ApproxHessian, kwargs..., ) end diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 9de011f60..5c4cb2520 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -9,8 +9,8 @@ At the moment only available with the `RAMSymbolic` implied type. # Constructor - SemWLS(; - observed, implied, + SemWLS( + observed::SemObserved, implied::SemImplied; wls_weight_matrix = nothing, wls_weight_matrix_mean = nothing, approximate_hessian = false, @@ -18,7 +18,7 @@ At the moment only available with the `RAMSymbolic` implied type. # Arguments - `observed`: the `SemObserved` part of the model -- `implied::SemImplied`: the implied part of the model +- `implied`: the `SemImplied` part of the model - `approximate_hessian::Bool`: should the hessian be swapped for an approximation - `wls_weight_matrix`: the weight matrix for weighted least squares. Defaults to GLS estimation (``0.5*(D^T*kron(S,S)*D)`` where D is the duplication matrix @@ -28,29 +28,37 @@ At the moment only available with the `RAMSymbolic` implied type. # Examples ```julia -my_wls = SemWLS(observed = my_observed, implied = my_implied) +my_wls = SemWLS(my_observed, my_implied) ``` # Interfaces Analytic gradients are available, and for models without a meanstructure also analytic hessians. """ -struct SemWLS{HE <: HessianEval, Vt, St, C} <: SemLossFunction +struct SemWLS{O, I, HE <: HessianEval, Vt, St, C} <: SemLoss{O, I} + observed::O + implied::I + hessianeval::HE V::Vt σₒ::St V_μ::C + + SemWLS(observed, implied, ::Type{HE}, args...) where {HE <: HessianEval} = + new{typeof(observed), typeof(implied), HE, map(typeof, args)...}( + observed, + implied, + HE(), + args..., + ) end ############################################################################################ ### Constructors ############################################################################################ -SemWLS{HE}(args...) where {HE <: HessianEval} = - SemWLS{HE, map(typeof, args)...}(HE(), args...) - -function SemWLS(; +function SemWLS( observed::SemObserved, - implied::SemImplied, + implied::SemImplied; wls_weight_matrix::Union{AbstractMatrix, Nothing} = nothing, wls_weight_matrix_mean::Union{AbstractMatrix, Nothing} = nothing, approximate_hessian::Bool = false, @@ -75,14 +83,19 @@ function SemWLS(; ArgumentError |> throw end + # check integrity + check_observed_vars(observed, implied) nobs_vars = nobserved_vars(observed) tril_ind = filter(x -> (x[1] >= x[2]), CartesianIndices(obs_cov(observed))) s = obs_cov(observed)[tril_ind] - size(s) == size(implied.Σ) || - throw(DimensionMismatch("SemWLS requires implied covariance to be in vech-ed form " * - "(vectorized lower triangular part of Σ matrix): $(size(s)) expected, $(size(implied.Σ)) found.\n" * - "$(nameof(typeof(implied))) must be constructed with vech=true.")) + size(s) == size(implied.Σ) || throw( + DimensionMismatch( + "SemWLS requires implied covariance to be in vech-ed form " * + "(vectorized lower triangular part of Σ matrix): $(size(s)) expected, $(size(implied.Σ)) found.\n" * + "$(nameof(typeof(implied))) must be constructed with vech=true.", + ), + ) # compute V here if isnothing(wls_weight_matrix) @@ -101,13 +114,12 @@ function SemWLS(; if MeanStruct(implied) == HasMeanStruct if isnothing(wls_weight_matrix_mean) - @warn "Computing WLS weight matrix for the meanstructure using obs_cov()" + @info "Computing WLS weight matrix for the meanstructure using obs_cov()" wls_weight_matrix_mean = inv(obs_cov(observed)) - else - size(wls_weight_matrix_mean) == (nobs_vars, nobs_vars) || DimensionMismatch( - "wls_weight_matrix_mean has to be of size $(nobs_vars)×$(nobs_vars)", - ) end + size(wls_weight_matrix_mean) == (nobs_vars, nobs_vars) || DimensionMismatch( + "wls_weight_matrix_mean has to be of size $(nobs_vars)×$(nobs_vars)", + ) else isnothing(wls_weight_matrix_mean) || @warn "Ignoring wls_weight_matrix_mean since meanstructure is disabled" @@ -115,31 +127,25 @@ function SemWLS(; end HE = approximate_hessian ? ApproxHessian : ExactHessian - return SemWLS{HE}(wls_weight_matrix, s, wls_weight_matrix_mean) + return SemWLS(observed, implied, HE, wls_weight_matrix, s, wls_weight_matrix_mean) end ############################################################################ ### methods ############################################################################ -function evaluate!( - objective, - gradient, - hessian, - semwls::SemWLS, - implied::SemImpliedSymbolic, - model::AbstractSemSingle, - par, -) +function evaluate!(objective, gradient, hessian, loss::SemWLS, par) + implied = SEM.implied(loss) + if !isnothing(hessian) && (MeanStruct(implied) === HasMeanStruct) error("hessian of WLS with meanstructure is not available") end - V = semwls.V + V = loss.V ∇σ = implied.∇Σ σ = implied.Σ - σₒ = semwls.σₒ + σₒ = loss.σₒ σ₋ = σₒ - σ isnothing(objective) || (objective = dot(σ₋, V, σ₋)) @@ -152,17 +158,17 @@ function evaluate!( gradient .*= -2 end isnothing(hessian) || (mul!(hessian, ∇σ' * V, ∇σ, 2, 0)) - if !isnothing(hessian) && (HessianEval(semwls) === ExactHessian) + if !isnothing(hessian) && (HessianEval(loss) === ExactHessian) ∇²Σ = implied.∇²Σ - J = -2 * (σ₋' * semwls.V)' + J = -2 * (σ₋' * loss.V)' implied.∇²Σ_eval!(∇²Σ, J, par) hessian .+= ∇²Σ end if MeanStruct(implied) === HasMeanStruct μ = implied.μ - μₒ = obs_mean(observed(model)) + μₒ = obs_mean(observed(loss)) μ₋ = μₒ - μ - V_μ = semwls.V_μ + V_μ = loss.V_μ if !isnothing(objective) objective += dot(μ₋, V_μ, μ₋) end @@ -179,23 +185,19 @@ end ############################################################################################ function update_observed( - lossfun::SemWLS, + loss::SemWLS, observed::SemObserved; recompute_V = true, kwargs..., ) if recompute_V - return SemWLS(; - observed = observed, - meanstructure = MeanStruct(kwargs[:implied]) == HasMeanStruct, - kwargs..., - ) + return SemWLS(observed, loss.implied; kwargs...) else - return SemWLS(; - observed = observed, - wls_weight_matrix = lossfun.V, - wls_weight_matrix_mean = lossfun.V_μ, - meanstructure = MeanStruct(kwargs[:implied]) == HasMeanStruct, + return SemWLS( + observed, + loss.implied; + wls_weight_matrix = loss.V, + wls_weight_matrix_mean = loss.V_μ, kwargs..., ) end diff --git a/src/loss/abstract.jl b/src/loss/abstract.jl new file mode 100644 index 000000000..bf8585d6a --- /dev/null +++ b/src/loss/abstract.jl @@ -0,0 +1,42 @@ +""" + observed(loss::SemLoss) -> SemObserved + +Returns the [*observed*](@ref SemObserved) part of a model. +""" +observed(loss::SemLoss) = loss.observed + +""" + implied(loss::SemLoss) -> SemImplied + +Returns the [*implied*](@ref SemImplied) part of a model. +""" +implied(loss::SemLoss) = loss.implied + +for f in (:nsamples, :obs_cov, :obs_mean) + @eval $f(loss::SemLoss) = $f(observed(loss)) +end + +for f in ( + :vars, + :nvars, + :latent_vars, + :nlatent_vars, + :observed_vars, + :nobserved_vars, + :params, + :nparams, +) + @eval $f(loss::SemLoss) = $f(implied(loss)) +end + +function check_observed_vars(observed::SemObserved, implied::SemImplied) + isnothing(observed_vars(implied)) || + observed_vars(observed) == observed_vars(implied) || + throw( + ArgumentError( + "Observed variables defined for \"observed\" and \"implied\" do not match.", + ), + ) +end + +check_observed_vars(sem::SemLoss) = check_observed_vars(observed(sem), implied(sem)) diff --git a/src/loss/constant/constant.jl b/src/loss/constant/constant.jl index 3aed5e27c..023076cc9 100644 --- a/src/loss/constant/constant.jl +++ b/src/loss/constant/constant.jl @@ -4,6 +4,8 @@ ### Types ############################################################################################ """ + SemConstant{C <: Number} <: AbstractLoss + Constant loss term. Can be used for comparability to other packages. # Constructor @@ -15,37 +17,27 @@ Constant loss term. Can be used for comparability to other packages. # Examples ```julia - my_constant = SemConstant(constant_loss = 42.0) + my_constant = SemConstant(42.0) ``` # Interfaces Analytic gradients and hessians are available. """ -struct SemConstant{C} <: SemLossFunction +struct SemConstant{C <: Number} <: AbstractLoss hessianeval::ExactHessian c::C -end -############################################################################################ -### Constructors -############################################################################################ - -function SemConstant(; constant_loss, kwargs...) - return SemConstant(ExactHessian(), constant_loss) + SemConstant(c::Number) = new{typeof(c)}(ExactHessian(), c) end -############################################################################################ -### methods -############################################################################################ +SemConstant(; constant_loss::Number, kwargs...) = SemConstant(constant_loss) -objective(constant::SemConstant, model::AbstractSem, par) = constant.c -gradient(constant::SemConstant, model::AbstractSem, par) = zero(par) -hessian(constant::SemConstant, model::AbstractSem, par) = - zeros(eltype(par), length(par), length(par)) +objective(loss::SemConstant, par) = convert(eltype(par), loss.c) +gradient(loss::SemConstant, par) = zero(par) +hessian(loss::SemConstant, par) = zeros(eltype(par), length(par), length(par)) ############################################################################################ ### Recommended methods ############################################################################################ -update_observed(loss_function::SemConstant, observed::SemObserved; kwargs...) = - loss_function +update_observed(loss::SemConstant, observed::SemObserved; kwargs...) = loss diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index 90cbcc231..3e2cfbff2 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -25,7 +25,7 @@ my_ridge = SemRidge(;α_ridge = 0.02, which_ridge = [:λ₁, :λ₂, :ω₂₃], # Interfaces Analytic gradients and hessians are available. """ -struct SemRidge{P, W1, W2, GT, HT} <: SemLossFunction +struct SemRidge{P, W1, W2, GT, HT} <: AbstractLoss hessianeval::ExactHessian α::P which::W1 @@ -74,15 +74,14 @@ end ### methods ############################################################################################ -objective(ridge::SemRidge, model::AbstractSem, par) = - @views ridge.α * sum(abs2, par[ridge.which]) +objective(ridge::SemRidge, par) = @views ridge.α * sum(abs2, par[ridge.which]) -function gradient(ridge::SemRidge, model::AbstractSem, par) +function gradient(ridge::SemRidge, par) @views ridge.gradient[ridge.which] .= (2 * ridge.α) * par[ridge.which] return ridge.gradient end -function hessian(ridge::SemRidge, model::AbstractSem, par) +function hessian(ridge::SemRidge, par) @views @. ridge.hessian[ridge.which_H] .= 2 * ridge.α return ridge.hessian end diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index 69915ffa9..23cef4e61 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -24,68 +24,61 @@ is_hessian_required(::EvaluationTargets{<:Any, <:Any, H}) where {H} = H (targets::EvaluationTargets)(arg_tuple::Tuple) = targets(arg_tuple...) """ - evaluate!(objective, gradient, hessian [, lossfun], model, params) + evaluate!(objective, gradient, hessian, loss::AbstractLoss, params) + evaluate!(objective, gradient, hessian, model::AbstractSem, params) Evaluates the objective, gradient, and/or Hessian at the given parameter vector. -If a loss function is passed, only this specific loss function is evaluated, otherwise, -the sum of all loss functions in the model is evaluated. + +If a single loss term (`loss`) is passed, only this specific term is evaluated, +otherwise, if the entire SEM `model` is passed, the weighted sum of all loss terms +in the model is evaluated. If objective, gradient or hessian are `nothing`, they are not evaluated. For example, since many numerical optimization algorithms don't require a Hessian, -the computation will be turned off by setting `hessian` to `nothing`. +its computation will be turned off by setting `hessian` to `nothing`. + +During the evaluation, the internal state of the loss term or of the model +could be modified. # Arguments - `objective`: a Number if the objective should be evaluated, otherwise `nothing` - `gradient`: a pre-allocated vector the gradient should be written to, otherwise `nothing` - `hessian`: a pre-allocated matrix the Hessian should be written to, otherwise `nothing` -- `lossfun::SemLossFunction`: loss function to evaluate +- `loss::AbstractLoss`: loss function to evaluate - `model::AbstractSem`: model to evaluate - `params`: vector of parameters # Implementing a new loss function -To implement a new loss function, a new method for `evaluate!` has to be defined. +To implement a new loss (subtype of `SemLoss` for SEM terms, or of `AbstractLoss` for +regularization terms), a new method for `evaluate!` has to be defined. This is explained in the online documentation on [Custom loss functions](@ref). """ function evaluate! end -# dispatch on SemImplied -evaluate!(objective, gradient, hessian, loss::SemLossFunction, model::AbstractSem, params) = - evaluate!(objective, gradient, hessian, loss, implied(model), model, params) - # fallback method -function evaluate!( - obj, - grad, - hess, - loss::SemLossFunction, - implied::SemImplied, - model, - params, -) - isnothing(obj) || (obj = objective(loss, implied, model, params)) - isnothing(grad) || copyto!(grad, gradient(loss, implied, model, params)) - isnothing(hess) || copyto!(hess, hessian(loss, implied, model, params)) +function evaluate!(obj, grad, hess, loss::AbstractLoss, params) + isnothing(obj) || (obj = objective(loss, params)) + isnothing(grad) || copyto!(grad, gradient(loss, params)) + isnothing(hess) || copyto!(hess, hessian(loss, params)) return obj end -# fallback methods -objective(f::SemLossFunction, implied::SemImplied, model, params) = - objective(f, model, params) -gradient(f::SemLossFunction, implied::SemImplied, model, params) = - gradient(f, model, params) -hessian(f::SemLossFunction, implied::SemImplied, model, params) = hessian(f, model, params) +evaluate!(obj, grad, hess, term::LossTerm, params) = + evaluate!(obj, grad, hess, loss(term), params) # fallback method for SemImplied that calls update_xxx!() methods -function update!(targets::EvaluationTargets, implied::SemImplied, model, params) - is_objective_required(targets) && update_objective!(implied, model, params) - is_gradient_required(targets) && update_gradient!(implied, model, params) - is_hessian_required(targets) && update_hessian!(implied, model, params) +function update!(targets::EvaluationTargets, implied::SemImplied, params) + is_objective_required(targets) && update_objective!(implied, params) + is_gradient_required(targets) && update_gradient!(implied, params) + is_hessian_required(targets) && update_hessian!(implied, params) end +const AbstractSemOrLoss = Union{AbstractSem, AbstractLoss} + # guess objective type -objective_type(model::AbstractSem, params::Any) = Float64 -objective_type(model::AbstractSem, params::AbstractVector{T}) where {T <: Number} = T -objective_zero(model::AbstractSem, params::Any) = zero(objective_type(model, params)) +objective_type(model::AbstractSemOrLoss, params::Any) = Float64 +objective_type(model::AbstractSemOrLoss, params::AbstractVector{T}) where {T <: Number} = T +objective_zero(model::AbstractSemOrLoss, params::Any) = zero(objective_type(model, params)) objective_type(objective::T, gradient, hessian) where {T <: Number} = T objective_type( @@ -101,145 +94,151 @@ objective_type( objective_zero(objective, gradient, hessian) = zero(objective_type(objective, gradient, hessian)) +evaluate!(objective, gradient, hessian, model::AbstractSem, params) = + error("evaluate!() for $(typeof(model)) is not implemented") + ############################################################################################ -# methods for AbstractSem +# methods for Sem ############################################################################################ -function evaluate!(objective, gradient, hessian, model::AbstractSemSingle, params) - targets = EvaluationTargets(objective, gradient, hessian) - # update implied state, its gradient and hessian (if required) - update!(targets, implied(model), model, params) - return evaluate!( - !isnothing(objective) ? zero(objective) : nothing, - gradient, - hessian, - loss(model), - model, - params, - ) -end +function evaluate!(objective, gradient, hessian, model::Sem, params) + # reset output + isnothing(objective) || (objective = objective_zero(objective, gradient, hessian)) + isnothing(gradient) || fill!(gradient, zero(eltype(gradient))) + isnothing(hessian) || fill!(hessian, zero(eltype(hessian))) -############################################################################################ -# methods for SemFiniteDiff -# (approximate gradient and hessian with finite differences of objective) -############################################################################################ + # gradient and hessian for individual terms + t_grad = isnothing(gradient) ? nothing : similar(gradient) + t_hess = isnothing(hessian) ? nothing : similar(hessian) + + # update implied states of all SemLoss terms before term calculation loop + # to make sure all terms use updated implied states + targets = EvaluationTargets(objective, gradient, hessian) + for term in loss_terms(model) + issemloss(term) && update!(targets, implied(term), params) + end -function evaluate!(objective, gradient, hessian, model::SemFiniteDiff, params) - function obj(p) - # recalculate implied state for p - update!(EvaluationTargets{true, false, false}(), implied(model), model, p) - evaluate!( - objective_zero(objective, gradient, hessian), - nothing, - nothing, - loss(model), - model, - p, + for term in loss_terms(model) + t_obj = evaluate!(objective, t_grad, t_hess, term, params) + #@show nameof(typeof(term)) t_obj + objective = accumulate_loss!( + objective, + gradient, + hessian, + weight(term), + t_obj, + t_grad, + t_hess, ) end - isnothing(gradient) || FiniteDiff.finite_difference_gradient!(gradient, obj, params) - isnothing(hessian) || FiniteDiff.finite_difference_hessian!(hessian, obj, params) - return !isnothing(objective) ? obj(params) : nothing + return objective end -objective(model::AbstractSem, params) = - evaluate!(objective_zero(model, params), nothing, nothing, model, params) - -############################################################################################ -# methods for SemLoss (weighted sum of individual SemLossFunctions) -############################################################################################ +# internal function to accumulate loss objective, gradient and hessian +function accumulate_loss!( + total_objective, + total_gradient, + total_hessian, + weight::Nothing, + objective, + gradient, + hessian, +) + isnothing(total_gradient) || (total_gradient .+= gradient) + isnothing(total_hessian) || (total_hessian .+= hessian) + return isnothing(total_objective) ? total_objective : (total_objective + objective) +end -function evaluate!(objective, gradient, hessian, loss::SemLoss, model::AbstractSem, params) - isnothing(objective) || (objective = zero(objective)) - isnothing(gradient) || fill!(gradient, zero(eltype(gradient))) - isnothing(hessian) || fill!(hessian, zero(eltype(hessian))) - f_grad = isnothing(gradient) ? nothing : similar(gradient) - f_hess = isnothing(hessian) ? nothing : similar(hessian) - for (f, weight) in zip(loss.functions, loss.weights) - f_obj = evaluate!(objective, f_grad, f_hess, f, model, params) - isnothing(objective) || (objective += weight * f_obj) - isnothing(gradient) || (gradient .+= weight * f_grad) - isnothing(hessian) || (hessian .+= weight * f_hess) - end - return objective +function accumulate_loss!( + total_objective, + total_gradient, + total_hessian, + weight::Number, + objective, + gradient, + hessian, +) + isnothing(total_gradient) || axpy!(weight, gradient, total_gradient) + isnothing(total_hessian) || axpy!(weight, hessian, total_hessian) + return isnothing(total_objective) ? total_objective : + (total_objective + weight * objective) end ############################################################################################ -# methods for SemEnsemble (weighted sum of individual AbstractSemSingle models) +# methods for SemFiniteDiff +# (approximate gradient and hessian with finite differences of objective) ############################################################################################ -function evaluate!(objective, gradient, hessian, ensemble::SemEnsemble, params) - isnothing(objective) || (objective = zero(objective)) - isnothing(gradient) || fill!(gradient, zero(eltype(gradient))) - isnothing(hessian) || fill!(hessian, zero(eltype(hessian))) - sem_grad = isnothing(gradient) ? nothing : similar(gradient) - sem_hess = isnothing(hessian) ? nothing : similar(hessian) - for (sem, weight) in zip(ensemble.sems, ensemble.weights) - sem_obj = evaluate!(objective, sem_grad, sem_hess, sem, params) - isnothing(objective) || (objective += weight * sem_obj) - isnothing(gradient) || (gradient .+= weight * sem_grad) - isnothing(hessian) || (hessian .+= weight * sem_hess) - end - return objective +# evaluate!() wrapper that does some housekeeping, if necessary +_evaluate!(args...) = evaluate!(args...) + +# update implied state, its gradient and hessian +function _evaluate!(objective, gradient, hessian, loss::SemLoss, params) + # note that any other Sem loss terms that are dependent on implied + # should be enumerated after the SemLoss term + # otherwise they would be using outdated implied state + update!(EvaluationTargets(objective, gradient, hessian), implied(loss), params) + return evaluate!(objective, gradient, hessian, loss, params) end +objective(model::AbstractSemOrLoss, params) = + _evaluate!(objective_zero(model, params), nothing, nothing, model, params) + +# throw an error by default if gradient! and hessian! are not implemented + +#= gradient!(model::AbstractSemOrLoss, par, model) = + throw(ArgumentError("gradient for $(nameof(typeof(model))) is not available")) + +hessian!(model::AbstractSemOrLoss, par, model) = + throw(ArgumentError("hessian for $(nameof(typeof(model))) is not available")) =# + ############################################################################################ # Documentation ############################################################################################ """ objective!(model::AbstractSem, params) -Returns the objective value at `params`. -The model object can be modified. +Calculates the objective value at `params`. -# Implementation -To implement a new `SemImplied` or `SemLossFunction` subtype, you need to add a method for - objective!(newtype::MyNewType, params, model::AbstractSemSingle) +The model object can be modified during calculation. -To implement a new `AbstractSem` subtype, you need to add a method for - objective!(model::MyNewType, params) +See also [`evaluate!`](@ref). """ function objective! end """ gradient!(gradient, model::AbstractSem, params) -Writes the gradient value at `params` to `gradient`. +Calculates the model's gradient at `params` and writes it to `gradient`. -# Implementation -To implement a new `SemImplied` or `SemLossFunction` type, you can add a method for - gradient!(newtype::MyNewType, params, model::AbstractSemSingle) +The model object can be modified during calculation. -To implement a new `AbstractSem` subtype, you can add a method for - gradient!(gradient, model::MyNewType, params) +See also [`evaluate!`](@ref). """ function gradient! end """ hessian!(hessian, model::AbstractSem, params) -Writes the hessian value at `params` to `hessian`. +Calculates the model's hessian at `params` and writes it to `hessian`. -# Implementation -To implement a new `SemImplied` or `SemLossFunction` type, you can add a method for - hessian!(newtype::MyNewType, params, model::AbstractSemSingle) +The model object can be modified during calculation. -To implement a new `AbstractSem` subtype, you can add a method for - hessian!(hessian, model::MyNewType, params) +See also [`evaluate!`](@ref). """ function hessian! end objective!(model::AbstractSem, params) = - evaluate!(objective_zero(model, params), nothing, nothing, model, params) + _evaluate!(objective_zero(model, params), nothing, nothing, model, params) gradient!(gradient, model::AbstractSem, params) = - evaluate!(nothing, gradient, nothing, model, params) + _evaluate!(nothing, gradient, nothing, model, params) hessian!(hessian, model::AbstractSem, params) = - evaluate!(nothing, nothing, hessian, model, params) + _evaluate!(nothing, nothing, hessian, model, params) objective_gradient!(gradient, model::AbstractSem, params) = - evaluate!(objective_zero(model, params), gradient, nothing, model, params) + _evaluate!(objective_zero(model, params), gradient, nothing, model, params) objective_hessian!(hessian, model::AbstractSem, params) = - evaluate!(objective_zero(model, params), nothing, hessian, model, params) + _evaluate!(objective_zero(model, params), nothing, hessian, model, params) gradient_hessian!(gradient, hessian, model::AbstractSem, params) = - evaluate!(nothing, gradient, hessian, model, params) + _evaluate!(nothing, gradient, hessian, model, params) objective_gradient_hessian!(gradient, hessian, model::AbstractSem, params) = - evaluate!(objective_zero(model, params), gradient, hessian, model, params) + _evaluate!(objective_zero(model, params), gradient, hessian, model, params) diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index 0c7913c48..6774e5494 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -137,13 +137,13 @@ fit(model::AbstractSem; engine::Symbol = :Optim, start_val = nothing, kwargs...) fit(optim::SemOptimizer, model::AbstractSem, start_params; kwargs...) = error("Optimizer $(optim) support not implemented.") -# FABIN3 is the default method for single models -prepare_start_params(start_val::Nothing, model::AbstractSemSingle; kwargs...) = - start_fabin3(model; kwargs...) - -# simple algorithm is the default method for ensembles -prepare_start_params(start_val::Nothing, model::AbstractSem; kwargs...) = - start_simple(model; kwargs...) +# defaults when no starting parameters are specified +function prepare_start_params(start_val::Nothing, model::AbstractSem; kwargs...) + sems = sem_terms(model) + # FABIN3 for single models, simple algorithm for ensembles + return length(sems) == 1 ? start_fabin3(loss(sems[1]); kwargs...) : + start_simple(model; kwargs...) +end # first argument is a function prepare_start_params(start_val, model::AbstractSem; kwargs...) = start_val(model; kwargs...) diff --git a/src/types.jl b/src/types.jl index 3a6b5fdf1..87b733cf4 100644 --- a/src/types.jl +++ b/src/types.jl @@ -1,14 +1,6 @@ ############################################################################################ # Define the basic type system ############################################################################################ -"Most abstract supertype for all SEMs" -abstract type AbstractSem end - -"Supertype for all single SEMs, e.g. SEMs that have at least the fields `observed`, `implied`, `loss`" -abstract type AbstractSemSingle{O, I, L} <: AbstractSem end - -"Supertype for all collections of multiple SEMs" -abstract type AbstractSemCollection <: AbstractSem end "Meanstructure trait for `SemImplied` subtypes" abstract type MeanStruct end @@ -36,48 +28,8 @@ HessianEval(::Type{T}) where {T} = HessianEval(semobj) = HessianEval(typeof(semobj)) -"Supertype for all loss functions of SEMs. If you want to implement a custom loss function, it should be a subtype of `SemLossFunction`." -abstract type SemLossFunction end - -""" - SemLoss(args...; loss_weights = nothing, ...) - -Constructs the loss field of a SEM. Can contain multiple `SemLossFunction`s, the model is optimized over their sum. -See also [`SemLossFunction`](@ref). - -# Arguments -- `args...`: Multiple `SemLossFunction`s. -- `loss_weights::Vector`: Weights for each loss function. Defaults to unweighted optimization. - -# Examples -```julia -my_ml_loss = SemML(...) -my_ridge_loss = SemRidge(...) -my_loss = SemLoss(SemML, SemRidge; loss_weights = [1.0, 2.0]) -``` -""" -mutable struct SemLoss{F <: Tuple, T} - functions::F - weights::T -end - -function SemLoss(functions...; loss_weights = nothing, kwargs...) - if !isnothing(loss_weights) - loss_weights = SemWeight.(loss_weights) - else - loss_weights = Tuple(SemWeight(nothing) for _ in 1:length(functions)) - end - - return SemLoss(functions, loss_weights) -end - -# weights for loss functions or models. If the weight is nothing, multiplication returns the second argument -struct SemWeight{T} - w::T -end - -Base.:*(x::SemWeight{Nothing}, y) = y -Base.:*(x::SemWeight, y) = x.w * y +"Supertype for all loss functions of SEMs. If you want to implement a custom loss function, it should be a subtype of `AbstractLoss`." +abstract type AbstractLoss end abstract type SemOptimizer{E} end @@ -85,6 +37,8 @@ abstract type SemOptimizer{E} end abstract type SemOptimizerResult{O <: SemOptimizer} end """ + abstract type SemObserved + Supertype of all objects that can serve as the observed field of a SEM. Pre-processes data and computes sufficient statistics for example. If you have a special kind of data, e.g. ordinal data, you should implement a subtype of SemObserved. @@ -103,169 +57,90 @@ abstract type SemImplied end abstract type SemImpliedSymbolic <: SemImplied end """ - Sem(;observed = SemObservedData, implied = RAM, loss = SemML, kwargs...) + abstract type SemLoss{O <: SemObserved, I <: SemImplied} <: AbstractLoss -Constructor for the basic `Sem` type. -All additional kwargs are passed down to the constructors for the observed, implied, and loss fields. +The base type for calculating the loss of the implied SEM model when explaining the observed data. -# Arguments -- `observed`: object of subtype `SemObserved` or a constructor. -- `implied`: object of subtype `SemImplied` or a constructor. -- `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such. - -Returns a Sem with fields -- `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref). -- `implied::SemImplied`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImplied`](@ref). -- `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref). +All subtypes of `SemLoss` should have the following fields: +- `observed::O`: object of subtype [`SemObserved`](@ref). +- `implied::I`: object of subtype [`SemImplied`](@ref). """ -mutable struct Sem{O <: SemObserved, I <: SemImplied, L <: SemLoss} <: - AbstractSemSingle{O, I, L} - observed::O - implied::I - loss::L -end +abstract type SemLoss{O <: SemObserved, I <: SemImplied} <: AbstractLoss end -############################################################################################ -# automatic differentiation -############################################################################################ """ - SemFiniteDiff(;observed = SemObservedData, implied = RAM, loss = SemML, kwargs...) + abstract type AbstractSem -A wrapper around [`Sem`](@ref) that substitutes dedicated evaluation of gradient and hessian with -finite difference approximation. +The base type for all SEMs. +""" +abstract type AbstractSem end -# Arguments -- `observed`: object of subtype `SemObserved` or a constructor. -- `implied`: object of subtype `SemImplied` or a constructor. -- `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such. - -Returns a Sem with fields -- `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref). -- `implied::SemImplied`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImplied`](@ref). -- `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref). """ -struct SemFiniteDiff{O <: SemObserved, I <: SemImplied, L <: SemLoss} <: - AbstractSemSingle{O, I, L} - observed::O - implied::I + struct LossTerm{L, I, W} + +A term of a [`Sem`](@ref) model that wraps [`AbstractLoss`](@ref) loss function of type `L`. +Loss term can have an optional *id* of type `I` and *weight* of numeric type `W`. +""" +struct LossTerm{L <: AbstractLoss, I <: Union{Symbol, Nothing}, W <: Union{Number, Nothing}} loss::L + id::I + weight::W end -############################################################################################ -# ensemble models -############################################################################################ """ - (1) SemEnsemble(models...; weights = nothing, groups = nothing, kwargs...) + Sem(loss_terms...; [params], kwargs...) - (2) SemEnsemble(;specification, data, groups, column = :group, kwargs...) +SEM model (including multi-group SEMs) that combines all the data, implied SEM structure +and regularization terms. -Constructor for ensemble models. (2) can be used to conveniently specify multigroup models. +All terms of the `Sem` object share the same set of parameters. +`Sem` implements the calculation of the weighted sum of its terms (the *objective* +function), as well as the gradient and Hessian of this sum. # Arguments -- `models...`: `AbstractSem`s. -- `weights::Vector`: Weights for each model. Defaults to the number of observed data points. -- `specification::EnsembleParameterTable`: Model specification. -- `data::DataFrame`: Observed data. Must contain a `column` of type `Vector{Symbol}` that contains the group. -- `groups::Vector{Symbol}`: Group names. -- `column::Symbol`: Name of the column in `data` that contains the group. - -All additional kwargs are passed down to the model parts. - -Returns a SemEnsemble with fields -- `n::Int`: Number of models. -- `sems::Tuple`: `AbstractSem`s. -- `weights::Vector`: Weights for each model. -- `param_labels::Vector`: Stores parameter labels and their position. - -For instructions on multigroup models, see the online documentation. +- `loss_terms...`: [`AbstractLoss`](@ref) objects, including SEM losses ([`SemLoss`](@ref)), + optionally can be a pair of a loss object and its numeric weight + +# Fields +- `loss_terms::Tuple`: a tuple of all loss functions and their weights +- `params::Vector{Symbol}`: the vector of parameter ids shared by all loss functions. """ -struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, I, G <: Vector{Symbol}} <: - AbstractSemCollection - n::N - sems::T - weights::V - param_labels::I - groups::G +struct Sem{L <: Tuple} <: AbstractSem + loss_terms::L + params::Vector{Symbol} end -# constructor from multiple models -function SemEnsemble(models...; weights = nothing, groups = nothing, kwargs...) - n = length(models) - # default weights - weights = isnothing(weights) ? multigroup_weights(models, n) : weights - # default group labels - groups = isnothing(groups) ? Symbol.(:g, 1:n) : groups - # check parameters equality - param_labels = SEM.param_labels(models[1]) - for model in models - if param_labels != SEM.param_labels(model) - throw(ErrorException("The parameters of your models do not match. \n - Maybe you tried to specify models of an ensemble via ParameterTables. \n - In that case, you may use RAMMatrices instead.")) - end - end - - return SemEnsemble(n, models, weights, param_labels, groups) -end +############################################################################################ +# automatic differentiation +############################################################################################ -# constructor from EnsembleParameterTable and data set -function SemEnsemble(; specification, data, groups, column = :group, kwargs...) - if specification isa EnsembleParameterTable - specification = convert(Dict{Symbol, RAMMatrices}, specification) - end - models = [] - for group in groups - ram_matrices = specification[group] - data_group = select(filter(r -> r[column] == group, data), Not(column)) - if iszero(nrow(data_group)) - error("Your data does not contain any observations from group `$(group)`.") - end - model = Sem(; specification = ram_matrices, data = data_group, kwargs...) - push!(models, model) - end - return SemEnsemble(models...; groups = groups, kwargs...) -end +""" + SemFiniteDiff(model::AbstractSem) -function multigroup_weights(models, n) - nsamples_total = sum(nsamples, models) - uniform_lossfun = check_single_lossfun(models...; throw_error = false) - if !uniform_lossfun - @info "Your ensemble model contains heterogeneous loss functions. - Default weights of (#samples per group/#total samples) will be used." - return [(nsamples(model)) / (nsamples_total) for model in models] - end - lossfun = models[1].loss.functions[1] - if !applicable(mg_correction, lossfun) - @info "We don't know how to choose group weights for the specified loss function. - Default weights of (#samples per group/#total samples) will be used." - return [(nsamples(model)) / (nsamples_total) for model in models] - end - c = mg_correction(lossfun) - return [(nsamples(model)+c) / (nsamples_total+n*c) for model in models] -end +A wrapper around [`AbstractSem`](@ref) that substitutes dedicated evaluation of gradient and +hessian with finite difference approximation. -param_labels(ensemble::SemEnsemble) = ensemble.param_labels +`SemFiniteDiff` could be used to enable gradient-based optimization of the SEM models +when the dedicated calculation of gradient and hessian are not available. +For approximation, it uses the *FiniteDiff.jl* package. +# Arguments +- `model::Sem`: the SEM model to wrap """ - n_models(ensemble::SemEnsemble) -> Integer +struct SemFiniteDiff{S <: AbstractSem} <: AbstractSem + model::S +end -Returns the number of models in an ensemble model. -""" -n_models(ensemble::SemEnsemble) = ensemble.n -""" - models(ensemble::SemEnsemble) -> Tuple{AbstractSem} +struct LossFiniteDiff{L <: AbstractLoss} <: AbstractLoss + loss::L +end -Returns the models in an ensemble model. -""" -models(ensemble::SemEnsemble) = ensemble.sems -""" - weights(ensemble::SemEnsemble) -> Vector +struct SemLossFiniteDiff{O, I, L <: SemLoss{O, I}} <: SemLoss{O, I} + loss::L +end -Returns the weights of an ensemble model. """ -weights(ensemble::SemEnsemble) = ensemble.weights + abstract type SemSpecification end -""" Base type for all SEM specifications. """ abstract type SemSpecification end diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index bb7db3b50..48723fbef 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -4,34 +4,36 @@ const SEM = StructuralEquationModels # ML estimation ############################################################################################ -model_g1 = Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic) +obs_g1 = SemObservedData(data = dat_g1, observed_vars = SEM.observed_vars(specification_g1)) +obs_g2 = SemObservedData(data = dat_g2, observed_vars = SEM.observed_vars(specification_g2)) -model_g2 = Sem(specification = specification_g2, data = dat_g2, implied = RAM) +model_ml_multigroup = Sem( + :Pasteur => SemML(obs_g1, RAMSymbolic(specification_g1)), + :Grant_White => SemML(obs_g2, RAM(specification_g2)), +) -@test SEM.param_labels(model_g1.implied.ram_matrices) == - SEM.param_labels(model_g2.implied.ram_matrices) +@testset "Sem API" begin + @test SEM.nsamples(model_ml_multigroup) == nsamples(obs_g1) + nsamples(obs_g2) + @test SEM.nsem_terms(model_ml_multigroup) == 2 + @test length(SEM.sem_terms(model_ml_multigroup)) == 2 +end -# test the different constructors -model_ml_multigroup = SemEnsemble(model_g1, model_g2; groups = [:Pasteur, :Grant_White]) -model_ml_multigroup2 = SemEnsemble( - specification = partable, - data = dat, - column = :school, - groups = [:Pasteur, :Grant_White], - loss = SemML, +# replace observed using Dict of data matrices +model_ml_multigroup3 = replace_observed( + model_ml_multigroup, + Dict(:Pasteur => dat_g1, :Grant_White => dat_g2), ) -model_ml_multigroup3 = replace_observed( - model_ml_multigroup2, - column = :school, - specification = partable, - data = dat, +# replace observed using DataFrame with group column +model_ml_multigroup4 = replace_observed( + model_ml_multigroup, + dat; + semterm_column = :school, ) # gradients @testset "ml_gradients_multigroup" begin test_gradient(model_ml_multigroup, start_test; atol = 1e-9) - test_gradient(model_ml_multigroup2, start_test; atol = 1e-9) end # fit @@ -44,50 +46,18 @@ end atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = fit(semoptimizer, model_ml_multigroup2) - update_estimate!(partable, solution) - test_estimates( - partable, - solution_lav[:parameter_estimates_ml]; - atol = 1e-4, - lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), - ) end @testset "replace_observed_multigroup" begin sem_fit_1 = fit(semoptimizer, model_ml_multigroup) - sem_fit_2 = fit(semoptimizer, model_ml_multigroup3) - @test sem_fit_1.solution ≈ sem_fit_2.solution + sem_fit_3 = fit(semoptimizer, model_ml_multigroup3) + @test sem_fit_1.solution ≈ sem_fit_3.solution + sem_fit_4 = fit(semoptimizer, model_ml_multigroup4) + @test sem_fit_1.solution ≈ sem_fit_4.solution end @testset "fitmeasures/se_ml" begin - solution_ml = fit(model_ml_multigroup) - test_fitmeasures( - fit_measures(solution_ml), - solution_lav[:fitmeasures_ml]; - rtol = 1e-2, - atol = 1e-7, - ) - test_fitmeasures( - Dict(:CFI => CFI(solution_ml)), - solution_lav[:fitmeasures_ml]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) - - update_se_hessian!(partable, solution_ml) - test_estimates( - partable, - solution_lav[:parameter_estimates_ml]; - atol = 1e-3, - col = :se, - lav_col = :se, - lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), - ) - - test_bootstrap(solution_ml, partable; rtol_hessian = 0.3) - smoketest_CI_z(solution_ml, partable) - - solution_ml = fit(model_ml_multigroup2) + solution_ml = fit(semoptimizer, model_ml_multigroup) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; @@ -118,15 +88,19 @@ end partable_s = sort_vars(partable) specification_s = convert(Dict{Symbol, RAMMatrices}, partable_s) +obs_g1_s = SemObservedData( + data = dat_g1, + observed_vars = SEM.observed_vars(specification_s[:Pasteur]), +) +obs_g2_s = SemObservedData( + data = dat_g2, + observed_vars = SEM.observed_vars(specification_s[:Grant_White]), +) -specification_g1_s = specification_s[:Pasteur] -specification_g2_s = specification_s[:Grant_White] - -model_g1 = Sem(specification = specification_g1_s, data = dat_g1, implied = RAMSymbolic) - -model_g2 = Sem(specification = specification_g2_s, data = dat_g2, implied = RAM) - -model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) +model_ml_multigroup = Sem( + SemML(obs_g1_s, RAMSymbolic(specification_s[:Pasteur])), + SemML(obs_g2_s, RAM(specification_s[:Grant_White])), +) # gradients @testset "ml_gradients_multigroup | sorted" begin @@ -142,7 +116,7 @@ grad_fd = FiniteDiff.finite_difference_gradient( # fit @testset "ml_solution_multigroup | sorted" begin - solution = fit(model_ml_multigroup) + solution = fit(semoptimizer, model_ml_multigroup) update_estimate!(partable_s, solution) test_estimates( partable_s, @@ -153,7 +127,7 @@ grad_fd = FiniteDiff.finite_difference_gradient( end @testset "fitmeasures/se_ml | sorted" begin - solution_ml = fit(model_ml_multigroup) + solution_ml = fit(semoptimizer, model_ml_multigroup) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; @@ -178,28 +152,26 @@ end end @testset "sorted | LowerTriangular A" begin - @test implied(model_ml_multigroup.sems[2]).A isa LowerTriangular + @test implied(SEM.sem_terms(model_ml_multigroup)[2]).A isa LowerTriangular end ############################################################################################ # ML estimation - user defined loss function ############################################################################################ -struct UserSemML <: SemLossFunction +struct UserSemML{O, I} <: SemLoss{O, I} hessianeval::ExactHessian - UserSemML() = new(ExactHessian()) -end - -############################################################################################ -### functors -############################################################################################ + observed::O + implied::I -using LinearAlgebra: isposdef, logdet, tr, inv + UserSemML(observed::SemObserved, implied::SemImplied) = + new{typeof(observed), typeof(implied)}(ExactHessian(), observed, implied) +end -function SEM.objective(ml::UserSemML, model::AbstractSem, params) - Σ = implied(model).Σ - Σₒ = SEM.obs_cov(observed(model)) +function SEM.objective(ml::UserSemML, params) + Σ = implied(ml).Σ + Σₒ = SEM.obs_cov(observed(ml)) if !isposdef(Σ) return Inf else @@ -208,24 +180,18 @@ function SEM.objective(ml::UserSemML, model::AbstractSem, params) end # models -model_g1 = Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic) - -model_g2 = SemFiniteDiff( - specification = specification_g2, - data = dat_g2, - implied = RAMSymbolic, - loss = UserSemML(), +model_ml_multigroup = Sem( + SemML(obs_g1, RAMSymbolic(specification_g1)), + SEM.FiniteDiffWrapper(UserSemML(obs_g2, RAMSymbolic(specification_g2))), ) -model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) - @testset "gradients_user_defined_loss" begin test_gradient(model_ml_multigroup, start_test; atol = 1e-9) end # fit @testset "solution_user_defined_loss" begin - solution = fit(model_ml_multigroup) + solution = fit(semoptimizer, model_ml_multigroup) update_estimate!(partable, solution) test_estimates( partable, @@ -239,25 +205,9 @@ end # GLS estimation ############################################################################################ -model_ls_g1 = Sem( - specification = specification_g1, - data = dat_g1, - implied = RAMSymbolic, - loss = SemWLS, -) - -model_ls_g2 = Sem( - specification = specification_g2, - data = dat_g2, - implied = RAMSymbolic, - loss = SemWLS, -) - -model_ls_multigroup = SemEnsemble( - model_ls_g1, - model_ls_g2; - groups = [:Pasteur, :Grant_White], - optimizer = semoptimizer, +model_ls_multigroup = Sem( + SemWLS(obs_g1, RAMSymbolic(specification_g1, vech = true)), + SemWLS(obs_g2, RAMSymbolic(specification_g2, vech = true)), ) @testset "ls_gradients_multigroup" begin @@ -265,7 +215,7 @@ model_ls_multigroup = SemEnsemble( end @testset "ls_solution_multigroup" begin - solution = fit(model_ls_multigroup) + solution = fit(semoptimizer, model_ls_multigroup) update_estimate!(partable, solution) test_estimates( partable, @@ -276,7 +226,7 @@ end end @testset "fitmeasures/se_ls" begin - solution_ls = fit(model_ls_multigroup) + solution_ls = fit(semoptimizer, model_ls_multigroup) test_fitmeasures( fit_measures(solution_ls), solution_lav[:fitmeasures_ls]; @@ -308,40 +258,27 @@ end ############################################################################################ if !isnothing(specification_miss_g1) - model_g1 = Sem( - specification = specification_miss_g1, - observed = SemObservedMissing, - loss = SemFIML, - data = dat_miss_g1, - implied = RAM, - meanstructure = true, - ) - - model_g2 = Sem( - specification = specification_miss_g2, - observed = SemObservedMissing, - loss = SemFIML, - data = dat_miss_g2, - implied = RAM, - meanstructure = true, - ) - - model_ml_multigroup = SemEnsemble(model_g1, model_g2) - model_ml_multigroup2 = SemEnsemble( - specification = partable_miss, - data = dat_missing, - column = :school, - groups = [:Pasteur, :Grant_White], - loss = SemFIML, - observed = SemObservedMissing, - meanstructure = true, + model_ml_multigroup = Sem( + SemFIML( + SemObservedMissing( + data = dat_miss_g1, + observed_vars = SEM.observed_vars(specification_miss_g1), + ), + RAM(specification_miss_g1), + ), + SemFIML( + SemObservedMissing( + data = dat_miss_g2, + observed_vars = SEM.observed_vars(specification_miss_g2), + ), + RAM(specification_miss_g2), + ), ) - model_ml_varonly = SemEnsemble( + model_ml_varonly = Sem( specification = partable_varonly, data = dat_missing, - column = :school, - groups = [:Pasteur, :Grant_White], + semterm_column = :school, loss = SemFIML, observed = SemObservedMissing, meanstructure = true, @@ -373,7 +310,6 @@ if !isnothing(specification_miss_g1) @testset "fiml_gradients_multigroup" begin test_gradient(model_ml_multigroup, start_test; atol = 1e-7) - test_gradient(model_ml_multigroup2, start_test; atol = 1e-7) end @testset "fiml_solution_multigroup" begin @@ -385,14 +321,6 @@ if !isnothing(specification_miss_g1) atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = fit(semoptimizer, model_ml_multigroup2) - update_estimate!(partable_miss, solution) - test_estimates( - partable_miss, - solution_lav[:parameter_estimates_fiml]; - atol = 1e-4, - lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), - ) end @testset "fitmeasures/se_fiml" begin diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index d2d468a98..6866eead5 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -2,103 +2,83 @@ ### models w.o. meanstructure ############################################################################################ -# observed --------------------------------------------------------------------------------- -observed = SemObservedData(specification = spec, data = dat) +semoptimizer = SemOptimizer(engine = opt_engine) -# implied -implied_ram = RAM(specification = spec) +model_ml = Sem(specification = spec, data = dat) +@test SEM.params(model_ml) == SEM.params(spec) -implied_ram_sym = RAMSymbolic(specification = spec) +model_ls_sym = + Sem(specification = spec, data = dat, implied = RAMSymbolic, vech = true, loss = SemWLS) -# loss functions --------------------------------------------------------------------------- -ml = SemML(observed = observed) +model_ml_sym = Sem(specification = spec, data = dat, implied = RAMSymbolic) -wls = SemWLS(observed = observed) - -ridge = SemRidge(α_ridge = 0.001, which_ridge = 16:20, nparams = 31) - -constant = SemConstant(constant_loss = 3.465) - -# loss ------------------------------------------------------------------------------------- -loss_ml = SemLoss(ml) - -loss_wls = SemLoss(wls) - -# optimizer ------------------------------------------------------------------------------------- -optimizer_obj = SemOptimizer(engine = opt_engine) - -# models ----------------------------------------------------------------------------------- - -model_ml = Sem(observed, implied_ram, loss_ml) - -model_ls_sym = Sem(observed, RAMSymbolic(specification = spec, vech = true), loss_wls) - -model_ml_sym = Sem(observed, implied_ram_sym, loss_ml) - -model_ridge = Sem(observed, implied_ram, SemLoss(ml, ridge)) +model_ml_ridge = Sem( + specification = spec, + data = dat, + loss = (SemML, SemRidge), + α_ridge = 0.001, + which_ridge = 16:20, +) -model_constant = Sem(observed, implied_ram, SemLoss(ml, constant)) +model_ml_const = Sem( + specification = spec, + data = dat, + loss = (SemML, SemConstant), + constant_loss = 3.465, +) -model_ml_weighted = - Sem(observed, implied_ram, SemLoss(ml; loss_weights = [nsamples(model_ml)])) +model_ml_weighted = Sem(SemML(SemObservedData(data = dat), RAM(spec)) => nsamples(model_ml)) ############################################################################################ ### test gradients ############################################################################################ -models = - [model_ml, model_ls_sym, model_ridge, model_constant, model_ml_sym, model_ml_weighted] -model_names = ["ml", "ls_sym", "ridge", "constant", "ml_sym", "ml_weighted"] +models = Dict( + "ml" => model_ml, + "ls_sym" => model_ls_sym, + "ml_ridge" => model_ml_ridge, + "ml_const" => model_ml_const, + "ml_sym" => model_ml_sym, + "ml_weighted" => model_ml_weighted, +) -for (model, name) in zip(models, model_names) - try - @testset "$(name)_gradient" begin - test_gradient(model, start_test; rtol = 1e-9) - end - catch - end +@testset "$(id)_gradient" for (id, model) in pairs(models) + test_gradient(model, start_test; rtol = 1e-9) end ############################################################################################ ### test solution ############################################################################################ -models = [model_ml, model_ls_sym, model_ml_sym, model_constant] -model_names = ["ml", "ls_sym", "ml_sym", "constant"] -solution_names = Symbol.("parameter_estimates_" .* ["ml", "ls", "ml", "ml"]) - -for (model, name, solution_name) in zip(models, model_names, solution_names) - try - @testset "$(name)_solution" begin - solution = fit(optimizer_obj, model) - update_estimate!(partable, solution) - test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) - end - catch - end +@testset "$(id)_solution" for id in ["ml", "ls_sym", "ml_sym", "ml_const"] + model = models[id] + solution = fit(semoptimizer, model) + sol_name = Symbol("parameter_estimates_", replace(id, r"_.+$" => "")) + update_estimate!(partable, solution) + test_estimates(partable, solution_lav[sol_name]; atol = 1e-2) end @testset "ridge_solution" begin - solution_ridge = fit(optimizer_obj, model_ridge) - solution_ml = fit(optimizer_obj, model_ml) - # solution_ridge_id = fit(optimizer_obj, model_ridge_id) - @test solution_ridge.minimum < solution_ml.minimum + 1 + solution_ridge = fit(semoptimizer, model_ml_ridge) + solution_ml = fit(semoptimizer, model_ml) + # solution_ridge_id = fit(model_ridge_id) + @test abs(solution_ridge.minimum - solution_ml.minimum) < 1 end # test constant objective value @testset "constant_objective_and_gradient" begin - @test (objective!(model_constant, start_test) - 3.465) ≈ + @test (objective!(model_ml_const, start_test) - 3.465) ≈ objective!(model_ml, start_test) grad = similar(start_test) grad2 = similar(start_test) - gradient!(grad, model_constant, start_test) + gradient!(grad, model_ml_const, start_test) gradient!(grad2, model_ml, start_test) @test grad ≈ grad2 end @testset "ml_solution_weighted" begin - solution_ml = fit(optimizer_obj, model_ml) - solution_ml_weighted = fit(optimizer_obj, model_ml_weighted) + solution_ml = fit(semoptimizer, model_ml) + solution_ml_weighted = fit(semoptimizer, model_ml_weighted) @test solution(solution_ml) ≈ solution(solution_ml_weighted) rtol = 1e-3 @test nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml) ≈ StructuralEquationModels.minimum(solution_ml_weighted) rtol = 1e-6 @@ -109,7 +89,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml" begin - solution_ml = fit(optimizer_obj, model_ml) + solution_ml = fit(semoptimizer, model_ml) test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) test_fitmeasures( Dict(:CFI => CFI(solution_ml)), @@ -128,20 +108,15 @@ end end @testset "fitmeasures/se_ls" begin - solution_ls = fit(optimizer_obj, model_ls_sym) + solution_ls = fit(semoptimizer, model_ls_sym) fm = fit_measures(solution_ls) test_fitmeasures( - fm, + merge(fm, Dict(:CFI => CFI(solution_ls))), solution_lav[:fitmeasures_ls]; atol = 1e-3, - fitmeasure_names = fitmeasure_names_ls, - ) - test_fitmeasures( - Dict(:CFI => CFI(solution_ls)), - solution_lav[:fitmeasures_ls]; - fitmeasure_names = Dict(:CFI => "cfi"), + fitmeasure_names = merge(fitmeasure_names_ls, Dict(:CFI => "cfi")) ) - @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) + @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) @suppress update_se_hessian!(partable, solution_ls) test_estimates( @@ -160,22 +135,22 @@ end if opt_engine == :Optim using Optim, LineSearches - optimizer_obj = SemOptimizer( - engine = opt_engine, - algorithm = Newton(; - linesearch = BackTracking(order = 3), - alphaguess = InitialHagerZhang(), - ), + model_ls = Sem( + data = dat, + specification = spec, + implied = RAMSymbolic, + loss = SemWLS, + vech = true, + hessian = true, ) - implied_sym_hessian_vech = - RAMSymbolic(specification = spec, vech = true, hessian = true) - - implied_sym_hessian = RAMSymbolic(specification = spec, hessian = true) - - model_ls = Sem(observed, implied_sym_hessian_vech, loss_wls) - - model_ml = Sem(observed, implied_sym_hessian, loss_ml) + model_ml = Sem( + data = dat, + specification = spec, + implied = RAMSymbolic, + loss = SemML, + hessian = true, + ) @testset "ml_hessians" begin test_hessian(model_ml, start_test; atol = 1e-4) @@ -186,13 +161,23 @@ if opt_engine == :Optim end @testset "ml_solution_hessian" begin - solution = fit(optimizer_obj, model_ml) + solution = fit(SemOptimizer(engine = :Optim, algorithm = Newton()), model_ml) + update_estimate!(partable, solution) test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-2) end @testset "ls_solution_hessian" begin - solution = fit(optimizer_obj, model_ls) + solution = fit( + SemOptimizer( + engine = :Optim, + algorithm = Newton( + linesearch = BackTracking(order = 3), + alphaguess = InitialHagerZhang(), + ), + ), + model_ls, + ) update_estimate!(partable, solution) test_estimates( partable, @@ -207,69 +192,47 @@ end ### meanstructure ############################################################################################ -# observed --------------------------------------------------------------------------------- -observed = SemObservedData(specification = spec_mean, data = dat, meanstructure = true) - -# implied -implied_ram = RAM(specification = spec_mean, meanstructure = true) - -implied_ram_sym = RAMSymbolic(specification = spec_mean, meanstructure = true) - -# loss functions --------------------------------------------------------------------------- -ml = SemML(observed = observed, meanstructure = true) - -wls = SemWLS(observed = observed, meanstructure = true) - -# loss ------------------------------------------------------------------------------------- -loss_ml = SemLoss(ml) - -loss_wls = SemLoss(wls) - -# optimizer ------------------------------------------------------------------------------------- -optimizer_obj = SemOptimizer(engine = opt_engine) +# models +model_ls = Sem( + data = dat, + specification = spec_mean, + implied = RAMSymbolic, + loss = SemWLS, + vech = true, +) -# models ----------------------------------------------------------------------------------- -model_ml = Sem(observed, implied_ram, loss_ml) +model_ml = Sem(data = dat, specification = spec_mean, implied = RAM, loss = SemML) -model_ls = Sem( - observed, - RAMSymbolic(specification = spec_mean, meanstructure = true, vech = true), - loss_wls, +model_ml_cov = Sem( + specification = spec, + observed = SemObservedCovariance, + obs_cov = cov(Matrix(dat)), + observed_vars = Symbol.(names(dat)), + nsamples = 75, ) -model_ml_sym = Sem(observed, implied_ram_sym, loss_ml) +model_ml_sym = + Sem(data = dat, specification = spec_mean, implied = RAMSymbolic, loss = SemML) ############################################################################################ ### test gradients ############################################################################################ -models = [model_ml, model_ls, model_ml_sym] -model_names = ["ml", "ls_sym", "ml_sym"] +models = Dict("ml" => model_ml, "ls_sym" => model_ls, "ml_sym" => model_ml_sym) -for (model, name) in zip(models, model_names) - try - @testset "$(name)_gradient_mean" begin - test_gradient(model, start_test_mean; rtol = 1e-9) - end - catch - end +@testset "$(id)_gradient_mean" for (id, model) in pairs(models) + test_gradient(model, start_test_mean; rtol = 1e-9) end ############################################################################################ ### test solution ############################################################################################ -solution_names = Symbol.("parameter_estimates_" .* ["ml", "ls", "ml"] .* "_mean") - -for (model, name, solution_name) in zip(models, model_names, solution_names) - try - @testset "$(name)_solution_mean" begin - solution = fit(optimizer_obj, model) - update_estimate!(partable_mean, solution) - test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) - end - catch - end +@testset "$(id)_solution_mean" for (id, model) in pairs(models) + solution = fit(semoptimizer, model, start_val = start_test_mean) + update_estimate!(partable_mean, solution) + sol_name = Symbol("parameter_estimates_", replace(id, r"_.+$" => ""), "_mean") + test_estimates(partable_mean, solution_lav[sol_name]; atol = 1e-2) end ############################################################################################ @@ -277,7 +240,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml_mean" begin - solution_ml = fit(optimizer_obj, model_ml) + solution_ml = fit(semoptimizer, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; @@ -300,20 +263,15 @@ end end @testset "fitmeasures/se_ls_mean" begin - solution_ls = fit(optimizer_obj, model_ls) + solution_ls = fit(semoptimizer, model_ls) fm = fit_measures(solution_ls) test_fitmeasures( - fm, + merge(fm, Dict(:CFI => CFI(solution_ls))), solution_lav[:fitmeasures_ls_mean]; atol = 1e-3, - fitmeasure_names = fitmeasure_names_ls, - ) - test_fitmeasures( - Dict(:CFI => CFI(solution_ls)), - solution_lav[:fitmeasures_ls_mean]; - fitmeasure_names = Dict(:CFI => "cfi"), + fitmeasure_names = merge(fitmeasure_names_ls, Dict(:CFI => "cfi")), ) - @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) + @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) @suppress update_se_hessian!(partable_mean, solution_ls) test_estimates( @@ -329,16 +287,22 @@ end ### fiml ############################################################################################ -observed = - SemObservedMissing(specification = spec_mean, data = dat_missing, rtol_em = 1e-10) - -fiml = SemFIML(observed = observed, specification = spec_mean) - -loss_fiml = SemLoss(fiml) - -model_ml = Sem(observed, implied_ram, loss_fiml) +# models +model_ml = Sem( + data = dat_missing, + observed = SemObservedMissing, + specification = spec_mean, + implied = RAM, + loss = SemFIML, +) -model_ml_sym = Sem(observed, implied_ram_sym, loss_fiml) +model_ml_sym = Sem( + data = dat_missing, + observed = SemObservedMissing, + specification = spec_mean, + implied = RAMSymbolic, + loss = SemFIML, +) ############################################################################################ ### test gradients @@ -357,13 +321,13 @@ end ############################################################################################ @testset "fiml_solution" begin - solution = fit(optimizer_obj, model_ml) + solution = fit(semoptimizer, model_ml) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @testset "fiml_solution_symbolic" begin - solution = fit(optimizer_obj, model_ml_sym) + solution = fit(semoptimizer, model_ml_sym, start_val = start_test_mean) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @@ -373,7 +337,7 @@ end ############################################################################################ @testset "fitmeasures/se_fiml" begin - solution_ml = fit(optimizer_obj, model_ml) + solution_ml = fit(semoptimizer, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; @@ -384,7 +348,7 @@ end test_estimates( partable_mean, solution_lav[:parameter_estimates_fiml]; - atol = 1e-3, + atol = 0.002, col = :se, lav_col = :se, ) diff --git a/test/examples/political_democracy/constraints.jl b/test/examples/political_democracy/constraints.jl index 7a6670fa3..0291e7ea5 100644 --- a/test/examples/political_democracy/constraints.jl +++ b/test/examples/political_democracy/constraints.jl @@ -50,7 +50,7 @@ end @test solution_constrained.solution[31] * solution_constrained.solution[30] >= (0.6 - 1e-8) - @test all(abs.(solution_constrained.solution) .< 10) - @test solution_constrained.optimization_result.result[3] == :FTOL_REACHED + @test all(p -> abs(p) < 10, solution_constrained.solution) + @test solution_constrained.optimization_result.result[3] == :FTOL_REACHED skip = true @test solution_constrained.minimum <= 21.21 + 0.01 end diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 1c1c42e54..25a6da914 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -5,21 +5,22 @@ semoptimizer = SemOptimizer(engine = opt_engine) model_ml = Sem(specification = spec, data = dat) -@test SEM.param_labels(model_ml.implied.ram_matrices) == SEM.param_labels(spec) +@test SEM.param_labels(model_ml) == SEM.param_labels(spec) model_ml_cov = Sem( specification = spec, observed = SemObservedCovariance, obs_cov = cov(Matrix(dat)), - obs_colnames = Symbol.(names(dat)), + observed_vars = Symbol.(names(dat)), nsamples = 75, ) -model_ls_sym = Sem(specification = spec, data = dat, implied = RAMSymbolic, loss = SemWLS) +model_ls_sym = + Sem(specification = spec, data = dat, implied = RAMSymbolic, vech = true, loss = SemWLS) model_ml_sym = Sem(specification = spec, data = dat, implied = RAMSymbolic) -model_ridge = Sem( +model_ml_ridge = Sem( specification = spec, data = dat, loss = (SemML, SemRidge), @@ -27,7 +28,7 @@ model_ridge = Sem( which_ridge = 16:20, ) -model_constant = Sem( +model_ml_const = Sem( specification = spec, data = dat, loss = (SemML, SemConstant), @@ -35,65 +36,52 @@ model_constant = Sem( ) model_ml_weighted = - Sem(specification = partable, data = dat, loss_weights = (nsamples(model_ml),)) + Sem(SemML(SemObservedData(data = dat), RAMSymbolic(spec)) => nsamples(model_ml)) ############################################################################################ ### test gradients ############################################################################################ -models = [ - model_ml, - model_ml_cov, - model_ls_sym, - model_ridge, - model_constant, - model_ml_sym, - model_ml_weighted, -] -model_names = ["ml", "ml_cov", "ls_sym", "ridge", "constant", "ml_sym", "ml_weighted"] - -for (model, name) in zip(models, model_names) - try - @testset "$(name)_gradient" begin - test_gradient(model, start_test; rtol = 1e-9) - end - catch - end +models = Dict( + "ml" => model_ml, + "ml_cov" => model_ml_cov, + "ls_sym" => model_ls_sym, + "ridge" => model_ml_ridge, + "ml_const" => model_ml_const, + "ml_sym" => model_ml_sym, + "ml_weighted" => model_ml_weighted, +) + +@testset "$(id)_gradient" for (id, model) in pairs(models) + test_gradient(model, start_test; rtol = 1e-9) end ############################################################################################ ### test solution ############################################################################################ -models = [model_ml, model_ml_cov, model_ls_sym, model_ml_sym, model_constant] -model_names = ["ml", "ml_cov", "ls_sym", "ml_sym", "constant"] -solution_names = Symbol.("parameter_estimates_" .* ["ml", "ml", "ls", "ml", "ml"]) - -for (model, name, solution_name) in zip(models, model_names, solution_names) - try - @testset "$(name)_solution" begin - solution = fit(semoptimizer, model) - update_estimate!(partable, solution) - test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) - end - catch - end +@testset "$(id)_solution" for id in ["ml", "ml_cov", "ls_sym", "ml_sym", "ml_const"] + model = models[id] + solution = fit(semoptimizer, model) + sol_name = Symbol("parameter_estimates_", replace(id, r"_.+$" => "")) + update_estimate!(partable, solution) + test_estimates(partable, solution_lav[sol_name]; atol = 1e-2) end @testset "ridge_solution" begin - solution_ridge = fit(semoptimizer, model_ridge) + solution_ridge = fit(semoptimizer, model_ml_ridge) solution_ml = fit(semoptimizer, model_ml) - # solution_ridge_id = fit(semoptimizer, model_ridge_id) + # solution_ridge_id = fit(model_ridge_id) @test abs(solution_ridge.minimum - solution_ml.minimum) < 1 end # test constant objective value @testset "constant_objective_and_gradient" begin - @test (objective!(model_constant, start_test) - 3.465) ≈ + @test (objective!(model_ml_const, start_test) - 3.465) ≈ objective!(model_ml, start_test) grad = similar(start_test) grad2 = similar(start_test) - gradient!(grad, model_constant, start_test) + gradient!(grad, model_ml_const, start_test) gradient!(grad2, model_ml, start_test) @test grad ≈ grad2 end @@ -101,12 +89,9 @@ end @testset "ml_solution_weighted" begin solution_ml = fit(semoptimizer, model_ml) solution_ml_weighted = fit(semoptimizer, model_ml_weighted) - @test isapprox(solution(solution_ml), solution(solution_ml_weighted), rtol = 1e-3) - @test isapprox( - nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml), - StructuralEquationModels.minimum(solution_ml_weighted), - rtol = 1e-6, - ) + @test solution(solution_ml) ≈ solution(solution_ml_weighted) rtol = 1e-3 + @test nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml) ≈ + StructuralEquationModels.minimum(solution_ml_weighted) rtol = 1e-6 end ############################################################################################ @@ -181,19 +166,14 @@ end ) # set seed for simulation Random.seed!(83472834) - colnames = Symbol.(names(example_data("political_democracy"))) # simulate data model_ml_new = replace_observed( model_ml, - data = rand(model_ml, params, 1_000_000), - specification = spec, - obs_colnames = colnames, + rand(model_ml, params, 1_000_000), ) model_ml_sym_new = replace_observed( model_ml_sym, - data = rand(model_ml_sym, params, 1_000_000), - specification = spec, - obs_colnames = colnames, + rand(model_ml_sym, params, 1_000_000), ) # fit models sol_ml = solution(fit(semoptimizer, model_ml_new)) @@ -211,23 +191,19 @@ if opt_engine == :Optim using Optim, LineSearches model_ls = Sem( - specification = spec, data = dat, - implied = RAMSymbolic, + specification = spec, + observed = SemObservedData, + implied = RAMSymbolic(spec, vech = true, hessian = true), loss = SemWLS, - hessian = true, - algorithm = Newton(; - linesearch = BackTracking(order = 3), - alphaguess = InitialHagerZhang(), - ), ) model_ml = Sem( - specification = spec, data = dat, - implied = RAMSymbolic, - hessian = true, - algorithm = Newton(), + specification = spec, + observed = SemObservedData, + implied = RAMSymbolic(spec, hessian = true), + loss = SemML, ) @testset "ml_hessians" begin @@ -239,13 +215,23 @@ if opt_engine == :Optim end @testset "ml_solution_hessian" begin - solution = fit(semoptimizer, model_ml) + solution = fit(SemOptimizer(engine = :Optim, algorithm = Newton()), model_ml) + update_estimate!(partable, solution) test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-2) end @testset "ls_solution_hessian" begin - solution = fit(semoptimizer, model_ls) + solution = fit( + SemOptimizer( + engine = :Optim, + algorithm = Newton( + linesearch = BackTracking(order = 3), + alphaguess = InitialHagerZhang(), + ), + ), + model_ls, + ) update_estimate!(partable, solution) test_estimates( partable, @@ -266,6 +252,7 @@ model_ls = Sem( specification = spec_mean, data = dat, implied = RAMSymbolic, + vech = true, loss = SemWLS, meanstructure = true, ) @@ -277,7 +264,7 @@ model_ml_cov = Sem( observed = SemObservedCovariance, obs_cov = cov(Matrix(dat)), obs_mean = vcat(mean(Matrix(dat), dims = 1)...), - obs_colnames = Symbol.(names(dat)), + observed_vars = Symbol.(names(dat)), meanstructure = true, nsamples = 75, ) @@ -289,33 +276,26 @@ model_ml_sym = ### test gradients ############################################################################################ -models = [model_ml, model_ml_cov, model_ls, model_ml_sym] -model_names = ["ml", "ml_cov", "ls_sym", "ml_sym"] +models = Dict( + "ml" => model_ml, + "ml_cov" => model_ml_cov, + "ls_sym" => model_ls, + "ml_sym" => model_ml_sym, +) -for (model, name) in zip(models, model_names) - try - @testset "$(name)_gradient_mean" begin - test_gradient(model, start_test_mean; rtol = 1e-9) - end - catch - end +@testset "$(id)_gradient_mean" for (id, model) in pairs(models) + test_gradient(model, start_test_mean; rtol = 1e-9) end ############################################################################################ ### test solution ############################################################################################ -solution_names = Symbol.("parameter_estimates_" .* ["ml", "ml", "ls", "ml"] .* "_mean") - -for (model, name, solution_name) in zip(models, model_names, solution_names) - try - @testset "$(name)_solution_mean" begin - solution = fit(semoptimizer, model) - update_estimate!(partable_mean, solution) - test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) - end - catch - end +@testset "$(id)_solution_mean" for (id, model) in pairs(models) + solution = fit(semoptimizer, model, start_val = start_test_mean) + update_estimate!(partable_mean, solution) + sol_name = Symbol("parameter_estimates_", replace(id, r"_.+$" => ""), "_mean") + test_estimates(partable_mean, solution_lav[sol_name]; atol = 1e-2) end ############################################################################################ @@ -395,21 +375,14 @@ end ) # set seed for simulation Random.seed!(83472834) - colnames = Symbol.(names(example_data("political_democracy"))) # simulate data model_ml_new = replace_observed( model_ml, - data = rand(model_ml, params, 1_000_000), - specification = spec, - obs_colnames = colnames, - meanstructure = true, + rand(model_ml, params, 1_000_000), ) model_ml_sym_new = replace_observed( model_ml_sym, - data = rand(model_ml_sym, params, 1_000_000), - specification = spec, - obs_colnames = colnames, - meanstructure = true, + rand(model_ml_sym, params, 1_000_000), ) # fit models sol_ml = solution(fit(semoptimizer, model_ml_new)) @@ -474,7 +447,7 @@ end end @testset "fiml_solution_symbolic" begin - solution = fit(semoptimizer, model_ml_sym) + solution = fit(semoptimizer, model_ml_sym, start_val = start_test_mean) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index a4bd7d5ff..ebaaae83b 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -1,5 +1,7 @@ using StructuralEquationModels, Distributions, Random, Optim, LineSearches +SEM = StructuralEquationModels + include( joinpath( chop(dirname(pathof(StructuralEquationModels)), tail = 3), @@ -7,7 +9,7 @@ include( ), ) -x = Symbol.("x", 1:13) +pars = Symbol.("x", 1:13) S = [ :x1 0 0 0 0 0 0 0 @@ -40,7 +42,7 @@ A = [ 0 0 0 0 0 0 0 0 ] -ram_matrices = RAMMatrices(; A = A, S = S, F = F, param_labels = x, vars = nothing) +ram_matrices = RAMMatrices(; A = A, S = S, F = F, param_labels = pars, vars = nothing) true_val = [ repeat([1], 8) @@ -53,19 +55,19 @@ start = [ repeat([0.5], 4) ] -implied_ml = RAMSymbolic(ram_matrices; start_val = start) +implied_sym = RAMSymbolic(ram_matrices) -implied_ml.Σ_eval!(implied_ml.Σ, true_val) +implied_sym.Σ_eval!(implied_sym.Σ, true_val) -true_dist = MultivariateNormal(implied_ml.Σ) +true_dist = MultivariateNormal(implied_sym.Σ) Random.seed!(1234) -x = transpose(rand(true_dist, 100_000)) -semobserved = SemObservedData(data = x, specification = nothing) +x = permutedims(rand(true_dist, 10^5), (2, 1)) + +observed = SemObservedData(data = x, specification = ram_matrices) -loss_ml = SemLoss(SemML(; observed = semobserved, nparams = length(start))) +model_ml = Sem(SemML(observed, implied_sym)) -model_ml = Sem(semobserved, implied_ml, loss_ml) objective!(model_ml, true_val) optimizer = SemOptimizer( @@ -73,6 +75,6 @@ optimizer = SemOptimizer( Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8), ) -solution_ml = fit(optimizer, model_ml) +solution_ml = fit(optimizer, model_ml, start_val = start) -@test true_val ≈ solution(solution_ml) atol = 0.05 +@test solution(solution_ml) ≈ true_val atol = 0.05 diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index fbe2a937f..87812fba2 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -68,9 +68,8 @@ end test_vars_api(implied(model), ram_matrices) test_params_api(implied(model), ram_matrices) - @test @inferred(loss(model)) isa SemLoss - semloss = loss(model).functions[1] - @test semloss isa SemML + @test @inferred(sem_term(model)) isa SemLoss + @test sem_term(model) isa losstype @test @inferred(nsamples(model)) == nsamples(obs) end From 9c82277f7809f073bb0e9c3d204e892a2c359887 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 11:17:31 -0700 Subject: [PATCH 308/364] params/param_labels(): use both as synonyms for now --- src/frontend/specification/Sem.jl | 1 + src/implied/abstract.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 684cfa62b..d89606b62 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -183,6 +183,7 @@ end ############################################################################################ params(model::AbstractSem) = model.params +param_labels(model::AbstractSem) = params(model) # alias """ loss_terms(model::AbstractSem) diff --git a/src/implied/abstract.jl b/src/implied/abstract.jl index d4868d746..e41e79f62 100644 --- a/src/implied/abstract.jl +++ b/src/implied/abstract.jl @@ -8,6 +8,7 @@ nobserved_vars(implied::SemImplied) = nobserved_vars(implied.ram_matrices) nlatent_vars(implied::SemImplied) = nlatent_vars(implied.ram_matrices) param_labels(implied::SemImplied) = param_labels(implied.ram_matrices) +params(implied::SemImplied) = param_labels(implied) nparams(implied::SemImplied) = nparams(implied.ram_matrices) # checks if the A matrix is acyclic From 2b5010e1eb3fc0e349de42f0b48e29b0be315576 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 11:17:31 -0700 Subject: [PATCH 309/364] check_same_semterm_type(): refactor check_single_lossfun() --- src/additional_functions/helper.jl | 29 --------------- src/frontend/fit/fitmeasures/RMSEA.jl | 2 +- src/frontend/fit/fitmeasures/chi2.jl | 17 ++------- src/frontend/fit/fitmeasures/minus2ll.jl | 2 +- src/frontend/specification/Sem.jl | 47 +++++++++++++++++++++++- 5 files changed, 50 insertions(+), 47 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index b3f5212b7..8f2342c3d 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -116,35 +116,6 @@ function nonunique(values::AbstractVector) return res end -# check that a model only has a single lossfun -function check_single_lossfun(model::AbstractSemSingle; throw_error) - if (length(model.loss.functions) > 1) & throw_error - @error "The model has $(length(sem.loss.functions)) loss functions. - Only a single loss function is supported." - end - return isone(length(model.loss.functions)) -end - -# check that all models use the same single loss function -function check_single_lossfun(models::AbstractSemSingle...; throw_error) - uniform = true - lossfun = models[1].loss.functions[1] - L = typeof(lossfun) - for (i, model) in enumerate(models) - uniform &= check_single_lossfun(model; throw_error = throw_error) - cur_lossfun = model.loss.functions[1] - if !isa(cur_lossfun, L) & throw_error - @error "Loss function for group #$i model is $(typeof(cur_lossfun)), expected $L. - Heterogeneous loss functions are not supported." - end - uniform &= isa(cur_lossfun, L) - end - return uniform -end - -check_single_lossfun(model::SemEnsemble; throw_error) = - check_single_lossfun(model.sems...; throw_error) - # scaling corrections for multigroup models mg_correction(::SemFIML) = 0 mg_correction(::SemML) = 0 diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index 7406b74cd..ac2d890d9 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -27,7 +27,7 @@ RMSEA_corr_scale(::Type{<:SemML}) = -1 RMSEA_corr_scale(::Type{<:SemWLS}) = -1 function RMSEA(fit::SemFit, model::AbstractSem) - term_type = check_single_lossfun(model; throw_error = true) + term_type = check_same_semterm_type(model; throw_error = true) n = nsamples(fit) + nsem_terms(model) * RMSEA_corr_scale(term_type) sqrt(nsem_terms(model)) * RMSEA(dof(fit), χ²(fit), n) end diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index 22d6c2e26..c56b9a2a0 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -14,21 +14,10 @@ with the *observed* covariance matrix. function χ²(fit::SemFit, model::AbstractSem) terms = sem_terms(model) - isempty(terms) && return 0.0 + @assert !isempty(terms) - term1 = _unwrap(loss(terms[1])) - L = typeof(term1).name - - # check that all SemLoss terms are of the same class (ML, FIML, WLS etc), ignore typeparams - for (i, term) in enumerate(terms) - lossterm = _unwrap(loss(term)) - @assert lossterm isa SemLoss - if typeof(_unwrap(lossterm)).name != L - @error "SemLoss term #$i is $(typeof(_unwrap(lossterm)).name), expected $L. Heterogeneous loss functions are not supported" - end - end - - return χ²(typeof(term1), fit, model) + L = check_same_semterm_type(model; throw_error = true) + return χ²(L, fit, model) end # bollen, p. 115, only correct for GLS weight matrix diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 3b353f5c7..1cdf5c07b 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -62,6 +62,6 @@ end ############################################################################################ function minus2ll(model::AbstractSem, fit::SemFit) - check_single_lossfun(model; throw_error = true) + check_same_semterm_type(model; throw_error = true) sum(Base.Fix2(minus2ll, fit) ∘ _unwrap ∘ loss, sem_terms(model)) end diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index d89606b62..c3e4bd2bb 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -51,8 +51,8 @@ end function multigroup_weights(models, n) nsamples_total = sum(nsamples, models) - uniform_lossfun = check_single_lossfun(models...; throw_error = false) - if !uniform_lossfun + semloss_type = check_same_semterm_type(semterms; throw_error = false) + if isnothing(semloss_type) @info """ Your ensemble model contains heterogeneous loss functions. Default weights of (#samples per group/#total samples) will be used @@ -258,6 +258,49 @@ function sem_term(model::AbstractSem, _::Nothing = nothing) error("Unreachable reached") end +# check that all models use the same single loss function +# returns the type of the single SEM loss function, SemLoss if there are multiple different SEM losses, +# nothing if there are no SEM terms. +# If throw_error=true, throws an error if there are multiple different SEM loss functions +check_same_semterm_type(model::AbstractSem; throw_error::Bool = true) = + check_same_semterm_type(sem_terms(model); throw_error = throw_error) + +# check that all models use the same single loss function +# returns the type of the single SEM loss function, +# nothing if there are multiple different SEM losses or no SEM terms. +# If throw_error=true, throws an error if there are multiple different SEM loss functions +function check_same_semterm_type(terms::Tuple; throw_error::Bool = true) + isempty(terms) && return nothing + + _semloss(term::SemLoss) = _unwrap(term) + _semloss(term::LossTerm) = _semloss(loss(term)) + _semloss(term) = throw(ArgumentError("SemLoss term expected, $(typeof(term)) found")) + _semloss_label(i::Integer, _::Union{SemLoss, LossTerm{<:SemLoss, Nothing}}) = "#$i" + _semloss_label(i::Integer, term::LossTerm{<:SemLoss, Symbol}) = "#$i ($(SEM.id(term)))" + + term1 = _semloss(terms[1]) + L = typeof(term1).name + + # check that all SemLoss terms are of the same class (ML, FIML, WLS etc), ignore typeparams + for (i, term) in enumerate(terms) + lossterm = _semloss(term) + @assert lossterm isa SemLoss + if typeof(lossterm).name != L + if throw_error + error( + "SemLoss term $(_semloss_label(i, term)) is $(typeof(lossterm).name), expected $L. Heterogeneous loss functions are not supported", + ) + else + return nothing + end + end + end + + # return the type of the first SEM term + # note that type params of the SEM terms might be different + return typeof(term1) +end + # wrappers arounds a single SemLoss term observed(model::AbstractSem, id::Nothing = nothing) = observed(sem_term(model, id)) implied(model::AbstractSem, id::Nothing = nothing) = implied(sem_term(model, id)) From c5f1216ac2edcdd9a44c50e0edc9a4996f50f061 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 11:17:31 -0700 Subject: [PATCH 310/364] update multi-group correction deduplicate the correction scale methods and move to Sem.jl --- src/additional_functions/helper.jl | 5 --- src/frontend/fit/fitmeasures/RMSEA.jl | 7 +--- src/frontend/specification/Sem.jl | 46 +++++++++++++++++---------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 8f2342c3d..5442357f0 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -115,8 +115,3 @@ function nonunique(values::AbstractVector) end return res end - -# scaling corrections for multigroup models -mg_correction(::SemFIML) = 0 -mg_correction(::SemML) = 0 -mg_correction(::SemWLS) = -1 diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index ac2d890d9..9d33e47e4 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -21,14 +21,9 @@ For multigroup models, the correction proposed by J.H. Steiger is applied """ RMSEA(fit::SemFit) = RMSEA(fit, fit.model) -# scaling corrections -RMSEA_corr_scale(::Type{<:SemFIML}) = 0 -RMSEA_corr_scale(::Type{<:SemML}) = -1 -RMSEA_corr_scale(::Type{<:SemWLS}) = -1 - function RMSEA(fit::SemFit, model::AbstractSem) term_type = check_same_semterm_type(model; throw_error = true) - n = nsamples(fit) + nsem_terms(model) * RMSEA_corr_scale(term_type) + n = nsamples(fit) + nsem_terms(model) * multigroup_correction_scale(term_type) sqrt(nsem_terms(model)) * RMSEA(dof(fit), χ²(fit), n) end diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index c3e4bd2bb..d8696e82e 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -45,32 +45,46 @@ function Base.show(io::IO, term::LossTerm) end end -############################################################################################ -# constructor for Sem types -############################################################################################ +# scaling corrections for multigroup models + +# fallback method for non-standard SemLoss type +multigroup_correction_scale(::Type{<:SemLoss}) = nothing -function multigroup_weights(models, n) - nsamples_total = sum(nsamples, models) +multigroup_correction_scale(::Type{<:SemFIML}) = 0 +multigroup_correction_scale(::Type{<:SemML}) = 0 +multigroup_correction_scale(::Type{<:SemWLS}) = -1 + +multigroup_correction_scale(loss::SemLoss) = multigroup_correction_scale(typeof(loss)) + +# calculate sem term weights for multigroup models +# correcting for the number of samples and the loss type +function multigroup_weights(semterms...) + n = length(semterms) + nsamples_total = sum(nsamples, semterms) semloss_type = check_same_semterm_type(semterms; throw_error = false) if isnothing(semloss_type) @info """ Your ensemble model contains heterogeneous loss functions. Default weights of (#samples per group/#total samples) will be used """ - return [(nsamples(model)) / (nsamples_total) for model in models] - end - lossfun = models[1].loss.functions[1] - if !applicable(mg_correction, lossfun) - @info """ - We don't know how to choose group weights for the specified loss function. - Default weights of (#samples per group/#total samples) will be used - """ - return [(nsamples(model)) / (nsamples_total) for model in models] + c = 0 + else + c = multigroup_correction_scale(semloss_type) + if isnothing(c) + @info """ + We don't know how to choose group weights for the specified loss function. + Default weights of (#samples per group/#total samples) will be used + """ + c = 0 + end end - c = mg_correction(lossfun) - return [(nsamples(model)+c) / (nsamples_total+n*c) for model in models] + return [(nsamples(term)+c) / (nsamples_total+n*c) for term in semterms] end +############################################################################################ +# constructor for Sem types +############################################################################################ + function Sem( loss_terms...; params::Union{Vector{Symbol}, Nothing} = nothing, From a59befdbf707b3e3b4da38de889a5984d02d1b88 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 11:18:19 -0700 Subject: [PATCH 311/364] replace_observed(): simplify & refactor remove update_observed!() --- ext/SEMNLOptExt/NLopt.jl | 7 -- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 7 -- src/StructuralEquationModels.jl | 1 - src/additional_functions/simulation.jl | 115 ------------------- src/frontend/specification/Sem.jl | 116 +++++++++++++++++++- src/implied/RAM/generic.jl | 17 --- src/implied/RAM/symbolic.jl | 20 ---- src/implied/empty.jl | 6 - src/loss/ML/FIML.jl | 8 -- src/loss/ML/ML.jl | 20 ---- src/loss/WLS/WLS.jl | 23 ---- src/loss/abstract.jl | 29 +++++ src/loss/constant/constant.jl | 6 - src/loss/regularization/ridge.jl | 6 - src/optimizer/Empty.jl | 6 - src/optimizer/optim.jl | 6 - 16 files changed, 140 insertions(+), 253 deletions(-) diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index 90004b907..876010305 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -107,13 +107,6 @@ function SemOptimizerNLopt(; ) end -############################################################################################ -### Recommended methods -############################################################################################ - -SEM.update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) = - optimizer - ############################################################################################ ### additional methods ############################################################################################ diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index 3ec324530..0937ee040 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -34,13 +34,6 @@ SemOptimizerProximal(; SEM.sem_optimizer_subtype(::Val{:Proximal}) = SemOptimizerProximal -############################################################################################ -### Recommended methods -############################################################################################ - -SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwargs...) = - optimizer - ############################################################################ ### Model fitting ############################################################################ diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index d98e79258..0dbcd16aa 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -205,7 +205,6 @@ export AbstractSem, z_test!, example_data, replace_observed, - update_observed, @StenoGraph, →, ←, diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 6d694c972..e85e9d5c4 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -1,118 +1,3 @@ -""" - (1) replace_observed(model::AbstractSemSingle; kwargs...) - - (2) replace_observed(model::AbstractSemSingle, observed; kwargs...) - - (3) replace_observed(model::SemEnsemble; column = :group, weights = nothing, kwargs...) - -Return a new model with swaped observed part. - -# Arguments -- `model::AbstractSemSingle`: model to swap the observed part of. -- `kwargs`: additional keyword arguments; typically includes `data` and `specification` -- `observed`: Either an object of subtype of `SemObserved` or a subtype of `SemObserved` - -# For SemEnsemble models: -- `column`: if a DataFrame is passed as `data = ...`, which column signifies the group? -- `weights`: how to weight the different sub-models, - defaults to number of samples per group in the new data -- `kwargs`: has to be a dict with keys equal to the group names. - For `data` can also be a DataFrame with `column` containing the group information, - and for `specification` can also be an `EnsembleParameterTable`. - -# Examples -See the online documentation on [Replace observed data](@ref). -""" -function replace_observed end - -""" - update_observed(to_update, observed::SemObserved; kwargs...) - -Update a `SemImplied`, `SemLossFunction` or `SemOptimizer` object to use a `SemObserved` object. - -# Examples -See the online documentation on [Replace observed data](@ref). - -# Implementation -You can provide a method for this function when defining a new type, for more information -on this see the online developer documentation on [Update observed data](@ref). -""" -function update_observed end - -############################################################################################ -# change observed (data) without reconstructing the whole model -############################################################################################ - -# don't change non-SEM terms -replace_observed(loss::AbstractLoss; kwargs...) = loss - -# use the same observed type as before -replace_observed(loss::SemLoss; kwargs...) = - replace_observed(loss, typeof(SEM.observed(loss)).name.wrapper; kwargs...) - -# construct a new observed type -replace_observed(loss::SemLoss, observed_type; kwargs...) = - replace_observed(loss, observed_type(; kwargs...); kwargs...) - -function replace_observed(loss::SemLoss, new_observed::SemObserved; kwargs...) - kwargs = Dict{Symbol, Any}(kwargs...) - old_observed = SEM.observed(loss) - implied = SEM.implied(loss) - - # get field types - kwargs[:observed_type] = typeof(new_observed) - kwargs[:old_observed_type] = typeof(old_observed) - - # update implied - new_implied = update_observed(implied, new_observed; kwargs...) - kwargs[:implied] = new_implied - kwargs[:implied_type] = typeof(new_implied) - kwargs[:nparams] = nparams(new_implied) - - # update loss - return update_observed(loss, new_observed; kwargs...) -end - -replace_observed(loss::LossTerm; kwargs...) = - LossTerm(replace_observed(loss.loss; kwargs...), loss.id, loss.weight) - -function replace_observed(sem::Sem; kwargs...) - updated_terms = Tuple(replace_observed(term; kwargs...) for term in loss_terms(sem)) - return Sem(updated_terms...) -end - -function replace_observed( - emodel::SemEnsemble; - column = :group, - weights = nothing, - kwargs..., -) - kwargs = Dict{Symbol, Any}(kwargs...) - # allow for EnsembleParameterTable to be passed as specification - if haskey(kwargs, :specification) && isa(kwargs[:specification], EnsembleParameterTable) - kwargs[:specification] = convert(Dict{Symbol, RAMMatrices}, kwargs[:specification]) - end - # allow for DataFrame with group variable "column" to be passed as new data - if haskey(kwargs, :data) && isa(kwargs[:data], DataFrame) - kwargs[:data] = Dict( - group => - select(filter(r -> r[column] == group, kwargs[:data]), Not(column)) for - group in emodel.groups - ) - end - # update each model for new data - models = emodel.sems - new_models = Tuple( - replace_observed(m; group_kwargs(g, kwargs)...) for - (m, g) in zip(models, emodel.groups) - ) - return SemEnsemble(new_models...; weights = weights, groups = emodel.groups) -end - -function group_kwargs(g, kwargs) - return Dict(k => kwargs[k][g] for k in keys(kwargs)) -end - ############################################################################################ # simulate data ############################################################################################ diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index d8696e82e..42ff2d3e8 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -436,12 +436,118 @@ function build_SemTerms(loss, observed, implied; kwargs...) end end -function update_observed(sem::Sem, new_observed; kwargs...) - new_terms = Tuple( - update_observed(lossterm.loss, new_observed; kwargs...) for - lossterm in loss_terms(sem) +############################################################## +# replace_observed: Sem level +############################################################## + +""" + replace_observed(model::Sem, observed::SemObserved) + replace_observed(model::Sem, data::AbstractDict{Symbol}) + replace_observed(model::Sem, data::AbstractDataFrame; [semterm_column]) + replace_observed(loss::SemLoss, observed::SemObserved) + replace_observed(loss::SemLoss, data::Union{AbstractMatrix, DataFrame}) + +Construct a new SEM model or SEM loss with replaced observed data. + +The SEM structure (implied covariance, loss type) is preserved; +only the observed data is swapped. + +# Single-term models + +Pass a `SemObserved` object, a data matrix, or a `DataFrame`: +```julia +replace_observed(model, new_data_matrix) +replace_observed(model, new_sem_observed) +replace_observed(model, new_df) +``` + +# Multi-term models + +Pass a `Dict{Symbol}` mapping term ids to data or `SemObserved` objects: +```julia +replace_observed(model, Dict(:g1 => data1, :g2 => data2)) +``` + +Or pass a `DataFrame` with a `semterm_column` identifying the group: +```julia +replace_observed(model, new_df; semterm_column = :group) +``` +""" +function replace_observed end + +function replace_observed(sem::Sem, data::Union{SemObserved, AbstractMatrix}) + nsem_terms(sem) > 1 && throw( + ArgumentError( + "Model contains $(nsem_terms(sem)) SEM terms. " * + "Use a Dict{Symbol} or a DataFrame with `semterm_column` to provide per-term data.", + ), + ) + updated_terms = Tuple(replace_observed(term, data) for term in loss_terms(sem)) + return Sem(updated_terms...) +end + +function replace_observed(sem::Sem, data::AbstractDict{Symbol}) + term_ids = Set( + if !isnothing(id(term)) + id(term) + else + "Multigroup replace_observed(sem, data::Dict) requires all SEM terms to have ids." |> + ArgumentError |> + throw + end for term in loss_terms(sem) if issemloss(term) + ) + # check for extra ids + extra_term_ids = setdiff(keys(data), term_ids) + isempty(extra_term_ids) || + @warn "Ignoring data with ids=$(collect(extra_term_ids)): no such SEM terms exist in the model" + + updated_terms = map(loss_terms(sem)) do term + issemloss(term) || return term + tid = id(term) + term_data = get(data, tid, nothing) + isnothing(term_data) && + throw(ArgumentError("No data provided for SEM term :$tid")) + return replace_observed(term, term_data) + end + return Sem(Tuple(updated_terms)...) +end + +function replace_observed(sem::Sem, data::AbstractVector) + nsem = nsem_terms(sem) + nsem == length(data) || throw( + ArgumentError( + "Length of data ($(length(data))) does not match number of SEM terms ($nsem)", + ), + ) + updated_terms = map(enumerate(loss_terms(sem))) do (i, term) + issemloss(term) ? replace_observed(term, data[i]) : term + end + return Sem(Tuple(updated_terms)...) +end + +function replace_observed( + sem::Sem, + data::AbstractDataFrame; + semterm_column::Union{Symbol, Nothing} = nothing, +) + if isnothing(semterm_column) + # single-term shortcut + nsem_terms(sem) > 1 && throw( + ArgumentError( + "Model contains $(nsem_terms(sem)) SEM terms. " * + "Provide `semterm_column` to specify which DataFrame column identifies the groups.", + ), + ) + updated_terms = Tuple(replace_observed(term, data) for term in loss_terms(sem)) + return Sem(updated_terms...) + end + + # multi-term: split DataFrame by semterm_column + terms_data = Dict( + g[semterm_column] => group_data for + (g, group_data) in pairs(groupby(data, semterm_column)) ) - return Sem(new_terms...) + return replace_observed(sem, terms_data) end ############################################################## diff --git a/src/implied/RAM/generic.jl b/src/implied/RAM/generic.jl index f1c1e08d0..1569b3415 100644 --- a/src/implied/RAM/generic.jl +++ b/src/implied/RAM/generic.jl @@ -179,20 +179,3 @@ function update!(targets::EvaluationTargets, implied::RAM, params) mul!(implied.μ, implied.F⨉I_A⁻¹, implied.M) end end - -############################################################################################ -### Recommended methods -############################################################################################ - -function update_observed(implied::RAM, observed::SemObserved; kwargs...) - if nobserved_vars(observed) == nobserved_vars(implied) - return implied - else - return RAM(; - observed = observed, - gradient_required = !isnothing(implied.∇A), - meanstructure = MeanStruct(implied) == HasMeanStruct, - kwargs..., - ) - end -end diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index 4c9bda914..52a192e64 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -190,26 +190,6 @@ function update!(targets::EvaluationTargets, implied::RAMSymbolic, par) end end -############################################################################################ -### Recommended methods -############################################################################################ - -function update_observed(implied::RAMSymbolic, observed::SemObserved; kwargs...) - if nobserved_vars(observed) == nobserved_vars(implied) - return implied - else - return RAMSymbolic(; - observed = observed, - vech = implied.Σ isa Vector, - gradient = !isnothing(implied.∇Σ), - hessian = !isnothing(implied.∇²Σ), - meanstructure = MeanStruct(implied) == HasMeanStruct, - approximate_hessian = isnothing(implied.∇²Σ), - kwargs..., - ) - end -end - ############################################################################################ ### additional functions ############################################################################################ diff --git a/src/implied/empty.jl b/src/implied/empty.jl index a327ee134..a650a07a9 100644 --- a/src/implied/empty.jl +++ b/src/implied/empty.jl @@ -46,9 +46,3 @@ end ############################################################################################ update!(targets::EvaluationTargets, implied::ImpliedEmpty, par) = nothing - -############################################################################################ -### Recommended methods -############################################################################################ - -update_observed(implied::ImpliedEmpty, observed::SemObserved; kwargs...) = implied diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index fdedf3984..15081e201 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -159,14 +159,6 @@ function evaluate!(objective, gradient, hessian, loss::SemFIML, params) return objective end - -############################################################################################ -### Recommended methods -############################################################################################ - -update_observed(loss::SemFIML, observed::SemObserved; kwargs...) = - SemFIML(; observed = observed, kwargs...) - ############################################################################################ ### additional functions ############################################################################################ diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 2d449d73c..9f3275443 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -235,23 +235,3 @@ function non_posdef_return(par) return typemax(eltype(par)) end end - -############################################################################################ -### recommended methods -############################################################################################ - -update_observed(loss::SemML, observed::SemObservedMissing; kwargs...) = - error("ML estimation does not work with missing data - use FIML instead") - -function update_observed(loss::SemML, observed::SemObserved; kwargs...) - if (obs_cov(loss) == obs_cov(observed)) && (obs_mean(loss) == obs_mean(observed)) - return loss # no change - else - return SemML( - observed, - loss.implied; - approximate_hessian = HessianEval(loss) == ApproxHessian, - kwargs..., - ) - end -end diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 5c4cb2520..8f4a109c5 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -179,26 +179,3 @@ function evaluate!(objective, gradient, hessian, loss::SemWLS, par) return objective end - -############################################################################################ -### Recommended methods -############################################################################################ - -function update_observed( - loss::SemWLS, - observed::SemObserved; - recompute_V = true, - kwargs..., -) - if recompute_V - return SemWLS(observed, loss.implied; kwargs...) - else - return SemWLS( - observed, - loss.implied; - wls_weight_matrix = loss.V, - wls_weight_matrix_mean = loss.V_μ, - kwargs..., - ) - end -end diff --git a/src/loss/abstract.jl b/src/loss/abstract.jl index bf8585d6a..bcd6d62b3 100644 --- a/src/loss/abstract.jl +++ b/src/loss/abstract.jl @@ -40,3 +40,32 @@ function check_observed_vars(observed::SemObserved, implied::SemImplied) end check_observed_vars(sem::SemLoss) = check_observed_vars(observed(sem), implied(sem)) + +############################################################################################ +# replace_observed: SemLoss, AbstractLoss, LossTerm +############################################################################################ + +function replace_observed(loss::SemLoss, new_observed::SemObserved) + old_obs = SEM.observed(loss) + observed_vars(old_obs) == observed_vars(new_observed) || throw( + ArgumentError( + "observed_vars of the new data do not match the model: " * + "expected $(observed_vars(old_obs)), got $(observed_vars(new_observed))", + ), + ) + return typeof(loss).name.wrapper(new_observed, SEM.implied(loss)) +end + +function replace_observed(loss::SemLoss, data::Union{AbstractMatrix, DataFrame}) + old_obs = SEM.observed(loss) + new_observed = + typeof(old_obs).name.wrapper(data = data, observed_vars = observed_vars(old_obs)) + return replace_observed(loss, new_observed) +end + +# non-SEM loss terms are unchanged +replace_observed(loss::AbstractLoss, ::Any) = loss + +# LossTerm: delegate to inner loss +replace_observed(term::LossTerm, data) = + LossTerm(replace_observed(loss(term), data), id(term), weight(term)) diff --git a/src/loss/constant/constant.jl b/src/loss/constant/constant.jl index 023076cc9..2aff0156d 100644 --- a/src/loss/constant/constant.jl +++ b/src/loss/constant/constant.jl @@ -35,9 +35,3 @@ SemConstant(; constant_loss::Number, kwargs...) = SemConstant(constant_loss) objective(loss::SemConstant, par) = convert(eltype(par), loss.c) gradient(loss::SemConstant, par) = zero(par) hessian(loss::SemConstant, par) = zeros(eltype(par), length(par), length(par)) - -############################################################################################ -### Recommended methods -############################################################################################ - -update_observed(loss::SemConstant, observed::SemObserved; kwargs...) = loss diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index 3e2cfbff2..813aff11c 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -85,9 +85,3 @@ function hessian(ridge::SemRidge, par) @views @. ridge.hessian[ridge.which_H] .= 2 * ridge.α return ridge.hessian end - -############################################################################################ -### Recommended methods -############################################################################################ - -update_observed(loss::SemRidge, observed::SemObserved; kwargs...) = loss diff --git a/src/optimizer/Empty.jl b/src/optimizer/Empty.jl index f95c067ce..fd36acb53 100644 --- a/src/optimizer/Empty.jl +++ b/src/optimizer/Empty.jl @@ -11,12 +11,6 @@ struct SemOptimizerEmpty <: SemOptimizer{:Empty} end sem_optimizer_subtype(::Val{:Empty}) = SemOptimizerEmpty -############################################################################################ -### Recommended methods -############################################################################################ - -update_observed(optimizer::SemOptimizerEmpty, observed::SemObserved; kwargs...) = optimizer - ############################################################################################ ### Pretty Printing ############################################################################################ diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 704131938..a0aae22ab 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -57,12 +57,6 @@ SemOptimizerOptim(; sem_optimizer_subtype(::Val{:Optim}) = SemOptimizerOptim -############################################################################################ -### Recommended methods -############################################################################################ - -update_observed(optimizer::SemOptimizerOptim, observed::SemObserved; kwargs...) = optimizer - ############################################################################################ ### additional methods ############################################################################################ From 62e5633d6ef0b11abc14417f079ca7deb5a87da8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 17:56:25 -0700 Subject: [PATCH 312/364] bootstrap: sync with Sem updates --- src/frontend/fit/standard_errors/bootstrap.jl | 348 +++++++++--------- test/examples/helper.jl | 27 +- test/examples/multigroup/build_models.jl | 4 +- .../political_democracy/constructor.jl | 14 +- 4 files changed, 199 insertions(+), 194 deletions(-) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index 845a209ee..ce84e923b 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -1,35 +1,118 @@ +# base type for accumulators of intermediate bootstrap results +abstract type BootstrapAccumulator end + +# internal function to run bootstrap +function bootstrap!( + acc::BootstrapAccumulator, + fitted::SemFit; + data = nothing, + engine = :Optim, + parallel = false, + fit_kwargs = Dict(), +) + sem = model(fitted) + data = isnothing(data) ? _bootstrap_data(sem) : data + start = solution(fitted) + + n_boot = n_bootstrap(acc) + + # fit to bootstrap samples + if !parallel + for i in 1:n_boot + new_fit = _fit_bootstrap_sample(sem, data, start; engine, fit_kwargs) + update!(acc, i, new_fit, nothing) + end + else + n_threads = Threads.nthreads() + # Pre-create one independent model copy per thread via deepcopy. + model_pool = Channel(n_threads) + for _ in 1:n_threads + put!(model_pool, deepcopy(sem)) + end + lk = ReentrantLock() + Threads.@threads for i in 1:n_boot + thread_model = take!(model_pool) + new_fit = _fit_bootstrap_sample(thread_model, data, start; engine, fit_kwargs) + update!(acc, i, new_fit, lk) + put!(model_pool, thread_model) + end + end + + return acc +end + +# a simple accumulator that just stores the statistic for each sample and whether it converged +struct SimpleBootstrapAccumulator{F} <: BootstrapAccumulator + statistic::F + samples::Vector{Any} + converged_mask::Vector{Bool} +end + +SimpleBootstrapAccumulator(statistic, n_boot::Integer) = + SimpleBootstrapAccumulator(statistic, Vector{Any}(undef, n_boot), fill(false, n_boot)) + +n_bootstrap(acc::SimpleBootstrapAccumulator) = length(acc.samples) + +function update!(acc::SimpleBootstrapAccumulator, i::Integer, fit::SemFit, _) + acc.samples[i] = acc.statistic(fit) + acc.converged_mask[i] = converged(fit) +end + +""" + struct BootstrapResult{T} + +Stores the output of a [`bootstrap`](@ref) call. +""" +struct BootstrapResult{T} + samples::Vector{T} + converged_mask::BitVector + n_boot::Int + n_converged::Int +end + +function Base.show(io::IO, result::BootstrapResult{T}) where {T} + println( + io, + "BootstrapResult{$(T)} with $(result.n_converged)/$(result.n_boot) converged samples", + ) +end + """ bootstrap( - fitted::SemFit, - specification::SemSpecification; + fitted::SemFit; statistic = solution, n_boot = 3000, data = nothing, engine = :Optim, parallel = false, - fit_kwargs = Dict(), - replace_kwargs = Dict()) + fit_kwargs = Dict() + ) -> BootstrapResult + +Bootstrap the samples and apply `statistic` function to each. -Return bootstrap samples for `statistic`. +Returns a [`BootstrapResult`](@ref) object containing the results of `statistic` +applied to each bootstrapped sample. + +Supports both single-group and multi-group models. +For multi-group models, each group is resampled independently. # Arguments - `fitted`: a fitted SEM. -- `specification`: a `ParameterTable` or `RAMMatrices` object passed to `replace_observed`. - `statistic`: any function that can be called on a `SemFit` object. The output will be returned as the bootstrap sample. - `n_boot`: number of boostrap samples -- `data`: data to sample from. Only needed if different than the data from `sem_fit` +- `data`: data to sample from. Only needed if different than the fitted model. + For multi-group models, pass a `Dict{Symbol}` mapping term ids to data matrices. - `engine`: optimizer engine, passed to `fit`. - `parallel`: if `true`, run bootstrap samples in parallel on all available threads. The number of threads is controlled by the `JULIA_NUM_THREADS` environment variable or the `--threads` flag when starting Julia. - `fit_kwargs` : a `Dict` controlling model fitting for each bootstrap sample, - passed to `fit` -- `replace_kwargs`: a `Dict` passed to `replace_observed` + passed to [`fit`](@ref) # Example ```julia -# 1000 boostrap samples of the minimum, fitted with :Optim +# 1000 bootstrap samples of the minimum, fitted with :Optim bootstrap( fitted; statistic = StructuralEquationModels.minimum, @@ -40,95 +123,74 @@ bootstrap( """ function bootstrap( fitted::SemFit, - specification::SemSpecification; - statistic = solution, + statistic = solution; n_boot = 3000, data = nothing, engine = :Optim, parallel = false, fit_kwargs = Dict(), - replace_kwargs = Dict(), ) - # access data and convert to matrix - data = prepare_data_bootstrap(data, fitted.model) - start = solution(fitted) - # pre-allocations - out = Vector{Any}(nothing, n_boot) - conv = fill(false, n_boot) - # fit to bootstrap samples - if !parallel - for i in 1:n_boot - sample_data = bootstrap_sample(data) - new_model = replace_observed( - fitted.model; - data = sample_data, - specification = specification, - replace_kwargs..., - ) - new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) - sample = statistic(new_fit) - c = converged(new_fit) - out[i] = sample - conv[i] = c - end - else - n_threads = Threads.nthreads() - # Pre-create one independent model copy per thread via deepcopy. - model_pool = Channel(n_threads) - for _ in 1:n_threads - put!(model_pool, deepcopy(fitted.model)) - end - # fit models in parallel - lk = ReentrantLock() - Threads.@threads for i in 1:n_boot - thread_model = take!(model_pool) - sample_data = bootstrap_sample(data) - new_model = replace_observed( - thread_model; - data = sample_data, - specification = specification, - replace_kwargs..., - ) - new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) - sample = statistic(new_fit) - c = converged(new_fit) - out[i] = sample - conv[i] = c - put!(model_pool, thread_model) - end - end - return Dict( - :samples => collect(a for a in out), - :n_boot => n_boot, - :n_converged => sum(conv), - :converged => conv, + acc = SimpleBootstrapAccumulator(statistic, n_boot) + bootstrap!(acc, fitted; data, engine, parallel, fit_kwargs) + return BootstrapResult( + [s for s in acc.samples], + convert(BitVector, acc.converged_mask), + n_bootstrap(acc), + sum(acc.converged_mask), ) end +# bootstrap accumulator for se_bootstrap() +# accumulates per-parameter sum and sum of squares across bootstrap samples +struct StdErrBootstrapAccumulator <: BootstrapAccumulator + n_boot::Int + sum::Vector{Float64} + squared_sum::Vector{Float64} + n_converged::Ref{Int} +end + +n_bootstrap(acc::StdErrBootstrapAccumulator) = acc.n_boot + +StdErrBootstrapAccumulator(n_params::Integer, n_boot::Integer) = + StdErrBootstrapAccumulator(n_boot, zeros(n_params), zeros(n_params), Ref(0)) + +function update!( + acc::StdErrBootstrapAccumulator, + i::Integer, + fit::SemFit, + lk::Union{Base.AbstractLock, Nothing}, +) + conv = converged(fit) + if conv + sol = solution(fit) + isnothing(lk) || lock(lk) + acc.n_converged[] += 1 + @. acc.sum += sol + @. acc.squared_sum += abs2(sol) + isnothing(lk) || unlock(lk) + end +end + """ - se_bootstrap( - fitted::SemFit, - specification::SemSpecification; - n_boot = 3000, - data = nothing, - parallel = false, - fit_kwargs = Dict(), - replace_kwargs = Dict()) + se_bootstrap(fitted::SemFit; n_boot = 3000, kwargs...) -Return bootstrap standard errors. +Calculate standard errors using bootstrap approach. + +Supports both single-group and multi-group models. +For multi-group models, each group is resampled independently. # Arguments - `fitted`: a fitted SEM. -- `specification`: a `ParameterTable` or `RAMMatrices` object passed to `replace_observed`. - `n_boot`: number of boostrap samples -- `data`: data to sample from. Only needed if different than the data from `sem_fit` +- `data`: data to sample from. Only needed if different than the fitted model. + For multi-group models, pass a `Dict{Symbol}` mapping term ids to data matrices. - `engine`: optimizer engine, passed to `fit`. - `parallel`: if `true`, run bootstrap samples in parallel on all available threads. The number of threads is controlled by the `JULIA_NUM_THREADS` environment variable or the `--threads` flag when starting Julia. -- `fit_kwargs` : a `Dict` controlling model fitting for each bootstrap sample, - passed to `sem_fit` -- `replace_kwargs`: a `Dict` passed to `replace_observed` +- `fit_kwargs` : a `Dict` controlling model fitting for each bootstrap sample, + passed to [`fit`](@ref) + # Example ```julia @@ -142,109 +204,53 @@ se_bootstrap( ) ``` """ -function se_bootstrap( - fitted::SemFit, - specification::SemSpecification; - n_boot = 3000, - data = nothing, - engine = :Optim, - parallel = false, - fit_kwargs = Dict(), - replace_kwargs = Dict(), -) - # access data and convert to matrix - data = prepare_data_bootstrap(data, fitted.model) - start = solution(fitted) - # pre-allocations - total_sum = zero(start) - total_squared_sum = zero(start) - n_conv = Ref(0) - # fit to bootstrap samples - if !parallel - for _ in 1:n_boot - sample_data = bootstrap_sample(data) - new_model = replace_observed( - fitted.model; - data = sample_data, - specification = specification, - replace_kwargs..., - ) - new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) - sol = solution(new_fit) - conv = converged(new_fit) - if conv - n_conv[] += 1 - @. total_sum += sol - @. total_squared_sum += sol^2 - end - end +function se_bootstrap(fitted::SemFit; n_boot = 3000, kwargs...) + acc = StdErrBootstrapAccumulator(nparams(fitted), n_boot) + bootstrap!(acc, fitted; kwargs...) + n_conv = acc.n_converged[] + @info "$n_conv models converged" + + if n_conv == 0 + @warn "No bootstrap samples converged. Returning NaN." + return fill(NaN, length(acc.sum)) else - n_threads = Threads.nthreads() - # Pre-create one independent model copy per thread via deepcopy. - model_pool = Channel(n_threads) - for _ in 1:n_threads - put!(model_pool, deepcopy(fitted.model)) - end - # fit models in parallel - lk = ReentrantLock() - Threads.@threads for _ in 1:n_boot - thread_model = take!(model_pool) - sample_data = bootstrap_sample(data) - new_model = replace_observed( - thread_model; - data = sample_data, - specification = specification, - replace_kwargs..., - ) - new_fit = fit(new_model; start_val = start, engine = engine, fit_kwargs...) - sol = solution(new_fit) - conv = converged(new_fit) - if conv - lock(lk) do - n_conv[] += 1 - @. total_sum += sol - @. total_squared_sum += sol^2 - end - end - put!(model_pool, thread_model) - end + return sqrt.(acc.squared_sum ./ n_conv - abs2.(acc.sum / n_conv)) end - # compute parameters - n_conv = n_conv[] - sd = sqrt.(total_squared_sum / n_conv - (total_sum / n_conv) .^ 2) - @info string(n_conv)*" models converged" - return sd end ############################################################################################ ### Helper Functions ############################################################################################ -function bootstrap_sample(data::Matrix) - nobs = size(data, 1) - index_new = rand(1:nobs, nobs) - data_new = data[index_new, :] - return data_new -end +""" + resample_with_replacement(data::AbstractMatrix) + resample_with_replacement(data::AbstractVector{<:AbstractMatrix}) -bootstrap_sample(data::Dict) = Dict(k => bootstrap_sample(data[k]) for k in keys(data)) +Resample rows of a data matrix with replacement (bootstrap sample). +For a vector of matrices (multi-group models), independently resamples each matrix. +""" +function resample_with_replacement(data::AbstractMatrix) + n = size(data, 1) + return data[rand(1:n, n), :] +end -function prepare_data_bootstrap(data, model::AbstractSemSingle) - if isnothing(data) - data = samples(observed(model)) - end - data = Matrix(data) - return data +function resample_with_replacement(data::AbstractVector{<:AbstractMatrix}) + return [resample_with_replacement(term_data) for term_data in data] end -function prepare_data_bootstrap(data, model::SemEnsemble) - sems = model.sems - groups = model.groups - if isnothing(data) - data = Dict(g => samples(observed(m)) for (g, m) in zip(groups, sems)) +# Extract data from a model for bootstrap resampling. +function _bootstrap_data(sem::AbstractSem) + terms = sem_terms(sem) + if length(terms) == 1 + return samples(observed(loss(terms[1]))) + else + return [samples(observed(loss(term))) for term in terms] end - data = Dict(k => Matrix(data[k]) for k in keys(data)) - return data end - +# Fit one bootstrap replicate: resample, replace observed data, fit. +function _fit_bootstrap_sample(sem_model, data, start; engine, fit_kwargs) + boot_data = resample_with_replacement(data) + boot_model = replace_observed(sem_model, boot_data) + return fit(boot_model; start_val = start, engine = engine, fit_kwargs...) +end diff --git a/test/examples/helper.jl b/test/examples/helper.jl index c4191fdb1..f14fec621 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -138,17 +138,17 @@ function test_estimates( end function test_bootstrap( - model_fit, - spec; + model_fit::SemFit; compare_hessian = true, rtol_hessian = 0.2, compare_bs = true, rtol_bs = 0.1, n_boot = 500, + seed = 32432, ) - @testset rng = Random.seed!(32432) "bootstrap" begin - se_bs = @suppress se_bootstrap(model_fit, spec; n_boot = n_boot) - # hessian and bootstrap se are close + @testset rng = Random.seed!(seed) "bootstrap" begin + se_bs = @suppress se_bootstrap(model_fit; n_boot = n_boot) + # hessian-based and bootstrap-based std.errors are close if compare_hessian se_he = @suppress se_hessian(model_fit) #println(maximum(abs.(se_he - se_bs))) @@ -156,10 +156,9 @@ function test_bootstrap( end # se_bootstrap and bootstrap |> se are close if compare_bs - bs_samples = bootstrap(model_fit, spec; n_boot = n_boot) - @test bs_samples[:n_converged] >= 0.95*n_boot - bs_samples = - cat(bs_samples[:samples][BitVector(bs_samples[:converged])]..., dims = 2) + bs_samples = bootstrap(model_fit; n_boot = n_boot) + @test bs_samples.n_converged >= 0.95*n_boot + bs_samples = reduce(hcat, bs_samples.samples[bs_samples.converged_mask]) se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2)) #println(maximum(abs.(se_bs_2 - se_bs))) @test isapprox(se_bs_2, se_bs, rtol = rtol_bs) @@ -167,14 +166,14 @@ function test_bootstrap( end end -function smoketest_bootstrap(model_fit, spec; n_boot = 5) - # hessian and bootstrap se are close - se_bs = se_bootstrap(model_fit, spec; n_boot = n_boot) - bs_samples = bootstrap(model_fit, spec; n_boot = n_boot) +function smoketest_bootstrap(model_fit::SemFit; n_boot = 5) + # just test that both methods succeed + se_bs = se_bootstrap(model_fit; n_boot = n_boot) + bs_samples = bootstrap(model_fit; n_boot = n_boot) return se_bs, bs_samples end -function smoketest_CI_z(model_fit, partable) +function smoketest_CI_z(model_fit::SemFit, partable) se_he = @suppress se_hessian(model_fit) normal_CI!(partable, model_fit, se_he) z_test!(partable, model_fit, se_he) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 48723fbef..462deab61 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -249,7 +249,7 @@ end lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - # test_bootstrap(solution_ls, partable; compare_bs = false, rtol_hessian = 0.3) + # test_bootstrap(solution_ls; compare_bs = false, rtol_hessian = 0.3) smoketest_CI_z(solution_ls, partable) end @@ -360,7 +360,7 @@ if !isnothing(specification_miss_g1) fitmeasure_names = Dict(:CFI => "cfi"), ) - test_bootstrap(solution, partable_miss; compare_bs = false, rtol_hessian = 0.5) + test_bootstrap(solution; compare_bs = false, rtol_hessian = 0.5) smoketest_CI_z(solution, partable_miss) update_se_hessian!(partable_miss, solution) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 25a6da914..759875b25 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -116,7 +116,7 @@ end lav_col = :se, ) - test_bootstrap(solution_ml, partable) + test_bootstrap(solution_ml) smoketest_CI_z(solution_ml, partable) end @@ -146,7 +146,7 @@ end lav_col = :se, ) - test_bootstrap(solution_ls, partable; compare_bs = false) + test_bootstrap(solution_ls; compare_bs = false) smoketest_CI_z(solution_ls, partable) end @@ -324,7 +324,7 @@ end lav_col = :se, ) - test_bootstrap(solution_ml, partable_mean) + test_bootstrap(solution_ml) smoketest_CI_z(solution_ml, partable_mean) end @@ -353,8 +353,8 @@ end lav_col = :se, ) - test_bootstrap(solution_ls, partable_mean, compare_bs = false) - # smoketest_bootstrap(solution_ls, partable_mean) + test_bootstrap(solution_ls, compare_bs = false) + # smoketest_bootstrap(solution_ls) smoketest_CI_z(solution_ls, partable_mean) end @@ -481,7 +481,7 @@ end lav_col = :se, ) - # test_bootstrap(solution_ml, partable_mean) # too much compute - smoketest_bootstrap(solution_ml, partable_mean) + # test_bootstrap(solution_ml) # too much compute + smoketest_bootstrap(solution_ml) smoketest_CI_z(solution_ml, partable_mean) end From 0aba9c1bd8300bf282711d49559970b7797d372b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 17:56:25 -0700 Subject: [PATCH 313/364] CFI: sync with Sem refactor --- src/frontend/fit/fitmeasures/CFI.jl | 47 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/src/frontend/fit/fitmeasures/CFI.jl b/src/frontend/fit/fitmeasures/CFI.jl index 9f3c5a2d9..e2bcb8c7d 100644 --- a/src/frontend/fit/fitmeasures/CFI.jl +++ b/src/frontend/fit/fitmeasures/CFI.jl @@ -5,11 +5,11 @@ Calculate the Comparative Fit Index (CFI). -The CFI ranges from 0-1 and measures how much better the model +The CFI ranges from 0-1 and measures how much better the model fits the data compared to a baseline model. If no baseline model is provided, a model with unconstrained variances (and means) is compaired against. -For multigroup models, variances (and means) per group are free +For multigroup models, variances (and means) per group are free without any equality constraints between groups. """ function CFI end @@ -35,34 +35,31 @@ function CFI(χ², dof, χ²₀, dof₀) end ### -function χ²_varonly(model::AbstractSemSingle) - check_single_lossfun(model; throw_error = true) - return χ²_varonly(model.loss.functions[1], model) -end - -function χ²_varonly(model::SemEnsemble) - check_single_lossfun(model; throw_error = true) - return sum(χ²_varonly, model.sems) +function χ²_varonly(model::AbstractSem) + check_same_semterm_type(model; throw_error = true) + return sum(sem_terms(model)) do semterm + χ²_varonly(_unwrap(loss(semterm))) + end end -function χ²_varonly(::SemML, model::AbstractSemSingle) - N⁻ = (nsamples(model) - 1) - S = obs_cov(observed(model)) +function χ²_varonly(loss::SemML) + N⁻ = (nsamples(loss) - 1) + S = obs_cov(observed(loss)) Σ₀ = Diagonal(S) - p = nobserved_vars(model) + p = nobserved_vars(loss) return N⁻*(logdet(Σ₀) + tr(inv(Σ₀)*S) - logdet(S) - p) end # for the optimal variance only model, we have to solve 1/2 tr((I-XS⁻¹)^2) with X diagonal -function χ²_varonly(::SemWLS, model) - N⁻ = (nsamples(model) - 1) - S⁻¹ = inv((obs_cov(observed(model)))) +function χ²_varonly(loss::SemWLS) + N⁻ = (nsamples(loss) - 1) + S⁻¹ = inv((obs_cov(observed(loss)))) Σ₀ = Diagonal(inv(S⁻¹ .* S⁻¹)*diag(S⁻¹)) return N⁻*0.5*tr((I - Σ₀*S⁻¹)^2) end # For FIML, an explicit bl model has to be passed -function χ²_varonly(::SemFIML, model) +function χ²_varonly(loss::SemFIML) """ Computing the CFI with FIML requires explicitely passing a fitted baseline model as CFI(fit::SemFit, fit_baseline::SemFit) @@ -71,12 +68,12 @@ function χ²_varonly(::SemFIML, model) throw end -function dof_varonly(model::AbstractSemSingle) - nparams_varonly = nobserved_vars(model) - if MeanStruct(model.implied) === HasMeanStruct - nparams_varonly *= 2 +function dof_varonly(model::AbstractSem) + return sum(sem_terms(model)) do semterm + nparams_varonly = nobserved_vars(semterm) + if MeanStruct(implied(semterm)) === HasMeanStruct + nparams_varonly *= 2 + end + return n_dp(loss(semterm)) - nparams_varonly end - return n_dp(model) - nparams_varonly end - -dof_varonly(model::SemEnsemble) = sum(dof_varonly, model.sems) From 77f1e522440a89ed4b53f09db87bfc172072cf3a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 17:57:46 -0700 Subject: [PATCH 314/364] test/build_models: remove redundant model --- test/examples/multigroup/build_models.jl | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 462deab61..329c55021 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -347,19 +347,6 @@ if !isnothing(specification_miss_g1) lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = fit(semoptimizer, model_ml_multigroup2) - test_fitmeasures( - fit_measures(solution), - solution_lav[:fitmeasures_fiml]; - rtol = 1e-3, - atol = 0, - ) - test_fitmeasures( - Dict(:CFI => CFI(solution, solution_varonly)), - solution_lav[:fitmeasures_fiml]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) - test_bootstrap(solution; compare_bs = false, rtol_hessian = 0.5) smoketest_CI_z(solution, partable_miss) From 54ce4adce8ab40e5752107fc9d198606f0c1a22a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 17:57:46 -0700 Subject: [PATCH 315/364] revert using --- test/examples/multigroup/multigroup.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index dd654731d..c8ae8c1f6 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -1,5 +1,5 @@ using StructuralEquationModels, Test, FiniteDiff, Suppressor -using LinearAlgebra: diagind, LowerTriangular +using LinearAlgebra: diagind, isposdef, logdet, tr, LowerTriangular using Statistics: var using Random From b2cfbeb70621e6bbc73bf2dd7478d6aded695ded Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 17:57:46 -0700 Subject: [PATCH 316/364] WLS: verbose option to suppress info about inv(obs_cov) --- src/loss/WLS/WLS.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 8f4a109c5..d04bc346e 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -62,6 +62,7 @@ function SemWLS( wls_weight_matrix::Union{AbstractMatrix, Nothing} = nothing, wls_weight_matrix_mean::Union{AbstractMatrix, Nothing} = nothing, approximate_hessian::Bool = false, + verbose::Bool = false, kwargs..., ) if observed isa SemObservedMissing @@ -114,7 +115,8 @@ function SemWLS( if MeanStruct(implied) == HasMeanStruct if isnothing(wls_weight_matrix_mean) - @info "Computing WLS weight matrix for the meanstructure using obs_cov()" + verbose && + @info "Computing WLS weight matrix for the meanstructure using obs_cov()" wls_weight_matrix_mean = inv(obs_cov(observed)) end size(wls_weight_matrix_mean) == (nobs_vars, nobs_vars) || DimensionMismatch( From 1cf97f51b9bae017d16b191f834c761e918f4a6f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 21 Mar 2026 17:57:46 -0700 Subject: [PATCH 317/364] docs: sync with Sem refactor --- docs/src/developer/implied.md | 30 ++++------- docs/src/developer/loss.md | 21 ++------ docs/src/developer/optimizer.md | 8 --- docs/src/developer/sem.md | 17 +++---- docs/src/internals/types.md | 20 +++++--- docs/src/performance/mixed_differentiation.md | 18 +++---- docs/src/performance/simulation.md | 17 +------ docs/src/tutorials/collection/collection.md | 51 +++++++++++++------ docs/src/tutorials/collection/multigroup.md | 17 ++++--- 9 files changed, 91 insertions(+), 108 deletions(-) diff --git a/docs/src/developer/implied.md b/docs/src/developer/implied.md index 056cd6638..6321decbc 100644 --- a/docs/src/developer/implied.md +++ b/docs/src/developer/implied.md @@ -13,9 +13,9 @@ end and a method to update!: ```julia -import StructuralEquationModels: objective! +import StructuralEquationModels: update! -function update!(targets::EvaluationTargets, implied::MyImplied, model::AbstractSemSingle, params) +function update!(targets::EvaluationTargets, implied::MyImplied, params) if is_objective_required(targets) ... @@ -31,11 +31,9 @@ function update!(targets::EvaluationTargets, implied::MyImplied, model::Abstract end ``` -As you can see, `update` gets passed as a first argument `targets`, which is telling us whether the objective value, gradient, and/or hessian are needed. +As you can see, `update!` gets passed as a first argument `targets`, which is telling us whether the objective value, gradient, and/or hessian are needed. We can then use the functions `is_..._required` and conditional on what the optimizer needs, we can compute and store things we want to make available to the loss functions. For example, as we have seen in [Second example - maximum likelihood](@ref), the `RAM` implied type computes the model-implied covariance matrix and makes it available via `implied.Σ`. - - Just as described in [Custom loss functions](@ref), you may define a constructor. Typically, this will depend on the `specification = ...` argument that can be a `ParameterTable` or a `RAMMatrices` object. We implement an `ImpliedEmpty` type in our package that does nothing but serving as an `implied` field in case you are using a loss function that does not need any implied type at all. You may use it as a template for defining your own implied type, as it also shows how to handle the specification objects: @@ -56,7 +54,7 @@ Empty placeholder for models that don't need an implied part. - `specification`: either a `RAMMatrices` or `ParameterTable` object # Examples -A multigroup model with ridge regularization could be specified as a `SemEnsemble` with one +A multigroup model with ridge regularization could be specified as a `Sem` with one model per group and an additional model with `ImpliedEmpty` and `SemRidge` for the regularization part. # Extended help @@ -75,26 +73,20 @@ end ### Constructors ############################################################################################ -function ImpliedEmpty(; - specification, - meanstruct = NoMeanStruct(), - hessianeval = ExactHessian(), +function ImpliedEmpty( + spec::SemSpecification; + hessianeval::HessianApprox = ExactHessian(), kwargs..., ) - return ImpliedEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) + ram_matrices = convert(RAMMatrices, spec) + return ImpliedEmpty(hessianeval, MeanStruct(ram_matrices), ram_matrices) end ############################################################################################ ### methods ############################################################################################ -update!(targets::EvaluationTargets, implied::ImpliedEmpty, par, model) = nothing - -############################################################################################ -### Recommended methods -############################################################################################ - -update_observed(implied::ImpliedEmpty, observed::SemObserved; kwargs...) = implied +update!(targets::EvaluationTargets, implied::ImpliedEmpty, par) = nothing ``` -As you see, similar to [Custom loss functions](@ref) we implement a method for `update_observed`. \ No newline at end of file +As you see, similar to [Custom loss functions](@ref) we implement a constructor. \ No newline at end of file diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index d6949842b..aa6a1e17f 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -11,9 +11,9 @@ Since we allow for the optimization of sums of loss functions, and the maximum l using StructuralEquationModels ``` -To define a new loss function, you have to define a new type that is a subtype of `SemLossFunction`: +To define a new loss function, you have to define a new type that is a subtype of `AbstractLoss`: ```@example loss -struct Ridge <: SemLossFunction +struct MyRidge <: AbstractLoss α I end @@ -25,8 +25,8 @@ Additionaly, we need to define a *method* of the function `evaluate!` to compute ```@example loss import StructuralEquationModels: evaluate! -evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, ridge::Ridge, model::AbstractSem, par) = - ridge.α * sum(i -> par[i]^2, ridge.I) +evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, ridge::MyRidge, par) = + ridge.α * sum(i -> abs2(par[i]), ridge.I) ``` The function `evaluate!` recognizes by the types of the arguments `objective`, `gradient` and `hessian` whether it should compute the objective value, gradient or hessian of the model w.r.t. the parameters. @@ -98,7 +98,7 @@ function evaluate!(objective, gradient, hessian::Nothing, ridge::Ridge, model::A gradient[ridge.I] .= 2 * ridge.α * par[ridge.I] end # compute objective - if !isnothing(objective) + if !isnothing(objective) return ridge.α * sum(i -> par[i]^2, ridge.I) end end @@ -166,17 +166,6 @@ end ## Additional functionality -### Update observed data - -If you are planing a simulation study where you have to fit the **same model** to many **different datasets**, it is computationally beneficial to not build the whole model completely new everytime you change your data. -Therefore, we provide a function to update the data of your model, `replace_observed(model(semfit); data = new_data)`. However, we can not know beforehand in what way your loss function depends on the specific datasets. The solution is to provide a method for `update_observed`. Since `Ridge` does not depend on the data at all, this is quite easy: - -```julia -import StructuralEquationModels: update_observed - -update_observed(ridge::Ridge, observed::SemObserved; kwargs...) = ridge -``` - ### Access additional information If you want to provide a way to query information about loss functions of your type, you can provide functions for that: diff --git a/docs/src/developer/optimizer.md b/docs/src/developer/optimizer.md index b5c9a6e09..4659ba5d5 100644 --- a/docs/src/developer/optimizer.md +++ b/docs/src/developer/optimizer.md @@ -25,12 +25,6 @@ struct MyoptResult{O <: SemOptimizerMyopt} <: SEM.SemOptimizerResult{O} ... end -############################################################################################ -### Recommended methods -############################################################################################ - -update_observed(optimizer::SemOptimizerMyopt, observed::SemObserved; kwargs...) = optimizer - ############################################################################################ ### additional methods ############################################################################################ @@ -43,8 +37,6 @@ and `SEM.sem_optimizer_subtype(::Val{:Myopt})` returns `SemOptimizerMyopt`. This instructs *SEM.jl* to use `SemOptimizerMyopt` when `:Myopt` is specified as the engine for model fitting: `fit(..., engine = :Myopt)`. -A method for `update_observed` and additional methods might be usefull, but are not necessary. - Now comes the essential part: we need to provide the [`fit`](@ref) method with `SemOptimizerMyopt` as the first positional argument. diff --git a/docs/src/developer/sem.md b/docs/src/developer/sem.md index c54ff26af..bb077f434 100644 --- a/docs/src/developer/sem.md +++ b/docs/src/developer/sem.md @@ -1,13 +1,14 @@ # Custom model types -The abstract supertype for all models is `AbstractSem`, which has two subtypes, `AbstractSemSingle{O, I, L}` and `AbstractSemCollection`. Currently, there are 2 subtypes of `AbstractSemSingle`: `Sem`, `SemFiniteDiff`. All subtypes of `AbstractSemSingle` should have at least observed, implied, loss and optimizer fields, and share their types (`{O, I, L}`) with the parametric abstract supertype. For example, the `SemFiniteDiff` type is implemented as +The abstract supertype for all models is [`AbstractSem`](@ref). Currently, there are 2 concrete subtypes: +`Sem{L <: Tuple}` and `SemFiniteDiff{S <: AbstractSem}`. +A `Sem` model holds a tuple of `LossTerm`s (each wrapping an `AbstractLoss`) and a vector of parameter labels. Both single-group and multigroup models are represented as `Sem`. + +`SemFiniteDiff` wraps any `AbstractSem` and substitutes dedicated gradient/hessian evaluation with finite difference approximation: ```julia -struct SemFiniteDiff{O <: SemObserved, I <: SemImplied, L <: SemLoss} <: - AbstractSemSingle{O, I, L} - observed::O - implied::I - loss::L +struct SemFiniteDiff{S <: AbstractSem} <: AbstractSem + model::S end ``` @@ -17,6 +18,4 @@ Additionally, you can change how objective/gradient/hessian values are computed evaluate!(objective, gradient, hessian, model::SemFiniteDiff, params) = ... ``` -Additionally, we can define constructors like the one in `"src/frontend/specification/Sem.jl"`. - -It is also possible to add new subtypes for `AbstractSemCollection`. \ No newline at end of file +Additionally, we can define constructors like the one in `"src/frontend/specification/Sem.jl"`. \ No newline at end of file diff --git a/docs/src/internals/types.md b/docs/src/internals/types.md index e70a52ca4..4b4cd4faa 100644 --- a/docs/src/internals/types.md +++ b/docs/src/internals/types.md @@ -2,12 +2,16 @@ The type hierarchy is implemented in `"src/types.jl"`. -`AbstractSem`: the most abstract type in our package -- `AbstractSemSingle{O, I, L} <: AbstractSem` is an abstract parametric type that is a supertype of all single models - - `Sem`: models that do not need automatic differentiation or finite difference approximation - - `SemFiniteDiff`: models whose gradients and/or hessians should be computed via finite difference approximation -- `AbstractSemCollection <: AbstractSem` is an abstract supertype of all models that contain multiple `AbstractSem` submodels +[`AbstractLoss`](@ref): is the base abstract type for all loss functions: +- `SemLoss{O <: SemObserved, I <: SemImplied}`: is the subtype of `AbstractLoss`, which is the + base for all SEM-specific loss functions ([`SemML`](@ref), [`SemWLS`](@ref) etc) that + evaluate how closely the implied covariation structure (represented by the object of type `I`) + matches the observed one (contained in the object of type `O`); +- regularizing terms (e.g. [`SemRidge`](@ref)) are implemented as subtypes of `AbstractLoss`. -Every `AbstractSemSingle` has to have `SemObserved`, `SemImplied`, and `SemLoss` fields (and can have additional fields). - -`SemLoss` is a container for multiple `SemLossFunctions`. \ No newline at end of file +[`AbstractSem`](@ref) is the base abstract type for all SEM models. It has two concrete subtypes: +- `Sem{L <: Tuple} <: AbstractSem`: the main SEM model type that implements a list of weighted +loss terms (using [`LossTerm`](@ref) wrapper around `AbstractLoss`) and allows modeling both single +and multi-group SEMs and combining them with regularization terms. +- `SemFiniteDiff{S <: AbstractSem} <: AbstractSem`: a wrapper around any `AbstractSem` that + substitutes dedicated gradient/hessian evaluation with finite difference approximation. diff --git a/docs/src/performance/mixed_differentiation.md b/docs/src/performance/mixed_differentiation.md index b7ae333b5..f33fa6ab2 100644 --- a/docs/src/performance/mixed_differentiation.md +++ b/docs/src/performance/mixed_differentiation.md @@ -2,22 +2,20 @@ This way of specifying our model is not ideal, however, because now also the maximum likelihood loss function lives inside a `SemFiniteDiff` model, and this means even though we have defined analytical gradients for it, we do not make use of them. -A more efficient way is therefore to specify our model as an ensemble model: +A more efficient way is therefore to specify our model as a combined model with multiple loss terms: ```julia -model_ml = Sem( - specification = partable, - data = data, - loss = SemML +ml_term = SemML( + SemObservedData(data = data, specification = partable), + RAMSymbolic(partable) ) -model_ridge = SemFiniteDiff( - specification = partable, - data = data, - loss = myridge +ridge_term = SemRidge( + α_ridge = 0.01, + which_ridge = params(ml_term) ) -model_ml_ridge = SemEnsemble(model_ml, model_ridge) +model_ml_ridge = Sem(ml_term, ridge_term) model_ml_ridge_fit = fit(model_ml_ridge) ``` diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index 85a0c0a0c..61a9d5ad7 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -57,19 +57,7 @@ model = Sem( data = data_1 ) -model_updated = replace_observed(model; data = data_2, specification = partable) -``` - -If you are building your models by parts, you can also update each part seperately with the function `update_observed`. -For example, - -```@example replace_observed - -new_observed = SemObservedData(;data = data_2, specification = partable) - -my_optimizer = SemOptimizer() - -new_optimizer = update_observed(my_optimizer, new_observed) +model_updated = replace_observed(model, data_2) ``` ## Multithreading @@ -90,7 +78,7 @@ model1 = Sem( data = data_1 ) -model2 = deepcopy(replace_observed(model; data = data_2, specification = partable)) +model2 = deepcopy(replace_observed(model, data_2)) models = [model1, model2] fits = Vector{SemFit}(undef, 2) @@ -104,5 +92,4 @@ end ```@docs replace_observed -update_observed ``` \ No newline at end of file diff --git a/docs/src/tutorials/collection/collection.md b/docs/src/tutorials/collection/collection.md index f60b7312c..2a8ea92c2 100644 --- a/docs/src/tutorials/collection/collection.md +++ b/docs/src/tutorials/collection/collection.md @@ -1,31 +1,52 @@ # Collections -With StructuralEquationModels.jl, you can fit weighted sums of structural equation models. -The most common use case for this are [Multigroup models](@ref). -Another use case may be optimizing the sum of loss functions for some of which you do know the analytic gradient, but not for others. -In this case, you can optimize the sum of a `Sem` and a `SemFiniteDiff` (or any other differentiation method). +With *StructuralEquationModels.jl*, you can fit weighted sums of structural equation models. +The most common use case for this are [Multigroup models](@ref). +Another use case may be optimizing the sum of loss functions for some of which you do know the analytic gradient, but not for others. +In this case, [`FiniteDiffWrapper`](@ref) can generate a wrapper around the specific `SemLoss` term. The wrapper loss term will +only use the objective of the original term to calculate its gradient using finite difference approximation. -To use this feature, you have to construct a `SemEnsemble` model, which is actually quite easy: +```julia +loss_1 = SemML(observed_1, implied_1) +loss_2 = SemML(observed_2, implied_2) +loss_2_findiff = FiniteDiffWrapper(loss_2) +``` + +To construct `Sem` from the the individual `SemLoss` (or other `AbstractLoss`) terms, they are +just passed to the `Sem` constructor: ```julia -# models -model_1 = Sem(...) +model = Sem(loss_1, loss_2) +model_findiff = Sem(loss_1, loss_2_findiff) +``` + +It is also possible to use finite difference for the entire `Sem` model: -model_2 = SemFiniteDiff(...) +```julia +model_findiff2 = FiniteDiffWrapper(model) +``` -model_3 = Sem(...) +The weighting scheme of the SEM loss terms is specified using `default_set_weights` argument of the `Sem` constructor. +The `:nsamples` scheme (the default) weights SEM terms by ``N_{term}/N_{total}``, i.e. each term is weighted by the number +of observations in its data (which matches the formula for multigroup models). +The weights for the loss terms (both SEM and regularization) can be explicitly specified the pair syntax `loss => weight`: -model_ensemble = SemEnsemble(model_1, model_2, model_3) +```julia +model_weighted = Sem(loss_1 => 0.5, loss_2 => 1.0) ``` -So you just construct the individual models (however you like) and pass them to `SemEnsemble`. -You may also pass a vector of weigths to `SemEnsemble`. By default, those are set to ``N_{model}/N_{total}``, i.e. each model is weighted by the number of observations in it's data (which matches the formula for multigroup models). +`Sem` support assigning unique identifier to each loss term, which is essential for complex multi-term model. +The syntax is `id => loss`, or `id => loss => weight`: -Multigroup models can also be specified via the graph interface; for an example, see [Multigroup models](@ref). +```julia +model2 = Sem(:main => loss_1, :alt => loss_2) +model2_weighted = Sem(:main => loss_1 => 0.5, :alt => loss_2 => 1.0) +``` # API - collections ```@docs -SemEnsemble -AbstractSemCollection +Sem +LossTerm +FiniteDiffWrapper ``` \ No newline at end of file diff --git a/docs/src/tutorials/collection/multigroup.md b/docs/src/tutorials/collection/multigroup.md index 16d3dcd77..04f1893d7 100644 --- a/docs/src/tutorials/collection/multigroup.md +++ b/docs/src/tutorials/collection/multigroup.md @@ -4,19 +4,20 @@ using StructuralEquationModels ``` -As an example, we will fit the model from [the `lavaan` tutorial](https://lavaan.ugent.be/tutorial/groups.html) with loadings constrained to equality across groups. +As an example, we will fit the model from [the `lavaan` tutorial](https://lavaan.ugent.be/tutorial/groups.html) +with loadings constrained to equality across groups. -We first load the example data. +We first load the example data. We have to make sure that the column indicating the group (here called `school`) is a vector of `Symbol`s, not strings - so we convert it. ```@setup mg dat = example_data("holzinger_swineford") -dat.school = ifelse.(dat.school .== "Pasteur", :Pasteur, :Grant_White) +dat.school = Symbol.(replace.(dat.school, "-" => "_")) ``` ```julia dat = example_data("holzinger_swineford") -dat.school = ifelse.(dat.school .== "Pasteur", :Pasteur, :Grant_White) +dat.school = Symbol.(replace.(dat.school, "-" => "_")) ``` We then specify our model via the graph interface: @@ -59,19 +60,19 @@ You can then use the resulting graph to specify an `EnsembleParameterTable` groups = [:Pasteur, :Grant_White] partable = EnsembleParameterTable( - graph, + graph, observed_vars = observed_vars, latent_vars = latent_vars, groups = groups) ``` -The parameter table can be used to create a `SemEnsemble` model: +The parameter table can be used to create a multigroup `Sem` model: ```@example mg; ansicolor = true -model_ml_multigroup = SemEnsemble( +model_ml_multigroup = Sem( specification = partable, data = dat, - column = :school, + semterm_column = :school, groups = groups) ``` From a402bebfd65c629619843985d1e4e7e49081f41e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 22 Mar 2026 13:19:46 -0700 Subject: [PATCH 318/364] test: fix formatting --- test/examples/multigroup/build_models.jl | 12 +++-------- .../political_democracy/constructor.jl | 20 ++++--------------- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 329c55021..6811cb40f 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -19,17 +19,11 @@ model_ml_multigroup = Sem( end # replace observed using Dict of data matrices -model_ml_multigroup3 = replace_observed( - model_ml_multigroup, - Dict(:Pasteur => dat_g1, :Grant_White => dat_g2), -) +model_ml_multigroup3 = + replace_observed(model_ml_multigroup, Dict(:Pasteur => dat_g1, :Grant_White => dat_g2)) # replace observed using DataFrame with group column -model_ml_multigroup4 = replace_observed( - model_ml_multigroup, - dat; - semterm_column = :school, -) +model_ml_multigroup4 = replace_observed(model_ml_multigroup, dat; semterm_column = :school) # gradients @testset "ml_gradients_multigroup" begin diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 759875b25..48ba1b960 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -167,14 +167,8 @@ end # set seed for simulation Random.seed!(83472834) # simulate data - model_ml_new = replace_observed( - model_ml, - rand(model_ml, params, 1_000_000), - ) - model_ml_sym_new = replace_observed( - model_ml_sym, - rand(model_ml_sym, params, 1_000_000), - ) + model_ml_new = replace_observed(model_ml, rand(model_ml, params, 1_000_000)) + model_ml_sym_new = replace_observed(model_ml_sym, rand(model_ml_sym, params, 1_000_000)) # fit models sol_ml = solution(fit(semoptimizer, model_ml_new)) sol_ml_sym = solution(fit(semoptimizer, model_ml_sym_new)) @@ -376,14 +370,8 @@ end # set seed for simulation Random.seed!(83472834) # simulate data - model_ml_new = replace_observed( - model_ml, - rand(model_ml, params, 1_000_000), - ) - model_ml_sym_new = replace_observed( - model_ml_sym, - rand(model_ml_sym, params, 1_000_000), - ) + model_ml_new = replace_observed(model_ml, rand(model_ml, params, 1_000_000)) + model_ml_sym_new = replace_observed(model_ml_sym, rand(model_ml_sym, params, 1_000_000)) # fit models sol_ml = solution(fit(semoptimizer, model_ml_new)) sol_ml_sym = solution(fit(semoptimizer, model_ml_sym_new)) From 42045dce0b082a29cc504c6abf73027175d1d41f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 22 Mar 2026 20:54:23 -0700 Subject: [PATCH 319/364] fit_measures(): support vectors of funcs also add CFI to the list --- src/frontend/fit/fitmeasures/fit_measures.jl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/frontend/fit/fitmeasures/fit_measures.jl b/src/frontend/fit/fitmeasures/fit_measures.jl index 185b348c0..7fabc9503 100644 --- a/src/frontend/fit/fitmeasures/fit_measures.jl +++ b/src/frontend/fit/fitmeasures/fit_measures.jl @@ -1,6 +1,8 @@ -fit_measures(fit) = fit_measures(fit, nparams, dof, AIC, BIC, RMSEA, χ², p_value, minus2ll) +const DEFAULT_FIT_MEASURES = [AIC, BIC, dof, χ², p_value, nparams, RMSEA, CFI] -fit_measures(fit, measures...) = Dict(Symbol(fn) => fn(fit) for fn in measures) +fit_measures(fit, measures::AbstractVector) = Dict(Symbol(fn) => fn(fit) for fn in measures) +fit_measures(fit, measures...) = fit_measures(fit, measures) +fit_measures(fit) = fit_measures(fit, DEFAULT_FIT_MEASURES) """ fit_measures(fit::SemFit, measures...) -> Dict{Symbol} @@ -20,6 +22,7 @@ fit_measures(semfit, nparams, dof, p_value) ``` # See also -[`AIC`](@ref), [`BIC`](@ref), [`RMSEA`](@ref), [`χ²`](@ref), [`p_value`](@ref), [`minus2ll`](@ref) +[`AIC`](@ref), [`BIC`](@ref), [`RMSEA`](@ref), [`χ²`](@ref), [`p_value`](@ref), +[`minus2ll`](@ref), [`CFI`](@ref) """ fit_measures From 4b5942f4b34872930bd1a5ff833693b4b2ea12a3 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 22 Mar 2026 20:58:00 -0700 Subject: [PATCH 320/364] test_fitmeasures(): refactor/simplify --- test/examples/helper.jl | 34 ++++++---- test/examples/multigroup/build_models.jl | 47 ++------------ test/examples/political_democracy/by_parts.jl | 36 ++--------- .../political_democracy/constructor.jl | 64 ++++--------------- 4 files changed, 42 insertions(+), 139 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index f14fec621..fed95f3c7 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -49,7 +49,8 @@ function test_hessian(model, params; rtol = 1e-4, atol = 0) @test hessian ≈ true_hessian rtol = rtol atol = atol end -fitmeasure_names_ml = Dict( +# map from the SEM.jl name of the fit measure to the lavaan's one +fitmeasure_semjl_to_lavaan = Dict( :AIC => "aic", :BIC => "bic", :dof => "df", @@ -57,26 +58,31 @@ fitmeasure_names_ml = Dict( :p_value => "pvalue", :nparams => "npar", :RMSEA => "rmsea", -) - -fitmeasure_names_ls = Dict( - :dof => "df", - :χ² => "chisq", - :p_value => "pvalue", - :nparams => "npar", - :RMSEA => "rmsea", + :CFI => "cfi", ) function test_fitmeasures( - measures, + fitted::SemFit, measures_lav; + fitmeasures::AbstractVector = SEM.DEFAULT_FIT_MEASURES, + fitted_baseline::Union{SemFit, Nothing} = nothing, rtol = 1e-4, atol = 0, - fitmeasure_names = fitmeasure_names_ml, ) - @testset "$name" for (key, name) in pairs(fitmeasure_names) - measure_lav = measures_lav.x[findfirst(==(name), measures_lav[!, 1])] - @test measures[key] ≈ measure_lav rtol = rtol atol = atol + @testset "$fn" for fn in fitmeasures + name = Symbol(fn) + # FIML CFI requires the baseline model + measure = + fn != CFI || isnothing(fitted_baseline) ? fn(fitted) : + fn(fitted, fitted_baseline) + lav_name = fitmeasure_semjl_to_lavaan[name] + lav_ix = findfirst(==(lav_name), measures_lav[!, 1]) + if isnothing(lav_ix) + @test ismissing(measure) + else + measure_lav = measures_lav.x[lav_ix] + @test measure ≈ measure_lav rtol = rtol atol = atol + end end end diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 6811cb40f..711287608 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -52,17 +52,7 @@ end @testset "fitmeasures/se_ml" begin solution_ml = fit(semoptimizer, model_ml_multigroup) - test_fitmeasures( - fit_measures(solution_ml), - solution_lav[:fitmeasures_ml]; - rtol = 1e-2, - atol = 1e-7, - ) - test_fitmeasures( - Dict(:CFI => CFI(solution_ml)), - solution_lav[:fitmeasures_ml]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) + test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7) update_se_hessian!(partable, solution_ml) test_estimates( @@ -122,17 +112,7 @@ end @testset "fitmeasures/se_ml | sorted" begin solution_ml = fit(semoptimizer, model_ml_multigroup) - test_fitmeasures( - fit_measures(solution_ml), - solution_lav[:fitmeasures_ml]; - rtol = 1e-2, - atol = 1e-7, - ) - test_fitmeasures( - Dict(:CFI => CFI(solution_ml)), - solution_lav[:fitmeasures_ml]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) + test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7) update_se_hessian!(partable_s, solution_ml) test_estimates( @@ -221,18 +201,7 @@ end @testset "fitmeasures/se_ls" begin solution_ls = fit(semoptimizer, model_ls_multigroup) - test_fitmeasures( - fit_measures(solution_ls), - solution_lav[:fitmeasures_ls]; - fitmeasure_names = fitmeasure_names_ls, - rtol = 1e-2, - atol = 1e-5, - ) - test_fitmeasures( - Dict(:CFI => CFI(solution_ls)), - solution_lav[:fitmeasures_ls]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) + test_fitmeasures(solution_ls, solution_lav[:fitmeasures_ls]; rtol = 1e-2, atol = 1e-5) @suppress update_se_hessian!(partable, solution_ls) test_estimates( @@ -319,18 +288,14 @@ if !isnothing(specification_miss_g1) @testset "fitmeasures/se_fiml" begin solution = fit(semoptimizer, model_ml_multigroup) + solution_varonly = fit(semoptimizer, model_ml_varonly) test_fitmeasures( - fit_measures(solution), + solution, solution_lav[:fitmeasures_fiml]; + fitted_baseline = solution_varonly, rtol = 1e-3, atol = 0, ) - solution_varonly = fit(semoptimizer, model_ml_varonly) - test_fitmeasures( - Dict(:CFI => CFI(solution, solution_varonly)), - solution_lav[:fitmeasures_fiml]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) update_se_hessian!(partable_miss, solution) test_estimates( partable_miss, diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 6866eead5..ef634a59f 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -90,12 +90,7 @@ end @testset "fitmeasures/se_ml" begin solution_ml = fit(semoptimizer, model_ml) - test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) - test_fitmeasures( - Dict(:CFI => CFI(solution_ml)), - solution_lav[:fitmeasures_ml]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) + test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml]; atol = 1e-3) update_se_hessian!(partable, solution_ml) test_estimates( @@ -109,14 +104,7 @@ end @testset "fitmeasures/se_ls" begin solution_ls = fit(semoptimizer, model_ls_sym) - fm = fit_measures(solution_ls) - test_fitmeasures( - merge(fm, Dict(:CFI => CFI(solution_ls))), - solution_lav[:fitmeasures_ls]; - atol = 1e-3, - fitmeasure_names = merge(fitmeasure_names_ls, Dict(:CFI => "cfi")) - ) - @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) + test_fitmeasures(solution_ls, solution_lav[:fitmeasures_ls]; atol = 1e-3) @suppress update_se_hessian!(partable, solution_ls) test_estimates( @@ -241,16 +229,7 @@ end @testset "fitmeasures/se_ml_mean" begin solution_ml = fit(semoptimizer, model_ml) - test_fitmeasures( - fit_measures(solution_ml), - solution_lav[:fitmeasures_ml_mean]; - atol = 1e-3, - ) - test_fitmeasures( - Dict(:CFI => CFI(solution_ml)), - solution_lav[:fitmeasures_ml_mean]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) + test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml_mean]; atol = 1e-3) update_se_hessian!(partable_mean, solution_ml) test_estimates( @@ -264,14 +243,7 @@ end @testset "fitmeasures/se_ls_mean" begin solution_ls = fit(semoptimizer, model_ls) - fm = fit_measures(solution_ls) - test_fitmeasures( - merge(fm, Dict(:CFI => CFI(solution_ls))), - solution_lav[:fitmeasures_ls_mean]; - atol = 1e-3, - fitmeasure_names = merge(fitmeasure_names_ls, Dict(:CFI => "cfi")), - ) - @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) + test_fitmeasures(solution_ls, solution_lav[:fitmeasures_ls_mean]; atol = 1e-3) @suppress update_se_hessian!(partable_mean, solution_ls) test_estimates( diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 48ba1b960..2efa5abeb 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -100,12 +100,7 @@ end @testset "fitmeasures/se_ml" begin solution_ml = fit(semoptimizer, model_ml) - test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) - test_fitmeasures( - Dict(:CFI => CFI(solution_ml)), - solution_lav[:fitmeasures_ml]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) + test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml]; atol = 1e-3) update_se_hessian!(partable, solution_ml) test_estimates( @@ -122,20 +117,7 @@ end @testset "fitmeasures/se_ls" begin solution_ls = fit(semoptimizer, model_ls_sym) - fm = fit_measures(solution_ls) - test_fitmeasures( - fm, - solution_lav[:fitmeasures_ls]; - atol = 1e-3, - fitmeasure_names = fitmeasure_names_ls, - ) - test_fitmeasures( - Dict(:CFI => CFI(solution_ls)), - solution_lav[:fitmeasures_ls]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) - - @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) + test_fitmeasures(solution_ls, solution_lav[:fitmeasures_ls]; atol = 1e-3) @suppress update_se_hessian!(partable, solution_ls) test_estimates( @@ -298,16 +280,7 @@ end @testset "fitmeasures/se_ml_mean" begin solution_ml = fit(semoptimizer, model_ml) - test_fitmeasures( - fit_measures(solution_ml), - solution_lav[:fitmeasures_ml_mean]; - atol = 0.002, - ) - test_fitmeasures( - Dict(:CFI => CFI(solution_ml)), - solution_lav[:fitmeasures_ml_mean]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) + test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml_mean]; atol = 0.002) update_se_hessian!(partable_mean, solution_ml) test_estimates( @@ -324,19 +297,7 @@ end @testset "fitmeasures/se_ls_mean" begin solution_ls = fit(semoptimizer, model_ls) - fm = fit_measures(solution_ls) - test_fitmeasures( - fm, - solution_lav[:fitmeasures_ls_mean]; - atol = 1e-3, - fitmeasure_names = fitmeasure_names_ls, - ) - @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) - test_fitmeasures( - Dict(:CFI => CFI(solution_ls)), - solution_lav[:fitmeasures_ls_mean]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) + test_fitmeasures(solution_ls, solution_lav[:fitmeasures_ls_mean]; atol = 1e-3) @suppress update_se_hessian!(partable_mean, solution_ls) test_estimates( @@ -410,6 +371,8 @@ if !ismissing(spec_varonly) loss = SemFIML, meanstructure = true, ) +else + model_varonly = nothing end ############################################################################################ @@ -446,19 +409,16 @@ end @testset "fitmeasures/se_fiml" begin solution_ml = fit(semoptimizer, model_ml) + solution_varonly = + !isnothing(model_varonly) ? fit(semoptimizer, model_varonly) : nothing test_fitmeasures( - fit_measures(solution_ml), + solution_ml, solution_lav[:fitmeasures_fiml]; + fitted_baseline = solution_varonly, + fitmeasures = !isnothing(solution_varonly) ? SEM.DEFAULT_FIT_MEASURES : + filter(!=(CFI), SEM.DEFAULT_FIT_MEASURES), atol = 1e-3, ) - if !ismissing(spec_varonly) - solution_varonly = fit(semoptimizer, model_varonly) - test_fitmeasures( - Dict(:CFI => CFI(solution_ml, solution_varonly)), - solution_lav[:fitmeasures_fiml]; - fitmeasure_names = Dict(:CFI => "cfi"), - ) - end update_se_hessian!(partable_mean, solution_ml) test_estimates( From 74390549f1d038d40a496f2f242f9715d8248505 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 22 Mar 2026 23:51:27 -0700 Subject: [PATCH 321/364] test/multigroup: small tweaks --- test/examples/multigroup/build_models.jl | 2 +- test/examples/multigroup/multigroup.jl | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 711287608..6c22a453e 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -240,7 +240,7 @@ if !isnothing(specification_miss_g1) model_ml_varonly = Sem( specification = partable_varonly, - data = dat_missing, + data = dat_miss, semterm_column = :school, loss = SemFIML, observed = SemObservedMissing, diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index c8ae8c1f6..35fe20e6b 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -10,17 +10,18 @@ const SEM = StructuralEquationModels include(joinpath(chop(dirname(pathof(SEM)), tail = 3), "test/examples/helper.jl")) dat = example_data("holzinger_swineford") -dat_missing = example_data("holzinger_swineford_missing") -solution_lav = example_data("holzinger_swineford_solution") +dat.school = Symbol.(replace.(dat.school, "-" => "_")) + +dat_miss = example_data("holzinger_swineford_missing") +dat_miss.school = Symbol.(replace.(dat_miss.school, "-" => "_")) -dat_g1 = dat[dat.school .== "Pasteur", :] -dat_g2 = dat[dat.school .== "Grant-White", :] +solution_lav = example_data("holzinger_swineford_solution") -dat_miss_g1 = dat_missing[dat_missing.school .== "Pasteur", :] -dat_miss_g2 = dat_missing[dat_missing.school .== "Grant-White", :] +dat_g1 = dat[dat.school .== :Pasteur, :] +dat_g2 = dat[dat.school .== :Grant_White, :] -dat.school = ifelse.(dat.school .== "Pasteur", :Pasteur, :Grant_White) -dat_missing.school = ifelse.(dat_missing.school .== "Pasteur", :Pasteur, :Grant_White) +dat_miss_g1 = dat_miss[dat_miss.school .== :Pasteur, :] +dat_miss_g2 = dat_miss[dat_miss.school .== :Grant_White, :] ############################################################################################ ### specification - RAMMatrices From 5b3398d671c8943f8812381640d6e44fb94e92dc Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 30 Mar 2026 15:51:19 -0700 Subject: [PATCH 322/364] finite_diff: replace_observed() calls replace_observed() for the underlying term --- src/frontend/finite_diff.jl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/frontend/finite_diff.jl b/src/frontend/finite_diff.jl index ee0a9bf96..bbf2cc683 100644 --- a/src/frontend/finite_diff.jl +++ b/src/frontend/finite_diff.jl @@ -2,6 +2,9 @@ _unwrap(wrapper::SemFiniteDiff) = wrapper.model params(wrapper::SemFiniteDiff) = params(wrapper.model) loss_terms(wrapper::SemFiniteDiff) = loss_terms(wrapper.model) +replace_observed(wrapper::SemFiniteDiff, data) = + SemFiniteDiff(replace_observed(wrapper.model, data)) + FiniteDiffLossWrappers = Union{LossFiniteDiff, SemLossFiniteDiff} _unwrap(term::AbstractLoss) = term @@ -9,6 +12,17 @@ _unwrap(wrapper::FiniteDiffLossWrappers) = wrapper.loss implied(wrapper::FiniteDiffLossWrappers) = implied(_unwrap(wrapper)) observed(wrapper::FiniteDiffLossWrappers) = observed(_unwrap(wrapper)) +replace_observed(wrapper::LossFiniteDiff, data) = + LossFiniteDiff(replace_observed(_unwrap(wrapper), data)) + +replace_observed(wrapper::SemLossFiniteDiff, new_observed::SemObserved) = + SemLossFiniteDiff(replace_observed(_unwrap(wrapper), new_observed)) + +replace_observed( + wrapper::SemLossFiniteDiff, + data::Union{AbstractMatrix, DataFrame}, +) = SemLossFiniteDiff(replace_observed(_unwrap(wrapper), data)) + FiniteDiffWrapper(model::AbstractSem) = SemFiniteDiff(model) FiniteDiffWrapper(loss::AbstractLoss) = LossFiniteDiff(loss) FiniteDiffWrapper(loss::SemLoss) = SemLossFiniteDiff(loss) From 9f3760999f4fb0a63cf0f28abda2f96e7475fcc6 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 30 Mar 2026 15:52:25 -0700 Subject: [PATCH 323/364] replace_observed(): support kwargs --- src/frontend/finite_diff.jl | 17 +++++++++-------- src/frontend/specification/Sem.jl | 19 +++++++++++-------- src/loss/abstract.jl | 13 +++++++------ 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/frontend/finite_diff.jl b/src/frontend/finite_diff.jl index bbf2cc683..0ecd4865a 100644 --- a/src/frontend/finite_diff.jl +++ b/src/frontend/finite_diff.jl @@ -2,8 +2,8 @@ _unwrap(wrapper::SemFiniteDiff) = wrapper.model params(wrapper::SemFiniteDiff) = params(wrapper.model) loss_terms(wrapper::SemFiniteDiff) = loss_terms(wrapper.model) -replace_observed(wrapper::SemFiniteDiff, data) = - SemFiniteDiff(replace_observed(wrapper.model, data)) +replace_observed(wrapper::SemFiniteDiff, data; kwargs...) = + SemFiniteDiff(replace_observed(wrapper.model, data; kwargs...)) FiniteDiffLossWrappers = Union{LossFiniteDiff, SemLossFiniteDiff} @@ -12,16 +12,17 @@ _unwrap(wrapper::FiniteDiffLossWrappers) = wrapper.loss implied(wrapper::FiniteDiffLossWrappers) = implied(_unwrap(wrapper)) observed(wrapper::FiniteDiffLossWrappers) = observed(_unwrap(wrapper)) -replace_observed(wrapper::LossFiniteDiff, data) = - LossFiniteDiff(replace_observed(_unwrap(wrapper), data)) +replace_observed(wrapper::LossFiniteDiff, data; kwargs...) = + LossFiniteDiff(replace_observed(_unwrap(wrapper), data; kwargs...)) -replace_observed(wrapper::SemLossFiniteDiff, new_observed::SemObserved) = - SemLossFiniteDiff(replace_observed(_unwrap(wrapper), new_observed)) +replace_observed(wrapper::SemLossFiniteDiff, new_observed::SemObserved; kwargs...) = + SemLossFiniteDiff(replace_observed(_unwrap(wrapper), new_observed; kwargs...)) replace_observed( wrapper::SemLossFiniteDiff, - data::Union{AbstractMatrix, DataFrame}, -) = SemLossFiniteDiff(replace_observed(_unwrap(wrapper), data)) + data::Union{AbstractMatrix, DataFrame}; + kwargs..., +) = SemLossFiniteDiff(replace_observed(_unwrap(wrapper), data; kwargs...)) FiniteDiffWrapper(model::AbstractSem) = SemFiniteDiff(model) FiniteDiffWrapper(loss::AbstractLoss) = LossFiniteDiff(loss) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 42ff2d3e8..01f5013cc 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -475,18 +475,19 @@ replace_observed(model, new_df; semterm_column = :group) """ function replace_observed end -function replace_observed(sem::Sem, data::Union{SemObserved, AbstractMatrix}) +function replace_observed(sem::Sem, data::Union{SemObserved, AbstractMatrix}; kwargs...) nsem_terms(sem) > 1 && throw( ArgumentError( "Model contains $(nsem_terms(sem)) SEM terms. " * "Use a Dict{Symbol} or a DataFrame with `semterm_column` to provide per-term data.", ), ) - updated_terms = Tuple(replace_observed(term, data) for term in loss_terms(sem)) + updated_terms = + Tuple(replace_observed(term, data; kwargs...) for term in loss_terms(sem)) return Sem(updated_terms...) end -function replace_observed(sem::Sem, data::AbstractDict{Symbol}) +function replace_observed(sem::Sem, data::AbstractDict{Symbol}; kwargs...) term_ids = Set( if !isnothing(id(term)) id(term) @@ -507,12 +508,12 @@ function replace_observed(sem::Sem, data::AbstractDict{Symbol}) term_data = get(data, tid, nothing) isnothing(term_data) && throw(ArgumentError("No data provided for SEM term :$tid")) - return replace_observed(term, term_data) + return replace_observed(term, term_data; kwargs...) end return Sem(Tuple(updated_terms)...) end -function replace_observed(sem::Sem, data::AbstractVector) +function replace_observed(sem::Sem, data::AbstractVector; kwargs...) nsem = nsem_terms(sem) nsem == length(data) || throw( ArgumentError( @@ -520,7 +521,7 @@ function replace_observed(sem::Sem, data::AbstractVector) ), ) updated_terms = map(enumerate(loss_terms(sem))) do (i, term) - issemloss(term) ? replace_observed(term, data[i]) : term + issemloss(term) ? replace_observed(term, data[i]; kwargs...) : term end return Sem(Tuple(updated_terms)...) end @@ -529,6 +530,7 @@ function replace_observed( sem::Sem, data::AbstractDataFrame; semterm_column::Union{Symbol, Nothing} = nothing, + kwargs..., ) if isnothing(semterm_column) # single-term shortcut @@ -538,7 +540,8 @@ function replace_observed( "Provide `semterm_column` to specify which DataFrame column identifies the groups.", ), ) - updated_terms = Tuple(replace_observed(term, data) for term in loss_terms(sem)) + updated_terms = + Tuple(replace_observed(term, data; kwargs...) for term in loss_terms(sem)) return Sem(updated_terms...) end @@ -547,7 +550,7 @@ function replace_observed( g[semterm_column] => group_data for (g, group_data) in pairs(groupby(data, semterm_column)) ) - return replace_observed(sem, terms_data) + return replace_observed(sem, terms_data; kwargs...) end ############################################################## diff --git a/src/loss/abstract.jl b/src/loss/abstract.jl index bcd6d62b3..2cd9f35d4 100644 --- a/src/loss/abstract.jl +++ b/src/loss/abstract.jl @@ -45,7 +45,7 @@ check_observed_vars(sem::SemLoss) = check_observed_vars(observed(sem), implied(s # replace_observed: SemLoss, AbstractLoss, LossTerm ############################################################################################ -function replace_observed(loss::SemLoss, new_observed::SemObserved) +function replace_observed(loss::SemLoss, new_observed::SemObserved; kwargs...) old_obs = SEM.observed(loss) observed_vars(old_obs) == observed_vars(new_observed) || throw( ArgumentError( @@ -53,19 +53,20 @@ function replace_observed(loss::SemLoss, new_observed::SemObserved) "expected $(observed_vars(old_obs)), got $(observed_vars(new_observed))", ), ) + # the default replace_observed() does not pass through kwargs to the ctor return typeof(loss).name.wrapper(new_observed, SEM.implied(loss)) end -function replace_observed(loss::SemLoss, data::Union{AbstractMatrix, DataFrame}) +function replace_observed(loss::SemLoss, data::Union{AbstractMatrix, DataFrame}; kwargs...) old_obs = SEM.observed(loss) new_observed = typeof(old_obs).name.wrapper(data = data, observed_vars = observed_vars(old_obs)) - return replace_observed(loss, new_observed) + return replace_observed(loss, new_observed; kwargs...) end # non-SEM loss terms are unchanged -replace_observed(loss::AbstractLoss, ::Any) = loss +replace_observed(loss::AbstractLoss, ::Any; kwargs...) = loss # LossTerm: delegate to inner loss -replace_observed(term::LossTerm, data) = - LossTerm(replace_observed(loss(term), data), id(term), weight(term)) +replace_observed(term::LossTerm, data; kwargs...) = + LossTerm(replace_observed(loss(term), data; kwargs...), id(term), weight(term)) From 64eca203847ba28851f9dad4a0e17ce157949d64 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 30 Mar 2026 15:53:29 -0700 Subject: [PATCH 324/364] replace_observed(SemWLS, ...; update_internal_state) the kwarg specifies whether to recalculate weights --- src/loss/WLS/WLS.jl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index d04bc346e..6c36aadd8 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -181,3 +181,17 @@ function evaluate!(objective, gradient, hessian, loss::SemWLS, par) return objective end + +function replace_observed( + loss::SemWLS, + new_observed::SemObserved; + update_internal_state::Bool = true, +) + # recompute weight matrices only if update_internal_state=true + return SemWLS( + new_observed, + SEM.implied(loss); + wls_weight_matrix = update_internal_state ? nothing : loss.V, + wls_weight_matrix_mean = update_internal_state ? nothing : loss.V_μ, + ) +end From 8e99cee457874cb2cdf8cf4c34ea52e76fb3815e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 30 Mar 2026 15:54:09 -0700 Subject: [PATCH 325/364] tests/model: replace_observed() kwargs passing --- test/unit_tests/model.jl | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index 87812fba2..e7f229aa3 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -1,5 +1,7 @@ using StructuralEquationModels, Test, Statistics +const SEM = StructuralEquationModels + dat = example_data("political_democracy") dat_missing = example_data("political_democracy_missing")[:, names(dat)] @@ -73,3 +75,34 @@ end @test @inferred(nsamples(model)) == nsamples(obs) end + +@testset "replace_observed() preserves WLS state through finite-diff wrappers" begin + model = Sem( + specification = ram_matrices, + observed = obs, + implied = RAMSymbolic, + loss = SemWLS, + ) + wls_loss = sem_term(model) + findiff_model = Sem(SEM.FiniteDiffWrapper(wls_loss)) + + new_data = randn(nsamples(obs), nobserved_vars(obs)) + + findiff_model_oldstate = + replace_observed(findiff_model, new_data; update_internal_state = false) + findiff_model_newstate = + replace_observed(findiff_model, new_data; update_internal_state = true) + + loss_orig = SEM._unwrap(sem_term(findiff_model)) + loss_oldstate = SEM._unwrap(sem_term(findiff_model_oldstate)) + loss_newstate = SEM._unwrap(sem_term(findiff_model_newstate)) + + @test loss_orig isa SemWLS + @test loss_oldstate isa SemWLS + @test loss_newstate isa SemWLS + @test loss_orig !== loss_oldstate + @test loss_orig !== loss_newstate + @test loss_oldstate.V === loss_orig.V + @test loss_newstate.V !== loss_orig.V + @test observed_vars(loss_oldstate) == observed_vars(loss_orig) +end From 828e2917161617634a315a1f5019be3bc5bcf97b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 30 Mar 2026 15:54:29 -0700 Subject: [PATCH 326/364] replace_observed(...; recompute_obs_state=true) --- src/frontend/fit/standard_errors/bootstrap.jl | 4 +++- src/loss/WLS/WLS.jl | 8 ++++---- test/unit_tests/model.jl | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index ce84e923b..0a1f39b4d 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -251,6 +251,8 @@ end # Fit one bootstrap replicate: resample, replace observed data, fit. function _fit_bootstrap_sample(sem_model, data, start; engine, fit_kwargs) boot_data = resample_with_replacement(data) - boot_model = replace_observed(sem_model, boot_data) + # we replace the observed data with the bootstrapped one, + # but preserve any internal state that is associated with the original data + boot_model = replace_observed(sem_model, boot_data; recompute_observed_state = true) return fit(boot_model; start_val = start, engine = engine, fit_kwargs...) end diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 6c36aadd8..9acb7de0b 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -185,13 +185,13 @@ end function replace_observed( loss::SemWLS, new_observed::SemObserved; - update_internal_state::Bool = true, + recompute_observed_state::Bool = true, ) - # recompute weight matrices only if update_internal_state=true + # recompute weight matrices only if recompute_observed_state=true return SemWLS( new_observed, SEM.implied(loss); - wls_weight_matrix = update_internal_state ? nothing : loss.V, - wls_weight_matrix_mean = update_internal_state ? nothing : loss.V_μ, + wls_weight_matrix = recompute_observed_state ? nothing : loss.V, + wls_weight_matrix_mean = recompute_observed_state ? nothing : loss.V_μ, ) end diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index e7f229aa3..dd1136f45 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -89,9 +89,9 @@ end new_data = randn(nsamples(obs), nobserved_vars(obs)) findiff_model_oldstate = - replace_observed(findiff_model, new_data; update_internal_state = false) + replace_observed(findiff_model, new_data; recompute_observed_state = false) findiff_model_newstate = - replace_observed(findiff_model, new_data; update_internal_state = true) + replace_observed(findiff_model, new_data; recompute_observed_state = true) loss_orig = SEM._unwrap(sem_term(findiff_model)) loss_oldstate = SEM._unwrap(sem_term(findiff_model_oldstate)) From 95eeb6fbec99f6cb9ac3a19fd25f7ec2fbcbb2fd Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 30 Mar 2026 17:57:10 -0700 Subject: [PATCH 327/364] tests/model: test multi-group data ctor --- src/frontend/specification/Sem.jl | 55 ++++++++++++++++++++++++++++--- test/unit_tests/model.jl | 35 ++++++++++++++++++++ 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 01f5013cc..2d2461662 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -335,6 +335,56 @@ function set_field_type_kwargs!(kwargs, observed, implied, loss, O, I) end end +# build ensemble/multi-group observed from the specification and Sem(...) kwargs +# used by Sem(...) and replace_observed() +function build_ensemble_observed(observed_type, spec::EnsembleParameterTable, kwargs) + if !haskey(kwargs, :data) + @warn """ + No data provided for ensemble SEM model. Each SEM term will be constructed with empty data. + To provide data for each term, pass a DataFrame with a column identifying the term groups or a Dict mapping term ids to data + """ + semterms_data = nothing + else + kwdata = kwargs[:data] + if isa(kwdata, AbstractDataFrame) + semterm_col = get(kwargs, :semterm_column, nothing) + isnothing(semterm_col) && + throw(ArgumentError("No semterm_column specified for ensemble data.")) + semterms_data = Dict( + g[semterm_col] => group_data for + (g, group_data) in pairs(groupby(kwdata, semterm_col)) + ) + elseif isa(kwdata, AbstractDict) + semterms_data = kwdata + else + """ + Unsupported data type for ensemble SEM model: $(typeof(kwdata)). + Provide a DataFrame with a column identifying the term groups or a Dict mapping term ids to data. + """ |> + ArgumentError |> + throw + end + unused_term_ids = setdiff(keys(semterms_data), keys(spec.tables)) + isempty(unused_term_ids) || + @warn "Ignoring data with ids=$(collect(unused_term_ids)): no such SEM terms exist" + end + + # construct SemObserved for each term + return Dict( + term_id => begin + term_kwargs = copy(kwargs) + if !isnothing(semterms_data) + term_data = get(semterms_data, term_id, nothing) + isnothing(term_data) && + throw(ArgumentError("No data provided for SEM term :$term_id")) + term_kwargs[:data] = term_data + delete!(term_kwargs, :semterm_column) + end + observed_type(; specification = term_spec, term_kwargs...) + end for (term_id, term_spec) in pairs(spec.tables) + ) +end + # construct Sem fields function get_fields!(kwargs, spec, observed, implied, loss) if !isa(spec, SemSpecification) @@ -344,10 +394,7 @@ function get_fields!(kwargs, spec, observed, implied, loss) # observed if !isa(observed, SemObserved) observed = if spec isa EnsembleParameterTable - Dict( - term_id => observed(; specification = term_spec, kwargs...) for - (term_id, term_spec) in pairs(spec.tables) - ) + build_ensemble_observed(observed, spec, kwargs) else observed(; specification = spec, kwargs...) end diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index dd1136f45..93ba5e80e 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -106,3 +106,38 @@ end @test loss_newstate.V !== loss_orig.V @test observed_vars(loss_oldstate) == observed_vars(loss_orig) end + +@testset "Sem(...; semterm_column=...) splits ensemble data by group" begin + dat_grouped = copy(dat[:, [:x1, :x2]]) + n_g1 = size(dat_grouped, 1) ÷ 2 + dat_grouped.group = [fill(:g1, n_g1); fill(:g2, size(dat_grouped, 1) - n_g1)] + + group_graph = @StenoGraph begin + f1 → fixed(1.0, 1.0) * x1 + label(:λ₂, :λ₂) * x2 + _(Symbol[:x1, :x2]) ↔ _(Symbol[:x1, :x2]) + _(Symbol[:f1]) ↔ _(Symbol[:f1]) + end + + grouped_partable = EnsembleParameterTable( + group_graph; + observed_vars = [:x1, :x2], + latent_vars = [:f1], + groups = [:g1, :g2], + ) + + grouped_model = Sem( + specification = grouped_partable, + data = dat_grouped, + semterm_column = :group, + observed = SemObservedData, + implied = RAM, + loss = SemML, + ) + + term_g1 = only(filter(term -> SEM.id(term) == :g1, SEM.loss_terms(grouped_model))) + term_g2 = only(filter(term -> SEM.id(term) == :g2, SEM.loss_terms(grouped_model))) + + @test nsamples(observed(term_g1)) == n_g1 + @test nsamples(observed(term_g2)) == size(dat_grouped, 1) - n_g1 + @test nsamples(grouped_model) == size(dat_grouped, 1) +end From 652d54496401b4abb329b55fadf3902e32c7b9c8 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Wed, 25 Mar 2026 14:41:30 +0100 Subject: [PATCH 328/364] SemFiniteDiff constructor to keep same Syntax --- src/types.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/types.jl b/src/types.jl index 87b733cf4..eb251a3b2 100644 --- a/src/types.jl +++ b/src/types.jl @@ -138,6 +138,9 @@ struct SemLossFiniteDiff{O, I, L <: SemLoss{O, I}} <: SemLoss{O, I} loss::L end +SemFiniteDiff(args...; kwargs...) = + SemFiniteDiff(Sem(args...; gradient = false, hessian = false, kwargs...)) + """ abstract type SemSpecification end From 27ce0b08129a475f6a209740a9adf92515567fb7 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Wed, 25 Mar 2026 22:40:37 +0100 Subject: [PATCH 329/364] Sem print methods --- src/frontend/specification/Sem.jl | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 2d2461662..db76acfae 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -28,20 +28,17 @@ for f in ( end function Base.show(io::IO, term::LossTerm) + print(io, nameof(losstype(term))) + print(io, "\n") if !isnothing(id(term)) - print(io, ":$(id(term)): ") + print(io, " - id: $(id(term)) \n") end - print(io, nameof(losstype(term))) if issemloss(term) - print( - io, - " ($(nsamples(term)) samples, $(nobserved_vars(term)) observed, $(nlatent_vars(term)) latent variables)", - ) + print(io, " - observed: $(nameof(typeof(observed(loss(term))))) \n") + print(io, " - implied: $(nameof(typeof(implied(loss(term))))) \n") end if !isnothing(weight(term)) - print(io, " w=$(round(weight(term), digits=3))") - else - print(io, " w=1") + print(io, " - weight: $(round(weight(term), digits=3))") end end @@ -604,12 +601,17 @@ end # pretty printing ############################################################## +_subtype_info(::Sem) = nothing +_subtype_info(::SemFiniteDiff) = "Finite Difference Approximation" + function Base.show(io::IO, sem::AbstractSem) - println(io, "Structural Equation Model ($(nameof(typeof(sem))))") - println(io, "- $(nparams(sem)) parameters") - println(io, "- Loss terms:") + print(io, "Structural Equation Model") + si = _subtype_info(sem) + isnothing(si) || print(io, " : "*si) + print("\n") + print(io, "- Loss Functions \n") for term in loss_terms(sem) - print(io, " - ") + print(io, " > ") print(io, term) println(io) end From 46dd2efa27beebf4bbb98b929f0064b74e23e6da Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 10 Apr 2026 18:36:06 +0200 Subject: [PATCH 330/364] add details method for AbstractSem --- src/frontend/fit/summary.jl | 43 ++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index c8495b799..fe7ea9307 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -1,3 +1,38 @@ +function details(sem::AbstractSem) + print("Structural Equation Model") + print(_subtype_info(sem)) + print("\n") + print("- Loss Functions \n") + for term in loss_terms(sem) + print(" > ") + details(term) + println() + end +end + +function details(term::LossTerm) + if !issemloss(term) + print(term.loss) + else + println("Structural Equation Model Loss ($(nameof(typeof(term.loss))))") + if !isnothing(id(term)) + print(" - id: $(id(term)) \n") + end + println( + " - Observed: $(nameof(typeof(observed(term)))) ($(nsamples(term)) samples)", + ) + println( + " - Implied: $(nameof(typeof(implied(term)))) ($(nparams(term)) parameters)", + ) + println( + " - Variables: $(nobserved_vars(term)) observed, $(nlatent_vars(term)) latent", + ) + if !isnothing(weight(term)) + print(" - weight: $(round(weight(term), digits=3))") + end + end +end + function details(sem_fit::SemFit; show_fitmeasures = false, color = :light_cyan, digits = 2) print("\n") println("Fitted Structural Equation Model") @@ -325,11 +360,13 @@ function Base.findall(fun::Function, partable::ParameterTable) end """ - (1) details(sem_fit::SemFit; show_fitmeasures = false) + (1) details(model::AbstractSem) + + (2) details(sem_fit::SemFit; show_fitmeasures = false) - (2) details(partable::AbstractParameterTable; ...) + (3) details(partable::AbstractParameterTable; ...) -Print information about (1) a fitted SEM or (2) a parameter table to stdout. +Print information about (1) a SEM, (2) a fitted SEM or (3) a parameter table to stdout. # Extended help ## Addition keyword arguments From c052d61da09c6bb71abe6e13a4c5ecaf28d95292 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sat, 11 Apr 2026 23:12:27 +0200 Subject: [PATCH 331/364] shorten model --- src/frontend/specification/Sem.jl | 41 ++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index db76acfae..12b8da159 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -28,17 +28,35 @@ for f in ( end function Base.show(io::IO, term::LossTerm) - print(io, nameof(losstype(term))) - print(io, "\n") - if !isnothing(id(term)) - print(io, " - id: $(id(term)) \n") - end - if issemloss(term) - print(io, " - observed: $(nameof(typeof(observed(loss(term))))) \n") - print(io, " - implied: $(nameof(typeof(implied(loss(term))))) \n") - end - if !isnothing(weight(term)) - print(io, " - weight: $(round(weight(term), digits=3))") + if (:compact => true) in io + if !isnothing(id(term)) + print(io, ":$(id(term)): ") + end + print(io, nameof(losstype(term))) + if issemloss(term) + print( + io, + " ($(nsamples(term)) samples, $(nobserved_vars(term)) observed, $(nlatent_vars(term)) latent variables)", + ) + end + if !isnothing(weight(term)) + print(io, " w=$(round(weight(term), digits=3))") + else + print(io, " w=1") + end + else + print(io, nameof(losstype(term))) + print(io, "\n") + if !isnothing(id(term)) + print(io, " - id: $(id(term)) \n") + end + if issemloss(term) + print(io, " - observed: $(nameof(typeof(observed(loss(term))))) \n") + print(io, " - implied: $(nameof(typeof(implied(loss(term))))) \n") + end + if !isnothing(weight(term)) + print(io, " - weight: $(round(weight(term), digits=3))") + end end end @@ -610,6 +628,7 @@ function Base.show(io::IO, sem::AbstractSem) isnothing(si) || print(io, " : "*si) print("\n") print(io, "- Loss Functions \n") + io = length(loss_terms(sem)) >= 5 ? IOContext(io, :compact => true) : io for term in loss_terms(sem) print(io, " > ") print(io, term) From b3242ffc18360329c169546150b92794c5ece242 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 12 Apr 2026 21:23:11 -0700 Subject: [PATCH 332/364] Sem(): remove SemWLS kw check logic --- src/frontend/specification/Sem.jl | 12 ++---------- test/unit_tests/model.jl | 2 ++ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 12b8da159..a3ccec58c 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -417,21 +417,13 @@ function get_fields!(kwargs, spec, observed, implied, loss) # implied if !isa(implied, SemImplied) - # FIXME remove this implicit logic - # SemWLS only accepts vech-ed implied covariance - if isa(loss, Type) && (loss <: SemWLS) && !haskey(kwargs, :vech) - implied_kwargs = copy(kwargs) - implied_kwargs[:vech] = true - else - implied_kwargs = kwargs - end implied = if spec isa EnsembleParameterTable Dict( - term_id => implied(term_spec; implied_kwargs...) for + term_id => implied(term_spec; kwargs...) for (term_id, term_spec) in pairs(spec.tables) ) else - implied(spec; implied_kwargs...) + implied(spec; kwargs...) end end diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index 93ba5e80e..c80a0c1b9 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -58,6 +58,7 @@ end observed = obs, implied = impliedtype, loss = losstype, + vech = losstype <: SemWLS && impliedtype <: RAMSymbolic ) @test model isa Sem @@ -82,6 +83,7 @@ end observed = obs, implied = RAMSymbolic, loss = SemWLS, + vech = true ) wls_loss = sem_term(model) findiff_model = Sem(SEM.FiniteDiffWrapper(wls_loss)) From 0c95fd6b9868d357558c30ff28e8769cc3f3d009 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 12 Apr 2026 21:32:48 -0700 Subject: [PATCH 333/364] Sem(): cleanup constructor * rename get_fields!() into build_sem_terms() for clarity * move set_field_type!() code into Sem() ctor since its not used outside --- src/frontend/specification/Sem.jl | 37 ++++++++++++++----------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index a3ccec58c..1c0c5f3fa 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -200,9 +200,19 @@ function Sem(; ) where {O, I, L} kwdict = Dict{Symbol, Any}(kwargs...) - set_field_type_kwargs!(kwdict, observed, implied, loss, O, I) + # add kwargs with type information + kwdict[:observed_type] = O <: Type ? observed : typeof(observed) + kwdict[:implied_type] = I <: Type ? implied : typeof(implied) + if loss isa SemLoss + kwdict[:loss_types] = + [aloss isa SemLoss ? typeof(aloss) : aloss for aloss in loss.functions] + elseif applicable(iterate, loss) + kwdict[:loss_types] = [aloss isa SemLoss ? typeof(aloss) : aloss for aloss in loss] + else + kwdict[:loss_types] = [loss isa SemLoss ? typeof(loss) : loss] + end - loss = get_fields!(kwdict, specification, observed, implied, loss) + loss = build_sem_terms(kwdict, specification, observed, implied, loss) return Sem(loss...) end @@ -337,19 +347,6 @@ vars(model::AbstractSem, id::Nothing = nothing) = vars(implied(model, id)) observed_vars(model::AbstractSem, id::Nothing = nothing) = observed_vars(implied(model, id)) latent_vars(model::AbstractSem, id::Nothing = nothing) = latent_vars(implied(model, id)) -function set_field_type_kwargs!(kwargs, observed, implied, loss, O, I) - kwargs[:observed_type] = O <: Type ? observed : typeof(observed) - kwargs[:implied_type] = I <: Type ? implied : typeof(implied) - if loss isa SemLoss - kwargs[:loss_types] = - [aloss isa SemLoss ? typeof(aloss) : aloss for aloss in loss.functions] - elseif applicable(iterate, loss) - kwargs[:loss_types] = [aloss isa SemLoss ? typeof(aloss) : aloss for aloss in loss] - else - kwargs[:loss_types] = [loss isa SemLoss ? typeof(loss) : loss] - end -end - # build ensemble/multi-group observed from the specification and Sem(...) kwargs # used by Sem(...) and replace_observed() function build_ensemble_observed(observed_type, spec::EnsembleParameterTable, kwargs) @@ -400,8 +397,8 @@ function build_ensemble_observed(observed_type, spec::EnsembleParameterTable, kw ) end -# construct Sem fields -function get_fields!(kwargs, spec, observed, implied, loss) +# called by Sem() ctor to construct its loss terms +function build_sem_terms(kwargs::AbstractDict, spec, observed, implied, loss) if !isa(spec, SemSpecification) spec = spec(; kwargs...) end @@ -430,13 +427,13 @@ function get_fields!(kwargs, spec, observed, implied, loss) # loss loss_kwargs = copy(kwargs) loss_kwargs[:nparams] = nparams(spec) - loss = build_SemTerms(loss, observed, implied; loss_kwargs...) + loss = build_sem_terms(loss, observed, implied; loss_kwargs...) return loss end -# construct loss field -function build_SemTerms(loss, observed, implied; kwargs...) +# construct loss terms for the given observed and implied +function build_sem_terms(loss, observed, implied; kwargs...) function build_SemLoss(aloss, observed, implied) if loss isa AbstractLoss return loss From eeaf32c5a4b68610aabe01b0e737418ac080c423 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 12 Apr 2026 22:59:47 -0700 Subject: [PATCH 334/364] show(::Sem): respect existing :compact key --- src/frontend/specification/Sem.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 1c0c5f3fa..be9c4fc3b 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -612,12 +612,15 @@ _subtype_info(::Sem) = nothing _subtype_info(::SemFiniteDiff) = "Finite Difference Approximation" function Base.show(io::IO, sem::AbstractSem) + # if not specified, use compact printing for larger models + if !haskey(io, :compact) && length(loss_terms(sem)) >= 5 + io = IOContext(io, :compact => true) + end print(io, "Structural Equation Model") si = _subtype_info(sem) isnothing(si) || print(io, " : "*si) print("\n") print(io, "- Loss Functions \n") - io = length(loss_terms(sem)) >= 5 ? IOContext(io, :compact => true) : io for term in loss_terms(sem) print(io, " > ") print(io, term) From 5eabccd582d44c61210e03883d5cfc0983eab371 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sat, 25 Apr 2026 11:04:55 +0200 Subject: [PATCH 335/364] add show method for AbstractSem --- src/frontend/specification/Sem.jl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index be9c4fc3b..2ecac9e71 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -612,6 +612,19 @@ _subtype_info(::Sem) = nothing _subtype_info(::SemFiniteDiff) = "Finite Difference Approximation" function Base.show(io::IO, sem::AbstractSem) + io = IOContext(io, :compact => true) + println(io, "Structural Equation Model ($(nameof(typeof(sem))))") + println(io, "- $(nparams(sem)) parameters") + println(io, "- Loss Functions:") + for term in loss_terms(sem) + print(io, " - ") + print(io, term) + println(io) + end +end + +# pretty prenting for console +function Base.show(io::IO, ::MIME"text/plain", sem::AbstractSem) # if not specified, use compact printing for larger models if !haskey(io, :compact) && length(loss_terms(sem)) >= 5 io = IOContext(io, :compact => true) From bcebd2b142f0583388f2425cee99de0aab73c95a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 3 May 2026 11:36:48 -0700 Subject: [PATCH 336/364] simulation.md: whitespace fixes --- docs/src/performance/simulation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index 61a9d5ad7..f5199f626 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -40,7 +40,7 @@ end partable = ParameterTable( graph, - latent_vars = latent_vars, + latent_vars = latent_vars, observed_vars = observed_vars ) ``` @@ -63,10 +63,10 @@ model_updated = replace_observed(model, data_2) ## Multithreading !!! danger "Thread safety" *This is only relevant when you are planning to fit updated models in parallel* - - Models generated by `replace_observed` may share the same objects in memory (e.g. some parts of + + Models generated by `replace_observed` may share the same objects in memory (e.g. some parts of `model` and `model_updated` are the same objects in memory.) - Therefore, fitting both of these models in parallel will lead to **race conditions**, + Therefore, fitting both of these models in parallel will lead to **race conditions**, possibly crashing your computer. To avoid these problems, you should copy `model` before updating it. From 130470536d307acd42a2814b2c637ada9c2418c4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 3 May 2026 12:32:32 -0700 Subject: [PATCH 337/364] replace_obs(sem): make sure Sem type is preserved Co-authored-by: Copilot --- src/frontend/specification/Sem.jl | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 2ecac9e71..bbf7dab83 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -491,6 +491,14 @@ end # replace_observed: Sem level ############################################################## +# internal function to create a copy of Sem with the loss term replaced +# used by the replace_observed() +_replace_loss_terms(sem::Sem, new_terms::Tuple) = + Sem{typeof(new_terms)}(new_terms, copy(params(sem))) + +_replace_loss_terms(sem::Sem, new_terms::AbstractVector) = + _replace_loss_terms(sem, Tuple(new_terms)) + """ replace_observed(model::Sem, observed::SemObserved) replace_observed(model::Sem, data::AbstractDict{Symbol}) @@ -533,9 +541,10 @@ function replace_observed(sem::Sem, data::Union{SemObserved, AbstractMatrix}; kw "Use a Dict{Symbol} or a DataFrame with `semterm_column` to provide per-term data.", ), ) - updated_terms = - Tuple(replace_observed(term, data; kwargs...) for term in loss_terms(sem)) - return Sem(updated_terms...) + updated_terms = map(loss_terms(sem)) do term + replace_observed(term, data; kwargs...) + end + return _replace_loss_terms(sem, updated_terms) end function replace_observed(sem::Sem, data::AbstractDict{Symbol}; kwargs...) @@ -561,7 +570,7 @@ function replace_observed(sem::Sem, data::AbstractDict{Symbol}; kwargs...) throw(ArgumentError("No data provided for SEM term :$tid")) return replace_observed(term, term_data; kwargs...) end - return Sem(Tuple(updated_terms)...) + return _replace_loss_terms(sem, updated_terms) end function replace_observed(sem::Sem, data::AbstractVector; kwargs...) @@ -574,7 +583,7 @@ function replace_observed(sem::Sem, data::AbstractVector; kwargs...) updated_terms = map(enumerate(loss_terms(sem))) do (i, term) issemloss(term) ? replace_observed(term, data[i]; kwargs...) : term end - return Sem(Tuple(updated_terms)...) + return _replace_loss_terms(sem, updated_terms) end function replace_observed( @@ -591,9 +600,10 @@ function replace_observed( "Provide `semterm_column` to specify which DataFrame column identifies the groups.", ), ) - updated_terms = - Tuple(replace_observed(term, data; kwargs...) for term in loss_terms(sem)) - return Sem(updated_terms...) + updated_terms = map(loss_terms(sem)) do term + replace_observed(term, data; kwargs...) + end + return _replace_loss_terms(sem, updated_terms) end # multi-term: split DataFrame by semterm_column From 6a906b3cc51cdd39fe77dc36626523ff2261fac1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 3 May 2026 12:34:19 -0700 Subject: [PATCH 338/364] replace_obs(sem): update docstring --- src/frontend/specification/Sem.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index bbf7dab83..529819902 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -509,7 +509,12 @@ _replace_loss_terms(sem::Sem, new_terms::AbstractVector) = Construct a new SEM model or SEM loss with replaced observed data. The SEM structure (implied covariance, loss type) is preserved; -only the observed data is swapped. +only the observed data is swapped. The new loss terms preserve the configuration +and share the implied state with the loss terms of the original SEM model. + +Keyword arguments: +- `recompute_observed_state::Bool = true`: loss terms should recompute observed-dependent + caches. Losses without such caches ignore this argument. # Single-term models From 5989dcf5d78536805884b74a673ab66d2c2b6b39 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 3 May 2026 12:35:42 -0700 Subject: [PATCH 339/364] replace_obs(loss): extract check_obs_vars() method Co-authored-by: Copilot --- src/loss/abstract.jl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/loss/abstract.jl b/src/loss/abstract.jl index 2cd9f35d4..2b4940ada 100644 --- a/src/loss/abstract.jl +++ b/src/loss/abstract.jl @@ -41,18 +41,20 @@ end check_observed_vars(sem::SemLoss) = check_observed_vars(observed(sem), implied(sem)) +function check_observed_vars(loss::SemLoss, new_observed::SemObserved) + observed_vars(new_observed) == observed_vars(SEM.observed(loss)) || throw( + ArgumentError( + "Observed variables of the loss term do not match the ones of the new observed data", + ), + ) +end + ############################################################################################ # replace_observed: SemLoss, AbstractLoss, LossTerm ############################################################################################ function replace_observed(loss::SemLoss, new_observed::SemObserved; kwargs...) - old_obs = SEM.observed(loss) - observed_vars(old_obs) == observed_vars(new_observed) || throw( - ArgumentError( - "observed_vars of the new data do not match the model: " * - "expected $(observed_vars(old_obs)), got $(observed_vars(new_observed))", - ), - ) + check_observed_vars(loss, new_observed) # the default replace_observed() does not pass through kwargs to the ctor return typeof(loss).name.wrapper(new_observed, SEM.implied(loss)) end From 1b3b8b5123f0d8668d8053ed649c0db625868ad4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 3 May 2026 20:17:19 -0700 Subject: [PATCH 340/364] test/multigroup: avoid clash with observed_vars() method --- test/examples/multigroup/multigroup.jl | 34 +++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 35fe20e6b..d095cf118 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -103,8 +103,8 @@ end # w.o. meanstructure ----------------------------------------------------------------------- -latent_vars = [:visual, :textual, :speed] -observed_vars = Symbol.(:x, 1:9) +lat_vars = [:visual, :textual, :speed] +obs_vars = Symbol.(:x, 1:9) graph = @StenoGraph begin # measurement model @@ -112,14 +112,14 @@ graph = @StenoGraph begin textual → fixed(1.0, 1.0) * x4 + label(:λ₅, :λ₅) * x5 + label(:λ₆, :λ₆) * x6 speed → fixed(1.0, 1.0) * x7 + label(:λ₈, :λ₈) * x8 + label(:λ₉, :λ₉) * x9 # variances and covariances - _(observed_vars) ↔ _(observed_vars) - _(latent_vars) ⇔ _(latent_vars) + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ⇔ _(lat_vars) end partable = EnsembleParameterTable( graph; - observed_vars = observed_vars, - latent_vars = latent_vars, + observed_vars = obs_vars, + latent_vars = lat_vars, groups = [:Pasteur, :Grant_White], ) @@ -130,8 +130,8 @@ specification_g2 = specification[:Grant_White] # w. meanstructure (fiml) ------------------------------------------------------------------ -latent_vars = [:visual, :textual, :speed] -observed_vars = Symbol.(:x, 1:9) +lat_vars = [:visual, :textual, :speed] +obs_vars = Symbol.(:x, 1:9) graph = @StenoGraph begin # measurement model @@ -139,16 +139,16 @@ graph = @StenoGraph begin textual → fixed(1.0, 1.0) * x4 + label(:λ₅, :λ₅) * x5 + label(:λ₆, :λ₆) * x6 speed → fixed(1.0, 1.0) * x7 + label(:λ₈, :λ₈) * x8 + label(:λ₉, :λ₉) * x9 # variances and covariances - _(observed_vars) ↔ _(observed_vars) - _(latent_vars) ⇔ _(latent_vars) + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ⇔ _(lat_vars) - Symbol(1) → _(observed_vars) + Symbol(1) → _(obs_vars) end partable_miss = EnsembleParameterTable( graph; - observed_vars = observed_vars, - latent_vars = latent_vars, + observed_vars = obs_vars, + latent_vars = lat_vars, groups = [:Pasteur, :Grant_White], ) @@ -159,14 +159,14 @@ specification_miss_g2 = specification_miss[:Grant_White] # CFI baseline model graph_varonly = @StenoGraph begin - _(observed_vars) ↔ _(observed_vars) - Symbol(1) → _(observed_vars) + _(obs_vars) ↔ _(obs_vars) + Symbol(1) → _(obs_vars) end partable_varonly = EnsembleParameterTable( graph_varonly; - observed_vars = observed_vars, - latent_vars = latent_vars, + observed_vars = obs_vars, + latent_vars = lat_vars, groups = [:Pasteur, :Grant_White], ) From ca8a517c89ac8dcdc4969e77573ce19082abf30c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 3 May 2026 20:26:30 -0700 Subject: [PATCH 341/364] SemLoss(observed, implied, refloss; kwarg...) ctor to allow replicating the whole refloss state, e.g. for replace_observed() Co-authored-by: Copilot --- docs/src/developer/loss.md | 38 ++++++++++-------------- src/loss/ML/FIML.jl | 16 ++++++++-- src/loss/ML/ML.jl | 11 +++++-- src/loss/WLS/WLS.jl | 20 +++++++++---- test/examples/multigroup/build_models.jl | 7 +++-- 5 files changed, 56 insertions(+), 36 deletions(-) diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index aa6a1e17f..8cdf2150f 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -136,31 +136,25 @@ Additionally, you may provide analytic hessians by writing a respective method f ## Convenient -To be able to build the model with the [Outer Constructor](@ref), you need to add a constructor for your loss function that only takes keyword arguments and allows for passing optional additional kewyword arguments. A constructor is just a function that creates a new instance of your type: +To be able to build the loss term, it needs a constructor. +Every `SemLoss` subtype should provide a constructor with 3 positional arguments: + * `observed::SemObserved`: the observed part of the model + * `implied::SemImplied`: the implied part of the model + * `refloss::Union{MyLoss, Nothing} = nothing`: optional loss term of the same type + to use as a reference for any loss-specific configuration. + +Any additional loss configuration details should be passed as optional keyword arguments. +If both `refloss` and the keyword arguments are provided, the keyword arguments take +precedence. This constructor is used internally by the functions like [`replace_observed`](@ref) +to rebuild the loss term with new observed data while preserving the implied state. ```julia -function MyLoss(;arg1 = ..., arg2, kwargs...) +function MyLoss( + observed::SemObserved, implied::SemImplied, refloss::Union{MyLoss, Nothing} = nothing; + kwarg1 = ..., kwarg2 = ..., kwargs... +) ... - return MyLoss(...) -end -``` - -All keyword arguments that a user passes to the Sem constructor are passed to your loss function. In addition, all previously constructed parts of the model (implied and observed part) are passed as keyword arguments as well as the number of parameters `n_par = ...`, so your constructor may depend on those. For example, the constructor for `SemML` in our package depends on the additional argument `meanstructure` as well as the observed part of the model to pre-allocate arrays of the same size as the observed covariance matrix and the observed mean vector: - -```julia -function SemML(;observed, meanstructure = false, approx_H = false, kwargs...) - - isnothing(obs_mean(observed)) ? - meandiff = nothing : - meandiff = copy(obs_mean(observed)) - - return SemML( - similar(obs_cov(observed)), - similar(obs_cov(observed)), - meandiff, - approx_H, - Val(meanstructure) - ) + return MyLoss(...) # internal MyLoss constructor end ``` diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 15081e201..74d5edfb4 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -75,13 +75,16 @@ Can handle observed data with missing values. # Constructor - SemFIML(observed::SemObservedMissing, implied::SemImplied) + SemFIML(observed::SemObservedMissing, implied::SemImplied, refloss = nothing) # Arguments - `observed::SemObservedMissing`: the observed part of the model (see [`SemObservedMissing`](@ref)) - `implied::SemImplied`: the implied part of the model (see [`SemImplied`](@ref)) +- `refloss::Union{SemFIML, Nothing}`: optional reference loss used to preserve + loss-specific configuration and share the internal state when rebuilding + a loss term, e.g. in [`replace_observed`](@ref) # Examples ```julia @@ -109,7 +112,12 @@ end ### Constructors ############################################################################################ -function SemFIML(observed::SemObservedMissing, implied::SemImplied; kwargs...) +function SemFIML( + observed::SemObservedMissing, + implied::SemImplied, + refloss::Union{SemFIML, Nothing} = nothing; + kwargs..., +) if MeanStruct(implied) === NoMeanStruct """ Full information maximum likelihood (FIML) can only be used with a meanstructure. @@ -124,8 +132,10 @@ function SemFIML(observed::SemObservedMissing, implied::SemImplied; kwargs...) observed, implied, [SemFIMLPattern(pat) for pat in observed.patterns], + # share the internal state with the refloss + !isnothing(refloss) ? refloss.imp_inv : zeros(nobserved_vars(observed), nobserved_vars(observed)), - CommutationMatrix(nvars(implied)), + !isnothing(refloss) ? refloss.commutator : CommutationMatrix(nvars(implied)), nothing, ) end diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 9f3275443..cf1198321 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -8,11 +8,14 @@ Maximum likelihood estimation. # Constructor - SemML(observed, implied; approximate_hessian = false) + SemML(observed, implied, refloss = nothing; approximate_hessian = false) # Arguments - `observed::SemObserved`: the observed part of the model - `implied::SemImplied`: [`SemImplied`](@ref) instance +- `refloss::Union{SemML, Nothing}`: optional reference loss used to preserve + loss-specific configuration and share the internal state when rebuilding + a loss term, e.g. in [`replace_observed`](@ref) - `approximate_hessian::Bool`: if hessian-based optimization is used, should the hessian be swapped for an approximation # Examples @@ -39,8 +42,10 @@ end function SemML( observed::SemObserved, - implied::SemImplied; - approximate_hessian::Bool = false, + implied::SemImplied, + refloss::Union{SemML, Nothing} = nothing; + approximate_hessian::Bool = !isnothing(refloss) ? + HessianEval(refloss) === ApproxHessian : false, kwargs..., ) if observed isa SemObservedMissing diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 9acb7de0b..d067e346a 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -10,7 +10,7 @@ At the moment only available with the `RAMSymbolic` implied type. # Constructor SemWLS( - observed::SemObserved, implied::SemImplied; + observed::SemObserved, implied::SemImplied, refloss = nothing; wls_weight_matrix = nothing, wls_weight_matrix_mean = nothing, approximate_hessian = false, @@ -19,6 +19,9 @@ At the moment only available with the `RAMSymbolic` implied type. # Arguments - `observed`: the `SemObserved` part of the model - `implied`: the `SemImplied` part of the model +- `refloss::Union{SemWLS, Nothing}`: optional reference loss used to preserve + loss-specific configuration and share the internal state when rebuilding + a loss term, e.g. in [`replace_observed`](@ref) - `approximate_hessian::Bool`: should the hessian be swapped for an approximation - `wls_weight_matrix`: the weight matrix for weighted least squares. Defaults to GLS estimation (``0.5*(D^T*kron(S,S)*D)`` where D is the duplication matrix @@ -58,10 +61,14 @@ end function SemWLS( observed::SemObserved, - implied::SemImplied; - wls_weight_matrix::Union{AbstractMatrix, Nothing} = nothing, - wls_weight_matrix_mean::Union{AbstractMatrix, Nothing} = nothing, - approximate_hessian::Bool = false, + implied::SemImplied, + refloss::Union{SemWLS, Nothing} = nothing; + wls_weight_matrix::Union{AbstractMatrix, Nothing} = !isnothing(refloss) ? refloss.V : + nothing, + wls_weight_matrix_mean::Union{AbstractMatrix, Nothing} = !isnothing(refloss) ? + refloss.V_μ : nothing, + approximate_hessian::Bool = !isnothing(refloss) ? + HessianEval(refloss) === ApproxHessian : false, verbose::Bool = false, kwargs..., ) @@ -190,7 +197,8 @@ function replace_observed( # recompute weight matrices only if recompute_observed_state=true return SemWLS( new_observed, - SEM.implied(loss); + SEM.implied(loss), + loss; wls_weight_matrix = recompute_observed_state ? nothing : loss.V, wls_weight_matrix_mean = recompute_observed_state ? nothing : loss.V_μ, ) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 6c22a453e..b538a4f46 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -139,8 +139,11 @@ struct UserSemML{O, I} <: SemLoss{O, I} observed::O implied::I - UserSemML(observed::SemObserved, implied::SemImplied) = - new{typeof(observed), typeof(implied)}(ExactHessian(), observed, implied) + UserSemML( + observed::SemObserved, + implied::SemImplied, + refloss::Union{UserSemML, Nothing} = nothing, + ) = new{typeof(observed), typeof(implied)}(ExactHessian(), observed, implied) end function SEM.objective(ml::UserSemML, params) From 6eed3094ccc025ae9e66541ea0c80807bd75eb23 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 3 May 2026 20:36:28 -0700 Subject: [PATCH 342/364] replace_observed(): use 3-arg SemLoss ctor Co-authored-by: Copilot --- docs/src/performance/simulation.md | 9 +++++++-- src/loss/abstract.jl | 12 ++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index f5199f626..3061c6564 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -3,8 +3,13 @@ ## Replace observed data In simulation studies, a common task is fitting the same model to many different datasets. It would be a waste of resources to reconstruct the complete model for each dataset. -We therefore provide the function `replace_observed` to change the `observed` part of a model, -without necessarily reconstructing the other parts. +We therefore provide the function [`replace_observed`](@ref) to change the `observed` part +of a model, without necessarily reconstructing the other parts. + +For `SemLoss` terms, `replace_observed()` constructs the new loss by passing the new observed +data, the current implied state, and the current loss (as `refloss`) to the appropriate loss +constructor. The new loss term therefore shares the implied state with the original one, as well +as loss-specific settings and, potentially, the internal state. For the [A first model](@ref), you would use it as diff --git a/src/loss/abstract.jl b/src/loss/abstract.jl index 2b4940ada..56a3af585 100644 --- a/src/loss/abstract.jl +++ b/src/loss/abstract.jl @@ -55,14 +55,18 @@ end function replace_observed(loss::SemLoss, new_observed::SemObserved; kwargs...) check_observed_vars(loss, new_observed) - # the default replace_observed() does not pass through kwargs to the ctor - return typeof(loss).name.wrapper(new_observed, SEM.implied(loss)) + # construct the new loss: + # 1) replace the observed + # 2) share the implied and its internal state with the original loss + # 3) replicate the current loss configuration/share its internal state + loss_ctor = typeof(loss).name.wrapper # get the loss constructor + return loss_ctor(new_observed, SEM.implied(loss), loss) end function replace_observed(loss::SemLoss, data::Union{AbstractMatrix, DataFrame}; kwargs...) old_obs = SEM.observed(loss) - new_observed = - typeof(old_obs).name.wrapper(data = data, observed_vars = observed_vars(old_obs)) + obs_ctor = typeof(old_obs).name.wrapper + new_observed = obs_ctor(data = data, observed_vars = observed_vars(old_obs)) return replace_observed(loss, new_observed; kwargs...) end From c209f8fcc37f6b39f0c6451381e209a4a69971f4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 3 May 2026 20:36:46 -0700 Subject: [PATCH 343/364] boostrap!(): deepcopy the sem Co-authored-by: Copilot --- src/frontend/fit/standard_errors/bootstrap.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index 0a1f39b4d..a8c1d3932 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -18,8 +18,9 @@ function bootstrap!( # fit to bootstrap samples if !parallel + bs_sem = deepcopy(sem) # avoid mutating the original model for i in 1:n_boot - new_fit = _fit_bootstrap_sample(sem, data, start; engine, fit_kwargs) + new_fit = _fit_bootstrap_sample(bs_sem, data, start; engine, fit_kwargs) update!(acc, i, new_fit, nothing) end else From 4f3ab7b1384aaee2fb750f15e4d956e17213ff59 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 3 May 2026 20:37:40 -0700 Subject: [PATCH 344/364] unit_tests/model: more config-preserving tests Co-authored-by: Copilot --- test/unit_tests/model.jl | 74 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index c80a0c1b9..b62f056d9 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -58,7 +58,7 @@ end observed = obs, implied = impliedtype, loss = losstype, - vech = losstype <: SemWLS && impliedtype <: RAMSymbolic + vech = losstype <: SemWLS && impliedtype <: RAMSymbolic, ) @test model isa Sem @@ -77,13 +77,20 @@ end @test @inferred(nsamples(model)) == nsamples(obs) end -@testset "replace_observed() preserves WLS state through finite-diff wrappers" begin +@testset "replace_observed() preserves WLS and approx_hessian=$(approx_hessian) state through finite-diff wrappers" for approx_hessian in + ( + false, + true, +) + expected_hessianeval = approx_hessian ? SEM.ApproxHessian : SEM.ExactHessian + model = Sem( specification = ram_matrices, observed = obs, implied = RAMSymbolic, loss = SemWLS, - vech = true + vech = true, + approximate_hessian = approx_hessian, ) wls_loss = sem_term(model) findiff_model = Sem(SEM.FiniteDiffWrapper(wls_loss)) @@ -104,11 +111,72 @@ end @test loss_newstate isa SemWLS @test loss_orig !== loss_oldstate @test loss_orig !== loss_newstate + @test SEM.HessianEval(loss_orig) === expected_hessianeval + @test SEM.HessianEval(loss_oldstate) === expected_hessianeval + @test SEM.HessianEval(loss_newstate) === expected_hessianeval @test loss_oldstate.V === loss_orig.V @test loss_newstate.V !== loss_orig.V @test observed_vars(loss_oldstate) == observed_vars(loss_orig) end +@testset "replace_observed() shares implied unless model is deepcopied and approx_hessian=$(approx_hessian)" for approx_hessian in + ( + false, + true, +) + expected_hessianeval = approx_hessian ? SEM.ApproxHessian : SEM.ExactHessian + + model = Sem( + specification = ram_matrices, + observed = obs, + implied = RAMSymbolic, + loss = SemML, + approximate_hessian = approx_hessian, + ) + + data_new = randn(nsamples(obs), nobserved_vars(obs)) + + model_new = replace_observed(model, data_new) + model_deepcopy = replace_observed(deepcopy(model), data_new) + + loss_orig = sem_term(model) + loss_new = sem_term(model_new) + loss_deepcopy = sem_term(model_deepcopy) + + @test SEM.HessianEval(loss_orig) === expected_hessianeval + @test SEM.HessianEval(loss_new) === expected_hessianeval + @test SEM.HessianEval(loss_deepcopy) === expected_hessianeval + @test implied(loss_new) === implied(loss_orig) + @test implied(loss_deepcopy) !== implied(loss_orig) +end + +@testset "replace_observed() preserves Sem container defaults" begin + data_g1 = dat[1:40, :] + data_g2 = dat[41:end, :] + + sem_multigroup = Sem( + :g1 => SemML( + SemObservedData(specification = ram_matrices, data = data_g1), + RAM(ram_matrices), + ), + :g2 => SemML( + SemObservedData(specification = ram_matrices, data = data_g2), + RAM(ram_matrices), + ); + default_sem_weights = :one, + ) + + sem_newobs = replace_observed( + sem_multigroup, + Dict(:g1 => randn(10, nobserved_vars(obs)), :g2 => randn(25, nobserved_vars(obs))), + ) + + @test all(isnothing, map(SEM.weight, SEM.loss_terms(sem_multigroup))) + @test all(isnothing, map(SEM.weight, SEM.loss_terms(sem_newobs))) + @test params(sem_newobs) == params(sem_multigroup) + @test params(sem_newobs) !== params(sem_multigroup) +end + @testset "Sem(...; semterm_column=...) splits ensemble data by group" begin dat_grouped = copy(dat[:, [:x1, :x2]]) n_g1 = size(dat_grouped, 1) ÷ 2 From 05f00f11ef1e0f8f239a09bc134ade680a379721 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 3 May 2026 20:39:35 -0700 Subject: [PATCH 345/364] tests: replace_observed(UserSemML) --- test/examples/multigroup/build_models.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index b538a4f46..47bdea224 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -178,6 +178,16 @@ end ) end +@testset "replace_observed_user_defined_loss" begin + wrapped_loss = SEM.FiniteDiffWrapper(UserSemML(obs_g2, RAMSymbolic(specification_g2))) + new_data = randn(nsamples(obs_g2), nobserved_vars(obs_g2)) + replaced_loss = SEM._unwrap(replace_observed(wrapped_loss, new_data)) + + @test replaced_loss isa UserSemML + @test observed_vars(replaced_loss) == observed_vars(obs_g2) + @test implied(replaced_loss) === implied(SEM._unwrap(wrapped_loss)) +end + ############################################################################################ # GLS estimation ############################################################################################ From cd44e6a20afc82bca630c42988aa918d63de65a2 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 3 May 2026 20:56:00 -0700 Subject: [PATCH 346/364] show(ParTable): fix formatting --- src/frontend/specification/ParameterTable.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index ecff7c4ec..c9b9dc24f 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -114,7 +114,10 @@ function Base.show(io::IO, partable::ParameterTable) pretty_table( io, as_matrix, - column_labels = [shown_columns, [eltype(partable.columns[col]) for col in shown_columns]], + column_labels = [ + shown_columns, + [eltype(partable.columns[col]) for col in shown_columns], + ], table_format = TextTableFormat(borders = text_table_borders__compact), # TODO switch to `missing` as non-specified values and suppress printing of `missing` instead formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v], From a677c1633187b3a791b59090934e70cb04f2fd65 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 11 May 2026 14:10:19 +0200 Subject: [PATCH 347/364] small bug fix --- src/frontend/fit/SemFit.jl | 4 ++-- src/frontend/specification/Sem.jl | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/frontend/fit/SemFit.jl b/src/frontend/fit/SemFit.jl index 1d2e82a60..04de8badc 100644 --- a/src/frontend/fit/SemFit.jl +++ b/src/frontend/fit/SemFit.jl @@ -30,12 +30,12 @@ end # pretty printing ############################################################################################ -function Base.show(io::IO, semfit::SemFit) +function Base.show(io::IO, m::MIME"text/plain", semfit::SemFit) print(io, "Fitted Structural Equation Model \n") print(io, "=============================================== \n") print(io, "--------------------- Model ------------------- \n") print(io, "\n") - print(io, semfit.model) + show(io, m, semfit.model) print(io, "\n") #print(io, "Objective value: $(round(semfit.minimum, digits = 4)) \n") print(io, "------------- Optimization result ------------- \n") diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 529819902..f53d50715 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -44,6 +44,7 @@ function Base.show(io::IO, term::LossTerm) else print(io, " w=1") end + println(io) else print(io, nameof(losstype(term))) print(io, "\n") @@ -55,7 +56,7 @@ function Base.show(io::IO, term::LossTerm) print(io, " - implied: $(nameof(typeof(implied(loss(term))))) \n") end if !isnothing(weight(term)) - print(io, " - weight: $(round(weight(term), digits=3))") + print(io, " - weight: $(round(weight(term), digits=3)) \n") end end end @@ -435,8 +436,8 @@ end # construct loss terms for the given observed and implied function build_sem_terms(loss, observed, implied; kwargs...) function build_SemLoss(aloss, observed, implied) - if loss isa AbstractLoss - return loss + if aloss isa AbstractLoss + return aloss elseif aloss <: SemLoss{O, I} where {O, I} return aloss(observed, implied; kwargs...) else @@ -634,7 +635,6 @@ function Base.show(io::IO, sem::AbstractSem) for term in loss_terms(sem) print(io, " - ") print(io, term) - println(io) end end @@ -652,6 +652,5 @@ function Base.show(io::IO, ::MIME"text/plain", sem::AbstractSem) for term in loss_terms(sem) print(io, " > ") print(io, term) - println(io) end end From d11bbdb3ce48adef21840d9bef9f3ce010618df7 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 16 Jun 2026 12:59:34 +0200 Subject: [PATCH 348/364] update docstrings for API changes --- src/frontend/finite_diff.jl | 13 +++++++++++++ src/loss/ML/FIML.jl | 2 +- src/observed/EM.jl | 4 ++-- src/types.jl | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/frontend/finite_diff.jl b/src/frontend/finite_diff.jl index 0ecd4865a..ac8fbf627 100644 --- a/src/frontend/finite_diff.jl +++ b/src/frontend/finite_diff.jl @@ -24,6 +24,19 @@ replace_observed( kwargs..., ) = SemLossFiniteDiff(replace_observed(_unwrap(wrapper), data; kwargs...)) +""" + FiniteDiffWrapper(model::AbstractSem) + FiniteDiffWrapper(loss::AbstractLoss) + +Wrap a SEM `model` or an individual `loss` term so that its gradient and Hessian +are approximated with finite differences of the objective (using the *FiniteDiff.jl* +package) instead of dedicated analytic evaluation. + +Wrapping the whole `model` returns a [`SemFiniteDiff`](@ref). Wrapping a single loss +term returns a loss wrapper that only uses the objective of the original term to +compute its gradient/Hessian, which is useful in [Collections](@ref) where analytic +gradients are available for some terms but not for others. +""" FiniteDiffWrapper(model::AbstractSem) = SemFiniteDiff(model) FiniteDiffWrapper(loss::AbstractLoss) = LossFiniteDiff(loss) FiniteDiffWrapper(loss::SemLoss) = SemLossFiniteDiff(loss) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 74d5edfb4..1f02ade18 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -68,7 +68,7 @@ function gradient!(JΣ, Jμ, patloss::SemFIMLPattern, pat::SemObservedMissingPat end """ - SemFIML{T, W} <: SemLossFunction + SemFIML{O, I, T, W} <: SemLoss{O, I} Full information maximum likelihood (FIML) estimation. Can handle observed data with missing values. diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 88af6112a..b495c8861 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -39,8 +39,8 @@ Estimate the covariance and the mean for data with missing values using the expectation maximization (EM) algorithm. # Arguments -- `patterns`: the observed data with missing values, grouped by missingness pattern (see [` - SemObservedMissingPattern`](@ref)) +- `patterns`: the observed data with missing values, grouped by missingness pattern + (each pattern is a `SemObservedMissingPattern`) - `max_iter_em`: the maximum number of EM iterations - `rtol_em`: the relative tolerance for convergence of the EM algorithm - `max_nsamples_em`: the maximum number of samples to use for each pattern in each EM iteration, diff --git a/src/types.jl b/src/types.jl index eb251a3b2..7742d3b76 100644 --- a/src/types.jl +++ b/src/types.jl @@ -16,7 +16,7 @@ MeanStruct(::Type{T}) where {T} = MeanStruct(semobj) = MeanStruct(typeof(semobj)) -"Hessian Evaluation trait for `SemImplied` and `SemLossFunction` subtypes" +"Hessian Evaluation trait for `SemImplied` and `AbstractLoss` subtypes" abstract type HessianEval end struct ApproxHessian <: HessianEval end struct ExactHessian <: HessianEval end From da85b5c635fc60ba40d8a5e8335c6c5c6cfe9d5b Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 16 Jun 2026 13:11:58 +0200 Subject: [PATCH 349/364] update docs for API changes --- docs/src/developer/loss.md | 39 ++++++++++++------- docs/src/internals/types.md | 2 +- docs/src/tutorials/collection/collection.md | 8 ++-- docs/src/tutorials/concept.md | 7 ++-- .../tutorials/construction/build_by_parts.md | 8 ++-- docs/src/tutorials/inspection/inspection.md | 1 + docs/src/tutorials/meanstructure.md | 6 +-- 7 files changed, 41 insertions(+), 30 deletions(-) diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index 8cdf2150f..a6b68663b 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -71,7 +71,7 @@ partable = ParameterTable( ) parameter_indices = getindex.([param_indices(partable)], [:a, :b, :c]) -myridge = Ridge(0.01, parameter_indices) +myridge = MyRidge(0.01, parameter_indices) model = SemFiniteDiff( specification = partable, @@ -82,16 +82,16 @@ model = SemFiniteDiff( model_fit = fit(model) ``` -This is one way of specifying the model - we now have **one model** with **multiple loss functions**. Because we did not provide a gradient for `Ridge`, we have to specify a `SemFiniteDiff` model that computes numerical gradients with finite difference approximation. +This is one way of specifying the model - we now have **one model** with **multiple loss functions**. Because we did not provide a gradient for `MyRidge`, we have to specify a `SemFiniteDiff` model that computes numerical gradients with finite difference approximation. -Note that the last argument to the `objective!` method is the whole model. Therefore, we can access everything that is stored inside our model everytime we compute the objective value for our loss function. Since ridge regularization is a very easy case, we do not need to do this. But maximum likelihood estimation for example depends on both the observed and the model implied covariance matrix. See [Second example - maximum likelihood](@ref) for information on how to do that. +Ridge regularization only depends on the parameters, so the `evaluate!` method above does not need anything else. Other loss functions, however, depend on the observed data and on what the model implies about it. Loss functions that compare the implied and the observed structure are subtypes of [`SemLoss`](@ref) and store their own `observed` and `implied` parts, which can be accessed inside `evaluate!` via `observed(loss)` and `implied(loss)`. See [Second example - maximum likelihood](@ref) for information on how to do that. ### Improve performance By far the biggest improvements in performance will result from specifying analytical gradients. We can do this for our example: ```@example loss -function evaluate!(objective, gradient, hessian::Nothing, ridge::Ridge, model::AbstractSem, par) +function evaluate!(objective, gradient, hessian::Nothing, ridge::MyRidge, par) # compute gradient if !isnothing(gradient) fill!(gradient, 0) @@ -165,8 +165,8 @@ end If you want to provide a way to query information about loss functions of your type, you can provide functions for that: ```julia -hyperparameter(ridge::Ridge) = ridge.α -regularization_indices(ridge::Ridge) = ridge.I +hyperparameter(ridge::MyRidge) = ridge.α +regularization_indices(ridge::MyRidge) = ridge.I ``` # Second example - maximum likelihood @@ -178,7 +178,9 @@ To keep it simple, we only cover models without a meanstructure. The maximum lik F_{ML} = \log \det \Sigma_i + \mathrm{tr}\left(\Sigma_{i}^{-1} \Sigma_o \right) ``` -where ``\Sigma_i`` is the model implied covariance matrix and ``\Sigma_o`` is the observed covariance matrix. We can query the model implied covariance matrix from the `implied` par of our model, and the observed covariance matrix from the `observed` path of our model. +where ``\Sigma_i`` is the model implied covariance matrix and ``\Sigma_o`` is the observed covariance matrix. We can query the model implied covariance matrix from the `implied` part of our loss term, and the observed covariance matrix from the `observed` part of our loss term. + +Since this loss function compares the implied and the observed structure, it is a subtype of [`SemLoss`](@ref) rather than a plain `AbstractLoss`. Every `SemLoss` stores its own `observed` and `implied` parts, which can be accessed inside `evaluate!` via `observed(loss)` and `implied(loss)`. To get information on what we can access from a certain `implied` or `observed` type, we can check it`s documentation an the pages [API - model parts](@ref) or via the help mode of the REPL: @@ -190,20 +192,27 @@ help?> RAM help?> SemObservedData ``` -We see that the model implied covariance matrix can be assessed as `Σ(implied)` and the observed covariance matrix as `obs_cov(observed)`. +We see that the model implied covariance matrix can be assessed as `implied(loss).Σ` and the observed covariance matrix as `obs_cov(observed(loss))`. -With this information, we write can implement maximum likelihood optimization as +A `SemLoss` subtype stores its `observed` and `implied` parts in the first two fields, and provides a constructor with the positional arguments `(observed, implied, refloss = nothing; kwargs...)` (see the [Convenient](@ref) section above). This constructor is used by the [`Sem`](@ref) constructor to build the loss term. With this information, we can implement maximum likelihood optimization as ```@example loss -struct MaximumLikelihood <: SemLossFunction end +struct MaximumLikelihood{O <: SemObserved, I <: SemImplied} <: SemLoss{O, I} + observed::O + implied::I +end + +# constructor used by the `Sem` constructor to build the loss term +MaximumLikelihood(observed::SemObserved, implied::SemImplied, refloss = nothing; kwargs...) = + MaximumLikelihood{typeof(observed), typeof(implied)}(observed, implied) using LinearAlgebra -import StructuralEquationModels: obs_cov, evaluate! +import StructuralEquationModels: evaluate! -function evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, semml::MaximumLikelihood, model::AbstractSem, par) +function evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, semml::MaximumLikelihood, par) # access the model implied and observed covariance matrices - Σᵢ = implied(model).Σ - Σₒ = obs_cov(observed(model)) + Σᵢ = implied(semml).Σ + Σₒ = obs_cov(observed(semml)) # compute the objective if isposdef(Symmetric(Σᵢ)) # is the model implied covariance matrix positive definite? return logdet(Σᵢ) + tr(inv(Σᵢ)*Σₒ) @@ -221,7 +230,7 @@ Let's specify and fit a model: model_ml = SemFiniteDiff( specification = partable, data = example_data("political_democracy"), - loss = MaximumLikelihood() + loss = MaximumLikelihood ) model_fit = fit(model_ml) diff --git a/docs/src/internals/types.md b/docs/src/internals/types.md index 4b4cd4faa..3bccb1f8f 100644 --- a/docs/src/internals/types.md +++ b/docs/src/internals/types.md @@ -11,7 +11,7 @@ The type hierarchy is implemented in `"src/types.jl"`. [`AbstractSem`](@ref) is the base abstract type for all SEM models. It has two concrete subtypes: - `Sem{L <: Tuple} <: AbstractSem`: the main SEM model type that implements a list of weighted -loss terms (using [`LossTerm`](@ref) wrapper around `AbstractLoss`) and allows modeling both single +loss terms (using [`LossTerm`](@ref StructuralEquationModels.LossTerm) wrapper around `AbstractLoss`) and allows modeling both single and multi-group SEMs and combining them with regularization terms. - `SemFiniteDiff{S <: AbstractSem} <: AbstractSem`: a wrapper around any `AbstractSem` that substitutes dedicated gradient/hessian evaluation with finite difference approximation. diff --git a/docs/src/tutorials/collection/collection.md b/docs/src/tutorials/collection/collection.md index 2a8ea92c2..407879e8e 100644 --- a/docs/src/tutorials/collection/collection.md +++ b/docs/src/tutorials/collection/collection.md @@ -3,7 +3,7 @@ With *StructuralEquationModels.jl*, you can fit weighted sums of structural equation models. The most common use case for this are [Multigroup models](@ref). Another use case may be optimizing the sum of loss functions for some of which you do know the analytic gradient, but not for others. -In this case, [`FiniteDiffWrapper`](@ref) can generate a wrapper around the specific `SemLoss` term. The wrapper loss term will +In this case, [`FiniteDiffWrapper`](@ref StructuralEquationModels.FiniteDiffWrapper) can generate a wrapper around the specific `SemLoss` term. The wrapper loss term will only use the objective of the original term to calculate its gradient using finite difference approximation. ```julia @@ -47,6 +47,8 @@ model2_weighted = Sem(:main => loss_1 => 0.5, :alt => loss_2 => 1.0) ```@docs Sem -LossTerm -FiniteDiffWrapper +SemFiniteDiff +AbstractSem +StructuralEquationModels.LossTerm +StructuralEquationModels.FiniteDiffWrapper ``` \ No newline at end of file diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index 49f0d404f..f05fca2f6 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -21,7 +21,7 @@ So everything that can be used as the 'observed' part has to be of type `SemObse Here is an overview on the available building blocks: -|[`SemObserved`](@ref) | [`SemImplied`](@ref) | [`SemLossFunction`](@ref) | [`SemOptimizer`](@ref) | +|[`SemObserved`](@ref) | [`SemImplied`](@ref) | [`AbstractLoss`](@ref) | [`SemOptimizer`](@ref) | |---------------------------------|-----------------------|---------------------------|----------------------------| | [`SemObservedData`](@ref) | [`RAM`](@ref) | [`SemML`](@ref) | [:Optim](@ref StructuralEquationModels.SemOptimizerOptim) | | [`SemObservedCovariance`](@ref) | [`RAMSymbolic`](@ref) | [`SemWLS`](@ref) | [:NLopt](@ref SEMNLOptExt.SemOptimizerNLopt) | @@ -42,7 +42,7 @@ There are two options at the moment: [`RAM`](@ref), which uses the reticular act ## The loss part aka `SemLoss` The loss part specifies the objective that is optimized to find the parameter estimates. -If it contains more then one loss function (aka [`SemLossFunction`](@ref))), we find the parameters by minimizing the sum of loss functions (for example in maximum likelihood estimation + ridge regularization). +If it contains more then one loss function (aka [`AbstractLoss`](@ref)), we find the parameters by minimizing the sum of loss functions (for example in maximum likelihood estimation + ridge regularization). Available loss functions are - [`SemML`](@ref): maximum likelihood estimation - [`SemWLS`](@ref): weighted least squares estimation @@ -75,6 +75,7 @@ SemObservedMissing samples observed_vars SemSpecification +em_mvn ``` ## implied @@ -89,8 +90,8 @@ ImpliedEmpty ## loss functions ```@docs +AbstractLoss SemLoss -SemLossFunction SemML SemFIML SemWLS diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 52e12f30b..afa51eeb0 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -51,19 +51,17 @@ Now, we construct the different parts: observed = SemObservedData(specification = partable, data = data) # implied ------------------------------------------------------------------------------ -implied_ram = RAM(specification = partable) +implied_ram = RAM(partable) # loss --------------------------------------------------------------------------------- -ml = SemML(observed = observed) - -loss_ml = SemLoss(ml) +ml = SemML(observed, implied_ram) # optimizer ---------------------------------------------------------------------------- optimizer = SemOptimizer() # model -------------------------------------------------------------------------------- -model_ml = Sem(observed, implied_ram, loss_ml) +model_ml = Sem(ml) fit(optimizer, model_ml) ``` \ No newline at end of file diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md index ff572eb58..a4d8adb1b 100644 --- a/docs/src/tutorials/inspection/inspection.md +++ b/docs/src/tutorials/inspection/inspection.md @@ -135,4 +135,5 @@ dof minus2ll p_value RMSEA +CFI ``` diff --git a/docs/src/tutorials/meanstructure.md b/docs/src/tutorials/meanstructure.md index 4e6d2a36a..1d8539355 100644 --- a/docs/src/tutorials/meanstructure.md +++ b/docs/src/tutorials/meanstructure.md @@ -106,11 +106,11 @@ For our example, ```@example meanstructure observed = SemObservedData(specification = partable, data = data, meanstructure = true) -implied_ram = RAM(specification = partable, meanstructure = true) +implied_ram = RAM(partable, meanstructure = true) -ml = SemML(observed = observed, meanstructure = true) +ml = SemML(observed, implied_ram) -model = Sem(observed, implied_ram, SemLoss(ml)) +model = Sem(ml) fit(model) ``` \ No newline at end of file From c54f3ae9ac5182540346cb19a170a7c77e1d72d3 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Wed, 17 Jun 2026 10:58:38 +0200 Subject: [PATCH 350/364] update SEM concept figures --- docs/src/assets/concept.svg | 155 +++++++++++++++++++++++++++++- docs/src/assets/concept_typed.svg | 153 ++++++++++++++++++++++++++++- docs/src/tutorials/concept.md | 2 +- 3 files changed, 307 insertions(+), 3 deletions(-) diff --git a/docs/src/assets/concept.svg b/docs/src/assets/concept.svg index f82e47943..fe6fd2081 100644 --- a/docs/src/assets/concept.svg +++ b/docs/src/assets/concept.svg @@ -1 +1,154 @@ - \ No newline at end of file + + + + + + + + Sem + + + lossfunction#1 + + observed + + implied + + + lossfunction#2 + + observed + + implied + + ... + + optimizer + + diff --git a/docs/src/assets/concept_typed.svg b/docs/src/assets/concept_typed.svg index e046819ae..4592fd87c 100644 --- a/docs/src/assets/concept_typed.svg +++ b/docs/src/assets/concept_typed.svg @@ -1 +1,152 @@ - \ No newline at end of file + + + + + + + + Sem + + + SemLoss + + SemObserved + + SemImplied + + + SemLoss + + SemObserved + + SemImplied + + AbstractLoss + + SemOptimizer + + diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index f05fca2f6..6c0d51a2f 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -13,7 +13,7 @@ For example, to build a model for maximum likelihood estimation with the NLopt o As you can see, a model can have as many loss functions as you want it to have. We always optimize over their (weighted) sum. So to build a model for ridge regularized full information maximum likelihood estimation, you would choose two loss functions, `SemFIML` and `SemRidge`. -In julia, everything has a type. To make more precise which objects can be used as the different building blocks, we require them to have a certain type: +To specify which objects can be used as the different building blocks, we require them to have a certain type: ![SEM concept typed](../assets/concept_typed.svg) From a333096d028d46310e88d82e4088e39f02f0ac14 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Wed, 17 Jun 2026 13:06:49 +0200 Subject: [PATCH 351/364] update dependency versions --- Project.toml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Project.toml b/Project.toml index eabc5b36f..8ca95785f 100644 --- a/Project.toml +++ b/Project.toml @@ -27,16 +27,15 @@ Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b" [compat] -julia = "1.9, 1.10, 1.11" +julia = "1.10, 1.11, 1.12" StenoGraphs = "0.5" DataFrames = "1" Distributions = "0.25" FiniteDiff = "2" -InteractiveUtils = "1.11.0" LineSearches = "7" -NLSolversBase = "7" +NLSolversBase = "8" NLopt = "0.6, 1" -Optim = "1" +Optim = "2" PrettyTables = "3" ProximalAlgorithms = "0.7" StatsBase = "0.33, 0.34" From 2e7fd30389a41f49914679830984b7b9c5b66de3 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Wed, 17 Jun 2026 16:17:53 +0200 Subject: [PATCH 352/364] update optim syntax --- src/optimizer/optim.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index a0aae22ab..e81017ed3 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -113,7 +113,7 @@ function fit( ) start_params = clamp.(start_params, lbounds, ubounds) result = Optim.optimize( - Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), + NLSolversBase.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), lbounds, ubounds, start_params, @@ -122,7 +122,7 @@ function fit( ) else result = Optim.optimize( - Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), + NLSolversBase.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), start_params, optim.algorithm, optim.options, From 4c545d4ac5926bb5545d020935545caca9f7892c Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 18 Jun 2026 11:06:45 +0200 Subject: [PATCH 353/364] add compat entries --- Project.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Project.toml b/Project.toml index 8ca95785f..14e394cd3 100644 --- a/Project.toml +++ b/Project.toml @@ -42,6 +42,8 @@ StatsBase = "0.33, 0.34" Symbolics = "4, 5, 6, 7" SymbolicUtils = "1.4 - 1.5, 1.7, 2, 3, 4" StatsAPI = "1" +DelimitedFiles = "1" +Statistics = "1" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From 5b6b10f7431569f35c7e2377f1acbe2e2d4e5362 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 18 Jun 2026 18:51:06 +0200 Subject: [PATCH 354/364] fix ml mg weights and chi2 weights to match lavaan --- src/frontend/fit/fitmeasures/chi2.jl | 7 +++++-- src/frontend/specification/Sem.jl | 19 +++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index c56b9a2a0..444d9678b 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -21,7 +21,9 @@ function χ²(fit::SemFit, model::AbstractSem) end # bollen, p. 115, only correct for GLS weight matrix -χ²(::Type{<:SemWLS}, fit::SemFit, model::AbstractSem) = (nsamples(model) - 1) * fit.minimum +# N - G (= Σ(Nᵍ-1)) multiplier; reduces to N-1 for a single group +χ²(::Type{<:SemWLS}, fit::SemFit, model::AbstractSem) = + (nsamples(model) - nsem_terms(model)) * fit.minimum function χ²(::Type{<:SemML}, fit::SemFit, model::AbstractSem) G = sum(loss_terms(model)) do term @@ -32,7 +34,8 @@ function χ²(::Type{<:SemML}, fit::SemFit, model::AbstractSem) return 0.0 end end - return (nsamples(model) - 1) * (fit.minimum - G) + # N - G (= Σ(Nᵍ-1)) multiplier; reduces to N-1 for a single group + return (nsamples(model) - nsem_terms(model)) * (fit.minimum - G) end function χ²(::Type{<:SemFIML}, fit::SemFit, model::AbstractSem) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index f53d50715..734b63c02 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -67,7 +67,9 @@ end multigroup_correction_scale(::Type{<:SemLoss}) = nothing multigroup_correction_scale(::Type{<:SemFIML}) = 0 -multigroup_correction_scale(::Type{<:SemML}) = 0 +# ML (like WLS) uses the Wishart convention: per-group (Nᵍ-1) weighting and N-G in the +# χ²/RMSEA. +multigroup_correction_scale(::Type{<:SemML}) = -1 multigroup_correction_scale(::Type{<:SemWLS}) = -1 multigroup_correction_scale(loss::SemLoss) = multigroup_correction_scale(typeof(loss)) @@ -104,9 +106,9 @@ end function Sem( loss_terms...; params::Union{Vector{Symbol}, Nothing} = nothing, - default_sem_weights = :nsamples, + default_sem_weights = :nsamples_corrected, ) - default_sem_weights ∈ [:nsamples, :uniform, :one] || + default_sem_weights ∈ [:nsamples_corrected, :nsamples, :uniform, :one] || throw(ArgumentError("Unsupported default_sem_weights=:$default_sem_weights")) # assemble a list of weighted losses and check params equality terms = Vector{LossTerm}() @@ -169,7 +171,16 @@ function Sem( if !has_sem_weights && nsems > 1 # set the weights of SEMs in the ensemble - if default_sem_weights == :nsamples + if default_sem_weights == :nsamples_corrected + # weight SEM terms by the number of samples, applying a loss-type-specific + # correction (see multigroup_correction_scale); consistent with the + # multigroup RMSEA sample-size correction + sem_idxs = [i for (i, term) in enumerate(terms) if issemloss(term)] + sem_weights = multigroup_weights(terms[sem_idxs]...) + for (k, i) in enumerate(sem_idxs) + terms[i] = LossTerm(loss(terms[i]), id(terms[i]), sem_weights[k]) + end + elseif default_sem_weights == :nsamples # weight SEM by the number of samples nsamples_total = sum(nsamples(term) for term in terms if issemloss(term)) for (i, term) in enumerate(terms) From cd017255b2ade28c346ab35c03f631f98761e703 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 18 Jun 2026 18:51:50 +0200 Subject: [PATCH 355/364] further doc updates --- docs/make.jl | 5 -- docs/src/developer/extending.md | 6 +- docs/src/developer/implied.md | 15 ++-- docs/src/developer/loss.md | 48 +++++++----- docs/src/internals/files.md | 27 ------- docs/src/internals/internals.md | 3 - docs/src/internals/types.md | 17 ----- docs/src/performance/parametric.md | 4 +- docs/src/tutorials/collection/collection.md | 16 ++-- docs/src/tutorials/collection/multigroup.md | 5 +- docs/src/tutorials/concept.md | 75 ++++++++++++++----- docs/src/tutorials/constraints/constraints.md | 2 +- .../construction/outer_constructor.md | 14 ++-- docs/src/tutorials/fitting/fitting.md | 7 +- 14 files changed, 123 insertions(+), 121 deletions(-) delete mode 100644 docs/src/internals/files.md delete mode 100644 docs/src/internals/internals.md delete mode 100644 docs/src/internals/types.md diff --git a/docs/make.jl b/docs/make.jl index f3824dd79..2990f8dbb 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -52,11 +52,6 @@ makedocs( "Starting values" => "performance/starting_values.md", "Parametric Types" => "performance/parametric.md", ], - "Internals and design" => [ - "Internals and design" => "internals/internals.md", - "files" => "internals/files.md", - "types" => "internals/types.md", - ], ], format = Documenter.HTML( prettyurls = get(ENV, "CI", nothing) == "true", diff --git a/docs/src/developer/extending.md b/docs/src/developer/extending.md index 5c3183da4..20f558def 100644 --- a/docs/src/developer/extending.md +++ b/docs/src/developer/extending.md @@ -1,10 +1,12 @@ # Extending the package -As discussed in the section on [Model Construction](@ref), every Structural Equation Model (`Sem`) consists of three (four with the optimizer) parts: +As discussed in [Our Concept of a Structural Equation Model](@ref), a `Sem` is built from one or more loss +terms, and each SEM loss function bundles an *observed* and an *implied* part: ![SEM concept typed](../assets/concept_typed.svg) -On the following pages, we will explain how you can define your own custom parts and "plug them in". There are certain things you **have to do** to define custom parts and some things you **can do** to have a more pleasent experience. In general, these requirements fall into the categories +On the following pages, we will explain how you can define your own custom parts (a loss function, an observed +type, an implied type, or an optimizer) and "plug them in". There are certain things you **have to do** to define custom parts and some things you **can do** to have a more pleasent experience. In general, these requirements fall into the categories - minimal (to use your custom part and fit a `Sem` with it) - use the outer constructor to build a model in a more convenient way - use additional functionality like standard errors, fit measures, etc. \ No newline at end of file diff --git a/docs/src/developer/implied.md b/docs/src/developer/implied.md index 6321decbc..f695e3e82 100644 --- a/docs/src/developer/implied.md +++ b/docs/src/developer/implied.md @@ -43,8 +43,7 @@ We implement an `ImpliedEmpty` type in our package that does nothing but serving ### Types ############################################################################################ """ -Empty placeholder for models that don't need an implied part. -(For example, models that only regularize parameters.) +Empty placeholder for loss functions that don't need an implied part. # Constructor @@ -55,7 +54,7 @@ Empty placeholder for models that don't need an implied part. # Examples A multigroup model with ridge regularization could be specified as a `Sem` with one -model per group and an additional model with `ImpliedEmpty` and `SemRidge` for the regularization part. +SEM term (`SemLoss`) per group and an additional `SemRidge` regularization term. # Extended help @@ -73,13 +72,13 @@ end ### Constructors ############################################################################################ -function ImpliedEmpty( - spec::SemSpecification; - hessianeval::HessianApprox = ExactHessian(), +function ImpliedEmpty(; + specification, + meanstruct = NoMeanStruct(), + hessianeval = ExactHessian(), kwargs..., ) - ram_matrices = convert(RAMMatrices, spec) - return ImpliedEmpty(hessianeval, MeanStruct(ram_matrices), ram_matrices) + return ImpliedEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) end ############################################################################################ diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index a6b68663b..e763364b2 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -136,28 +136,34 @@ Additionally, you may provide analytic hessians by writing a respective method f ## Convenient -To be able to build the loss term, it needs a constructor. -Every `SemLoss` subtype should provide a constructor with 3 positional arguments: - * `observed::SemObserved`: the observed part of the model - * `implied::SemImplied`: the implied part of the model - * `refloss::Union{MyLoss, Nothing} = nothing`: optional loss term of the same type - to use as a reference for any loss-specific configuration. - -Any additional loss configuration details should be passed as optional keyword arguments. -If both `refloss` and the keyword arguments are provided, the keyword arguments take -precedence. This constructor is used internally by the functions like [`replace_observed`](@ref) -to rebuild the loss term with new observed data while preserving the implied state. +In the minimal example above we built `myridge` ourselves and passed the ready-made instance to +the model via `loss = (SemML, myridge)`. Alternatively, you can let the outer [`Sem`](@ref) +constructor build the loss term for you: pass the loss *type* instead of an instance and provide +a keyword constructor. ```julia -function MyLoss( - observed::SemObserved, implied::SemImplied, refloss::Union{MyLoss, Nothing} = nothing; - kwarg1 = ..., kwarg2 = ..., kwargs... +MyRidge(; α_ridge, which_ridge, kwargs...) = MyRidge(α_ridge, which_ridge) +``` + +Any keyword arguments passed to `Sem(...)` are forwarded to this constructor (along with some that +the model supplies automatically, such as `nparams`), so the loss can be configured directly from +the model call: + +```julia +model = SemFiniteDiff( + specification = partable, + data = example_data("political_democracy"), + loss = (SemML, MyRidge), + α_ridge = 0.01, + which_ridge = parameter_indices, ) - ... - return MyLoss(...) # internal MyLoss constructor -end ``` +Note that, being a plain `AbstractLoss`, `MyRidge` neither stores nor receives an `observed` or +`implied` part — it depends only on the parameters. +SEM-specific loss functions are constructed differently; see +[Second example - maximum likelihood](@ref). + ## Additional functionality ### Access additional information @@ -194,7 +200,13 @@ help?> SemObservedData We see that the model implied covariance matrix can be assessed as `implied(loss).Σ` and the observed covariance matrix as `obs_cov(observed(loss))`. -A `SemLoss` subtype stores its `observed` and `implied` parts in the first two fields, and provides a constructor with the positional arguments `(observed, implied, refloss = nothing; kwargs...)` (see the [Convenient](@ref) section above). This constructor is used by the [`Sem`](@ref) constructor to build the loss term. With this information, we can implement maximum likelihood optimization as +Unlike a plain `AbstractLoss`, a `SemLoss` subtype stores its `observed` and `implied` parts (in its first two fields), and the [`Sem`](@ref) constructor builds it for you. To support this, every `SemLoss` subtype should provide a constructor with three positional arguments: + * `observed::SemObserved`: the observed part of the loss term + * `implied::SemImplied`: the implied part of the loss term + * `refloss::Union{MaximumLikelihood, Nothing} = nothing`: an optional existing loss term of the + same type, used as a reference for any loss-specific configuration. + +Any additional configuration is passed as optional keyword arguments; if both `refloss` and keyword arguments are given, the keyword arguments take precedence. This constructor is also used by [`replace_observed`](@ref) to rebuild the loss term with new observed data while sharing the implied state. With this, we can implement maximum likelihood optimization as ```@example loss struct MaximumLikelihood{O <: SemObserved, I <: SemImplied} <: SemLoss{O, I} diff --git a/docs/src/internals/files.md b/docs/src/internals/files.md deleted file mode 100644 index 4c2338393..000000000 --- a/docs/src/internals/files.md +++ /dev/null @@ -1,27 +0,0 @@ -# Files - -We briefly describe the file and folder structure of the package. - -## Source code - -Source code is in the `"src"` folder: - -`"src"` -- `"StructuralEquationModels.jl"` defines the module and the exported objects -- `"types.jl"` defines all abstract types and the basic type hierarchy -- `"objective_gradient_hessian.jl"` contains methods for computing objective, gradient and hessian values for different model types as well as generic fallback methods -- The folders `"observed"`, `"implied"`, and `"loss"` contain implementations of specific subtypes (for example, the `"loss"` folder contains a file `"ML.jl"` that implements the `SemML` loss function). -- `"optimizer"` contains connections to different optimization backends (aka methods for `fit`) - - `"optim.jl"`: connection to the `Optim.jl` package -- `"frontend"` contains user-facing functions - - `"specification"` contains functionality for model specification - - `"fit"` contains functionality for model assessment, like fit measures and standard errors -- `"additional_functions"` contains helper functions for simulations, loading artifacts (example data) and various other things - -Code for the package extentions can be found in the `"ext"` folder: -- `"SEMNLOptExt"` for connection to `NLopt.jl`. -- `"SEMProximalOptExt"` for connection to `ProximalAlgorithms.jl`. - -## Tests and Documentation - -Tests are in the `"test"` folder, documentation in the `"docs"` folder. \ No newline at end of file diff --git a/docs/src/internals/internals.md b/docs/src/internals/internals.md deleted file mode 100644 index 18f82dbca..000000000 --- a/docs/src/internals/internals.md +++ /dev/null @@ -1,3 +0,0 @@ -# Internals and Design - -On the following pages, we document some technical information about the package. Those informations are no prerequisite for extending the package (as decribed in the developer documentation)!, but they may be useful. \ No newline at end of file diff --git a/docs/src/internals/types.md b/docs/src/internals/types.md deleted file mode 100644 index 3bccb1f8f..000000000 --- a/docs/src/internals/types.md +++ /dev/null @@ -1,17 +0,0 @@ -# Type hierarchy - -The type hierarchy is implemented in `"src/types.jl"`. - -[`AbstractLoss`](@ref): is the base abstract type for all loss functions: -- `SemLoss{O <: SemObserved, I <: SemImplied}`: is the subtype of `AbstractLoss`, which is the - base for all SEM-specific loss functions ([`SemML`](@ref), [`SemWLS`](@ref) etc) that - evaluate how closely the implied covariation structure (represented by the object of type `I`) - matches the observed one (contained in the object of type `O`); -- regularizing terms (e.g. [`SemRidge`](@ref)) are implemented as subtypes of `AbstractLoss`. - -[`AbstractSem`](@ref) is the base abstract type for all SEM models. It has two concrete subtypes: -- `Sem{L <: Tuple} <: AbstractSem`: the main SEM model type that implements a list of weighted -loss terms (using [`LossTerm`](@ref StructuralEquationModels.LossTerm) wrapper around `AbstractLoss`) and allows modeling both single -and multi-group SEMs and combining them with regularization terms. -- `SemFiniteDiff{S <: AbstractSem} <: AbstractSem`: a wrapper around any `AbstractSem` that - substitutes dedicated gradient/hessian evaluation with finite difference approximation. diff --git a/docs/src/performance/parametric.md b/docs/src/performance/parametric.md index 9c6be382b..ce1781ab6 100644 --- a/docs/src/performance/parametric.md +++ b/docs/src/performance/parametric.md @@ -25,7 +25,7 @@ giving each field a type and adding them as parameters to our type declaration. Recall our example from [Custom loss functions](@ref): ```julia -struct Ridge <: SemLossFunction +struct Ridge <: AbstractLoss α I end @@ -34,7 +34,7 @@ end We could also declare it as a parametric type: ```julia -struct ParametricRidge{X, Y} <: SemLossFunction +struct ParametricRidge{X, Y} <: AbstractLoss α::X I::Y end diff --git a/docs/src/tutorials/collection/collection.md b/docs/src/tutorials/collection/collection.md index 407879e8e..423ac5bc7 100644 --- a/docs/src/tutorials/collection/collection.md +++ b/docs/src/tutorials/collection/collection.md @@ -26,16 +26,22 @@ It is also possible to use finite difference for the entire `Sem` model: model_findiff2 = FiniteDiffWrapper(model) ``` -The weighting scheme of the SEM loss terms is specified using `default_set_weights` argument of the `Sem` constructor. -The `:nsamples` scheme (the default) weights SEM terms by ``N_{term}/N_{total}``, i.e. each term is weighted by the number -of observations in its data (which matches the formula for multigroup models). -The weights for the loss terms (both SEM and regularization) can be explicitly specified the pair syntax `loss => weight`: +The weighting scheme of the SEM loss terms is specified using the `default_sem_weights` argument of the `Sem` constructor. +The available schemes are: +- `:nsamples_corrected` (the default): like `:nsamples`, but applies a loss-type-specific correction + to the sample counts (e.g. ``N_{term} - 1`` for maximum likelihood and weighted least squares). For FIML the correction is zero, so it coincides with `:nsamples`, +- `:nsamples`: weights each SEM term by ``N_{term}/N_{total}``, i.e. by the (uncorrected) number of + observations in its data, +- `:uniform`: weights each of the ``k`` SEM terms by ``1/k``, +- `:one`: leaves all SEM terms unweighted. + +The weights for the loss terms (both SEM and regularization) can also be explicitly specified using the pair syntax `loss => weight`: ```julia model_weighted = Sem(loss_1 => 0.5, loss_2 => 1.0) ``` -`Sem` support assigning unique identifier to each loss term, which is essential for complex multi-term model. +`Sem` supports assigning a unique identifier to each loss term, which is useful for complex multi-term models. The syntax is `id => loss`, or `id => loss => weight`: ```julia diff --git a/docs/src/tutorials/collection/multigroup.md b/docs/src/tutorials/collection/multigroup.md index 04f1893d7..ccd3bdd65 100644 --- a/docs/src/tutorials/collection/multigroup.md +++ b/docs/src/tutorials/collection/multigroup.md @@ -52,7 +52,7 @@ graph = @StenoGraph begin end ``` -You can pass multiple arguments to `fix()` and `label()` for each group. Parameters with the same label (within and across groups) are constrained to be equal. To fix a parameter in one group, but estimate it freely in the other, you may write `fix(NaN, 4.3)`. +You can pass multiple arguments to `fixed()` and `label()` for each group. Parameters with the same label (within and across groups) are constrained to be equal. To fix a parameter in one group, but estimate it freely in the other, you may write `fixed(NaN, 4.3)`. You can then use the resulting graph to specify an `EnsembleParameterTable` @@ -72,8 +72,7 @@ The parameter table can be used to create a multigroup `Sem` model: model_ml_multigroup = Sem( specification = partable, data = dat, - semterm_column = :school, - groups = groups) + semterm_column = :school) ``` !!! note "A different way to specify" diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index 6c0d51a2f..39f189d2e 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -1,23 +1,44 @@ # Our Concept of a Structural Equation Model -In our package, every Structural Equation Model (`Sem`) consists of three parts (four, if you count the optimizer): +In our package, a structural equation model (a [`Sem`](@ref)) is built from one or more **loss terms**. +Fitting the model means finding the parameters that minimize the (weighted) sum of all of its loss terms. +This simple idea is remarkably general: within the same structure it covers a single SEM fit by maximum +likelihood, a regularized SEM (e.g. maximum likelihood plus a ridge penalty), and multigroup models +(one SEM term per group). ![SEM concept](../assets/concept.svg) -Those parts are interchangable building blocks (like 'Legos'), i.e. there are different pieces available you can choose as the `observed` slot of the model, and stick them together with other pieces that can serve as the `implied` part. +A loss term is anything of type [`AbstractLoss`](@ref) — a function that maps the model parameters to a +number that should be minimized. There are two kinds of loss terms: -The `observed` part is for observed data, the `implied` part is what the model implies about your data (e.g. the model implied covariance matrix), and the loss part compares the observed data and implied properties (e.g. weighted least squares difference between the observed and implied covariance matrix). -The optimizer part is not part of the model itself, but it is needed to fit the model as it connects to the optimization backend (e.g. the type of optimization algorithm used). +- **SEM loss functions** ([`SemLoss`](@ref)), such as [`SemML`](@ref), [`SemWLS`](@ref) and [`SemFIML`](@ref), + measure how well the model explains the data. To do so, each `SemLoss` *bundles its own observed part + (the data) and implied part (what the model implies about the data)*. They are the heart of a SEM. +- **Other loss functions**, such as the regularization terms [`SemRidge`](@ref) and [`SemConstant`](@ref), + depend only on the parameters and therefore need neither an observed nor an implied part. -For example, to build a model for maximum likelihood estimation with the NLopt optimization suite as a backend you would choose `SemML` as a loss function and `SemOptimizerNLopt` as the optimizer. +Because a model is just a (weighted) sum of loss terms, you can freely combine them. +For example, ridge-regularized full information maximum likelihood estimation is a model with two loss terms, +a [`SemFIML`](@ref) term and a [`SemRidge`](@ref) term. A two-group model is a model with two [`SemML`](@ref) +terms, one per group, weighted by the respective sample sizes. -As you can see, a model can have as many loss functions as you want it to have. We always optimize over their (weighted) sum. So to build a model for ridge regularized full information maximum likelihood estimation, you would choose two loss functions, `SemFIML` and `SemRidge`. +All models are subtypes of [`AbstractSem`](@ref). The default [`Sem`](@ref) computes the weighted sum of its +loss terms together with their (analytic) gradients. [`SemFiniteDiff`](@ref) is an alternative that +approximates the gradient with finite differences, which is useful for loss functions that do not provide an +analytic gradient. -To specify which objects can be used as the different building blocks, we require them to have a certain type: +## The parts of a SEM loss + +Each SEM loss function ([`SemLoss`](@ref)) is itself composed of interchangeable building blocks (like 'Legos'): +an *observed* part and an *implied* part. To make precise which objects can play each role, we require them to +have a certain type: ![SEM concept typed](../assets/concept_typed.svg) -So everything that can be used as the 'observed' part has to be of type `SemObserved`. +So everything that can serve as the *observed* part has to be of type [`SemObserved`](@ref), everything that can +serve as the *implied* part has to be of type [`SemImplied`](@ref), and the loss function that combines them is a +[`SemLoss`](@ref). To fit the model, you additionally choose a [`SemOptimizer`](@ref); it connects to the +numerical optimization backend but is not itself part of the model. Here is an overview on the available building blocks: @@ -29,28 +50,44 @@ Here is an overview on the available building blocks: | | | [`SemRidge`](@ref) | | | | | [`SemConstant`](@ref) | | -The rest of this page explains the building blocks for each part. First, we explain every part and give an overview on the different options that are available. After that, the [API - model parts](@ref) section serves as a reference for detailed explanations about the different options. -(How to stick them together to a final model is explained in the section on [Model Construction](@ref).) +The rest of this page explains each building block and the available options. After that, the +[API - model parts](@ref) section serves as a reference for detailed explanations. +(How to stick the building blocks together into a final model is explained in the section on +[Model Construction](@ref).) ## The observed part aka [`SemObserved`](@ref) -The *observed* part contains all necessary information about the observed data. Currently, we have three options: [`SemObservedData`](@ref) for fully observed datasets, [`SemObservedCovariance`](@ref) for observed covariances (and means) and [`SemObservedMissing`](@ref) for data that contains missing values. +The *observed* part contains all necessary information about the observed data, and pre-computes the statistics +a loss function needs from it — for example the observed covariance matrix, or the different patterns of +missingness used for full information maximum likelihood (FIML) estimation. +Currently, we have three options: [`SemObservedData`](@ref) for fully observed datasets, +[`SemObservedCovariance`](@ref) for observed covariances (and means) and [`SemObservedMissing`](@ref) for data +that contains missing values. ## The implied part aka [`SemImplied`](@ref) -The *implied* part is what your model implies about the data, for example, the model-implied covariance matrix. -There are two options at the moment: [`RAM`](@ref), which uses the reticular action model to compute the model implied covariance matrix, and [`RAMSymbolic`](@ref) which does the same but symbolically pre-computes part of the model, which increases subsequent performance in model fitting (see [Symbolic precomputation](@ref)). There is also a third option, [`ImpliedEmpty`](@ref) that can serve as a 'placeholder' for models that do not need an implied part. - -## The loss part aka `SemLoss` -The loss part specifies the objective that is optimized to find the parameter estimates. -If it contains more then one loss function (aka [`AbstractLoss`](@ref)), we find the parameters by minimizing the sum of loss functions (for example in maximum likelihood estimation + ridge regularization). +The *implied* part defines how the model-implied statistics (for example, the model-implied covariance matrix +and mean vector) are computed from the parameters. +There are two options at the moment: [`RAM`](@ref), which uses the reticular action model to compute the model +implied covariance matrix, and [`RAMSymbolic`](@ref) which does the same but symbolically pre-computes part of +the model, which increases subsequent performance in model fitting (see [Symbolic precomputation](@ref)). There +is also a third option, [`ImpliedEmpty`](@ref) that can serve as a 'placeholder' for loss terms that do not need +an implied part. + +## The loss functions aka [`AbstractLoss`](@ref) +The loss terms specify the objective that is minimized to find the parameter estimates; a model minimizes the +(weighted) sum of all its loss terms. +SEM loss functions ([`SemLoss`](@ref)) compare what the model implies to the observed data, while regularization +terms depend only on the parameters. Available loss functions are - [`SemML`](@ref): maximum likelihood estimation - [`SemWLS`](@ref): weighted least squares estimation - [`SemFIML`](@ref): full-information maximum likelihood estimation - [`SemRidge`](@ref): ridge regularization +- [`SemConstant`](@ref): adds a constant to the objective -## The optimizer part aka `SemOptimizer` -The optimizer part of a model connects to the numerical optimization backend used to fit the model. +## The optimizer aka [`SemOptimizer`](@ref) +The optimizer connects to the numerical optimization backend used to fit the model. It is not part of the model +itself, but it is chosen when fitting (see [Model fitting](@ref)). It can be used to control options like the optimization algorithm, linesearch, stopping criteria, etc. There are currently three available engines (i.e., backends used to carry out the numerical optimization), [`:Optim`](@ref StructuralEquationModels.SemOptimizerOptim) connecting to the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) backend, [`:NLopt`](@ref SEMNLOptExt.SemOptimizerNLopt) connecting to the [NLopt.jl](https://github.com/JuliaOpt/NLopt.jl) backend and [`:Proximal`](@ref SEMProximalOptExt.SemOptimizerProximal) connecting to [ProximalAlgorithms.jl](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl). For more information about the available options see also the tutorials about [Using Optim.jl](@ref) and [Using NLopt.jl](@ref), as well as [Constrained optimization](@ref) and [Regularization](@ref) . diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index 32bb6a529..e50fd2ef5 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -76,7 +76,7 @@ parind[:y3y7] # 29 ``` The bound constraint is easy to specify: just give a vector of upper or lower bounds for each parameter. -In our example, only the parameter labeled `:λₗ` has an upper bound, and the number of total parameters is `n_par(model) = 31`, so +In our example, only the parameter labeled `:λₗ` has an upper bound, and the number of total parameters is `nparams(model) = 31`, so ```@example constraints upper_bounds = fill(Inf, 31) diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md index e0c69ef3c..cd982ecca 100644 --- a/docs/src/tutorials/construction/outer_constructor.md +++ b/docs/src/tutorials/construction/outer_constructor.md @@ -12,13 +12,12 @@ model = Sem( Structural Equation Model - Loss Functions - SemML -- Fields - observed: SemObservedData - implied: RAM + > SemML + - observed: SemObservedData + - implied: RAM ``` -The output of this call tells you exactly what model you just constructed (i.e. what the loss functions, observed, implied and optimizer parts are). +The output of this call tells you exactly what model you just constructed (i.e. what the loss functions and their observed and implied parts are). As you can see, by default, we use maximum likelihood estimation abd the RAM implied type. To choose something different, you can provide it as a keyword argument: @@ -40,11 +39,12 @@ model = Sem( specification = partable, data = data, implied = RAMSymbolic, - loss = SemWLS, - optimizer = SemOptimizer + loss = SemWLS ) ``` +The optimizer is not part of the model itself; it is chosen when fitting (see [Model fitting](@ref)). + In the section on [Our Concept of a Structural Equation Model](@ref), we go over the different options you have for each part of the model, and in [API - model parts](@ref) we explain each option in detail. Let's make another example: to use full information maximum likelihood information (FIML), we use diff --git a/docs/src/tutorials/fitting/fitting.md b/docs/src/tutorials/fitting/fitting.md index 1af03ce8e..c14d9b993 100644 --- a/docs/src/tutorials/fitting/fitting.md +++ b/docs/src/tutorials/fitting/fitting.md @@ -13,10 +13,9 @@ Fitted Structural Equation Model Structural Equation Model - Loss Functions - SemML -- Fields - observed: SemObservedData - implied: RAM + > SemML + - observed: SemObservedData + - implied: RAM ------------- Optimization result ------------- From 8e895b18efd300ca5945ce5640856d995f4a650d Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 19 Jun 2026 16:07:57 +0200 Subject: [PATCH 356/364] remove references to ordinal data --- docs/src/tutorials/specification/graph_interface.md | 2 +- src/types.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/tutorials/specification/graph_interface.md b/docs/src/tutorials/specification/graph_interface.md index 62eeef00b..fe03cf304 100644 --- a/docs/src/tutorials/specification/graph_interface.md +++ b/docs/src/tutorials/specification/graph_interface.md @@ -105,4 +105,4 @@ end The syntax to specify graphs (`@StenoGraph`) may seem a bit strange if you are not familiar with the julia language. It is called a **macro**, but explaining this concept in detail is beyond this documentation (and not necessary to understand to specify models). However, if you want to know more about it, you may have a look at the respective part of the [manual](https://docs.julialang.org/en/v1/manual/metaprogramming/#man-macros). ### The StenoGraphs Package -Behind the scenes, we are using the [StenoGraphs](https://github.com/aaronpeikert/StenoGraphs.jl) package to specify our graphs. It makes a domain specific language available that allows you to specify graphs with arbitrary information attached to its edges and nodes (for structural equation models, this may be the name or the value of a parameter). Is also allows you to specify your own types to "attach" to the graph, called a `Modifier`. So if you contemplate about writing your own modifier (e.g., to mark a variable as ordinal, an effect as quadratic, ...), please refer to the `StenoGraphs` [documentation](https://aaronpeikert.github.io/StenoGraphs.jl/dev/). \ No newline at end of file +Behind the scenes, we are using the [StenoGraphs](https://github.com/aaronpeikert/StenoGraphs.jl) package to specify our graphs. It makes a domain specific language available that allows you to specify graphs with arbitrary information attached to its edges and nodes (for structural equation models, this may be the name or the value of a parameter). Is also allows you to specify your own types to "attach" to the graph, called a `Modifier`. So if you contemplate about writing your own modifier, please refer to the `StenoGraphs` [documentation](https://aaronpeikert.github.io/StenoGraphs.jl/dev/). \ No newline at end of file diff --git a/src/types.jl b/src/types.jl index 7742d3b76..d0218b14f 100644 --- a/src/types.jl +++ b/src/types.jl @@ -41,7 +41,7 @@ abstract type SemOptimizerResult{O <: SemOptimizer} end Supertype of all objects that can serve as the observed field of a SEM. Pre-processes data and computes sufficient statistics for example. -If you have a special kind of data, e.g. ordinal data, you should implement a subtype of SemObserved. +If you have a special kind of data, you should implement a subtype of SemObserved. """ abstract type SemObserved end From 85f339e86fc07b6c93b4b971ac77edc1c3f1e6f7 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sat, 21 Mar 2026 13:36:44 +0100 Subject: [PATCH 357/364] add return type to evaluate docstring --- src/objective_gradient_hessian.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index 23cef4e61..4a8ec649e 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -40,6 +40,8 @@ its computation will be turned off by setting `hessian` to `nothing`. During the evaluation, the internal state of the loss term or of the model could be modified. +Returns the objective value or `nothing`. + # Arguments - `objective`: a Number if the objective should be evaluated, otherwise `nothing` - `gradient`: a pre-allocated vector the gradient should be written to, otherwise `nothing` From e8bdc5ef8e25f41ace46aa752d0eda78656a7d24 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sat, 20 Jun 2026 11:18:35 +0200 Subject: [PATCH 358/364] add docs for p-values and CI --- docs/src/tutorials/inspection/inspection.md | 33 +++++++++++++++++-- .../standard_errors/confidence_intervals.jl | 2 +- src/frontend/fit/standard_errors/z_test.jl | 2 +- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md index a4d8adb1b..cc4d92e3d 100644 --- a/docs/src/tutorials/inspection/inspection.md +++ b/docs/src/tutorials/inspection/inspection.md @@ -81,7 +81,7 @@ update_estimate!(partable, model_fit) details(partable) ``` -We can also update the `ParameterTable` object with other information via [`update_partable!`](@ref). For example, if we want to compare hessian-based and bootstrap-based standard errors, we may write +We can also update the `ParameterTable` object with other information via [`update_partable!`](@ref). For example, we can obtain standard errors from the inverse Hessian with [`se_hessian`](@ref) or by bootstrapping with [`se_bootstrap`](@ref), and add both to the table to compare them: ```@example colored; ansicolor = true se_bs = se_bootstrap(model_fit; n_boot = 20) @@ -93,6 +93,22 @@ update_partable!(partable, :se_bootstrap, model_fit, se_bs) details(partable) ``` +From a vector of standard errors we can also compute *p*-values and confidence intervals for the parameter estimates. +[`z_test!`](@ref) adds the two-sided *p*-value of the test that each parameter is zero (using `z = estimate / se`), and [`normal_CI!`](@ref) adds the lower and upper bounds of a normal-theory confidence interval (95% by default). +Both update the `ParameterTable` in place: + +```@example colored; ansicolor = true +z_test!(partable, model_fit, se_he) +normal_CI!(partable, model_fit, se_he) + +details(partable; show_columns = [:to, :estimate, :p_value, :ci_lower, :ci_upper]) +``` + +The non-mutating variants [`z_test`](@ref) and [`normal_CI`](@ref) return the values instead of writing them to the table. + +Beyond standard errors, [`bootstrap`](@ref) draws bootstrap samples of an arbitrary statistic of a fitted model (not only the parameter estimates), while [`se_bootstrap`](@ref) is a convenience wrapper returning bootstrapped standard errors. +Both support parallel resampling across the available Julia threads via the `parallel = true` keyword. + ## Export results You may convert a `ParameterTable` to a `DataFrame` and use the [`DataFrames`](https://github.com/JuliaData/DataFrames.jl) package for further analysis (or to save it to your hard drive). @@ -100,7 +116,7 @@ You may convert a `ParameterTable` to a `DataFrame` and use the [`DataFrames`](h ```@example colored; ansicolor = true using DataFrames -parameters_df = DataFrame(partable) +parameters_df = DataFrame(partable); ``` # API - model inspection @@ -137,3 +153,16 @@ p_value RMSEA CFI ``` + +## Standard errors and inference + +```@docs +se_hessian +se_bootstrap +bootstrap +StructuralEquationModels.BootstrapResult +z_test +z_test! +normal_CI +normal_CI! +``` diff --git a/src/frontend/fit/standard_errors/confidence_intervals.jl b/src/frontend/fit/standard_errors/confidence_intervals.jl index 20bf58a73..59c6bb7dd 100644 --- a/src/frontend/fit/standard_errors/confidence_intervals.jl +++ b/src/frontend/fit/standard_errors/confidence_intervals.jl @@ -10,7 +10,7 @@ Return normal-theory confidence intervals for all model parameters. - `fitted`: a fitted SEM. - `se`: standard errors for each parameter, e.g. from [`se_hessian`](@ref) or [`se_bootstrap`](@ref). -- `partable`: a [`ParameterTable`](@ref) to write confidence intervals to. +- `partable`: a `ParameterTable` to write confidence intervals to. - `α`: significance level. Defaults to `0.05` (95% intervals). - `name_lower`: column name for the lower bound in `partable`. Defaults to `:ci_lower`. - `name_upper`: column name for the upper bound in `partable`. Defaults to `:ci_upper`. diff --git a/src/frontend/fit/standard_errors/z_test.jl b/src/frontend/fit/standard_errors/z_test.jl index 27bebf147..d912fd867 100644 --- a/src/frontend/fit/standard_errors/z_test.jl +++ b/src/frontend/fit/standard_errors/z_test.jl @@ -13,7 +13,7 @@ Tests the null hypothesis that each parameter is zero using the test statistic - `fitted`: a fitted SEM. - `se`: standard errors for each parameter, e.g. from [`se_hessian`](@ref) or [`se_bootstrap`](@ref). -- `partable`: a [`ParameterTable`](@ref) to write p-values to. +- `partable`: a `ParameterTable` to write p-values to. - `name`: column name for the p-values in `partable`. Defaults to `:p_value`. # Returns From f186e4112c2aedcffca7e339869a1daef74d3a0a Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sat, 20 Jun 2026 12:44:20 +0200 Subject: [PATCH 359/364] fix ImpliedEmpty and add ParameterTable to docs --- docs/src/developer/implied.md | 8 ++++---- docs/src/tutorials/collection/multigroup.md | 8 +++++++- docs/src/tutorials/specification/parameter_table.md | 8 +++++++- src/implied/RAM/generic.jl | 2 +- src/implied/RAM/symbolic.jl | 4 ++-- src/implied/empty.jl | 6 +++--- test/unit_tests/model.jl | 9 +++++++++ 7 files changed, 33 insertions(+), 12 deletions(-) diff --git a/docs/src/developer/implied.md b/docs/src/developer/implied.md index f695e3e82..17b75dfd0 100644 --- a/docs/src/developer/implied.md +++ b/docs/src/developer/implied.md @@ -34,7 +34,7 @@ end As you can see, `update!` gets passed as a first argument `targets`, which is telling us whether the objective value, gradient, and/or hessian are needed. We can then use the functions `is_..._required` and conditional on what the optimizer needs, we can compute and store things we want to make available to the loss functions. For example, as we have seen in [Second example - maximum likelihood](@ref), the `RAM` implied type computes the model-implied covariance matrix and makes it available via `implied.Σ`. -Just as described in [Custom loss functions](@ref), you may define a constructor. Typically, this will depend on the `specification = ...` argument that can be a `ParameterTable` or a `RAMMatrices` object. +Just as described in [Custom loss functions](@ref), you may define a constructor. It should take the model specification (a `ParameterTable` or a `RAMMatrices` object) as the first positional argument, like the built-in [`RAM`](@ref) — this is how the outer `Sem` constructor builds the implied part when you select it by type (e.g. `implied = MyImplied`). We implement an `ImpliedEmpty` type in our package that does nothing but serving as an `implied` field in case you are using a loss function that does not need any implied type at all. You may use it as a template for defining your own implied type, as it also shows how to handle the specification objects: @@ -47,7 +47,7 @@ Empty placeholder for loss functions that don't need an implied part. # Constructor - ImpliedEmpty(;specification, kwargs...) + ImpliedEmpty(specification; kwargs...) # Arguments - `specification`: either a `RAMMatrices` or `ParameterTable` object @@ -72,8 +72,8 @@ end ### Constructors ############################################################################################ -function ImpliedEmpty(; - specification, +function ImpliedEmpty( + specification::SemSpecification; meanstruct = NoMeanStruct(), hessianeval = ExactHessian(), kwargs..., diff --git a/docs/src/tutorials/collection/multigroup.md b/docs/src/tutorials/collection/multigroup.md index ccd3bdd65..d085b04a9 100644 --- a/docs/src/tutorials/collection/multigroup.md +++ b/docs/src/tutorials/collection/multigroup.md @@ -86,4 +86,10 @@ update_estimate!(partable, sem_fit) details(partable) ``` -Other things you can query about your fitted model (fit measures, standard errors, etc.) are described in the section [Model inspection](@ref) and work the same way for multigroup models. \ No newline at end of file +Other things you can query about your fitted model (fit measures, standard errors, etc.) are described in the section [Model inspection](@ref) and work the same way for multigroup models. + +# API + +```@docs +EnsembleParameterTable +``` \ No newline at end of file diff --git a/docs/src/tutorials/specification/parameter_table.md b/docs/src/tutorials/specification/parameter_table.md index 62c45c9a4..61549b40c 100644 --- a/docs/src/tutorials/specification/parameter_table.md +++ b/docs/src/tutorials/specification/parameter_table.md @@ -6,4 +6,10 @@ As lavaan also uses parameter tables to store model specifications, we are worki ## Convert from and to RAMMatrices To convert a RAMMatrices object to a ParameterTable, simply use `partable = ParameterTable(rammatrices)`. -To convert an object of type `ParameterTable` to RAMMatrices, you can use `ram_matrices = RAMMatrices(partable)`. \ No newline at end of file +To convert an object of type `ParameterTable` to RAMMatrices, you can use `ram_matrices = RAMMatrices(partable)`. + +# API + +```@docs +ParameterTable +``` \ No newline at end of file diff --git a/src/implied/RAM/generic.jl b/src/implied/RAM/generic.jl index 1569b3415..90d886f98 100644 --- a/src/implied/RAM/generic.jl +++ b/src/implied/RAM/generic.jl @@ -6,7 +6,7 @@ Model implied covariance and means via RAM notation. # Constructor - RAM(; specification, gradient = true, kwargs...) + RAM(specification; gradient = true, kwargs...) # Arguments - `specification`: either a `RAMMatrices` or `ParameterTable` object diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index 52a192e64..552cd65d9 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -6,8 +6,8 @@ Subtype of `SemImplied` that implements the RAM notation with symbolic precomput # Constructor - RAMSymbolic(; - specification, + RAMSymbolic( + specification; vech = false, gradient = true, hessian = false, diff --git a/src/implied/empty.jl b/src/implied/empty.jl index a650a07a9..3f1c3ef5b 100644 --- a/src/implied/empty.jl +++ b/src/implied/empty.jl @@ -7,7 +7,7 @@ Empty placeholder for models that don't need an implied part. # Constructor - ImpliedEmpty(;specification, kwargs...) + ImpliedEmpty(specification; kwargs...) # Arguments - `specification`: either a `RAMMatrices` or `ParameterTable` object @@ -32,8 +32,8 @@ end ### Constructors ############################################################################################ -function ImpliedEmpty(; - specification, +function ImpliedEmpty( + specification::SemSpecification; meanstruct = NoMeanStruct(), hessianeval = ExactHessian(), kwargs..., diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index b62f056d9..ee9b07532 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -211,3 +211,12 @@ end @test nsamples(observed(term_g2)) == size(dat_grouped, 1) - n_g1 @test nsamples(grouped_model) == size(dat_grouped, 1) end + +@testset "ImpliedEmpty" begin + # positional `specification`, consistent with RAM/RAMSymbolic + @test ImpliedEmpty(ram_matrices) isa ImpliedEmpty + + # implied can be selected by type through the outer `Sem` constructor (like `implied = RAM`) + model = Sem(specification = ram_matrices, data = dat, implied = ImpliedEmpty) + @test implied(model) isa ImpliedEmpty +end From d81dcf0b5de203213bdefa9ed6959988c1ea7ebf Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sat, 20 Jun 2026 12:46:23 +0200 Subject: [PATCH 360/364] bump version number --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 14e394cd3..19fa518df 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "StructuralEquationModels" uuid = "383ca8c5-e4ff-4104-b0a9-f7b279deed53" authors = ["Maximilian Ernst", "Aaron Peikert"] -version = "0.4.2" +version = "0.5.0" [deps] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" From bba0d25c0fa3d1fd65a7f6f5c391216096f0dd3e Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sat, 20 Jun 2026 20:16:14 +0200 Subject: [PATCH 361/364] fix FIML fit_measures --- src/frontend/fit/fitmeasures/CFI.jl | 13 ++++--------- test/examples/political_democracy/by_parts.jl | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/frontend/fit/fitmeasures/CFI.jl b/src/frontend/fit/fitmeasures/CFI.jl index e2bcb8c7d..36871b89e 100644 --- a/src/frontend/fit/fitmeasures/CFI.jl +++ b/src/frontend/fit/fitmeasures/CFI.jl @@ -58,15 +58,10 @@ function χ²_varonly(loss::SemWLS) return N⁻*0.5*tr((I - Σ₀*S⁻¹)^2) end -# For FIML, an explicit bl model has to be passed -function χ²_varonly(loss::SemFIML) - """ - Computing the CFI with FIML requires explicitely passing a fitted baseline model as - CFI(fit::SemFit, fit_baseline::SemFit) - """ |> - ArgumentError |> - throw -end +# For FIML, the variance-only baseline cannot be derived automatically, so the CFI is +# `missing` unless an explicit baseline model is passed via `CFI(fit, fit_baseline)`. +# Returning `missing` (instead of throwing) keeps `fit_measures()` usable for FIML models. +χ²_varonly(loss::SemFIML) = missing function dof_varonly(model::AbstractSem) return sum(sem_terms(model)) do semterm diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index ef634a59f..e3cfbf115 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -5,7 +5,7 @@ semoptimizer = SemOptimizer(engine = opt_engine) model_ml = Sem(specification = spec, data = dat) -@test SEM.params(model_ml) == SEM.params(spec) +@test SEM.param_labels(model_ml) == SEM.param_labels(spec) model_ls_sym = Sem(specification = spec, data = dat, implied = RAMSymbolic, vech = true, loss = SemWLS) From 92346ee781ab37e8fc80ae72730cefef6def5877 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sat, 20 Jun 2026 23:02:26 +0200 Subject: [PATCH 362/364] fix docs typo --- docs/src/developer/optimizer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/developer/optimizer.md b/docs/src/developer/optimizer.md index 4659ba5d5..538d9e0f5 100644 --- a/docs/src/developer/optimizer.md +++ b/docs/src/developer/optimizer.md @@ -32,7 +32,7 @@ end options(optimizer::SemOptimizerMyopt) = optimizer.options ``` -Note that `SemOptimizerMyopt` is defined as a subtype of [`SemOptimizer{:Myopt}`](@ref SEM.SemOptimizer)`, +Note that `SemOptimizerMyopt` is defined as a subtype of [`SemOptimizer{:Myopt}`](@ref SEM.SemOptimizer), and `SEM.sem_optimizer_subtype(::Val{:Myopt})` returns `SemOptimizerMyopt`. This instructs *SEM.jl* to use `SemOptimizerMyopt` when `:Myopt` is specified as the engine for model fitting: `fit(..., engine = :Myopt)`. From 1dba1f0b4c1b97bbd72bfd0e0b4937bbeb307517 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sat, 20 Jun 2026 23:13:01 +0200 Subject: [PATCH 363/364] fix fiml fitmeasures test --- test/examples/political_democracy/by_parts.jl | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index e3cfbf115..27de965ec 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -276,6 +276,17 @@ model_ml_sym = Sem( loss = SemFIML, ) +if !ismissing(spec_varonly) + model_varonly = Sem( + data = dat_missing, + observed = SemObservedMissing, + specification = spec_varonly, + loss = SemFIML, + ) +else + model_varonly = nothing +end + ############################################################################################ ### test gradients ############################################################################################ @@ -310,9 +321,14 @@ end @testset "fitmeasures/se_fiml" begin solution_ml = fit(semoptimizer, model_ml) + solution_varonly = + !isnothing(model_varonly) ? fit(semoptimizer, model_varonly) : nothing test_fitmeasures( - fit_measures(solution_ml), + solution_ml, solution_lav[:fitmeasures_fiml]; + fitted_baseline = solution_varonly, + fitmeasures = !isnothing(solution_varonly) ? SEM.DEFAULT_FIT_MEASURES : + filter(!=(CFI), SEM.DEFAULT_FIT_MEASURES), atol = 1e-3, ) From f6fcdb7e8c307d14fcb90565dfe85c5ae16428c3 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 21 Jun 2026 15:41:04 +0200 Subject: [PATCH 364/364] improve explanation on replace_observed --- docs/src/developer/loss.md | 91 +++++++++++++++++++++++++++++- docs/src/performance/simulation.md | 42 +++++++++++++- 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index e763364b2..d9b89ab95 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -248,4 +248,93 @@ model_ml = SemFiniteDiff( model_fit = fit(model_ml) ``` -If you want to differentiate your own loss functions via automatic differentiation, check out the [AutoDiffSEM](https://github.com/StructuralEquationModels/AutoDiffSEM) package. +## Supporting `replace_observed` + +[`replace_observed`](@ref) swaps the observed data of a model while keeping the rest of +the model (specification, implied type, loss configuration) intact. It is the backbone of +[Simulation studies](@ref) and the [`bootstrap`](@ref), where the same model is fitted to +many datasets and rebuilding it from scratch each time would be wasteful. + +### The default mechanism + +For a `SemLoss` term, the generic implementation rebuilds the term by calling its three-argument +constructor with the new observed data, the *original* implied part, and the *original* loss term +as `refloss`: + +```julia +# simplified; see src/loss/abstract.jl +function replace_observed(loss::SemLoss, new_observed::SemObserved; kwargs...) + loss_ctor = typeof(loss).name.wrapper # e.g. `MaximumLikelihood` + return loss_ctor(new_observed, implied(loss), loss) # third arg is the `refloss` +end +``` + +This is exactly the three-argument constructor every `SemLoss` already provides (see +[Second example - maximum likelihood](@ref)). The `refloss` argument is what makes this work +without re-deriving anything: the new term inherits the loss-specific configuration from the +reference term and shares its implied state (and, where applicable, internal buffers). The implied +part is shared rather than copied because it depends only on the model specification, not on the data. + +Because of this, **a loss term that does not cache anything derived from the observed data needs no +extra code** — implementing the three-argument constructor is enough, and `replace_observed` works +out of the box. `MaximumLikelihood` above is such a case: it reads `obs_cov(observed(loss))` on every +evaluation and stores nothing, so it even ignores `refloss` entirely and is already fully compatible. + +### Plain `AbstractLoss` terms (no observed part) + +The mechanism above only applies to [`SemLoss`](@ref) terms, which carry an `observed` part. A plain +[`AbstractLoss`](@ref) term — like the `MyRidge` regularizer from the [Minimal](@ref) example — depends +only on the parameters and has no notion of observed data. There is therefore nothing to swap, and +`replace_observed` returns such terms unchanged: + +```julia +# src/loss/abstract.jl — fallback for non-SEM loss terms +replace_observed(loss::AbstractLoss, ::Any; kwargs...) = loss +``` + +This is handled by the default fallback, so **you do not need to write anything** for your own +`AbstractLoss` types: when a model mixes SEM and non-SEM loss terms (e.g. `loss = (SemML, MyRidge)`), +`replace_observed` rebuilds the `SemML` term with the new data and carries the `MyRidge` term over +as-is. The `recompute_observed_state` keyword is likewise accepted and ignored. + +If your regularizer *does* need to know about the data, the idiomatic solution is to make it a +`SemLoss` (so it owns an `observed` part and participates in the rebuild) rather than to specialize +`replace_observed` on a plain `AbstractLoss`. + +### When you need a custom method + +You need to specialize `replace_observed` when your loss term **precomputes and stores a quantity +derived from the observed data**. The default mechanism inherits that quantity from the `refloss`, +so after swapping in new data the cached value would be stale. + +[`SemWLS`](@ref) is the canonical example. Its weight matrix `V` defaults to the GLS weights computed +from the *observed* covariance matrix and is stored on the term. If `replace_observed` simply reused +`refloss.V`, the new term would weight the new data with the old data's weights. `SemWLS` therefore +overrides `replace_observed` to recompute the weights from the new data by default, while exposing a +`recompute_observed_state` keyword to opt out: + +```julia +# src/loss/WLS/WLS.jl +function replace_observed( + loss::SemWLS, + new_observed::SemObserved; + recompute_observed_state::Bool = true, +) + return SemWLS( + new_observed, + implied(loss), + loss; + # pass `nothing` to recompute from the new data, or reuse the old matrices + wls_weight_matrix = recompute_observed_state ? nothing : loss.V, + wls_weight_matrix_mean = recompute_observed_state ? nothing : loss.V_μ, + ) +end +``` + +Note how the override still goes through the three-argument constructor with `loss` as `refloss`, +so all *other* configuration (e.g. the choice of approximate vs. analytic Hessian) is still +inherited automatically — the custom method only intervenes for the observed-dependent caches. + +The `recompute_observed_state` keyword is a convention shared by all `replace_observed` methods: it +is forwarded from the model-level call down to every loss term, and terms without observed-dependent +caches simply ignore it. diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index 3061c6564..a33232ee4 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -65,6 +65,46 @@ model = Sem( model_updated = replace_observed(model, data_2) ``` +`replace_observed` accepts a data matrix, a `DataFrame`, or a ready-made `SemObserved` object, and +works for multigroup/collection models too (pass a `Dict` mapping term ids to data, or a `DataFrame` +together with a `semterm_column`). See the [API](@ref simulation_api) below for all signatures. + +### Recomputing observed-dependent state + +Some loss terms cache quantities that are derived from the observed data. The most prominent example +is weighted least squares ([`SemWLS`](@ref)), whose weight matrix defaults to the GLS weights computed +from the observed covariance matrix. By default, `replace_observed` **recomputes** these quantities +from the new data, which is what you want in most simulation studies: + +```@example replace_observed +model_wls = Sem( + specification = partable, + data = data_1, + implied = RAMSymbolic, + vech = true, + loss = SemWLS, +) + +# weight matrix recomputed from `data_2` (default) +model_wls_updated = replace_observed(model_wls, data_2) +``` + +If instead you want to keep the original observed-dependent state — e.g. fit every replication with +the *same* fixed weight matrix — pass `recompute_observed_state = false`: + +```@example replace_observed +model_wls_fixed = replace_observed(model_wls, data_2; recompute_observed_state = false) +``` + +Loss terms without observed-dependent caches (such as `SemML`) ignore this keyword. + +!!! tip "Simulating with your own loss type" + If you run simulation studies with a custom loss function, see + [Supporting `replace_observed`](@ref) in the [Custom loss functions](@ref) chapter. It explains + how `replace_observed` rebuilds a loss term, when the default behavior is enough, and how to + implement a custom `replace_observed` method (with `recompute_observed_state`) if your loss + caches quantities derived from the observed data. + ## Multithreading !!! danger "Thread safety" *This is only relevant when you are planning to fit updated models in parallel* @@ -93,7 +133,7 @@ Threads.@threads for i in 1:2 end ``` -## API +## [API](@id simulation_api) ```@docs replace_observed