diff --git a/Project.toml b/Project.toml
index 376347083..19fa518df 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,13 +1,14 @@
name = "StructuralEquationModels"
uuid = "383ca8c5-e4ff-4104-b0a9-f7b279deed53"
authors = ["Maximilian Ernst", "Aaron Peikert"]
-version = "0.4.2"
+version = "0.5.0"
[deps]
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@@ -15,6 +16,7 @@ NLSolversBase = "d41bc354-129a-5804-8e4c-c37616107c6c"
Optim = "429524aa-4258-5aef-a3af-852621145aeb"
Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
@@ -25,21 +27,23 @@ Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7"
SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b"
[compat]
-julia = "1.9, 1.10, 1.11"
-StenoGraphs = "0.2 - 0.3, 0.4.1 - 0.5"
+julia = "1.10, 1.11, 1.12"
+StenoGraphs = "0.5"
DataFrames = "1"
Distributions = "0.25"
FiniteDiff = "2"
LineSearches = "7"
-NLSolversBase = "7"
+NLSolversBase = "8"
NLopt = "0.6, 1"
-Optim = "1"
-PrettyTables = "2"
+Optim = "2"
+PrettyTables = "3"
ProximalAlgorithms = "0.7"
StatsBase = "0.33, 0.34"
-Symbolics = "4, 5, 6"
-SymbolicUtils = "1.4 - 1.5, 1.7, 2, 3"
+Symbolics = "4, 5, 6, 7"
+SymbolicUtils = "1.4 - 1.5, 1.7, 2, 3, 4"
StatsAPI = "1"
+DelimitedFiles = "1"
+Statistics = "1"
[extras]
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
diff --git a/docs/make.jl b/docs/make.jl
index 1bb68c4da..2990f8dbb 100644
--- a/docs/make.jl
+++ b/docs/make.jl
@@ -1,7 +1,12 @@
-using Documenter, StructuralEquationModels
+using Documenter, StructuralEquationModels, NLopt, ProximalAlgorithms, ProximalOperators
+
+SEM = StructuralEquationModels
+SEMNLOptExt = Base.get_extension(StructuralEquationModels, :SEMNLOptExt)
+SEMProximalOptExt = Base.get_extension(StructuralEquationModels, :SEMProximalOptExt)
makedocs(
sitename = "StructuralEquationModels.jl",
+ modules = [SEM, SEMNLOptExt, SEMProximalOptExt],
pages = [
"index.md",
"Tutorials" => [
@@ -47,11 +52,6 @@ makedocs(
"Starting values" => "performance/starting_values.md",
"Parametric Types" => "performance/parametric.md",
],
- "Internals and design" => [
- "Internals and design" => "internals/internals.md",
- "files" => "internals/files.md",
- "types" => "internals/types.md",
- ],
],
format = Documenter.HTML(
prettyurls = get(ENV, "CI", nothing) == "true",
@@ -60,6 +60,7 @@ makedocs(
collapselevel = 1,
),
doctest = false,
+ checkdocs = :none,
)
# doctest(StructuralEquationModels, fix=true)
diff --git a/docs/src/assets/concept.svg b/docs/src/assets/concept.svg
index fa222a0d9..fe6fd2081 100644
--- a/docs/src/assets/concept.svg
+++ b/docs/src/assets/concept.svg
@@ -1,171 +1,154 @@
+
+
diff --git a/docs/src/assets/concept_typed.svg b/docs/src/assets/concept_typed.svg
index 88a0d8566..4592fd87c 100644
--- a/docs/src/assets/concept_typed.svg
+++ b/docs/src/assets/concept_typed.svg
@@ -1,171 +1,152 @@
+
+
diff --git a/docs/src/developer/extending.md b/docs/src/developer/extending.md
index 5c3183da4..20f558def 100644
--- a/docs/src/developer/extending.md
+++ b/docs/src/developer/extending.md
@@ -1,10 +1,12 @@
# Extending the package
-As discussed in the section on [Model Construction](@ref), every Structural Equation Model (`Sem`) consists of three (four with the optimizer) parts:
+As discussed in [Our Concept of a Structural Equation Model](@ref), a `Sem` is built from one or more loss
+terms, and each SEM loss function bundles an *observed* and an *implied* part:
-On the following pages, we will explain how you can define your own custom parts and "plug them in". There are certain things you **have to do** to define custom parts and some things you **can do** to have a more pleasent experience. In general, these requirements fall into the categories
+On the following pages, we will explain how you can define your own custom parts (a loss function, an observed
+type, an implied type, or an optimizer) and "plug them in". There are certain things you **have to do** to define custom parts and some things you **can do** to have a more pleasent experience. In general, these requirements fall into the categories
- minimal (to use your custom part and fit a `Sem` with it)
- use the outer constructor to build a model in a more convenient way
- use additional functionality like standard errors, fit measures, etc.
\ No newline at end of file
diff --git a/docs/src/developer/implied.md b/docs/src/developer/implied.md
index 056cd6638..17b75dfd0 100644
--- a/docs/src/developer/implied.md
+++ b/docs/src/developer/implied.md
@@ -13,9 +13,9 @@ end
and a method to update!:
```julia
-import StructuralEquationModels: objective!
+import StructuralEquationModels: update!
-function update!(targets::EvaluationTargets, implied::MyImplied, model::AbstractSemSingle, params)
+function update!(targets::EvaluationTargets, implied::MyImplied, params)
if is_objective_required(targets)
...
@@ -31,12 +31,10 @@ function update!(targets::EvaluationTargets, implied::MyImplied, model::Abstract
end
```
-As you can see, `update` gets passed as a first argument `targets`, which is telling us whether the objective value, gradient, and/or hessian are needed.
+As you can see, `update!` gets passed as a first argument `targets`, which is telling us whether the objective value, gradient, and/or hessian are needed.
We can then use the functions `is_..._required` and conditional on what the optimizer needs, we can compute and store things we want to make available to the loss functions. For example, as we have seen in [Second example - maximum likelihood](@ref), the `RAM` implied type computes the model-implied covariance matrix and makes it available via `implied.Σ`.
-
-
-Just as described in [Custom loss functions](@ref), you may define a constructor. Typically, this will depend on the `specification = ...` argument that can be a `ParameterTable` or a `RAMMatrices` object.
+Just as described in [Custom loss functions](@ref), you may define a constructor. It should take the model specification (a `ParameterTable` or a `RAMMatrices` object) as the first positional argument, like the built-in [`RAM`](@ref) — this is how the outer `Sem` constructor builds the implied part when you select it by type (e.g. `implied = MyImplied`).
We implement an `ImpliedEmpty` type in our package that does nothing but serving as an `implied` field in case you are using a loss function that does not need any implied type at all. You may use it as a template for defining your own implied type, as it also shows how to handle the specification objects:
@@ -45,19 +43,18 @@ We implement an `ImpliedEmpty` type in our package that does nothing but serving
### Types
############################################################################################
"""
-Empty placeholder for models that don't need an implied part.
-(For example, models that only regularize parameters.)
+Empty placeholder for loss functions that don't need an implied part.
# Constructor
- ImpliedEmpty(;specification, kwargs...)
+ ImpliedEmpty(specification; kwargs...)
# Arguments
- `specification`: either a `RAMMatrices` or `ParameterTable` object
# Examples
-A multigroup model with ridge regularization could be specified as a `SemEnsemble` with one
-model per group and an additional model with `ImpliedEmpty` and `SemRidge` for the regularization part.
+A multigroup model with ridge regularization could be specified as a `Sem` with one
+SEM term (`SemLoss`) per group and an additional `SemRidge` regularization term.
# Extended help
@@ -75,8 +72,8 @@ end
### Constructors
############################################################################################
-function ImpliedEmpty(;
- specification,
+function ImpliedEmpty(
+ specification::SemSpecification;
meanstruct = NoMeanStruct(),
hessianeval = ExactHessian(),
kwargs...,
@@ -88,13 +85,7 @@ end
### methods
############################################################################################
-update!(targets::EvaluationTargets, implied::ImpliedEmpty, par, model) = nothing
-
-############################################################################################
-### Recommended methods
-############################################################################################
-
-update_observed(implied::ImpliedEmpty, observed::SemObserved; kwargs...) = implied
+update!(targets::EvaluationTargets, implied::ImpliedEmpty, par) = nothing
```
-As you see, similar to [Custom loss functions](@ref) we implement a method for `update_observed`.
\ No newline at end of file
+As you see, similar to [Custom loss functions](@ref) we implement a constructor.
\ No newline at end of file
diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md
index d6949842b..d9b89ab95 100644
--- a/docs/src/developer/loss.md
+++ b/docs/src/developer/loss.md
@@ -11,9 +11,9 @@ Since we allow for the optimization of sums of loss functions, and the maximum l
using StructuralEquationModels
```
-To define a new loss function, you have to define a new type that is a subtype of `SemLossFunction`:
+To define a new loss function, you have to define a new type that is a subtype of `AbstractLoss`:
```@example loss
-struct Ridge <: SemLossFunction
+struct MyRidge <: AbstractLoss
α
I
end
@@ -25,8 +25,8 @@ Additionaly, we need to define a *method* of the function `evaluate!` to compute
```@example loss
import StructuralEquationModels: evaluate!
-evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, ridge::Ridge, model::AbstractSem, par) =
- ridge.α * sum(i -> par[i]^2, ridge.I)
+evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, ridge::MyRidge, par) =
+ ridge.α * sum(i -> abs2(par[i]), ridge.I)
```
The function `evaluate!` recognizes by the types of the arguments `objective`, `gradient` and `hessian` whether it should compute the objective value, gradient or hessian of the model w.r.t. the parameters.
@@ -71,7 +71,7 @@ partable = ParameterTable(
)
parameter_indices = getindex.([param_indices(partable)], [:a, :b, :c])
-myridge = Ridge(0.01, parameter_indices)
+myridge = MyRidge(0.01, parameter_indices)
model = SemFiniteDiff(
specification = partable,
@@ -82,23 +82,23 @@ model = SemFiniteDiff(
model_fit = fit(model)
```
-This is one way of specifying the model - we now have **one model** with **multiple loss functions**. Because we did not provide a gradient for `Ridge`, we have to specify a `SemFiniteDiff` model that computes numerical gradients with finite difference approximation.
+This is one way of specifying the model - we now have **one model** with **multiple loss functions**. Because we did not provide a gradient for `MyRidge`, we have to specify a `SemFiniteDiff` model that computes numerical gradients with finite difference approximation.
-Note that the last argument to the `objective!` method is the whole model. Therefore, we can access everything that is stored inside our model everytime we compute the objective value for our loss function. Since ridge regularization is a very easy case, we do not need to do this. But maximum likelihood estimation for example depends on both the observed and the model implied covariance matrix. See [Second example - maximum likelihood](@ref) for information on how to do that.
+Ridge regularization only depends on the parameters, so the `evaluate!` method above does not need anything else. Other loss functions, however, depend on the observed data and on what the model implies about it. Loss functions that compare the implied and the observed structure are subtypes of [`SemLoss`](@ref) and store their own `observed` and `implied` parts, which can be accessed inside `evaluate!` via `observed(loss)` and `implied(loss)`. See [Second example - maximum likelihood](@ref) for information on how to do that.
### Improve performance
By far the biggest improvements in performance will result from specifying analytical gradients. We can do this for our example:
```@example loss
-function evaluate!(objective, gradient, hessian::Nothing, ridge::Ridge, model::AbstractSem, par)
+function evaluate!(objective, gradient, hessian::Nothing, ridge::MyRidge, par)
# compute gradient
if !isnothing(gradient)
fill!(gradient, 0)
gradient[ridge.I] .= 2 * ridge.α * par[ridge.I]
end
# compute objective
- if !isnothing(objective)
+ if !isnothing(objective)
return ridge.α * sum(i -> par[i]^2, ridge.I)
end
end
@@ -136,54 +136,43 @@ Additionally, you may provide analytic hessians by writing a respective method f
## Convenient
-To be able to build the model with the [Outer Constructor](@ref), you need to add a constructor for your loss function that only takes keyword arguments and allows for passing optional additional kewyword arguments. A constructor is just a function that creates a new instance of your type:
+In the minimal example above we built `myridge` ourselves and passed the ready-made instance to
+the model via `loss = (SemML, myridge)`. Alternatively, you can let the outer [`Sem`](@ref)
+constructor build the loss term for you: pass the loss *type* instead of an instance and provide
+a keyword constructor.
```julia
-function MyLoss(;arg1 = ..., arg2, kwargs...)
- ...
- return MyLoss(...)
-end
+MyRidge(; α_ridge, which_ridge, kwargs...) = MyRidge(α_ridge, which_ridge)
```
-All keyword arguments that a user passes to the Sem constructor are passed to your loss function. In addition, all previously constructed parts of the model (implied and observed part) are passed as keyword arguments as well as the number of parameters `n_par = ...`, so your constructor may depend on those. For example, the constructor for `SemML` in our package depends on the additional argument `meanstructure` as well as the observed part of the model to pre-allocate arrays of the same size as the observed covariance matrix and the observed mean vector:
+Any keyword arguments passed to `Sem(...)` are forwarded to this constructor (along with some that
+the model supplies automatically, such as `nparams`), so the loss can be configured directly from
+the model call:
```julia
-function SemML(;observed, meanstructure = false, approx_H = false, kwargs...)
-
- isnothing(obs_mean(observed)) ?
- meandiff = nothing :
- meandiff = copy(obs_mean(observed))
-
- return SemML(
- similar(obs_cov(observed)),
- similar(obs_cov(observed)),
- meandiff,
- approx_H,
- Val(meanstructure)
- )
-end
+model = SemFiniteDiff(
+ specification = partable,
+ data = example_data("political_democracy"),
+ loss = (SemML, MyRidge),
+ α_ridge = 0.01,
+ which_ridge = parameter_indices,
+)
```
-## Additional functionality
-
-### Update observed data
-
-If you are planing a simulation study where you have to fit the **same model** to many **different datasets**, it is computationally beneficial to not build the whole model completely new everytime you change your data.
-Therefore, we provide a function to update the data of your model, `replace_observed(model(semfit); data = new_data)`. However, we can not know beforehand in what way your loss function depends on the specific datasets. The solution is to provide a method for `update_observed`. Since `Ridge` does not depend on the data at all, this is quite easy:
-
-```julia
-import StructuralEquationModels: update_observed
+Note that, being a plain `AbstractLoss`, `MyRidge` neither stores nor receives an `observed` or
+`implied` part — it depends only on the parameters.
+SEM-specific loss functions are constructed differently; see
+[Second example - maximum likelihood](@ref).
-update_observed(ridge::Ridge, observed::SemObserved; kwargs...) = ridge
-```
+## Additional functionality
### Access additional information
If you want to provide a way to query information about loss functions of your type, you can provide functions for that:
```julia
-hyperparameter(ridge::Ridge) = ridge.α
-regularization_indices(ridge::Ridge) = ridge.I
+hyperparameter(ridge::MyRidge) = ridge.α
+regularization_indices(ridge::MyRidge) = ridge.I
```
# Second example - maximum likelihood
@@ -195,7 +184,9 @@ To keep it simple, we only cover models without a meanstructure. The maximum lik
F_{ML} = \log \det \Sigma_i + \mathrm{tr}\left(\Sigma_{i}^{-1} \Sigma_o \right)
```
-where ``\Sigma_i`` is the model implied covariance matrix and ``\Sigma_o`` is the observed covariance matrix. We can query the model implied covariance matrix from the `implied` par of our model, and the observed covariance matrix from the `observed` path of our model.
+where ``\Sigma_i`` is the model implied covariance matrix and ``\Sigma_o`` is the observed covariance matrix. We can query the model implied covariance matrix from the `implied` part of our loss term, and the observed covariance matrix from the `observed` part of our loss term.
+
+Since this loss function compares the implied and the observed structure, it is a subtype of [`SemLoss`](@ref) rather than a plain `AbstractLoss`. Every `SemLoss` stores its own `observed` and `implied` parts, which can be accessed inside `evaluate!` via `observed(loss)` and `implied(loss)`.
To get information on what we can access from a certain `implied` or `observed` type, we can check it`s documentation an the pages [API - model parts](@ref) or via the help mode of the REPL:
@@ -207,20 +198,33 @@ help?> RAM
help?> SemObservedData
```
-We see that the model implied covariance matrix can be assessed as `Σ(implied)` and the observed covariance matrix as `obs_cov(observed)`.
+We see that the model implied covariance matrix can be assessed as `implied(loss).Σ` and the observed covariance matrix as `obs_cov(observed(loss))`.
-With this information, we write can implement maximum likelihood optimization as
+Unlike a plain `AbstractLoss`, a `SemLoss` subtype stores its `observed` and `implied` parts (in its first two fields), and the [`Sem`](@ref) constructor builds it for you. To support this, every `SemLoss` subtype should provide a constructor with three positional arguments:
+ * `observed::SemObserved`: the observed part of the loss term
+ * `implied::SemImplied`: the implied part of the loss term
+ * `refloss::Union{MaximumLikelihood, Nothing} = nothing`: an optional existing loss term of the
+ same type, used as a reference for any loss-specific configuration.
+
+Any additional configuration is passed as optional keyword arguments; if both `refloss` and keyword arguments are given, the keyword arguments take precedence. This constructor is also used by [`replace_observed`](@ref) to rebuild the loss term with new observed data while sharing the implied state. With this, we can implement maximum likelihood optimization as
```@example loss
-struct MaximumLikelihood <: SemLossFunction end
+struct MaximumLikelihood{O <: SemObserved, I <: SemImplied} <: SemLoss{O, I}
+ observed::O
+ implied::I
+end
+
+# constructor used by the `Sem` constructor to build the loss term
+MaximumLikelihood(observed::SemObserved, implied::SemImplied, refloss = nothing; kwargs...) =
+ MaximumLikelihood{typeof(observed), typeof(implied)}(observed, implied)
using LinearAlgebra
-import StructuralEquationModels: obs_cov, evaluate!
+import StructuralEquationModels: evaluate!
-function evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, semml::MaximumLikelihood, model::AbstractSem, par)
+function evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, semml::MaximumLikelihood, par)
# access the model implied and observed covariance matrices
- Σᵢ = implied(model).Σ
- Σₒ = obs_cov(observed(model))
+ Σᵢ = implied(semml).Σ
+ Σₒ = obs_cov(observed(semml))
# compute the objective
if isposdef(Symmetric(Σᵢ)) # is the model implied covariance matrix positive definite?
return logdet(Σᵢ) + tr(inv(Σᵢ)*Σₒ)
@@ -238,10 +242,99 @@ Let's specify and fit a model:
model_ml = SemFiniteDiff(
specification = partable,
data = example_data("political_democracy"),
- loss = MaximumLikelihood()
+ loss = MaximumLikelihood
)
model_fit = fit(model_ml)
```
-If you want to differentiate your own loss functions via automatic differentiation, check out the [AutoDiffSEM](https://github.com/StructuralEquationModels/AutoDiffSEM) package.
+## Supporting `replace_observed`
+
+[`replace_observed`](@ref) swaps the observed data of a model while keeping the rest of
+the model (specification, implied type, loss configuration) intact. It is the backbone of
+[Simulation studies](@ref) and the [`bootstrap`](@ref), where the same model is fitted to
+many datasets and rebuilding it from scratch each time would be wasteful.
+
+### The default mechanism
+
+For a `SemLoss` term, the generic implementation rebuilds the term by calling its three-argument
+constructor with the new observed data, the *original* implied part, and the *original* loss term
+as `refloss`:
+
+```julia
+# simplified; see src/loss/abstract.jl
+function replace_observed(loss::SemLoss, new_observed::SemObserved; kwargs...)
+ loss_ctor = typeof(loss).name.wrapper # e.g. `MaximumLikelihood`
+ return loss_ctor(new_observed, implied(loss), loss) # third arg is the `refloss`
+end
+```
+
+This is exactly the three-argument constructor every `SemLoss` already provides (see
+[Second example - maximum likelihood](@ref)). The `refloss` argument is what makes this work
+without re-deriving anything: the new term inherits the loss-specific configuration from the
+reference term and shares its implied state (and, where applicable, internal buffers). The implied
+part is shared rather than copied because it depends only on the model specification, not on the data.
+
+Because of this, **a loss term that does not cache anything derived from the observed data needs no
+extra code** — implementing the three-argument constructor is enough, and `replace_observed` works
+out of the box. `MaximumLikelihood` above is such a case: it reads `obs_cov(observed(loss))` on every
+evaluation and stores nothing, so it even ignores `refloss` entirely and is already fully compatible.
+
+### Plain `AbstractLoss` terms (no observed part)
+
+The mechanism above only applies to [`SemLoss`](@ref) terms, which carry an `observed` part. A plain
+[`AbstractLoss`](@ref) term — like the `MyRidge` regularizer from the [Minimal](@ref) example — depends
+only on the parameters and has no notion of observed data. There is therefore nothing to swap, and
+`replace_observed` returns such terms unchanged:
+
+```julia
+# src/loss/abstract.jl — fallback for non-SEM loss terms
+replace_observed(loss::AbstractLoss, ::Any; kwargs...) = loss
+```
+
+This is handled by the default fallback, so **you do not need to write anything** for your own
+`AbstractLoss` types: when a model mixes SEM and non-SEM loss terms (e.g. `loss = (SemML, MyRidge)`),
+`replace_observed` rebuilds the `SemML` term with the new data and carries the `MyRidge` term over
+as-is. The `recompute_observed_state` keyword is likewise accepted and ignored.
+
+If your regularizer *does* need to know about the data, the idiomatic solution is to make it a
+`SemLoss` (so it owns an `observed` part and participates in the rebuild) rather than to specialize
+`replace_observed` on a plain `AbstractLoss`.
+
+### When you need a custom method
+
+You need to specialize `replace_observed` when your loss term **precomputes and stores a quantity
+derived from the observed data**. The default mechanism inherits that quantity from the `refloss`,
+so after swapping in new data the cached value would be stale.
+
+[`SemWLS`](@ref) is the canonical example. Its weight matrix `V` defaults to the GLS weights computed
+from the *observed* covariance matrix and is stored on the term. If `replace_observed` simply reused
+`refloss.V`, the new term would weight the new data with the old data's weights. `SemWLS` therefore
+overrides `replace_observed` to recompute the weights from the new data by default, while exposing a
+`recompute_observed_state` keyword to opt out:
+
+```julia
+# src/loss/WLS/WLS.jl
+function replace_observed(
+ loss::SemWLS,
+ new_observed::SemObserved;
+ recompute_observed_state::Bool = true,
+)
+ return SemWLS(
+ new_observed,
+ implied(loss),
+ loss;
+ # pass `nothing` to recompute from the new data, or reuse the old matrices
+ wls_weight_matrix = recompute_observed_state ? nothing : loss.V,
+ wls_weight_matrix_mean = recompute_observed_state ? nothing : loss.V_μ,
+ )
+end
+```
+
+Note how the override still goes through the three-argument constructor with `loss` as `refloss`,
+so all *other* configuration (e.g. the choice of approximate vs. analytic Hessian) is still
+inherited automatically — the custom method only intervenes for the observed-dependent caches.
+
+The `recompute_observed_state` keyword is a convention shared by all `replace_observed` methods: it
+is forwarded from the model-level call down to every loss term, and terms without observed-dependent
+caches simply ignore it.
diff --git a/docs/src/developer/optimizer.md b/docs/src/developer/optimizer.md
index 9e01ac87c..538d9e0f5 100644
--- a/docs/src/developer/optimizer.md
+++ b/docs/src/developer/optimizer.md
@@ -1,74 +1,75 @@
# Custom optimizer types
-The optimizer part of a model connects it to the optimization backend.
-Let's say we want to implement a new optimizer as `SemOptimizerName`. The first part of the implementation is very similar to loss functions, so we just show the implementation of `SemOptimizerOptim` here as a reference:
+The optimizer part of a model connects it to the optimization engine.
+Let's say we want to implement a new optimizer as `SemOptimizerMyopt`.
```julia
############################################################################################
### Types and Constructor
############################################################################################
-mutable struct SemOptimizerName{A, B} <: SemOptimizer{:Name}
+struct SemOptimizerMyopt{A, B} <: SemOptimizer{:Myopt}
algorithm::A
options::B
end
-SemOptimizer{:Name}(args...; kwargs...) = SemOptimizerName(args...; kwargs...)
+SEM.sem_optimizer_subtype(::Val{:Myopt}) = SemOptimizerMyopt
-SemOptimizerName(;
- algorithm = LBFGS(),
- options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8),
+SemOptimizerMyopt(;
+ algorithm = ...,
+ options = ...,
kwargs...,
-) = SemOptimizerName(algorithm, options)
+) = SemOptimizerMyopt(algorithm, options)
-############################################################################################
-### Recommended methods
-############################################################################################
-
-update_observed(optimizer::SemOptimizerName, observed::SemObserved; kwargs...) = optimizer
+struct MyoptResult{O <: SemOptimizerMyopt} <: SEM.SemOptimizerResult{O}
+ optimizer::O
+ ...
+end
############################################################################################
### additional methods
############################################################################################
-algorithm(optimizer::SemOptimizerName) = optimizer.algorithm
-options(optimizer::SemOptimizerName) = optimizer.options
+options(optimizer::SemOptimizerMyopt) = optimizer.options
```
-Note that your optimizer is a subtype of `SemOptimizer{:Name}`, where you can choose a `:Name` that can later be used as a keyword argument to `fit(engine = :Name)`.
-Similarly, `SemOptimizer{:Name}(args...; kwargs...) = SemOptimizerName(args...; kwargs...)` should be defined as well as a constructor that uses only keyword arguments:
+Note that `SemOptimizerMyopt` is defined as a subtype of [`SemOptimizer{:Myopt}`](@ref SEM.SemOptimizer),
+and `SEM.sem_optimizer_subtype(::Val{:Myopt})` returns `SemOptimizerMyopt`.
+This instructs *SEM.jl* to use `SemOptimizerMyopt` when `:Myopt` is specified as the engine for
+model fitting: `fit(..., engine = :Myopt)`.
-```julia
-SemOptimizerName(;
- algorithm = LBFGS(),
- options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8),
- kwargs...,
-) = SemOptimizerName(algorithm, options)
-```
-A method for `update_observed` and additional methods might be usefull, but are not necessary.
-
-Now comes the substantive part: We need to provide a method for `fit`:
+Now comes the essential part: we need to provide the [`fit`](@ref) method with `SemOptimizerMyopt`
+as the first positional argument.
```julia
function fit(
- optim::SemOptimizerName,
+ optim::SemOptimizerMyopt,
model::AbstractSem,
start_params::AbstractVector;
kwargs...,
)
- optimization_result = ...
+ # ... prepare the Myopt optimization problem
- ...
+ myopt_res = ... # fit the problem with the Myopt engine
+ minimum = ... # extract the minimum from myopt_res
+ minimizer = ... # extract the solution (parameter estimates)
+ optim_result = MyoptResult(optim, myopt_res, ...) # store the original Myopt result and params
- return SemFit(minimum, minimizer, start_params, model, optimization_result)
+ return SemFit(minimum, minimizer, start_params, model, optim_result)
end
```
-The method has to return a `SemFit` object that consists of the minimum of the objective at the solution, the minimizer (aka parameter estimates), the starting values, the model and the optimization result (which may be anything you desire for your specific backend).
+This method is responsible for converting the SEM into the format required by your optimization engine,
+running the optimization, extracting the solution and returning the `SemFit` object, which should package:
+* the minimum of the objective at the solution
+* the minimizer (the vector of the SEM parameter estimates)
+* the starting values
+* the SEM model
+* `MyoptResult` object with any relevant engine-specific details you want to preserve
-In addition, you might want to provide methods to access properties of your optimization result:
+In addition, you might want to provide methods to access engine-specific properties stored in `MyoptResult`:
```julia
-optimizer(res::MyOptimizationResult) = ...
-n_iterations(res::MyOptimizationResult) = ...
-convergence(res::MyOptimizationResult) = ...
-```
\ No newline at end of file
+algorithm_name(res::MyoptResult) = ...
+n_iterations(res::MyoptResult) = ...
+convergence(res::MyoptResult) = ...
+```
diff --git a/docs/src/developer/sem.md b/docs/src/developer/sem.md
index c54ff26af..bb077f434 100644
--- a/docs/src/developer/sem.md
+++ b/docs/src/developer/sem.md
@@ -1,13 +1,14 @@
# Custom model types
-The abstract supertype for all models is `AbstractSem`, which has two subtypes, `AbstractSemSingle{O, I, L}` and `AbstractSemCollection`. Currently, there are 2 subtypes of `AbstractSemSingle`: `Sem`, `SemFiniteDiff`. All subtypes of `AbstractSemSingle` should have at least observed, implied, loss and optimizer fields, and share their types (`{O, I, L}`) with the parametric abstract supertype. For example, the `SemFiniteDiff` type is implemented as
+The abstract supertype for all models is [`AbstractSem`](@ref). Currently, there are 2 concrete subtypes:
+`Sem{L <: Tuple}` and `SemFiniteDiff{S <: AbstractSem}`.
+A `Sem` model holds a tuple of `LossTerm`s (each wrapping an `AbstractLoss`) and a vector of parameter labels. Both single-group and multigroup models are represented as `Sem`.
+
+`SemFiniteDiff` wraps any `AbstractSem` and substitutes dedicated gradient/hessian evaluation with finite difference approximation:
```julia
-struct SemFiniteDiff{O <: SemObserved, I <: SemImplied, L <: SemLoss} <:
- AbstractSemSingle{O, I, L}
- observed::O
- implied::I
- loss::L
+struct SemFiniteDiff{S <: AbstractSem} <: AbstractSem
+ model::S
end
```
@@ -17,6 +18,4 @@ Additionally, you can change how objective/gradient/hessian values are computed
evaluate!(objective, gradient, hessian, model::SemFiniteDiff, params) = ...
```
-Additionally, we can define constructors like the one in `"src/frontend/specification/Sem.jl"`.
-
-It is also possible to add new subtypes for `AbstractSemCollection`.
\ No newline at end of file
+Additionally, we can define constructors like the one in `"src/frontend/specification/Sem.jl"`.
\ No newline at end of file
diff --git a/docs/src/internals/files.md b/docs/src/internals/files.md
deleted file mode 100644
index 4c2338393..000000000
--- a/docs/src/internals/files.md
+++ /dev/null
@@ -1,27 +0,0 @@
-# Files
-
-We briefly describe the file and folder structure of the package.
-
-## Source code
-
-Source code is in the `"src"` folder:
-
-`"src"`
-- `"StructuralEquationModels.jl"` defines the module and the exported objects
-- `"types.jl"` defines all abstract types and the basic type hierarchy
-- `"objective_gradient_hessian.jl"` contains methods for computing objective, gradient and hessian values for different model types as well as generic fallback methods
-- The folders `"observed"`, `"implied"`, and `"loss"` contain implementations of specific subtypes (for example, the `"loss"` folder contains a file `"ML.jl"` that implements the `SemML` loss function).
-- `"optimizer"` contains connections to different optimization backends (aka methods for `fit`)
- - `"optim.jl"`: connection to the `Optim.jl` package
-- `"frontend"` contains user-facing functions
- - `"specification"` contains functionality for model specification
- - `"fit"` contains functionality for model assessment, like fit measures and standard errors
-- `"additional_functions"` contains helper functions for simulations, loading artifacts (example data) and various other things
-
-Code for the package extentions can be found in the `"ext"` folder:
-- `"SEMNLOptExt"` for connection to `NLopt.jl`.
-- `"SEMProximalOptExt"` for connection to `ProximalAlgorithms.jl`.
-
-## Tests and Documentation
-
-Tests are in the `"test"` folder, documentation in the `"docs"` folder.
\ No newline at end of file
diff --git a/docs/src/internals/internals.md b/docs/src/internals/internals.md
deleted file mode 100644
index 18f82dbca..000000000
--- a/docs/src/internals/internals.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# Internals and Design
-
-On the following pages, we document some technical information about the package. Those informations are no prerequisite for extending the package (as decribed in the developer documentation)!, but they may be useful.
\ No newline at end of file
diff --git a/docs/src/internals/types.md b/docs/src/internals/types.md
deleted file mode 100644
index e70a52ca4..000000000
--- a/docs/src/internals/types.md
+++ /dev/null
@@ -1,13 +0,0 @@
-# Type hierarchy
-
-The type hierarchy is implemented in `"src/types.jl"`.
-
-`AbstractSem`: the most abstract type in our package
-- `AbstractSemSingle{O, I, L} <: AbstractSem` is an abstract parametric type that is a supertype of all single models
- - `Sem`: models that do not need automatic differentiation or finite difference approximation
- - `SemFiniteDiff`: models whose gradients and/or hessians should be computed via finite difference approximation
-- `AbstractSemCollection <: AbstractSem` is an abstract supertype of all models that contain multiple `AbstractSem` submodels
-
-Every `AbstractSemSingle` has to have `SemObserved`, `SemImplied`, and `SemLoss` fields (and can have additional fields).
-
-`SemLoss` is a container for multiple `SemLossFunctions`.
\ No newline at end of file
diff --git a/docs/src/performance/mixed_differentiation.md b/docs/src/performance/mixed_differentiation.md
index b7ae333b5..f33fa6ab2 100644
--- a/docs/src/performance/mixed_differentiation.md
+++ b/docs/src/performance/mixed_differentiation.md
@@ -2,22 +2,20 @@
This way of specifying our model is not ideal, however, because now also the maximum likelihood loss function lives inside a `SemFiniteDiff` model, and this means even though we have defined analytical gradients for it, we do not make use of them.
-A more efficient way is therefore to specify our model as an ensemble model:
+A more efficient way is therefore to specify our model as a combined model with multiple loss terms:
```julia
-model_ml = Sem(
- specification = partable,
- data = data,
- loss = SemML
+ml_term = SemML(
+ SemObservedData(data = data, specification = partable),
+ RAMSymbolic(partable)
)
-model_ridge = SemFiniteDiff(
- specification = partable,
- data = data,
- loss = myridge
+ridge_term = SemRidge(
+ α_ridge = 0.01,
+ which_ridge = params(ml_term)
)
-model_ml_ridge = SemEnsemble(model_ml, model_ridge)
+model_ml_ridge = Sem(ml_term, ridge_term)
model_ml_ridge_fit = fit(model_ml_ridge)
```
diff --git a/docs/src/performance/parametric.md b/docs/src/performance/parametric.md
index 9c6be382b..ce1781ab6 100644
--- a/docs/src/performance/parametric.md
+++ b/docs/src/performance/parametric.md
@@ -25,7 +25,7 @@ giving each field a type and adding them as parameters to our type declaration.
Recall our example from [Custom loss functions](@ref):
```julia
-struct Ridge <: SemLossFunction
+struct Ridge <: AbstractLoss
α
I
end
@@ -34,7 +34,7 @@ end
We could also declare it as a parametric type:
```julia
-struct ParametricRidge{X, Y} <: SemLossFunction
+struct ParametricRidge{X, Y} <: AbstractLoss
α::X
I::Y
end
diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md
index d268853f6..a33232ee4 100644
--- a/docs/src/performance/simulation.md
+++ b/docs/src/performance/simulation.md
@@ -3,8 +3,13 @@
## Replace observed data
In simulation studies, a common task is fitting the same model to many different datasets.
It would be a waste of resources to reconstruct the complete model for each dataset.
-We therefore provide the function `replace_observed` to change the `observed` part of a model,
-without necessarily reconstructing the other parts.
+We therefore provide the function [`replace_observed`](@ref) to change the `observed` part
+of a model, without necessarily reconstructing the other parts.
+
+For `SemLoss` terms, `replace_observed()` constructs the new loss by passing the new observed
+data, the current implied state, and the current loss (as `refloss`) to the appropriate loss
+constructor. The new loss term therefore shares the implied state with the original one, as well
+as loss-specific settings and, potentially, the internal state.
For the [A first model](@ref), you would use it as
@@ -40,7 +45,7 @@ end
partable = ParameterTable(
graph,
- latent_vars = latent_vars,
+ latent_vars = latent_vars,
observed_vars = observed_vars
)
```
@@ -57,28 +62,56 @@ model = Sem(
data = data_1
)
-model_updated = replace_observed(model; data = data_2, specification = partable)
+model_updated = replace_observed(model, data_2)
```
-If you are building your models by parts, you can also update each part seperately with the function `update_observed`.
-For example,
+`replace_observed` accepts a data matrix, a `DataFrame`, or a ready-made `SemObserved` object, and
+works for multigroup/collection models too (pass a `Dict` mapping term ids to data, or a `DataFrame`
+together with a `semterm_column`). See the [API](@ref simulation_api) below for all signatures.
+
+### Recomputing observed-dependent state
+
+Some loss terms cache quantities that are derived from the observed data. The most prominent example
+is weighted least squares ([`SemWLS`](@ref)), whose weight matrix defaults to the GLS weights computed
+from the observed covariance matrix. By default, `replace_observed` **recomputes** these quantities
+from the new data, which is what you want in most simulation studies:
```@example replace_observed
+model_wls = Sem(
+ specification = partable,
+ data = data_1,
+ implied = RAMSymbolic,
+ vech = true,
+ loss = SemWLS,
+)
-new_observed = SemObservedData(;data = data_2, specification = partable)
+# weight matrix recomputed from `data_2` (default)
+model_wls_updated = replace_observed(model_wls, data_2)
+```
-my_optimizer = SemOptimizerOptim()
+If instead you want to keep the original observed-dependent state — e.g. fit every replication with
+the *same* fixed weight matrix — pass `recompute_observed_state = false`:
-new_optimizer = update_observed(my_optimizer, new_observed)
+```@example replace_observed
+model_wls_fixed = replace_observed(model_wls, data_2; recompute_observed_state = false)
```
+Loss terms without observed-dependent caches (such as `SemML`) ignore this keyword.
+
+!!! tip "Simulating with your own loss type"
+ If you run simulation studies with a custom loss function, see
+ [Supporting `replace_observed`](@ref) in the [Custom loss functions](@ref) chapter. It explains
+ how `replace_observed` rebuilds a loss term, when the default behavior is enough, and how to
+ implement a custom `replace_observed` method (with `recompute_observed_state`) if your loss
+ caches quantities derived from the observed data.
+
## Multithreading
!!! danger "Thread safety"
*This is only relevant when you are planning to fit updated models in parallel*
-
- Models generated by `replace_observed` may share the same objects in memory (e.g. some parts of
+
+ Models generated by `replace_observed` may share the same objects in memory (e.g. some parts of
`model` and `model_updated` are the same objects in memory.)
- Therefore, fitting both of these models in parallel will lead to **race conditions**,
+ Therefore, fitting both of these models in parallel will lead to **race conditions**,
possibly crashing your computer.
To avoid these problems, you should copy `model` before updating it.
@@ -90,7 +123,7 @@ model1 = Sem(
data = data_1
)
-model2 = deepcopy(replace_observed(model; data = data_2, specification = partable))
+model2 = deepcopy(replace_observed(model, data_2))
models = [model1, model2]
fits = Vector{SemFit}(undef, 2)
@@ -100,9 +133,8 @@ Threads.@threads for i in 1:2
end
```
-## API
+## [API](@id simulation_api)
```@docs
replace_observed
-update_observed
```
\ No newline at end of file
diff --git a/docs/src/tutorials/backends/nlopt.md b/docs/src/tutorials/backends/nlopt.md
index feb5c8f48..8e6b83bee 100644
--- a/docs/src/tutorials/backends/nlopt.md
+++ b/docs/src/tutorials/backends/nlopt.md
@@ -1,31 +1,21 @@
# Using NLopt.jl
-[`SemOptimizerNLopt`](@ref) implements the connection to `NLopt.jl`.
-It is only available if the `NLopt` package is loaded alongside `StructuralEquationModels.jl` in the running Julia session.
-It takes a bunch of arguments:
+When [`NLopt.jl`](https://github.com/jump-dev/NLopt.jl) is loaded in the running Julia session,
+it can be used by the [`SemOptimizer`](@ref) by specifying `engine = :NLopt`
+(see [NLopt-specific options](@ref SEMNLOptExt.SemOptimizerNLopt)).
+Among other things, `NLopt` enables constrained optimization of SEMs, which is
+explained in the [Constrained optimization](@ref) section.
-```julia
- • algorithm: optimization algorithm
-
- • options::Dict{Symbol, Any}: options for the optimization algorithm
-
- • local_algorithm: local optimization algorithm
-
- • local_options::Dict{Symbol, Any}: options for the local optimization algorithm
-
- • equality_constraints::Vector{NLoptConstraint}: vector of equality constraints
-
- • inequality_constraints::Vector{NLoptConstraint}: vector of inequality constraints
-```
-Constraints are explained in the section on [Constrained optimization](@ref).
-
-The defaults are LBFGS as the optimization algorithm and the standard options from `NLopt.jl`.
-We can choose something different:
+We can override the default *NLopt* algorithm (LFBGS) and instead use
+the *augmented lagrangian* method with LBFGS as the *local* optimization algorithm,
+stop at a maximum of 200 evaluations and use a relative tolerance of
+the objective value of `1e-6` as the stopping criterion for the local algorithm:
```julia
using NLopt
-my_optimizer = SemOptimizerNLopt(;
+my_optimizer = SemOptimizer(;
+ engine = :NLopt,
algorithm = :AUGLAG,
options = Dict(:maxeval => 200),
local_algorithm = :LD_LBFGS,
@@ -33,15 +23,14 @@ my_optimizer = SemOptimizerNLopt(;
)
```
-This uses an augmented lagrangian method with LBFGS as the local optimization algorithm, stops at a maximum of 200 evaluations and uses a relative tolerance of the objective value of `1e-6` as the stopping criterion for the local algorithm.
-
To see how to use the optimizer to actually fit a model now, check out the [Model fitting](@ref) section.
-In the NLopt docs, you can find explanations about the different [algorithms](https://nlopt.readthedocs.io/en/latest/NLopt_Algorithms/) and a [tutorial](https://nlopt.readthedocs.io/en/latest/NLopt_Introduction/) that also explains the different options.
+In the *NLopt* docs, you can find details about the [optimization algorithms](https://nlopt.readthedocs.io/en/latest/NLopt_Algorithms/),
+and the [tutorial](https://nlopt.readthedocs.io/en/latest/NLopt_Introduction/) that demonstrates how to tweak their behavior.
-To choose an algorithm, just pass its name without the 'NLOPT\_' prefix (for example, 'NLOPT\_LD\_SLSQP' can be used by passing `algorithm = :LD_SLSQP`).
+To choose an algorithm, just pass its name without the `NLOPT_` prefix (for example, `NLOPT_LD_SLSQP` can be used by passing `algorithm = :LD_SLSQP`).
-The README of the [julia package](https://github.com/JuliaOpt/NLopt.jl) may also be helpful, and provides a list of options:
+The *README* of [*NLopt.jl*](https://github.com/JuliaOpt/NLopt.jl) may also be helpful, and provides a list of options:
- `algorithm`
- `stopval`
diff --git a/docs/src/tutorials/backends/optim.md b/docs/src/tutorials/backends/optim.md
index cf287e773..b1b07885e 100644
--- a/docs/src/tutorials/backends/optim.md
+++ b/docs/src/tutorials/backends/optim.md
@@ -1,23 +1,23 @@
# Using Optim.jl
-[`SemOptimizerOptim`](@ref) implements the connection to `Optim.jl`.
-It takes two arguments, `algorithm` and `options`.
-The defaults are LBFGS as the optimization algorithm and the standard options from `Optim.jl`.
-We can load the `Optim` and `LineSearches` packages to choose something different:
+[*Optim.jl*](https://github.com/JuliaNLSolvers/Optim.jl) is the default optimization engine of *SEM.jl*,
+see [`SEM.SemOptimizerOptim`](@ref) for a full list of its parameters.
+It defaults to the LBFGS optimization, but we can load the `Optim` and `LineSearches` packages
+and specify BFGS (!not L-BFGS) with a back-tracking linesearch and Hager-Zhang initial step length guess:
```julia
using Optim, LineSearches
-my_optimizer = SemOptimizerOptim(
+my_optimizer = SemOptimizer(
algorithm = BFGS(
- linesearch = BackTracking(order=3),
+ linesearch = BackTracking(order=3),
alphaguess = InitialHagerZhang()
- ),
- options = Optim.Options(show_trace = true)
- )
+ ),
+ options = Optim.Options(show_trace = true)
+)
```
-This optimizer will use BFGS (!not L-BFGS) with a back tracking linesearch and a certain initial step length guess. Also, the trace of the optimization will be printed to the console.
+Note that we used `options` to print the optimization progress to the console.
To see how to use the optimizer to actually fit a model now, check out the [Model fitting](@ref) section.
diff --git a/docs/src/tutorials/collection/collection.md b/docs/src/tutorials/collection/collection.md
index f60b7312c..423ac5bc7 100644
--- a/docs/src/tutorials/collection/collection.md
+++ b/docs/src/tutorials/collection/collection.md
@@ -1,31 +1,60 @@
# Collections
-With StructuralEquationModels.jl, you can fit weighted sums of structural equation models.
-The most common use case for this are [Multigroup models](@ref).
-Another use case may be optimizing the sum of loss functions for some of which you do know the analytic gradient, but not for others.
-In this case, you can optimize the sum of a `Sem` and a `SemFiniteDiff` (or any other differentiation method).
+With *StructuralEquationModels.jl*, you can fit weighted sums of structural equation models.
+The most common use case for this are [Multigroup models](@ref).
+Another use case may be optimizing the sum of loss functions for some of which you do know the analytic gradient, but not for others.
+In this case, [`FiniteDiffWrapper`](@ref StructuralEquationModels.FiniteDiffWrapper) can generate a wrapper around the specific `SemLoss` term. The wrapper loss term will
+only use the objective of the original term to calculate its gradient using finite difference approximation.
-To use this feature, you have to construct a `SemEnsemble` model, which is actually quite easy:
+```julia
+loss_1 = SemML(observed_1, implied_1)
+loss_2 = SemML(observed_2, implied_2)
+loss_2_findiff = FiniteDiffWrapper(loss_2)
+```
+
+To construct `Sem` from the the individual `SemLoss` (or other `AbstractLoss`) terms, they are
+just passed to the `Sem` constructor:
```julia
-# models
-model_1 = Sem(...)
+model = Sem(loss_1, loss_2)
+model_findiff = Sem(loss_1, loss_2_findiff)
+```
-model_2 = SemFiniteDiff(...)
+It is also possible to use finite difference for the entire `Sem` model:
-model_3 = Sem(...)
+```julia
+model_findiff2 = FiniteDiffWrapper(model)
+```
-model_ensemble = SemEnsemble(model_1, model_2, model_3)
+The weighting scheme of the SEM loss terms is specified using the `default_sem_weights` argument of the `Sem` constructor.
+The available schemes are:
+- `:nsamples_corrected` (the default): like `:nsamples`, but applies a loss-type-specific correction
+ to the sample counts (e.g. ``N_{term} - 1`` for maximum likelihood and weighted least squares). For FIML the correction is zero, so it coincides with `:nsamples`,
+- `:nsamples`: weights each SEM term by ``N_{term}/N_{total}``, i.e. by the (uncorrected) number of
+ observations in its data,
+- `:uniform`: weights each of the ``k`` SEM terms by ``1/k``,
+- `:one`: leaves all SEM terms unweighted.
+
+The weights for the loss terms (both SEM and regularization) can also be explicitly specified using the pair syntax `loss => weight`:
+
+```julia
+model_weighted = Sem(loss_1 => 0.5, loss_2 => 1.0)
```
-So you just construct the individual models (however you like) and pass them to `SemEnsemble`.
-You may also pass a vector of weigths to `SemEnsemble`. By default, those are set to ``N_{model}/N_{total}``, i.e. each model is weighted by the number of observations in it's data (which matches the formula for multigroup models).
+`Sem` supports assigning a unique identifier to each loss term, which is useful for complex multi-term models.
+The syntax is `id => loss`, or `id => loss => weight`:
-Multigroup models can also be specified via the graph interface; for an example, see [Multigroup models](@ref).
+```julia
+model2 = Sem(:main => loss_1, :alt => loss_2)
+model2_weighted = Sem(:main => loss_1 => 0.5, :alt => loss_2 => 1.0)
+```
# API - collections
```@docs
-SemEnsemble
-AbstractSemCollection
+Sem
+SemFiniteDiff
+AbstractSem
+StructuralEquationModels.LossTerm
+StructuralEquationModels.FiniteDiffWrapper
```
\ No newline at end of file
diff --git a/docs/src/tutorials/collection/multigroup.md b/docs/src/tutorials/collection/multigroup.md
index 16d3dcd77..d085b04a9 100644
--- a/docs/src/tutorials/collection/multigroup.md
+++ b/docs/src/tutorials/collection/multigroup.md
@@ -4,19 +4,20 @@
using StructuralEquationModels
```
-As an example, we will fit the model from [the `lavaan` tutorial](https://lavaan.ugent.be/tutorial/groups.html) with loadings constrained to equality across groups.
+As an example, we will fit the model from [the `lavaan` tutorial](https://lavaan.ugent.be/tutorial/groups.html)
+with loadings constrained to equality across groups.
-We first load the example data.
+We first load the example data.
We have to make sure that the column indicating the group (here called `school`) is a vector of `Symbol`s, not strings - so we convert it.
```@setup mg
dat = example_data("holzinger_swineford")
-dat.school = ifelse.(dat.school .== "Pasteur", :Pasteur, :Grant_White)
+dat.school = Symbol.(replace.(dat.school, "-" => "_"))
```
```julia
dat = example_data("holzinger_swineford")
-dat.school = ifelse.(dat.school .== "Pasteur", :Pasteur, :Grant_White)
+dat.school = Symbol.(replace.(dat.school, "-" => "_"))
```
We then specify our model via the graph interface:
@@ -51,7 +52,7 @@ graph = @StenoGraph begin
end
```
-You can pass multiple arguments to `fix()` and `label()` for each group. Parameters with the same label (within and across groups) are constrained to be equal. To fix a parameter in one group, but estimate it freely in the other, you may write `fix(NaN, 4.3)`.
+You can pass multiple arguments to `fixed()` and `label()` for each group. Parameters with the same label (within and across groups) are constrained to be equal. To fix a parameter in one group, but estimate it freely in the other, you may write `fixed(NaN, 4.3)`.
You can then use the resulting graph to specify an `EnsembleParameterTable`
@@ -59,20 +60,19 @@ You can then use the resulting graph to specify an `EnsembleParameterTable`
groups = [:Pasteur, :Grant_White]
partable = EnsembleParameterTable(
- graph,
+ graph,
observed_vars = observed_vars,
latent_vars = latent_vars,
groups = groups)
```
-The parameter table can be used to create a `SemEnsemble` model:
+The parameter table can be used to create a multigroup `Sem` model:
```@example mg; ansicolor = true
-model_ml_multigroup = SemEnsemble(
+model_ml_multigroup = Sem(
specification = partable,
data = dat,
- column = :school,
- groups = groups)
+ semterm_column = :school)
```
!!! note "A different way to specify"
@@ -86,4 +86,10 @@ update_estimate!(partable, sem_fit)
details(partable)
```
-Other things you can query about your fitted model (fit measures, standard errors, etc.) are described in the section [Model inspection](@ref) and work the same way for multigroup models.
\ No newline at end of file
+Other things you can query about your fitted model (fit measures, standard errors, etc.) are described in the section [Model inspection](@ref) and work the same way for multigroup models.
+
+# API
+
+```@docs
+EnsembleParameterTable
+```
\ No newline at end of file
diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md
index 035144d62..39f189d2e 100644
--- a/docs/src/tutorials/concept.md
+++ b/docs/src/tutorials/concept.md
@@ -1,58 +1,95 @@
# Our Concept of a Structural Equation Model
-In our package, every Structural Equation Model (`Sem`) consists of three parts (four, if you count the optimizer):
+In our package, a structural equation model (a [`Sem`](@ref)) is built from one or more **loss terms**.
+Fitting the model means finding the parameters that minimize the (weighted) sum of all of its loss terms.
+This simple idea is remarkably general: within the same structure it covers a single SEM fit by maximum
+likelihood, a regularized SEM (e.g. maximum likelihood plus a ridge penalty), and multigroup models
+(one SEM term per group).
-Those parts are interchangable building blocks (like 'Legos'), i.e. there are different pieces available you can choose as the `observed` slot of the model, and stick them together with other pieces that can serve as the `implied` part.
+A loss term is anything of type [`AbstractLoss`](@ref) — a function that maps the model parameters to a
+number that should be minimized. There are two kinds of loss terms:
-The `observed` part is for observed data, the `implied` part is what the model implies about your data (e.g. the model implied covariance matrix), and the loss part compares the observed data and implied properties (e.g. weighted least squares difference between the observed and implied covariance matrix).
-The optimizer part is not part of the model itself, but it is needed to fit the model as it connects to the optimization backend (e.g. the type of optimization algorithm used).
+- **SEM loss functions** ([`SemLoss`](@ref)), such as [`SemML`](@ref), [`SemWLS`](@ref) and [`SemFIML`](@ref),
+ measure how well the model explains the data. To do so, each `SemLoss` *bundles its own observed part
+ (the data) and implied part (what the model implies about the data)*. They are the heart of a SEM.
+- **Other loss functions**, such as the regularization terms [`SemRidge`](@ref) and [`SemConstant`](@ref),
+ depend only on the parameters and therefore need neither an observed nor an implied part.
-For example, to build a model for maximum likelihood estimation with the NLopt optimization suite as a backend you would choose `SemML` as a loss function and `SemOptimizerNLopt` as the optimizer.
+Because a model is just a (weighted) sum of loss terms, you can freely combine them.
+For example, ridge-regularized full information maximum likelihood estimation is a model with two loss terms,
+a [`SemFIML`](@ref) term and a [`SemRidge`](@ref) term. A two-group model is a model with two [`SemML`](@ref)
+terms, one per group, weighted by the respective sample sizes.
-As you can see, a model can have as many loss functions as you want it to have. We always optimize over their (weighted) sum. So to build a model for ridge regularized full information maximum likelihood estimation, you would choose two loss functions, `SemFIML` and `SemRidge`.
+All models are subtypes of [`AbstractSem`](@ref). The default [`Sem`](@ref) computes the weighted sum of its
+loss terms together with their (analytic) gradients. [`SemFiniteDiff`](@ref) is an alternative that
+approximates the gradient with finite differences, which is useful for loss functions that do not provide an
+analytic gradient.
-In julia, everything has a type. To make more precise which objects can be used as the different building blocks, we require them to have a certain type:
+## The parts of a SEM loss
+
+Each SEM loss function ([`SemLoss`](@ref)) is itself composed of interchangeable building blocks (like 'Legos'):
+an *observed* part and an *implied* part. To make precise which objects can play each role, we require them to
+have a certain type:
-So everything that can be used as the 'observed' part has to be of type `SemObserved`.
+So everything that can serve as the *observed* part has to be of type [`SemObserved`](@ref), everything that can
+serve as the *implied* part has to be of type [`SemImplied`](@ref), and the loss function that combines them is a
+[`SemLoss`](@ref). To fit the model, you additionally choose a [`SemOptimizer`](@ref); it connects to the
+numerical optimization backend but is not itself part of the model.
Here is an overview on the available building blocks:
-|[`SemObserved`](@ref) | [`SemImplied`](@ref) | [`SemLossFunction`](@ref) | [`SemOptimizer`](@ref) |
-|---------------------------------|-----------------------|---------------------------|-------------------------------|
-| [`SemObservedData`](@ref) | [`RAM`](@ref) | [`SemML`](@ref) | [`SemOptimizerOptim`](@ref) |
-| [`SemObservedCovariance`](@ref) | [`RAMSymbolic`](@ref) | [`SemWLS`](@ref) | [`SemOptimizerNLopt`](@ref) |
-| [`SemObservedMissing`](@ref) | [`ImpliedEmpty`](@ref)| [`SemFIML`](@ref) | |
-| | | [`SemRidge`](@ref) | |
-| | | [`SemConstant`](@ref) | |
+|[`SemObserved`](@ref) | [`SemImplied`](@ref) | [`AbstractLoss`](@ref) | [`SemOptimizer`](@ref) |
+|---------------------------------|-----------------------|---------------------------|----------------------------|
+| [`SemObservedData`](@ref) | [`RAM`](@ref) | [`SemML`](@ref) | [:Optim](@ref StructuralEquationModels.SemOptimizerOptim) |
+| [`SemObservedCovariance`](@ref) | [`RAMSymbolic`](@ref) | [`SemWLS`](@ref) | [:NLopt](@ref SEMNLOptExt.SemOptimizerNLopt) |
+| [`SemObservedMissing`](@ref) | [`ImpliedEmpty`](@ref)| [`SemFIML`](@ref) | [:Proximal](@ref SEMProximalOptExt.SemOptimizerProximal) |
+| | | [`SemRidge`](@ref) | |
+| | | [`SemConstant`](@ref) | |
-The rest of this page explains the building blocks for each part. First, we explain every part and give an overview on the different options that are available. After that, the [API - model parts](@ref) section serves as a reference for detailed explanations about the different options.
-(How to stick them together to a final model is explained in the section on [Model Construction](@ref).)
+The rest of this page explains each building block and the available options. After that, the
+[API - model parts](@ref) section serves as a reference for detailed explanations.
+(How to stick the building blocks together into a final model is explained in the section on
+[Model Construction](@ref).)
## The observed part aka [`SemObserved`](@ref)
-The *observed* part contains all necessary information about the observed data. Currently, we have three options: [`SemObservedData`](@ref) for fully observed datasets, [`SemObservedCovariance`](@ref) for observed covariances (and means) and [`SemObservedMissing`](@ref) for data that contains missing values.
+The *observed* part contains all necessary information about the observed data, and pre-computes the statistics
+a loss function needs from it — for example the observed covariance matrix, or the different patterns of
+missingness used for full information maximum likelihood (FIML) estimation.
+Currently, we have three options: [`SemObservedData`](@ref) for fully observed datasets,
+[`SemObservedCovariance`](@ref) for observed covariances (and means) and [`SemObservedMissing`](@ref) for data
+that contains missing values.
## The implied part aka [`SemImplied`](@ref)
-The *implied* part is what your model implies about the data, for example, the model-implied covariance matrix.
-There are two options at the moment: [`RAM`](@ref), which uses the reticular action model to compute the model implied covariance matrix, and [`RAMSymbolic`](@ref) which does the same but symbolically pre-computes part of the model, which increases subsequent performance in model fitting (see [Symbolic precomputation](@ref)). There is also a third option, [`ImpliedEmpty`](@ref) that can serve as a 'placeholder' for models that do not need an implied part.
-
-## The loss part aka `SemLoss`
-The loss part specifies the objective that is optimized to find the parameter estimates.
-If it contains more then one loss function (aka [`SemLossFunction`](@ref))), we find the parameters by minimizing the sum of loss functions (for example in maximum likelihood estimation + ridge regularization).
+The *implied* part defines how the model-implied statistics (for example, the model-implied covariance matrix
+and mean vector) are computed from the parameters.
+There are two options at the moment: [`RAM`](@ref), which uses the reticular action model to compute the model
+implied covariance matrix, and [`RAMSymbolic`](@ref) which does the same but symbolically pre-computes part of
+the model, which increases subsequent performance in model fitting (see [Symbolic precomputation](@ref)). There
+is also a third option, [`ImpliedEmpty`](@ref) that can serve as a 'placeholder' for loss terms that do not need
+an implied part.
+
+## The loss functions aka [`AbstractLoss`](@ref)
+The loss terms specify the objective that is minimized to find the parameter estimates; a model minimizes the
+(weighted) sum of all its loss terms.
+SEM loss functions ([`SemLoss`](@ref)) compare what the model implies to the observed data, while regularization
+terms depend only on the parameters.
Available loss functions are
- [`SemML`](@ref): maximum likelihood estimation
- [`SemWLS`](@ref): weighted least squares estimation
- [`SemFIML`](@ref): full-information maximum likelihood estimation
- [`SemRidge`](@ref): ridge regularization
+- [`SemConstant`](@ref): adds a constant to the objective
-## The optimizer part aka `SemOptimizer`
-The optimizer part of a model connects to the numerical optimization backend used to fit the model.
-It can be used to control options like the optimization algorithm, linesearch, stopping criteria, etc.
-There are currently three available backends, [`SemOptimizerOptim`](@ref) connecting to the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) backend, [`SemOptimizerNLopt`](@ref) connecting to the [NLopt.jl](https://github.com/JuliaOpt/NLopt.jl) backend and [`SemOptimizerProximal`](@ref) connecting to [ProximalAlgorithms.jl](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl).
+## The optimizer aka [`SemOptimizer`](@ref)
+The optimizer connects to the numerical optimization backend used to fit the model. It is not part of the model
+itself, but it is chosen when fitting (see [Model fitting](@ref)).
+It can be used to control options like the optimization algorithm, linesearch, stopping criteria, etc.
+There are currently three available engines (i.e., backends used to carry out the numerical optimization), [`:Optim`](@ref StructuralEquationModels.SemOptimizerOptim) connecting to the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) backend, [`:NLopt`](@ref SEMNLOptExt.SemOptimizerNLopt) connecting to the [NLopt.jl](https://github.com/JuliaOpt/NLopt.jl) backend and [`:Proximal`](@ref SEMProximalOptExt.SemOptimizerProximal) connecting to [ProximalAlgorithms.jl](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl).
For more information about the available options see also the tutorials about [Using Optim.jl](@ref) and [Using NLopt.jl](@ref), as well as [Constrained optimization](@ref) and [Regularization](@ref) .
# What to do next
@@ -75,6 +112,7 @@ SemObservedMissing
samples
observed_vars
SemSpecification
+em_mvn
```
## implied
@@ -89,8 +127,8 @@ ImpliedEmpty
## loss functions
```@docs
+AbstractLoss
SemLoss
-SemLossFunction
SemML
SemFIML
SemWLS
@@ -101,8 +139,11 @@ SemConstant
## optimizer
```@docs
+optimizer_engines
+optimizer_engine
+optimizer_engine_doc
SemOptimizer
-SemOptimizerOptim
-SemOptimizerNLopt
-SemOptimizerProximal
+SEM.SemOptimizerOptim
+SEMNLOptExt.SemOptimizerNLopt
+SEMProximalOptExt.SemOptimizerProximal
```
\ No newline at end of file
diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md
index 338803cb3..e50fd2ef5 100644
--- a/docs/src/tutorials/constraints/constraints.md
+++ b/docs/src/tutorials/constraints/constraints.md
@@ -1,10 +1,15 @@
# Constrained optimization
-## Using the NLopt backend
+*SEM.jl* allows to fit models with additional constraints imposed on the parameters.
+
+## Using the NLopt engine
+
+*NLopt.jl* is one of *SEM.jl* optimization engines that supports constrained optimization.
+In the example below we show how to specify constraints for the *SEM* model when using *NLopt*.
### Define an example model
-Let's revisit our model from [A first model](@ref):
+Let's revisit our model from [A first model](@ref) and fit it first without constraints:
```@example constraints
using StructuralEquationModels
@@ -38,7 +43,7 @@ end
partable = ParameterTable(
graph,
- latent_vars = latent_vars,
+ latent_vars = latent_vars,
observed_vars = observed_vars)
data = example_data("political_democracy")
@@ -57,39 +62,40 @@ details(partable)
### Define the constraints
-Let's introduce some constraints:
+Let's introduce some constraints (they are not based on any real properties of the underlying study and serve only as an example):
1. **Equality constraint**: The covariances `y3 ↔ y7` and `y8 ↔ y4` should sum up to `1`.
2. **Inequality constraint**: The difference between the loadings `dem60 → y2` and `dem60 → y3` should be smaller than `0.1`
3. **Bound constraint**: The directed effect from `ind60 → dem65` should be smaller than `0.5`
-(Of course those constaints only serve an illustratory purpose.)
-
-We first need to get the indices of the respective parameters that are invoved in the constraints.
-We can look up their labels in the output above, and retrieve their indices as
+Since *NLopt* does not have access to the SEM parameter names, its constaints are defined on the vector of all SEM parameters.
+We have to look up the indices of the parameters involved in the constraints to construct the respective functions.
```@example constraints
parind = param_indices(model)
parind[:y3y7] # 29
```
-The bound constraint is easy to specify: Just give a vector of upper or lower bounds that contains the bound for each parameter. In our example, only the parameter labeled `:λₗ` has an upper bound, and the number of total parameters is `n_par(model) = 31`, so we define
+The bound constraint is easy to specify: just give a vector of upper or lower bounds for each parameter.
+In our example, only the parameter labeled `:λₗ` has an upper bound, and the number of total parameters is `nparams(model) = 31`, so
```@example constraints
upper_bounds = fill(Inf, 31)
upper_bounds[parind[:λₗ]] = 0.5
```
-The equailty and inequality constraints have to be reformulated to be of the form `x = 0` or `x ≤ 0`:
-1. `y3 ↔ y7 + y8 ↔ y4 - 1 = 0`
-2. `dem60 → y2 - dem60 → y3 - 0.1 ≤ 0`
+The equailty and inequality constraints have to be reformulated in the `f(θ) = 0` or `f(θ) ≤ 0` form,
+where `θ` is the vector of SEM parameters:
+1. `f(θ) = 0`, where `f(θ) = y3 ↔ y7 + y8 ↔ y4 - 1`
+2. `g(θ) ≤ 0`, where `g(θ) = dem60 → y2 - dem60 → y3 - 0.1`
-Now they can be defined as functions of the parameter vector:
+If the optimization algorithm needs gradients, it will pass the `gradient` vector that is of the same size as the parameters,
+and the constraint function has to calculate the gradient in-place.
```@example constraints
parind[:y3y7] # 29
parind[:y8y4] # 30
# θ[29] + θ[30] - 1 = 0.0
-function eq_constraint(θ, gradient)
+function f(θ, gradient)
if length(gradient) > 0
gradient .= 0.0
gradient[29] = 1.0
@@ -101,7 +107,7 @@ end
parind[:λ₂] # 3
parind[:λ₃] # 4
# θ[3] - θ[4] - 0.1 ≤ 0
-function ineq_constraint(θ, gradient)
+function g(θ, gradient)
if length(gradient) > 0
gradient .= 0.0
gradient[3] = 1.0
@@ -111,49 +117,43 @@ function ineq_constraint(θ, gradient)
end
```
-If the algorithm needs gradients at an iteration, it will pass the vector `gradient` that is of the same size as the parameters.
-With `if length(gradient) > 0` we check if the algorithm needs gradients, and if it does, we fill the `gradient` vector with the gradients
-of the constraint w.r.t. the parameters.
-
-In NLopt, vector-valued constraints are also possible, but we refer to the documentation for that.
+In *NLopt*, vector-valued constraints are also possible, but we refer to the documentation for that.
### Fit the model
-We now have everything together to specify and fit our model. First, we specify our optimizer backend as
+Now we can construct the *SemOptimizer* that will use the *NLopt* engine for constrained optimization.
```@example constraints
using NLopt
-constrained_optimizer = SemOptimizerNLopt(
+constrained_optimizer = SemOptimizer(
+ engine = :NLopt,
algorithm = :AUGLAG,
options = Dict(:upper_bounds => upper_bounds, :xtol_abs => 1e-4),
local_algorithm = :LD_LBFGS,
- equality_constraints = NLoptConstraint(;f = eq_constraint, tol = 1e-8),
- inequality_constraints = NLoptConstraint(;f = ineq_constraint, tol = 1e-8),
+ equality_constraints = (f => 1e-8),
+ inequality_constraints = (g => 1e-8),
)
```
-As you see, the equality constraints and inequality constraints are passed as keyword arguments, and the bounds are passed as options for the (outer) optimization algorithm.
-Additionally, for equality and inequality constraints, a feasibility tolerance can be specified that controls if a solution can be accepted, even if it violates the constraints by a small amount.
+As you see, the equality and inequality constraints are passed as keyword arguments, and the bounds are passed as options for the (outer) optimization algorithm.
+Additionally, for equality and inequality constraints, a feasibility tolerance can be specified that controls if a solution can be accepted, even if it violates the constraints by a small amount.
Especially for equality constraints, it is recommended to allow for a small positive tolerance.
In this example, we set both tolerances to `1e-8`.
!!! warning "Convergence criteria"
We have often observed that the default convergence criteria in NLopt lead to non-convergence flags.
Indeed, this example does not convergence with default criteria.
- As you see above, we used a realively liberal absolute tolerance in the optimization parameters of 1e-4.
- This should not be a problem in most cases, as the sampling variance in (almost all) structural equation models
+ As you see above, we used a relatively liberal absolute tolerance in the optimization parameters of 1e-4.
+ This should not be a problem in most cases, as the sampling variance in (almost all) structural equation models
should lead to uncertainty in the parameter estimates that are orders of magnitude larger.
We nontheless recommend choosing a convergence criterion with care (i.e. w.r.t. the scale of your parameters),
inspecting the solutions for plausibility, and comparing them to unconstrained solutions.
-```@example constraints
-model_constrained = Sem(
- specification = partable,
- data = data
-)
+We now have everything to fit our model under constraints:
-model_fit_constrained = fit(constrained_optimizer, model_constrained)
+```@example constraints
+model_fit_constrained = fit(constrained_optimizer, model)
```
As you can see, the optimizer converged (`:XTOL_REACHED`) and investigating the solution yields
@@ -162,14 +162,14 @@ As you can see, the optimizer converged (`:XTOL_REACHED`) and investigating the
update_partable!(
partable,
:estimate_constr,
- param_labels(model_fit_constrained),
- solution(model_fit_constrained),
- )
+ model_fit_constrained,
+ solution(model_fit_constrained),
+)
details(partable)
```
-As we can see, the constrained solution is very close to the original solution (compare the columns estimate and estimate_constr), with the difference that the constrained parameters fulfill their constraints.
+As we can see, the constrained solution is very close to the original solution (compare the columns estimate and estimate_constr), with the difference that the constrained parameters fulfill their constraints.
As all parameters are estimated simultaneously, it is expexted that some unconstrained parameters are also affected (e.g., the constraint on `dem60 → y2` leads to a higher estimate of the residual variance `y2 ↔ y2`).
## Using the Optim.jl backend
diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md
index 6b6b59ac9..afa51eeb0 100644
--- a/docs/src/tutorials/construction/build_by_parts.md
+++ b/docs/src/tutorials/construction/build_by_parts.md
@@ -40,7 +40,7 @@ end
partable = ParameterTable(
graph,
- latent_vars = lat_vars,
+ latent_vars = lat_vars,
observed_vars = obs_vars)
```
@@ -51,19 +51,17 @@ Now, we construct the different parts:
observed = SemObservedData(specification = partable, data = data)
# implied ------------------------------------------------------------------------------
-implied_ram = RAM(specification = partable)
+implied_ram = RAM(partable)
# loss ---------------------------------------------------------------------------------
-ml = SemML(observed = observed)
-
-loss_ml = SemLoss(ml)
+ml = SemML(observed, implied_ram)
# optimizer ----------------------------------------------------------------------------
-optimizer = SemOptimizerOptim()
+optimizer = SemOptimizer()
# model --------------------------------------------------------------------------------
-model_ml = Sem(observed, implied_ram, loss_ml)
+model_ml = Sem(ml)
fit(optimizer, model_ml)
```
\ No newline at end of file
diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md
index e27724307..cd982ecca 100644
--- a/docs/src/tutorials/construction/outer_constructor.md
+++ b/docs/src/tutorials/construction/outer_constructor.md
@@ -12,13 +12,12 @@ model = Sem(
Structural Equation Model
- Loss Functions
- SemML
-- Fields
- observed: SemObservedData
- implied: RAM
+ > SemML
+ - observed: SemObservedData
+ - implied: RAM
```
-The output of this call tells you exactly what model you just constructed (i.e. what the loss functions, observed, implied and optimizer parts are).
+The output of this call tells you exactly what model you just constructed (i.e. what the loss functions and their observed and implied parts are).
As you can see, by default, we use maximum likelihood estimation abd the RAM implied type.
To choose something different, you can provide it as a keyword argument:
@@ -40,11 +39,12 @@ model = Sem(
specification = partable,
data = data,
implied = RAMSymbolic,
- loss = SemWLS,
- optimizer = SemOptimizerOptim
+ loss = SemWLS
)
```
+The optimizer is not part of the model itself; it is chosen when fitting (see [Model fitting](@ref)).
+
In the section on [Our Concept of a Structural Equation Model](@ref), we go over the different options you have for each part of the model, and in [API - model parts](@ref) we explain each option in detail.
Let's make another example: to use full information maximum likelihood information (FIML), we use
diff --git a/docs/src/tutorials/fitting/fitting.md b/docs/src/tutorials/fitting/fitting.md
index fff06abaa..c14d9b993 100644
--- a/docs/src/tutorials/fitting/fitting.md
+++ b/docs/src/tutorials/fitting/fitting.md
@@ -7,19 +7,17 @@ model_fit = fit(model)
# output
-Fitted Structural Equation Model
-===============================================
---------------------- Model -------------------
+Fitted Structural Equation Model
+===============================================
+--------------------- Model -------------------
-Structural Equation Model
-- Loss Functions
- SemML
-- Fields
- observed: SemObservedData
- implied: RAM
- optimizer: SemOptimizerOptim
+Structural Equation Model
+- Loss Functions
+ > SemML
+ - observed: SemObservedData
+ - implied: RAM
-------------- Optimization result -------------
+------------- Optimization result -------------
* Status: success
@@ -60,7 +58,7 @@ The available keyword arguments are listed in the sections [Using Optim.jl](@ref
Alternative, you can also explicitely define a `SemOptimizer` and pass it as the first argument to `fit`:
```julia
-my_optimizer = SemOptimizerOptim(algorithm = BFGS())
+my_optimizer = SemOptimizer(algorithm = BFGS())
fit(my_optimizer, model)
```
diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md
index 170e4c075..cc4d92e3d 100644
--- a/docs/src/tutorials/inspection/inspection.md
+++ b/docs/src/tutorials/inspection/inspection.md
@@ -81,18 +81,34 @@ update_estimate!(partable, model_fit)
details(partable)
```
-We can also update the `ParameterTable` object with other information via [`update_partable!`](@ref). For example, if we want to compare hessian-based and bootstrap-based standard errors, we may write
+We can also update the `ParameterTable` object with other information via [`update_partable!`](@ref). For example, we can obtain standard errors from the inverse Hessian with [`se_hessian`](@ref) or by bootstrapping with [`se_bootstrap`](@ref), and add both to the table to compare them:
```@example colored; ansicolor = true
se_bs = se_bootstrap(model_fit; n_boot = 20)
se_he = se_hessian(model_fit)
-update_partable!(partable, :se_hessian, param_labels(model_fit), se_he)
-update_partable!(partable, :se_bootstrap, param_labels(model_fit), se_bs)
+update_partable!(partable, :se_hessian, model_fit, se_he)
+update_partable!(partable, :se_bootstrap, model_fit, se_bs)
details(partable)
```
+From a vector of standard errors we can also compute *p*-values and confidence intervals for the parameter estimates.
+[`z_test!`](@ref) adds the two-sided *p*-value of the test that each parameter is zero (using `z = estimate / se`), and [`normal_CI!`](@ref) adds the lower and upper bounds of a normal-theory confidence interval (95% by default).
+Both update the `ParameterTable` in place:
+
+```@example colored; ansicolor = true
+z_test!(partable, model_fit, se_he)
+normal_CI!(partable, model_fit, se_he)
+
+details(partable; show_columns = [:to, :estimate, :p_value, :ci_lower, :ci_upper])
+```
+
+The non-mutating variants [`z_test`](@ref) and [`normal_CI`](@ref) return the values instead of writing them to the table.
+
+Beyond standard errors, [`bootstrap`](@ref) draws bootstrap samples of an arbitrary statistic of a fitted model (not only the parameter estimates), while [`se_bootstrap`](@ref) is a convenience wrapper returning bootstrapped standard errors.
+Both support parallel resampling across the available Julia threads via the `parallel = true` keyword.
+
## Export results
You may convert a `ParameterTable` to a `DataFrame` and use the [`DataFrames`](https://github.com/JuliaData/DataFrames.jl) package for further analysis (or to save it to your hard drive).
@@ -100,7 +116,7 @@ You may convert a `ParameterTable` to a `DataFrame` and use the [`DataFrames`](h
```@example colored; ansicolor = true
using DataFrames
-parameters_df = DataFrame(partable)
+parameters_df = DataFrame(partable);
```
# API - model inspection
@@ -135,4 +151,18 @@ dof
minus2ll
p_value
RMSEA
+CFI
+```
+
+## Standard errors and inference
+
+```@docs
+se_hessian
+se_bootstrap
+bootstrap
+StructuralEquationModels.BootstrapResult
+z_test
+z_test!
+normal_CI
+normal_CI!
```
diff --git a/docs/src/tutorials/meanstructure.md b/docs/src/tutorials/meanstructure.md
index b2da5029a..1d8539355 100644
--- a/docs/src/tutorials/meanstructure.md
+++ b/docs/src/tutorials/meanstructure.md
@@ -40,7 +40,7 @@ end
partable = ParameterTable(
graph,
- latent_vars = latent_vars,
+ latent_vars = latent_vars,
observed_vars = observed_vars)
```
@@ -78,7 +78,7 @@ end
partable = ParameterTable(
graph,
- latent_vars = latent_vars,
+ latent_vars = latent_vars,
observed_vars = observed_vars)
```
@@ -106,11 +106,11 @@ For our example,
```@example meanstructure
observed = SemObservedData(specification = partable, data = data, meanstructure = true)
-implied_ram = RAM(specification = partable, meanstructure = true)
+implied_ram = RAM(partable, meanstructure = true)
-ml = SemML(observed = observed, meanstructure = true)
+ml = SemML(observed, implied_ram)
-model = Sem(observed, implied_ram, SemLoss(ml))
+model = Sem(ml)
fit(model)
```
\ No newline at end of file
diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md
index f743ac79c..79e301c21 100644
--- a/docs/src/tutorials/regularization/regularization.md
+++ b/docs/src/tutorials/regularization/regularization.md
@@ -2,10 +2,11 @@
## Setup
-For ridge regularization, you can simply use `SemRidge` as an additional loss function
+For ridge regularization, you can simply use `SemRidge` as an additional loss function
(for example, a model with the loss functions `SemML` and `SemRidge` corresponds to ridge-regularized maximum likelihood estimation).
-For lasso, elastic net and (far) beyond, you can load the `ProximalAlgorithms.jl` and `ProximalOperators.jl` packages alongside `StructuralEquationModels`:
+You can define lasso, elastic net and other forms of regularization using [`ProximalOperators.jl`](https://github.com/JuliaFirstOrder/ProximalOperators.jl)
+and optimize the SEM with so-called *proximal optimization* algorithms from [`ProximalAlgorithms.jl`](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl).
```@setup reg
using StructuralEquationModels, ProximalAlgorithms, ProximalOperators
@@ -19,24 +20,22 @@ Pkg.add("ProximalOperators")
using StructuralEquationModels, ProximalAlgorithms, ProximalOperators
```
-## `SemOptimizerProximal`
+## Proximal optimization
-To estimate regularized models, we provide a "building block" for the optimizer part, called `SemOptimizerProximal`.
-It connects our package to the [`ProximalAlgorithms.jl`](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) optimization backend, providing so-called proximal optimization algorithms.
-Those can handle, amongst other things, various forms of regularization.
-
-It can be used as
+With the *ProximalAlgorithms* package loaded, it is now possible to use the `:Proximal`
+optimization engine in `SemOptimizer` for estimating regularized models.
```julia
-SemOptimizerProximal(
+SemOptimizer(;
+ engine = :Proximal,
algorithm = ProximalAlgorithms.PANOC(),
operator_g,
operator_h = nothing
- )
+)
```
-The proximal operator (aka the regularization function) can be passed as `operator_g`.
-The available Algorithms are listed [here](https://juliafirstorder.github.io/ProximalAlgorithms.jl/stable/guide/implemented_algorithms/).
+The *proximal operator* (aka the *regularization function*) is passed as `operator_g`, see [available operators](https://juliafirstorder.github.io/ProximalOperators.jl/stable/functions/).
+The `algorithm` is chosen from one of the [available algorithms](https://juliafirstorder.github.io/ProximalAlgorithms.jl/stable/guide/implemented_algorithms/).
## First example - lasso
@@ -69,7 +68,7 @@ end
partable = ParameterTable(
graph,
- latent_vars = latent_vars,
+ latent_vars = latent_vars,
observed_vars = observed_vars
)
@@ -84,8 +83,8 @@ model = Sem(
We labeled the covariances between the items because we want to regularize those:
```@example reg
-ind = getindex.(
- [param_indices(model)],
+cov_inds = getindex.(
+ Ref(param_indices(model)),
[:cov_15, :cov_24, :cov_26, :cov_37, :cov_48, :cov_68])
```
@@ -96,22 +95,16 @@ The lasso penalty is defined as
\sum \lambda_i \lvert \theta_i \rvert
```
-From the previously linked [documentation](https://juliafirstorder.github.io/ProximalOperators.jl/stable/functions/#ProximalOperators.NormL1), we find that lasso regularization is named `NormL1` in the `ProximalOperators` package, and that we can pass an array of hyperparameters (`λ`) to control the amount of regularization for each parameter. To regularize only the observed item covariances, we define `λ` as
+In `ProximalOperators.jl`, lasso regularization is represented by the [`NormL1`](https://juliafirstorder.github.io/ProximalOperators.jl/stable/functions/#ProximalOperators.NormL1) operator. It allows controlling the amount of
+regularization individually for each SEM model parameter via the vector of hyperparameters (`λ`).
+To regularize only the observed item covariances, we define `λ` as
```@example reg
-λ = zeros(31); λ[ind] .= 0.02
-```
+λ = zeros(31); λ[cov_inds] .= 0.02
-and use `SemOptimizerProximal`.
-
-```@example reg
-optimizer_lasso = SemOptimizerProximal(
+optimizer_lasso = SemOptimizer(
+ engine = :Proximal,
operator_g = NormL1(λ)
- )
-
-model_lasso = Sem(
- specification = partable,
- data = data
)
```
@@ -119,7 +112,7 @@ Let's fit the regularized model
```@example reg
-fit_lasso = fit(optimizer_lasso, model_lasso)
+fit_lasso = fit(optimizer_lasso, model)
```
and compare the solution to unregularizted estimates:
@@ -129,45 +122,43 @@ sem_fit = fit(model)
update_estimate!(partable, sem_fit)
-update_partable!(partable, :estimate_lasso, param_labels(fit_lasso), solution(fit_lasso))
+update_partable!(partable, :estimate_lasso, fit_lasso, solution(fit_lasso))
details(partable)
```
-Instead of explicitely defining a `SemOptimizerProximal` object, you can also pass `engine = :Proximal` and additional keyword arguments to `fit`:
+Instead of explicitly defining a `SemOptimizer` object, you can also pass `engine = :Proximal`
+and additional keyword arguments directly to the `fit` function:
```@example reg
-sem_fit = fit(model; engine = :Proximal, operator_g = NormL1(λ))
+fit_lasso2 = fit(model; engine = :Proximal, operator_g = NormL1(λ))
```
## Second example - mixed l1 and l0 regularization
You can choose to penalize different parameters with different types of regularization functions.
-Let's use the lasso again on the covariances, but additionally penalyze the error variances of the observed items via l0 regularization.
+Let's use the *lasso* (*l1*) again on the covariances, but additionally penalize the error variances of
+the observed items via *l0* regularization.
-The l0 penalty is defined as
+The *l0* penalty is defined as
```math
-\lambda \mathrm{nnz}(\theta)
+l_0 = \lambda \mathrm{nnz}(\theta)
```
-To define a sup of separable proximal operators (i.e. no parameter is penalized twice),
-we can use [`SlicedSeparableSum`](https://juliafirstorder.github.io/ProximalOperators.jl/stable/calculus/#ProximalOperators.SlicedSeparableSum) from the `ProximalOperators` package:
+Since we apply *l1* and *l0* to the disjoint sets of parameters, this regularization can be
+represented as a sum of *separable proximal operators* (i.e. no parameter is penalized twice)
+implemented by the [`SlicedSeparableSum`](https://juliafirstorder.github.io/ProximalOperators.jl/stable/calculus/#ProximalOperators.SlicedSeparableSum) operator:
```@example reg
-prox_operator = SlicedSeparableSum((NormL0(20.0), NormL1(0.02), NormL0(0.0)), ([ind], [9:11], [vcat(1:8, 12:25)]))
-
-model_mixed = Sem(
- specification = partable,
- data = data,
-)
+l0_and_l1_reg = SlicedSeparableSum((NormL0(20.0), NormL1(0.02), NormL0(0.0)), ([cov_inds], [9:11], [vcat(1:8, 12:25)]))
-fit_mixed = fit(model_mixed; engine = :Proximal, operator_g = prox_operator)
+fit_mixed = fit(model; engine = :Proximal, operator_g = l0_and_l1_reg)
```
Let's again compare the different results:
```@example reg
-update_partable!(partable, :estimate_mixed, param_labels(fit_mixed), solution(fit_mixed))
+update_partable!(partable, :estimate_mixed, fit_mixed, solution(fit_mixed))
details(partable)
```
\ No newline at end of file
diff --git a/docs/src/tutorials/specification/graph_interface.md b/docs/src/tutorials/specification/graph_interface.md
index 75e1d1b6d..fe03cf304 100644
--- a/docs/src/tutorials/specification/graph_interface.md
+++ b/docs/src/tutorials/specification/graph_interface.md
@@ -1,6 +1,6 @@
# Graph interface
-## Workflow
+## Workflow
As discussed before, when using the graph interface, you can specify your model as a graph
```julia
@@ -17,7 +17,7 @@ lat_vars = ...
partable = ParameterTable(
graph,
- latent_vars = lat_vars,
+ latent_vars = lat_vars,
observed_vars = obs_vars)
model = Sem(
@@ -32,7 +32,7 @@ In general, there are two different types of parameters: **directed** and **indi
We allow multiple variables on both sides of an arrow, for example `x → [y z]` or `[a b] → [c d]`. The later specifies element wise edges; that is its the same as `a → c; b → d`. If you want edges corresponding to the cross-product, we have the double lined arrow `[a b] ⇒ [c d]`, corresponding to `a → c; a → d; b → c; b → d`. The undirected arrows ↔ (element-wise) and ⇔ (crossproduct) behave the same way.
!!! note "Unicode symbols in julia"
- The `→` symbol is a unicode symbol allowed in julia (among many others; see this [list](https://docs.julialang.org/en/v1/manual/unicode-input/)). You can enter it in the julia REPL or the vscode IDE by typing `\to` followed by hitting `tab`. Similarly,
+ The `→` symbol is a unicode symbol allowed in julia (among many others; see this [list](https://docs.julialang.org/en/v1/manual/unicode-input/)). You can enter it in the julia REPL or the vscode IDE by typing `\to` followed by hitting `tab`. Similarly,
- `←` = `\leftarrow`,
- `↔` = `\leftrightarrow`,
- `⇒` = `\Rightarrow`,
@@ -54,7 +54,7 @@ graph = @StenoGraph begin
ξ₃ ↔ fixed(1.0)*ξ₃
end
```
-would
+would
- fix the directed effects from `ξ₁` to `x1` and from `ξ₂` to `x2` to `1`
- leave the directed effect from `ξ₃` to `x7` free but instead restrict the variance of `ξ₃` to `1`
- give the effect from `ξ₁` to `x3` the label `:a` (which can be convenient later if you want to retrieve information from your model about that specific parameter)
@@ -66,7 +66,7 @@ As you saw above and in the [A first model](@ref) example, the graph object need
```julia
partable = ParameterTable(
graph,
- latent_vars = lat_vars,
+ latent_vars = lat_vars,
observed_vars = obs_vars)
```
@@ -85,7 +85,7 @@ The variable names (`:x1`) have to be symbols, the syntax `:something` creates a
_(lat_vars) ⇔ _(lat_vars)
end
```
-creates undirected effects coresponding to
+creates undirected effects coresponding to
1. the variances of all observed variables and
2. the variances plus covariances of all latent variables
So if you want to work with a subset of variables, simply specify a vector of symbols `somevars = [...]`, and inside the graph specification, refer to them as `_(somevars)`.
@@ -105,4 +105,4 @@ end
The syntax to specify graphs (`@StenoGraph`) may seem a bit strange if you are not familiar with the julia language. It is called a **macro**, but explaining this concept in detail is beyond this documentation (and not necessary to understand to specify models). However, if you want to know more about it, you may have a look at the respective part of the [manual](https://docs.julialang.org/en/v1/manual/metaprogramming/#man-macros).
### The StenoGraphs Package
-Behind the scenes, we are using the [StenoGraphs](https://github.com/aaronpeikert/StenoGraphs.jl) package to specify our graphs. It makes a domain specific language available that allows you to specify graphs with arbitrary information attached to its edges and nodes (for structural equation models, this may be the name or the value of a parameter). Is also allows you to specify your own types to "attach" to the graph, called a `Modifier`. So if you contemplate about writing your own modifier (e.g., to mark a variable as ordinal, an effect as quadratic, ...), please refer to the `StenoGraphs` [documentation](https://aaronpeikert.github.io/StenoGraphs.jl/dev/).
\ No newline at end of file
+Behind the scenes, we are using the [StenoGraphs](https://github.com/aaronpeikert/StenoGraphs.jl) package to specify our graphs. It makes a domain specific language available that allows you to specify graphs with arbitrary information attached to its edges and nodes (for structural equation models, this may be the name or the value of a parameter). Is also allows you to specify your own types to "attach" to the graph, called a `Modifier`. So if you contemplate about writing your own modifier, please refer to the `StenoGraphs` [documentation](https://aaronpeikert.github.io/StenoGraphs.jl/dev/).
\ No newline at end of file
diff --git a/docs/src/tutorials/specification/parameter_table.md b/docs/src/tutorials/specification/parameter_table.md
index c328a3b1a..61549b40c 100644
--- a/docs/src/tutorials/specification/parameter_table.md
+++ b/docs/src/tutorials/specification/parameter_table.md
@@ -5,5 +5,11 @@ As lavaan also uses parameter tables to store model specifications, we are worki
## Convert from and to RAMMatrices
-To convert a RAMMatrices object to a ParameterTable, simply use `partable = ParameterTable(rammatrices)`.
-To convert an object of type `ParameterTable` to RAMMatrices, you can use `ram_matrices = RAMMatrices(partable)`.
\ No newline at end of file
+To convert a RAMMatrices object to a ParameterTable, simply use `partable = ParameterTable(rammatrices)`.
+To convert an object of type `ParameterTable` to RAMMatrices, you can use `ram_matrices = RAMMatrices(partable)`.
+
+# API
+
+```@docs
+ParameterTable
+```
\ No newline at end of file
diff --git a/docs/src/tutorials/specification/ram_matrices.md b/docs/src/tutorials/specification/ram_matrices.md
index abe76ea6f..2730ff4bf 100644
--- a/docs/src/tutorials/specification/ram_matrices.md
+++ b/docs/src/tutorials/specification/ram_matrices.md
@@ -1,6 +1,6 @@
# RAMMatrices interface
-Models can also be specified by an object of type `RAMMatrices`.
+Models can also be specified by an object of type `RAMMatrices`.
The RAM (reticular action model) specification corresponds to three matrices; the `A` matrix containing all directed parameters, the `S` matrix containing all undirected parameters, and the `F` matrix filtering out latent variables from the model implied covariance.
The model implied covariance matrix for the observed variables of a SEM is then computed as
@@ -56,9 +56,9 @@ A =[0 0 0 0 0 0 0 0 0 0 0 1.0 0 0
θ = Symbol.(:θ, 1:31)
spec = RAMMatrices(;
- A = A,
- S = S,
- F = F,
+ A = A,
+ S = S,
+ F = F,
param_labels = θ,
vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65]
)
@@ -71,9 +71,9 @@ model = Sem(
Let's look at this step by step:
-First, we specify the `A`, `S` and `F`-Matrices.
-For a free parameter, we write a `Symbol` like `:θ1` (or any other symbol we like) to the corresponding place in the respective matrix, to constrain parameters to be equal we just use the same `Symbol` in the respective entries.
-To fix a parameter (as in the `A`-Matrix above), we just write down the number we want to fix it to.
+First, we specify the `A`, `S` and `F`-Matrices.
+For a free parameter, we write a `Symbol` like `:θ1` (or any other symbol we like) to the corresponding place in the respective matrix, to constrain parameters to be equal we just use the same `Symbol` in the respective entries.
+To fix a parameter (as in the `A`-Matrix above), we just write down the number we want to fix it to.
All other entries are 0.
Second, we specify a vector of symbols containing our parameters:
@@ -82,14 +82,14 @@ Second, we specify a vector of symbols containing our parameters:
θ = Symbol.(:θ, 1:31)
```
-Third, we construct an object of type `RAMMatrices`, passing our matrices and parameters, as well as the column names of our matrices.
+Third, we construct an object of type `RAMMatrices`, passing our matrices and parameters, as well as the column names of our matrices.
Those are quite important, as they will be used to rearrange your data to match it to your `RAMMatrices` specification.
```julia
spec = RAMMatrices(;
- A = A,
- S = S,
- F = F,
+ A = A,
+ S = S,
+ F = F,
param_labels = θ,
vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65]
)
@@ -109,7 +109,7 @@ According to the RAM, model implied mean values of the observed variables are co
```math
\mu = F(I-A)^{-1}M
```
-where `M` is a vector of mean parameters. To estimate the means of the observed variables in our example (and set the latent means to `0`), we would specify the model just as before but add
+where `M` is a vector of mean parameters. To estimate the means of the observed variables in our example (and set the latent means to `0`), we would specify the model just as before but add
```julia
...
@@ -128,5 +128,5 @@ spec = RAMMatrices(;
## Convert from and to ParameterTables
-To convert a RAMMatrices object to a ParameterTable, simply use `partable = ParameterTable(ram_matrices)`.
+To convert a RAMMatrices object to a ParameterTable, simply use `partable = ParameterTable(ram_matrices)`.
To convert an object of type `ParameterTable` to RAMMatrices, you can use `ram_matrices = RAMMatrices(partable)`.
\ No newline at end of file
diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl
index c5e0ad6cb..876010305 100644
--- a/ext/SEMNLOptExt/NLopt.jl
+++ b/ext/SEMNLOptExt/NLopt.jl
@@ -1,74 +1,146 @@
-Base.convert(
- ::Type{NLoptConstraint},
- tuple::NamedTuple{(:f, :tol), Tuple{F, T}},
-) where {F, T} = NLoptConstraint(tuple.f, tuple.tol)
+############################################################################################
+### Types
+############################################################################################
+
+const NLoptConstraint = Pair{Any, Number}
+
+struct SemOptimizerNLopt <: SemOptimizer{:NLopt}
+ algorithm::Symbol
+ local_algorithm::Union{Symbol, Nothing}
+ options::Dict{Symbol, Any}
+ local_options::Dict{Symbol, Any}
+ equality_constraints::Vector{NLoptConstraint}
+ inequality_constraints::Vector{NLoptConstraint}
+end
+
+SEM.sem_optimizer_subtype(::Val{:NLopt}) = SemOptimizerNLopt
############################################################################################
### Constructor
############################################################################################
+"""
+ SemOptimizer(;
+ engine = :NLopt,
+ algorithm = :LD_LBFGS,
+ options = Dict{Symbol, Any}(),
+ local_algorithm = nothing,
+ local_options = Dict{Symbol, Any}(),
+ equality_constraints = nothing,
+ inequality_constraints = nothing,
+ constraint_tol::Number = 0.0,
+ kwargs...)
+
+Uses *NLopt.jl* as the optimization engine. For more information on the available algorithms
+and options, see the [*NLopt.jl*](https://github.com/JuliaOpt/NLopt.jl) package and
+the [NLopt docs](https://nlopt.readthedocs.io/en/latest/).
+
+# Arguments
+- `algorithm`: optimization algorithm.
+- `options::Dict{Symbol, Any}`: options for the optimization algorithm
+- `local_algorithm`: local optimization algorithm
+- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm
+- `equality_constraints: optional equality constraints
+- `inequality_constraints:: optional inequality constraints
+- `constraint_tol::Number`: default tolerance for constraints
+
+## Constraints specification
+
+Equality and inequality constraints arguments could be a single constraint or any
+iterable constraints container (e.g. vector or tuple).
+Each constraint could be a function or any other callable object that
+takes the two input arguments:
+ - the vector of the model parameters;
+ - the array for the in-place calculation of the constraint gradient.
+To override the default tolerance, the constraint can be specified
+as a pair of the function and its tolerance: `constraint_func => tol`.
+For information on how to use inequality and equality constraints,
+see [Constrained optimization](@ref) in our online documentation.
+
+# Example
+```julia
+my_optimizer = SemOptimizer(engine = :NLopt)
+
+# constrained optimization with augmented lagrangian
+my_constrained_optimizer = SemOptimizer(;
+ engine = :NLopt,
+ algorithm = :AUGLAG,
+ local_algorithm = :LD_LBFGS,
+ local_options = Dict(:ftol_rel => 1e-6),
+ inequality_constraints = (my_constraint => tol),
+)
+```
+
+# Interfaces
+- `algorithm(::SemOptimizerNLopt)`
+- `local_algorithm(::SemOptimizerNLopt)`
+- `options(::SemOptimizerNLopt)`
+- `local_options(::SemOptimizerNLopt)`
+- `equality_constraints(::SemOptimizerNLopt)`
+- `inequality_constraints(::SemOptimizerNLopt)`
+"""
function SemOptimizerNLopt(;
algorithm = :LD_LBFGS,
local_algorithm = nothing,
options = Dict{Symbol, Any}(),
local_options = Dict{Symbol, Any}(),
- equality_constraints = Vector{NLoptConstraint}(),
- inequality_constraints = Vector{NLoptConstraint}(),
- kwargs...,
+ equality_constraints = nothing,
+ inequality_constraints = nothing,
+ constraint_tol::Number = 0.0,
+ kwargs..., # FIXME remove the sink for unused kwargs
)
- applicable(iterate, equality_constraints) && !isa(equality_constraints, NamedTuple) ||
- (equality_constraints = [equality_constraints])
- applicable(iterate, inequality_constraints) &&
- !isa(inequality_constraints, NamedTuple) ||
- (inequality_constraints = [inequality_constraints])
+ constraint(f::Any) = f => constraint_tol
+ constraint(f_and_tol::Pair) = f_and_tol
+
+ constraints(::Nothing) = Vector{NLoptConstraint}()
+ constraints(constraints) =
+ applicable(iterate, constraints) && !isa(constraints, Pair) ?
+ [constraint(constr) for constr in constraints] : [constraint(constraints)]
+
return SemOptimizerNLopt(
algorithm,
local_algorithm,
options,
local_options,
- convert.(NLoptConstraint, equality_constraints),
- convert.(NLoptConstraint, inequality_constraints),
+ constraints(equality_constraints),
+ constraints(inequality_constraints),
)
end
-SEM.SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...)
-
-############################################################################################
-### Recommended methods
-############################################################################################
-
-SEM.update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) =
- optimizer
-
############################################################################################
### additional methods
############################################################################################
-SEM.algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm
local_algorithm(optimizer::SemOptimizerNLopt) = optimizer.local_algorithm
SEM.options(optimizer::SemOptimizerNLopt) = optimizer.options
local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options
equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints
inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints
-mutable struct NLoptResult
+# wrapper for the NLopt optimization result
+struct NLoptResult <: SEM.SemOptimizerResult{SemOptimizerNLopt}
+ optimizer::SemOptimizerNLopt
result::Any
problem::Any
end
-SEM.optimizer(res::NLoptResult) = res.problem.algorithm
+SEM.algorithm_name(res::NLoptResult) = res.problem.algorithm
SEM.n_iterations(res::NLoptResult) = res.problem.numevals
SEM.convergence(res::NLoptResult) = res.result[3]
+function SEM.converged(res::NLoptResult)
+ flag = res.result[3]
+ return flag ∈ [:SUCCESS, :STOPVAL_REACHED, :FTOL_REACHED, :XTOL_REACHED]
+end
-# construct SemFit from fitted NLopt object
-function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, opt)
- return SemFit(
- optimization_result[1],
- optimization_result[2],
- start_val,
- model,
- NLoptResult(optimization_result, opt),
- )
+# construct NLopt.jl problem
+function NLopt_problem(algorithm, options, npar)
+ problem = Opt(algorithm, npar)
+
+ for (key, val) in pairs(options)
+ setproperty!(problem, key, val)
+ end
+
+ return problem
end
# fit method
@@ -78,11 +150,8 @@ function SEM.fit(
start_params::AbstractVector;
kwargs...,
)
-
- # construct the NLopt problem
- opt = construct_NLopt_problem(optim.algorithm, optim.options, length(start_params))
- set_NLopt_constraints!(opt, optim)
- opt.min_objective =
+ problem = NLopt_problem(optim.algorithm, optim.options, nparams(model))
+ problem.min_objective =
(par, G) -> SEM.evaluate!(
zero(eltype(par)),
!isnothing(G) && !isempty(G) ? G : nothing,
@@ -90,43 +159,28 @@ function SEM.fit(
model,
par,
)
+ for (f, tol) in optim.inequality_constraints
+ inequality_constraint!(problem, f, tol)
+ end
+ for (f, tol) in optim.equality_constraints
+ equality_constraint!(problem, f, tol)
+ end
if !isnothing(optim.local_algorithm)
- opt_local = construct_NLopt_problem(
- optim.local_algorithm,
- optim.local_options,
- length(start_params),
- )
- opt.local_optimizer = opt_local
+ problem.local_optimizer =
+ NLopt_problem(optim.local_algorithm, optim.local_options, nparams(model))
end
# fit
- result = NLopt.optimize(opt, start_params)
-
- return SemFit_NLopt(result, model, start_params, opt)
-end
-
-############################################################################################
-### additional functions
-############################################################################################
-
-function construct_NLopt_problem(algorithm, options, npar)
- opt = Opt(algorithm, npar)
+ result = NLopt.optimize(problem, start_params)
- for (key, val) in pairs(options)
- setproperty!(opt, key, val)
- end
-
- return opt
-end
-
-function set_NLopt_constraints!(opt::Opt, optimizer::SemOptimizerNLopt)
- for con in optimizer.inequality_constraints
- inequality_constraint!(opt, con.f, con.tol)
- end
- for con in optimizer.equality_constraints
- equality_constraint!(opt, con.f, con.tol)
- end
+ return SemFit(
+ result[1], # minimum
+ result[2], # optimal params
+ start_val,
+ model,
+ NLoptResult(optim, result, problem),
+ )
end
############################################################################################
diff --git a/ext/SEMNLOptExt/SEMNLOptExt.jl b/ext/SEMNLOptExt/SEMNLOptExt.jl
index c79fc2b86..61c41338b 100644
--- a/ext/SEMNLOptExt/SEMNLOptExt.jl
+++ b/ext/SEMNLOptExt/SEMNLOptExt.jl
@@ -1,10 +1,11 @@
module SEMNLOptExt
using StructuralEquationModels, NLopt
-using StructuralEquationModels: SemOptimizerNLopt, NLoptConstraint
SEM = StructuralEquationModels
+export SemOptimizerNLopt
+
include("NLopt.jl")
end
diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl
index 0d4748e3a..0937ee040 100644
--- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl
+++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl
@@ -1,5 +1,30 @@
-SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...)
+############################################################################################
+### Types
+############################################################################################
+mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal}
+ algorithm::A
+ operator_g::B
+ operator_h::C
+end
+
+"""
+ SemOptimizerProximal(;
+ algorithm = ProximalAlgorithms.PANOC(),
+ operator_g,
+ operator_h = nothing,
+ kwargs...,
+ )
+
+Connects to `ProximalAlgorithms.jl` as the optimization backend. For more information on
+the available algorithms and options, see the online docs on [Regularization](@ref) and
+the documentation of [*ProximalAlgorithms.jl*](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) /
+[ProximalOperators.jl](https://github.com/JuliaFirstOrder/ProximalOperators.jl).
+# Arguments
+- `algorithm`: proximal optimization algorithm.
+- `operator_g`: proximal operator (e.g., regularization penalty)
+- `operator_h`: optional second proximal operator
+"""
SemOptimizerProximal(;
algorithm = ProximalAlgorithms.PANOC(),
operator_g,
@@ -7,28 +32,12 @@ SemOptimizerProximal(;
kwargs...,
) = SemOptimizerProximal(algorithm, operator_g, operator_h)
-############################################################################################
-### Recommended methods
-############################################################################################
-
-SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwargs...) =
- optimizer
-
-############################################################################################
-### additional methods
-############################################################################################
-
-SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm
+SEM.sem_optimizer_subtype(::Val{:Proximal}) = SemOptimizerProximal
############################################################################
-### Pretty Printing
+### Model fitting
############################################################################
-function Base.show(io::IO, struct_inst::SemOptimizerProximal)
- print_type_name(io, struct_inst)
- print_field_types(io, struct_inst)
-end
-
## connect to ProximalAlgorithms.jl
function ProximalAlgorithms.value_and_gradient(model::AbstractSem, params)
grad = similar(params)
@@ -36,8 +45,11 @@ function ProximalAlgorithms.value_and_gradient(model::AbstractSem, params)
return obj, grad
end
-mutable struct ProximalResult
- result::Any
+# wrapper for the Proximal optimization result
+struct ProximalResult{O <: SemOptimizer{:Proximal}} <: SEM.SemOptimizerResult{O}
+ optimizer::O
+ minimum::Float64
+ n_iterations::Int
end
function SEM.fit(
@@ -47,10 +59,10 @@ function SEM.fit(
kwargs...,
)
if isnothing(optim.operator_h)
- solution, iterations =
+ solution, niterations =
optim.algorithm(x0 = start_params, f = model, g = optim.operator_g)
else
- solution, iterations = optim.algorithm(
+ solution, niterations = optim.algorithm(
x0 = start_params,
f = model,
g = optim.operator_g,
@@ -58,36 +70,38 @@ function SEM.fit(
)
end
- minimum = objective!(model, solution)
+ optim_res = ProximalResult(optim, objective!(model, solution), niterations)
- optimization_result = Dict(
- :minimum => minimum,
- :iterations => iterations,
- :algorithm => optim.algorithm,
- :operator_g => optim.operator_g,
- )
+ return SemFit(optim_res.minimum, solution, start_params, model, optim_res)
+end
- isnothing(optim.operator_h) ||
- push!(optimization_result, :operator_h => optim.operator_h)
+############################################################################################
+### additional methods
+############################################################################################
- return SemFit(
- minimum,
- solution,
- start_params,
- model,
- ProximalResult(optimization_result),
- )
-end
+SEM.algorithm_name(res::ProximalResult) = SEM.algorithm_name(res.optimizer.algorithm)
+SEM.algorithm_name(
+ ::ProximalAlgorithms.IterativeAlgorithm{I, H, S, D, K},
+) where {I, H, S, D, K} = nameof(I)
+SEM.convergence(
+ ::ProximalResult,
+) = "No standard convergence criteria for proximal \n algorithms available."
+SEM.converged(::ProximalResult) = missing
+SEM.n_iterations(res::ProximalResult) = res.n_iterations
############################################################################################
# pretty printing
############################################################################################
+function Base.show(io::IO, struct_inst::SemOptimizerProximal)
+ print_type_name(io, struct_inst)
+ print_field_types(io, struct_inst)
+end
+
function Base.show(io::IO, result::ProximalResult)
- print(io, "Minimum: $(round(result.result[:minimum]; digits = 2)) \n")
- print(io, "No. evaluations: $(result.result[:iterations]) \n")
- print(io, "Operator: $(nameof(typeof(result.result[:operator_g]))) \n")
- if haskey(result.result, :operator_h)
- print(io, "Second Operator: $(nameof(typeof(result.result[:operator_h]))) \n")
- end
+ print(io, "Minimum: $(round(result.minimum; digits = 2)) \n")
+ print(io, "No. evaluations: $(result.n_iterations) \n")
+ print(io, "Operator: $(nameof(typeof(result.optimizer.operator_g))) \n")
+ op_h = result.optimizer.operator_h
+ isnothing(op_h) || print(io, "Second Operator: $(nameof(typeof(op_h))) \n")
end
diff --git a/ext/SEMProximalOptExt/SEMProximalOptExt.jl b/ext/SEMProximalOptExt/SEMProximalOptExt.jl
index 192944fef..bedf1920e 100644
--- a/ext/SEMProximalOptExt/SEMProximalOptExt.jl
+++ b/ext/SEMProximalOptExt/SEMProximalOptExt.jl
@@ -1,8 +1,10 @@
module SEMProximalOptExt
using StructuralEquationModels
+using StructuralEquationModels: print_type_name, print_field_types
using ProximalAlgorithms
-using StructuralEquationModels: SemOptimizerProximal, print_type_name, print_field_types
+
+export SemOptimizerProximal
SEM = StructuralEquationModels
diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl
index f6068dc50..0dbcd16aa 100644
--- a/src/StructuralEquationModels.jl
+++ b/src/StructuralEquationModels.jl
@@ -14,11 +14,14 @@ using LinearAlgebra,
StenoGraphs,
LazyArtifacts,
DelimitedFiles,
- DataFrames
+ DataFrames,
+ ProgressMeter
import StatsAPI: params, coef, coefnames, dof, fit, nobs, coeftable
-export StenoGraphs, @StenoGraph, meld
+using InteractiveUtils: subtypes
+
+export @StenoGraph, meld
const SEM = StructuralEquationModels
@@ -41,6 +44,7 @@ include("frontend/specification/EnsembleParameterTable.jl")
include("frontend/specification/StenoGraphs.jl")
include("frontend/fit/summary.jl")
include("frontend/StatsAPI.jl")
+include("frontend/finite_diff.jl")
# pretty printing
include("frontend/pretty_printing.jl")
# observed
@@ -50,26 +54,28 @@ include("observed/covariance.jl")
include("observed/missing_pattern.jl")
include("observed/missing.jl")
include("observed/EM.jl")
-# constructor
-include("frontend/specification/Sem.jl")
-include("frontend/specification/documentation.jl")
# implied
include("implied/abstract.jl")
include("implied/RAM/symbolic.jl")
include("implied/RAM/generic.jl")
include("implied/empty.jl")
# loss
+include("loss/abstract.jl")
include("loss/ML/ML.jl")
include("loss/ML/FIML.jl")
include("loss/regularization/ridge.jl")
include("loss/WLS/WLS.jl")
include("loss/constant/constant.jl")
+# constructor
+include("frontend/specification/Sem.jl")
+include("frontend/specification/documentation.jl")
# optimizer
include("optimizer/abstract.jl")
include("optimizer/Empty.jl")
include("optimizer/optim.jl")
# helper functions
include("additional_functions/helper.jl")
+include("additional_functions/start_val/common.jl")
include("additional_functions/start_val/start_fabin3.jl")
include("additional_functions/start_val/start_simple.jl")
include("additional_functions/artifacts.jl")
@@ -82,23 +88,20 @@ include("frontend/fit/fitmeasures/dof.jl")
include("frontend/fit/fitmeasures/minus2ll.jl")
include("frontend/fit/fitmeasures/p.jl")
include("frontend/fit/fitmeasures/RMSEA.jl")
+include("frontend/fit/fitmeasures/CFI.jl")
include("frontend/fit/fitmeasures/fit_measures.jl")
# standard errors
include("frontend/fit/standard_errors/hessian.jl")
include("frontend/fit/standard_errors/bootstrap.jl")
-# extensions
-include("package_extensions/SEMNLOptExt.jl")
-include("package_extensions/SEMProximalOptExt.jl")
+include("frontend/fit/standard_errors/z_test.jl")
+include("frontend/fit/standard_errors/confidence_intervals.jl")
export AbstractSem,
- AbstractSemSingle,
- AbstractSemCollection,
coef,
coefnames,
coeftable,
Sem,
SemFiniteDiff,
- SemEnsemble,
MeanStruct,
NoMeanStruct,
HasMeanStruct,
@@ -113,8 +116,8 @@ export AbstractSem,
start_val,
start_fabin3,
start_simple,
+ AbstractLoss,
SemLoss,
- SemLossFunction,
SemML,
SemFIML,
em_mvn,
@@ -122,11 +125,16 @@ export AbstractSem,
SemConstant,
SemWLS,
loss,
+ nsem_terms,
+ sem_terms,
+ sem_term,
SemOptimizer,
- SemOptimizerEmpty,
- SemOptimizerOptim,
optimizer,
+ optimizer_engine,
+ optimizer_engine_doc,
+ optimizer_engines,
n_iterations,
+ converged,
convergence,
SemObserved,
SemObservedData,
@@ -186,18 +194,20 @@ export AbstractSem,
minus2ll,
p_value,
RMSEA,
+ CFI,
EmMVNModel,
se_hessian,
+ bootstrap,
se_bootstrap,
+ normal_CI,
+ normal_CI!,
+ z_test,
+ z_test!,
example_data,
replace_observed,
- update_observed,
@StenoGraph,
→,
←,
↔,
- ⇔,
- SemOptimizerNLopt,
- NLoptConstraint,
- SemOptimizerProximal
+ ⇔
end
diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl
index 6cbcb0573..5442357f0 100644
--- a/src/additional_functions/helper.jl
+++ b/src/additional_functions/helper.jl
@@ -14,14 +14,8 @@ function neumann_series(mat::SparseMatrixCSC; maxiter::Integer = size(mat, 1))
return inverse
end
-function batch_inv!(fun, model)
- for i in 1:size(fun.inverses, 1)
- fun.inverses[i] .= LinearAlgebra.inv!(fun.choleskys[i])
- end
-end
-
# computes A*S*B -> C, where ind gives the entries of S that are 1
-function sparse_outer_mul!(C, A, B, ind)
+function sparse_outer_mul!(C, A, B, ind)
fill!(C, 0.0)
for i in 1:length(ind)
BLAS.ger!(1.0, A[:, ind[i][1]], B[ind[i][2], :], C)
@@ -29,14 +23,14 @@ function sparse_outer_mul!(C, A, B, ind)
end
# computes A*∇m, where ∇m ind gives the entries of ∇m that are 1
-function sparse_outer_mul!(C, A, ind)
+function sparse_outer_mul!(C, A, ind)
fill!(C, 0.0)
@views C .= sum(A[:, ind], dims = 2)
return C
end
# computes A*S*B -> C, where ind gives the entries of S that are 1
-function sparse_outer_mul!(C, A, B::Vector, ind)
+function sparse_outer_mul!(C, A, B::Vector, ind)
fill!(C, 0.0)
@views @inbounds for i in 1:length(ind)
C .+= B[ind[i][2]] .* A[:, ind[i][1]]
@@ -75,6 +69,38 @@ function elimination_matrix(n::Integer)
return L
end
+# truncate eigenvalues of a symmetric matrix and return the result
+function trunc_eigvals(
+ mtx::AbstractMatrix{T},
+ min_eigval::Number;
+ mtx_label::AbstractString = "matrix",
+ verbose::Bool = false,
+) where {T}
+ size(mtx, 1) == size(mtx, 2) ||
+ throw(ArgumentError("Matrix must be square, $(size(mtx)) given"))
+ issymmetric(mtx) || throw(ArgumentError("Matrix must be symmetric"))
+
+ # eigen decomposition of the mtx
+ mtx_eig = eigen(convert(Matrix{T}, mtx))
+ verbose &&
+ @info "min(eigvals($mtx_label))=$(Base.minimum(mtx_eig.values)), N(eigvals < $min_eigval) = $(sum(<(min_eigval), mtx_eig.values))"
+
+ eigmin = Base.minimum(mtx_eig.values)
+ if eigmin < min_eigval
+ # substitute small eigvals with min_eigval
+ eigvals_mtx = Diagonal(max.(mtx_eig.values, min_eigval))
+ newmtx = mtx_eig.vectors * eigvals_mtx * mtx_eig.vectors'
+ StatsBase._symmetrize!(newmtx)
+ if verbose
+ Δmtx = newmtx .- mtx
+ @info "Δ($mtx_label, posdef)=$(norm(Δmtx, 2)), min,max(Δᵢ)=$(extrema(Δmtx))"
+ end
+ return newmtx
+ else
+ return mtx
+ end
+end
+
# returns the vector of non-unique values in the order of appearance
# each non-unique values is reported once
function nonunique(values::AbstractVector)
diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl
index 1031e349e..8cd89032e 100644
--- a/src/additional_functions/params_array.jl
+++ b/src/additional_functions/params_array.jl
@@ -199,11 +199,7 @@ materialize!(
kwargs...,
) = materialize!(parent(dest), src, params; kwargs...)
-function sparse_materialize(
- ::Type{T},
- arr::ParamsMatrix,
- params::AbstractVector,
-) where {T}
+function sparse_materialize(::Type{T}, arr::ParamsMatrix, params::AbstractVector) where {T}
nparams(arr) == length(params) || throw(
DimensionMismatch(
"Number of values ($(length(params))) does not match the number of parameter ($(nparams(arr)))",
@@ -251,8 +247,8 @@ sparse_gradient(arr::ParamsArray{T}) where {T} = sparse_gradient(T, arr)
# range of parameters that are referenced in the matrix
function params_range(arr::ParamsArray; allow_gaps::Bool = false)
- first_i = findfirst(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:nparams(arr)-1)
- last_i = findlast(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:nparams(arr)-1)
+ first_i = findfirst(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:(nparams(arr)-1))
+ last_i = findlast(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:(nparams(arr)-1))
if !allow_gaps && !isnothing(first_i) && !isnothing(last_i)
for i in first_i:last_i
diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl
index da3e6a906..e85e9d5c4 100644
--- a/src/additional_functions/simulation.jl
+++ b/src/additional_functions/simulation.jl
@@ -1,134 +1,39 @@
-"""
- (1) replace_observed(model::AbstractSemSingle; kwargs...)
-
- (2) replace_observed(model::AbstractSemSingle, observed; kwargs...)
-
-Return a new model with swaped observed part.
-
-# Arguments
-- `model::AbstractSemSingle`: model to swap the observed part of.
-- `kwargs`: additional keyword arguments; typically includes `data` and `specification`
-- `observed`: Either an object of subtype of `SemObserved` or a subtype of `SemObserved`
-
-# Examples
-See the online documentation on [Replace observed data](@ref).
-"""
-function replace_observed end
-
-"""
- update_observed(to_update, observed::SemObserved; kwargs...)
-
-Update a `SemImplied`, `SemLossFunction` or `SemOptimizer` object to use a `SemObserved` object.
-
-# Examples
-See the online documentation on [Replace observed data](@ref).
-
-# Implementation
-You can provide a method for this function when defining a new type, for more information
-on this see the online developer documentation on [Update observed data](@ref).
-"""
-function update_observed end
-
-############################################################################################
-# change observed (data) without reconstructing the whole model
-############################################################################################
-
-# use the same observed type as before
-replace_observed(model::AbstractSemSingle; kwargs...) =
- replace_observed(model, typeof(observed(model)).name.wrapper; kwargs...)
-
-# construct a new observed type
-replace_observed(model::AbstractSemSingle, observed_type; kwargs...) =
- replace_observed(model, observed_type(; kwargs...); kwargs...)
-
-replace_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) =
- replace_observed(
- model,
- observed(model),
- implied(model),
- loss(model),
- new_observed;
- kwargs...,
- )
-
-function replace_observed(
- model::AbstractSemSingle,
- old_observed,
- implied,
- loss,
- new_observed::SemObserved;
- kwargs...,
-)
- kwargs = Dict{Symbol, Any}(kwargs...)
-
- # get field types
- kwargs[:observed_type] = typeof(new_observed)
- kwargs[:old_observed_type] = typeof(old_observed)
- kwargs[:implied_type] = typeof(implied)
- kwargs[:loss_types] = [typeof(lossfun) for lossfun in loss.functions]
-
- # update implied
- implied = update_observed(implied, new_observed; kwargs...)
- kwargs[:implied] = implied
- kwargs[:nparams] = nparams(implied)
-
- # update loss
- loss = update_observed(loss, new_observed; kwargs...)
- kwargs[:loss] = loss
-
- #new_implied = update_observed(model.implied, new_observed; kwargs...)
-
- return Sem(
- new_observed,
- update_observed(model.implied, new_observed; kwargs...),
- update_observed(model.loss, new_observed; kwargs...),
- )
-end
-
-function update_observed(loss::SemLoss, new_observed; kwargs...)
- new_functions = Tuple(
- update_observed(lossfun, new_observed; kwargs...) for lossfun in loss.functions
- )
- return SemLoss(new_functions, loss.weights)
-end
-
############################################################################################
# simulate data
############################################################################################
"""
- (1) rand(model::AbstractSemSingle, params, n)
-
- (2) rand(model::AbstractSemSingle, n)
+ rand(sem::Union{Sem, SemLoss, SemImplied}, [params], n)
-Sample normally distributed data from the model-implied covariance matrix and mean vector.
+Sample from the multivariate normal distribution implied by the SEM model.
# Arguments
-- `model::AbstractSemSingle`: model to simulate from.
-- `params`: parameter values to simulate from.
-- `n::Integer`: Number of samples.
+- `sem`: SEM model to use. Ensemble models with multiple SEM terms are not supported.
+- `params`: optional SEM model parameters to simulate from, otherwise uses the
+ current state of implied covariances and means.
+- `n::Integer`: Number of samples to draw.
# Examples
```julia
rand(model, start_simple(model), 100)
```
"""
-function Distributions.rand(
- model::AbstractSemSingle{O, I, L},
- params,
- n::Integer,
-) where {O, I <: Union{RAM, RAMSymbolic}, L}
- update!(EvaluationTargets{true, false, false}(), model.implied, model, params)
- return rand(model, n)
-end
-
-function Distributions.rand(
- model::AbstractSemSingle{O, I, L},
- n::Integer,
-) where {O, I <: Union{RAM, RAMSymbolic}, L}
- if MeanStruct(model.implied) === NoMeanStruct
- data = permutedims(rand(MvNormal(Symmetric(model.implied.Σ)), n))
- elseif MeanStruct(model.implied) === HasMeanStruct
- data = permutedims(rand(MvNormal(model.implied.μ, Symmetric(model.implied.Σ)), n))
+function Distributions.rand(implied::SemImplied, params, n::Integer)
+ if !isnothing(params)
+ # update the implied covariances with the new model params
+ update!(EvaluationTargets{true, false, false}(), implied, params)
+ end
+ Σ = Symmetric(implied.Σ)
+ if MeanStruct(implied) === NoMeanStruct
+ return permutedims(rand(MvNormal(Σ), n))
+ elseif MeanStruct(implied) === HasMeanStruct
+ return permutedims(rand(MvNormal(implied.μ, Σ), n))
end
- return data
end
+
+Distributions.rand(loss::SemLoss, params, n::Integer) = rand(SEM.implied(loss), params, n)
+
+Distributions.rand(model::Sem, params, n::Integer) = rand(sem_term(model), params, n)
+
+# rand() overloads without SEM params
+Distributions.rand(implied::Union{SemImplied, SemLoss, Sem}, n::Integer) =
+ Distributions.rand(implied, nothing, n)
diff --git a/src/additional_functions/start_val/common.jl b/src/additional_functions/start_val/common.jl
new file mode 100644
index 000000000..92c85d6f5
--- /dev/null
+++ b/src/additional_functions/start_val/common.jl
@@ -0,0 +1,17 @@
+
+# start values for SEM Models (including ensembles)
+function start_values(f, model::AbstractSem; kwargs...)
+ start_vals = fill(0.0, nparams(model))
+
+ # initialize parameters using the SEM loss terms
+ # (first SEM loss term that sets given parameter to nonzero value)
+ for term in loss_terms(model)
+ issemloss(term) || continue
+ term_start_vals = f(loss(term); kwargs...)
+ for (i, val) in enumerate(term_start_vals)
+ iszero(val) || (start_vals[i] = val)
+ end
+ end
+
+ return start_vals
+end
diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl
index ab79d9ada..543370285 100644
--- a/src/additional_functions/start_val/start_fabin3.jl
+++ b/src/additional_functions/start_val/start_fabin3.jl
@@ -7,21 +7,17 @@ Not available for ensemble models.
function start_fabin3 end
# splice model and loss functions
-function start_fabin3(model::AbstractSemSingle; kwargs...)
- return start_fabin3(model.observed, model.implied, model.loss.functions..., kwargs...)
+function start_fabin3(model::SemLoss; kwargs...)
+ return start_fabin3(model.observed, model.implied; kwargs...)
end
-function start_fabin3(observed, implied, args...; kwargs...)
- return start_fabin3(implied.ram_matrices, obs_cov(observed), obs_mean(observed))
-end
-
-# SemObservedMissing
-function start_fabin3(observed::SemObservedMissing, implied, args...; kwargs...)
- if !observed.em_model.fitted
- em_mvn!(observed; kwargs...)
- end
-
- return start_fabin3(implied.ram_matrices, observed.em_model.Σ, observed.em_model.μ)
+function start_fabin3(observed::SemObserved, implied::SemImplied; kwargs...)
+ return start_fabin3(
+ implied.ram_matrices,
+ obs_cov(observed),
+ # ignore observed means if no meansturcture
+ !isnothing(implied.ram_matrices.M) ? obs_mean(observed) : nothing,
+ )
end
function start_fabin3(
@@ -170,3 +166,6 @@ end
function is_in_Λ(ind_vec, F_ind)
return any(ind -> !(ind[2] ∈ F_ind) && (ind[1] ∈ F_ind), ind_vec)
end
+
+# ensembles
+start_fabin3(model::AbstractSem; kwargs...) = start_values(start_fabin3, model; kwargs...)
diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl
index 4fbc8719c..afdbf92ee 100644
--- a/src/additional_functions/start_val/start_simple.jl
+++ b/src/additional_functions/start_val/start_simple.jl
@@ -15,34 +15,11 @@ Return a vector of simple starting values.
"""
function start_simple end
-# Single Models ----------------------------------------------------------------------------
-function start_simple(model::AbstractSemSingle; kwargs...)
- return start_simple(model.observed, model.implied, model.loss.functions...; kwargs...)
-end
-
-function start_simple(observed, implied, args...; kwargs...)
- return start_simple(implied.ram_matrices; kwargs...)
-end
-
-# Ensemble Models --------------------------------------------------------------------------
-function start_simple(model::SemEnsemble; kwargs...)
- start_vals = []
-
- for sem in model.sems
- push!(start_vals, start_simple(sem; kwargs...))
- end
-
- has_start_val = [.!iszero.(start_val) for start_val in start_vals]
+start_simple(model::SemLoss; kwargs...) =
+ start_simple(observed(model), implied(model); kwargs...)
- start_val = similar(start_vals[1])
- start_val .= 0.0
-
- for (j, indices) in enumerate(has_start_val)
- start_val[indices] .= start_vals[j][indices]
- end
-
- return start_val
-end
+start_simple(observed::SemObserved, implied::SemImplied; kwargs...) =
+ start_simple(implied.ram_matrices; kwargs...)
function start_simple(
ram_matrices::RAMMatrices;
@@ -103,3 +80,6 @@ function start_simple(
end
return start_val
end
+
+# multigroup models
+start_simple(model::AbstractSem; kwargs...) = start_values(start_simple, model; kwargs...)
diff --git a/src/frontend/StatsAPI.jl b/src/frontend/StatsAPI.jl
index edd677e34..b00c451af 100644
--- a/src/frontend/StatsAPI.jl
+++ b/src/frontend/StatsAPI.jl
@@ -13,11 +13,7 @@ Note that the function combines the duplicate occurences of the
same parameter in `partable` and will raise an error if the
values do not match.
"""
-function params!(
- out::AbstractVector,
- partable::ParameterTable,
- col::Symbol = :estimate,
-)
+function params!(out::AbstractVector, partable::ParameterTable, col::Symbol = :estimate)
(length(out) == nparams(partable)) || throw(
DimensionMismatch(
"The length of parameter values vector ($(length(out))) does not match the number of parameters ($(nparams(partable)))",
@@ -75,4 +71,8 @@ Synonymous to [`nsamples`](@ref).
"""
nobs(model::AbstractSem) = nsamples(model)
-coeftable(model::AbstractSem; level::Real=0.95) = throw(ArgumentError("StructuralEquationModels does not support the `CoefTable` interface; see [`ParameterTable`](@ref) instead."))
\ No newline at end of file
+coeftable(model::AbstractSem; level::Real = 0.95) = throw(
+ ArgumentError(
+ "StructuralEquationModels does not support the `CoefTable` interface; see [`ParameterTable`](@ref) instead.",
+ ),
+)
diff --git a/src/frontend/finite_diff.jl b/src/frontend/finite_diff.jl
new file mode 100644
index 000000000..ac8fbf627
--- /dev/null
+++ b/src/frontend/finite_diff.jl
@@ -0,0 +1,63 @@
+_unwrap(wrapper::SemFiniteDiff) = wrapper.model
+params(wrapper::SemFiniteDiff) = params(wrapper.model)
+loss_terms(wrapper::SemFiniteDiff) = loss_terms(wrapper.model)
+
+replace_observed(wrapper::SemFiniteDiff, data; kwargs...) =
+ SemFiniteDiff(replace_observed(wrapper.model, data; kwargs...))
+
+FiniteDiffLossWrappers = Union{LossFiniteDiff, SemLossFiniteDiff}
+
+_unwrap(term::AbstractLoss) = term
+_unwrap(wrapper::FiniteDiffLossWrappers) = wrapper.loss
+implied(wrapper::FiniteDiffLossWrappers) = implied(_unwrap(wrapper))
+observed(wrapper::FiniteDiffLossWrappers) = observed(_unwrap(wrapper))
+
+replace_observed(wrapper::LossFiniteDiff, data; kwargs...) =
+ LossFiniteDiff(replace_observed(_unwrap(wrapper), data; kwargs...))
+
+replace_observed(wrapper::SemLossFiniteDiff, new_observed::SemObserved; kwargs...) =
+ SemLossFiniteDiff(replace_observed(_unwrap(wrapper), new_observed; kwargs...))
+
+replace_observed(
+ wrapper::SemLossFiniteDiff,
+ data::Union{AbstractMatrix, DataFrame};
+ kwargs...,
+) = SemLossFiniteDiff(replace_observed(_unwrap(wrapper), data; kwargs...))
+
+"""
+ FiniteDiffWrapper(model::AbstractSem)
+ FiniteDiffWrapper(loss::AbstractLoss)
+
+Wrap a SEM `model` or an individual `loss` term so that its gradient and Hessian
+are approximated with finite differences of the objective (using the *FiniteDiff.jl*
+package) instead of dedicated analytic evaluation.
+
+Wrapping the whole `model` returns a [`SemFiniteDiff`](@ref). Wrapping a single loss
+term returns a loss wrapper that only uses the objective of the original term to
+compute its gradient/Hessian, which is useful in [Collections](@ref) where analytic
+gradients are available for some terms but not for others.
+"""
+FiniteDiffWrapper(model::AbstractSem) = SemFiniteDiff(model)
+FiniteDiffWrapper(loss::AbstractLoss) = LossFiniteDiff(loss)
+FiniteDiffWrapper(loss::SemLoss) = SemLossFiniteDiff(loss)
+
+function evaluate!(
+ objective,
+ gradient,
+ hessian,
+ sem::Union{SemFiniteDiff, FiniteDiffLossWrappers},
+ params,
+)
+ wrapped = _unwrap(sem)
+ obj(p) = _evaluate!(
+ objective_zero(objective, gradient, hessian),
+ nothing,
+ nothing,
+ wrapped,
+ p,
+ )
+ isnothing(gradient) || FiniteDiff.finite_difference_gradient!(gradient, obj, params)
+ isnothing(hessian) || FiniteDiff.finite_difference_hessian!(hessian, obj, params)
+ # FIXME if objective is not calculated, SemLoss implied states may not correspond to params
+ return !isnothing(objective) ? obj(params) : nothing
+end
diff --git a/src/frontend/fit/SemFit.jl b/src/frontend/fit/SemFit.jl
index 438da4da6..04de8badc 100644
--- a/src/frontend/fit/SemFit.jl
+++ b/src/frontend/fit/SemFit.jl
@@ -13,9 +13,10 @@ Fitted structural equation model.
- `model(::SemFit)`
- `optimization_result(::SemFit)`
-- `optimizer(::SemFit)` -> optimization algorithm
+- `algorithm_name(::SemFit)` -> optimization algorithm
- `n_iterations(::SemFit)` -> number of iterations
-- `convergence(::SemFit)` -> convergence properties
+- `convergence(::SemFit)` -> convergence flags
+- `converged(::SemFit)` -> convergence success
"""
mutable struct SemFit{Mi, So, St, Mo, O}
minimum::Mi
@@ -29,16 +30,20 @@ end
# pretty printing
############################################################################################
-function Base.show(io::IO, semfit::SemFit)
+function Base.show(io::IO, m::MIME"text/plain", semfit::SemFit)
print(io, "Fitted Structural Equation Model \n")
print(io, "=============================================== \n")
print(io, "--------------------- Model ------------------- \n")
print(io, "\n")
- print(io, semfit.model)
+ show(io, m, semfit.model)
print(io, "\n")
#print(io, "Objective value: $(round(semfit.minimum, digits = 4)) \n")
print(io, "------------- Optimization result ------------- \n")
print(io, "\n")
+ print(io, "engine: ")
+ print(io, optimizer_engine(semfit))
+ print(io, "\n")
+ print(io, "\n")
print(io, semfit.optimization_result)
end
@@ -52,12 +57,19 @@ nsamples(fit::SemFit) = nsamples(fit.model)
# access fields
minimum(sem_fit::SemFit) = sem_fit.minimum
+"""
+ solution(sem_fit::SemFit)
+
+Returns the vector of parameter estimates from a fitted SEM.
+"""
solution(sem_fit::SemFit) = sem_fit.solution
start_val(sem_fit::SemFit) = sem_fit.start_val
model(sem_fit::SemFit) = sem_fit.model
optimization_result(sem_fit::SemFit) = sem_fit.optimization_result
# optimizer properties
-optimizer(sem_fit::SemFit) = optimizer(optimization_result(sem_fit))
+optimizer_engine(sem_fit::SemFit) = optimizer_engine(optimization_result(sem_fit))
+algorithm_name(sem_fit::SemFit) = algorithm_name(optimization_result(sem_fit))
n_iterations(sem_fit::SemFit) = n_iterations(optimization_result(sem_fit))
convergence(sem_fit::SemFit) = convergence(optimization_result(sem_fit))
+converged(sem_fit::SemFit) = converged(optimization_result(sem_fit))
diff --git a/src/frontend/fit/fitmeasures/AIC.jl b/src/frontend/fit/fitmeasures/AIC.jl
index f26f1f4dc..91bc09810 100644
--- a/src/frontend/fit/fitmeasures/AIC.jl
+++ b/src/frontend/fit/fitmeasures/AIC.jl
@@ -1,6 +1,9 @@
"""
- AIC(sem_fit::SemFit)
+ AIC(fit::SemFit)
-Return the akaike information criterion.
+Calculate the *AIC* ([*Akaike information criterion*](https://en.wikipedia.org/wiki/Akaike_information_criterion)).
+
+# See also
+[`fit_measures`](@ref)
"""
-AIC(sem_fit::SemFit) = minus2ll(sem_fit) + 2nparams(sem_fit)
+AIC(fit::SemFit) = minus2ll(fit) + 2nparams(fit)
diff --git a/src/frontend/fit/fitmeasures/BIC.jl b/src/frontend/fit/fitmeasures/BIC.jl
index 20638f4e4..44190375a 100644
--- a/src/frontend/fit/fitmeasures/BIC.jl
+++ b/src/frontend/fit/fitmeasures/BIC.jl
@@ -1,6 +1,9 @@
"""
- BIC(sem_fit::SemFit)
+ BIC(fit::SemFit)
-Return the bayesian information criterion.
+Calculate the *BIC* ([*Bayesian information criterion*](https://en.wikipedia.org/wiki/Bayesian_information_criterion)).
+
+# See also
+[`fit_measures`](@ref)
"""
-BIC(sem_fit::SemFit) = minus2ll(sem_fit) + log(nsamples(sem_fit)) * nparams(sem_fit)
+BIC(fit::SemFit) = minus2ll(fit) + log(nsamples(fit)) * nparams(fit)
diff --git a/src/frontend/fit/fitmeasures/CFI.jl b/src/frontend/fit/fitmeasures/CFI.jl
new file mode 100644
index 000000000..36871b89e
--- /dev/null
+++ b/src/frontend/fit/fitmeasures/CFI.jl
@@ -0,0 +1,74 @@
+"""
+ (1) CFI(fit::SemFit, fit_baseline::SemFit)
+
+ (2) CFI(fit::SemFit)
+
+Calculate the Comparative Fit Index (CFI).
+
+The CFI ranges from 0-1 and measures how much better the model
+fits the data compared to a baseline model.
+If no baseline model is provided, a model with unconstrained
+variances (and means) is compaired against.
+For multigroup models, variances (and means) per group are free
+without any equality constraints between groups.
+"""
+function CFI end
+
+# if the user provides a baseline model
+CFI(fit::SemFit, fit_baseline::SemFit) =
+ CFI(χ²(fit), dof(fit), χ²(fit_baseline), dof(fit_baseline))
+
+# no baseline -> variance only model
+CFI(fit::SemFit) = CFI(fit, fit.model)
+
+function CFI(fit::SemFit, model::AbstractSem)
+ dof₀ = dof_varonly(model)
+ χ²₀ = χ²_varonly(model)
+ return CFI(χ²(fit), dof(fit), χ²₀, dof₀)
+end
+
+# basic CFI function
+function CFI(χ², dof, χ²₀, dof₀)
+ λ = χ² - dof
+ λ₀ = χ²₀ - dof₀
+ return 1 - maximum([λ, 0])/maximum([λ, λ₀, 0])
+end
+
+###
+function χ²_varonly(model::AbstractSem)
+ check_same_semterm_type(model; throw_error = true)
+ return sum(sem_terms(model)) do semterm
+ χ²_varonly(_unwrap(loss(semterm)))
+ end
+end
+
+function χ²_varonly(loss::SemML)
+ N⁻ = (nsamples(loss) - 1)
+ S = obs_cov(observed(loss))
+ Σ₀ = Diagonal(S)
+ p = nobserved_vars(loss)
+ return N⁻*(logdet(Σ₀) + tr(inv(Σ₀)*S) - logdet(S) - p)
+end
+
+# for the optimal variance only model, we have to solve 1/2 tr((I-XS⁻¹)^2) with X diagonal
+function χ²_varonly(loss::SemWLS)
+ N⁻ = (nsamples(loss) - 1)
+ S⁻¹ = inv((obs_cov(observed(loss))))
+ Σ₀ = Diagonal(inv(S⁻¹ .* S⁻¹)*diag(S⁻¹))
+ return N⁻*0.5*tr((I - Σ₀*S⁻¹)^2)
+end
+
+# For FIML, the variance-only baseline cannot be derived automatically, so the CFI is
+# `missing` unless an explicit baseline model is passed via `CFI(fit, fit_baseline)`.
+# Returning `missing` (instead of throwing) keeps `fit_measures()` usable for FIML models.
+χ²_varonly(loss::SemFIML) = missing
+
+function dof_varonly(model::AbstractSem)
+ return sum(sem_terms(model)) do semterm
+ nparams_varonly = nobserved_vars(semterm)
+ if MeanStruct(implied(semterm)) === HasMeanStruct
+ nparams_varonly *= 2
+ end
+ return n_dp(loss(semterm)) - nparams_varonly
+ end
+end
diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl
index b9fff648e..9d33e47e4 100644
--- a/src/frontend/fit/fitmeasures/RMSEA.jl
+++ b/src/frontend/fit/fitmeasures/RMSEA.jl
@@ -1,18 +1,31 @@
"""
- RMSEA(sem_fit::SemFit)
+ RMSEA(fit::SemFit)
-Return the RMSEA.
-"""
-function RMSEA end
+Calculate the RMSEA ([*Root Mean Squared Error of Approximation*](https://meth.psychopen.eu/index.php/meth/article/download/2333/2333.html?inline=1#sec1)).
+
+Uses the formula
+```math
+\\mathrm{RMSEA} = \\sqrt{\\frac{\\chi^2 - N_{\\mathrm{df}}}{N_{\\mathrm{obs}} * N_{\\mathrm{df}}}},
+```
+where *χ²* is the chi-squared statistic, ``N_{\\mathrm{df}}`` is the degrees of freedom,
+and ``N_{\\mathrm{obs}}`` is the (corrected) number of observations
+for the SEM model.
-RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) =
- RMSEA(dof(sem_fit), χ²(sem_fit), nsamples(sem_fit))
+# See also
+[`fit_measures`](@ref), [`χ²`](@ref), [`dof`](@ref)
-RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}) =
- sqrt(length(sem_fit.model.sems)) * RMSEA(dof(sem_fit), χ²(sem_fit), nsamples(sem_fit))
+# Extended help
-function RMSEA(dof, chi2, nsamples)
- rmsea = (chi2 - dof) / (nsamples * dof)
- rmsea > 0 ? nothing : rmsea = 0
- return sqrt(rmsea)
+For multigroup models, the correction proposed by J.H. Steiger is applied
+(see [Steiger, J. H. (1998). *A note on multiple sample extensions of the RMSEA fit index*](https://doi.org/10.1080/10705519809540115)).
+"""
+RMSEA(fit::SemFit) = RMSEA(fit, fit.model)
+
+function RMSEA(fit::SemFit, model::AbstractSem)
+ term_type = check_same_semterm_type(model; throw_error = true)
+ n = nsamples(fit) + nsem_terms(model) * multigroup_correction_scale(term_type)
+ sqrt(nsem_terms(model)) * RMSEA(dof(fit), χ²(fit), n)
end
+
+RMSEA(dof::Number, chi2::Number, nsamples::Number) =
+ sqrt(max((chi2 - dof) / (nsamples * dof), 0.0))
diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl
index 333783f95..444d9678b 100644
--- a/src/frontend/fit/fitmeasures/chi2.jl
+++ b/src/frontend/fit/fitmeasures/chi2.jl
@@ -1,89 +1,45 @@
"""
- χ²(sem_fit::SemFit)
+ χ²(fit::SemFit)
-Return the χ² value.
-"""
-function χ² end
-
-############################################################################################
-# Single Models
-############################################################################################
-
-# SemFit splices loss functions ------------------------------------------------------------
-χ²(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) = χ²(
- sem_fit,
- sem_fit.model.observed,
- sem_fit.model.implied,
- sem_fit.model.loss.functions...,
-)
-
-# RAM + SemML
-χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemML) =
- (nsamples(sem_fit) - 1) *
- (sem_fit.minimum - logdet(observed.obs_cov) - nobserved_vars(observed))
-
-# bollen, p. 115, only correct for GLS weight matrix
-χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemWLS) =
- (nsamples(sem_fit) - 1) * sem_fit.minimum
-
-# FIML
-function χ²(sem_fit::SemFit, observed::SemObservedMissing, imp, loss_ml::SemFIML)
- ll_H0 = minus2ll(sem_fit)
- ll_H1 = minus2ll(observed)
- chi2 = ll_H0 - ll_H1
- return chi2
-end
-
-############################################################################################
-# Collections
-############################################################################################
+Calculate the *χ²* (*chi-square*) value for the `fit`.
-# SemFit splices loss functions ------------------------------------------------------------
-χ²(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}) =
- χ²(sem_fit, sem_fit.model, sem_fit.model.sems[1].loss.functions[1])
+The *χ²* is a test statistic for the SEM goodness-of-fit.
+It compares the *implied* covariance matrix of the SEM model
+with the *observed* covariance matrix.
-function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemWLS}
- check_ensemble_length(model)
- check_lossfun_types(model, L)
- return (nsamples(model) - 1) * sem_fit.minimum
-end
-
-function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemML}
- check_ensemble_length(model)
- check_lossfun_types(model, L)
- F_G = sem_fit.minimum
- F_G -= sum([
- w * (logdet(m.observed.obs_cov) + nobserved_vars(m.observed)) for
- (w, m) in zip(model.weights, model.sems)
- ])
- return (nsamples(model) - 1) * F_G
-end
-
-function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemFIML}
- check_ensemble_length(model)
- check_lossfun_types(model, L)
+# See also
+[`fit_measures`](@ref)
+"""
+χ²(fit::SemFit) = χ²(fit, fit.model)
- ll_H0 = minus2ll(sem_fit)
- ll_H1 = sum(minus2ll.(observed.(models(model))))
- chi2 = ll_H0 - ll_H1
+function χ²(fit::SemFit, model::AbstractSem)
+ terms = sem_terms(model)
+ @assert !isempty(terms)
- return chi2
+ L = check_same_semterm_type(model; throw_error = true)
+ return χ²(L, fit, model)
end
-function check_ensemble_length(model)
- for sem in model.sems
- if length(sem.loss.functions) > 1
- @error "A model for one of the groups contains multiple loss functions."
+# bollen, p. 115, only correct for GLS weight matrix
+# N - G (= Σ(Nᵍ-1)) multiplier; reduces to N-1 for a single group
+χ²(::Type{<:SemWLS}, fit::SemFit, model::AbstractSem) =
+ (nsamples(model) - nsem_terms(model)) * fit.minimum
+
+function χ²(::Type{<:SemML}, fit::SemFit, model::AbstractSem)
+ G = sum(loss_terms(model)) do term
+ if issemloss(term)
+ data = observed(term)
+ something(weight(term), 1.0) * (logdet(obs_cov(data)) + nobserved_vars(data))
+ else
+ return 0.0
end
end
+ # N - G (= Σ(Nᵍ-1)) multiplier; reduces to N-1 for a single group
+ return (nsamples(model) - nsem_terms(model)) * (fit.minimum - G)
end
-function check_lossfun_types(model, type)
- for sem in model.sems
- for lossfun in sem.loss.functions
- if !isa(lossfun, type)
- @error "Your model(s) contain multiple lossfunctions with differing types."
- end
- end
- end
+function χ²(::Type{<:SemFIML}, fit::SemFit, model::AbstractSem)
+ ll_H0 = minus2ll(fit)
+ ll_H1 = sum(minus2ll ∘ observed, sem_terms(model))
+ return ll_H0 - ll_H1
end
diff --git a/src/frontend/fit/fitmeasures/dof.jl b/src/frontend/fit/fitmeasures/dof.jl
index 3df49d89d..49b7febf1 100644
--- a/src/frontend/fit/fitmeasures/dof.jl
+++ b/src/frontend/fit/fitmeasures/dof.jl
@@ -1,22 +1,33 @@
"""
- dof(sem_fit::SemFit)
+ dof(fit::SemFit)
dof(model::AbstractSem)
-Return the degrees of freedom.
+Get the *degrees of freedom* for the SEM model.
+
+The *degrees of freedom* for the SEM with *N* observed variables is the difference
+between the number of non-redundant elements in the observed covariance matrix
+(*½N(N+1)*) and the number of model parameters, *q* ([`nparams(model)`](@ref nparams)).
+If the SEM also models the observed means, the formula becomes *½N(N+1) + N - q*.
+
+# See also
+[`fit_measures`](@ref)
"""
function dof end
-dof(sem_fit::SemFit) = dof(sem_fit.model)
+dof(fit::SemFit) = dof(fit.model)
dof(model::AbstractSem) = n_dp(model) - nparams(model)
-function n_dp(model::AbstractSemSingle)
- nvars = nobserved_vars(model)
+# length of Σ and μ (if present)
+function n_dp(implied::SemImplied)
+ nvars = nobserved_vars(implied)
ndp = 0.5(nvars^2 + nvars)
- if !isnothing(model.implied.μ)
+ if !isnothing(implied.μ)
ndp += nvars
end
return ndp
end
-n_dp(model::SemEnsemble) = sum(n_dp.(model.sems))
+n_dp(term::SemLoss) = n_dp(implied(term))
+
+n_dp(model::AbstractSem) = sum(n_dp ∘ loss, sem_terms(model))
diff --git a/src/frontend/fit/fitmeasures/fit_measures.jl b/src/frontend/fit/fitmeasures/fit_measures.jl
index afdde173b..7fabc9503 100644
--- a/src/frontend/fit/fitmeasures/fit_measures.jl
+++ b/src/frontend/fit/fitmeasures/fit_measures.jl
@@ -1,19 +1,28 @@
-fit_measures(sem_fit) =
- fit_measures(sem_fit, nparams, dof, AIC, BIC, RMSEA, χ², p_value, minus2ll)
+const DEFAULT_FIT_MEASURES = [AIC, BIC, dof, χ², p_value, nparams, RMSEA, CFI]
-function fit_measures(sem_fit, args...)
- measures = Dict{Symbol, Union{Float64, Missing}}()
+fit_measures(fit, measures::AbstractVector) = Dict(Symbol(fn) => fn(fit) for fn in measures)
+fit_measures(fit, measures...) = fit_measures(fit, measures)
+fit_measures(fit) = fit_measures(fit, DEFAULT_FIT_MEASURES)
- for arg in args
- push!(measures, Symbol(arg) => arg(sem_fit))
- end
+"""
+ fit_measures(fit::SemFit, measures...) -> Dict{Symbol}
- return measures
-end
+Calculate fit measures for the SEM solution.
-"""
- fit_measures(sem_fit, args...)
+The `measures` are the functions that calculate fit measures for a given SEM solution
+([`SemFit`](@ref) object). If no `measures` are specified, the default set of measures is used.
+
+Returns a dictionary of the fit measures, where the keys are the function names.
+
+# Examples
+
+```julia
+fit_measures(semfit)
+fit_measures(semfit, nparams, dof, p_value)
+```
-Return a default set of fit measures or the fit measures passed as `args...`.
+# See also
+[`AIC`](@ref), [`BIC`](@ref), [`RMSEA`](@ref), [`χ²`](@ref), [`p_value`](@ref),
+[`minus2ll`](@ref), [`CFI`](@ref)
"""
-function fit_measures end
+fit_measures
diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl
index ab4d24e53..1cdf5c07b 100644
--- a/src/frontend/fit/fitmeasures/minus2ll.jl
+++ b/src/frontend/fit/fitmeasures/minus2ll.jl
@@ -1,60 +1,55 @@
"""
- minus2ll(sem_fit::SemFit)
+ minus2ll(fit::SemFit)
-Return the negative 2* log likelihood.
+Calculate the *-2⋅log(likelihood(fit))*.
+
+# See also
+[`fit_measures`](@ref)
"""
-function minus2ll end
+minus2ll(fit::SemFit) = minus2ll(fit.model, fit)
############################################################################################
-# Single Models
+# Single SEM Terms Models
############################################################################################
-# SemFit splices loss functions ------------------------------------------------------------
-minus2ll(
- sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O},
-) = minus2ll(
- sem_fit,
- sem_fit.model.observed,
- sem_fit.model.implied,
- sem_fit.model.loss.functions...,
-)
-
-minus2ll(sem_fit::SemFit, obs, imp, args...) = minus2ll(sem_fit.minimum, obs, imp, args...)
+function minus2ll(term::SemLoss, fit::SemFit)
+ minimum = objective(term, fit.solution)
+ return minus2ll(term, minimum)
+end
-# SemML ------------------------------------------------------------------------------------
-minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, loss_ml::SemML) =
- nsamples(obs) * (minimum + log(2π) * nobserved_vars(obs))
+minus2ll(term::SemML, minimum::Number) =
+ nsamples(term) * (minimum + log(2π) * nobserved_vars(term))
-# WLS --------------------------------------------------------------------------------------
-minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, loss_ml::SemWLS) = missing
+minus2ll(term::SemWLS, minimum::Number) = missing
# compute likelihood for missing data - H0 -------------------------------------------------
# -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n
-function minus2ll(minimum::Number, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemFIML)
- F = minimum * nsamples(observed)
- F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), observed.patterns)
+function minus2ll(term::SemFIML, minimum::Number)
+ obs = observed(term)::SemObservedMissing
+ F = minimum * nsamples(obs)
+ F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), obs.patterns)
return F
end
# compute likelihood for missing data - H1 -------------------------------------------------
# -2ll = ∑ log(2π)*(nᵢ + mᵢ) + ln(Σᵢ) + (mᵢ - μᵢ)ᵀ Σᵢ⁻¹ (mᵢ - μᵢ)) + tr(SᵢΣᵢ)
function minus2ll(observed::SemObservedMissing)
- # fit EM-based mean and cov if not yet fitted
- # FIXME EM could be very computationally expensive
- observed.em_model.fitted || em_mvn!(observed)
-
- Σ = observed.em_model.Σ
- μ = observed.em_model.μ
+ Σ, μ = obs_cov(observed), obs_mean(observed)
+ # FIXME: this code is duplicate to objective(fiml, ...)
F = sum(observed.patterns) do pat
# implied covariance/mean
Σᵢ = Σ[pat.measured_mask, pat.measured_mask]
Σᵢ_chol = cholesky!(Σᵢ)
ld = logdet(Σᵢ_chol)
Σᵢ⁻¹ = LinearAlgebra.inv!(Σᵢ_chol)
- meandiffᵢ = pat.measured_mean - μ[pat.measured_mask]
+ μ_diffᵢ = pat.measured_mean - μ[pat.measured_mask]
- F_one_pattern(meandiffᵢ, Σᵢ⁻¹, pat.measured_cov, ld, nsamples(pat))
+ F_pat = ld + dot(μ_diffᵢ, Σᵢ⁻¹, μ_diffᵢ)
+ if nsamples(pat) > 1
+ F_pat += dot(pat.measured_cov, Σᵢ⁻¹)
+ end
+ F_pat * nsamples(pat)
end
F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), observed.patterns)
@@ -63,19 +58,10 @@ function minus2ll(observed::SemObservedMissing)
end
############################################################################################
-# Collection
+# Multi-group
############################################################################################
-minus2ll(minimum, model::AbstractSemSingle) =
- minus2ll(minimum, model.observed, model.implied, model.loss.functions...)
-
-function minus2ll(
- sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O},
-)
- m2ll = 0.0
- for sem in sem_fit.model.sems
- minimum = objective!(sem, sem_fit.solution)
- m2ll += minus2ll(minimum, sem)
- end
- return m2ll
+function minus2ll(model::AbstractSem, fit::SemFit)
+ check_same_semterm_type(model; throw_error = true)
+ sum(Base.Fix2(minus2ll, fit) ∘ _unwrap ∘ loss, sem_terms(model))
end
diff --git a/src/frontend/fit/fitmeasures/p.jl b/src/frontend/fit/fitmeasures/p.jl
index 8c69d5ec2..50cf7220c 100644
--- a/src/frontend/fit/fitmeasures/p.jl
+++ b/src/frontend/fit/fitmeasures/p.jl
@@ -1,6 +1,9 @@
"""
- p(sem_fit::SemFit)
+ p_value(fit::SemFit)
-Return the p value computed from the χ² test statistic.
+Calculate the *p*-value for the *χ²* test statistic.
+
+# See also
+[`fit_measures`](@ref), [`χ²`](@ref)
"""
-p_value(sem_fit::SemFit) = 1 - cdf(Chisq(dof(sem_fit)), χ²(sem_fit))
+p_value(fit::SemFit) = ccdf(Chisq(dof(fit)), χ²(fit))
diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl
index 4589dc020..a8c1d3932 100644
--- a/src/frontend/fit/standard_errors/bootstrap.jl
+++ b/src/frontend/fit/standard_errors/bootstrap.jl
@@ -1,81 +1,259 @@
+# base type for accumulators of intermediate bootstrap results
+abstract type BootstrapAccumulator end
+
+# internal function to run bootstrap
+function bootstrap!(
+ acc::BootstrapAccumulator,
+ fitted::SemFit;
+ data = nothing,
+ engine = :Optim,
+ parallel = false,
+ fit_kwargs = Dict(),
+)
+ sem = model(fitted)
+ data = isnothing(data) ? _bootstrap_data(sem) : data
+ start = solution(fitted)
+
+ n_boot = n_bootstrap(acc)
+
+ # fit to bootstrap samples
+ if !parallel
+ bs_sem = deepcopy(sem) # avoid mutating the original model
+ for i in 1:n_boot
+ new_fit = _fit_bootstrap_sample(bs_sem, data, start; engine, fit_kwargs)
+ update!(acc, i, new_fit, nothing)
+ end
+ else
+ n_threads = Threads.nthreads()
+ # Pre-create one independent model copy per thread via deepcopy.
+ model_pool = Channel(n_threads)
+ for _ in 1:n_threads
+ put!(model_pool, deepcopy(sem))
+ end
+ lk = ReentrantLock()
+ Threads.@threads for i in 1:n_boot
+ thread_model = take!(model_pool)
+ new_fit = _fit_bootstrap_sample(thread_model, data, start; engine, fit_kwargs)
+ update!(acc, i, new_fit, lk)
+ put!(model_pool, thread_model)
+ end
+ end
+
+ return acc
+end
+
+# a simple accumulator that just stores the statistic for each sample and whether it converged
+struct SimpleBootstrapAccumulator{F} <: BootstrapAccumulator
+ statistic::F
+ samples::Vector{Any}
+ converged_mask::Vector{Bool}
+end
+
+SimpleBootstrapAccumulator(statistic, n_boot::Integer) =
+ SimpleBootstrapAccumulator(statistic, Vector{Any}(undef, n_boot), fill(false, n_boot))
+
+n_bootstrap(acc::SimpleBootstrapAccumulator) = length(acc.samples)
+
+function update!(acc::SimpleBootstrapAccumulator, i::Integer, fit::SemFit, _)
+ acc.samples[i] = acc.statistic(fit)
+ acc.converged_mask[i] = converged(fit)
+end
+
+"""
+ struct BootstrapResult{T}
+
+Stores the output of a [`bootstrap`](@ref) call.
"""
- se_bootstrap(sem_fit::SemFit; n_boot = 3000, data = nothing, kwargs...)
+struct BootstrapResult{T}
+ samples::Vector{T}
+ converged_mask::BitVector
+ n_boot::Int
+ n_converged::Int
+end
+
+function Base.show(io::IO, result::BootstrapResult{T}) where {T}
+ println(
+ io,
+ "BootstrapResult{$(T)} with $(result.n_converged)/$(result.n_boot) converged samples",
+ )
+end
+
+"""
+ bootstrap(
+ fitted::SemFit;
+ statistic = solution,
+ n_boot = 3000,
+ data = nothing,
+ engine = :Optim,
+ parallel = false,
+ fit_kwargs = Dict()
+ ) -> BootstrapResult
+
+Bootstrap the samples and apply `statistic` function to each.
-Return boorstrap standard errors.
-Only works for single models.
+Returns a [`BootstrapResult`](@ref) object containing the results of `statistic`
+applied to each bootstrapped sample.
+
+Supports both single-group and multi-group models.
+For multi-group models, each group is resampled independently.
# Arguments
+- `fitted`: a fitted SEM.
+- `statistic`: any function that can be called on a `SemFit` object.
+ The output will be returned as the bootstrap sample.
- `n_boot`: number of boostrap samples
-- `data`: data to sample from. Only needed if different than the data from `sem_fit`
-- `kwargs...`: passed down to `replace_observed`
+- `data`: data to sample from. Only needed if different than the fitted model.
+ For multi-group models, pass a `Dict{Symbol}` mapping term ids to data matrices.
+- `engine`: optimizer engine, passed to `fit`.
+- `parallel`: if `true`, run bootstrap samples in parallel on all available threads.
+ The number of threads is controlled by the `JULIA_NUM_THREADS` environment variable or
+ the `--threads` flag when starting Julia.
+- `fit_kwargs` : a `Dict` controlling model fitting for each bootstrap sample,
+ passed to [`fit`](@ref)
+
+# Example
+```julia
+# 1000 bootstrap samples of the minimum, fitted with :Optim
+bootstrap(
+ fitted;
+ statistic = StructuralEquationModels.minimum,
+ n_boot = 1000,
+ engine = :Optim,
+)
+```
"""
-function se_bootstrap(
- semfit::SemFit;
+function bootstrap(
+ fitted::SemFit,
+ statistic = solution;
n_boot = 3000,
data = nothing,
- specification = nothing,
- kwargs...,
+ engine = :Optim,
+ parallel = false,
+ fit_kwargs = Dict(),
)
- if model(semfit) isa AbstractSemCollection
- throw(
- ArgumentError(
- "bootstrap standard errors for ensemble models are not available yet",
- ),
- )
- end
+ acc = SimpleBootstrapAccumulator(statistic, n_boot)
+ bootstrap!(acc, fitted; data, engine, parallel, fit_kwargs)
+ return BootstrapResult(
+ [s for s in acc.samples],
+ convert(BitVector, acc.converged_mask),
+ n_bootstrap(acc),
+ sum(acc.converged_mask),
+ )
+end
- if isnothing(data)
- data = samples(observed(model(semfit)))
- end
+# bootstrap accumulator for se_bootstrap()
+# accumulates per-parameter sum and sum of squares across bootstrap samples
+struct StdErrBootstrapAccumulator <: BootstrapAccumulator
+ n_boot::Int
+ sum::Vector{Float64}
+ squared_sum::Vector{Float64}
+ n_converged::Ref{Int}
+end
- data = prepare_data_bootstrap(data)
+n_bootstrap(acc::StdErrBootstrapAccumulator) = acc.n_boot
- start = solution(semfit)
+StdErrBootstrapAccumulator(n_params::Integer, n_boot::Integer) =
+ StdErrBootstrapAccumulator(n_boot, zeros(n_params), zeros(n_params), Ref(0))
- new_solution = zero(start)
- sum = zero(start)
- squared_sum = zero(start)
+function update!(
+ acc::StdErrBootstrapAccumulator,
+ i::Integer,
+ fit::SemFit,
+ lk::Union{Base.AbstractLock, Nothing},
+)
+ conv = converged(fit)
+ if conv
+ sol = solution(fit)
+ isnothing(lk) || lock(lk)
+ acc.n_converged[] += 1
+ @. acc.sum += sol
+ @. acc.squared_sum += abs2(sol)
+ isnothing(lk) || unlock(lk)
+ end
+end
- n_failed = 0.0
+"""
+ se_bootstrap(fitted::SemFit; n_boot = 3000, kwargs...)
- converged = true
+Calculate standard errors using bootstrap approach.
- for _ in 1:n_boot
- sample_data = bootstrap_sample(data)
- new_model = replace_observed(
- model(semfit);
- data = sample_data,
- specification = specification,
- kwargs...,
- )
+Supports both single-group and multi-group models.
+For multi-group models, each group is resampled independently.
- new_solution .= 0.0
+# Arguments
+- `fitted`: a fitted SEM.
+- `n_boot`: number of boostrap samples
+- `data`: data to sample from. Only needed if different than the fitted model.
+ For multi-group models, pass a `Dict{Symbol}` mapping term ids to data matrices.
+- `engine`: optimizer engine, passed to `fit`.
+- `parallel`: if `true`, run bootstrap samples in parallel on all available threads.
+ The number of threads is controlled by the `JULIA_NUM_THREADS` environment variable or
+ the `--threads` flag when starting Julia.
+- `fit_kwargs` : a `Dict` controlling model fitting for each bootstrap sample,
+ passed to [`fit`](@ref)
- try
- new_solution = solution(fit(new_model; start_val = start))
- catch
- n_failed += 1
- end
- @. sum += new_solution
- @. squared_sum += new_solution^2
+# Example
+```julia
+# 1000 boostrap samples, fitted with :NLopt
+using NLopt
+
+se_bootstrap(
+ fitted;
+ n_boot = 1000,
+ engine = :NLopt,
+)
+```
+"""
+function se_bootstrap(fitted::SemFit; n_boot = 3000, kwargs...)
+ acc = StdErrBootstrapAccumulator(nparams(fitted), n_boot)
+ bootstrap!(acc, fitted; kwargs...)
+ n_conv = acc.n_converged[]
+ @info "$n_conv models converged"
- converged = true
+ if n_conv == 0
+ @warn "No bootstrap samples converged. Returning NaN."
+ return fill(NaN, length(acc.sum))
+ else
+ return sqrt.(acc.squared_sum ./ n_conv - abs2.(acc.sum / n_conv))
end
+end
+
+############################################################################################
+### Helper Functions
+############################################################################################
- n_conv = n_boot - n_failed
- sd = sqrt.(squared_sum / n_conv - (sum / n_conv) .^ 2)
- print("Number of nonconverged models: ", n_failed, "\n")
- return sd
+"""
+ resample_with_replacement(data::AbstractMatrix)
+ resample_with_replacement(data::AbstractVector{<:AbstractMatrix})
+
+Resample rows of a data matrix with replacement (bootstrap sample).
+For a vector of matrices (multi-group models), independently resamples each matrix.
+"""
+function resample_with_replacement(data::AbstractMatrix)
+ n = size(data, 1)
+ return data[rand(1:n, n), :]
end
-function prepare_data_bootstrap(data)
- return Matrix(data)
+function resample_with_replacement(data::AbstractVector{<:AbstractMatrix})
+ return [resample_with_replacement(term_data) for term_data in data]
+end
+
+# Extract data from a model for bootstrap resampling.
+function _bootstrap_data(sem::AbstractSem)
+ terms = sem_terms(sem)
+ if length(terms) == 1
+ return samples(observed(loss(terms[1])))
+ else
+ return [samples(observed(loss(term))) for term in terms]
+ end
end
-function bootstrap_sample(data)
- nobs = size(data, 1)
- index_new = rand(1:nobs, nobs)
- data_new = data[index_new, :]
- return data_new
+# Fit one bootstrap replicate: resample, replace observed data, fit.
+function _fit_bootstrap_sample(sem_model, data, start; engine, fit_kwargs)
+ boot_data = resample_with_replacement(data)
+ # we replace the observed data with the bootstrapped one,
+ # but preserve any internal state that is associated with the original data
+ boot_model = replace_observed(sem_model, boot_data; recompute_observed_state = true)
+ return fit(boot_model; start_val = start, engine = engine, fit_kwargs...)
end
diff --git a/src/frontend/fit/standard_errors/confidence_intervals.jl b/src/frontend/fit/standard_errors/confidence_intervals.jl
new file mode 100644
index 000000000..59c6bb7dd
--- /dev/null
+++ b/src/frontend/fit/standard_errors/confidence_intervals.jl
@@ -0,0 +1,43 @@
+_doc_normal_CI = """
+ (1) normal_CI(fitted, se; α = 0.05, name_lower = :ci_lower, name_upper = :ci_upper)
+
+ (2) normal_CI!(partable, fitted, se; α = 0.05, name_lower = :ci_lower, name_upper = :ci_upper)
+
+Return normal-theory confidence intervals for all model parameters.
+`normal_CI!` additionally writes the result into `partable`.
+
+# Arguments
+- `fitted`: a fitted SEM.
+- `se`: standard errors for each parameter, e.g. from [`se_hessian`](@ref) or
+ [`se_bootstrap`](@ref).
+- `partable`: a `ParameterTable` to write confidence intervals to.
+- `α`: significance level. Defaults to `0.05` (95% intervals).
+- `name_lower`: column name for the lower bound in `partable`. Defaults to `:ci_lower`.
+- `name_upper`: column name for the upper bound in `partable`. Defaults to `:ci_upper`.
+
+# Returns
+- a `Dict` with keys `name_lower` and `name_upper`, each mapping to a vector of bounds
+ over all parameters.
+"""
+
+@doc "$(_doc_normal_CI)"
+function normal_CI(fitted, se; α = 0.05, name_lower = :ci_lower, name_upper = :ci_upper)
+ qnt = quantile(Normal(0, 1), 1-α/2);
+ sol = solution(fitted)
+ return Dict(name_lower => sol - qnt*se, name_upper => sol + qnt*se)
+end
+
+@doc "$(_doc_normal_CI)"
+function normal_CI!(
+ partable,
+ fitted,
+ se;
+ α = 0.05,
+ name_lower = :ci_lower,
+ name_upper = :ci_upper,
+)
+ cis = normal_CI(fitted, se; α, name_lower, name_upper)
+ update_partable!(partable, name_lower, fitted, cis[name_lower])
+ update_partable!(partable, name_upper, fitted, cis[name_upper])
+ return cis
+end
diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl
index 6ae53407f..80b96d337 100644
--- a/src/frontend/fit/standard_errors/hessian.jl
+++ b/src/frontend/fit/standard_errors/hessian.jl
@@ -35,20 +35,21 @@ function se_hessian(fit::SemFit; method = :finitediff)
end
# Addition functions -------------------------------------------------------------
-function H_scaling(model::AbstractSemSingle)
- if length(model.loss.functions) > 1
- @warn "Hessian scaling for multiple loss functions is not implemented yet"
- end
- return H_scaling(model.loss.functions[1], model)
-end
-
-H_scaling(lossfun::SemML, model::AbstractSemSingle) = 2 / (nsamples(model) - 1)
+H_scaling(loss::SemML) = 2 / (nsamples(loss) - 1)
-function H_scaling(lossfun::SemWLS, model::AbstractSemSingle)
+function H_scaling(loss::SemWLS)
@warn "Standard errors for WLS are only correct if a GLS weight matrix (the default) is used."
- return 2 / (nsamples(model) - 1)
+ return 2 / (nsamples(loss) - 1)
end
-H_scaling(lossfun::SemFIML, model::AbstractSemSingle) = 2 / nsamples(model)
+H_scaling(loss::SemFIML) = 2 / nsamples(loss)
-H_scaling(model::SemEnsemble) = 2 / nsamples(model)
+function H_scaling(model::AbstractSem)
+ semterms = SEM.sem_terms(model)
+ if length(semterms) > 1
+ #@warn "Hessian scaling for multiple loss functions is not implemented yet"
+ return 2 / nsamples(model)
+ else
+ return length(semterms) >= 1 ? H_scaling(loss(semterms[1])) : 1.0
+ end
+end
diff --git a/src/frontend/fit/standard_errors/z_test.jl b/src/frontend/fit/standard_errors/z_test.jl
new file mode 100644
index 000000000..d912fd867
--- /dev/null
+++ b/src/frontend/fit/standard_errors/z_test.jl
@@ -0,0 +1,36 @@
+_doc_z_test = """
+ (1) z_test(fitted, se)
+
+ (2) z_test!(partable, fitted, se, name = :p_value)
+
+Return two-sided p-values from a z-test for each model parameter.
+
+Tests the null hypothesis that each parameter is zero using the test statistic
+`z = estimate / se`, which is compared against a standard normal distribution.
+`z_test!` additionally writes the result into `partable`.
+
+# Arguments
+- `fitted`: a fitted SEM.
+- `se`: standard errors for each parameter, e.g. from [`se_hessian`](@ref) or
+ [`se_bootstrap`](@ref).
+- `partable`: a `ParameterTable` to write p-values to.
+- `name`: column name for the p-values in `partable`. Defaults to `:p_value`.
+
+# Returns
+- a vector of p-values.
+"""
+
+@doc "$(_doc_z_test)"
+function z_test(fitted, se)
+ dev = solution(fitted) ./ se
+ dist = Normal(0, 1)
+ p = 2*ccdf.(dist, abs.(dev))
+ return p
+end
+
+@doc "$(_doc_z_test)"
+function z_test!(partable, fitted, se, name = :p_value)
+ p = z_test(fitted, se)
+ update_partable!(partable, name, fitted, p)
+ return p
+end
diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl
index 3071d5653..fe7ea9307 100644
--- a/src/frontend/fit/summary.jl
+++ b/src/frontend/fit/summary.jl
@@ -1,3 +1,38 @@
+function details(sem::AbstractSem)
+ print("Structural Equation Model")
+ print(_subtype_info(sem))
+ print("\n")
+ print("- Loss Functions \n")
+ for term in loss_terms(sem)
+ print(" > ")
+ details(term)
+ println()
+ end
+end
+
+function details(term::LossTerm)
+ if !issemloss(term)
+ print(term.loss)
+ else
+ println("Structural Equation Model Loss ($(nameof(typeof(term.loss))))")
+ if !isnothing(id(term))
+ print(" - id: $(id(term)) \n")
+ end
+ println(
+ " - Observed: $(nameof(typeof(observed(term)))) ($(nsamples(term)) samples)",
+ )
+ println(
+ " - Implied: $(nameof(typeof(implied(term)))) ($(nparams(term)) parameters)",
+ )
+ println(
+ " - Variables: $(nobserved_vars(term)) observed, $(nlatent_vars(term)) latent",
+ )
+ if !isnothing(weight(term))
+ print(" - weight: $(round(weight(term), digits=3))")
+ end
+ end
+end
+
function details(sem_fit::SemFit; show_fitmeasures = false, color = :light_cyan, digits = 2)
print("\n")
println("Fitted Structural Equation Model")
@@ -7,7 +42,8 @@ function details(sem_fit::SemFit; show_fitmeasures = false, color = :light_cyan,
color = color,
)
print("\n")
- println("Optimization algorithm: $(optimizer(sem_fit))")
+ println("Optimization engine: $(optimizer_engine(sem_fit))")
+ println("Optimization algorithm: $(algorithm_name(sem_fit))")
println("Convergence: $(convergence(sem_fit))")
println("No. iterations/evaluations: $(n_iterations(sem_fit))")
print("\n")
@@ -113,10 +149,10 @@ function details(
print("\n")
pretty_table(
loading_array;
- header = header_cols,
- tf = PrettyTables.tf_borderless,
+ column_labels = header_cols,
+ table_format = TextTableFormat(borders = text_table_borders__borderless),
alignment = :l,
- formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v,
+ formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v],
)
print("\n")
end
@@ -151,10 +187,10 @@ function details(
print("\n")
pretty_table(
regression_array;
- header = regression_columns,
- tf = PrettyTables.tf_borderless,
+ column_labels = regression_columns,
+ table_format = TextTableFormat(borders = text_table_borders__borderless),
alignment = :l,
- formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v,
+ formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v],
)
print("\n")
@@ -178,10 +214,10 @@ function details(
print("\n")
pretty_table(
var_array;
- header = var_columns,
- tf = PrettyTables.tf_borderless,
+ column_labels = var_columns,
+ table_format = TextTableFormat(borders = text_table_borders__borderless),
alignment = :l,
- formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v,
+ formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v],
)
print("\n")
@@ -205,10 +241,10 @@ function details(
print("\n")
pretty_table(
covar_array;
- header = covar_columns,
- tf = PrettyTables.tf_borderless,
+ column_labels = covar_columns,
+ table_format = TextTableFormat(borders = text_table_borders__borderless),
alignment = :l,
- formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v,
+ formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v],
)
print("\n")
@@ -235,10 +271,10 @@ function details(
print("\n")
pretty_table(
mean_array;
- header = mean_columns,
- tf = PrettyTables.tf_borderless,
+ column_labels = mean_columns,
+ table_format = TextTableFormat(borders = text_table_borders__borderless),
alignment = :l,
- formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v,
+ formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v],
)
print("\n")
end
@@ -324,11 +360,13 @@ function Base.findall(fun::Function, partable::ParameterTable)
end
"""
- (1) details(sem_fit::SemFit; show_fitmeasures = false)
+ (1) details(model::AbstractSem)
+
+ (2) details(sem_fit::SemFit; show_fitmeasures = false)
- (2) details(partable::AbstractParameterTable; ...)
+ (3) details(partable::AbstractParameterTable; ...)
-Print information about (1) a fitted SEM or (2) a parameter table to stdout.
+Print information about (1) a SEM, (2) a fitted SEM or (3) a parameter table to stdout.
# Extended help
## Addition keyword arguments
diff --git a/src/frontend/pretty_printing.jl b/src/frontend/pretty_printing.jl
index 2fa970f24..7b6975f68 100644
--- a/src/frontend/pretty_printing.jl
+++ b/src/frontend/pretty_printing.jl
@@ -32,9 +32,11 @@ end
# Loss Function, Implied, Observed, Optimizer
##############################################################
-function Base.show(io::IO, struct_inst::SemLossFunction)
- print_type_name(io, struct_inst)
- print_field_types(io, struct_inst)
+function Base.show(io::IO, sem::SemLoss)
+ println(io, "Structural Equation Model Loss ($(nameof(typeof(sem))))")
+ println(io, "- Observed: $(nameof(typeof(observed(sem)))) ($(nsamples(sem)) samples)")
+ println(io, "- Implied: $(nameof(typeof(implied(sem)))) ($(nparams(sem)) parameters)")
+ println(io, "- Variables: $(nobserved_vars(sem)) observed, $(nlatent_vars(sem)) latent")
end
function Base.show(io::IO, struct_inst::SemImplied)
diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl
index 14169dd94..227e902ff 100644
--- a/src/frontend/specification/EnsembleParameterTable.jl
+++ b/src/frontend/specification/EnsembleParameterTable.jl
@@ -30,7 +30,7 @@ function EnsembleParameterTable(
param_labels = if isnothing(param_labels)
# collect all SEM parameters in ensemble if not specified
# and apply the set to all partables
- unique(mapreduce(SEM.param_labels, vcat, values(spec_ensemble), init = Vector{Symbol}()))
+ mapreduce(SEM.param_labels, vcat, values(spec_ensemble), init = Symbol[]) |> unique
else
copy(param_labels)
end
diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl
index 2af269372..c9b9dc24f 100644
--- a/src/frontend/specification/ParameterTable.jl
+++ b/src/frontend/specification/ParameterTable.jl
@@ -34,7 +34,9 @@ function ParameterTable(
latent_vars::Union{AbstractVector{Symbol}, Nothing} = nothing,
param_labels::Union{AbstractVector{Symbol}, Nothing} = nothing,
)
- param_labels = isnothing(param_labels) ? unique!(filter(!=(:const), columns[:label])) : copy(param_labels)
+ param_labels =
+ isnothing(param_labels) ? unique!(filter(!=(:const), columns[:label])) :
+ copy(param_labels)
check_param_labels(param_labels, columns[:label])
return ParameterTable(
columns,
@@ -112,10 +114,13 @@ function Base.show(io::IO, partable::ParameterTable)
pretty_table(
io,
as_matrix,
- header = (shown_columns, [eltype(partable.columns[col]) for col in shown_columns]),
- tf = PrettyTables.tf_compact,
+ column_labels = [
+ shown_columns,
+ [eltype(partable.columns[col]) for col in shown_columns],
+ ],
+ table_format = TextTableFormat(borders = text_table_borders__compact),
# TODO switch to `missing` as non-specified values and suppress printing of `missing` instead
- formatters = (v, i, j) -> isa(v, Number) && isnan(v) ? "" : v,
+ formatters = [(v, i, j) -> isa(v, Number) && isnan(v) ? "" : v],
)
print(io, "Latent Variables: $(partable.latent_vars) \n")
@@ -288,14 +293,25 @@ function update_partable!(
end
"""
- update_partable!(partable::AbstractParameterTable, param_labels::Vector{Symbol}, params, column)
-
-Write parameter `values` into `column` of `partable`.
-
-The `param_labels` and `params` vectors define the pairs of
-parameters, which are being matched to the `:param` column
-of the `partable`.
+ (1) update_partable!(partable::AbstractParameterTable, column, fitted:SemFit, params, default = nothing)
+
+ (2) update_partable!(partable::AbstractParameterTable, column, param_labels::Vector{Symbol}, params, default = nothing)
+
+Add a new column to a parameter table.
+`column` is the name of the column, `params` contains the values of the new column,
+and `fitted` or `param_labels` is used to match the values to the correct parameter labels.
+The `default` value is used if a parameter in `partable` does not occur in `param_labels`.
"""
+function update_partable!(
+ partable::AbstractParameterTable,
+ column::Symbol,
+ fitted::SemFit,
+ params::AbstractVector,
+ default::Any = nothing,
+)
+ update_partable!(partable, column, param_labels(fitted), params, default)
+end
+
function update_partable!(
partable::ParameterTable,
column::Symbol,
@@ -389,7 +405,6 @@ function update_se_hessian!(
return update_partable!(partable, :se, param_labels(fit), se)
end
-
"""
lavaan_params!(out::AbstractVector, partable_lav,
partable::ParameterTable,
@@ -438,8 +453,8 @@ function lavaan_params!(
lav_ind = findallrows(
r ->
r[:lhs] == String(to) &&
- r[:op] == "~1" &&
- (isnothing(lav_group) || r[:group] == lav_group),
+ r[:op] == "~1" &&
+ (isnothing(lav_group) || r[:group] == lav_group),
partable_lav,
)
else
@@ -458,20 +473,20 @@ function lavaan_params!(
lav_ind = findallrows(
r ->
(
- (r[:lhs] == String(from) && r[:rhs] == String(to)) ||
- (r[:lhs] == String(to) && r[:rhs] == String(from))
- ) &&
- r[:op] == lav_type &&
- (isnothing(lav_group) || r[:group] == lav_group),
+ (r[:lhs] == String(from) && r[:rhs] == String(to)) ||
+ (r[:lhs] == String(to) && r[:rhs] == String(from))
+ ) &&
+ r[:op] == lav_type &&
+ (isnothing(lav_group) || r[:group] == lav_group),
partable_lav,
)
else
lav_ind = findallrows(
r ->
r[:lhs] == String(from) &&
- r[:rhs] == String(to) &&
- r[:op] == lav_type &&
- (isnothing(lav_group) || r[:group] == lav_group),
+ r[:rhs] == String(to) &&
+ r[:op] == lav_type &&
+ (isnothing(lav_group) || r[:group] == lav_group),
partable_lav,
)
end
@@ -524,10 +539,4 @@ lavaan_params(
partable::ParameterTable,
lav_col::Symbol = :est,
lav_group = nothing,
-) = lavaan_params!(
- fill(NaN, nparams(partable)),
- partable_lav,
- partable,
- lav_col,
- lav_group,
-)
+) = lavaan_params!(fill(NaN, nparams(partable)), partable_lav, partable, lav_col, lav_group)
diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl
index 53858abd4..734b63c02 100644
--- a/src/frontend/specification/Sem.jl
+++ b/src/frontend/specification/Sem.jl
@@ -1,7 +1,208 @@
+losstype(::Type{<:LossTerm{L, W, I}}) where {L, W, I} = L
+losstype(term::LossTerm) = losstype(typeof(term))
+loss(term::LossTerm) = term.loss
+weight(term::LossTerm) = term.weight
+id(term::LossTerm) = term.id
+
+"""
+ issemloss(term::LossTerm) -> Bool
+
+Check if a SEM model term is a SEM loss function ([`SemLoss`](@ref)).
+"""
+issemloss(term::LossTerm) = isa(loss(term), SemLoss)
+
+for f in (
+ :implied,
+ :observed,
+ :nsamples,
+ :observed_vars,
+ :nobserved_vars,
+ :vars,
+ :nvars,
+ :latent_vars,
+ :nlatent_vars,
+ :params,
+ :nparams,
+)
+ @eval $f(term::LossTerm) = $f(loss(term))
+end
+
+function Base.show(io::IO, term::LossTerm)
+ if (:compact => true) in io
+ if !isnothing(id(term))
+ print(io, ":$(id(term)): ")
+ end
+ print(io, nameof(losstype(term)))
+ if issemloss(term)
+ print(
+ io,
+ " ($(nsamples(term)) samples, $(nobserved_vars(term)) observed, $(nlatent_vars(term)) latent variables)",
+ )
+ end
+ if !isnothing(weight(term))
+ print(io, " w=$(round(weight(term), digits=3))")
+ else
+ print(io, " w=1")
+ end
+ println(io)
+ else
+ print(io, nameof(losstype(term)))
+ print(io, "\n")
+ if !isnothing(id(term))
+ print(io, " - id: $(id(term)) \n")
+ end
+ if issemloss(term)
+ print(io, " - observed: $(nameof(typeof(observed(loss(term))))) \n")
+ print(io, " - implied: $(nameof(typeof(implied(loss(term))))) \n")
+ end
+ if !isnothing(weight(term))
+ print(io, " - weight: $(round(weight(term), digits=3)) \n")
+ end
+ end
+end
+
+# scaling corrections for multigroup models
+
+# fallback method for non-standard SemLoss type
+multigroup_correction_scale(::Type{<:SemLoss}) = nothing
+
+multigroup_correction_scale(::Type{<:SemFIML}) = 0
+# ML (like WLS) uses the Wishart convention: per-group (Nᵍ-1) weighting and N-G in the
+# χ²/RMSEA.
+multigroup_correction_scale(::Type{<:SemML}) = -1
+multigroup_correction_scale(::Type{<:SemWLS}) = -1
+
+multigroup_correction_scale(loss::SemLoss) = multigroup_correction_scale(typeof(loss))
+
+# calculate sem term weights for multigroup models
+# correcting for the number of samples and the loss type
+function multigroup_weights(semterms...)
+ n = length(semterms)
+ nsamples_total = sum(nsamples, semterms)
+ semloss_type = check_same_semterm_type(semterms; throw_error = false)
+ if isnothing(semloss_type)
+ @info """
+ Your ensemble model contains heterogeneous loss functions.
+ Default weights of (#samples per group/#total samples) will be used
+ """
+ c = 0
+ else
+ c = multigroup_correction_scale(semloss_type)
+ if isnothing(c)
+ @info """
+ We don't know how to choose group weights for the specified loss function.
+ Default weights of (#samples per group/#total samples) will be used
+ """
+ c = 0
+ end
+ end
+ return [(nsamples(term)+c) / (nsamples_total+n*c) for term in semterms]
+end
+
############################################################################################
# constructor for Sem types
############################################################################################
+function Sem(
+ loss_terms...;
+ params::Union{Vector{Symbol}, Nothing} = nothing,
+ default_sem_weights = :nsamples_corrected,
+)
+ default_sem_weights ∈ [:nsamples_corrected, :nsamples, :uniform, :one] ||
+ throw(ArgumentError("Unsupported default_sem_weights=:$default_sem_weights"))
+ # assemble a list of weighted losses and check params equality
+ terms = Vector{LossTerm}()
+ params = !isnothing(params) ? copy(params) : params
+ has_sem_weights = false
+ nsems = 0
+ for inp_term in loss_terms
+ if inp_term isa AbstractLoss # term
+ term = inp_term
+ term_w = nothing
+ term_id = nothing
+ elseif inp_term isa Pair
+ if inp_term[1] isa AbstractLoss # term => weight
+ term, term_w = inp_term
+ term_id = nothing
+ elseif inp_term[2] isa AbstractLoss # id => term
+ term_id, term = inp_term
+ term_w = nothing
+ elseif inp_term[2] isa Pair # id => term => weight
+ term_id, (term, term_w) = inp_term
+ isa(term, AbstractLoss) || throw(
+ ArgumentError(
+ "AbstractLoss expected as a second argument of a loss term double pair (id => loss => weight), $(nameof(typeof(term))) found",
+ ),
+ )
+ end
+ elseif inp_term isa LossTerm
+ term_id = id(inp_term)
+ term = loss(inp_term)
+ term_w = weight(inp_term)
+ else
+ "[id =>] AbstractLoss [=> weight] expected as a loss term, $(nameof(typeof(inp_term))) found" |>
+ ArgumentError |>
+ throw
+ end
+
+ if term isa SemLoss
+ nsems += 1
+ has_sem_weights |= !isnothing(term_w)
+ # check integrity
+ if isnothing(params)
+ params = SEM.params(term)
+ elseif params != SEM.params(term)
+ # FIXME the suggestion might no longer be relevant, since ParTable also stores params order
+ """
+ The parameters of your SEM models do not match.
+ Maybe you tried to specify models of an ensemble via ParameterTables.
+ In that case, you may use RAMMatrices instead.
+ """ |> error
+ end
+ check_observed_vars(term)
+ elseif !(term isa AbstractLoss)
+ "AbstractLoss term expected at $(length(terms)+1) position, $(nameof(typeof(term))) found" |>
+ ArgumentError |>
+ throw
+ end
+ push!(terms, LossTerm(term, term_id, term_w))
+ end
+ isnothing(params) && throw(ErrorException("No SEM models provided."))
+
+ if !has_sem_weights && nsems > 1
+ # set the weights of SEMs in the ensemble
+ if default_sem_weights == :nsamples_corrected
+ # weight SEM terms by the number of samples, applying a loss-type-specific
+ # correction (see multigroup_correction_scale); consistent with the
+ # multigroup RMSEA sample-size correction
+ sem_idxs = [i for (i, term) in enumerate(terms) if issemloss(term)]
+ sem_weights = multigroup_weights(terms[sem_idxs]...)
+ for (k, i) in enumerate(sem_idxs)
+ terms[i] = LossTerm(loss(terms[i]), id(terms[i]), sem_weights[k])
+ end
+ elseif default_sem_weights == :nsamples
+ # weight SEM by the number of samples
+ nsamples_total = sum(nsamples(term) for term in terms if issemloss(term))
+ for (i, term) in enumerate(terms)
+ if issemloss(term)
+ terms[i] =
+ LossTerm(loss(term), id(term), nsamples(term) / nsamples_total)
+ end
+ end
+ elseif default_sem_weights == :uniform # uniform weights
+ for (i, term) in enumerate(terms)
+ if issemloss(term)
+ terms[i] = LossTerm(loss(term), id(term), 1 / nsems)
+ end
+ end
+ elseif default_sem_weights == :one # do nothing
+ end
+ end
+
+ terms_tuple = Tuple(terms)
+ return Sem{typeof(terms_tuple)}(terms_tuple, params)
+end
+
function Sem(;
specification = ParameterTable,
observed::O = SemObservedData,
@@ -11,193 +212,456 @@ function Sem(;
) where {O, I, L}
kwdict = Dict{Symbol, Any}(kwargs...)
- set_field_type_kwargs!(kwdict, observed, implied, loss, O, I)
-
- observed, implied, loss = get_fields!(kwdict, specification, observed, implied, loss)
+ # add kwargs with type information
+ kwdict[:observed_type] = O <: Type ? observed : typeof(observed)
+ kwdict[:implied_type] = I <: Type ? implied : typeof(implied)
+ if loss isa SemLoss
+ kwdict[:loss_types] =
+ [aloss isa SemLoss ? typeof(aloss) : aloss for aloss in loss.functions]
+ elseif applicable(iterate, loss)
+ kwdict[:loss_types] = [aloss isa SemLoss ? typeof(aloss) : aloss for aloss in loss]
+ else
+ kwdict[:loss_types] = [loss isa SemLoss ? typeof(loss) : loss]
+ end
- sem = Sem(observed, implied, loss)
+ loss = build_sem_terms(kwdict, specification, observed, implied, loss)
- return sem
+ return Sem(loss...)
end
+############################################################################################
+# functions
+############################################################################################
+
+params(model::AbstractSem) = model.params
+param_labels(model::AbstractSem) = params(model) # alias
+
"""
- implied(model::AbstractSemSingle) -> SemImplied
+ loss_terms(model::AbstractSem)
-Returns the [*implied*](@ref SemImplied) part of a model.
+Returns a tuple of all [`LossTerm`](@ref) weighted terms in the SEM model.
+
+See also [`sem_terms`](@ref), [`loss_term`](@ref).
"""
-implied(model::AbstractSemSingle) = model.implied
+loss_terms(model::AbstractSem) = model.loss_terms
+nloss_terms(model::AbstractSem) = length(loss_terms(model))
-nvars(model::AbstractSemSingle) = nvars(implied(model))
-nobserved_vars(model::AbstractSemSingle) = nobserved_vars(implied(model))
-nlatent_vars(model::AbstractSemSingle) = nlatent_vars(implied(model))
+"""
+ sem_terms(model::AbstractSem)
-vars(model::AbstractSemSingle) = vars(implied(model))
-observed_vars(model::AbstractSemSingle) = observed_vars(implied(model))
-latent_vars(model::AbstractSemSingle) = latent_vars(implied(model))
+Returns a tuple of all weighted SEM terms in the SEM model.
-param_labels(model::AbstractSemSingle) = param_labels(implied(model))
-nparams(model::AbstractSemSingle) = nparams(implied(model))
+In comparison to [`loss_terms`](@ref) that returns all model terms, including e.g.
+regularization terms, this function returns only the [`SemLoss`] terms.
+See also [`loss_terms`](@ref), [`sem_term`](@ref).
"""
- observed(model::AbstractSemSingle) -> SemObserved
+sem_terms(model::AbstractSem) = Tuple(term for term in loss_terms(model) if issemloss(term))
+nsem_terms(model::AbstractSem) = sum(issemloss, loss_terms(model))
+
+nsamples(model::AbstractSem) =
+ sum(term -> issemloss(term) ? nsamples(term) : 0, loss_terms(model))
-Returns the [*observed*](@ref SemObserved) part of a model.
"""
-observed(model::AbstractSemSingle) = model.observed
+ loss_term(model::AbstractSem, id::Any) -> AbstractLoss
-nsamples(model::AbstractSemSingle) = nsamples(observed(model))
+Returns the loss term with the specified `id` from the `model`.
+Throws an error if the model has no term with the specified `id`.
+See also [`loss_terms`](@ref).
"""
- loss(model::AbstractSemSingle) -> SemLoss
+function loss_term(model::AbstractSem, id::Any)
+ for term in loss_terms(model)
+ if SEM.id(term) == id
+ return loss(term)
+ end
+ end
+ error("No loss term with id=$id found")
+end
-Returns the [*loss*](@ref SemLoss) function of a model.
"""
-loss(model::AbstractSemSingle) = model.loss
+ sem_term(model::AbstractSem, [id]) -> SemLoss
-# sum of samples in all sub-models
-nsamples(ensemble::SemEnsemble) = sum(nsamples, ensemble.sems)
+Returns the SEM loss term with the specified `id` from the `model`.
+Throws an error if the model has no term with the specified `id` or
+if it is not of a [`SemLoss`](@ref) type.
-function SemFiniteDiff(;
- specification = ParameterTable,
- observed::O = SemObservedData,
- implied::I = RAM,
- loss::L = SemML,
- kwargs...,
-) where {O, I, L}
- kwdict = Dict{Symbol, Any}(kwargs...)
+If no `id` is specified and the model contains only one SEM term, the term is returned.
+Throws an error if the model contains multiple SEM terms.
- set_field_type_kwargs!(kwdict, observed, implied, loss, O, I)
+See also [`loss_term`](@ref), [`sem_terms`](@ref).
+"""
+function sem_term(model::AbstractSem, id::Any)
+ term = loss_term(model, id)
+ issemloss(term) || error("Loss term with id=$id ($(typeof(term))) is not a SEM term")
+ return term
+end
- observed, implied, loss = get_fields!(kwdict, specification, observed, implied, loss)
+function sem_term(model::AbstractSem, _::Nothing = nothing)
+ if nsem_terms(model) != 1
+ error(
+ "Model contains $(nsem_terms(model)) SEM terms, you have to specify a specific term",
+ )
+ end
+ for term in loss_terms(model)
+ issemloss(term) && return loss(term)
+ end
+ error("Unreachable reached")
+end
- sem = SemFiniteDiff(observed, implied, loss)
+# check that all models use the same single loss function
+# returns the type of the single SEM loss function, SemLoss if there are multiple different SEM losses,
+# nothing if there are no SEM terms.
+# If throw_error=true, throws an error if there are multiple different SEM loss functions
+check_same_semterm_type(model::AbstractSem; throw_error::Bool = true) =
+ check_same_semterm_type(sem_terms(model); throw_error = throw_error)
+
+# check that all models use the same single loss function
+# returns the type of the single SEM loss function,
+# nothing if there are multiple different SEM losses or no SEM terms.
+# If throw_error=true, throws an error if there are multiple different SEM loss functions
+function check_same_semterm_type(terms::Tuple; throw_error::Bool = true)
+ isempty(terms) && return nothing
+
+ _semloss(term::SemLoss) = _unwrap(term)
+ _semloss(term::LossTerm) = _semloss(loss(term))
+ _semloss(term) = throw(ArgumentError("SemLoss term expected, $(typeof(term)) found"))
+ _semloss_label(i::Integer, _::Union{SemLoss, LossTerm{<:SemLoss, Nothing}}) = "#$i"
+ _semloss_label(i::Integer, term::LossTerm{<:SemLoss, Symbol}) = "#$i ($(SEM.id(term)))"
+
+ term1 = _semloss(terms[1])
+ L = typeof(term1).name
+
+ # check that all SemLoss terms are of the same class (ML, FIML, WLS etc), ignore typeparams
+ for (i, term) in enumerate(terms)
+ lossterm = _semloss(term)
+ @assert lossterm isa SemLoss
+ if typeof(lossterm).name != L
+ if throw_error
+ error(
+ "SemLoss term $(_semloss_label(i, term)) is $(typeof(lossterm).name), expected $L. Heterogeneous loss functions are not supported",
+ )
+ else
+ return nothing
+ end
+ end
+ end
- return sem
+ # return the type of the first SEM term
+ # note that type params of the SEM terms might be different
+ return typeof(term1)
end
-############################################################################################
-# functions
-############################################################################################
-
-function set_field_type_kwargs!(kwargs, observed, implied, loss, O, I)
- kwargs[:observed_type] = O <: Type ? observed : typeof(observed)
- kwargs[:implied_type] = I <: Type ? implied : typeof(implied)
- if loss isa SemLoss
- kwargs[:loss_types] = [
- lossfun isa SemLossFunction ? typeof(lossfun) : lossfun for
- lossfun in loss.functions
- ]
- elseif applicable(iterate, loss)
- kwargs[:loss_types] =
- [lossfun isa SemLossFunction ? typeof(lossfun) : lossfun for lossfun in loss]
+# wrappers arounds a single SemLoss term
+observed(model::AbstractSem, id::Nothing = nothing) = observed(sem_term(model, id))
+implied(model::AbstractSem, id::Nothing = nothing) = implied(sem_term(model, id))
+vars(model::AbstractSem, id::Nothing = nothing) = vars(implied(model, id))
+observed_vars(model::AbstractSem, id::Nothing = nothing) = observed_vars(implied(model, id))
+latent_vars(model::AbstractSem, id::Nothing = nothing) = latent_vars(implied(model, id))
+
+# build ensemble/multi-group observed from the specification and Sem(...) kwargs
+# used by Sem(...) and replace_observed()
+function build_ensemble_observed(observed_type, spec::EnsembleParameterTable, kwargs)
+ if !haskey(kwargs, :data)
+ @warn """
+ No data provided for ensemble SEM model. Each SEM term will be constructed with empty data.
+ To provide data for each term, pass a DataFrame with a column identifying the term groups or a Dict mapping term ids to data
+ """
+ semterms_data = nothing
else
- kwargs[:loss_types] = [loss isa SemLossFunction ? typeof(loss) : loss]
+ kwdata = kwargs[:data]
+ if isa(kwdata, AbstractDataFrame)
+ semterm_col = get(kwargs, :semterm_column, nothing)
+ isnothing(semterm_col) &&
+ throw(ArgumentError("No semterm_column specified for ensemble data."))
+ semterms_data = Dict(
+ g[semterm_col] => group_data for
+ (g, group_data) in pairs(groupby(kwdata, semterm_col))
+ )
+ elseif isa(kwdata, AbstractDict)
+ semterms_data = kwdata
+ else
+ """
+ Unsupported data type for ensemble SEM model: $(typeof(kwdata)).
+ Provide a DataFrame with a column identifying the term groups or a Dict mapping term ids to data.
+ """ |>
+ ArgumentError |>
+ throw
+ end
+ unused_term_ids = setdiff(keys(semterms_data), keys(spec.tables))
+ isempty(unused_term_ids) ||
+ @warn "Ignoring data with ids=$(collect(unused_term_ids)): no such SEM terms exist"
end
+
+ # construct SemObserved for each term
+ return Dict(
+ term_id => begin
+ term_kwargs = copy(kwargs)
+ if !isnothing(semterms_data)
+ term_data = get(semterms_data, term_id, nothing)
+ isnothing(term_data) &&
+ throw(ArgumentError("No data provided for SEM term :$term_id"))
+ term_kwargs[:data] = term_data
+ delete!(term_kwargs, :semterm_column)
+ end
+ observed_type(; specification = term_spec, term_kwargs...)
+ end for (term_id, term_spec) in pairs(spec.tables)
+ )
end
-# construct Sem fields
-function get_fields!(kwargs, specification, observed, implied, loss)
- if !isa(specification, SemSpecification)
- specification = specification(; kwargs...)
+# called by Sem() ctor to construct its loss terms
+function build_sem_terms(kwargs::AbstractDict, spec, observed, implied, loss)
+ if !isa(spec, SemSpecification)
+ spec = spec(; kwargs...)
end
# observed
if !isa(observed, SemObserved)
- observed = observed(; specification, kwargs...)
+ observed = if spec isa EnsembleParameterTable
+ build_ensemble_observed(observed, spec, kwargs)
+ else
+ observed(; specification = spec, kwargs...)
+ end
end
- kwargs[:observed] = observed
# implied
if !isa(implied, SemImplied)
- implied = implied(; specification, kwargs...)
+ implied = if spec isa EnsembleParameterTable
+ Dict(
+ term_id => implied(term_spec; kwargs...) for
+ (term_id, term_spec) in pairs(spec.tables)
+ )
+ else
+ implied(spec; kwargs...)
+ end
end
- kwargs[:implied] = implied
- kwargs[:nparams] = nparams(implied)
-
# loss
- loss = get_SemLoss(loss; specification, kwargs...)
- kwargs[:loss] = loss
+ loss_kwargs = copy(kwargs)
+ loss_kwargs[:nparams] = nparams(spec)
+ loss = build_sem_terms(loss, observed, implied; loss_kwargs...)
- return observed, implied, loss
+ return loss
end
-# construct loss field
-function get_SemLoss(loss; kwargs...)
+# construct loss terms for the given observed and implied
+function build_sem_terms(loss, observed, implied; kwargs...)
+ function build_SemLoss(aloss, observed, implied)
+ if aloss isa AbstractLoss
+ return aloss
+ elseif aloss <: SemLoss{O, I} where {O, I}
+ return aloss(observed, implied; kwargs...)
+ else
+ return aloss(; kwargs...)
+ end
+ end
+
if loss isa SemLoss
- nothing
+ return loss
elseif applicable(iterate, loss)
- loss_out = []
- for lossfun in loss
- if isa(lossfun, SemLossFunction)
- push!(loss_out, lossfun)
- else
- lossfun = lossfun(; kwargs...)
- push!(loss_out, lossfun)
- end
- end
- loss = SemLoss(loss_out...; kwargs...)
+ return [build_SemLoss(aloss, observed, implied) for aloss in loss]
else
- if !isa(loss, SemLossFunction)
- loss = SemLoss(loss(; kwargs...); kwargs...)
+ if isa(observed, AbstractDict) && isa(implied, AbstractDict)
+ observed_ids = Set(keys(observed))
+ implied_ids = Set(keys(implied))
+ if observed_ids != implied_ids
+ """"
+ The term ids of the observed and the implied data do not match.
+ Observed term ids: $(observed_ids), implied term ids: $(implied_ids)
+ """ |>
+ ArgumentError |>
+ throw
+ end
+ loss_out = [
+ begin
+ term_implied = implied[term_id]
+ if observed_vars(term_observed) != observed_vars(term_implied)
+ "observed_vars differ between the observed and the implied for the term $term_id" |>
+ ArgumentError |>
+ throw
+ end
+ LossTerm(
+ build_SemLoss(loss, term_observed, term_implied),
+ term_id,
+ nothing,
+ )
+ end for (term_id, term_observed) in pairs(observed)
+ ]
+ return loss_out
else
- loss = SemLoss(loss; kwargs...)
+ if observed_vars(observed) != observed_vars(implied)
+ "observed_vars differ between the observed and the implied" |>
+ ArgumentError |>
+ throw
+ end
+ return (build_SemLoss(loss, observed, implied),)
end
end
- return loss
end
##############################################################
-# pretty printing
+# replace_observed: Sem level
##############################################################
-function Base.show(io::IO, sem::Sem{O, I, L}) where {O, I, L}
- lossfuntypes = @. string(nameof(typeof(sem.loss.functions)))
- lossfuntypes = " " .* lossfuntypes .* ("\n")
- print(io, "Structural Equation Model \n")
- print(io, "- Loss Functions \n")
- print(io, lossfuntypes...)
- print(io, "- Fields \n")
- print(io, " observed: $(nameof(O)) \n")
- print(io, " implied: $(nameof(I)) \n")
-end
+# internal function to create a copy of Sem with the loss term replaced
+# used by the replace_observed()
+_replace_loss_terms(sem::Sem, new_terms::Tuple) =
+ Sem{typeof(new_terms)}(new_terms, copy(params(sem)))
-function Base.show(io::IO, sem::SemFiniteDiff{O, I, L}) where {O, I, L}
- lossfuntypes = @. string(nameof(typeof(sem.loss.functions)))
- lossfuntypes = " " .* lossfuntypes .* ("\n")
- print(io, "Structural Equation Model : Finite Diff Approximation\n")
- print(io, "- Loss Functions \n")
- print(io, lossfuntypes...)
- print(io, "- Fields \n")
- print(io, " observed: $(nameof(O)) \n")
- print(io, " implied: $(nameof(I)) \n")
+_replace_loss_terms(sem::Sem, new_terms::AbstractVector) =
+ _replace_loss_terms(sem, Tuple(new_terms))
+
+"""
+ replace_observed(model::Sem, observed::SemObserved)
+ replace_observed(model::Sem, data::AbstractDict{Symbol})
+ replace_observed(model::Sem, data::AbstractDataFrame; [semterm_column])
+ replace_observed(loss::SemLoss, observed::SemObserved)
+ replace_observed(loss::SemLoss, data::Union{AbstractMatrix, DataFrame})
+
+Construct a new SEM model or SEM loss with replaced observed data.
+
+The SEM structure (implied covariance, loss type) is preserved;
+only the observed data is swapped. The new loss terms preserve the configuration
+and share the implied state with the loss terms of the original SEM model.
+
+Keyword arguments:
+- `recompute_observed_state::Bool = true`: loss terms should recompute observed-dependent
+ caches. Losses without such caches ignore this argument.
+
+# Single-term models
+
+Pass a `SemObserved` object, a data matrix, or a `DataFrame`:
+```julia
+replace_observed(model, new_data_matrix)
+replace_observed(model, new_sem_observed)
+replace_observed(model, new_df)
+```
+
+# Multi-term models
+
+Pass a `Dict{Symbol}` mapping term ids to data or `SemObserved` objects:
+```julia
+replace_observed(model, Dict(:g1 => data1, :g2 => data2))
+```
+
+Or pass a `DataFrame` with a `semterm_column` identifying the group:
+```julia
+replace_observed(model, new_df; semterm_column = :group)
+```
+"""
+function replace_observed end
+
+function replace_observed(sem::Sem, data::Union{SemObserved, AbstractMatrix}; kwargs...)
+ nsem_terms(sem) > 1 && throw(
+ ArgumentError(
+ "Model contains $(nsem_terms(sem)) SEM terms. " *
+ "Use a Dict{Symbol} or a DataFrame with `semterm_column` to provide per-term data.",
+ ),
+ )
+ updated_terms = map(loss_terms(sem)) do term
+ replace_observed(term, data; kwargs...)
+ end
+ return _replace_loss_terms(sem, updated_terms)
end
-function Base.show(io::IO, loss::SemLoss)
- lossfuntypes = @. string(nameof(typeof(loss.functions)))
- lossfuntypes = " " .* lossfuntypes .* ("\n")
- print(io, "SemLoss \n")
- print(io, "- Loss Functions \n")
- print(io, lossfuntypes...)
- print(io, "- Weights \n")
- for weight in loss.weights
- if isnothing(weight.w)
- print(io, " one \n")
+function replace_observed(sem::Sem, data::AbstractDict{Symbol}; kwargs...)
+ term_ids = Set(
+ if !isnothing(id(term))
+ id(term)
else
- print(io, "$(round.(weight.w, digits = 2)) \n")
+ "Multigroup replace_observed(sem, data::Dict) requires all SEM terms to have ids." |>
+ ArgumentError |>
+ throw
+ end for term in loss_terms(sem) if issemloss(term)
+ )
+ # check for extra ids
+ extra_term_ids = setdiff(keys(data), term_ids)
+ isempty(extra_term_ids) ||
+ @warn "Ignoring data with ids=$(collect(extra_term_ids)): no such SEM terms exist in the model"
+
+ updated_terms = map(loss_terms(sem)) do term
+ issemloss(term) || return term
+ tid = id(term)
+ term_data = get(data, tid, nothing)
+ isnothing(term_data) &&
+ throw(ArgumentError("No data provided for SEM term :$tid"))
+ return replace_observed(term, term_data; kwargs...)
+ end
+ return _replace_loss_terms(sem, updated_terms)
+end
+
+function replace_observed(sem::Sem, data::AbstractVector; kwargs...)
+ nsem = nsem_terms(sem)
+ nsem == length(data) || throw(
+ ArgumentError(
+ "Length of data ($(length(data))) does not match number of SEM terms ($nsem)",
+ ),
+ )
+ updated_terms = map(enumerate(loss_terms(sem))) do (i, term)
+ issemloss(term) ? replace_observed(term, data[i]; kwargs...) : term
+ end
+ return _replace_loss_terms(sem, updated_terms)
+end
+
+function replace_observed(
+ sem::Sem,
+ data::AbstractDataFrame;
+ semterm_column::Union{Symbol, Nothing} = nothing,
+ kwargs...,
+)
+ if isnothing(semterm_column)
+ # single-term shortcut
+ nsem_terms(sem) > 1 && throw(
+ ArgumentError(
+ "Model contains $(nsem_terms(sem)) SEM terms. " *
+ "Provide `semterm_column` to specify which DataFrame column identifies the groups.",
+ ),
+ )
+ updated_terms = map(loss_terms(sem)) do term
+ replace_observed(term, data; kwargs...)
end
+ return _replace_loss_terms(sem, updated_terms)
end
+
+ # multi-term: split DataFrame by semterm_column
+ terms_data = Dict(
+ g[semterm_column] => group_data for
+ (g, group_data) in pairs(groupby(data, semterm_column))
+ )
+ return replace_observed(sem, terms_data; kwargs...)
end
-function Base.show(io::IO, models::SemEnsemble)
- print(io, "SemEnsemble \n")
- print(io, "- Number of Models: $(models.n) \n")
- print(io, "- Weights: $(round.(models.weights, digits = 2)) \n")
+##############################################################
+# pretty printing
+##############################################################
- print(io, "\n", "Models: \n")
- print(io, "===============================================", "\n")
- for (model, i) in zip(models.sems, 1:models.n)
- print(io, "---------------------- ", i, " ----------------------", "\n")
- print(io, model)
+_subtype_info(::Sem) = nothing
+_subtype_info(::SemFiniteDiff) = "Finite Difference Approximation"
+
+function Base.show(io::IO, sem::AbstractSem)
+ io = IOContext(io, :compact => true)
+ println(io, "Structural Equation Model ($(nameof(typeof(sem))))")
+ println(io, "- $(nparams(sem)) parameters")
+ println(io, "- Loss Functions:")
+ for term in loss_terms(sem)
+ print(io, " - ")
+ print(io, term)
+ end
+end
+
+# pretty prenting for console
+function Base.show(io::IO, ::MIME"text/plain", sem::AbstractSem)
+ # if not specified, use compact printing for larger models
+ if !haskey(io, :compact) && length(loss_terms(sem)) >= 5
+ io = IOContext(io, :compact => true)
+ end
+ print(io, "Structural Equation Model")
+ si = _subtype_info(sem)
+ isnothing(si) || print(io, " : "*si)
+ print("\n")
+ print(io, "- Loss Functions \n")
+ for term in loss_terms(sem)
+ print(io, " > ")
+ print(io, term)
end
end
diff --git a/src/frontend/specification/checks.jl b/src/frontend/specification/checks.jl
index 5ef41c59d..2d00be26d 100644
--- a/src/frontend/specification/checks.jl
+++ b/src/frontend/specification/checks.jl
@@ -4,8 +4,11 @@ function check_param_labels(
param_refs::Union{AbstractVector{Symbol}, Nothing},
)
dup_param_labels = nonunique(param_labels)
- isempty(dup_param_labels) ||
- throw(ArgumentError("Duplicate parameter labels detected: $(join(dup_param_labels, ", "))"))
+ isempty(dup_param_labels) || throw(
+ ArgumentError(
+ "Duplicate parameter labels detected: $(join(dup_param_labels, ", "))",
+ ),
+ )
any(==(:const), param_labels) &&
throw(ArgumentError("Parameters constain reserved :const name"))
diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl
index e46620fbc..a3a8d2659 100644
--- a/src/frontend/specification/documentation.jl
+++ b/src/frontend/specification/documentation.jl
@@ -37,7 +37,6 @@ Return the vector of parameter labels (in the same order as [`params`](@ref)).
"""
param_labels(spec::SemSpecification) = spec.param_labels
-
"""
`ParameterTable`s contain the specification of a structural equation model.
diff --git a/src/implied/RAM/generic.jl b/src/implied/RAM/generic.jl
index fd2ef61b5..90d886f98 100644
--- a/src/implied/RAM/generic.jl
+++ b/src/implied/RAM/generic.jl
@@ -6,14 +6,10 @@ Model implied covariance and means via RAM notation.
# Constructor
- RAM(;specification,
- meanstructure = false,
- gradient = true,
- kwargs...)
+ RAM(specification; gradient = true, kwargs...)
# Arguments
- `specification`: either a `RAMMatrices` or `ParameterTable` object
-- `meanstructure::Bool`: does the model have a meanstructure?
- `gradient::Bool`: is gradient-based optimization used
# Extended help
@@ -53,9 +49,9 @@ Vector of indices of each parameter in the respective RAM matrix:
- `ram.M_indices`
Additional interfaces
-- `ram.F⨉I_A⁻¹` -> ``F(I-A)^{-1}``
-- `ram.F⨉I_A⁻¹S` -> ``F(I-A)^{-1}S``
-- `ram.I_A` -> ``I-A``
+- `F⨉I_A⁻¹(::RAM)` -> ``F(I-A)^{-1}``
+- `F⨉I_A⁻¹S(::RAM)` -> ``F(I-A)^{-1}S``
+- `I_A(::RAM)` -> ``I-A``
Only available in gradient! calls:
- `ram.I_A⁻¹` -> ``(I-A)^{-1}``
@@ -90,13 +86,13 @@ end
### Constructors
############################################################################################
-function RAM(;
- specification::SemSpecification,
+function RAM(
+ spec::SemSpecification;
+ #vech = false,
gradient_required = true,
- meanstructure = false,
kwargs...,
)
- ram_matrices = convert(RAMMatrices, specification)
+ ram_matrices = convert(RAMMatrices, spec)
# get dimensions of the model
n_par = nparams(ram_matrices)
@@ -124,13 +120,8 @@ function RAM(;
end
# μ
- if meanstructure
+ if !isnothing(ram_matrices.M)
MS = HasMeanStruct
- !isnothing(ram_matrices.M) || throw(
- ArgumentError(
- "You set `meanstructure = true`, but your model specification contains no mean parameters.",
- ),
- )
M_pre = materialize(ram_matrices.M, rand_params)
∇M = gradient_required ? sparse_gradient(ram_matrices.M) : nothing
μ = zeros(n_obs)
@@ -152,7 +143,7 @@ function RAM(;
F⨉I_A⁻¹,
F⨉I_A⁻¹S,
I_A,
- copy(I_A),
+ similar(I_A),
∇A,
∇S,
∇M,
@@ -163,11 +154,11 @@ end
### methods
############################################################################################
-function update!(targets::EvaluationTargets, implied::RAM, model::AbstractSemSingle, param_labels)
- materialize!(implied.A, implied.ram_matrices.A, param_labels)
- materialize!(implied.S, implied.ram_matrices.S, param_labels)
+function update!(targets::EvaluationTargets, implied::RAM, params)
+ materialize!(implied.A, implied.ram_matrices.A, params)
+ materialize!(implied.S, implied.ram_matrices.S, params)
if !isnothing(implied.M)
- materialize!(implied.M, implied.ram_matrices.M, param_labels)
+ materialize!(implied.M, implied.ram_matrices.M, params)
end
parent(implied.I_A) .= .-implied.A
@@ -188,15 +179,3 @@ function update!(targets::EvaluationTargets, implied::RAM, model::AbstractSemSin
mul!(implied.μ, implied.F⨉I_A⁻¹, implied.M)
end
end
-
-############################################################################################
-### Recommended methods
-############################################################################################
-
-function update_observed(implied::RAM, observed::SemObserved; kwargs...)
- if nobserved_vars(observed) == size(implied.Σ, 1)
- return implied
- else
- return RAM(; observed = observed, kwargs...)
- end
-end
diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl
index 9634bfa89..552cd65d9 100644
--- a/src/implied/RAM/symbolic.jl
+++ b/src/implied/RAM/symbolic.jl
@@ -6,18 +6,16 @@ Subtype of `SemImplied` that implements the RAM notation with symbolic precomput
# Constructor
- RAMSymbolic(;
- specification,
+ RAMSymbolic(
+ specification;
vech = false,
gradient = true,
hessian = false,
approximate_hessian = false,
- meanstructure = false,
kwargs...)
# Arguments
- `specification`: either a `RAMMatrices` or `ParameterTable` object
-- `meanstructure::Bool`: does the model have a meanstructure?
- `gradient::Bool`: is gradient-based optimization used
- `hessian::Bool`: is hessian-based optimization used
- `approximate_hessian::Bool`: for hessian based optimization: should the hessian be approximated
@@ -37,13 +35,13 @@ Jacobians (only available in gradient! calls)
- `ram.∇Σ` -> ``∂vec(Σ)/∂θᵀ``
- `ram.∇μ` -> ``∂μ/∂θᵀ``
-- `ram.∇Σ_function` -> function to overwrite `∇Σ` in place,
- i.e. `∇Σ_function(∇Σ, θ)`. Typically, you do not want to use this but simply
+- `∇Σ_eval!(::RAMSymbolic)` -> function to evaluate `∇Σ` in place,
+ i.e. `∇Σ_eval!(∇Σ, θ)`. Typically, you do not want to use this but simply
query `ram.∇Σ`.
Hessians
The computation of hessians is more involved.
-Therefore, we desribe it in the online documentation,
+Therefore, we desribe it in the online documentation,
and the respective interfaces are omitted here.
## RAM notation
@@ -56,23 +54,19 @@ and for models with a meanstructure, the model implied means are computed as
\mu = F(I-A)^{-1}M
```
"""
-struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5} <:
- SemImpliedSymbolic
+struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, V2, F4, A4, F5, A5} <: SemImpliedSymbolic
meanstruct::MS
hessianeval::ExactHessian
- Σ_function::F1
- ∇Σ_function::F2
- ∇²Σ_function::F3
+ Σ_eval!::F1
+ ∇Σ_eval!::F2
+ ∇²Σ_eval!::F3
Σ::A1
∇Σ::A2
∇²Σ::A3
- Σ_symbolic::S1
- ∇Σ_symbolic::S2
- ∇²Σ_symbolic::S3
ram_matrices::V2
- μ_function::F4
+ μ_eval!::F4
μ::A4
- ∇μ_function::F5
+ ∇μ_eval!::F5
∇μ::A5
RAMSymbolic{MS}(args...) where {MS <: MeanStruct} =
@@ -83,18 +77,16 @@ end
### Constructors
############################################################################################
-function RAMSymbolic(;
- specification::SemSpecification,
- loss_types = nothing,
- vech = false,
- simplify_symbolics = false,
- gradient = true,
- hessian = false,
- meanstructure = false,
- approximate_hessian = false,
+function RAMSymbolic(
+ spec::SemSpecification;
+ vech::Bool = false,
+ simplify_symbolics::Bool = false,
+ gradient::Bool = true,
+ hessian::Bool = false,
+ approximate_hessian::Bool = false,
kwargs...,
)
- ram_matrices = convert(RAMMatrices, specification)
+ ram_matrices = convert(RAMMatrices, spec)
n_par = nparams(ram_matrices)
par = (Symbolics.@variables θ[1:n_par])[1]
@@ -104,88 +96,78 @@ function RAMSymbolic(;
M = !isnothing(ram_matrices.M) ? materialize(Num, ram_matrices.M, par) : nothing
F = ram_matrices.F
- if !isnothing(loss_types) && any(T -> T <: SemWLS, loss_types)
- vech = true
- end
-
I_A⁻¹ = neumann_series(A)
# Σ
- Σ_symbolic = eval_Σ_symbolic(S, I_A⁻¹, F; vech = vech, simplify = simplify_symbolics)
- #print(Symbolics.build_function(Σ_symbolic)[2])
- Σ_function = Symbolics.build_function(Σ_symbolic, par, expression = Val{false})[2]
- Σ = zeros(size(Σ_symbolic))
- precompile(Σ_function, (typeof(Σ), Vector{Float64}))
+ Σ_sym = eval_Σ_symbolic(S, I_A⁻¹, F; vech, simplify = simplify_symbolics)
+ #print(Symbolics.build_function(Σ_sym)[2])
+ Σ_eval! = Symbolics.build_function(Σ_sym, par, expression = Val{false})[2]
+ Σ = zeros(size(Σ_sym))
+ precompile(Σ_eval!, (typeof(Σ), Vector{Float64}))
# ∇Σ
if gradient
- ∇Σ_symbolic = Symbolics.sparsejacobian(vec(Σ_symbolic), [par...])
- ∇Σ_function = Symbolics.build_function(∇Σ_symbolic, par, expression = Val{false})[2]
- constr = findnz(∇Σ_symbolic)
- ∇Σ = sparse(constr[1], constr[2], fill(1.0, nnz(∇Σ_symbolic)), size(∇Σ_symbolic)...)
- precompile(∇Σ_function, (typeof(∇Σ), Vector{Float64}))
+ ∇Σ_sym = Symbolics.sparsejacobian(vec(Σ_sym), [par...])
+ ∇Σ_eval! = Symbolics.build_function(∇Σ_sym, par, expression = Val{false})[2]
+ constr = findnz(∇Σ_sym)
+ ∇Σ = sparse(constr[1], constr[2], fill(1.0, nnz(∇Σ_sym)), size(∇Σ_sym)...)
+ precompile(∇Σ_eval!, (typeof(∇Σ), Vector{Float64}))
else
- ∇Σ_symbolic = nothing
- ∇Σ_function = nothing
+ ∇Σ_eval! = nothing
∇Σ = nothing
end
if hessian && !approximate_hessian
- n_sig = length(Σ_symbolic)
- ∇²Σ_symbolic_vec = [Symbolics.sparsehessian(σᵢ, [par...]) for σᵢ in vec(Σ_symbolic)]
+ n_sig = length(Σ_sym)
+ ∇²Σ_sym_vec = [Symbolics.sparsehessian(σᵢ, [par...]) for σᵢ in vec(Σ_sym)]
@variables J[1:n_sig]
- ∇²Σ_symbolic = zeros(Num, n_par, n_par)
+ ∇²Σ_sym = zeros(Num, n_par, n_par)
for i in 1:n_sig
- ∇²Σ_symbolic += J[i] * ∇²Σ_symbolic_vec[i]
+ ∇²Σ_sym += J[i] * ∇²Σ_sym_vec[i]
end
- ∇²Σ_function =
- Symbolics.build_function(∇²Σ_symbolic, J, par, expression = Val{false})[2]
+ ∇²Σ_eval! = Symbolics.build_function(∇²Σ_sym, J, par, expression = Val{false})[2]
∇²Σ = zeros(n_par, n_par)
else
- ∇²Σ_symbolic = nothing
- ∇²Σ_function = nothing
+ ∇²Σ_sym = nothing
+ ∇²Σ_eval! = nothing
∇²Σ = nothing
end
# μ
- if meanstructure
+ if !isnothing(ram_matrices.M)
MS = HasMeanStruct
- μ_symbolic = eval_μ_symbolic(M, I_A⁻¹, F; simplify = simplify_symbolics)
- μ_function = Symbolics.build_function(μ_symbolic, par, expression = Val{false})[2]
- μ = zeros(size(μ_symbolic))
+ μ_sym = eval_μ_symbolic(M, I_A⁻¹, F; simplify = simplify_symbolics)
+ μ_eval! = Symbolics.build_function(μ_sym, par, expression = Val{false})[2]
+ μ = zeros(size(μ_sym))
if gradient
- ∇μ_symbolic = Symbolics.jacobian(μ_symbolic, [par...])
- ∇μ_function =
- Symbolics.build_function(∇μ_symbolic, par, expression = Val{false})[2]
+ ∇μ_sym = Symbolics.jacobian(μ_sym, [par...])
+ ∇μ_eval! = Symbolics.build_function(∇μ_sym, par, expression = Val{false})[2]
∇μ = zeros(size(F, 1), size(par, 1))
else
- ∇μ_function = nothing
+ ∇μ_eval! = nothing
∇μ = nothing
end
else
MS = NoMeanStruct
- μ_function = nothing
+ μ_eval! = nothing
μ = nothing
- ∇μ_function = nothing
+ ∇μ_eval! = nothing
∇μ = nothing
end
return RAMSymbolic{MS}(
- Σ_function,
- ∇Σ_function,
- ∇²Σ_function,
+ Σ_eval!,
+ ∇Σ_eval!,
+ ∇²Σ_eval!,
Σ,
∇Σ,
∇²Σ,
- Σ_symbolic,
- ∇Σ_symbolic,
- ∇²Σ_symbolic,
ram_matrices,
- μ_function,
+ μ_eval!,
μ,
- ∇μ_function,
+ ∇μ_eval!,
∇μ,
)
end
@@ -194,46 +176,32 @@ end
### objective, gradient, hessian
############################################################################################
-function update!(
- targets::EvaluationTargets,
- implied::RAMSymbolic,
- model::AbstractSemSingle,
- par,
-)
- implied.Σ_function(implied.Σ, par)
+function update!(targets::EvaluationTargets, implied::RAMSymbolic, par)
+ implied.Σ_eval!(implied.Σ, par)
if MeanStruct(implied) === HasMeanStruct
- implied.μ_function(implied.μ, par)
+ implied.μ_eval!(implied.μ, par)
end
if is_gradient_required(targets) || is_hessian_required(targets)
- implied.∇Σ_function(implied.∇Σ, par)
+ implied.∇Σ_eval!(implied.∇Σ, par)
if MeanStruct(implied) === HasMeanStruct
- implied.∇μ_function(implied.∇μ, par)
+ implied.∇μ_eval!(implied.∇μ, par)
end
end
end
-############################################################################################
-### Recommended methods
-############################################################################################
-
-function update_observed(implied::RAMSymbolic, observed::SemObserved; kwargs...)
- if nobserved_vars(observed) == size(implied.Σ, 1)
- return implied
- else
- return RAMSymbolic(; observed = observed, kwargs...)
- end
-end
-
############################################################################################
### additional functions
############################################################################################
# expected covariations of observed vars
-function eval_Σ_symbolic(S, I_A⁻¹, F; vech = false, simplify = false)
+function eval_Σ_symbolic(S, I_A⁻¹, F; vech::Bool = false, simplify::Bool = false)
Σ = F * I_A⁻¹ * S * permutedims(I_A⁻¹) * permutedims(F)
Σ = Array(Σ)
- vech && (Σ = Σ[tril(trues(size(F, 1), size(F, 1)))])
+ if vech
+ n = size(Σ, 1)
+ Σ = [Σ[i, j] for j in 1:n for i in j:n]
+ end
if simplify
Threads.@threads for i in eachindex(Σ)
Σ[i] = Symbolics.simplify(Σ[i])
diff --git a/src/implied/abstract.jl b/src/implied/abstract.jl
index d4868d746..e41e79f62 100644
--- a/src/implied/abstract.jl
+++ b/src/implied/abstract.jl
@@ -8,6 +8,7 @@ nobserved_vars(implied::SemImplied) = nobserved_vars(implied.ram_matrices)
nlatent_vars(implied::SemImplied) = nlatent_vars(implied.ram_matrices)
param_labels(implied::SemImplied) = param_labels(implied.ram_matrices)
+params(implied::SemImplied) = param_labels(implied)
nparams(implied::SemImplied) = nparams(implied.ram_matrices)
# checks if the A matrix is acyclic
diff --git a/src/implied/empty.jl b/src/implied/empty.jl
index 82a6c9469..3f1c3ef5b 100644
--- a/src/implied/empty.jl
+++ b/src/implied/empty.jl
@@ -7,14 +7,14 @@ Empty placeholder for models that don't need an implied part.
# Constructor
- ImpliedEmpty(;specification, kwargs...)
+ ImpliedEmpty(specification; kwargs...)
# Arguments
- `specification`: either a `RAMMatrices` or `ParameterTable` object
# Examples
-A multigroup model with ridge regularization could be specified as a `SemEnsemble` with one
-model per group and an additional model with `ImpliedEmpty` and `SemRidge` for the regularization part.
+A multigroup model with ridge regularization could be specified as a `Sem` with one
+SEM term (`SemLoss`) per group and an additional `SemRidge` regularization term.
# Extended help
@@ -32,8 +32,8 @@ end
### Constructors
############################################################################################
-function ImpliedEmpty(;
- specification,
+function ImpliedEmpty(
+ specification::SemSpecification;
meanstruct = NoMeanStruct(),
hessianeval = ExactHessian(),
kwargs...,
@@ -45,10 +45,4 @@ end
### methods
############################################################################################
-update!(targets::EvaluationTargets, implied::ImpliedEmpty, par, model) = nothing
-
-############################################################################################
-### Recommended methods
-############################################################################################
-
-update_observed(implied::ImpliedEmpty, observed::SemObserved; kwargs...) = implied
+update!(targets::EvaluationTargets, implied::ImpliedEmpty, par) = nothing
diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl
index ea8da6b37..1f02ade18 100644
--- a/src/loss/ML/FIML.jl
+++ b/src/loss/ML/FIML.jl
@@ -1,38 +1,107 @@
############################################################################################
### Types
############################################################################################
+
+# state of SemFIML for a specific missing pattern (`SemObservedMissingPattern` type)
+struct SemFIMLPattern{T}
+ ∇ind::Vector{Int} # indices of co-observed variable pairs
+ Σ⁻¹::Matrix{T} # preallocated inverse of implied cov
+ logdet::Ref{T} # logdet of implied cov
+ μ_diff::Vector{T} # implied mean difference
+end
+
+# allocate arrays for pattern FIML
+function SemFIMLPattern(pat::SemObservedMissingPattern)
+ nobs_vars = nobserved_vars(pat)
+ nmeas_vars = nmeasured_vars(pat)
+
+ # generate linear indicies of co-observed variable pairs for each pattern
+ Σ_linind = LinearIndices((nobs_vars, nobs_vars))
+ pat_vars = findall(pat.measured_mask)
+ ∇ind = vec(Σ_linind[pat_vars, pat_vars])
+
+ return SemFIMLPattern(∇ind, zeros(nmeas_vars, nmeas_vars), Ref(NaN), zeros(nmeas_vars))
+end
+
+function prepare!(
+ patloss::SemFIMLPattern,
+ pat::SemObservedMissingPattern,
+ Σ::AbstractMatrix,
+ μ::AbstractVector,
+)
+ @inbounds @. @views begin
+ patloss.Σ⁻¹ = Σ[pat.measured_mask, pat.measured_mask]
+ patloss.μ_diff = pat.measured_mean - μ[pat.measured_mask]
+ end
+ Σ_chol = cholesky!(Symmetric(patloss.Σ⁻¹))
+ patloss.logdet[] = logdet(Σ_chol)
+ LinearAlgebra.inv!(Σ_chol) # updates loss.Σ⁻¹
+ return patloss
+end
+
+prepare!(patloss::SemFIMLPattern, pat::SemObservedMissingPattern, implied::SemImplied) =
+ prepare!(patloss, pat, implied.Σ, implied.μ)
+
+function objective(patloss::SemFIMLPattern{T}, pat::SemObservedMissingPattern) where {T}
+ F = patloss.logdet[] + dot(patloss.μ_diff, patloss.Σ⁻¹, patloss.μ_diff)
+ if nsamples(pat) > 1
+ F += dot(pat.measured_cov, patloss.Σ⁻¹)
+ F *= nsamples(pat)
+ end
+ return F
+end
+
+function gradient!(JΣ, Jμ, patloss::SemFIMLPattern, pat::SemObservedMissingPattern)
+ Σ⁻¹ = Symmetric(patloss.Σ⁻¹)
+ μ_diff⨉Σ⁻¹ = patloss.μ_diff' * Σ⁻¹
+ if nsamples(pat) > 1
+ JΣ_pat = Σ⁻¹ * (I - pat.measured_cov * Σ⁻¹ - patloss.μ_diff * μ_diff⨉Σ⁻¹)
+ JΣ_pat .*= nsamples(pat)
+ else
+ JΣ_pat = Σ⁻¹ * (I - patloss.μ_diff * μ_diff⨉Σ⁻¹)
+ end
+ @inbounds vec(JΣ)[patloss.∇ind] .+= vec(JΣ_pat)
+
+ lmul!(2 * nsamples(pat), μ_diff⨉Σ⁻¹)
+ @inbounds Jμ[pat.measured_mask] .+= μ_diff⨉Σ⁻¹'
+ return nothing
+end
+
"""
-Full information maximum likelihood estimation. Can handle observed data with missings.
+ SemFIML{O, I, T, W} <: SemLoss{O, I}
+
+Full information maximum likelihood (FIML) estimation.
+Can handle observed data with missing values.
# Constructor
- SemFIML(;observed, specification, kwargs...)
+ SemFIML(observed::SemObservedMissing, implied::SemImplied, refloss = nothing)
# Arguments
- `observed::SemObservedMissing`: the observed part of the model
-- `specification`: either a `RAMMatrices` or `ParameterTable` object
+ (see [`SemObservedMissing`](@ref))
+- `implied::SemImplied`: the implied part of the model
+ (see [`SemImplied`](@ref))
+- `refloss::Union{SemFIML, Nothing}`: optional reference loss used to preserve
+ loss-specific configuration and share the internal state when rebuilding
+ a loss term, e.g. in [`replace_observed`](@ref)
# Examples
```julia
-my_fiml = SemFIML(observed = my_observed, specification = my_parameter_table)
+my_fiml = SemFIML(my_observed, my_implied)
```
# Interfaces
Analytic gradients are available.
"""
-mutable struct SemFIML{INV, C, L, O, M, IM, I, T, W} <: SemLossFunction
+struct SemFIML{O, I, T, W} <: SemLoss{O, I}
hessianeval::ExactHessian
- inverses::INV #preallocated inverses of imp_cov
- choleskys::C #preallocated choleskys
- logdets::L #logdets of implied covmats
- ∇ind::O
+ observed::O
+ implied::I
+ patterns::Vector{SemFIMLPattern{T}}
- imp_mean::IM
- meandiff::M
- imp_inv::I
-
- mult::T
+ imp_inv::Matrix{T} # implied inverse
commutator::CommutationMatrix
@@ -43,39 +112,30 @@ end
### Constructors
############################################################################################
-function SemFIML(; observed::SemObservedMissing, specification, kwargs...)
- inverses =
- [zeros(nmeasured_vars(pat), nmeasured_vars(pat)) for pat in observed.patterns]
- choleskys = Array{Cholesky{Float64, Array{Float64, 2}}, 1}(undef, length(inverses))
-
- n_patterns = length(observed.patterns)
- logdets = zeros(n_patterns)
-
- imp_mean = [zeros(nmeasured_vars(pat)) for pat in observed.patterns]
- meandiff = [zeros(nmeasured_vars(pat)) for pat in observed.patterns]
-
- nobs_vars = nobserved_vars(observed)
- imp_inv = zeros(nobs_vars, nobs_vars)
- mult = similar.(inverses)
-
- # generate linear indicies of co-observed variable pairs for each pattern
- Σ_linind = LinearIndices((nobs_vars, nobs_vars))
- ∇ind = map(observed.patterns) do pat
- pat_vars = findall(pat.measured_mask)
- vec(Σ_linind[pat_vars, pat_vars])
+function SemFIML(
+ observed::SemObservedMissing,
+ implied::SemImplied,
+ refloss::Union{SemFIML, Nothing} = nothing;
+ kwargs...,
+)
+ if MeanStruct(implied) === NoMeanStruct
+ """
+ Full information maximum likelihood (FIML) can only be used with a meanstructure.
+ Did you forget to set `Sem(..., meanstructure = true)`?
+ """ |>
+ ArgumentError |>
+ throw
end
return SemFIML(
ExactHessian(),
- inverses,
- choleskys,
- logdets,
- ∇ind,
- imp_mean,
- meandiff,
- imp_inv,
- mult,
- CommutationMatrix(nvars(specification)),
+ observed,
+ implied,
+ [SemFIMLPattern(pat) for pat in observed.patterns],
+ # share the internal state with the refloss
+ !isnothing(refloss) ? refloss.imp_inv :
+ zeros(nobserved_vars(observed), nobserved_vars(observed)),
+ !isnothing(refloss) ? refloss.commutator : CommutationMatrix(nvars(implied)),
nothing,
)
end
@@ -84,76 +144,47 @@ end
### methods
############################################################################################
-function evaluate!(
- objective,
- gradient,
- hessian,
- semfiml::SemFIML,
- implied::SemImplied,
- model::AbstractSemSingle,
- param_labels,
-)
+function evaluate!(objective, gradient, hessian, loss::SemFIML, params)
isnothing(hessian) || error("Hessian not implemented for FIML")
- if !check_fiml(semfiml, model)
- isnothing(objective) || (objective = non_posdef_return(param_labels))
+ implied = SEM.implied(loss)
+ observed = SEM.observed(loss)
+
+ copyto!(loss.imp_inv, implied.Σ)
+ Σ_chol = cholesky!(Symmetric(loss.imp_inv); check = false)
+
+ if !isposdef(Σ_chol)
+ isnothing(objective) || (objective = non_posdef_return(params))
isnothing(gradient) || fill!(gradient, 1)
return objective
end
- prepare_SemFIML!(semfiml, model)
+ @inbounds for (patloss, pat) in zip(loss.patterns, observed.patterns)
+ prepare!(patloss, pat, implied)
+ end
- scale = inv(nsamples(observed(model)))
- isnothing(objective) ||
- (objective = scale * F_FIML(observed(model), semfiml, model, param_labels))
- isnothing(gradient) ||
- (∇F_FIML!(gradient, observed(model), semfiml, model); gradient .*= scale)
+ scale = inv(nsamples(observed))
+ isnothing(objective) || (objective = scale * F_FIML(eltype(params), loss))
+ isnothing(gradient) || (∇F_FIML!(gradient, loss); gradient .*= scale)
return objective
end
-
-############################################################################################
-### Recommended methods
-############################################################################################
-
-update_observed(lossfun::SemFIML, observed::SemObserved; kwargs...) =
- SemFIML(; observed = observed, kwargs...)
-
############################################################################################
### additional functions
############################################################################################
-function F_one_pattern(meandiff, inverse, obs_cov, logdet, N)
- F = logdet + dot(meandiff, inverse, meandiff)
- if N > one(N)
- F += dot(obs_cov, inverse)
- end
- return F * N
-end
-
-function ∇F_one_pattern(μ_diff, Σ⁻¹, S, obs_mask, ∇ind, N, Jμ, JΣ, model)
- diff⨉inv = μ_diff' * Σ⁻¹
-
- if N > one(N)
- JΣ[∇ind] .+= N * vec(Σ⁻¹ * (I - S * Σ⁻¹ - μ_diff * diff⨉inv))
- @. Jμ[obs_mask] += (N * 2 * diff⨉inv)'
-
- else
- JΣ[∇ind] .+= vec(Σ⁻¹ * (I - μ_diff * diff⨉inv))
- @. Jμ[obs_mask] += (2 * diff⨉inv)'
- end
+function ∇F_fiml_outer!(G, JΣ, Jμ, loss::SemFIML{O, I}) where {O, I <: SemImpliedSymbolic}
+ mul!(G, loss.implied.∇Σ', JΣ) # should be transposed
+ mul!(G, loss.implied.∇μ', Jμ, -1, 1)
end
-function ∇F_fiml_outer!(G, JΣ, Jμ, implied::SemImpliedSymbolic, model, semfiml)
- mul!(G, implied.∇Σ', JΣ) # should be transposed
- mul!(G, implied.∇μ', Jμ, -1, 1)
-end
+function ∇F_fiml_outer!(G, JΣ, Jμ, loss::SemFIML)
+ implied = loss.implied
-function ∇F_fiml_outer!(G, JΣ, Jμ, implied, model, semfiml)
Iₙ = sparse(1.0I, size(implied.A)...)
P = kron(implied.F⨉I_A⁻¹, implied.F⨉I_A⁻¹)
Q = kron(implied.S * implied.I_A⁻¹', Iₙ)
- Q .+= semfiml.commutator * Q
+ Q .+= loss.commutator * Q
∇Σ = P * (implied.∇S + Q * implied.∇A)
@@ -165,70 +196,20 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, implied, model, semfiml)
mul!(G, ∇μ', Jμ, -1, 1)
end
-function F_FIML(observed::SemObservedMissing, semfiml, model, param_labels)
- F = zero(eltype(param_labels))
- for (i, pat) in enumerate(observed.patterns)
- F += F_one_pattern(
- semfiml.meandiff[i],
- semfiml.inverses[i],
- pat.measured_cov,
- semfiml.logdets[i],
- nsamples(pat),
- )
+function F_FIML(::Type{T}, loss::SemFIML) where {T}
+ F = zero(T)
+ for (patloss, pat) in zip(loss.patterns, loss.observed.patterns)
+ F += objective(patloss, pat)
end
return F
end
-function ∇F_FIML!(G, observed::SemObservedMissing, semfiml, model)
- Jμ = zeros(nobserved_vars(model))
- JΣ = zeros(nobserved_vars(model)^2)
-
- for (i, pat) in enumerate(observed.patterns)
- ∇F_one_pattern(
- semfiml.meandiff[i],
- semfiml.inverses[i],
- pat.measured_cov,
- pat.measured_mask,
- semfiml.∇ind[i],
- nsamples(pat),
- Jμ,
- JΣ,
- model,
- )
- end
- return ∇F_fiml_outer!(G, JΣ, Jμ, implied(model), model, semfiml)
-end
+function ∇F_FIML!(G, loss::SemFIML)
+ Jμ = zeros(nobserved_vars(loss))
+ JΣ = zeros(nobserved_vars(loss)^2)
-function prepare_SemFIML!(semfiml, model)
- copy_per_pattern!(semfiml, model)
- batch_cholesky!(semfiml, model)
- #batch_sym_inv_update!(semfiml, model)
- batch_inv!(semfiml, model)
- for (i, pat) in enumerate(observed(model).patterns)
- semfiml.meandiff[i] .= pat.measured_mean .- semfiml.imp_mean[i]
+ for (patloss, pat) in zip(loss.patterns, loss.observed.patterns)
+ gradient!(JΣ, Jμ, patloss, pat)
end
-end
-
-function copy_per_pattern!(fiml::SemFIML, model::AbstractSem)
- Σ = implied(model).Σ
- μ = implied(model).μ
- data = observed(model)
- @inbounds @views for (i, pat) in enumerate(data.patterns)
- fiml.inverses[i] .= Σ[pat.measured_mask, pat.measured_mask]
- fiml.imp_mean[i] .= μ[pat.measured_mask]
- end
-end
-
-function batch_cholesky!(semfiml, model)
- for i in 1:size(semfiml.inverses, 1)
- semfiml.choleskys[i] = cholesky!(Symmetric(semfiml.inverses[i]))
- semfiml.logdets[i] = logdet(semfiml.choleskys[i])
- end
- return true
-end
-
-function check_fiml(semfiml, model)
- copyto!(semfiml.imp_inv, implied(model).Σ)
- a = cholesky!(Symmetric(semfiml.imp_inv); check = false)
- return isposdef(a)
+ ∇F_fiml_outer!(G, JΣ, Jμ, loss)
end
diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl
index ec5eb997c..cf1198321 100644
--- a/src/loss/ML/ML.jl
+++ b/src/loss/ML/ML.jl
@@ -8,42 +8,78 @@ Maximum likelihood estimation.
# Constructor
- SemML(;observed, meanstructure = false, approximate_hessian = false, kwargs...)
+ SemML(observed, implied, refloss = nothing; approximate_hessian = false)
# Arguments
- `observed::SemObserved`: the observed part of the model
-- `meanstructure::Bool`: does the model have a meanstructure?
+- `implied::SemImplied`: [`SemImplied`](@ref) instance
+- `refloss::Union{SemML, Nothing}`: optional reference loss used to preserve
+ loss-specific configuration and share the internal state when rebuilding
+ a loss term, e.g. in [`replace_observed`](@ref)
- `approximate_hessian::Bool`: if hessian-based optimization is used, should the hessian be swapped for an approximation
# Examples
```julia
-my_ml = SemML(observed = my_observed)
+my_ml = SemML(my_observed, my_implied)
```
# Interfaces
Analytic gradients are available, and for models without a meanstructure
-and RAMSymbolic implied type, also analytic hessians.
+and `RAMSymbolic` implied type, also analytic hessians.
"""
-struct SemML{HE <: HessianEval, INV, M, M2} <: SemLossFunction
+struct SemML{O, I, HE <: HessianEval, INV, M, M2} <: SemLoss{O, I}
+ observed::O
+ implied::I
hessianeval::HE
Σ⁻¹::INV
Σ⁻¹Σₒ::M
meandiff::M2
-
- SemML{HE}(args...) where {HE <: HessianEval} =
- new{HE, map(typeof, args)...}(HE(), args...)
end
############################################################################################
### Constructors
############################################################################################
-function SemML(; observed::SemObserved, approximate_hessian::Bool = false, kwargs...)
+function SemML(
+ observed::SemObserved,
+ implied::SemImplied,
+ refloss::Union{SemML, Nothing} = nothing;
+ approximate_hessian::Bool = !isnothing(refloss) ?
+ HessianEval(refloss) === ApproxHessian : false,
+ kwargs...,
+)
+ if observed isa SemObservedMissing
+ @warn """
+ ML estimation with `SemObservedMissing` will use an approximate covariance and mean estimated with EM algorithm.
+ For more accurate results, consider using full information maximum likelihood (FIML) estimation or remove missing values in your data.
+ A FIML model can be constructed with
+ Sem(
+ ...,
+ observed = SemObservedMissing,
+ loss = SemFIML,
+ meanstructure = true
+ )
+ """
+ end
+ # check integrity
+ check_observed_vars(observed, implied)
+
+ he = approximate_hessian ? ApproxHessian() : ExactHessian()
obsmean = obs_mean(observed)
obscov = obs_cov(observed)
meandiff = isnothing(obsmean) ? nothing : copy(obsmean)
- return SemML{approximate_hessian ? ApproxHessian : ExactHessian}(
+ return SemML{
+ typeof(observed),
+ typeof(implied),
+ typeof(he),
+ typeof(obscov),
+ typeof(obscov),
+ typeof(meandiff),
+ }(
+ observed,
+ implied,
+ he,
similar(obscov),
similar(obscov),
meandiff,
@@ -61,20 +97,20 @@ function evaluate!(
objective,
gradient,
hessian,
- semml::SemML,
- implied::SemImpliedSymbolic,
- model::AbstractSemSingle,
+ loss::SemML{<:Any, <:SemImpliedSymbolic},
par,
)
+ implied = SEM.implied(loss)
+
if !isnothing(hessian)
(MeanStruct(implied) === HasMeanStruct) &&
throw(DomainError(H, "hessian of ML + meanstructure is not available"))
end
Σ = implied.Σ
- Σₒ = obs_cov(observed(model))
- Σ⁻¹Σₒ = semml.Σ⁻¹Σₒ
- Σ⁻¹ = semml.Σ⁻¹
+ Σₒ = obs_cov(observed(loss))
+ Σ⁻¹Σₒ = loss.Σ⁻¹Σₒ
+ Σ⁻¹ = loss.Σ⁻¹
copyto!(Σ⁻¹, Σ)
Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false)
@@ -92,7 +128,7 @@ function evaluate!(
if MeanStruct(implied) === HasMeanStruct
μ = implied.μ
- μₒ = obs_mean(observed(model))
+ μₒ = obs_mean(observed(loss))
μ₋ = μₒ - μ
isnothing(objective) || (objective += dot(μ₋, Σ⁻¹, μ₋))
@@ -111,13 +147,12 @@ function evaluate!(
mul!(gradient, ∇Σ', J')
end
if !isnothing(hessian)
- if HessianEval(semml) === ApproxHessian
+ if HessianEval(loss) === ApproxHessian
mul!(hessian, ∇Σ' * kron(Σ⁻¹, Σ⁻¹), ∇Σ, 2, 0)
else
- ∇²Σ_function! = implied.∇²Σ_function
∇²Σ = implied.∇²Σ
# inner
- ∇²Σ_function!(∇²Σ, J, par)
+ implied.∇²Σ_eval!(∇²Σ, J, par)
# outer
H_outer = kron(2Σ⁻¹ΣₒΣ⁻¹ - Σ⁻¹, Σ⁻¹)
mul!(hessian, ∇Σ' * H_outer, ∇Σ)
@@ -131,24 +166,17 @@ end
############################################################################################
### Non-Symbolic Implied Types
-function evaluate!(
- objective,
- gradient,
- hessian,
- semml::SemML,
- implied::RAM,
- model::AbstractSemSingle,
- par,
-)
+function evaluate!(objective, gradient, hessian, loss::SemML, par)
if !isnothing(hessian)
error("hessian of ML + non-symbolic implied type is not available")
end
- Σ = implied.Σ
- Σₒ = obs_cov(observed(model))
- Σ⁻¹Σₒ = semml.Σ⁻¹Σₒ
- Σ⁻¹ = semml.Σ⁻¹
+ implied = SEM.implied(loss)
+ Σ = implied.Σ
+ Σₒ = obs_cov(observed(loss))
+ Σ⁻¹Σₒ = loss.Σ⁻¹Σₒ
+ Σ⁻¹ = loss.Σ⁻¹
copyto!(Σ⁻¹, Σ)
Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false)
if !isposdef(Σ_chol)
@@ -167,7 +195,7 @@ function evaluate!(
if MeanStruct(implied) === HasMeanStruct
μ = implied.μ
- μₒ = obs_mean(observed(model))
+ μₒ = obs_mean(observed(loss))
μ₋ = μₒ - μ
objective += dot(μ₋, Σ⁻¹, μ₋)
end
@@ -186,7 +214,7 @@ function evaluate!(
if MeanStruct(implied) === HasMeanStruct
μ = implied.μ
- μₒ = obs_mean(observed(model))
+ μₒ = obs_mean(observed(loss))
∇M = implied.∇M
M = implied.M
μ₋ = μₒ - μ
@@ -212,18 +240,3 @@ function non_posdef_return(par)
return typemax(eltype(par))
end
end
-
-############################################################################################
-### recommended methods
-############################################################################################
-
-update_observed(lossfun::SemML, observed::SemObservedMissing; kwargs...) =
- error("ML estimation does not work with missing data - use FIML instead")
-
-function update_observed(lossfun::SemML, observed::SemObserved; kwargs...)
- if size(lossfun.Σ⁻¹) == size(obs_cov(observed))
- return lossfun
- else
- return SemML(; observed = observed, kwargs...)
- end
-end
diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl
index dd5be4874..d067e346a 100644
--- a/src/loss/WLS/WLS.jl
+++ b/src/loss/WLS/WLS.jl
@@ -9,9 +9,8 @@ At the moment only available with the `RAMSymbolic` implied type.
# Constructor
- SemWLS(;
- observed,
- meanstructure = false,
+ SemWLS(
+ observed::SemObserved, implied::SemImplied, refloss = nothing;
wls_weight_matrix = nothing,
wls_weight_matrix_mean = nothing,
approximate_hessian = false,
@@ -19,7 +18,10 @@ At the moment only available with the `RAMSymbolic` implied type.
# Arguments
- `observed`: the `SemObserved` part of the model
-- `meanstructure::Bool`: does the model have a meanstructure?
+- `implied`: the `SemImplied` part of the model
+- `refloss::Union{SemWLS, Nothing}`: optional reference loss used to preserve
+ loss-specific configuration and share the internal state when rebuilding
+ a loss term, e.g. in [`replace_observed`](@ref)
- `approximate_hessian::Bool`: should the hessian be swapped for an approximation
- `wls_weight_matrix`: the weight matrix for weighted least squares.
Defaults to GLS estimation (``0.5*(D^T*kron(S,S)*D)`` where D is the duplication matrix
@@ -29,37 +31,79 @@ At the moment only available with the `RAMSymbolic` implied type.
# Examples
```julia
-my_wls = SemWLS(observed = my_observed)
+my_wls = SemWLS(my_observed, my_implied)
```
# Interfaces
Analytic gradients are available, and for models without a meanstructure also analytic hessians.
"""
-struct SemWLS{HE <: HessianEval, Vt, St, C} <: SemLossFunction
+struct SemWLS{O, I, HE <: HessianEval, Vt, St, C} <: SemLoss{O, I}
+ observed::O
+ implied::I
+
hessianeval::HE
V::Vt
σₒ::St
V_μ::C
+
+ SemWLS(observed, implied, ::Type{HE}, args...) where {HE <: HessianEval} =
+ new{typeof(observed), typeof(implied), HE, map(typeof, args)...}(
+ observed,
+ implied,
+ HE(),
+ args...,
+ )
end
############################################################################################
### Constructors
############################################################################################
-SemWLS{HE}(args...) where {HE <: HessianEval} =
- SemWLS{HE, map(typeof, args)...}(HE(), args...)
-
-function SemWLS(;
- observed,
- wls_weight_matrix = nothing,
- wls_weight_matrix_mean = nothing,
- approximate_hessian = false,
- meanstructure = false,
+function SemWLS(
+ observed::SemObserved,
+ implied::SemImplied,
+ refloss::Union{SemWLS, Nothing} = nothing;
+ wls_weight_matrix::Union{AbstractMatrix, Nothing} = !isnothing(refloss) ? refloss.V :
+ nothing,
+ wls_weight_matrix_mean::Union{AbstractMatrix, Nothing} = !isnothing(refloss) ?
+ refloss.V_μ : nothing,
+ approximate_hessian::Bool = !isnothing(refloss) ?
+ HessianEval(refloss) === ApproxHessian : false,
+ verbose::Bool = false,
kwargs...,
)
+ if observed isa SemObservedMissing
+ @warn """
+ WLS estimation with `SemObservedMissing` will use an approximate covariance and mean estimated with EM algorithm.
+ For more accurate results, consider using full information maximum likelihood (FIML) estimation or remove missing values in your data.
+ A FIML model can be constructed with
+ Sem(
+ ...,
+ observed = SemObservedMissing,
+ loss = SemFIML,
+ meanstructure = true
+ )
+ """
+ end
+
+ if !(implied isa RAMSymbolic)
+ "WLS estimation is only available with the implied type RAMSymbolic at the moment." |>
+ ArgumentError |>
+ throw
+ end
+ # check integrity
+ check_observed_vars(observed, implied)
+
nobs_vars = nobserved_vars(observed)
tril_ind = filter(x -> (x[1] >= x[2]), CartesianIndices(obs_cov(observed)))
s = obs_cov(observed)[tril_ind]
+ size(s) == size(implied.Σ) || throw(
+ DimensionMismatch(
+ "SemWLS requires implied covariance to be in vech-ed form " *
+ "(vectorized lower triangular part of Σ matrix): $(size(s)) expected, $(size(implied.Σ)) found.\n" *
+ "$(nameof(typeof(implied))) must be constructed with vech=true.",
+ ),
+ )
# compute V here
if isnothing(wls_weight_matrix)
@@ -73,15 +117,18 @@ function SemWLS(;
"wls_weight_matrix has to be of size $(length(tril_ind))×$(length(tril_ind))",
)
end
+ size(wls_weight_matrix) == (length(s), length(s)) ||
+ DimensionMismatch("wls_weight_matrix has to be of size $(length(s))×$(length(s))")
- if meanstructure
+ if MeanStruct(implied) == HasMeanStruct
if isnothing(wls_weight_matrix_mean)
+ verbose &&
+ @info "Computing WLS weight matrix for the meanstructure using obs_cov()"
wls_weight_matrix_mean = inv(obs_cov(observed))
- else
- size(wls_weight_matrix_mean) == (nobs_vars, nobs_vars) || DimensionMismatch(
- "wls_weight_matrix_mean has to be of size $(nobs_vars)×$(nobs_vars)",
- )
end
+ size(wls_weight_matrix_mean) == (nobs_vars, nobs_vars) || DimensionMismatch(
+ "wls_weight_matrix_mean has to be of size $(nobs_vars)×$(nobs_vars)",
+ )
else
isnothing(wls_weight_matrix_mean) ||
@warn "Ignoring wls_weight_matrix_mean since meanstructure is disabled"
@@ -89,31 +136,25 @@ function SemWLS(;
end
HE = approximate_hessian ? ApproxHessian : ExactHessian
- return SemWLS{HE}(wls_weight_matrix, s, wls_weight_matrix_mean)
+ return SemWLS(observed, implied, HE, wls_weight_matrix, s, wls_weight_matrix_mean)
end
############################################################################
### methods
############################################################################
-function evaluate!(
- objective,
- gradient,
- hessian,
- semwls::SemWLS,
- implied::SemImpliedSymbolic,
- model::AbstractSemSingle,
- par,
-)
+function evaluate!(objective, gradient, hessian, loss::SemWLS, par)
+ implied = SEM.implied(loss)
+
if !isnothing(hessian) && (MeanStruct(implied) === HasMeanStruct)
error("hessian of WLS with meanstructure is not available")
end
- V = semwls.V
+ V = loss.V
∇σ = implied.∇Σ
σ = implied.Σ
- σₒ = semwls.σₒ
+ σₒ = loss.σₒ
σ₋ = σₒ - σ
isnothing(objective) || (objective = dot(σ₋, V, σ₋))
@@ -126,18 +167,17 @@ function evaluate!(
gradient .*= -2
end
isnothing(hessian) || (mul!(hessian, ∇σ' * V, ∇σ, 2, 0))
- if !isnothing(hessian) && (HessianEval(semwls) === ExactHessian)
- ∇²Σ_function! = implied.∇²Σ_function
+ if !isnothing(hessian) && (HessianEval(loss) === ExactHessian)
∇²Σ = implied.∇²Σ
- J = -2 * (σ₋' * semwls.V)'
- ∇²Σ_function!(∇²Σ, J, par)
+ J = -2 * (σ₋' * loss.V)'
+ implied.∇²Σ_eval!(∇²Σ, J, par)
hessian .+= ∇²Σ
end
if MeanStruct(implied) === HasMeanStruct
μ = implied.μ
- μₒ = obs_mean(observed(model))
+ μₒ = obs_mean(observed(loss))
μ₋ = μₒ - μ
- V_μ = semwls.V_μ
+ V_μ = loss.V_μ
if !isnothing(objective)
objective += dot(μ₋, V_μ, μ₋)
end
@@ -149,9 +189,17 @@ function evaluate!(
return objective
end
-############################################################################################
-### Recommended methods
-############################################################################################
-
-update_observed(lossfun::SemWLS, observed::SemObserved; kwargs...) =
- SemWLS(; observed = observed, kwargs...)
+function replace_observed(
+ loss::SemWLS,
+ new_observed::SemObserved;
+ recompute_observed_state::Bool = true,
+)
+ # recompute weight matrices only if recompute_observed_state=true
+ return SemWLS(
+ new_observed,
+ SEM.implied(loss),
+ loss;
+ wls_weight_matrix = recompute_observed_state ? nothing : loss.V,
+ wls_weight_matrix_mean = recompute_observed_state ? nothing : loss.V_μ,
+ )
+end
diff --git a/src/loss/abstract.jl b/src/loss/abstract.jl
new file mode 100644
index 000000000..56a3af585
--- /dev/null
+++ b/src/loss/abstract.jl
@@ -0,0 +1,78 @@
+"""
+ observed(loss::SemLoss) -> SemObserved
+
+Returns the [*observed*](@ref SemObserved) part of a model.
+"""
+observed(loss::SemLoss) = loss.observed
+
+"""
+ implied(loss::SemLoss) -> SemImplied
+
+Returns the [*implied*](@ref SemImplied) part of a model.
+"""
+implied(loss::SemLoss) = loss.implied
+
+for f in (:nsamples, :obs_cov, :obs_mean)
+ @eval $f(loss::SemLoss) = $f(observed(loss))
+end
+
+for f in (
+ :vars,
+ :nvars,
+ :latent_vars,
+ :nlatent_vars,
+ :observed_vars,
+ :nobserved_vars,
+ :params,
+ :nparams,
+)
+ @eval $f(loss::SemLoss) = $f(implied(loss))
+end
+
+function check_observed_vars(observed::SemObserved, implied::SemImplied)
+ isnothing(observed_vars(implied)) ||
+ observed_vars(observed) == observed_vars(implied) ||
+ throw(
+ ArgumentError(
+ "Observed variables defined for \"observed\" and \"implied\" do not match.",
+ ),
+ )
+end
+
+check_observed_vars(sem::SemLoss) = check_observed_vars(observed(sem), implied(sem))
+
+function check_observed_vars(loss::SemLoss, new_observed::SemObserved)
+ observed_vars(new_observed) == observed_vars(SEM.observed(loss)) || throw(
+ ArgumentError(
+ "Observed variables of the loss term do not match the ones of the new observed data",
+ ),
+ )
+end
+
+############################################################################################
+# replace_observed: SemLoss, AbstractLoss, LossTerm
+############################################################################################
+
+function replace_observed(loss::SemLoss, new_observed::SemObserved; kwargs...)
+ check_observed_vars(loss, new_observed)
+ # construct the new loss:
+ # 1) replace the observed
+ # 2) share the implied and its internal state with the original loss
+ # 3) replicate the current loss configuration/share its internal state
+ loss_ctor = typeof(loss).name.wrapper # get the loss constructor
+ return loss_ctor(new_observed, SEM.implied(loss), loss)
+end
+
+function replace_observed(loss::SemLoss, data::Union{AbstractMatrix, DataFrame}; kwargs...)
+ old_obs = SEM.observed(loss)
+ obs_ctor = typeof(old_obs).name.wrapper
+ new_observed = obs_ctor(data = data, observed_vars = observed_vars(old_obs))
+ return replace_observed(loss, new_observed; kwargs...)
+end
+
+# non-SEM loss terms are unchanged
+replace_observed(loss::AbstractLoss, ::Any; kwargs...) = loss
+
+# LossTerm: delegate to inner loss
+replace_observed(term::LossTerm, data; kwargs...) =
+ LossTerm(replace_observed(loss(term), data; kwargs...), id(term), weight(term))
diff --git a/src/loss/constant/constant.jl b/src/loss/constant/constant.jl
index 3aed5e27c..2aff0156d 100644
--- a/src/loss/constant/constant.jl
+++ b/src/loss/constant/constant.jl
@@ -4,6 +4,8 @@
### Types
############################################################################################
"""
+ SemConstant{C <: Number} <: AbstractLoss
+
Constant loss term. Can be used for comparability to other packages.
# Constructor
@@ -15,37 +17,21 @@ Constant loss term. Can be used for comparability to other packages.
# Examples
```julia
- my_constant = SemConstant(constant_loss = 42.0)
+ my_constant = SemConstant(42.0)
```
# Interfaces
Analytic gradients and hessians are available.
"""
-struct SemConstant{C} <: SemLossFunction
+struct SemConstant{C <: Number} <: AbstractLoss
hessianeval::ExactHessian
c::C
-end
-
-############################################################################################
-### Constructors
-############################################################################################
-function SemConstant(; constant_loss, kwargs...)
- return SemConstant(ExactHessian(), constant_loss)
+ SemConstant(c::Number) = new{typeof(c)}(ExactHessian(), c)
end
-############################################################################################
-### methods
-############################################################################################
-
-objective(constant::SemConstant, model::AbstractSem, par) = constant.c
-gradient(constant::SemConstant, model::AbstractSem, par) = zero(par)
-hessian(constant::SemConstant, model::AbstractSem, par) =
- zeros(eltype(par), length(par), length(par))
-
-############################################################################################
-### Recommended methods
-############################################################################################
+SemConstant(; constant_loss::Number, kwargs...) = SemConstant(constant_loss)
-update_observed(loss_function::SemConstant, observed::SemObserved; kwargs...) =
- loss_function
+objective(loss::SemConstant, par) = convert(eltype(par), loss.c)
+gradient(loss::SemConstant, par) = zero(par)
+hessian(loss::SemConstant, par) = zeros(eltype(par), length(par), length(par))
diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl
index 90cbcc231..813aff11c 100644
--- a/src/loss/regularization/ridge.jl
+++ b/src/loss/regularization/ridge.jl
@@ -25,7 +25,7 @@ my_ridge = SemRidge(;α_ridge = 0.02, which_ridge = [:λ₁, :λ₂, :ω₂₃],
# Interfaces
Analytic gradients and hessians are available.
"""
-struct SemRidge{P, W1, W2, GT, HT} <: SemLossFunction
+struct SemRidge{P, W1, W2, GT, HT} <: AbstractLoss
hessianeval::ExactHessian
α::P
which::W1
@@ -74,21 +74,14 @@ end
### methods
############################################################################################
-objective(ridge::SemRidge, model::AbstractSem, par) =
- @views ridge.α * sum(abs2, par[ridge.which])
+objective(ridge::SemRidge, par) = @views ridge.α * sum(abs2, par[ridge.which])
-function gradient(ridge::SemRidge, model::AbstractSem, par)
+function gradient(ridge::SemRidge, par)
@views ridge.gradient[ridge.which] .= (2 * ridge.α) * par[ridge.which]
return ridge.gradient
end
-function hessian(ridge::SemRidge, model::AbstractSem, par)
+function hessian(ridge::SemRidge, par)
@views @. ridge.hessian[ridge.which_H] .= 2 * ridge.α
return ridge.hessian
end
-
-############################################################################################
-### Recommended methods
-############################################################################################
-
-update_observed(loss::SemRidge, observed::SemObserved; kwargs...) = loss
diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl
index 06f39329f..4a8ec649e 100644
--- a/src/objective_gradient_hessian.jl
+++ b/src/objective_gradient_hessian.jl
@@ -23,44 +23,64 @@ is_hessian_required(::EvaluationTargets{<:Any, <:Any, H}) where {H} = H
(targets::EvaluationTargets)(arg_tuple::Tuple) = targets(arg_tuple...)
-# dispatch on SemImplied
-evaluate!(objective, gradient, hessian, loss::SemLossFunction, model::AbstractSem, params) =
- evaluate!(objective, gradient, hessian, loss, implied(model), model, params)
+"""
+ evaluate!(objective, gradient, hessian, loss::AbstractLoss, params)
+ evaluate!(objective, gradient, hessian, model::AbstractSem, params)
+
+Evaluates the objective, gradient, and/or Hessian at the given parameter vector.
+
+If a single loss term (`loss`) is passed, only this specific term is evaluated,
+otherwise, if the entire SEM `model` is passed, the weighted sum of all loss terms
+in the model is evaluated.
+
+If objective, gradient or hessian are `nothing`, they are not evaluated.
+For example, since many numerical optimization algorithms don't require a Hessian,
+its computation will be turned off by setting `hessian` to `nothing`.
+
+During the evaluation, the internal state of the loss term or of the model
+could be modified.
+
+Returns the objective value or `nothing`.
+
+# Arguments
+- `objective`: a Number if the objective should be evaluated, otherwise `nothing`
+- `gradient`: a pre-allocated vector the gradient should be written to, otherwise `nothing`
+- `hessian`: a pre-allocated matrix the Hessian should be written to, otherwise `nothing`
+- `loss::AbstractLoss`: loss function to evaluate
+- `model::AbstractSem`: model to evaluate
+- `params`: vector of parameters
+
+# Implementing a new loss function
+To implement a new loss (subtype of `SemLoss` for SEM terms, or of `AbstractLoss` for
+regularization terms), a new method for `evaluate!` has to be defined.
+This is explained in the online documentation on [Custom loss functions](@ref).
+"""
+function evaluate! end
# fallback method
-function evaluate!(
- obj,
- grad,
- hess,
- loss::SemLossFunction,
- implied::SemImplied,
- model,
- params,
-)
- isnothing(obj) || (obj = objective(loss, implied, model, params))
- isnothing(grad) || copyto!(grad, gradient(loss, implied, model, params))
- isnothing(hess) || copyto!(hess, hessian(loss, implied, model, params))
+function evaluate!(obj, grad, hess, loss::AbstractLoss, params)
+ isnothing(obj) || (obj = objective(loss, params))
+ isnothing(grad) || copyto!(grad, gradient(loss, params))
+ isnothing(hess) || copyto!(hess, hessian(loss, params))
return obj
end
-# fallback methods
-objective(f::SemLossFunction, implied::SemImplied, model, params) =
- objective(f, model, params)
-gradient(f::SemLossFunction, implied::SemImplied, model, params) =
- gradient(f, model, params)
-hessian(f::SemLossFunction, implied::SemImplied, model, params) = hessian(f, model, params)
+evaluate!(obj, grad, hess, term::LossTerm, params) =
+ evaluate!(obj, grad, hess, loss(term), params)
# fallback method for SemImplied that calls update_xxx!() methods
-function update!(targets::EvaluationTargets, implied::SemImplied, model, params)
- is_objective_required(targets) && update_objective!(implied, model, params)
- is_gradient_required(targets) && update_gradient!(implied, model, params)
- is_hessian_required(targets) && update_hessian!(implied, model, params)
+function update!(targets::EvaluationTargets, implied::SemImplied, params)
+ is_objective_required(targets) && update_objective!(implied, params)
+ is_gradient_required(targets) && update_gradient!(implied, params)
+ is_hessian_required(targets) && update_hessian!(implied, params)
end
+const AbstractSemOrLoss = Union{AbstractSem, AbstractLoss}
+
# guess objective type
-objective_type(model::AbstractSem, params::Any) = Float64
-objective_type(model::AbstractSem, params::AbstractVector{T}) where {T <: Number} = T
-objective_zero(model::AbstractSem, params::Any) = zero(objective_type(model, params))
+objective_type(model::AbstractSemOrLoss, params::Any) = Float64
+objective_type(model::AbstractSemOrLoss, params::AbstractVector{T}) where {T <: Number} = T
+objective_zero(model::AbstractSemOrLoss, params::Any) = zero(objective_type(model, params))
objective_type(objective::T, gradient, hessian) where {T <: Number} = T
objective_type(
@@ -76,145 +96,151 @@ objective_type(
objective_zero(objective, gradient, hessian) =
zero(objective_type(objective, gradient, hessian))
+evaluate!(objective, gradient, hessian, model::AbstractSem, params) =
+ error("evaluate!() for $(typeof(model)) is not implemented")
+
############################################################################################
-# methods for AbstractSem
+# methods for Sem
############################################################################################
-function evaluate!(objective, gradient, hessian, model::AbstractSemSingle, params)
- targets = EvaluationTargets(objective, gradient, hessian)
- # update implied state, its gradient and hessian (if required)
- update!(targets, implied(model), model, params)
- return evaluate!(
- !isnothing(objective) ? zero(objective) : nothing,
- gradient,
- hessian,
- loss(model),
- model,
- params,
- )
-end
+function evaluate!(objective, gradient, hessian, model::Sem, params)
+ # reset output
+ isnothing(objective) || (objective = objective_zero(objective, gradient, hessian))
+ isnothing(gradient) || fill!(gradient, zero(eltype(gradient)))
+ isnothing(hessian) || fill!(hessian, zero(eltype(hessian)))
-############################################################################################
-# methods for SemFiniteDiff
-# (approximate gradient and hessian with finite differences of objective)
-############################################################################################
+ # gradient and hessian for individual terms
+ t_grad = isnothing(gradient) ? nothing : similar(gradient)
+ t_hess = isnothing(hessian) ? nothing : similar(hessian)
+
+ # update implied states of all SemLoss terms before term calculation loop
+ # to make sure all terms use updated implied states
+ targets = EvaluationTargets(objective, gradient, hessian)
+ for term in loss_terms(model)
+ issemloss(term) && update!(targets, implied(term), params)
+ end
-function evaluate!(objective, gradient, hessian, model::SemFiniteDiff, params)
- function obj(p)
- # recalculate implied state for p
- update!(EvaluationTargets{true, false, false}(), implied(model), model, p)
- evaluate!(
- objective_zero(objective, gradient, hessian),
- nothing,
- nothing,
- loss(model),
- model,
- p,
+ for term in loss_terms(model)
+ t_obj = evaluate!(objective, t_grad, t_hess, term, params)
+ #@show nameof(typeof(term)) t_obj
+ objective = accumulate_loss!(
+ objective,
+ gradient,
+ hessian,
+ weight(term),
+ t_obj,
+ t_grad,
+ t_hess,
)
end
- isnothing(gradient) || FiniteDiff.finite_difference_gradient!(gradient, obj, params)
- isnothing(hessian) || FiniteDiff.finite_difference_hessian!(hessian, obj, params)
- return !isnothing(objective) ? obj(params) : nothing
+ return objective
end
-objective(model::AbstractSem, params) =
- evaluate!(objective_zero(model, params), nothing, nothing, model, params)
-
-############################################################################################
-# methods for SemLoss (weighted sum of individual SemLossFunctions)
-############################################################################################
+# internal function to accumulate loss objective, gradient and hessian
+function accumulate_loss!(
+ total_objective,
+ total_gradient,
+ total_hessian,
+ weight::Nothing,
+ objective,
+ gradient,
+ hessian,
+)
+ isnothing(total_gradient) || (total_gradient .+= gradient)
+ isnothing(total_hessian) || (total_hessian .+= hessian)
+ return isnothing(total_objective) ? total_objective : (total_objective + objective)
+end
-function evaluate!(objective, gradient, hessian, loss::SemLoss, model::AbstractSem, params)
- isnothing(objective) || (objective = zero(objective))
- isnothing(gradient) || fill!(gradient, zero(eltype(gradient)))
- isnothing(hessian) || fill!(hessian, zero(eltype(hessian)))
- f_grad = isnothing(gradient) ? nothing : similar(gradient)
- f_hess = isnothing(hessian) ? nothing : similar(hessian)
- for (f, weight) in zip(loss.functions, loss.weights)
- f_obj = evaluate!(objective, f_grad, f_hess, f, model, params)
- isnothing(objective) || (objective += weight * f_obj)
- isnothing(gradient) || (gradient .+= weight * f_grad)
- isnothing(hessian) || (hessian .+= weight * f_hess)
- end
- return objective
+function accumulate_loss!(
+ total_objective,
+ total_gradient,
+ total_hessian,
+ weight::Number,
+ objective,
+ gradient,
+ hessian,
+)
+ isnothing(total_gradient) || axpy!(weight, gradient, total_gradient)
+ isnothing(total_hessian) || axpy!(weight, hessian, total_hessian)
+ return isnothing(total_objective) ? total_objective :
+ (total_objective + weight * objective)
end
############################################################################################
-# methods for SemEnsemble (weighted sum of individual AbstractSemSingle models)
+# methods for SemFiniteDiff
+# (approximate gradient and hessian with finite differences of objective)
############################################################################################
-function evaluate!(objective, gradient, hessian, ensemble::SemEnsemble, params)
- isnothing(objective) || (objective = zero(objective))
- isnothing(gradient) || fill!(gradient, zero(eltype(gradient)))
- isnothing(hessian) || fill!(hessian, zero(eltype(hessian)))
- sem_grad = isnothing(gradient) ? nothing : similar(gradient)
- sem_hess = isnothing(hessian) ? nothing : similar(hessian)
- for (sem, weight) in zip(ensemble.sems, ensemble.weights)
- sem_obj = evaluate!(objective, sem_grad, sem_hess, sem, params)
- isnothing(objective) || (objective += weight * sem_obj)
- isnothing(gradient) || (gradient .+= weight * sem_grad)
- isnothing(hessian) || (hessian .+= weight * sem_hess)
- end
- return objective
+# evaluate!() wrapper that does some housekeeping, if necessary
+_evaluate!(args...) = evaluate!(args...)
+
+# update implied state, its gradient and hessian
+function _evaluate!(objective, gradient, hessian, loss::SemLoss, params)
+ # note that any other Sem loss terms that are dependent on implied
+ # should be enumerated after the SemLoss term
+ # otherwise they would be using outdated implied state
+ update!(EvaluationTargets(objective, gradient, hessian), implied(loss), params)
+ return evaluate!(objective, gradient, hessian, loss, params)
end
+objective(model::AbstractSemOrLoss, params) =
+ _evaluate!(objective_zero(model, params), nothing, nothing, model, params)
+
+# throw an error by default if gradient! and hessian! are not implemented
+
+#= gradient!(model::AbstractSemOrLoss, par, model) =
+ throw(ArgumentError("gradient for $(nameof(typeof(model))) is not available"))
+
+hessian!(model::AbstractSemOrLoss, par, model) =
+ throw(ArgumentError("hessian for $(nameof(typeof(model))) is not available")) =#
+
############################################################################################
# Documentation
############################################################################################
"""
objective!(model::AbstractSem, params)
-Returns the objective value at `params`.
-The model object can be modified.
+Calculates the objective value at `params`.
-# Implementation
-To implement a new `SemImplied` or `SemLossFunction` subtype, you need to add a method for
- objective!(newtype::MyNewType, params, model::AbstractSemSingle)
+The model object can be modified during calculation.
-To implement a new `AbstractSem` subtype, you need to add a method for
- objective!(model::MyNewType, params)
+See also [`evaluate!`](@ref).
"""
function objective! end
"""
gradient!(gradient, model::AbstractSem, params)
-Writes the gradient value at `params` to `gradient`.
+Calculates the model's gradient at `params` and writes it to `gradient`.
-# Implementation
-To implement a new `SemImplied` or `SemLossFunction` type, you can add a method for
- gradient!(newtype::MyNewType, params, model::AbstractSemSingle)
+The model object can be modified during calculation.
-To implement a new `AbstractSem` subtype, you can add a method for
- gradient!(gradient, model::MyNewType, params)
+See also [`evaluate!`](@ref).
"""
function gradient! end
"""
hessian!(hessian, model::AbstractSem, params)
-Writes the hessian value at `params` to `hessian`.
+Calculates the model's hessian at `params` and writes it to `hessian`.
-# Implementation
-To implement a new `SemImplied` or `SemLossFunction` type, you can add a method for
- hessian!(newtype::MyNewType, params, model::AbstractSemSingle)
+The model object can be modified during calculation.
-To implement a new `AbstractSem` subtype, you can add a method for
- hessian!(hessian, model::MyNewType, params)
+See also [`evaluate!`](@ref).
"""
function hessian! end
objective!(model::AbstractSem, params) =
- evaluate!(objective_zero(model, params), nothing, nothing, model, params)
+ _evaluate!(objective_zero(model, params), nothing, nothing, model, params)
gradient!(gradient, model::AbstractSem, params) =
- evaluate!(nothing, gradient, nothing, model, params)
+ _evaluate!(nothing, gradient, nothing, model, params)
hessian!(hessian, model::AbstractSem, params) =
- evaluate!(nothing, nothing, hessian, model, params)
+ _evaluate!(nothing, nothing, hessian, model, params)
objective_gradient!(gradient, model::AbstractSem, params) =
- evaluate!(objective_zero(model, params), gradient, nothing, model, params)
+ _evaluate!(objective_zero(model, params), gradient, nothing, model, params)
objective_hessian!(hessian, model::AbstractSem, params) =
- evaluate!(objective_zero(model, params), nothing, hessian, model, params)
+ _evaluate!(objective_zero(model, params), nothing, hessian, model, params)
gradient_hessian!(gradient, hessian, model::AbstractSem, params) =
- evaluate!(nothing, gradient, hessian, model, params)
+ _evaluate!(nothing, gradient, hessian, model, params)
objective_gradient_hessian!(gradient, hessian, model::AbstractSem, params) =
- evaluate!(objective_zero(model, params), gradient, hessian, model, params)
+ _evaluate!(objective_zero(model, params), gradient, hessian, model, params)
diff --git a/src/observed/EM.jl b/src/observed/EM.jl
index 46d0622be..b495c8861 100644
--- a/src/observed/EM.jl
+++ b/src/observed/EM.jl
@@ -26,157 +26,271 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. =#
-# outer function ---------------------------------------------------------------------------
"""
- em_mvn!(;
- observed::SemObservedMissing,
- start_em = start_em_observed,
- max_iter_em = 100,
- rtol_em = 1e-4,
- kwargs...)
-
-Estimates the covariance matrix and mean vector of the normal distribution via expectation maximization for `observed`.
-Overwrites the statistics stored in `observed`.
+ em_mvn(patterns::AbstractVector{SemObservedMissingPattern};
+ max_iter_em = 100,
+ rtol_em = 1e-4,
+ max_nsamples_em = nothing,
+ min_eigval = nothing,
+ start_em = start_em_observed,
+ start_kwargs...)
+
+Estimate the covariance and the mean for data with missing values using
+the expectation maximization (EM) algorithm.
+
+# Arguments
+- `patterns`: the observed data with missing values, grouped by missingness pattern
+ (each pattern is a `SemObservedMissingPattern`)
+- `max_iter_em`: the maximum number of EM iterations
+- `rtol_em`: the relative tolerance for convergence of the EM algorithm
+- `max_nsamples_em`: the maximum number of samples to use for each pattern in each EM iteration,
+ by default all samples are used, but for large datasets it may be desirable to use a random
+ subset of the data for each pattern in each EM iteration to speed up the algorithm
+- `min_eigval`: the minimum eigenvalue for the covariance matrix;
+ if not `nothing`, the covariance matrix is regularized in each EM iteration to ensure that
+ all eigenvalues are not smaller than `min_eigval`, which can help with convergence;
+- `start_em`: the function to generate starting values for the EM algorithm, by default
+ `start_em_observed` which uses the mean and covariance of the full cases if available
+- `start_kwargs...`: keyword arguments to pass to the `start_em` function
+
+Returns the tuple of the covariance matrix and the mean vector for the estimated
+multivariate normal (MVN) distribution.
+
+# References
+
+Based on the EM algorithm for MVN-distributed data with missing values
+adapted from the supplementary material to the book *Machine Learning: A Probabilistic Perspective*,
+copyright (2010) Kevin Murphy and Matt Dunham: see
+[*gaussMissingFitEm.m*](https://github.com/probml/pmtk3/blob/master/toolbox/BasicModels/gauss/sub/gaussMissingFitEm.m) and
+[*emAlgo.m*](https://github.com/probml/pmtk3/blob/master/toolbox/Algorithms/optimization/emAlgo.m) scripts.
"""
-function em_mvn!(
- observed::SemObservedMissing;
+function em_mvn(
+ patterns::AbstractVector{<:SemObservedMissingPattern};
+ max_iter_em::Integer = 100,
+ rtol_em::Number = 1e-4,
+ max_nsamples_em::Union{Integer, Nothing} = nothing,
+ min_eigval::Union{Number, Nothing} = nothing,
+ verbose::Bool = false,
start_em = start_em_observed,
- max_iter_em = 100,
- rtol_em = 1e-4,
- kwargs...,
+ start_kwargs...,
)
- nvars = nobserved_vars(observed)
- nsamps = nsamples(observed)
-
- # preallocate stuff
- 𝔼x_pre = zeros(nvars)
- 𝔼xxᵀ_pre = zeros(nvars, nvars)
-
- ### precompute for full cases
- fullpat = observed.patterns[1]
- if nmissed_vars(fullpat) == 0
- for row in eachrow(fullpat.data)
- 𝔼x_pre += row
- 𝔼xxᵀ_pre += row * row'
+ nobs_vars = nobserved_vars(patterns[1])
+
+ verbose && @info "Estimating N(μ, Σ) for complete observations..."
+ 𝔼x_full = zeros(nobs_vars)
+ 𝔼xxᵀ_full = zeros(nobs_vars, nobs_vars)
+ nsamples_full = 0
+ for pat in patterns
+ if nmissed_vars(pat) == 0
+ 𝔼x_full .+= sum(pat.data, dims = 2)
+ mul!(𝔼xxᵀ_full, pat.data, pat.data', 1, 1)
+ nsamples_full += nsamples(pat)
end
end
+ if nsamples_full == 0
+ @warn "No full cases in data"
+ end
- # initialize
- em_model = start_em(observed; kwargs...)
- em_model_prev = EmMVNModel(zeros(nvars, nvars), zeros(nvars), false)
- iter = 1
- done = false
- 𝔼x = zeros(nvars)
- 𝔼xxᵀ = zeros(nvars, nvars)
-
- while !done
- em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xxᵀ_pre)
- em_mvn_Mstep!(em_model, nsamps, 𝔼x, 𝔼xxᵀ)
-
- if iter > max_iter_em
- done = true
- @warn "EM Algorithm for MVN missing data did not converge. Likelihood for FIML is not interpretable.
- Maybe try passing different starting values via 'start_em = ...' "
- elseif iter > 1
- done =
- isapprox(em_model_prev.μ, em_model.μ; rtol = rtol_em) &
- isapprox(em_model_prev.Σ, em_model.Σ; rtol = rtol_em)
+ verbose && @info "Estimating initial μ and Σ..."
+ Σ₀, μ = start_em(patterns; start_kwargs...)
+ Σ = convert(Matrix, Σ₀)
+ @assert all(isfinite, Σ) all(isfinite, μ)
+ Σ_prev, μ_prev = copy(Σ), copy(μ)
+
+ iter = 0
+ converged = false
+ Δμ_rel = NaN
+ ΔΣ_rel = NaN
+ progress = Progress(
+ max_iter_em,
+ dt = 1.0,
+ showspeed = true,
+ desc = "EM inference of MVN(μ, Σ)",
+ )
+ while !converged && (iter < max_iter_em)
+ em_step!(
+ Σ,
+ μ,
+ Σ_prev,
+ μ_prev,
+ patterns,
+ 𝔼xxᵀ_full,
+ 𝔼x_full,
+ nsamples_full;
+ max_nsamples_em,
+ min_eigval,
+ )
+
+ if iter > 0
+ Δμ = norm(μ - μ_prev)
+ ΔΣ = norm(Σ - Σ_prev)
+ Δμ_rel = Δμ / max(norm(μ_prev), norm(μ))
+ ΔΣ_rel = ΔΣ / max(norm(Σ_prev), norm(Σ))
+ #@info "Iteration #$iter: ΔΣ=$(ΔΣ) ΔΣ/Σ=$(ΔΣ_rel) Δμ=$(Δμ) Δμ/μ=$(Δμ_rel)"
+ # converged = isapprox(ll, ll_prev; rtol = rtol)
+ converged = ΔΣ_rel <= rtol_em && Δμ_rel <= rtol_em
end
+ if !converged
+ Σ, Σ_prev = Σ_prev, Σ
+ μ, μ_prev = μ_prev, μ
+ end
+ iter += 1
+ next!(progress, step = 1, showvalues = [("ΔΣ/Σ", ΔΣ_rel), ("Δμ/μ", Δμ_rel)])
+ end
+ finish!(progress)
- iter = iter + 1
- em_model_prev.μ, em_model_prev.Σ = em_model.μ, em_model.Σ
+ if !converged
+ @warn "EM inference for MVN missing data did not converge in $iter iterations.\n" *
+ "Final tolerances: ΔΣ/Σ=$(ΔΣ_rel), Δμ/μ=$(Δμ_rel).\n" *
+ "Likelihood for FIML is not interpretable.\n" *
+ "Maybe try passing different starting values via 'start_em = ...' "
+ else
+ verbose &&
+ @info "EM for MVN missing data converged in $iter iterations: ΔΣ/Σ=$(ΔΣ_rel), Δμ/μ=$(Δμ_rel)."
end
- # update EM Mode in observed
- observed.em_model.Σ .= em_model.Σ
- observed.em_model.μ .= em_model.μ
- observed.em_model.fitted = true
+ StatsBase._symmetrize!(Σ)
- return nothing
+ return Σ, μ
end
-# E and M step -----------------------------------------------------------------------------
-
-function em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xxᵀ_pre)
- 𝔼x .= 0.0
- 𝔼xxᵀ .= 0.0
-
- 𝔼xᵢ = copy(𝔼x)
- 𝔼xxᵀᵢ = copy(𝔼xxᵀ)
-
- μ = em_model.μ
- Σ = em_model.Σ
+# E and M steps combined
+function em_step!(
+ Σ::AbstractMatrix,
+ μ::AbstractVector,
+ Σ₀::AbstractMatrix,
+ μ₀::AbstractVector,
+ patterns::AbstractVector{<:SemObservedMissingPattern},
+ 𝔼xxᵀ_full::AbstractMatrix,
+ 𝔼x_full::AbstractVector,
+ nsamples_full::Integer;
+ max_nsamples_em::Union{Integer, Nothing} = nothing,
+ min_eigval::Union{Number, Nothing} = nothing,
+)
+ # E step: update 𝔼x and 𝔼xxᵀ
+ copy!(μ, 𝔼x_full)
+ copy!(Σ, 𝔼xxᵀ_full)
+ nsamples_used = nsamples_full
+ mul!(Σ, μ₀, μ₀', -nsamples_used, 1)
+ axpy!(-nsamples_used, μ₀, μ)
# Compute the expected sufficient statistics
- for pat in observed.patterns
- (nmissed_vars(pat) == 0) && continue # skip full cases
+ for pat in patterns
+ (nmissed_vars(pat) == 0) && continue # full cases already accounted for
# observed and unobserved vars
u = pat.miss_mask
o = pat.measured_mask
- # precompute for pattern
- Σoo = Σ[o, o]
- Σuo = Σ[u, o]
- μu = μ[u]
- μo = μ[o]
+ # compute cholesky to speed-up ldiv!()
+ Σ₀oo_chol = cholesky(Symmetric(Σ₀[o, o]))
+ Σ₀uo = Σ₀[u, o]
+ μ₀u = μ₀[u]
+ μ₀o = μ₀[o]
+
+ # get pattern observations
+ nsamples_pat =
+ !isnothing(max_nsamples_em) ? min(max_nsamples_em, nsamples(pat)) :
+ nsamples(pat)
+ zo =
+ nsamples_pat < nsamples(pat) ?
+ pat.data[:, sort!(sample(1:nsamples(pat), nsamples_pat, replace = false))] :
+ copy(pat.data)
+ zo .-= μ₀o # subtract current mean from observations
+
+ 𝔼zo = sum(zo, dims = 2)
+ 𝔼zu = fill!(similar(μ₀u), 0)
+
+ 𝔼zzᵀuo = fill!(similar(Σ₀uo), 0)
+ 𝔼zzᵀuu = nsamples_pat * Σ₀[u, u]
+ mul!(𝔼zzᵀuu, Σ₀uo, Σ₀oo_chol \ Σ₀uo', -nsamples_pat, 1)
+
+ # loop through observations
+ yᵢo = similar(μ₀o)
+ 𝔼zᵢu = similar(μ₀u)
+ @inbounds for zᵢo in eachcol(zo)
+ ldiv!(yᵢo, Σ₀oo_chol, zᵢo)
+ mul!(𝔼zᵢu, Σ₀uo, yᵢo)
+ mul!(𝔼zzᵀuu, 𝔼zᵢu, 𝔼zᵢu', 1, 1)
+ mul!(𝔼zzᵀuo, 𝔼zᵢu, zᵢo', 1, 1)
+ 𝔼zu .+= 𝔼zᵢu
+ end
+ # correct 𝔼zzᵀ by adding back μ₀×𝔼z' + 𝔼z'×μ₀
+ mul!(𝔼zzᵀuo, μ₀u, 𝔼zo', 1, 1)
+ mul!(𝔼zzᵀuo, 𝔼zu, μ₀o', 1, 1)
+
+ mul!(𝔼zzᵀuu, μ₀u, 𝔼zu', 1, 1)
+ mul!(𝔼zzᵀuu, 𝔼zu, μ₀u', 1, 1)
- V = Σ[u, u] - Σuo * (Σoo \ Σ[o, u])
+ 𝔼zzᵀoo = zo * zo'
+ mul!(𝔼zzᵀoo, μ₀o, 𝔼zo', 1, 1)
+ mul!(𝔼zzᵀoo, 𝔼zo, μ₀o', 1, 1)
- # loop trough data
- for rowdata in eachrow(pat.data)
- m = μu + Σuo * (Σoo \ (rowdata - μo))
+ # update Σ and μ
+ Σ[o, o] .+= 𝔼zzᵀoo
+ Σ[u, o] .+= 𝔼zzᵀuo
+ Σ[o, u] .+= 𝔼zzᵀuo'
+ Σ[u, u] .+= 𝔼zzᵀuu
- 𝔼xᵢ[u] = m
- 𝔼xᵢ[o] = rowdata
- 𝔼xxᵀᵢ[u, u] = 𝔼xᵢ[u] * 𝔼xᵢ[u]' + V
- 𝔼xxᵀᵢ[o, o] = 𝔼xᵢ[o] * 𝔼xᵢ[o]'
- 𝔼xxᵀᵢ[o, u] = 𝔼xᵢ[o] * 𝔼xᵢ[u]'
- 𝔼xxᵀᵢ[u, o] = 𝔼xᵢ[u] * 𝔼xᵢ[o]'
+ μ[o] .+= 𝔼zo
+ μ[u] .+= 𝔼zu
- 𝔼x .+= 𝔼xᵢ
- 𝔼xxᵀ .+= 𝔼xxᵀᵢ
- end
+ nsamples_used += nsamples_pat
end
- 𝔼x .+= 𝔼x_pre
- 𝔼xxᵀ .+= 𝔼xxᵀ_pre
-end
+ # M step: update Σ and μ
+ lmul!(1 / nsamples_used, Σ)
+ lmul!(1 / nsamples_used, μ)
+ # at this point μ = μ - μ₀
+ # and Σ = Σ + (μ - μ₀)×(μ - μ₀)' - μ₀×μ₀'
+ mul!(Σ, μ, μ₀', -1, 1)
+ mul!(Σ, μ₀, μ', -1, 1)
+ mul!(Σ, μ, μ', -1, 1)
+ μ .+= μ₀
+
+ StatsBase._symmetrize!(Σ) # correct numerical errors
+ if !isnothing(min_eigval)
+ # try to fix non-positive-definite Σ
+ reg_Σ = trunc_eigvals(Σ, min_eigval)
+ # if no truncation was done, reg_Σ will be the same object
+ (reg_Σ === Σ) || copyto!(Σ, reg_Σ)
+ end
-function em_mvn_Mstep!(em_model, nsamples, 𝔼x, 𝔼xxᵀ)
- em_model.μ = 𝔼x / nsamples
- Σ = Symmetric(𝔼xxᵀ / nsamples - em_model.μ * em_model.μ')
- em_model.Σ = Σ
- return nothing
+ return Σ, μ
end
# generate starting values -----------------------------------------------------------------
# use μ and Σ of full cases
-function start_em_observed(observed::SemObservedMissing; kwargs...)
- fullpat = observed.patterns[1]
- if (nmissed_vars(fullpat) == 0) && (nobserved_vars(fullpat) > 1)
+function start_em_observed(patterns::AbstractVector{<:SemObservedMissingPattern}; kwargs...)
+ fullpat = patterns[1]
+ if (nmissed_vars(fullpat) == 0) && (nsamples(fullpat) > 1)
μ = copy(fullpat.measured_mean)
- Σ = copy(Symmetric(fullpat.measured_cov))
+ Σ = copy(parent(fullpat.measured_cov))
if !isposdef(Σ)
- Σ = Matrix(Diagonal(Σ))
+ Σ = Diagonal(Σ)
end
- return EmMVNModel(Σ, μ, false)
+ return Σ, μ
else
- return start_em_simple(observed, kwargs...)
+ return start_em_simple(patterns, kwargs...)
end
end
# use μ = O and Σ = I
-function start_em_simple(observed::SemObservedMissing; kwargs...)
- nvars = nobserved_vars(observed)
- μ = zeros(nvars)
- Σ = rand(nvars, nvars)
+function start_em_simple(patterns::AbstractVector{<:SemObservedMissingPattern}; kwargs...)
+ nobs_vars = nobserved_vars(first(patterns))
+ μ = zeros(nobs_vars)
+ Σ = rand(nobs_vars, nobs_vars)
Σ = Σ * Σ'
- return EmMVNModel(Σ, μ, false)
+ return Σ, μ
end
# set to passed values
-function start_em_set(observed::SemObservedMissing; model_em, kwargs...)
- return em_model
+function start_em_set(
+ patterns::AbstractVector{<:SemObservedMissingPattern};
+ obs_cov::AbstractMatrix,
+ obs_mean::AbstractVector,
+ kwargs...,
+)
+ return copy(obs_cov), copy(obs_mean)
end
diff --git a/src/observed/data.jl b/src/observed/data.jl
index 7ba38edf5..30d433e03 100644
--- a/src/observed/data.jl
+++ b/src/observed/data.jl
@@ -42,6 +42,22 @@ function SemObservedData(;
prepare_data(data, observed_vars, specification; observed_var_prefix)
obs_mean, obs_cov = mean_and_cov(data, 1)
+ if any(ismissing, data)
+ """
+ Your dataset contains missing values.
+ Remove missing values or use full information maximum likelihood (FIML) estimation.
+ A FIML model can be constructed with
+ Sem(
+ ...,
+ observed = SemObservedMissing,
+ loss = SemFIML,
+ meanstructure = true
+ )
+ """ |>
+ ArgumentError |>
+ throw
+ end
+
return SemObservedData(data, obs_vars, obs_cov, vec(obs_mean), size(data, 1))
end
diff --git a/src/observed/missing.jl b/src/observed/missing.jl
index ac8b7ea5f..52115cf33 100644
--- a/src/observed/missing.jl
+++ b/src/observed/missing.jl
@@ -2,19 +2,10 @@
### Types
############################################################################################
-# Type to store Expectation Maximization result --------------------------------------------
-mutable struct EmMVNModel{A, b, B}
- Σ::A
- μ::b
- fitted::B
-end
-
-# FIXME type unstable
-obs_mean(em::EmMVNModel) = ifelse(em.fitted, em.μ, nothing)
-obs_cov(em::EmMVNModel) = ifelse(em.fitted, em.Σ, nothing)
-
"""
-For observed data with missing values.
+ SemObservedMissing{T <: Real, S <: Real} <: SemObserved
+
+[`SemObserved`](@ref) implementation for data with missing values.
# Constructor
@@ -22,35 +13,31 @@ For observed data with missing values.
data,
observed_vars = nothing,
specification = nothing,
- kwargs...)
+ lazy_cov = true,
+ em_kwargs...)
# Arguments
- `data`: observed data
- `observed_vars::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame)
- `specification`: optional SEM model specification ([`SemSpecification`](@ref))
-
-# Extended help
-## Interfaces
-- `nsamples(::SemObservedMissing)` -> number of samples (data points)
-- `nobserved_vars(::SemObservedMissing)` -> number of observed variables
-
-- `samples(::SemObservedMissing)` -> data matrix (contains both measured and missing values)
-
-## Expectation maximization
-`em_mvn!(::SemObservedMissing)` can be called to fit a covariance matrix and mean vector to the data
-using an expectation maximization (EM) algorithm under the assumption of multivariate normality.
-After, the following methods are available:
-- `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via EM
-- `obs_cov(::SemObservedData)` -> EM covariance matrix
-- `obs_mean(::SemObservedData)` -> EM mean vector
+- `lazy_cov::Bool`: whether to defer covariance and mean calculation until requested (default: `true`)
+- `em_kwargs...`: keyword arguments to pass to the EM algorithm (see [`em_mvn`](@ref))
+
+`SemObservedMissing` could be used in combination with [`SemFIML`](@ref) loss for the
+*full information maximum likelihood* (FIML) to fit SEM with missing data.
+It could also be used with other loss functions, e.g. [`SemML`](@ref);
+in that case the approximated observed covariance and mean would be calculated using
+the *EM* algorithm (see [`em_mvn`](@ref)).
"""
-struct SemObservedMissing{T <: Real, S <: Real, E <: EmMVNModel} <: SemObserved
+struct SemObservedMissing{T <: Real, S <: Real} <: SemObserved
data::Matrix{Union{T, Missing}}
observed_vars::Vector{Symbol}
nsamples::Int
patterns::Vector{SemObservedMissingPattern{T, S}}
- em_model::E
+ em_kwargs::AbstractDict
+ obs_cov::Ref{Matrix{S}}
+ obs_mean::Ref{Vector{S}}
end
############################################################################################
@@ -62,7 +49,8 @@ function SemObservedMissing(;
observed_vars::Union{AbstractVector, Nothing} = nothing,
specification::Union{SemSpecification, Nothing} = nothing,
observed_var_prefix::Union{Symbol, AbstractString} = :obs,
- kwargs...,
+ lazy_cov::Bool = true,
+ em_kwargs...,
)
data, obs_vars, _ =
prepare_data(data, observed_vars, specification; observed_var_prefix)
@@ -83,22 +71,52 @@ function SemObservedMissing(;
]
sort!(patterns, by = nmissed_vars)
- # allocate EM model (but don't fit)
- em_model = EmMVNModel(zeros(nobs_vars, nobs_vars), zeros(nobs_vars), false)
+ S = isempty(patterns) ? Float64 : eltype(patterns[1].measured_mean)
+ if lazy_cov
+ # defer EM covariance and mean calculation until requested
+ em_cov_ref = Ref{Matrix{S}}()
+ em_mean_ref = Ref{Vector{S}}()
+ else
+ em_cov, em_mean = em_mvn(patterns; em_kwargs...)
+ em_cov_ref, em_mean_ref = Ref(em_cov), Ref(em_mean)
+ end
return SemObservedMissing(
convert(Matrix{Union{nonmissingtype(eltype(data)), Missing}}, data),
obs_vars,
nsamples,
patterns,
- em_model,
+ em_kwargs, # remember EM kwargs for calculate_cov!
+ em_cov_ref,
+ em_mean_ref,
)
end
-############################################################################################
-### Additional methods
-############################################################################################
+"""
+ calculate_cov!(observed::SemObservedMissing; em_kwargs...)
+
+Force calculation of the observed mean and covariance using the EM algorithm.
-em_model(observed::SemObservedMissing) = observed.em_model
-obs_mean(observed::SemObservedMissing) = obs_mean(em_model(observed))
-obs_cov(observed::SemObservedMissing) = obs_cov(em_model(observed))
+# Arguments
+- `observed`: the observed data with missing values (see [`SemObservedMissing`](@ref))
+- `em_kwargs...`: keyword arguments for the EM algorithm (see [`em_mvn`](@ref)),
+ the values provided here override the EM arguments passed to the
+ [`SemObservedMissing`](@ref) constructor
+"""
+function calculate_cov!(observed::SemObservedMissing; em_kwargs...)
+ em_kwargs = merge(observed.em_kwargs, em_kwargs)
+ em_cov, em_mean = em_mvn(observed.patterns; em_kwargs...)
+ observed.obs_cov[] = em_cov
+ observed.obs_mean[] = em_mean
+ return observed
+end
+
+function obs_cov(observed::SemObservedMissing{<:Any, S}) where {S}
+ isassigned(observed.obs_cov) || calculate_cov!(observed)
+ return observed.obs_cov[]::Matrix{S}
+end
+
+function obs_mean(observed::SemObservedMissing{<:Any, S}) where {S}
+ isassigned(observed.obs_mean) || calculate_cov!(observed)
+ return observed.obs_mean[]::Vector{S}
+end
diff --git a/src/observed/missing_pattern.jl b/src/observed/missing_pattern.jl
index 6ac6a360b..8d9a1eb93 100644
--- a/src/observed/missing_pattern.jl
+++ b/src/observed/missing_pattern.jl
@@ -5,10 +5,10 @@ struct SemObservedMissingPattern{T, S}
measured_mask::BitVector # measured vars mask
miss_mask::BitVector # missing vars mask
rows::Vector{Int} # rows in original data
- data::Matrix{T} # non-missing submatrix of data
+ data::Matrix{T} # non-missing submatrix of data (vars × observations)
measured_mean::Vector{S} # means of measured vars
- measured_cov::Matrix{S} # covariance of measured vars
+ measured_cov::Symmetric{S, Matrix{S}} # covariance of measured vars
end
function SemObservedMissingPattern(
@@ -32,9 +32,9 @@ function SemObservedMissingPattern(
measured_mask,
.!measured_mask,
rows,
- pat_data,
+ permutedims(pat_data),
dropdims(pat_mean, dims = 1),
- pat_cov,
+ Symmetric(pat_cov),
)
end
diff --git a/src/optimizer/Empty.jl b/src/optimizer/Empty.jl
index 1bf0c30ac..fd36acb53 100644
--- a/src/optimizer/Empty.jl
+++ b/src/optimizer/Empty.jl
@@ -1,27 +1,15 @@
############################################################################################
### Types
############################################################################################
-"""
-Empty placeholder for models that don't need
-an optimizer part.
-# Constructor
+"""
+ SemOptimizer(engine = :Empty)
- SemOptimizerEmpty()
+Constructs a dummy placeholder optimizer for models that don't need it.
"""
struct SemOptimizerEmpty <: SemOptimizer{:Empty} end
-############################################################################################
-### Constructor
-############################################################################################
-
-SemOptimizer{:Empty}() = SemOptimizerEmpty()
-
-############################################################################################
-### Recommended methods
-############################################################################################
-
-update_observed(optimizer::SemOptimizerEmpty, observed::SemObserved; kwargs...) = optimizer
+sem_optimizer_subtype(::Val{:Empty}) = SemOptimizerEmpty
############################################################################################
### Pretty Printing
diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl
index 3b1e98842..6774e5494 100644
--- a/src/optimizer/abstract.jl
+++ b/src/optimizer/abstract.jl
@@ -1,6 +1,96 @@
+const optimizer_engine_dependencies =
+ Dict(:NLopt => ["NLopt"], :Proximal => ["ProximalAlgorithms"])
+
+# throw unsupported engine error
+function throw_engine_error(E)
+ if typeof(E) !== Symbol
+ throw(ArgumentError("engine argument must be a Symbol."))
+ elseif haskey(optimizer_engine_dependencies, E)
+ error(
+ "optimizer \":$E\" requires \"using $(join(optimizer_engine_dependencies[E], ", "))\".",
+ )
+ else
+ error("optimizer engine \":$E\" is not supported.")
+ end
+end
+
+# return the type implementing SemOptimizer{engine}
+# the method should be overridden in the extension
+sem_optimizer_subtype(engine::Symbol) = sem_optimizer_subtype(Val(engine))
+
+# fallback method for unsupported engines
+sem_optimizer_subtype(::Val{E}) where {E} = throw_engine_error(E)
+
+"""
+ SemOptimizer(args...; engine::Symbol = :Optim, kwargs...)
+
+Constructs a `SemOptimizer` object that can be passed to [`fit`](@ref) for specifying aspects
+of the numerical optimization involved in fitting a SEM.
+
+The keyword `engine` controlls which Julia package is used, with `:Optim` being the default.
+- [`optimizer_engines()`](@ref optimizer_engines) prints a list of currently available engines.
+- [`optimizer_engine_doc(EngineName)`](@ref optimizer_engine_doc) prints information on the usage of a specific engine.
+
+More engines become available if specific packages are loaded, for example
+[*NLopt.jl*](https://github.com/JuliaOpt/NLopt.jl) (also see [Constrained optimization](@ref)
+in the online documentation) or
+[*ProximalAlgorithms.jl*](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl)
+(also see [Regularization](@ref) in the online documentation).
+
+The arguments `args...` and `kwargs...` are engine-specific and control further
+aspects of the optimization process, such as the algorithm, convergence criteria or constraints.
+Information on those can be accessed with [`optimizer_engine_doc`](@ref).
+
+[Custom optimizer types](@ref) shows how to connect the *SEM.jl* package to a completely new optimization engine.
+"""
+SemOptimizer
+
+# default constructor that dispatches to the engine-specific type
+SemOptimizer(::Val{E}, args...; kwargs...) where {E} =
+ sem_optimizer_subtype(E)(args...; kwargs...)
+
+SemOptimizer{E}(args...; kwargs...) where {E} = SemOptimizer(Val(E), args...; kwargs...)
+
+SemOptimizer(args...; engine::Symbol = :Optim, kwargs...) =
+ SemOptimizer(Val(engine), args...; kwargs...)
+
+"""
+ optimizer_engine(::Type{<:SemOptimizer})
+ optimizer_engine(::SemOptimizer)
+
+Returns the engine name (`Symbol`) for a [`SemOptimizer`](@ref) instance or subtype.
+"""
+optimizer_engine(::Type{<:SemOptimizer{E}}) where {E} = E
+optimizer_engine(optim::SemOptimizer) = optimizer_engine(typeof(optim))
+
+"""
+ optimizer_engines()
+
+Returns a vector of optimizer engines supported by the `engine` keyword argument of
+the [`SemOptimizer`](@ref) constructor.
+
+The list of engines depends on the Julia packages loaded (with the `using` directive)
+into the current session.
+"""
+optimizer_engines() =
+ Symbol[optimizer_engine(opt_type) for opt_type in subtypes(SemOptimizer)]
+
+"""
+ optimizer_engine_doc(engine::Symbol)
+
+Shows documentation for the optimizer engine.
+
+For a list of available engines, call [`optimizer_engines`](@ref).
+"""
+optimizer_engine_doc(engine) = Base.Docs.doc(sem_optimizer_subtype(engine))
+
+optimizer(result::SemOptimizerResult) = result.optimizer
+
+optimizer_engine(result::SemOptimizerResult) = optimizer_engine(result.optimizer)
+
"""
fit([optim::SemOptimizer], model::AbstractSem;
- [engine::Symbol], start_val = start_val, kwargs...)
+ [engine::Symbol], start_val = start_val, kwargs...)
Return the fitted `model`.
@@ -41,23 +131,22 @@ function fit(optim::SemOptimizer, model::AbstractSem; start_val = nothing, kwarg
end
fit(model::AbstractSem; engine::Symbol = :Optim, start_val = nothing, kwargs...) =
-fit(SemOptimizer(; engine, kwargs...), model; start_val, kwargs...)
+ fit(SemOptimizer(; engine, kwargs...), model; start_val, kwargs...)
# fallback method
fit(optim::SemOptimizer, model::AbstractSem, start_params; kwargs...) =
error("Optimizer $(optim) support not implemented.")
-# FABIN3 is the default method for single models
-prepare_start_params(start_val::Nothing, model::AbstractSemSingle; kwargs...) =
- start_fabin3(model; kwargs...)
-
-# simple algorithm is the default method for ensembles
-prepare_start_params(start_val::Nothing, model::AbstractSem; kwargs...) =
- start_simple(model; kwargs...)
+# defaults when no starting parameters are specified
+function prepare_start_params(start_val::Nothing, model::AbstractSem; kwargs...)
+ sems = sem_terms(model)
+ # FABIN3 for single models, simple algorithm for ensembles
+ return length(sems) == 1 ? start_fabin3(loss(sems[1]); kwargs...) :
+ start_simple(model; kwargs...)
+end
# first argument is a function
-prepare_start_params(start_val, model::AbstractSem; kwargs...) =
- start_val(model; kwargs...)
+prepare_start_params(start_val, model::AbstractSem; kwargs...) = start_val(model; kwargs...)
function prepare_start_params(start_val::AbstractVector, model::AbstractSem; kwargs...)
(length(start_val) == nparams(model)) || throw(
@@ -65,11 +154,24 @@ function prepare_start_params(start_val::AbstractVector, model::AbstractSem; kwa
"The length of `start_val` vector ($(length(start_val))) does not match the number of model parameters ($(nparams(model))).",
),
)
+ (eltype(start_val) <: Number) || throw(
+ TypeError(
+ :prepare_start_params,
+ "start_val elements must be numeric",
+ Number,
+ eltype(start_val),
+ ),
+ )
return start_val
end
function prepare_start_params(start_val::AbstractDict, model::AbstractSem; kwargs...)
- return [start_val[param] for param in params(model)] # convert to a vector
+ # convert to a vector
+ return prepare_start_params(
+ [start_val[param] for param in params(model)],
+ model;
+ kwargs...,
+ )
end
# get from the ParameterTable (potentially from a different model with match param names)
diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl
index bd57942d8..e81017ed3 100644
--- a/src/optimizer/optim.jl
+++ b/src/optimizer/optim.jl
@@ -3,34 +3,36 @@
############################################################################################
### Types and Constructor
############################################################################################
-"""
- SemOptimizerOptim{A, B} <: SemOptimizer{:Optim}
-
-Connects to `Optim.jl` as the optimization backend.
-# Constructor
+# SemOptimizer for the Optim.jl
+mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim}
+ algorithm::A
+ options::B
+end
- SemOptimizerOptim(;
+"""
+ SemOptimizer(;
+ engine = :Optim,
algorithm = LBFGS(),
options = Optim.Options(;f_reltol = 1e-10, x_abstol = 1.5e-8),
kwargs...)
+Connects to *Optim.jl* as the optimization engine.
+
+For more information on the available algorithms and options,
+see the [*Optim.jl* docs](https://julianlsolvers.github.io/Optim.jl/stable/).
+
# Arguments
-- `algorithm`: optimization algorithm from `Optim.jl`
+- `algorithm`: optimization algorithm from *Optim.jl*
- `options::Optim.Options`: options for the optimization algorithm
-# Usage
-All algorithms and options from the Optim.jl library are available, for more information see
-the Optim.jl online documentation.
-
# Examples
```julia
-my_optimizer = SemOptimizerOptim()
-
# hessian based optimization with backtracking linesearch and modified initial step size
using Optim, LineSearches
-my_newton_optimizer = SemOptimizerOptim(
+my_newton_optimizer = SemOptimizer(
+ engine = :Optim,
algorithm = Newton(
;linesearch = BackTracking(order=3),
alphaguess = InitialHagerZhang()
@@ -38,10 +40,7 @@ my_newton_optimizer = SemOptimizerOptim(
)
```
-# Extended help
-
-## Constrained optimization
-
+# Constrained optimization
When using the `Fminbox` or `SAMIN` constrained optimization algorithms,
the vector or dictionary of lower and upper bounds for each model parameter can be specified
via `lower_bounds` and `upper_bounds` keyword arguments.
@@ -49,58 +48,40 @@ Alternatively, the `lower_bound` and `upper_bound` keyword arguments can be used
the default bound for all non-variance model parameters,
and the `variance_lower_bound` and `variance_upper_bound` keyword --
for the variance parameters (the diagonal of the *S* matrix).
-
-## Interfaces
-- `algorithm(::SemOptimizerOptim)`
-- `options(::SemOptimizerOptim)`
-
-## Implementation
-
-Subtype of `SemOptimizer`.
"""
-mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim}
- algorithm::A
- options::B
-end
-
-SemOptimizer{:Optim}(args...; kwargs...) = SemOptimizerOptim(args...; kwargs...)
-
SemOptimizerOptim(;
algorithm = LBFGS(),
- options = Optim.Options(;f_reltol = 1e-10, x_abstol = 1.5e-8),
+ options = Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8),
kwargs...,
) = SemOptimizerOptim(algorithm, options)
-############################################################################################
-### Recommended methods
-############################################################################################
-
-update_observed(optimizer::SemOptimizerOptim, observed::SemObserved; kwargs...) = optimizer
+sem_optimizer_subtype(::Val{:Optim}) = SemOptimizerOptim
############################################################################################
### additional methods
############################################################################################
-algorithm(optimizer::SemOptimizerOptim) = optimizer.algorithm
options(optimizer::SemOptimizerOptim) = optimizer.options
-function SemFit(
- optimization_result::Optim.MultivariateOptimizationResults,
- model::AbstractSem,
- start_val,
-)
- return SemFit(
- optimization_result.minimum,
- optimization_result.minimizer,
- start_val,
- model,
- optimization_result,
- )
+# wrapper for the Optim.jl result
+struct SemOptimResult{O <: SemOptimizerOptim} <: SemOptimizerResult{O}
+ optimizer::O
+ result::Optim.MultivariateOptimizationResults
end
-optimizer(res::Optim.MultivariateOptimizationResults) = Optim.summary(res)
-n_iterations(res::Optim.MultivariateOptimizationResults) = Optim.iterations(res)
-convergence(res::Optim.MultivariateOptimizationResults) = Optim.converged(res)
+algorithm_name(res::SemOptimResult) = Optim.summary(res.result)
+n_iterations(res::SemOptimResult) = Optim.iterations(res.result)
+function convergence(res::SemOptimResult)
+ flags = res.result.stopped_by
+ active_flags = Symbol[]
+ for key in keys(flags)
+ if flags[key]
+ push!(active_flags, key)
+ end
+ end
+ return active_flags
+end
+converged(res::SemOptimResult) = Optim.converged(res.result)
function fit(
optim::SemOptimizerOptim,
@@ -132,7 +113,7 @@ function fit(
)
start_params = clamp.(start_params, lbounds, ubounds)
result = Optim.optimize(
- Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)),
+ NLSolversBase.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)),
lbounds,
ubounds,
start_params,
@@ -141,11 +122,23 @@ function fit(
)
else
result = Optim.optimize(
- Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)),
+ NLSolversBase.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)),
start_params,
optim.algorithm,
optim.options,
)
end
- return SemFit(result, model, start_params)
+ return SemFit(
+ result.minimum,
+ result.minimizer,
+ start_params,
+ model,
+ SemOptimResult(optim, result),
+ )
end
+
+############################################################################################
+# pretty printing
+############################################################################################
+
+Base.show(io::IO, result::SemOptimResult) = Base.show(io, result.result)
diff --git a/src/package_extensions/SEMNLOptExt.jl b/src/package_extensions/SEMNLOptExt.jl
deleted file mode 100644
index 64c4cff04..000000000
--- a/src/package_extensions/SEMNLOptExt.jl
+++ /dev/null
@@ -1,65 +0,0 @@
-"""
-Connects to `NLopt.jl` as the optimization backend.
-Only usable if `NLopt.jl` is loaded in the current Julia session!
-
-# Constructor
-
- SemOptimizerNLopt(;
- algorithm = :LD_LBFGS,
- options = Dict{Symbol, Any}(),
- local_algorithm = nothing,
- local_options = Dict{Symbol, Any}(),
- equality_constraints = Vector{NLoptConstraint}(),
- inequality_constraints = Vector{NLoptConstraint}(),
- kwargs...)
-
-# Arguments
-- `algorithm`: optimization algorithm.
-- `options::Dict{Symbol, Any}`: options for the optimization algorithm
-- `local_algorithm`: local optimization algorithm
-- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm
-- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints
-- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints
-
-# Example
-```julia
-my_optimizer = SemOptimizerNLopt()
-
-# constrained optimization with augmented lagrangian
-my_constrained_optimizer = SemOptimizerNLopt(;
- algorithm = :AUGLAG,
- local_algorithm = :LD_LBFGS,
- local_options = Dict(:ftol_rel => 1e-6),
- inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0),
-)
-```
-
-# Usage
-All algorithms and options from the NLopt library are available, for more information see
-the NLopt.jl package and the NLopt online documentation.
-For information on how to use inequality and equality constraints,
-see [Constrained optimization](@ref) in our online documentation.
-
-# Extended help
-
-## Interfaces
-- `algorithm(::SemOptimizerNLopt)`
-- `local_algorithm(::SemOptimizerNLopt)`
-- `options(::SemOptimizerNLopt)`
-- `local_options(::SemOptimizerNLopt)`
-- `equality_constraints(::SemOptimizerNLopt)`
-- `inequality_constraints(::SemOptimizerNLopt)`
-"""
-struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt}
- algorithm::A
- local_algorithm::A2
- options::B
- local_options::B2
- equality_constraints::C
- inequality_constraints::C
-end
-
-Base.@kwdef struct NLoptConstraint
- f::Any
- tol = 0.0
-end
diff --git a/src/package_extensions/SEMProximalOptExt.jl b/src/package_extensions/SEMProximalOptExt.jl
deleted file mode 100644
index ad4c2da2a..000000000
--- a/src/package_extensions/SEMProximalOptExt.jl
+++ /dev/null
@@ -1,27 +0,0 @@
-"""
-Connects to `ProximalAlgorithms.jl` as the optimization backend.
-Can be used for regularized SEM, for a tutorial see the online docs on [Regularization](@ref).
-
-# Constructor
-
- SemOptimizerProximal(;
- algorithm = ProximalAlgorithms.PANOC(),
- operator_g,
- operator_h = nothing,
- kwargs...,
-
-# Arguments
-- `algorithm`: optimization algorithm.
-- `operator_g`: proximal operator (e.g., regularization penalty)
-- `operator_h`: optional second proximal operator
-
-# Usage
-All algorithms and operators from `ProximalAlgorithms.jl` are available,
-for more information see the online docs on [Regularization](@ref) and
-the documentation of `ProximalAlgorithms.jl` / `ProximalOperators.jl`.
-"""
-mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal}
- algorithm::A
- operator_g::B
- operator_h::C
-end
diff --git a/src/types.jl b/src/types.jl
index 44d472ebf..d0218b14f 100644
--- a/src/types.jl
+++ b/src/types.jl
@@ -1,14 +1,6 @@
############################################################################################
# Define the basic type system
############################################################################################
-"Most abstract supertype for all SEMs"
-abstract type AbstractSem end
-
-"Supertype for all single SEMs, e.g. SEMs that have at least the fields `observed`, `implied`, `loss`"
-abstract type AbstractSemSingle{O, I, L} <: AbstractSem end
-
-"Supertype for all collections of multiple SEMs"
-abstract type AbstractSemCollection <: AbstractSem end
"Meanstructure trait for `SemImplied` subtypes"
abstract type MeanStruct end
@@ -24,7 +16,7 @@ MeanStruct(::Type{T}) where {T} =
MeanStruct(semobj) = MeanStruct(typeof(semobj))
-"Hessian Evaluation trait for `SemImplied` and `SemLossFunction` subtypes"
+"Hessian Evaluation trait for `SemImplied` and `AbstractLoss` subtypes"
abstract type HessianEval end
struct ApproxHessian <: HessianEval end
struct ExactHessian <: HessianEval end
@@ -36,71 +28,20 @@ HessianEval(::Type{T}) where {T} =
HessianEval(semobj) = HessianEval(typeof(semobj))
-"Supertype for all loss functions of SEMs. If you want to implement a custom loss function, it should be a subtype of `SemLossFunction`."
-abstract type SemLossFunction end
-
-"""
- SemLoss(args...; loss_weights = nothing, ...)
-
-Constructs the loss field of a SEM. Can contain multiple `SemLossFunction`s, the model is optimized over their sum.
-See also [`SemLossFunction`](@ref).
-
-# Arguments
-- `args...`: Multiple `SemLossFunction`s.
-- `loss_weights::Vector`: Weights for each loss function. Defaults to unweighted optimization.
-
-# Examples
-```julia
-my_ml_loss = SemML(...)
-my_ridge_loss = SemRidge(...)
-my_loss = SemLoss(SemML, SemRidge; loss_weights = [1.0, 2.0])
-```
-"""
-mutable struct SemLoss{F <: Tuple, T}
- functions::F
- weights::T
-end
-
-function SemLoss(functions...; loss_weights = nothing, kwargs...)
- if !isnothing(loss_weights)
- loss_weights = SemWeight.(loss_weights)
- else
- loss_weights = Tuple(SemWeight(nothing) for _ in 1:length(functions))
- end
+"Supertype for all loss functions of SEMs. If you want to implement a custom loss function, it should be a subtype of `AbstractLoss`."
+abstract type AbstractLoss end
- return SemLoss(functions, loss_weights)
-end
-
-# weights for loss functions or models. If the weight is nothing, multiplication returns the second argument
-struct SemWeight{T}
- w::T
-end
-
-Base.:*(x::SemWeight{Nothing}, y) = y
-Base.:*(x::SemWeight, y) = x.w * y
-
-"""
-Supertype of all objects that can serve as the `optimizer` field of a SEM.
-Connects the SEM to its optimization backend and controls options like the optimization algorithm.
-If you want to connect the SEM package to a new optimization backend, you should implement a subtype of SemOptimizer.
-"""
abstract type SemOptimizer{E} end
-engine(::Type{SemOptimizer{E}}) where {E} = E
-engine(optimizer::SemOptimizer) = engine(typeof(optimizer))
-
-SemOptimizer(args...; engine::Symbol = :Optim, kwargs...) =
- SemOptimizer{engine}(args...; kwargs...)
-
-# fallback optimizer constructor
-function SemOptimizer{E}(args...; kwargs...) where {E}
- throw(ErrorException("$E optimizer is not supported."))
-end
+# wrapper around optimization result
+abstract type SemOptimizerResult{O <: SemOptimizer} end
"""
+ abstract type SemObserved
+
Supertype of all objects that can serve as the observed field of a SEM.
Pre-processes data and computes sufficient statistics for example.
-If you have a special kind of data, e.g. ordinal data, you should implement a subtype of SemObserved.
+If you have a special kind of data, you should implement a subtype of SemObserved.
"""
abstract type SemObserved end
@@ -116,153 +57,93 @@ abstract type SemImplied end
abstract type SemImpliedSymbolic <: SemImplied end
"""
- Sem(;observed = SemObservedData, implied = RAM, loss = SemML, kwargs...)
+ abstract type SemLoss{O <: SemObserved, I <: SemImplied} <: AbstractLoss
-Constructor for the basic `Sem` type.
-All additional kwargs are passed down to the constructors for the observed, implied, and loss fields.
+The base type for calculating the loss of the implied SEM model when explaining the observed data.
-# Arguments
-- `observed`: object of subtype `SemObserved` or a constructor.
-- `implied`: object of subtype `SemImplied` or a constructor.
-- `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such.
-
-Returns a Sem with fields
-- `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref).
-- `implied::SemImplied`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImplied`](@ref).
-- `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref).
+All subtypes of `SemLoss` should have the following fields:
+- `observed::O`: object of subtype [`SemObserved`](@ref).
+- `implied::I`: object of subtype [`SemImplied`](@ref).
"""
-mutable struct Sem{O <: SemObserved, I <: SemImplied, L <: SemLoss} <:
- AbstractSemSingle{O, I, L}
- observed::O
- implied::I
- loss::L
-end
+abstract type SemLoss{O <: SemObserved, I <: SemImplied} <: AbstractLoss end
-############################################################################################
-# automatic differentiation
-############################################################################################
"""
- SemFiniteDiff(;observed = SemObservedData, implied = RAM, loss = SemML, kwargs...)
+ abstract type AbstractSem
-A wrapper around [`Sem`](@ref) that substitutes dedicated evaluation of gradient and hessian with
-finite difference approximation.
+The base type for all SEMs.
+"""
+abstract type AbstractSem end
-# Arguments
-- `observed`: object of subtype `SemObserved` or a constructor.
-- `implied`: object of subtype `SemImplied` or a constructor.
-- `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such.
-
-Returns a Sem with fields
-- `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref).
-- `implied::SemImplied`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImplied`](@ref).
-- `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref).
"""
-struct SemFiniteDiff{O <: SemObserved, I <: SemImplied, L <: SemLoss} <:
- AbstractSemSingle{O, I, L}
- observed::O
- implied::I
+ struct LossTerm{L, I, W}
+
+A term of a [`Sem`](@ref) model that wraps [`AbstractLoss`](@ref) loss function of type `L`.
+Loss term can have an optional *id* of type `I` and *weight* of numeric type `W`.
+"""
+struct LossTerm{L <: AbstractLoss, I <: Union{Symbol, Nothing}, W <: Union{Number, Nothing}}
loss::L
+ id::I
+ weight::W
end
-############################################################################################
-# ensemble models
-############################################################################################
"""
- (1) SemEnsemble(models...; weights = nothing, kwargs...)
+ Sem(loss_terms...; [params], kwargs...)
- (2) SemEnsemble(;specification, data, groups, column = :group, kwargs...)
+SEM model (including multi-group SEMs) that combines all the data, implied SEM structure
+and regularization terms.
-Constructor for ensemble models. (2) can be used to conveniently specify multigroup models.
+All terms of the `Sem` object share the same set of parameters.
+`Sem` implements the calculation of the weighted sum of its terms (the *objective*
+function), as well as the gradient and Hessian of this sum.
# Arguments
-- `models...`: `AbstractSem`s.
-- `weights::Vector`: Weights for each model. Defaults to the number of observed data points.
-- `specification::EnsembleParameterTable`: Model specification.
-- `data::DataFrame`: Observed data. Must contain a `column` of type `Vector{Symbol}` that contains the group.
-- `groups::Vector{Symbol}`: Group names.
-- `column::Symbol`: Name of the column in `data` that contains the group.
-
-All additional kwargs are passed down to the model parts.
-
-Returns a SemEnsemble with fields
-- `n::Int`: Number of models.
-- `sems::Tuple`: `AbstractSem`s.
-- `weights::Vector`: Weights for each model.
-- `param_labels::Vector`: Stores parameter labels and their position.
-
-For instructions on multigroup models, see the online documentation.
+- `loss_terms...`: [`AbstractLoss`](@ref) objects, including SEM losses ([`SemLoss`](@ref)),
+ optionally can be a pair of a loss object and its numeric weight
+
+# Fields
+- `loss_terms::Tuple`: a tuple of all loss functions and their weights
+- `params::Vector{Symbol}`: the vector of parameter ids shared by all loss functions.
"""
-struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, I} <: AbstractSemCollection
- n::N
- sems::T
- weights::V
- param_labels::I
+struct Sem{L <: Tuple} <: AbstractSem
+ loss_terms::L
+ params::Vector{Symbol}
end
-# constructor from multiple models
-function SemEnsemble(models...; weights = nothing, kwargs...)
- n = length(models)
+############################################################################################
+# automatic differentiation
+############################################################################################
- # default weights
+"""
+ SemFiniteDiff(model::AbstractSem)
- if isnothing(weights)
- nsamples_total = sum(nsamples, models)
- weights = [nsamples(model) / nsamples_total for model in models]
- end
+A wrapper around [`AbstractSem`](@ref) that substitutes dedicated evaluation of gradient and
+hessian with finite difference approximation.
- # check parameters equality
- param_labels = SEM.param_labels(models[1])
- for model in models
- if param_labels != SEM.param_labels(model)
- throw(ErrorException("The parameters of your models do not match. \n
- Maybe you tried to specify models of an ensemble via ParameterTables. \n
- In that case, you may use RAMMatrices instead."))
- end
- end
+`SemFiniteDiff` could be used to enable gradient-based optimization of the SEM models
+when the dedicated calculation of gradient and hessian are not available.
+For approximation, it uses the *FiniteDiff.jl* package.
- return SemEnsemble(n, models, weights, param_labels)
+# Arguments
+- `model::Sem`: the SEM model to wrap
+"""
+struct SemFiniteDiff{S <: AbstractSem} <: AbstractSem
+ model::S
end
-# constructor from EnsembleParameterTable and data set
-function SemEnsemble(; specification, data, groups, column = :group, kwargs...)
- if specification isa EnsembleParameterTable
- specification = convert(Dict{Symbol, RAMMatrices}, specification)
- end
- models = []
- for group in groups
- ram_matrices = specification[group]
- data_group = select(filter(r -> r[column] == group, data), Not(column))
- if iszero(nrow(data_group))
- error("Your data does not contain any observations from group `$(group)`.")
- end
- model = Sem(; specification = ram_matrices, data = data_group, kwargs...)
- push!(models, model)
- end
- return SemEnsemble(models...; weights = nothing, kwargs...)
+struct LossFiniteDiff{L <: AbstractLoss} <: AbstractLoss
+ loss::L
end
-param_labels(ensemble::SemEnsemble) = ensemble.param_labels
-
-"""
- n_models(ensemble::SemEnsemble) -> Integer
-
-Returns the number of models in an ensemble model.
-"""
-n_models(ensemble::SemEnsemble) = ensemble.n
-"""
- models(ensemble::SemEnsemble) -> Tuple{AbstractSem}
+struct SemLossFiniteDiff{O, I, L <: SemLoss{O, I}} <: SemLoss{O, I}
+ loss::L
+end
-Returns the models in an ensemble model.
-"""
-models(ensemble::SemEnsemble) = ensemble.sems
-"""
- weights(ensemble::SemEnsemble) -> Vector
+SemFiniteDiff(args...; kwargs...) =
+ SemFiniteDiff(Sem(args...; gradient = false, hessian = false, kwargs...))
-Returns the weights of an ensemble model.
"""
-weights(ensemble::SemEnsemble) = ensemble.weights
+ abstract type SemSpecification end
-"""
Base type for all SEM specifications.
"""
abstract type SemSpecification end
diff --git a/test/examples/helper.jl b/test/examples/helper.jl
index 4ff9bd507..fed95f3c7 100644
--- a/test/examples/helper.jl
+++ b/test/examples/helper.jl
@@ -1,4 +1,5 @@
using LinearAlgebra: norm
+using Suppressor
function is_extended_tests()
return lowercase(get(ENV, "JULIA_EXTENDED_TESTS", "false")) == "true"
@@ -48,7 +49,8 @@ function test_hessian(model, params; rtol = 1e-4, atol = 0)
@test hessian ≈ true_hessian rtol = rtol atol = atol
end
-fitmeasure_names_ml = Dict(
+# map from the SEM.jl name of the fit measure to the lavaan's one
+fitmeasure_semjl_to_lavaan = Dict(
:AIC => "aic",
:BIC => "bic",
:dof => "df",
@@ -56,26 +58,31 @@ fitmeasure_names_ml = Dict(
:p_value => "pvalue",
:nparams => "npar",
:RMSEA => "rmsea",
-)
-
-fitmeasure_names_ls = Dict(
- :dof => "df",
- :χ² => "chisq",
- :p_value => "pvalue",
- :nparams => "npar",
- :RMSEA => "rmsea",
+ :CFI => "cfi",
)
function test_fitmeasures(
- measures,
+ fitted::SemFit,
measures_lav;
+ fitmeasures::AbstractVector = SEM.DEFAULT_FIT_MEASURES,
+ fitted_baseline::Union{SemFit, Nothing} = nothing,
rtol = 1e-4,
atol = 0,
- fitmeasure_names = fitmeasure_names_ml,
)
- @testset "$name" for (key, name) in pairs(fitmeasure_names)
- measure_lav = measures_lav.x[findfirst(==(name), measures_lav[!, 1])]
- @test measures[key] ≈ measure_lav rtol = rtol atol = atol
+ @testset "$fn" for fn in fitmeasures
+ name = Symbol(fn)
+ # FIML CFI requires the baseline model
+ measure =
+ fn != CFI || isnothing(fitted_baseline) ? fn(fitted) :
+ fn(fitted, fitted_baseline)
+ lav_name = fitmeasure_semjl_to_lavaan[name]
+ lav_ix = findfirst(==(lav_name), measures_lav[!, 1])
+ if isnothing(lav_ix)
+ @test ismissing(measure)
+ else
+ measure_lav = measures_lav.x[lav_ix]
+ @test measure ≈ measure_lav rtol = rtol atol = atol
+ end
end
end
@@ -90,12 +97,8 @@ function test_estimates(
skip::Bool = false,
)
actual = StructuralEquationModels.params(partable, col)
- expected = StructuralEquationModels.lavaan_params(
- partable_lav,
- partable,
- lav_col,
- lav_group,
- )
+ expected =
+ StructuralEquationModels.lavaan_params(partable_lav, partable, lav_col, lav_group)
@test !any(isnan, actual)
@test !any(isnan, expected)
@@ -139,3 +142,45 @@ function test_estimates(
@test actual ≈ expected rtol = rtol atol = atol norm = Base.Fix2(norm, Inf)
end
end
+
+function test_bootstrap(
+ model_fit::SemFit;
+ compare_hessian = true,
+ rtol_hessian = 0.2,
+ compare_bs = true,
+ rtol_bs = 0.1,
+ n_boot = 500,
+ seed = 32432,
+)
+ @testset rng = Random.seed!(seed) "bootstrap" begin
+ se_bs = @suppress se_bootstrap(model_fit; n_boot = n_boot)
+ # hessian-based and bootstrap-based std.errors are close
+ if compare_hessian
+ se_he = @suppress se_hessian(model_fit)
+ #println(maximum(abs.(se_he - se_bs)))
+ @test isapprox(se_bs, se_he, rtol = rtol_hessian)
+ end
+ # se_bootstrap and bootstrap |> se are close
+ if compare_bs
+ bs_samples = bootstrap(model_fit; n_boot = n_boot)
+ @test bs_samples.n_converged >= 0.95*n_boot
+ bs_samples = reduce(hcat, bs_samples.samples[bs_samples.converged_mask])
+ se_bs_2 = sqrt.(var(bs_samples, corrected = false, dims = 2))
+ #println(maximum(abs.(se_bs_2 - se_bs)))
+ @test isapprox(se_bs_2, se_bs, rtol = rtol_bs)
+ end
+ end
+end
+
+function smoketest_bootstrap(model_fit::SemFit; n_boot = 5)
+ # just test that both methods succeed
+ se_bs = se_bootstrap(model_fit; n_boot = n_boot)
+ bs_samples = bootstrap(model_fit; n_boot = n_boot)
+ return se_bs, bs_samples
+end
+
+function smoketest_CI_z(model_fit::SemFit, partable)
+ se_he = @suppress se_hessian(model_fit)
+ normal_CI!(partable, model_fit, se_he)
+ z_test!(partable, model_fit, se_he)
+end
diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl
index f6a7a230d..47bdea224 100644
--- a/test/examples/multigroup/build_models.jl
+++ b/test/examples/multigroup/build_models.jl
@@ -4,26 +4,30 @@ const SEM = StructuralEquationModels
# ML estimation
############################################################################################
-model_g1 = Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic)
+obs_g1 = SemObservedData(data = dat_g1, observed_vars = SEM.observed_vars(specification_g1))
+obs_g2 = SemObservedData(data = dat_g2, observed_vars = SEM.observed_vars(specification_g2))
-model_g2 = Sem(specification = specification_g2, data = dat_g2, implied = RAM)
+model_ml_multigroup = Sem(
+ :Pasteur => SemML(obs_g1, RAMSymbolic(specification_g1)),
+ :Grant_White => SemML(obs_g2, RAM(specification_g2)),
+)
-@test SEM.param_labels(model_g1.implied.ram_matrices) == SEM.param_labels(model_g2.implied.ram_matrices)
+@testset "Sem API" begin
+ @test SEM.nsamples(model_ml_multigroup) == nsamples(obs_g1) + nsamples(obs_g2)
+ @test SEM.nsem_terms(model_ml_multigroup) == 2
+ @test length(SEM.sem_terms(model_ml_multigroup)) == 2
+end
-# test the different constructors
-model_ml_multigroup = SemEnsemble(model_g1, model_g2)
-model_ml_multigroup2 = SemEnsemble(
- specification = partable,
- data = dat,
- column = :school,
- groups = [:Pasteur, :Grant_White],
- loss = SemML,
-)
+# replace observed using Dict of data matrices
+model_ml_multigroup3 =
+ replace_observed(model_ml_multigroup, Dict(:Pasteur => dat_g1, :Grant_White => dat_g2))
+
+# replace observed using DataFrame with group column
+model_ml_multigroup4 = replace_observed(model_ml_multigroup, dat; semterm_column = :school)
# gradients
@testset "ml_gradients_multigroup" begin
test_gradient(model_ml_multigroup, start_test; atol = 1e-9)
- test_gradient(model_ml_multigroup2, start_test; atol = 1e-9)
end
# fit
@@ -36,41 +40,20 @@ end
atol = 1e-4,
lav_groups = Dict(:Pasteur => 1, :Grant_White => 2),
)
- solution = fit(semoptimizer, model_ml_multigroup2)
- update_estimate!(partable, solution)
- test_estimates(
- partable,
- solution_lav[:parameter_estimates_ml];
- atol = 1e-4,
- lav_groups = Dict(:Pasteur => 1, :Grant_White => 2),
- )
+end
+
+@testset "replace_observed_multigroup" begin
+ sem_fit_1 = fit(semoptimizer, model_ml_multigroup)
+ sem_fit_3 = fit(semoptimizer, model_ml_multigroup3)
+ @test sem_fit_1.solution ≈ sem_fit_3.solution
+ sem_fit_4 = fit(semoptimizer, model_ml_multigroup4)
+ @test sem_fit_1.solution ≈ sem_fit_4.solution
end
@testset "fitmeasures/se_ml" begin
- solution_ml = fit(model_ml_multigroup)
- test_fitmeasures(
- fit_measures(solution_ml),
- solution_lav[:fitmeasures_ml];
- rtol = 1e-2,
- atol = 1e-7,
- )
- update_se_hessian!(partable, solution_ml)
- test_estimates(
- partable,
- solution_lav[:parameter_estimates_ml];
- atol = 1e-3,
- col = :se,
- lav_col = :se,
- lav_groups = Dict(:Pasteur => 1, :Grant_White => 2),
- )
+ solution_ml = fit(semoptimizer, model_ml_multigroup)
+ test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7)
- solution_ml = fit(model_ml_multigroup2)
- test_fitmeasures(
- fit_measures(solution_ml),
- solution_lav[:fitmeasures_ml];
- rtol = 1e-2,
- atol = 1e-7,
- )
update_se_hessian!(partable, solution_ml)
test_estimates(
partable,
@@ -89,15 +72,19 @@ end
partable_s = sort_vars(partable)
specification_s = convert(Dict{Symbol, RAMMatrices}, partable_s)
+obs_g1_s = SemObservedData(
+ data = dat_g1,
+ observed_vars = SEM.observed_vars(specification_s[:Pasteur]),
+)
+obs_g2_s = SemObservedData(
+ data = dat_g2,
+ observed_vars = SEM.observed_vars(specification_s[:Grant_White]),
+)
-specification_g1_s = specification_s[:Pasteur]
-specification_g2_s = specification_s[:Grant_White]
-
-model_g1 = Sem(specification = specification_g1_s, data = dat_g1, implied = RAMSymbolic)
-
-model_g2 = Sem(specification = specification_g2_s, data = dat_g2, implied = RAM)
-
-model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer)
+model_ml_multigroup = Sem(
+ SemML(obs_g1_s, RAMSymbolic(specification_s[:Pasteur])),
+ SemML(obs_g2_s, RAM(specification_s[:Grant_White])),
+)
# gradients
@testset "ml_gradients_multigroup | sorted" begin
@@ -113,7 +100,7 @@ grad_fd = FiniteDiff.finite_difference_gradient(
# fit
@testset "ml_solution_multigroup | sorted" begin
- solution = fit(model_ml_multigroup)
+ solution = fit(semoptimizer, model_ml_multigroup)
update_estimate!(partable_s, solution)
test_estimates(
partable_s,
@@ -124,13 +111,8 @@ grad_fd = FiniteDiff.finite_difference_gradient(
end
@testset "fitmeasures/se_ml | sorted" begin
- solution_ml = fit(model_ml_multigroup)
- test_fitmeasures(
- fit_measures(solution_ml),
- solution_lav[:fitmeasures_ml];
- rtol = 1e-2,
- atol = 1e-7,
- )
+ solution_ml = fit(semoptimizer, model_ml_multigroup)
+ test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7)
update_se_hessian!(partable_s, solution_ml)
test_estimates(
@@ -144,28 +126,29 @@ end
end
@testset "sorted | LowerTriangular A" begin
- @test implied(model_ml_multigroup.sems[2]).A isa LowerTriangular
+ @test implied(SEM.sem_terms(model_ml_multigroup)[2]).A isa LowerTriangular
end
############################################################################################
# ML estimation - user defined loss function
############################################################################################
-struct UserSemML <: SemLossFunction
+struct UserSemML{O, I} <: SemLoss{O, I}
hessianeval::ExactHessian
- UserSemML() = new(ExactHessian())
-end
+ observed::O
+ implied::I
-############################################################################################
-### functors
-############################################################################################
-
-using LinearAlgebra: isposdef, logdet, tr, inv
+ UserSemML(
+ observed::SemObserved,
+ implied::SemImplied,
+ refloss::Union{UserSemML, Nothing} = nothing,
+ ) = new{typeof(observed), typeof(implied)}(ExactHessian(), observed, implied)
+end
-function SEM.objective(ml::UserSemML, model::AbstractSem, params)
- Σ = implied(model).Σ
- Σₒ = SEM.obs_cov(observed(model))
+function SEM.objective(ml::UserSemML, params)
+ Σ = implied(ml).Σ
+ Σₒ = SEM.obs_cov(observed(ml))
if !isposdef(Σ)
return Inf
else
@@ -174,24 +157,18 @@ function SEM.objective(ml::UserSemML, model::AbstractSem, params)
end
# models
-model_g1 = Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic)
-
-model_g2 = SemFiniteDiff(
- specification = specification_g2,
- data = dat_g2,
- implied = RAMSymbolic,
- loss = UserSemML(),
+model_ml_multigroup = Sem(
+ SemML(obs_g1, RAMSymbolic(specification_g1)),
+ SEM.FiniteDiffWrapper(UserSemML(obs_g2, RAMSymbolic(specification_g2))),
)
-model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer)
-
@testset "gradients_user_defined_loss" begin
test_gradient(model_ml_multigroup, start_test; atol = 1e-9)
end
# fit
@testset "solution_user_defined_loss" begin
- solution = fit(model_ml_multigroup)
+ solution = fit(semoptimizer, model_ml_multigroup)
update_estimate!(partable, solution)
test_estimates(
partable,
@@ -201,32 +178,31 @@ end
)
end
+@testset "replace_observed_user_defined_loss" begin
+ wrapped_loss = SEM.FiniteDiffWrapper(UserSemML(obs_g2, RAMSymbolic(specification_g2)))
+ new_data = randn(nsamples(obs_g2), nobserved_vars(obs_g2))
+ replaced_loss = SEM._unwrap(replace_observed(wrapped_loss, new_data))
+
+ @test replaced_loss isa UserSemML
+ @test observed_vars(replaced_loss) == observed_vars(obs_g2)
+ @test implied(replaced_loss) === implied(SEM._unwrap(wrapped_loss))
+end
+
############################################################################################
# GLS estimation
############################################################################################
-model_ls_g1 = Sem(
- specification = specification_g1,
- data = dat_g1,
- implied = RAMSymbolic,
- loss = SemWLS,
+model_ls_multigroup = Sem(
+ SemWLS(obs_g1, RAMSymbolic(specification_g1, vech = true)),
+ SemWLS(obs_g2, RAMSymbolic(specification_g2, vech = true)),
)
-model_ls_g2 = Sem(
- specification = specification_g2,
- data = dat_g2,
- implied = RAMSymbolic,
- loss = SemWLS,
-)
-
-model_ls_multigroup = SemEnsemble(model_ls_g1, model_ls_g2; optimizer = semoptimizer)
-
@testset "ls_gradients_multigroup" begin
test_gradient(model_ls_multigroup, start_test; atol = 1e-9)
end
@testset "ls_solution_multigroup" begin
- solution = fit(model_ls_multigroup)
+ solution = fit(semoptimizer, model_ls_multigroup)
update_estimate!(partable, solution)
test_estimates(
partable,
@@ -237,14 +213,8 @@ end
end
@testset "fitmeasures/se_ls" begin
- solution_ls = fit(model_ls_multigroup)
- test_fitmeasures(
- fit_measures(solution_ls),
- solution_lav[:fitmeasures_ls];
- fitmeasure_names = fitmeasure_names_ls,
- rtol = 1e-2,
- atol = 1e-5,
- )
+ solution_ls = fit(semoptimizer, model_ls_multigroup)
+ test_fitmeasures(solution_ls, solution_lav[:fitmeasures_ls]; rtol = 1e-2, atol = 1e-5)
@suppress update_se_hessian!(partable, solution_ls)
test_estimates(
@@ -255,6 +225,8 @@ end
lav_col = :se,
lav_groups = Dict(:Pasteur => 1, :Grant_White => 2),
)
+ # test_bootstrap(solution_ls; compare_bs = false, rtol_hessian = 0.3)
+ smoketest_CI_z(solution_ls, partable)
end
############################################################################################
@@ -262,30 +234,27 @@ end
############################################################################################
if !isnothing(specification_miss_g1)
- model_g1 = Sem(
- specification = specification_miss_g1,
- observed = SemObservedMissing,
- loss = SemFIML,
- data = dat_miss_g1,
- implied = RAM,
- meanstructure = true,
+ model_ml_multigroup = Sem(
+ SemFIML(
+ SemObservedMissing(
+ data = dat_miss_g1,
+ observed_vars = SEM.observed_vars(specification_miss_g1),
+ ),
+ RAM(specification_miss_g1),
+ ),
+ SemFIML(
+ SemObservedMissing(
+ data = dat_miss_g2,
+ observed_vars = SEM.observed_vars(specification_miss_g2),
+ ),
+ RAM(specification_miss_g2),
+ ),
)
- model_g2 = Sem(
- specification = specification_miss_g2,
- observed = SemObservedMissing,
- loss = SemFIML,
- data = dat_miss_g2,
- implied = RAM,
- meanstructure = true,
- )
-
- model_ml_multigroup = SemEnsemble(model_g1, model_g2)
- model_ml_multigroup2 = SemEnsemble(
- specification = partable_miss,
- data = dat_missing,
- column = :school,
- groups = [:Pasteur, :Grant_White],
+ model_ml_varonly = Sem(
+ specification = partable_varonly,
+ data = dat_miss,
+ semterm_column = :school,
loss = SemFIML,
observed = SemObservedMissing,
meanstructure = true,
@@ -317,7 +286,6 @@ if !isnothing(specification_miss_g1)
@testset "fiml_gradients_multigroup" begin
test_gradient(model_ml_multigroup, start_test; atol = 1e-7)
- test_gradient(model_ml_multigroup2, start_test; atol = 1e-7)
end
@testset "fiml_solution_multigroup" begin
@@ -329,21 +297,15 @@ if !isnothing(specification_miss_g1)
atol = 1e-4,
lav_groups = Dict(:Pasteur => 1, :Grant_White => 2),
)
- solution = fit(semoptimizer, model_ml_multigroup2)
- update_estimate!(partable_miss, solution)
- test_estimates(
- partable_miss,
- solution_lav[:parameter_estimates_fiml];
- atol = 1e-4,
- lav_groups = Dict(:Pasteur => 1, :Grant_White => 2),
- )
end
@testset "fitmeasures/se_fiml" begin
solution = fit(semoptimizer, model_ml_multigroup)
+ solution_varonly = fit(semoptimizer, model_ml_varonly)
test_fitmeasures(
- fit_measures(solution),
+ solution,
solution_lav[:fitmeasures_fiml];
+ fitted_baseline = solution_varonly,
rtol = 1e-3,
atol = 0,
)
@@ -357,13 +319,9 @@ if !isnothing(specification_miss_g1)
lav_groups = Dict(:Pasteur => 1, :Grant_White => 2),
)
- solution = fit(semoptimizer, model_ml_multigroup2)
- test_fitmeasures(
- fit_measures(solution),
- solution_lav[:fitmeasures_fiml];
- rtol = 1e-3,
- atol = 0,
- )
+ test_bootstrap(solution; compare_bs = false, rtol_hessian = 0.5)
+ smoketest_CI_z(solution, partable_miss)
+
update_se_hessian!(partable_miss, solution)
test_estimates(
partable_miss,
diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl
index 239bf713c..d095cf118 100644
--- a/test/examples/multigroup/multigroup.jl
+++ b/test/examples/multigroup/multigroup.jl
@@ -1,22 +1,27 @@
using StructuralEquationModels, Test, FiniteDiff, Suppressor
-using LinearAlgebra: diagind, LowerTriangular
+using LinearAlgebra: diagind, isposdef, logdet, tr, LowerTriangular
+using Statistics: var
+using Random
+
+Random.seed!(948723)
const SEM = StructuralEquationModels
include(joinpath(chop(dirname(pathof(SEM)), tail = 3), "test/examples/helper.jl"))
dat = example_data("holzinger_swineford")
-dat_missing = example_data("holzinger_swineford_missing")
-solution_lav = example_data("holzinger_swineford_solution")
+dat.school = Symbol.(replace.(dat.school, "-" => "_"))
-dat_g1 = dat[dat.school.=="Pasteur", :]
-dat_g2 = dat[dat.school.=="Grant-White", :]
+dat_miss = example_data("holzinger_swineford_missing")
+dat_miss.school = Symbol.(replace.(dat_miss.school, "-" => "_"))
-dat_miss_g1 = dat_missing[dat_missing.school.=="Pasteur", :]
-dat_miss_g2 = dat_missing[dat_missing.school.=="Grant-White", :]
+solution_lav = example_data("holzinger_swineford_solution")
-dat.school = ifelse.(dat.school .== "Pasteur", :Pasteur, :Grant_White)
-dat_missing.school = ifelse.(dat_missing.school .== "Pasteur", :Pasteur, :Grant_White)
+dat_g1 = dat[dat.school .== :Pasteur, :]
+dat_g2 = dat[dat.school .== :Grant_White, :]
+
+dat_miss_g1 = dat_miss[dat_miss.school .== :Pasteur, :]
+dat_miss_g2 = dat_miss[dat_miss.school .== :Grant_White, :]
############################################################################################
### specification - RAMMatrices
@@ -86,7 +91,7 @@ start_test = [
fill(0.05, 3)
fill(0.01, 3)
]
-semoptimizer = SemOptimizerOptim()
+semoptimizer = SemOptimizer()
@testset "RAMMatrices | constructor | Optim" begin
include("build_models.jl")
@@ -98,8 +103,8 @@ end
# w.o. meanstructure -----------------------------------------------------------------------
-latent_vars = [:visual, :textual, :speed]
-observed_vars = Symbol.(:x, 1:9)
+lat_vars = [:visual, :textual, :speed]
+obs_vars = Symbol.(:x, 1:9)
graph = @StenoGraph begin
# measurement model
@@ -107,14 +112,14 @@ graph = @StenoGraph begin
textual → fixed(1.0, 1.0) * x4 + label(:λ₅, :λ₅) * x5 + label(:λ₆, :λ₆) * x6
speed → fixed(1.0, 1.0) * x7 + label(:λ₈, :λ₈) * x8 + label(:λ₉, :λ₉) * x9
# variances and covariances
- _(observed_vars) ↔ _(observed_vars)
- _(latent_vars) ⇔ _(latent_vars)
+ _(obs_vars) ↔ _(obs_vars)
+ _(lat_vars) ⇔ _(lat_vars)
end
partable = EnsembleParameterTable(
graph;
- observed_vars = observed_vars,
- latent_vars = latent_vars,
+ observed_vars = obs_vars,
+ latent_vars = lat_vars,
groups = [:Pasteur, :Grant_White],
)
@@ -125,8 +130,8 @@ specification_g2 = specification[:Grant_White]
# w. meanstructure (fiml) ------------------------------------------------------------------
-latent_vars = [:visual, :textual, :speed]
-observed_vars = Symbol.(:x, 1:9)
+lat_vars = [:visual, :textual, :speed]
+obs_vars = Symbol.(:x, 1:9)
graph = @StenoGraph begin
# measurement model
@@ -134,16 +139,16 @@ graph = @StenoGraph begin
textual → fixed(1.0, 1.0) * x4 + label(:λ₅, :λ₅) * x5 + label(:λ₆, :λ₆) * x6
speed → fixed(1.0, 1.0) * x7 + label(:λ₈, :λ₈) * x8 + label(:λ₉, :λ₉) * x9
# variances and covariances
- _(observed_vars) ↔ _(observed_vars)
- _(latent_vars) ⇔ _(latent_vars)
+ _(obs_vars) ↔ _(obs_vars)
+ _(lat_vars) ⇔ _(lat_vars)
- Symbol(1) → _(observed_vars)
+ Symbol(1) → _(obs_vars)
end
partable_miss = EnsembleParameterTable(
graph;
- observed_vars = observed_vars,
- latent_vars = latent_vars,
+ observed_vars = obs_vars,
+ latent_vars = lat_vars,
groups = [:Pasteur, :Grant_White],
)
@@ -152,6 +157,19 @@ specification_miss = convert(Dict{Symbol, RAMMatrices}, partable_miss)
specification_miss_g1 = specification_miss[:Pasteur]
specification_miss_g2 = specification_miss[:Grant_White]
+# CFI baseline model
+graph_varonly = @StenoGraph begin
+ _(obs_vars) ↔ _(obs_vars)
+ Symbol(1) → _(obs_vars)
+end
+
+partable_varonly = EnsembleParameterTable(
+ graph_varonly;
+ observed_vars = obs_vars,
+ latent_vars = lat_vars,
+ groups = [:Pasteur, :Grant_White],
+)
+
start_test = [
fill(0.5, 6)
fill(1.0, 9)
@@ -169,7 +187,7 @@ start_test = [
0.01
0.05
]
-semoptimizer = SemOptimizerOptim()
+semoptimizer = SemOptimizer()
@testset "Graph → Partable → RAMMatrices | constructor | Optim" begin
include("build_models.jl")
diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl
index 3397b5f0a..27de965ec 100644
--- a/test/examples/political_democracy/by_parts.jl
+++ b/test/examples/political_democracy/by_parts.jl
@@ -2,103 +2,83 @@
### models w.o. meanstructure
############################################################################################
-# observed ---------------------------------------------------------------------------------
-observed = SemObservedData(specification = spec, data = dat)
+semoptimizer = SemOptimizer(engine = opt_engine)
-# implied
-implied_ram = RAM(specification = spec)
+model_ml = Sem(specification = spec, data = dat)
+@test SEM.param_labels(model_ml) == SEM.param_labels(spec)
-implied_ram_sym = RAMSymbolic(specification = spec)
+model_ls_sym =
+ Sem(specification = spec, data = dat, implied = RAMSymbolic, vech = true, loss = SemWLS)
-# loss functions ---------------------------------------------------------------------------
-ml = SemML(observed = observed)
+model_ml_sym = Sem(specification = spec, data = dat, implied = RAMSymbolic)
-wls = SemWLS(observed = observed)
-
-ridge = SemRidge(α_ridge = 0.001, which_ridge = 16:20, nparams = 31)
-
-constant = SemConstant(constant_loss = 3.465)
-
-# loss -------------------------------------------------------------------------------------
-loss_ml = SemLoss(ml)
-
-loss_wls = SemLoss(wls)
-
-# optimizer -------------------------------------------------------------------------------------
-optimizer_obj = SemOptimizer(engine = opt_engine)
-
-# models -----------------------------------------------------------------------------------
-
-model_ml = Sem(observed, implied_ram, loss_ml)
-
-model_ls_sym = Sem(observed, RAMSymbolic(specification = spec, vech = true), loss_wls)
-
-model_ml_sym = Sem(observed, implied_ram_sym, loss_ml)
-
-model_ridge = Sem(observed, implied_ram, SemLoss(ml, ridge))
+model_ml_ridge = Sem(
+ specification = spec,
+ data = dat,
+ loss = (SemML, SemRidge),
+ α_ridge = 0.001,
+ which_ridge = 16:20,
+)
-model_constant = Sem(observed, implied_ram, SemLoss(ml, constant))
+model_ml_const = Sem(
+ specification = spec,
+ data = dat,
+ loss = (SemML, SemConstant),
+ constant_loss = 3.465,
+)
-model_ml_weighted =
- Sem(observed, implied_ram, SemLoss(ml; loss_weights = [nsamples(model_ml)]))
+model_ml_weighted = Sem(SemML(SemObservedData(data = dat), RAM(spec)) => nsamples(model_ml))
############################################################################################
### test gradients
############################################################################################
-models =
- [model_ml, model_ls_sym, model_ridge, model_constant, model_ml_sym, model_ml_weighted]
-model_names = ["ml", "ls_sym", "ridge", "constant", "ml_sym", "ml_weighted"]
+models = Dict(
+ "ml" => model_ml,
+ "ls_sym" => model_ls_sym,
+ "ml_ridge" => model_ml_ridge,
+ "ml_const" => model_ml_const,
+ "ml_sym" => model_ml_sym,
+ "ml_weighted" => model_ml_weighted,
+)
-for (model, name) in zip(models, model_names)
- try
- @testset "$(name)_gradient" begin
- test_gradient(model, start_test; rtol = 1e-9)
- end
- catch
- end
+@testset "$(id)_gradient" for (id, model) in pairs(models)
+ test_gradient(model, start_test; rtol = 1e-9)
end
############################################################################################
### test solution
############################################################################################
-models = [model_ml, model_ls_sym, model_ml_sym, model_constant]
-model_names = ["ml", "ls_sym", "ml_sym", "constant"]
-solution_names = Symbol.("parameter_estimates_" .* ["ml", "ls", "ml", "ml"])
-
-for (model, name, solution_name) in zip(models, model_names, solution_names)
- try
- @testset "$(name)_solution" begin
- solution = fit(optimizer_obj, model)
- update_estimate!(partable, solution)
- test_estimates(partable, solution_lav[solution_name]; atol = 1e-2)
- end
- catch
- end
+@testset "$(id)_solution" for id in ["ml", "ls_sym", "ml_sym", "ml_const"]
+ model = models[id]
+ solution = fit(semoptimizer, model)
+ sol_name = Symbol("parameter_estimates_", replace(id, r"_.+$" => ""))
+ update_estimate!(partable, solution)
+ test_estimates(partable, solution_lav[sol_name]; atol = 1e-2)
end
@testset "ridge_solution" begin
- solution_ridge = fit(optimizer_obj, model_ridge)
- solution_ml = fit(optimizer_obj, model_ml)
- # solution_ridge_id = fit(optimizer_obj, model_ridge_id)
- @test solution_ridge.minimum < solution_ml.minimum + 1
+ solution_ridge = fit(semoptimizer, model_ml_ridge)
+ solution_ml = fit(semoptimizer, model_ml)
+ # solution_ridge_id = fit(model_ridge_id)
+ @test abs(solution_ridge.minimum - solution_ml.minimum) < 1
end
# test constant objective value
@testset "constant_objective_and_gradient" begin
- @test (objective!(model_constant, start_test) - 3.465) ≈
+ @test (objective!(model_ml_const, start_test) - 3.465) ≈
objective!(model_ml, start_test)
grad = similar(start_test)
grad2 = similar(start_test)
- gradient!(grad, model_constant, start_test)
+ gradient!(grad, model_ml_const, start_test)
gradient!(grad2, model_ml, start_test)
@test grad ≈ grad2
end
@testset "ml_solution_weighted" begin
- solution_ml = fit(optimizer_obj, model_ml)
- solution_ml_weighted = fit(optimizer_obj, model_ml_weighted)
+ solution_ml = fit(semoptimizer, model_ml)
+ solution_ml_weighted = fit(semoptimizer, model_ml_weighted)
@test solution(solution_ml) ≈ solution(solution_ml_weighted) rtol = 1e-3
@test nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml) ≈
StructuralEquationModels.minimum(solution_ml_weighted) rtol = 1e-6
@@ -109,8 +89,8 @@ end
############################################################################################
@testset "fitmeasures/se_ml" begin
- solution_ml = fit(optimizer_obj, model_ml)
- test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3)
+ solution_ml = fit(semoptimizer, model_ml)
+ test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml]; atol = 1e-3)
update_se_hessian!(partable, solution_ml)
test_estimates(
@@ -123,15 +103,8 @@ end
end
@testset "fitmeasures/se_ls" begin
- solution_ls = fit(optimizer_obj, model_ls_sym)
- fm = fit_measures(solution_ls)
- test_fitmeasures(
- fm,
- solution_lav[:fitmeasures_ls];
- atol = 1e-3,
- fitmeasure_names = fitmeasure_names_ls,
- )
- @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing)
+ solution_ls = fit(semoptimizer, model_ls_sym)
+ test_fitmeasures(solution_ls, solution_lav[:fitmeasures_ls]; atol = 1e-3)
@suppress update_se_hessian!(partable, solution_ls)
test_estimates(
@@ -150,22 +123,22 @@ end
if opt_engine == :Optim
using Optim, LineSearches
- optimizer_obj = SemOptimizer(
- engine = opt_engine,
- algorithm = Newton(;
- linesearch = BackTracking(order = 3),
- alphaguess = InitialHagerZhang(),
- ),
+ model_ls = Sem(
+ data = dat,
+ specification = spec,
+ implied = RAMSymbolic,
+ loss = SemWLS,
+ vech = true,
+ hessian = true,
)
- implied_sym_hessian_vech =
- RAMSymbolic(specification = spec, vech = true, hessian = true)
-
- implied_sym_hessian = RAMSymbolic(specification = spec, hessian = true)
-
- model_ls = Sem(observed, implied_sym_hessian_vech, loss_wls)
-
- model_ml = Sem(observed, implied_sym_hessian, loss_ml)
+ model_ml = Sem(
+ data = dat,
+ specification = spec,
+ implied = RAMSymbolic,
+ loss = SemML,
+ hessian = true,
+ )
@testset "ml_hessians" begin
test_hessian(model_ml, start_test; atol = 1e-4)
@@ -176,13 +149,23 @@ if opt_engine == :Optim
end
@testset "ml_solution_hessian" begin
- solution = fit(optimizer_obj, model_ml)
+ solution = fit(SemOptimizer(engine = :Optim, algorithm = Newton()), model_ml)
+
update_estimate!(partable, solution)
test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-2)
end
@testset "ls_solution_hessian" begin
- solution = fit(optimizer_obj, model_ls)
+ solution = fit(
+ SemOptimizer(
+ engine = :Optim,
+ algorithm = Newton(
+ linesearch = BackTracking(order = 3),
+ alphaguess = InitialHagerZhang(),
+ ),
+ ),
+ model_ls,
+ )
update_estimate!(partable, solution)
test_estimates(
partable,
@@ -197,69 +180,47 @@ end
### meanstructure
############################################################################################
-# observed ---------------------------------------------------------------------------------
-observed = SemObservedData(specification = spec_mean, data = dat, meanstructure = true)
-
-# implied
-implied_ram = RAM(specification = spec_mean, meanstructure = true)
-
-implied_ram_sym = RAMSymbolic(specification = spec_mean, meanstructure = true)
-
-# loss functions ---------------------------------------------------------------------------
-ml = SemML(observed = observed, meanstructure = true)
-
-wls = SemWLS(observed = observed, meanstructure = true)
-
-# loss -------------------------------------------------------------------------------------
-loss_ml = SemLoss(ml)
-
-loss_wls = SemLoss(wls)
-
-# optimizer -------------------------------------------------------------------------------------
-optimizer_obj = SemOptimizer(engine = opt_engine)
+# models
+model_ls = Sem(
+ data = dat,
+ specification = spec_mean,
+ implied = RAMSymbolic,
+ loss = SemWLS,
+ vech = true,
+)
-# models -----------------------------------------------------------------------------------
-model_ml = Sem(observed, implied_ram, loss_ml)
+model_ml = Sem(data = dat, specification = spec_mean, implied = RAM, loss = SemML)
-model_ls = Sem(
- observed,
- RAMSymbolic(specification = spec_mean, meanstructure = true, vech = true),
- loss_wls,
+model_ml_cov = Sem(
+ specification = spec,
+ observed = SemObservedCovariance,
+ obs_cov = cov(Matrix(dat)),
+ observed_vars = Symbol.(names(dat)),
+ nsamples = 75,
)
-model_ml_sym = Sem(observed, implied_ram_sym, loss_ml)
+model_ml_sym =
+ Sem(data = dat, specification = spec_mean, implied = RAMSymbolic, loss = SemML)
############################################################################################
### test gradients
############################################################################################
-models = [model_ml, model_ls, model_ml_sym]
-model_names = ["ml", "ls_sym", "ml_sym"]
+models = Dict("ml" => model_ml, "ls_sym" => model_ls, "ml_sym" => model_ml_sym)
-for (model, name) in zip(models, model_names)
- try
- @testset "$(name)_gradient_mean" begin
- test_gradient(model, start_test_mean; rtol = 1e-9)
- end
- catch
- end
+@testset "$(id)_gradient_mean" for (id, model) in pairs(models)
+ test_gradient(model, start_test_mean; rtol = 1e-9)
end
############################################################################################
### test solution
############################################################################################
-solution_names = Symbol.("parameter_estimates_" .* ["ml", "ls", "ml"] .* "_mean")
-
-for (model, name, solution_name) in zip(models, model_names, solution_names)
- try
- @testset "$(name)_solution_mean" begin
- solution = fit(optimizer_obj, model)
- update_estimate!(partable_mean, solution)
- test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2)
- end
- catch
- end
+@testset "$(id)_solution_mean" for (id, model) in pairs(models)
+ solution = fit(semoptimizer, model, start_val = start_test_mean)
+ update_estimate!(partable_mean, solution)
+ sol_name = Symbol("parameter_estimates_", replace(id, r"_.+$" => ""), "_mean")
+ test_estimates(partable_mean, solution_lav[sol_name]; atol = 1e-2)
end
############################################################################################
@@ -267,12 +228,8 @@ end
############################################################################################
@testset "fitmeasures/se_ml_mean" begin
- solution_ml = fit(optimizer_obj, model_ml)
- test_fitmeasures(
- fit_measures(solution_ml),
- solution_lav[:fitmeasures_ml_mean];
- atol = 1e-3,
- )
+ solution_ml = fit(semoptimizer, model_ml)
+ test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml_mean]; atol = 1e-3)
update_se_hessian!(partable_mean, solution_ml)
test_estimates(
@@ -285,15 +242,8 @@ end
end
@testset "fitmeasures/se_ls_mean" begin
- solution_ls = fit(optimizer_obj, model_ls)
- fm = fit_measures(solution_ls)
- test_fitmeasures(
- fm,
- solution_lav[:fitmeasures_ls_mean];
- atol = 1e-3,
- fitmeasure_names = fitmeasure_names_ls,
- )
- @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing)
+ solution_ls = fit(semoptimizer, model_ls)
+ test_fitmeasures(solution_ls, solution_lav[:fitmeasures_ls_mean]; atol = 1e-3)
@suppress update_se_hessian!(partable_mean, solution_ls)
test_estimates(
@@ -309,15 +259,33 @@ end
### fiml
############################################################################################
-observed = SemObservedMissing(specification = spec_mean, data = dat_missing)
-
-fiml = SemFIML(observed = observed, specification = spec_mean)
-
-loss_fiml = SemLoss(fiml)
+# models
+model_ml = Sem(
+ data = dat_missing,
+ observed = SemObservedMissing,
+ specification = spec_mean,
+ implied = RAM,
+ loss = SemFIML,
+)
-model_ml = Sem(observed, implied_ram, loss_fiml)
+model_ml_sym = Sem(
+ data = dat_missing,
+ observed = SemObservedMissing,
+ specification = spec_mean,
+ implied = RAMSymbolic,
+ loss = SemFIML,
+)
-model_ml_sym = Sem(observed, implied_ram_sym, loss_fiml)
+if !ismissing(spec_varonly)
+ model_varonly = Sem(
+ data = dat_missing,
+ observed = SemObservedMissing,
+ specification = spec_varonly,
+ loss = SemFIML,
+ )
+else
+ model_varonly = nothing
+end
############################################################################################
### test gradients
@@ -336,13 +304,13 @@ end
############################################################################################
@testset "fiml_solution" begin
- solution = fit(optimizer_obj, model_ml)
+ solution = fit(semoptimizer, model_ml)
update_estimate!(partable_mean, solution)
test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2)
end
@testset "fiml_solution_symbolic" begin
- solution = fit(optimizer_obj, model_ml_sym)
+ solution = fit(semoptimizer, model_ml_sym, start_val = start_test_mean)
update_estimate!(partable_mean, solution)
test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2)
end
@@ -352,10 +320,15 @@ end
############################################################################################
@testset "fitmeasures/se_fiml" begin
- solution_ml = fit(optimizer_obj, model_ml)
+ solution_ml = fit(semoptimizer, model_ml)
+ solution_varonly =
+ !isnothing(model_varonly) ? fit(semoptimizer, model_varonly) : nothing
test_fitmeasures(
- fit_measures(solution_ml),
+ solution_ml,
solution_lav[:fitmeasures_fiml];
+ fitted_baseline = solution_varonly,
+ fitmeasures = !isnothing(solution_varonly) ? SEM.DEFAULT_FIT_MEASURES :
+ filter(!=(CFI), SEM.DEFAULT_FIT_MEASURES),
atol = 1e-3,
)
@@ -363,7 +336,7 @@ end
test_estimates(
partable_mean,
solution_lav[:parameter_estimates_fiml];
- atol = 1e-3,
+ atol = 0.002,
col = :se,
lav_col = :se,
)
diff --git a/test/examples/political_democracy/constraints.jl b/test/examples/political_democracy/constraints.jl
index cc1b0874d..0291e7ea5 100644
--- a/test/examples/political_democracy/constraints.jl
+++ b/test/examples/political_democracy/constraints.jl
@@ -26,8 +26,8 @@ constrained_optimizer = SemOptimizer(;
algorithm = :AUGLAG,
local_algorithm = :LD_LBFGS,
options = Dict(:xtol_rel => 1e-4),
- # equality_constraints = (f = eq_constraint, tol = 1e-14),
- inequality_constraints = (f = ineq_constraint, tol = 0.0),
+ # equality_constraints = (eq_constraint => 1e-14),
+ inequality_constraints = (ineq_constraint => 0.0),
)
@test constrained_optimizer isa SemOptimizer{:NLopt}
@@ -50,7 +50,7 @@ end
@test solution_constrained.solution[31] * solution_constrained.solution[30] >=
(0.6 - 1e-8)
- @test all(abs.(solution_constrained.solution) .< 10)
- @test solution_constrained.optimization_result.result[3] == :FTOL_REACHED
+ @test all(p -> abs(p) < 10, solution_constrained.solution)
+ @test solution_constrained.optimization_result.result[3] == :FTOL_REACHED skip = true
@test solution_constrained.minimum <= 21.21 + 0.01
end
diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl
index 7a8adc72e..2efa5abeb 100644
--- a/test/examples/political_democracy/constructor.jl
+++ b/test/examples/political_democracy/constructor.jl
@@ -1,6 +1,3 @@
-using Statistics: cov, mean
-using Random, NLopt
-
############################################################################################
### models w.o. meanstructure
############################################################################################
@@ -8,21 +5,22 @@ using Random, NLopt
semoptimizer = SemOptimizer(engine = opt_engine)
model_ml = Sem(specification = spec, data = dat)
-@test SEM.param_labels(model_ml.implied.ram_matrices) == SEM.param_labels(spec)
+@test SEM.param_labels(model_ml) == SEM.param_labels(spec)
model_ml_cov = Sem(
specification = spec,
observed = SemObservedCovariance,
obs_cov = cov(Matrix(dat)),
- obs_colnames = Symbol.(names(dat)),
+ observed_vars = Symbol.(names(dat)),
nsamples = 75,
)
-model_ls_sym = Sem(specification = spec, data = dat, implied = RAMSymbolic, loss = SemWLS)
+model_ls_sym =
+ Sem(specification = spec, data = dat, implied = RAMSymbolic, vech = true, loss = SemWLS)
model_ml_sym = Sem(specification = spec, data = dat, implied = RAMSymbolic)
-model_ridge = Sem(
+model_ml_ridge = Sem(
specification = spec,
data = dat,
loss = (SemML, SemRidge),
@@ -30,7 +28,7 @@ model_ridge = Sem(
which_ridge = 16:20,
)
-model_constant = Sem(
+model_ml_const = Sem(
specification = spec,
data = dat,
loss = (SemML, SemConstant),
@@ -38,65 +36,52 @@ model_constant = Sem(
)
model_ml_weighted =
- Sem(specification = partable, data = dat, loss_weights = (nsamples(model_ml),))
+ Sem(SemML(SemObservedData(data = dat), RAMSymbolic(spec)) => nsamples(model_ml))
############################################################################################
### test gradients
############################################################################################
-models = [
- model_ml,
- model_ml_cov,
- model_ls_sym,
- model_ridge,
- model_constant,
- model_ml_sym,
- model_ml_weighted,
-]
-model_names = ["ml", "ml_cov", "ls_sym", "ridge", "constant", "ml_sym", "ml_weighted"]
-
-for (model, name) in zip(models, model_names)
- try
- @testset "$(name)_gradient" begin
- test_gradient(model, start_test; rtol = 1e-9)
- end
- catch
- end
+models = Dict(
+ "ml" => model_ml,
+ "ml_cov" => model_ml_cov,
+ "ls_sym" => model_ls_sym,
+ "ridge" => model_ml_ridge,
+ "ml_const" => model_ml_const,
+ "ml_sym" => model_ml_sym,
+ "ml_weighted" => model_ml_weighted,
+)
+
+@testset "$(id)_gradient" for (id, model) in pairs(models)
+ test_gradient(model, start_test; rtol = 1e-9)
end
############################################################################################
### test solution
############################################################################################
-models = [model_ml, model_ml_cov, model_ls_sym, model_ml_sym, model_constant]
-model_names = ["ml", "ml_cov", "ls_sym", "ml_sym", "constant"]
-solution_names = Symbol.("parameter_estimates_" .* ["ml", "ml", "ls", "ml", "ml"])
-
-for (model, name, solution_name) in zip(models, model_names, solution_names)
- try
- @testset "$(name)_solution" begin
- solution = fit(semoptimizer, model)
- update_estimate!(partable, solution)
- test_estimates(partable, solution_lav[solution_name]; atol = 1e-2)
- end
- catch
- end
+@testset "$(id)_solution" for id in ["ml", "ml_cov", "ls_sym", "ml_sym", "ml_const"]
+ model = models[id]
+ solution = fit(semoptimizer, model)
+ sol_name = Symbol("parameter_estimates_", replace(id, r"_.+$" => ""))
+ update_estimate!(partable, solution)
+ test_estimates(partable, solution_lav[sol_name]; atol = 1e-2)
end
@testset "ridge_solution" begin
- solution_ridge = fit(semoptimizer, model_ridge)
+ solution_ridge = fit(semoptimizer, model_ml_ridge)
solution_ml = fit(semoptimizer, model_ml)
- # solution_ridge_id = fit(semoptimizer, model_ridge_id)
+ # solution_ridge_id = fit(model_ridge_id)
@test abs(solution_ridge.minimum - solution_ml.minimum) < 1
end
# test constant objective value
@testset "constant_objective_and_gradient" begin
- @test (objective!(model_constant, start_test) - 3.465) ≈
+ @test (objective!(model_ml_const, start_test) - 3.465) ≈
objective!(model_ml, start_test)
grad = similar(start_test)
grad2 = similar(start_test)
- gradient!(grad, model_constant, start_test)
+ gradient!(grad, model_ml_const, start_test)
gradient!(grad2, model_ml, start_test)
@test grad ≈ grad2
end
@@ -104,12 +89,9 @@ end
@testset "ml_solution_weighted" begin
solution_ml = fit(semoptimizer, model_ml)
solution_ml_weighted = fit(semoptimizer, model_ml_weighted)
- @test isapprox(solution(solution_ml), solution(solution_ml_weighted), rtol = 1e-3)
- @test isapprox(
- nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml),
- StructuralEquationModels.minimum(solution_ml_weighted),
- rtol = 1e-6,
- )
+ @test solution(solution_ml) ≈ solution(solution_ml_weighted) rtol = 1e-3
+ @test nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml) ≈
+ StructuralEquationModels.minimum(solution_ml_weighted) rtol = 1e-6
end
############################################################################################
@@ -118,7 +100,7 @@ end
@testset "fitmeasures/se_ml" begin
solution_ml = fit(semoptimizer, model_ml)
- test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3)
+ test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml]; atol = 1e-3)
update_se_hessian!(partable, solution_ml)
test_estimates(
@@ -128,18 +110,14 @@ end
col = :se,
lav_col = :se,
)
+
+ test_bootstrap(solution_ml)
+ smoketest_CI_z(solution_ml, partable)
end
@testset "fitmeasures/se_ls" begin
solution_ls = fit(semoptimizer, model_ls_sym)
- fm = fit_measures(solution_ls)
- test_fitmeasures(
- fm,
- solution_lav[:fitmeasures_ls];
- atol = 1e-3,
- fitmeasure_names = fitmeasure_names_ls,
- )
- @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll])
+ test_fitmeasures(solution_ls, solution_lav[:fitmeasures_ls]; atol = 1e-3)
@suppress update_se_hessian!(partable, solution_ls)
test_estimates(
@@ -149,6 +127,9 @@ end
col = :se,
lav_col = :se,
)
+
+ test_bootstrap(solution_ls; compare_bs = false)
+ smoketest_CI_z(solution_ls, partable)
end
############################################################################################
@@ -167,20 +148,9 @@ end
)
# set seed for simulation
Random.seed!(83472834)
- colnames = Symbol.(names(example_data("political_democracy")))
# simulate data
- model_ml_new = replace_observed(
- model_ml,
- data = rand(model_ml, params, 1_000_000),
- specification = spec,
- obs_colnames = colnames,
- )
- model_ml_sym_new = replace_observed(
- model_ml_sym,
- data = rand(model_ml_sym, params, 1_000_000),
- specification = spec,
- obs_colnames = colnames,
- )
+ model_ml_new = replace_observed(model_ml, rand(model_ml, params, 1_000_000))
+ model_ml_sym_new = replace_observed(model_ml_sym, rand(model_ml_sym, params, 1_000_000))
# fit models
sol_ml = solution(fit(semoptimizer, model_ml_new))
sol_ml_sym = solution(fit(semoptimizer, model_ml_sym_new))
@@ -197,23 +167,19 @@ if opt_engine == :Optim
using Optim, LineSearches
model_ls = Sem(
- specification = spec,
data = dat,
- implied = RAMSymbolic,
+ specification = spec,
+ observed = SemObservedData,
+ implied = RAMSymbolic(spec, vech = true, hessian = true),
loss = SemWLS,
- hessian = true,
- algorithm = Newton(;
- linesearch = BackTracking(order = 3),
- alphaguess = InitialHagerZhang(),
- ),
)
model_ml = Sem(
- specification = spec,
data = dat,
- implied = RAMSymbolic,
- hessian = true,
- algorithm = Newton(),
+ specification = spec,
+ observed = SemObservedData,
+ implied = RAMSymbolic(spec, hessian = true),
+ loss = SemML,
)
@testset "ml_hessians" begin
@@ -225,13 +191,23 @@ if opt_engine == :Optim
end
@testset "ml_solution_hessian" begin
- solution = fit(semoptimizer, model_ml)
+ solution = fit(SemOptimizer(engine = :Optim, algorithm = Newton()), model_ml)
+
update_estimate!(partable, solution)
test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-2)
end
@testset "ls_solution_hessian" begin
- solution = fit(semoptimizer, model_ls)
+ solution = fit(
+ SemOptimizer(
+ engine = :Optim,
+ algorithm = Newton(
+ linesearch = BackTracking(order = 3),
+ alphaguess = InitialHagerZhang(),
+ ),
+ ),
+ model_ls,
+ )
update_estimate!(partable, solution)
test_estimates(
partable,
@@ -252,6 +228,7 @@ model_ls = Sem(
specification = spec_mean,
data = dat,
implied = RAMSymbolic,
+ vech = true,
loss = SemWLS,
meanstructure = true,
)
@@ -263,7 +240,7 @@ model_ml_cov = Sem(
observed = SemObservedCovariance,
obs_cov = cov(Matrix(dat)),
obs_mean = vcat(mean(Matrix(dat), dims = 1)...),
- obs_colnames = Symbol.(names(dat)),
+ observed_vars = Symbol.(names(dat)),
meanstructure = true,
nsamples = 75,
)
@@ -275,33 +252,26 @@ model_ml_sym =
### test gradients
############################################################################################
-models = [model_ml, model_ml_cov, model_ls, model_ml_sym]
-model_names = ["ml", "ml_cov", "ls_sym", "ml_sym"]
+models = Dict(
+ "ml" => model_ml,
+ "ml_cov" => model_ml_cov,
+ "ls_sym" => model_ls,
+ "ml_sym" => model_ml_sym,
+)
-for (model, name) in zip(models, model_names)
- try
- @testset "$(name)_gradient_mean" begin
- test_gradient(model, start_test_mean; rtol = 1e-9)
- end
- catch
- end
+@testset "$(id)_gradient_mean" for (id, model) in pairs(models)
+ test_gradient(model, start_test_mean; rtol = 1e-9)
end
############################################################################################
### test solution
############################################################################################
-solution_names = Symbol.("parameter_estimates_" .* ["ml", "ml", "ls", "ml"] .* "_mean")
-
-for (model, name, solution_name) in zip(models, model_names, solution_names)
- try
- @testset "$(name)_solution_mean" begin
- solution = fit(semoptimizer, model)
- update_estimate!(partable_mean, solution)
- test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2)
- end
- catch
- end
+@testset "$(id)_solution_mean" for (id, model) in pairs(models)
+ solution = fit(semoptimizer, model, start_val = start_test_mean)
+ update_estimate!(partable_mean, solution)
+ sol_name = Symbol("parameter_estimates_", replace(id, r"_.+$" => ""), "_mean")
+ test_estimates(partable_mean, solution_lav[sol_name]; atol = 1e-2)
end
############################################################################################
@@ -310,11 +280,7 @@ end
@testset "fitmeasures/se_ml_mean" begin
solution_ml = fit(semoptimizer, model_ml)
- test_fitmeasures(
- fit_measures(solution_ml),
- solution_lav[:fitmeasures_ml_mean];
- atol = 0.002,
- )
+ test_fitmeasures(solution_ml, solution_lav[:fitmeasures_ml_mean]; atol = 0.002)
update_se_hessian!(partable_mean, solution_ml)
test_estimates(
@@ -324,18 +290,14 @@ end
col = :se,
lav_col = :se,
)
+
+ test_bootstrap(solution_ml)
+ smoketest_CI_z(solution_ml, partable_mean)
end
@testset "fitmeasures/se_ls_mean" begin
solution_ls = fit(semoptimizer, model_ls)
- fm = fit_measures(solution_ls)
- test_fitmeasures(
- fm,
- solution_lav[:fitmeasures_ls_mean];
- atol = 1e-3,
- fitmeasure_names = fitmeasure_names_ls,
- )
- @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll])
+ test_fitmeasures(solution_ls, solution_lav[:fitmeasures_ls_mean]; atol = 1e-3)
@suppress update_se_hessian!(partable_mean, solution_ls)
test_estimates(
@@ -345,6 +307,10 @@ end
col = :se,
lav_col = :se,
)
+
+ test_bootstrap(solution_ls, compare_bs = false)
+ # smoketest_bootstrap(solution_ls)
+ smoketest_CI_z(solution_ls, partable_mean)
end
############################################################################################
@@ -364,22 +330,9 @@ end
)
# set seed for simulation
Random.seed!(83472834)
- colnames = Symbol.(names(example_data("political_democracy")))
# simulate data
- model_ml_new = replace_observed(
- model_ml,
- data = rand(model_ml, params, 1_000_000),
- specification = spec,
- obs_colnames = colnames,
- meanstructure = true,
- )
- model_ml_sym_new = replace_observed(
- model_ml_sym,
- data = rand(model_ml_sym, params, 1_000_000),
- specification = spec,
- obs_colnames = colnames,
- meanstructure = true,
- )
+ model_ml_new = replace_observed(model_ml, rand(model_ml, params, 1_000_000))
+ model_ml_sym_new = replace_observed(model_ml_sym, rand(model_ml_sym, params, 1_000_000))
# fit models
sol_ml = solution(fit(semoptimizer, model_ml_new))
sol_ml_sym = solution(fit(semoptimizer, model_ml_sym_new))
@@ -410,6 +363,18 @@ model_ml_sym = Sem(
meanstructure = true,
)
+if !ismissing(spec_varonly)
+ model_varonly = Sem(
+ specification = spec_varonly,
+ data = dat_missing,
+ observed = SemObservedMissing,
+ loss = SemFIML,
+ meanstructure = true,
+ )
+else
+ model_varonly = nothing
+end
+
############################################################################################
### test gradients
############################################################################################
@@ -433,7 +398,7 @@ end
end
@testset "fiml_solution_symbolic" begin
- solution = fit(semoptimizer, model_ml_sym)
+ solution = fit(semoptimizer, model_ml_sym, start_val = start_test_mean)
update_estimate!(partable_mean, solution)
test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2)
end
@@ -444,9 +409,14 @@ end
@testset "fitmeasures/se_fiml" begin
solution_ml = fit(semoptimizer, model_ml)
+ solution_varonly =
+ !isnothing(model_varonly) ? fit(semoptimizer, model_varonly) : nothing
test_fitmeasures(
- fit_measures(solution_ml),
+ solution_ml,
solution_lav[:fitmeasures_fiml];
+ fitted_baseline = solution_varonly,
+ fitmeasures = !isnothing(solution_varonly) ? SEM.DEFAULT_FIT_MEASURES :
+ filter(!=(CFI), SEM.DEFAULT_FIT_MEASURES),
atol = 1e-3,
)
@@ -458,4 +428,8 @@ end
col = :se,
lav_col = :se,
)
+
+ # test_bootstrap(solution_ml) # too much compute
+ smoketest_bootstrap(solution_ml)
+ smoketest_CI_z(solution_ml, partable_mean)
end
diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl
index ad06e0fcd..8c8d6c36e 100644
--- a/test/examples/political_democracy/political_democracy.jl
+++ b/test/examples/political_democracy/political_democracy.jl
@@ -1,4 +1,8 @@
using StructuralEquationModels, Test, Suppressor, FiniteDiff
+using Statistics: cov, mean, var
+using Random, NLopt
+
+Random.seed!(464577)
SEM = StructuralEquationModels
@@ -102,6 +106,8 @@ partable_mean = ParameterTable(spec_mean)
@test SEM.param_labels(partable_mean) == SEM.param_labels(spec_mean)
+spec_varonly = missing
+
start_test = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3)]
start_test_mean =
[fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3); fill(0.1, 7)]
@@ -142,6 +148,7 @@ spec_mean = ParameterTable(spec_mean)
partable = spec
partable_mean = spec_mean
+spec_varonly = missing
opt_engine = :Optim
@testset "RAMMatrices → ParameterTable | constructor | Optim" begin
@@ -226,6 +233,15 @@ sort_vars!(spec_mean)
partable_mean = spec_mean
+# varonly model for CFI
+graph_varonly = @StenoGraph begin
+ _(observed_vars) ↔ _(observed_vars)
+ Symbol(1) → _(observed_vars)
+end
+
+spec_varonly =
+ ParameterTable(graph_varonly, latent_vars = latent_vars, observed_vars = observed_vars)
+
start_test = [fill(0.5, 8); fill(0.05, 3); fill(1.0, 11); fill(0.05, 9)]
start_test_mean =
[fill(0.5, 8); fill(0.05, 3); fill(1.0, 11); fill(0.05, 3); fill(0.05, 13)]
diff --git a/test/examples/proximal/l0.jl b/test/examples/proximal/l0.jl
index 374f8e58a..8542ac458 100644
--- a/test/examples/proximal/l0.jl
+++ b/test/examples/proximal/l0.jl
@@ -1,5 +1,3 @@
-using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators
-
# load data
dat = example_data("political_democracy")
@@ -47,7 +45,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
fit_prox = fit(model_prox, engine = :Proximal, operator_g = prox_operator)
@testset "l0 | solution_unregularized" begin
- @test fit_prox.optimization_result.result[:iterations] < 1000
+ @test n_iterations(fit_prox.optimization_result) < 1000
@test maximum(abs.(solution(sem_fit) - solution(fit_prox))) < 0.002
end
@@ -59,9 +57,10 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
fit_prox = fit(model_prox, engine = :Proximal, operator_g = prox_operator)
@testset "l0 | solution_regularized" begin
- @test fit_prox.optimization_result.result[:iterations] < 1000
+ @test n_iterations(fit_prox.optimization_result) < 1000
@test solution(fit_prox)[31] == 0.0
@test abs(
- StructuralEquationModels.minimum(fit_prox) - StructuralEquationModels.minimum(sem_fit),
+ StructuralEquationModels.minimum(fit_prox) -
+ StructuralEquationModels.minimum(sem_fit),
) < 1.0
end
diff --git a/test/examples/proximal/lasso.jl b/test/examples/proximal/lasso.jl
index beb5cf529..9138f6884 100644
--- a/test/examples/proximal/lasso.jl
+++ b/test/examples/proximal/lasso.jl
@@ -1,5 +1,3 @@
-using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators
-
# load data
dat = example_data("political_democracy")
@@ -45,7 +43,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
fit_prox = fit(model_prox, engine = :Proximal, operator_g = NormL1(λ))
@testset "lasso | solution_unregularized" begin
- @test fit_prox.optimization_result.result[:iterations] < 1000
+ @test n_iterations(fit_prox.optimization_result) < 1000
@test maximum(abs.(solution(sem_fit) - solution(fit_prox))) < 0.002
end
@@ -57,7 +55,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
fit_prox = fit(model_prox, engine = :Proximal, operator_g = NormL1(λ))
@testset "lasso | solution_regularized" begin
- @test fit_prox.optimization_result.result[:iterations] < 1000
+ @test n_iterations(fit_prox.optimization_result) < 1000
@test all(solution(fit_prox)[16:20] .< solution(sem_fit)[16:20])
@test StructuralEquationModels.minimum(fit_prox) -
StructuralEquationModels.minimum(sem_fit) < 0.03
diff --git a/test/examples/proximal/proximal.jl b/test/examples/proximal/proximal.jl
index 40e72a1ef..84a9162cb 100644
--- a/test/examples/proximal/proximal.jl
+++ b/test/examples/proximal/proximal.jl
@@ -1,3 +1,5 @@
+using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators, Suppressor
+
@testset "Ridge" begin
include("ridge.jl")
end
diff --git a/test/examples/proximal/ridge.jl b/test/examples/proximal/ridge.jl
index fd7ae113d..61b7fa12a 100644
--- a/test/examples/proximal/ridge.jl
+++ b/test/examples/proximal/ridge.jl
@@ -1,5 +1,3 @@
-using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators, Suppressor
-
# load data
dat = example_data("political_democracy")
diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl
index a7b4cec9a..ebaaae83b 100644
--- a/test/examples/recover_parameters/recover_parameters_twofact.jl
+++ b/test/examples/recover_parameters/recover_parameters_twofact.jl
@@ -1,5 +1,7 @@
using StructuralEquationModels, Distributions, Random, Optim, LineSearches
+SEM = StructuralEquationModels
+
include(
joinpath(
chop(dirname(pathof(StructuralEquationModels)), tail = 3),
@@ -7,7 +9,7 @@ include(
),
)
-x = Symbol.("x", 1:13)
+pars = Symbol.("x", 1:13)
S = [
:x1 0 0 0 0 0 0 0
@@ -40,7 +42,7 @@ A = [
0 0 0 0 0 0 0 0
]
-ram_matrices = RAMMatrices(; A = A, S = S, F = F, param_labels = x, vars = nothing)
+ram_matrices = RAMMatrices(; A = A, S = S, F = F, param_labels = pars, vars = nothing)
true_val = [
repeat([1], 8)
@@ -53,26 +55,26 @@ start = [
repeat([0.5], 4)
]
-implied_ml = RAMSymbolic(; specification = ram_matrices, start_val = start)
+implied_sym = RAMSymbolic(ram_matrices)
-implied_ml.Σ_function(implied_ml.Σ, true_val)
+implied_sym.Σ_eval!(implied_sym.Σ, true_val)
-true_dist = MultivariateNormal(implied_ml.Σ)
+true_dist = MultivariateNormal(implied_sym.Σ)
Random.seed!(1234)
-x = transpose(rand(true_dist, 100_000))
-semobserved = SemObservedData(data = x, specification = nothing)
+x = permutedims(rand(true_dist, 10^5), (2, 1))
+
+observed = SemObservedData(data = x, specification = ram_matrices)
-loss_ml = SemLoss(SemML(; observed = semobserved, nparams = length(start)))
+model_ml = Sem(SemML(observed, implied_sym))
-model_ml = Sem(semobserved, implied_ml, loss_ml)
objective!(model_ml, true_val)
-optimizer = SemOptimizerOptim(
+optimizer = SemOptimizer(
BFGS(; linesearch = BackTracking(order = 3), alphaguess = InitialHagerZhang()),# m = 100),
Optim.Options(; f_reltol = 1e-10, x_abstol = 1.5e-8),
)
-solution_ml = fit(optimizer, model_ml)
+solution_ml = fit(optimizer, model_ml, start_val = start)
-@test true_val ≈ solution(solution_ml) atol = 0.05
+@test solution(solution_ml) ≈ true_val atol = 0.05
diff --git a/test/unit_tests/StatsAPI.jl b/test/unit_tests/StatsAPI.jl
index 8648fc363..5907ee7b5 100644
--- a/test/unit_tests/StatsAPI.jl
+++ b/test/unit_tests/StatsAPI.jl
@@ -5,10 +5,7 @@ end
partable = ParameterTable(graph, observed_vars = [:a, :b], latent_vars = Symbol[])
update_partable!(partable, :estimate, param_labels(partable), [3.1415])
data = randn(100, 2)
-model = Sem(
- specification = partable,
- data = data
-)
+model = Sem(specification = partable, data = data)
model_fit = fit(model)
@testset "params" begin
@@ -25,5 +22,5 @@ end
end
@testset "coeftable" begin
- @test_throws "StructuralEquationModels does not support" coeftable(model)
-end
\ No newline at end of file
+ @test_throws "StructuralEquationModels does not support" coeftable(model)
+end
diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl
index 183b067f5..7f024e110 100644
--- a/test/unit_tests/data_input_formats.jl
+++ b/test/unit_tests/data_input_formats.jl
@@ -93,8 +93,7 @@ function test_observed(
@test @inferred(obs_mean(observed)) == dat_mean
end
else
- # FIXME @inferred is broken for EM cov/mean since it may return nothing if EM was not run
- @test @inferred(obs_mean(observed)) isa AbstractVector{Float64} broken = true # EM-based means
+ @test @inferred(obs_mean(observed)) isa AbstractVector{Float64} # EM-based means
end
else
@test @inferred(obs_mean(observed)) === nothing skip = true
diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl
index 2bf5dedaf..ee9b07532 100644
--- a/test/unit_tests/model.jl
+++ b/test/unit_tests/model.jl
@@ -1,5 +1,7 @@
using StructuralEquationModels, Test, Statistics
+const SEM = StructuralEquationModels
+
dat = example_data("political_democracy")
dat_missing = example_data("political_democracy_missing")[:, names(dat)]
@@ -25,7 +27,6 @@ graph = @StenoGraph begin
y8 ↔ y4 + y6
end
-
ram_matrices =
RAMMatrices(ParameterTable(graph, observed_vars = obs_vars, latent_vars = lat_vars))
@@ -47,14 +48,17 @@ function test_params_api(semobj, spec::SemSpecification)
@test @inferred(param_labels(semobj)) == param_labels(spec)
end
-@testset "Sem(implied=$impliedtype, loss=$losstype)" for impliedtype in (RAM, RAMSymbolic),
- losstype in (SemML, SemWLS)
-
+@testset "Sem(implied=$impliedtype, loss=$losstype)" for (impliedtype, losstype) in [
+ (RAM, SemML),
+ (RAMSymbolic, SemML),
+ (RAMSymbolic, SemWLS),
+]
model = Sem(
specification = ram_matrices,
observed = obs,
implied = impliedtype,
loss = losstype,
+ vech = losstype <: SemWLS && impliedtype <: RAMSymbolic,
)
@test model isa Sem
@@ -67,9 +71,152 @@ end
test_vars_api(implied(model), ram_matrices)
test_params_api(implied(model), ram_matrices)
- @test @inferred(loss(model)) isa SemLoss
- semloss = loss(model).functions[1]
- @test semloss isa losstype
+ @test @inferred(sem_term(model)) isa SemLoss
+ @test sem_term(model) isa losstype
@test @inferred(nsamples(model)) == nsamples(obs)
end
+
+@testset "replace_observed() preserves WLS and approx_hessian=$(approx_hessian) state through finite-diff wrappers" for approx_hessian in
+ (
+ false,
+ true,
+)
+ expected_hessianeval = approx_hessian ? SEM.ApproxHessian : SEM.ExactHessian
+
+ model = Sem(
+ specification = ram_matrices,
+ observed = obs,
+ implied = RAMSymbolic,
+ loss = SemWLS,
+ vech = true,
+ approximate_hessian = approx_hessian,
+ )
+ wls_loss = sem_term(model)
+ findiff_model = Sem(SEM.FiniteDiffWrapper(wls_loss))
+
+ new_data = randn(nsamples(obs), nobserved_vars(obs))
+
+ findiff_model_oldstate =
+ replace_observed(findiff_model, new_data; recompute_observed_state = false)
+ findiff_model_newstate =
+ replace_observed(findiff_model, new_data; recompute_observed_state = true)
+
+ loss_orig = SEM._unwrap(sem_term(findiff_model))
+ loss_oldstate = SEM._unwrap(sem_term(findiff_model_oldstate))
+ loss_newstate = SEM._unwrap(sem_term(findiff_model_newstate))
+
+ @test loss_orig isa SemWLS
+ @test loss_oldstate isa SemWLS
+ @test loss_newstate isa SemWLS
+ @test loss_orig !== loss_oldstate
+ @test loss_orig !== loss_newstate
+ @test SEM.HessianEval(loss_orig) === expected_hessianeval
+ @test SEM.HessianEval(loss_oldstate) === expected_hessianeval
+ @test SEM.HessianEval(loss_newstate) === expected_hessianeval
+ @test loss_oldstate.V === loss_orig.V
+ @test loss_newstate.V !== loss_orig.V
+ @test observed_vars(loss_oldstate) == observed_vars(loss_orig)
+end
+
+@testset "replace_observed() shares implied unless model is deepcopied and approx_hessian=$(approx_hessian)" for approx_hessian in
+ (
+ false,
+ true,
+)
+ expected_hessianeval = approx_hessian ? SEM.ApproxHessian : SEM.ExactHessian
+
+ model = Sem(
+ specification = ram_matrices,
+ observed = obs,
+ implied = RAMSymbolic,
+ loss = SemML,
+ approximate_hessian = approx_hessian,
+ )
+
+ data_new = randn(nsamples(obs), nobserved_vars(obs))
+
+ model_new = replace_observed(model, data_new)
+ model_deepcopy = replace_observed(deepcopy(model), data_new)
+
+ loss_orig = sem_term(model)
+ loss_new = sem_term(model_new)
+ loss_deepcopy = sem_term(model_deepcopy)
+
+ @test SEM.HessianEval(loss_orig) === expected_hessianeval
+ @test SEM.HessianEval(loss_new) === expected_hessianeval
+ @test SEM.HessianEval(loss_deepcopy) === expected_hessianeval
+ @test implied(loss_new) === implied(loss_orig)
+ @test implied(loss_deepcopy) !== implied(loss_orig)
+end
+
+@testset "replace_observed() preserves Sem container defaults" begin
+ data_g1 = dat[1:40, :]
+ data_g2 = dat[41:end, :]
+
+ sem_multigroup = Sem(
+ :g1 => SemML(
+ SemObservedData(specification = ram_matrices, data = data_g1),
+ RAM(ram_matrices),
+ ),
+ :g2 => SemML(
+ SemObservedData(specification = ram_matrices, data = data_g2),
+ RAM(ram_matrices),
+ );
+ default_sem_weights = :one,
+ )
+
+ sem_newobs = replace_observed(
+ sem_multigroup,
+ Dict(:g1 => randn(10, nobserved_vars(obs)), :g2 => randn(25, nobserved_vars(obs))),
+ )
+
+ @test all(isnothing, map(SEM.weight, SEM.loss_terms(sem_multigroup)))
+ @test all(isnothing, map(SEM.weight, SEM.loss_terms(sem_newobs)))
+ @test params(sem_newobs) == params(sem_multigroup)
+ @test params(sem_newobs) !== params(sem_multigroup)
+end
+
+@testset "Sem(...; semterm_column=...) splits ensemble data by group" begin
+ dat_grouped = copy(dat[:, [:x1, :x2]])
+ n_g1 = size(dat_grouped, 1) ÷ 2
+ dat_grouped.group = [fill(:g1, n_g1); fill(:g2, size(dat_grouped, 1) - n_g1)]
+
+ group_graph = @StenoGraph begin
+ f1 → fixed(1.0, 1.0) * x1 + label(:λ₂, :λ₂) * x2
+ _(Symbol[:x1, :x2]) ↔ _(Symbol[:x1, :x2])
+ _(Symbol[:f1]) ↔ _(Symbol[:f1])
+ end
+
+ grouped_partable = EnsembleParameterTable(
+ group_graph;
+ observed_vars = [:x1, :x2],
+ latent_vars = [:f1],
+ groups = [:g1, :g2],
+ )
+
+ grouped_model = Sem(
+ specification = grouped_partable,
+ data = dat_grouped,
+ semterm_column = :group,
+ observed = SemObservedData,
+ implied = RAM,
+ loss = SemML,
+ )
+
+ term_g1 = only(filter(term -> SEM.id(term) == :g1, SEM.loss_terms(grouped_model)))
+ term_g2 = only(filter(term -> SEM.id(term) == :g2, SEM.loss_terms(grouped_model)))
+
+ @test nsamples(observed(term_g1)) == n_g1
+ @test nsamples(observed(term_g2)) == size(dat_grouped, 1) - n_g1
+ @test nsamples(grouped_model) == size(dat_grouped, 1)
+end
+
+@testset "ImpliedEmpty" begin
+ # positional `specification`, consistent with RAM/RAMSymbolic
+ @test ImpliedEmpty(ram_matrices) isa ImpliedEmpty
+
+ # implied can be selected by type through the outer `Sem` constructor (like `implied = RAM`)
+ model = Sem(specification = ram_matrices, data = dat, implied = ImpliedEmpty)
+ @test implied(model) isa ImpliedEmpty
+end
diff --git a/test/unit_tests/unit_tests.jl b/test/unit_tests/unit_tests.jl
index 7189addd4..4d7dad7cf 100644
--- a/test/unit_tests/unit_tests.jl
+++ b/test/unit_tests/unit_tests.jl
@@ -7,7 +7,7 @@ available_tests = Dict(
"data_input_formats" => "SemObserved",
"specification" => "SemSpecification",
"model" => "Sem model",
- "StatsAPI" => "StatsAPI"
+ "StatsAPI" => "StatsAPI",
)
# Determine which tests to run based on command-line arguments
diff --git a/test/unit_tests/unit_tests_interactive.jl b/test/unit_tests/unit_tests_interactive.jl
index cf082fa60..10a384403 100644
--- a/test/unit_tests/unit_tests_interactive.jl
+++ b/test/unit_tests/unit_tests_interactive.jl
@@ -7,4 +7,4 @@ try
catch e
@warn "Error initializing Test Env" exception=(e, catch_backtrace())
end
-include("unit_tests.jl")
\ No newline at end of file
+include("unit_tests.jl")