From 3d56cc76c75f5c8939adb256cdefb4469f4c2f4c Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 3 Apr 2026 17:22:42 -0700 Subject: [PATCH 01/11] feat: started adding support for multiple datatypes --- src/ASDF.jl | 70 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index 17d031c..e3379f6 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -278,6 +278,8 @@ const host_byteorder = reinterpret(UInt8, UInt16[1])[1] == 1 ? Byteorder_little """ Careful, there is also `Base.DataType`, which is a different type. + +[Specification](https://www.asdf-format.org/projects/asdf-standard/en/latest/generated/stsci.edu/asdf/core/datatype-1.0.0.html). """ @enum Datatype begin Datatype_bool8 @@ -348,13 +350,50 @@ Datatype(type::Type) = type_datatype_dict[type] ################################################################################ +""" +The other variant of [Datatype](@ref). +""" +struct StructuredField + name::String + datatype::Datatype + byteorder::Byteorder + #shape::Vector{Int64} +end + +struct StructuredDatatype + fields::Vector{StructuredField} +end + +function Base.Type(sd::StructuredDatatype) + names = Tuple(Symbol(f.name) for f in sd.fields) + types = Tuple{(Type(f.datatype) for f in sd.fields)...} + return NamedTuple{names, types} +end + +# For parsing raw YAML +function parse_asdf_datatype(val::AbstractString)::Union{Datatype, StructuredDatatype} + return Datatype(val) +end + +function parse_asdf_datatype(val::AbstractVector)::Union{Datatype, StructuredDatatype} + fields = map(val) do d + name = d["name"]::String + dt = Datatype(d["datatype"]::String) + bo = haskey(d, "byteorder") ? Byteorder(d["byteorder"]::String) : host_byteorder + StructuredField(name, dt, bo) + end + return StructuredDatatype(fields) +end + +################################################################################ + struct NDArray lazy_block_headers::LazyBlockHeaders source::Union{Nothing,Int64,AbstractString} data::Union{Nothing,AbstractArray} shape::Vector{Int64} - datatype::Datatype + datatype::Union{Datatype,StructuredDatatype} byteorder::Byteorder offset::Int64 strides::Vector{Int64} # stored in ASDF (Python/C) order, not in Julia (Fortran) order @@ -365,7 +404,7 @@ struct NDArray source::Union{Nothing,Int64,AbstractString}, data::Union{Nothing,AbstractArray}, shape::Vector{Int64}, - datatype::Datatype, + datatype::Union{Datatype,StructuredDatatype}, byteorder::Byteorder, offset::Int64, strides::Vector{Int64}, @@ -405,7 +444,7 @@ function NDArray( source::Union{Nothing,Integer}, data::Union{Nothing,AbstractArray}, shape::AbstractVector{<:Integer}, - datatype::Union{Datatype,AbstractString}, + datatype::Union{Datatype,StructuredDatatype,AbstractString,AbstractVector}, byteorder::Union{Nothing,Byteorder,AbstractString}, offset::Union{Nothing,Integer}=0, strides::Union{Nothing,<:AbstractVector{<:Integer}}=nothing, @@ -413,8 +452,8 @@ function NDArray( if source isa Integer source = Int64(source) end - if datatype isa AbstractString - datatype = Datatype(datatype) + if datatype isa AbstractString || datatype isa AbstractVector + datatype = parse_asdf_datatype(datatype) end if data !== nothing # Convert arrays of arrays into multi-dimensional arrays @@ -447,7 +486,7 @@ function make_construct_yaml_ndarray(block_headers::LazyBlockHeaders) source = get(mapping, "source", nothing)::Union{Nothing,Integer} data = get(mapping, "data", nothing)::Union{Nothing,AbstractVector} shape = mapping["shape"]::AbstractVector{<:Integer} - datatype = mapping["datatype"]::AbstractString + datatype = mapping["datatype"] # No annotation needed, `parse_asdf_datatype` handles dispatch byteorder = get(mapping, "byteorder", nothing)::Union{Nothing,AbstractString} offset = get(mapping, "offset", nothing)::Union{Nothing,Integer} strides = get(mapping, "strides", nothing)::Union{Nothing,AbstractVector{<:Integer}} @@ -475,13 +514,26 @@ function Base.getindex(ndarray::NDArray) strides = (1, reverse(ndarray.strides)...) data = StridedView(data, Int.(shape), Int.(strides), Int(ndarray.offset)) # Impose datatype - data = reinterpret(Type(ndarray.datatype), data) + NT = Type(ndarray.datatype) + data = reinterpret(NT, data) # Remove the new dimension again data = reshape(data, shape[2:end]) # Correct byteorder if necessary. # Do this after imposing the datatype since byteorder depends on the datatype. - if ndarray.byteorder != host_byteorder - map!(bswap, data, data) + if ndarray.datatype isa StructuredDatatype + needs_swap = any(f.byteorder != host_byteorder for f in ndarray.datatype.fields) + if needs_swap + data = map(data) do elem + swapped = map(ndarray.datatype.fields, Tuple(elem)) do field, val + field.byteorder != host_byteorder ? bswap(val) : val + end + NT(swapped) + end + end + else + if ndarray.byteorder != host_byteorder + map!(bswap, data, data) + end end else # Caught in the constructor for `NDArray`. This branch would imply that From 333f2bfd3b2d21587dbf26b777804b202e58bc22 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 3 Apr 2026 18:12:39 -0700 Subject: [PATCH 02/11] feat: start adding ascii and ucs support --- src/ASDF.jl | 65 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index e3379f6..e0cf98a 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -348,6 +348,32 @@ Base.show(io::IO, datatype::Datatype) = show(io, string(datatype)) Base.Type(datatype::Datatype) = datatype_type_dict[datatype] Datatype(type::Type) = type_datatype_dict[type] +# For parsing raw YAML +function parse_asdf_datatype(val::AbstractString) + return Datatype(val) +end + +function parse_asdf_datatype(val::AbstractVector) + # 2-element [kind, length] form: ["ascii", 4] or ["ucs4", 4] + if length(val) == 2 && val[1] isa AbstractString && val[2] isa Integer + n = Int(val[2]) + if val[1] == "ascii" + return AsciiDatatype(n) + elseif val[1] == "ucs4" + return Ucs4Datatype(n) + else + error("Unknown string datatype kind: $(val[1])") + end + end + fields = map(val) do d + name = d["name"]::String + dt = parse_asdf_datatype(d["datatype"]) + bo = haskey(d, "byteorder") ? Byteorder(d["byteorder"]::String) : host_byteorder + StructuredField(name, dt, bo) + end + return StructuredDatatype(fields) +end + ################################################################################ """ @@ -355,7 +381,7 @@ The other variant of [Datatype](@ref). """ struct StructuredField name::String - datatype::Datatype + datatype::{Datatype, AsciiDatatype, Ucs4Datatype} byteorder::Byteorder #shape::Vector{Int64} end @@ -370,21 +396,19 @@ function Base.Type(sd::StructuredDatatype) return NamedTuple{names, types} end -# For parsing raw YAML -function parse_asdf_datatype(val::AbstractString)::Union{Datatype, StructuredDatatype} - return Datatype(val) +################################################################################ + +struct AsciiDatatype + length::Int # Bytes (1 per char) end -function parse_asdf_datatype(val::AbstractVector)::Union{Datatype, StructuredDatatype} - fields = map(val) do d - name = d["name"]::String - dt = Datatype(d["datatype"]::String) - bo = haskey(d, "byteorder") ? Byteorder(d["byteorder"]::String) : host_byteorder - StructuredField(name, dt, bo) - end - return StructuredDatatype(fields) +struct Ucs4Datatype + length::Int # Characters (4 bytes each) end +Base.Type(dt::AsciiDatatype) = NTuple{dt.length, UInt8} +Base.Type(dt::Ucs4Datatype) = NTuple{dt.length, UInt32} + ################################################################################ struct NDArray @@ -393,7 +417,7 @@ struct NDArray source::Union{Nothing,Int64,AbstractString} data::Union{Nothing,AbstractArray} shape::Vector{Int64} - datatype::Union{Datatype,StructuredDatatype} + datatype::Union{Datatype,StructuredDatatype,AsciiDatatype,Ucs4Datatype} byteorder::Byteorder offset::Int64 strides::Vector{Int64} # stored in ASDF (Python/C) order, not in Julia (Fortran) order @@ -404,7 +428,7 @@ struct NDArray source::Union{Nothing,Int64,AbstractString}, data::Union{Nothing,AbstractArray}, shape::Vector{Int64}, - datatype::Union{Datatype,StructuredDatatype}, + datatype::Union{Datatype,StructuredDatatype,AsciiDatatype,Ucs4Datatype} byteorder::Byteorder, offset::Int64, strides::Vector{Int64}, @@ -444,7 +468,7 @@ function NDArray( source::Union{Nothing,Integer}, data::Union{Nothing,AbstractArray}, shape::AbstractVector{<:Integer}, - datatype::Union{Datatype,StructuredDatatype,AbstractString,AbstractVector}, + datatype::Union{Datatype,StructuredDatatype,AsciiDatatype,Ucs4Datatype,AbstractString,AbstractVector}, byteorder::Union{Nothing,Byteorder,AbstractString}, offset::Union{Nothing,Integer}=0, strides::Union{Nothing,<:AbstractVector{<:Integer}}=nothing, @@ -520,7 +544,11 @@ function Base.getindex(ndarray::NDArray) data = reshape(data, shape[2:end]) # Correct byteorder if necessary. # Do this after imposing the datatype since byteorder depends on the datatype. - if ndarray.datatype isa StructuredDatatype + if ndarray.datatype isa Datatype + if ndarray.byteorder != host_byteorder + map!(bswap, data, data) + end + elseif ndarray.datatype isa StructuredDatatype needs_swap = any(f.byteorder != host_byteorder for f in ndarray.datatype.fields) if needs_swap data = map(data) do elem @@ -530,11 +558,16 @@ function Base.getindex(ndarray::NDArray) NT(swapped) end end + elseif ndarray.datatype isa Ucs4Datatype + if ndarray.byteorder != host_byteorder + data = map(elem -> map(bswap, elem), data) + end else if ndarray.byteorder != host_byteorder map!(bswap, data, data) end end + # AsciiDatatype: no byteswap needed (single bytes have no endianness) else # Caught in the constructor for `NDArray`. This branch would imply that # `ndarray` is in invalid state; neither `source` nor `data` is given. From fbcdadb02a6b831f7e3e1758a7b8df14e6d9f03a Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 3 Apr 2026 22:23:11 -0700 Subject: [PATCH 03/11] feat: ascii + ucs support up --- src/ASDF.jl | 71 +++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index e0cf98a..e24f401 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -348,32 +348,19 @@ Base.show(io::IO, datatype::Datatype) = show(io, string(datatype)) Base.Type(datatype::Datatype) = datatype_type_dict[datatype] Datatype(type::Type) = type_datatype_dict[type] -# For parsing raw YAML -function parse_asdf_datatype(val::AbstractString) - return Datatype(val) +################################################################################ + +struct AsciiDatatype + length::Int # Bytes (1 per char) end -function parse_asdf_datatype(val::AbstractVector) - # 2-element [kind, length] form: ["ascii", 4] or ["ucs4", 4] - if length(val) == 2 && val[1] isa AbstractString && val[2] isa Integer - n = Int(val[2]) - if val[1] == "ascii" - return AsciiDatatype(n) - elseif val[1] == "ucs4" - return Ucs4Datatype(n) - else - error("Unknown string datatype kind: $(val[1])") - end - end - fields = map(val) do d - name = d["name"]::String - dt = parse_asdf_datatype(d["datatype"]) - bo = haskey(d, "byteorder") ? Byteorder(d["byteorder"]::String) : host_byteorder - StructuredField(name, dt, bo) - end - return StructuredDatatype(fields) +struct Ucs4Datatype + length::Int # Characters (4 bytes each) end +Base.Type(dt::AsciiDatatype) = NTuple{dt.length, UInt8} +Base.Type(dt::Ucs4Datatype) = NTuple{dt.length, UInt32} + ################################################################################ """ @@ -381,7 +368,7 @@ The other variant of [Datatype](@ref). """ struct StructuredField name::String - datatype::{Datatype, AsciiDatatype, Ucs4Datatype} + datatype::Union{Datatype, AsciiDatatype, Ucs4Datatype} byteorder::Byteorder #shape::Vector{Int64} end @@ -398,17 +385,32 @@ end ################################################################################ -struct AsciiDatatype - length::Int # Bytes (1 per char) +# For parsing raw YAML +function parse_asdf_datatype(val::AbstractString) + return Datatype(val) end -struct Ucs4Datatype - length::Int # Characters (4 bytes each) +function parse_asdf_datatype(val::AbstractVector) + # 2-element [kind, length] form: ["ascii", 4] or ["ucs4", 4] + if length(val) == 2 && val[1] isa AbstractString && val[2] isa Integer + n = Int(val[2]) + if val[1] == "ascii" + return AsciiDatatype(n) + elseif val[1] == "ucs4" + return Ucs4Datatype(n) + else + error("Unknown string datatype kind: $(val[1])") + end + end + fields = map(val) do d + name = d["name"]::String + dt = parse_asdf_datatype(d["datatype"]) + bo = haskey(d, "byteorder") ? Byteorder(d["byteorder"]::String) : host_byteorder + StructuredField(name, dt, bo) + end + return StructuredDatatype(fields) end -Base.Type(dt::AsciiDatatype) = NTuple{dt.length, UInt8} -Base.Type(dt::Ucs4Datatype) = NTuple{dt.length, UInt32} - ################################################################################ struct NDArray @@ -428,7 +430,7 @@ struct NDArray source::Union{Nothing,Int64,AbstractString}, data::Union{Nothing,AbstractArray}, shape::Vector{Int64}, - datatype::Union{Datatype,StructuredDatatype,AsciiDatatype,Ucs4Datatype} + datatype::Union{Datatype,StructuredDatatype,AsciiDatatype,Ucs4Datatype}, byteorder::Byteorder, offset::Int64, strides::Vector{Int64}, @@ -558,16 +560,15 @@ function Base.getindex(ndarray::NDArray) NT(swapped) end end + elseif ndarray.datatype isa AsciiDatatype + # AsciiDatatype: no byteswap needed (single bytes have no endianness) elseif ndarray.datatype isa Ucs4Datatype if ndarray.byteorder != host_byteorder data = map(elem -> map(bswap, elem), data) end else - if ndarray.byteorder != host_byteorder - map!(bswap, data, data) - end + error("Unhandled datatype type: $(typeof(ndarray.datatype))") end - # AsciiDatatype: no byteswap needed (single bytes have no endianness) else # Caught in the constructor for `NDArray`. This branch would imply that # `ndarray` is in invalid state; neither `source` nor `data` is given. From d23567ae4c9d27ad481404e66823dcae66b12559 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 3 Apr 2026 22:38:16 -0700 Subject: [PATCH 04/11] feat: finish updates to getindex and ChunkedNDArray --- src/ASDF.jl | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index e24f401..df2372d 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -551,11 +551,20 @@ function Base.getindex(ndarray::NDArray) map!(bswap, data, data) end elseif ndarray.datatype isa StructuredDatatype - needs_swap = any(f.byteorder != host_byteorder for f in ndarray.datatype.fields) + needs_swap = any( + !(f.datatype isa AsciiDatatype) && f.byteorder != host_byteorder + for f in ndarray.datatype.fields + ) if needs_swap data = map(data) do elem swapped = map(ndarray.datatype.fields, Tuple(elem)) do field, val - field.byteorder != host_byteorder ? bswap(val) : val + if field.datatype isa AsciiDatatype + val # bytes have no endianness + elseif field.byteorder != host_byteorder + bswap(val) + else + val + end end NT(swapped) end @@ -618,7 +627,7 @@ end struct ChunkedNDArray shape::Vector{Int64} - datatype::Datatype + datatype::Union{Datatype, StructuredDatatype, AsciiDatatype, Ucs4Datatype} chunks::AbstractVector{NDArrayChunk} function ChunkedNDArray(shape::Vector{Int64}, datatype::Datatype, chunks::Vector{NDArrayChunk}) @@ -645,9 +654,9 @@ struct ChunkedNDArray end function ChunkedNDArray( - shape::AbstractVector{<:Integer}, datatype::Union{Datatype,AbstractString}, chunks::AbstractVector{NDArrayChunk} + shape::AbstractVector{<:Integer}, datatype::Union{Datatype,StructuredDatatype,AsciiDatatype,Ucs4Datatype,AbstractString,AbstractVector}, chunks::AbstractVector{NDArrayChunk} ) - if datatype isa AbstractString + if datatype isa AbstractString || datatype isa AbstractVector datatype = Datatype(datatype) end return ChunkedNDArray(Vector{Int64}(shape), datatype, chunks) @@ -657,7 +666,7 @@ function make_construct_yaml_chunked_ndarray(block_headers::LazyBlockHeaders) function construct_yaml_chunked_ndarray(constructor::YAML.Constructor, node::YAML.Node) mapping = YAML.construct_mapping(constructor, node) shape = mapping["shape"]::AbstractVector{<:Integer} - datatype = mapping["datatype"]::AbstractString + datatype = mapping["datatype"] chunks = mapping["chunks"]::AbstractVector{NDArrayChunk} return ChunkedNDArray(shape, datatype, chunks) end From 1e624e1b68437e6ce2802834514edb22de6c4c24 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 3 Apr 2026 23:27:17 -0700 Subject: [PATCH 05/11] feat: support writing --- src/ASDF.jl | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index df2372d..44047fe 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -348,6 +348,8 @@ Base.show(io::IO, datatype::Datatype) = show(io, string(datatype)) Base.Type(datatype::Datatype) = datatype_type_dict[datatype] Datatype(type::Type) = type_datatype_dict[type] +asdf_datatype_yaml(dt::Datatype) = string(dt) # "float32", "uint8", etc. + ################################################################################ struct AsciiDatatype @@ -361,6 +363,9 @@ end Base.Type(dt::AsciiDatatype) = NTuple{dt.length, UInt8} Base.Type(dt::Ucs4Datatype) = NTuple{dt.length, UInt32} +asdf_datatype_yaml(dt::AsciiDatatype) = ["ascii", dt.length] +asdf_datatype_yaml(dt::Ucs4Datatype) = ["ucs4", dt.length] + ################################################################################ """ @@ -383,6 +388,13 @@ function Base.Type(sd::StructuredDatatype) return NamedTuple{names, types} end +function asdf_datatype_yaml(dt::StructuredDatatype) + return map(dt.fields) do f + # To-do: replace with OrderedDict after https://github.com/JuliaAstro/ASDF.jl/pull/26 + Dict("name" => f.name, "datatype" => asdf_datatype_yaml(f.datatype), "byteorder" => string(f.byteorder)) + end +end + ################################################################################ # For parsing raw YAML @@ -594,6 +606,11 @@ function Base.getindex(ndarray::NDArray) return data::AbstractArray end +function YAML._print(io::IO, val::NDArray, level::Int = 0, ignore_level::Bool = false) + # TODO: Get compression from underlying header block? + YAML._print(io, NDArrayWrapper(val[]; compression = C_None), level, ignore_level) +end + ################################################################################ struct NDArrayChunk @@ -778,7 +795,23 @@ Base.isempty(blocks::Blocks) = isempty(blocks.arrays) && isempty(blocks.position # This means that `write_file` is not thread-safe. const blocks::Blocks = Blocks() +function infer_asdf_datatype(T::Type)::Union{Datatype, AsciiDatatype, Ucs4Datatype, StructuredDatatype} + if T <: NTuple{N, UInt8} where N + return AsciiDatatype(fieldcount(T)) + elseif T <: NTuple{N, UInt32} where N + return Ucs4Datatype(fieldcount(T)) + elseif T <: NamedTuple + fields = map(fieldnames(T), fieldtypes(T)) do name, FT + StructuredField(string(name), infer_asdf_datatype(FT), host_byteorder) + end + return StructuredDatatype(collect(fields)) + else + return Datatype(T) # existing dict lookup, errors on unknown types + end +end + function YAML._print(io::IO, val::NDArrayWrapper, level::Int=0, ignore_level::Bool=false) + datatype = infer_asdf_datatype(eltype(val.array)) if val.inline data = val.array # Split multidimensional arrays into array-of-arrays @@ -786,7 +819,7 @@ function YAML._print(io::IO, val::NDArrayWrapper, level::Int=0, ignore_level::Bo ndarray = Dict( :data => data, :shape => collect(reverse(size(val.array)))::Vector{<:Integer}, - :datatype => string(Datatype(eltype(val.array))), + :datatype => asdf_datatype_yaml(datatype),#string(Datatype(eltype(val.array))), # :offset => 0::Integer, # :strides => ::Vector{Int64}, ) @@ -798,7 +831,7 @@ function YAML._print(io::IO, val::NDArrayWrapper, level::Int=0, ignore_level::Bo ndarray = Dict( :source => source::Integer, :shape => collect(reverse(size(val.array)))::Vector{<:Integer}, - :datatype => string(Datatype(eltype(val.array))), + :datatype => asdf_datatype_yaml(datatype), :byteorder => string(host_byteorder::Byteorder), # :offset => 0::Integer, # :strides => ::Vector{Int64}, @@ -890,7 +923,6 @@ function write_file(filename::AbstractString, document::Dict{Any,Any}) header_size = 48 flags = 0 # not streamed compression = compression_keys[array.compression] - data_size = sizeof(array.array) # Write block # TODO: create function write_block @@ -903,6 +935,8 @@ function write_file(filename::AbstractString, document::Dict{Any,Any}) # Reinterpret as UInt8 input = reinterpret(UInt8, input) + data_size = UInt64(length(input)) + # TODO: Write directly to file if array.compression == C_None data = input From 84e6eca092897144a4cc35735bc78e258800c37c Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 4 Apr 2026 00:49:14 -0700 Subject: [PATCH 06/11] test: started adding reference tests from asdf-format repo --- test/test-reference.jl | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 test/test-reference.jl diff --git a/test/test-reference.jl b/test/test-reference.jl new file mode 100644 index 0000000..40acb0d --- /dev/null +++ b/test/test-reference.jl @@ -0,0 +1,40 @@ +compare(field1::ASDF.NDArray, field2::ASDF.NDArray) = isequal(field1[], field2[]) +compare(field1, field2) = isequal(field1, field2) + +function test_fields(af1, af2) + for ((k1, v1), (k2, v2)) in zip(af1.metadata, af2.metadata) + if occursin("asdf", k1) + # Skip non-data entries + else + @test compare(v1, v2) + end + end +end + +function roundtrip(fpath) + af = ASDF.load_file(fpath; extensions = false, validate_checksum = true) + fpath_roundtrip = replace(fpath, ".asdf" => "_roundtrip.asdf") + ASDF.write_file(fpath_roundtrip, af.metadata) + af_roundtrip = ASDF.load_file(fpath_roundtrip; extensions = false, validate_checksum = true) + return af_roundtrip, af +end + +function test_references(references) + for reference in references + @testset "$(reference)" begin + af_roundtrip, af = roundtrip(joinpath("data", reference * ".asdf")) + test_fields(af_roundtrip, af) + end + end +end + +function yea() + references = [ + "anchor", + "ascii", + "basic", + "complex", + ] + + test_references(references) +end From 69981dbe10836119123894b0ff90218d1e7c4c3b Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 4 Apr 2026 01:16:26 -0700 Subject: [PATCH 07/11] tes: up --- test/data/asdf-1.6.0/anchor.asdf | 17 ++++++++ test/data/asdf-1.6.0/anchor_roundtrip.asdf | 37 ++++++++++++++++ test/data/asdf-1.6.0/ascii.asdf | Bin 0 -> 772 bytes test/data/asdf-1.6.0/ascii_roundtrip.asdf | Bin 0 -> 984 bytes test/data/asdf-1.6.0/basic.asdf | Bin 0 -> 824 bytes test/data/asdf-1.6.0/basic_roundtrip.asdf | Bin 0 -> 1024 bytes test/data/asdf-1.6.0/complex.asdf | Bin 0 -> 6060 bytes test/data/asdf-1.6.0/complex_roundtrip.asdf | Bin 0 -> 6285 bytes test/data/asdf-1.6.0/compressed.asdf | Bin 0 -> 1351 bytes .../data/asdf-1.6.0/compressed_roundtrip.asdf | Bin 0 -> 3168 bytes test/data/asdf-1.6.0/endian.asdf | Bin 0 -> 1245 bytes test/data/asdf-1.6.0/endian_roundtrip.asdf | Bin 0 -> 1455 bytes test/data/asdf-1.6.0/exploded.asdf | 20 +++++++++ test/data/asdf-1.6.0/exploded0000.asdf | Bin 0 -> 735 bytes test/data/asdf-1.6.0/float.asdf | Bin 0 -> 1484 bytes test/data/asdf-1.6.0/float_roundtrip.asdf | Bin 0 -> 1709 bytes test/data/asdf-1.6.0/int.asdf | Bin 0 -> 2545 bytes test/data/asdf-1.6.0/int_roundtrip.asdf | Bin 0 -> 2835 bytes test/data/asdf-1.6.0/scalars.asdf | 18 ++++++++ test/data/asdf-1.6.0/scalars_roundtrip.asdf | 36 +++++++++++++++ test/data/asdf-1.6.0/shared.asdf | Bin 0 -> 943 bytes test/data/asdf-1.6.0/shared_roundtrip.asdf | Bin 0 -> 1214 bytes test/data/asdf-1.6.0/stream.asdf | Bin 0 -> 1243 bytes test/data/asdf-1.6.0/structured.asdf | Bin 0 -> 928 bytes .../data/asdf-1.6.0/structured_roundtrip.asdf | Bin 0 -> 1197 bytes test/data/asdf-1.6.0/unicode_bmp.asdf | Bin 0 -> 961 bytes .../asdf-1.6.0/unicode_bmp_roundtrip.asdf | Bin 0 -> 1186 bytes test/data/asdf-1.6.0/unicode_spp.asdf | Bin 0 -> 945 bytes .../asdf-1.6.0/unicode_spp_roundtrip.asdf | Bin 0 -> 1170 bytes .../{ => data}/blue_upchan_gain.00000000.asdf | Bin test/{ => data}/chunking.asdf | 0 test/test-read.jl | 2 +- test/test-read_chunked.jl | 2 +- test/test-reference.jl | 41 ++++++++++++------ 34 files changed, 157 insertions(+), 16 deletions(-) create mode 100644 test/data/asdf-1.6.0/anchor.asdf create mode 100644 test/data/asdf-1.6.0/anchor_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/ascii.asdf create mode 100644 test/data/asdf-1.6.0/ascii_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/basic.asdf create mode 100644 test/data/asdf-1.6.0/basic_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/complex.asdf create mode 100644 test/data/asdf-1.6.0/complex_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/compressed.asdf create mode 100644 test/data/asdf-1.6.0/compressed_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/endian.asdf create mode 100644 test/data/asdf-1.6.0/endian_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/exploded.asdf create mode 100644 test/data/asdf-1.6.0/exploded0000.asdf create mode 100644 test/data/asdf-1.6.0/float.asdf create mode 100644 test/data/asdf-1.6.0/float_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/int.asdf create mode 100644 test/data/asdf-1.6.0/int_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/scalars.asdf create mode 100644 test/data/asdf-1.6.0/scalars_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/shared.asdf create mode 100644 test/data/asdf-1.6.0/shared_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/stream.asdf create mode 100644 test/data/asdf-1.6.0/structured.asdf create mode 100644 test/data/asdf-1.6.0/structured_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/unicode_bmp.asdf create mode 100644 test/data/asdf-1.6.0/unicode_bmp_roundtrip.asdf create mode 100644 test/data/asdf-1.6.0/unicode_spp.asdf create mode 100644 test/data/asdf-1.6.0/unicode_spp_roundtrip.asdf rename test/{ => data}/blue_upchan_gain.00000000.asdf (100%) rename test/{ => data}/chunking.asdf (100%) diff --git a/test/data/asdf-1.6.0/anchor.asdf b/test/data/asdf-1.6.0/anchor.asdf new file mode 100644 index 0000000..09ac78e --- /dev/null +++ b/test/data/asdf-1.6.0/anchor.asdf @@ -0,0 +1,17 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.6.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.1.0 +asdf_library: !core/software-1.0.0 {author: The ASDF Developers, homepage: 'http://github.com/asdf-format/asdf', + name: asdf, version: 4.1.0} +history: + extensions: + - !core/extension_metadata-1.0.0 + extension_class: asdf.extension._manifest.ManifestExtension + extension_uri: asdf://asdf-format.org/core/extensions/core-1.6.0 + manifest_software: !core/software-1.0.0 {name: asdf_standard, version: 1.1.1} + software: !core/software-1.0.0 {name: asdf, version: 4.1.0} +a: &id001 {abc: 123} +b: *id001 +... diff --git a/test/data/asdf-1.6.0/anchor_roundtrip.asdf b/test/data/asdf-1.6.0/anchor_roundtrip.asdf new file mode 100644 index 0000000..ced3593 --- /dev/null +++ b/test/data/asdf-1.6.0/anchor_roundtrip.asdf @@ -0,0 +1,37 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.2.0 +# This is an ASDF file +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- +!core/asdf-1.1.0 +history: + extensions: + - software: + name: "asdf" + version: "4.1.0" + manifest_software: + name: "asdf_standard" + version: "1.1.1" + extension_class: "asdf.extension._manifest.ManifestExtension" + extension_uri: "asdf://asdf-format.org/core/extensions/core-1.6.0" +b: + abc: 123 +asdf/library: !core/software-1.0.0 + version: "2.0.0" + name: "ASDF.jl" + author: "Erik Schnetter " + homepage: "https://github.com/JuliaAstro/ASDF.jl" +asdf_library: + name: "asdf" + author: "The ASDF Developers" + homepage: "http://github.com/asdf-format/asdf" + version: "4.1.0" +a: + abc: 123 +... +#ASDF BLOCK INDEX +%YAML 1.1 +--- +[] +... diff --git a/test/data/asdf-1.6.0/ascii.asdf b/test/data/asdf-1.6.0/ascii.asdf new file mode 100644 index 0000000000000000000000000000000000000000..75099226ce98c88bac5aca37192f79ceeae58d33 GIT binary patch literal 772 zcma)4O-|cD6lT%YNIgQOhY&()<8e|!7T%efRuusx$f5`#^I}izkz!9YGloVLw<)~< zo8F_lUZI!ho3R~Iq)K?>_y2wKy|)uzBp+eKhkVF(oSj}w~7kLw*$>$*pf&jKNrA6rcfU2p}ndw5#H0o8EU^#~?3LCoqcAL_Et?t7RMi5_m9EfwdvCx1YOJW* z0T1AoW=K^>uOr3MH(Ivc^d7HP`v|Ud5qb&NLyh-|gxiB7(PN~&jQ^AMNr%)}taH@) zh9GV|qkBK`<%2f*J3*a7Jpl)yan)@ez^F!Nv!bpTtg6DrDT1a&YH|P}WH$CXUa-BY@LVXgn1zi*q7JP zCg-vga9kj+Oc(^*WVvsU*BRFt(}HJMh)ku1Nfn%U2d|@xsf8MQ2lMC|>;m#-XwamR ziL42!I`Dko_jXgIMTcgyEwN=qrPrYcAU>g}ja0QE3&01Xa{S1(=xp-Uyb>Yo5#U~% zM9yoWsm%Hx5T*-Myq38znEV$eai?-++j!LD*ae3ElCT|LWk>GA zLz`h#qfoRmOFGRQHvA7ncTG<8q2cZ>ai;uAVPNQ~#sbF*Ne)AY(-uKh# z`OawP_s8qa;rs5k`uX+S)txUtZtmT^`FyL5w{o=?6i%lvpPs|>%Xso?>w3{0dh__` K=>A)esD1%K03PN5 literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/basic.asdf b/test/data/asdf-1.6.0/basic.asdf new file mode 100644 index 0000000000000000000000000000000000000000..b39b0f5a7a44fb60a25baf1bb9552f2bd27be268 GIT binary patch literal 824 zcma)5&2H2%5N@GRDX+j~s;cy|adxG$LOGF=QdLchME5|2s>qvoHDNl!6R_vzzc9;oH)yZgw$N-$KT9+{>IsjF5-7EVna6ccCDXYOrrBR`VdocOv$~= z=qw?BjeSB>rFGL%PBqud&>b5!!yB#z zL1-e}@&*f~Lzol-S|+i$7NuGUZ3a-NN-X$Xgs@*AF2W#~ODvix%T(13nkijzZ2kR# z2T=2h(01?uuBnGqbqEKR&7D_BgNm3COngRB+e2fcp86`ARswm>k#{T!vR0C2B{!xU z!d4|Lt#~bG!XP_y&QU+)pZZ42uA1)Cs?}YBDLoHb30F-`8xsk;2TP)#k@hnFPu4#= zqy~ANbDggVV%IaeYdb!^(OSI|)GE{yup=~Iy3GR^)#!{?VR>5|E*BKZ8pj8enJ!T% zoeNrgDG^ISIa6>8`SkkAV~ly<4wKW3;l|(J8yBwZtIOme{qo}bkLdp8*W}mF9=_?E rC(b@~cFWmk&OUc`+u8OHfWzed+fz6>kB>gCJ%4)Ro)2TXfaUia>;UQY literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/basic_roundtrip.asdf b/test/data/asdf-1.6.0/basic_roundtrip.asdf new file mode 100644 index 0000000000000000000000000000000000000000..435b67cb6478790e1464050900dfdc2c9d139f76 GIT binary patch literal 1024 zcmaJ=&2H2%5H9~xUV)3-s+@?k3uRSBRn?R%2$~j&-2)XukvDNRh9q9ulfu3W&w#|8 zx8M=Dao`2G;7ROs`4Lg1@%(+?cxJpC&Eh9yOhDZL5@4mnAYeK#d=0G0naZi+EQgZkGSgJZ;KE-)f^-d2J| z8k7tM5-c`##TtY^7G$VkUh;mgV}8jixiTyB4?;7F4L-#7`>Av*iwRfvw!20t^}e1Y zr(2_~-@jw44)W1O@+tj%_uKdAAt#lNa7!zI9W^)$)aV%Wntgf;8N0Bx%y_R-*OY zMwqN`%#~+nXM=O?ejLq#&&qCz#IUv1U5mqbuc6CHe8^OI! z6c{Hf(SvBKzZ1pDrJ&RAMLXeU)Cewi(sZXWH@DeMJG*O@cE4v6ZS><_n40m$OXVQg z4tug2i7y2YM5Ei^ZUl9>c~tInle8~RWLxwwjkd9vfYG{}ZS|rwTo2RGx|A&=A+7dS zm?Sny#f4N_y>Pp`5hZEmmJY5ni5I^ccWtzs$5hK6qSB8y=S&HmXkxgSz&*eObF^5- zqOuO_&`Q#9dp(TTM+$=LsXj7_Cl?xt*Mg!#>E2G%Y*!a)1gYY1PhUU~B>mmEEmu;L zY~0@nZa*dBy&k8+>xl&>iql%WyUqzOUI?DPD`^` zKTVb^CpgW*v@{c^S?g{V9p%dLN5PHrgdTz%9L9{_O7yvWHI*7DdZlF{m3gxg!59=qjF1KTx7wbk3W9SJQJwc-<2tDjYBO0AU)y#)_ zSce`EdO+v_p$9a2vL>TpZ4Olnn2+c|L%Y$FvC$JWdV=P>Kxx=x@Nl{wHGu5qyu!_S z5uaPn(7_s7(Rt~G8bEgFVGKQt(TGKQSRagdp$3p0dZr5yi}bJ#Js{2t9P=a`b0_;$ zt^WMS+f9-tp+Zl=V+F@2IvFAF@&5ht1u^DyOAY(+**DTL<+JF*j0mt<6 zlj$K2Z4@y^&xr$Yt7lN>Q7E%d{23G-#Gnx|&yG2aao$|@3}Q|?^nf_8LFmX{G$P{J zu?J(LH?aQX*rtAFKlCKt=mAk%K~rOcLoK6|Q4jk@x)2jrx0u%y-CYTnSTuT@&@&p? z$1aa5m$~>Y3W%KG2~j9W6J1 zi_VL6(GhO54d8nxj)WJQ$Fw5k$c-Fe9>=#KgK^Rf6Px?MV#{v3;c1N z{>WKBa@NoBed~an^~*Gm^XGHc%f}*rI`8~(pWZL?Ie)s&`{g?QO}rc3@BI3{$eC9; zbsVZ6a@LQW{v6l(kvo6hyXCq6w?F#w(D~DO=Z~EJl+z#MoFA_11mD917SxY=JSVx2 z^v8L3zsx^l4R9a*`Fz6HG55=O`XhHf4#x9xo&LyKKh8UU`y5$3wLgt_{**g^%ISmo zeWX8!)=tUir}f8p`Xi@5a{6;zeUlTX{VzBCwy~zqUj2Rh?%G28?;qa#EQo*l^P8`E znE%G7@?D4KOicg_=^>wc?NDjhUAvbgVYeY)eD!1D1a-xqnIKm0=_?OFL_ zKDsL6oF^9fdzu-0&Y$C6y;(o%v3}&NAM?dv5G@tXQ-1$>JBmeKAO}rCbKfOQd6=24Bt9;NL=ldRa z{xUkz{f^st*2l-m`Z1n)J)V(!{xqNSr=0i8I@6`keP860{=B>ck857)`^fv{JpFN= z{(Me9J{I|Nbx?nft3T?~2f2^*N2EWW`@YDF{JD3d{v20-)Tcjk`XiqynIG7LYd7D0 r^G5K_?Un2IM}A$GpU}&*!R4i@{H-q4UKdzcT)He!uhna1{H*scha`y& literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/complex_roundtrip.asdf b/test/data/asdf-1.6.0/complex_roundtrip.asdf new file mode 100644 index 0000000000000000000000000000000000000000..98544577c194d953b358c38747a715ca568cef21 GIT binary patch literal 6285 zcmeHM%Wfk@6m4k)3$OACHHl^yw;%C@$g-k%a6mAbK_Wo{jgZx`%W;G4wz|thvU0EZ z4Q2})<^x!;0DgcKZ217PU@<#Wx9Xmy5+Kxo%U(xJy8p* zVYReF&)w6r_R~)LQ=#+lGR(L|WegLouGYpB#TYgQ^z)*qBJ6;1k4+7DA1 z_0>@JlU^3aN#(Hg;IsCV6D?IMJveK>FYXH!T{JV5_2N+WXH6PiDV59R(*0hNN~@-O z>%jd;Me1rQTfN$TQ`pxOB#o}h)~^ug(-hmiWIP?o%UbJlXsq?O zX#yl?ujewQVYHc?Q{C6fbdg026Jn`HIjP}=^VjW-qAX(@p=%Dih!{TM z(8rk4rp7FdSpmyANYZhn!X&+@m{oDYnc+J9`!qRZ|M1_eQ0?|Bw2XIf`-0eS(2P+J z^?x3%)qCS;d$*5fHo_=APa|D|TEA3ph7q(A-f-M4Gq3(2dwedXh|*M7Z##KK&E`kect*UJ{3EWYsg z*81xu%a@Pznf`5gUyL_f4+ zeXl)#j<2irKvA)Q&9TfyV_wxq6 znLjV?!s8ql`aWWP(GGuThd-agkB>$ETprk;cwdRQ$F{7 zfkVe}_^}_Bcc%^~6#4OSGk@MGgFL#^2Jfw~I~94|{ZCh{FX{m6>vP{1_@+NEF0Z%m z1NuIq{?HD8Xoo+a!;gi7kH6BcgnCo$JrnBi}Hl` zH=jE9g!Q?^XTSXZbN8+G(d9Rd?VZ;57>?;^{82~7o%3^U+$|P!pl&x;!$Q8tXkm>& zi*QX(fAZf(kH*opiu6o(4qOnodIq^Y(Edz6?a!drp)qs;sU0`9(KF3AXFY?|rfVg~?av_TQU4WB#Pb`!%Pn^k4GDsK;%-;vw8HuV z7Oo;!>fNr=xk4pV;%H~k#aNU-vm%Mss`!=G2?SGlZq0HDy^W>#rbI^<%vb9 zKqWvM6{-_UOEU6{tQ0~rQWd~1aY-#p&B-rFEh^Sg$jHx4El5mHwNg;eC@Cqh($`PV zEXgQM(o4?Ig;{lnjgtL!d37 zkSwVz0P-^PO3X}vB1x4csrf}IslbTO$t)?!Nd<}(XC#7@L>n4e#Bx;ulMNxGK|w-> z(RzA%T$i1EycrA_fB<9>hz5eo5DG@KFw{%6?z?h6mwBhd#N+Z>YWV@@r`^%8=Pfc8MIp;n%GygC0XXR#c}I{oqk1|g6F(4 zS#EuJ=DDJ6#dBWUJl}nwRj+8{{~znUV_q3e-+gH6x}vSwF|Vzz?>^X@SF}0$&MUL+ zyARKOSG2wQ&TG5x)d!m8K5cNfd1cuDd%pUg`;+H=+LCVb+VXt$!S1+Eo5J_JGF@MN zc=o+d+sgO6wtfHmK)c0YL#kTTc31xb-CWX{muMWyUo_#*H6aGinR_v zTU95qbaZq~aAZMgRZ+ literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/compressed_roundtrip.asdf b/test/data/asdf-1.6.0/compressed_roundtrip.asdf new file mode 100644 index 0000000000000000000000000000000000000000..d7f37941b40d508825bbb3252186a974428969c0 GIT binary patch literal 3168 zcmeI#X;afc7zc2?HQx8F3y3F4X|1RgkwQU{OQD{Kb=z#4XqwdBEl@mo<%e>{@hkX3 zXZ!$u3g5Y#{8JE*aePguGr#O+_j&eeCT-77<%USo>ax0wc70u#%4f%O*~y%$OzBFJ zcU(c#m--}ZTy{N1dK@WhB9ll^VV6xIsc%zmTbxl_I?N7A!g7N|uhH=+J2t99C5?`J z_9kg1l2$T8ijr$FyPna#6NYJ;Mr$eHEQ+RDtHdg+40t1B5W?mq^MxDuLYD|Ji3rMa zj&c^2b)8SEEJNCKz_zfFuAF6DcT@Fg9W%aBrM_EcLKglSrV!66?zTLh9%C}T(Hvi) zERVuB3R}$eWH3N&VY$mO4J-OmSEtMhB zLh9|{w4rY4oh8JklFCL+FOloZ?zB!Tf;um$8hyXmkSyRfCw}=bOJr0LGHdF zQzge|QZi0@M10v-sZ!UoN*Hr1*GOT%-rL?yxj(9B9k3jL-d}BQ0Gm zKYiAtwB+KEcOSoes;+zA_wDQVnGcWS=4)`V7Oq>-a|6m7;U>5lZh;2e3b(=Sa0lE8 zcfs9o58Mm)!Ts<6JO~fL!|(_^3Xj3#@B};w32cRJupOR)9q=?f1JA;9@I1T#JD~|J zn1JzyB~ebni!co@!ESgNUV&F(23~_b@H*^;H((#k!hSdab8rw2!JF_F9EP{y2polD za2!s+JMb=?gi|mN@4@@<0elD_!T3N=qx=NUz^AYPpFs+Xumo+$U>R1R16}wWzJMOA zLLUaO246xB1(dK3XW<;2hYh#@U%}V#4SWk1;p$UEFU0CoBRUSNPmO3DR-YQtI{fLW n5&lOY{i72DBV>3yH~4Vb8@_&xpT63i?o4&1Q=K!0emD66k<^6K literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/endian.asdf b/test/data/asdf-1.6.0/endian.asdf new file mode 100644 index 0000000000000000000000000000000000000000..1e72f02a12cd4068305f2d2d70d46f7169e7683b GIT binary patch literal 1245 zcmd6nOK;Oa5XZwC%p1y^!iEr1q-2w{DFxqBQV?od5Xk|AkhO_7wx!q`?QQ~5)eE13 zk5F;uj5u>b;=mbk-~^w6|Jq3il`49~($Aim-FfVcwV7(8_JEXWiI%L4k)1}fx?Zb3 zt-*K)#_aRzS{-uP$~LPHNt$Tp*-EQ+K)KtunQ}Wtt57JAbX!PXGo zLwmHWL`T13l4A=pk`30^zK}L)`kcg-)cAmRMUP9BC%y=Ik9piCIbZ9ZT`YQm_WLc` z7GaX8BV@>QEa&nTArT9)+a%7D0b&FqvPs2MvuXK((gKxW8-J~NWEPdt#-&fJPRKQL znPy3q*fQnfw7X2HBnh3kP$y(j(BVqcwUMwgPO{CvFN4Gz=ZU+O9D)kz72_68f~r_Y z5oQjih;a^Q+{T^S`lk<$(kybBbf+3(##7#mBkn%Xl>exqt-u@SHyI=wjs{PN5`Edm zBuv?+k$TvJ7DRes9+uWnb0J-hxg!P)%4cT%MP>e_)eW@nLP>t7odTEt6*vjAEjFdp z`m$VKO_frAKU*Vp^8H0CTR&%>e4c&x@jLN0TBG$n`0fS!zUk;3zl- zj)N26Bsc|5gEQbPm;vX&d2j(-1ed^Na0Ofiv)~%I4hTqt8z2L2f-JZNa$pYJ-recy zmFB0ncYn+c-q6PTuirkvGxgse4K>CW6j`o6zQ0Nyt=CqbP2Enof~*2rT3o>U-qNCF GX8jxXjv0dh literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/endian_roundtrip.asdf b/test/data/asdf-1.6.0/endian_roundtrip.asdf new file mode 100644 index 0000000000000000000000000000000000000000..a44613df7f3e1f0cabc1057d4a98b4acc24bfbe9 GIT binary patch literal 1455 zcmds%$!-%d7{}R{Vc&OfTM9+$I89rs5>*u`DF~VtiR1u6C~`7R#xNO2wo}NJr{EEI z0M0xBR}P#x@f!TU*;*DM#E~dJ&G_@@Zy!%o8@2na%xCzFH$m^MMzgw6t3IhgI7>p- z43%a0iy8z}-8%0PE4SGu(9O+Krx*I}ycO*X3! z*cfx7SFz4^6qmuULZ?ff@B7|ZM;n=1gEkz)tJ7vw@feeDoQ$o~v85GbKC`;(UJE1B zHOXSpmlZZn2FDX65^hRE#~`04oB2jx#HuT;YyHtoE6*xFw%s1ZSk5Mdh|aKeoSR3sI4Glg@vDnu!x`;%Jkc*$xRvQ z0mkw-4$ZkUc0Tzm=)hLq**tU3z0WK4wH-4%e*C@9mc_F^-~9A$@!QnqTi*Ea z$ literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/exploded.asdf b/test/data/asdf-1.6.0/exploded.asdf new file mode 100644 index 0000000..157f895 --- /dev/null +++ b/test/data/asdf-1.6.0/exploded.asdf @@ -0,0 +1,20 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.6.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.1.0 +asdf_library: !core/software-1.0.0 {author: The ASDF Developers, homepage: 'http://github.com/asdf-format/asdf', + name: asdf, version: 4.1.0} +history: + extensions: + - !core/extension_metadata-1.0.0 + extension_class: asdf.extension._manifest.ManifestExtension + extension_uri: asdf://asdf-format.org/core/extensions/core-1.6.0 + manifest_software: !core/software-1.0.0 {name: asdf_standard, version: 1.1.1} + software: !core/software-1.0.0 {name: asdf, version: 4.1.0} +data: !core/ndarray-1.1.0 + source: exploded0000.asdf + datatype: int64 + byteorder: little + shape: [8] +... diff --git a/test/data/asdf-1.6.0/exploded0000.asdf b/test/data/asdf-1.6.0/exploded0000.asdf new file mode 100644 index 0000000000000000000000000000000000000000..054f2af56dca0ff7431847e8f7077a845126cd50 GIT binary patch literal 735 zcma)4OHRWu5G{WxS776+D%~_rD^ZZLA|(Z(rbVJ%z$$VRr-{_WmF*M|>PR@`8+>Q!m1YpM{|>L=0yNq-8fI~^Zwhr^?2~&e0~(^8_L*J oxTSDg;f}&xg?kFKMFoB5@^lC;#`fU4vZZ1bH68Shdz$3;1)8|dG5`Po literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/float.asdf b/test/data/asdf-1.6.0/float.asdf new file mode 100644 index 0000000000000000000000000000000000000000..f4c2dc98b91b6676c6169d5e2318e673f89232c5 GIT binary patch literal 1484 zcmb7E&1(}u6yKAF^MV&n~|`RJX>O}S7$ySB(_&&by{Ht z8L)+@8_8tF5q6`fmAtAXvOK>^MW_!9s)W>N6}*vnkSswC$Ez76gVn64w!?)7m4F+& zEm#c?Iqu@pJ65yG1T9lRWtHI46=IfL%DE)bIw6`_r8UQ5TxfHVG8L2Tif{OijD~)~ zX~`xu&#xrI5q6UDa1TY`JzxCZ9H=2S5n-8h#0GL zNhyD?Q_;4Ar=K6|U0~Cz@85h|K3NM2c-{tSZGeChptORZ1(E@K8usXLxa_O?;=}u| zu3y{y@lJpC_}&+Y#no1njTQh(0J%{>SAd|M#^Zlxzlfo*I{53!2lvSKolm(3{-vik zUp|t@7%x|AC&Dd>L5!O~h@25-*X4P3O`_5x@o-gqf6G3NDhoS)B6UFrJ!!Y@`$kg<_r`0AWb>UF>sw;P|4uOCbYl1&m1i z1xSo6NHDX2_!AIfVnEEOu(82C+vlcGq0$M<*7JM!?)%<*_nvO9z0o<3JY2?Qdk+44 z8{PJLr+u}fjTJDWZp0+gFG*199I}|AW~9_eHe8p;AatZ6NkBw^1to!sXuy4m8FyRu z{Pp(Qs)q9H`EL6Hnn8*T8d6E0VH%7Zu-mmA$FXO8E@+0Py*06BRa{IOHbV5CqKRZY zk)R>uAjv~@mk65aU`xo5Hqa~p&ZbUa+@b=uX?qPY&BBl*ETmHP{tDAGS($Cqco3s! z7K-?KeoUmy!@z|Z_Y5Om%c+-5N`GRE1Y;5GA^1Rj5$&D8t_X#7QU_?c=RWVauHCWU1FwGOjD1DjPhbT z$SW6At)o(T|4h{kYlqFMTA9wOa%ifw8)McNME6}@LDM2d3HN$Rk4M&1dGEn=;M*~9 zld+08lyF(FJ7~j?5~>sxs430w*};&o82fzKf(;QL(h(Uz3!BLrFcppa;Ne~x$BeY4 z65K6FAlRM>RwMj207V4dh-P!!p<6WOBRvua5b9Q&;sb0a>$-F#VZ&bD`7D`UY8~}p zsv7lh?S?V7-!HAMSY>Pfb0t>``S~sV@cPArV_#3)Kl$m*_x^r&YbVoxY-jSDkw3F% z*T~%dvYGik%jI$FKaS#57OPdrhu-Fs$FFYQ*!}X(d;av%$9#Rqvfk+YcHq!p z*0f&L-@!Q+sD|aJ=;jfJizTmCuavxng?j0h H4K45sorUV5 literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/int.asdf b/test/data/asdf-1.6.0/int.asdf new file mode 100644 index 0000000000000000000000000000000000000000..193a41793ac28f5f3a645e4b9401ebf462759c7c GIT binary patch literal 2545 zcmcImOK1~87!I~n2|ZZwB#2W=snBMV&8A7ShC*tp)V`ptqLvya$s}1#vT=8|52b7` z9=upk#7jZJLs6-M;zdC(UVI$-@G5xlAnKvkOQn=`X0yBI*)6EK{4@Xj-}n9V%^%p7 z@Mz>XU>OhNakkKPbaX5{6bX++NYYP|t!Kl717yuQTgSpDKr_HF#miWZ3k*sqJd_hj zx6|cvf#$dr4wW7D*?h3{7K{qlYrxnXOydXcl3ZfEX6KBqc?VXgmXDxgf$=5GI<1IU$K; z%nTZfUQ?tDuSh}xP41(+Rk#ErN-4K)p+zd|d(s4b2ikXLZKi6BXeNL#ab}CXYe_YxY5vBsgOX006S6NOAHaULt0e^FYYqE|Q4NU?fTj zM4mq;q)0+e!#pV-=O&zHaY*pmar?`;S(+&#&j=W2P%VC+9Y0IGH~Z0>=je5IE1=k# z!RFmmA3C_OR?qlQ`1?}IUUKJeJNqL^(9!rUOq9U#t?RfVVr9{1N{z< zqx4s2Y^`|tGgdwvzWKFoxNoz4ap;sf=co$YV*suA>C3CfFB-0$KI(oxy>R>Kd#yUo z5;BGVw_t-IWW^>Fdlp-D9bEZ2@*sM3J*jGQq@?O4>kX_GC(gf|+Pm)hj@9Lrrsd6$ zrqw`NYNq<;{`pI9mfmfwElBkfHR=j(Cd!rAiZ$})y3)}BMs;Z{x zx21Jewf7keRy^?Q_0p|&t?tRU`TJY^qe((Q?py1 zLjqRG*&IFpJAM6Q3-k;OAL|EwLy_JyMPFLvbIR!gY`_!HgWpF(m)E0*fSwF;9Nh(j dGu}5dAUwK1S!DF`8oYx^c!`+&U*j= literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/int_roundtrip.asdf b/test/data/asdf-1.6.0/int_roundtrip.asdf new file mode 100644 index 0000000000000000000000000000000000000000..63bbb773dad2b5ef7d75b96dfaa9716a2fda14b3 GIT binary patch literal 2835 zcmcIm&5ILB6d#;HrSY%}dtI!VK@dzj=}smSV-t*<*+m^k*crtS#!!<~(xshrtg0TJ zz06$qB&>)R!NVRzLi2u^ zReHQMTmBVUX4*^}Vxs)UM!!6WRu2YX7#}%T@3#k<5wR(~Z z+^S(#Ga11(;gXtP|+3tpursRaeFP8rE$(ipA! zjD+5#whUau0*j?KDbj2Kg}V@g;iY7FT!dnk0ve+&k+H`Ngt#BDDk-3}TrVsM;v?#Gs7>;rifU&LtuKGFc>^-zJR1 zzUwzh8#llND3OdbsBnV?v+6fPICY;jvFNVBU{M-Uza{mvfOD(|G|&MWY(Q#Bry59Y zL*`NVAxLswkQN++$3$?yF3w>V5d-9WMg)lZoIVscaf{Xo7nQHNsl;^xJcOgi24vle zH`MtmRS$C><-=5Bvl2%%4@IG;Vn`c(1dK)=;(`rxV*0F+>eiid%)9Th!g6a<;|!P@ zTQ@avB|Fs8EA~8~jB7ok7bc@h%?WhEK&h>Zlhuo3t2jk!>%p;dYw=9z#R+>i758p* zNyHVXZi!xr$@>hls!AKKGxM|u64mBYjGjTWRkuZiAPhM?UhZu)G4-IWi(v3s-@{e7 zrtuF19>patn4eNXdO6nV^9^mD(+W$Clf}8;pWNF{{rfC|__e~!hN)N3c+oD%BKl|&<)ptM7?xZI^9=P_m zbEDJgEK8jg(Ch1AE9}Uj=AS-1xOr(;XZX(B3)epuZY@d=9Z>iV*2Cae4mJPrN(3*q zUwkusc>n6IYcofdSNezD6xt1``Q_{TxBuDv=g~c>d*`oQzWcb--)?hcr{>ikE}VUM z^U?d~SJ=q>Np4HBR=xmLQ5^Z$_RUm}M3!ovKlSgqE&nI~6U$GY{`h3o0iyfTtj-&s z{~OM&IErjHX>z5K64?#wCP%@G;r4^s{YFLmrVsye5bd8SPaRwHg$bXLC+)PAPuS^v sHep-YT*8)1`N&Nq?2K*6M_xX%*~x@Gk(ROv*vP>~ZZe;kH=toY0hPrMm;e9( literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/scalars.asdf b/test/data/asdf-1.6.0/scalars.asdf new file mode 100644 index 0000000..f18c657 --- /dev/null +++ b/test/data/asdf-1.6.0/scalars.asdf @@ -0,0 +1,18 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.6.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.1.0 +asdf_library: !core/software-1.0.0 {author: The ASDF Developers, homepage: 'http://github.com/asdf-format/asdf', + name: asdf, version: 4.1.0} +history: + extensions: + - !core/extension_metadata-1.0.0 + extension_class: asdf.extension._manifest.ManifestExtension + extension_uri: asdf://asdf-format.org/core/extensions/core-1.6.0 + manifest_software: !core/software-1.0.0 {name: asdf_standard, version: 1.1.1} + software: !core/software-1.0.0 {name: asdf, version: 4.1.0} +float: 3.14 +int: 42 +string: foo +... diff --git a/test/data/asdf-1.6.0/scalars_roundtrip.asdf b/test/data/asdf-1.6.0/scalars_roundtrip.asdf new file mode 100644 index 0000000..6a59d0f --- /dev/null +++ b/test/data/asdf-1.6.0/scalars_roundtrip.asdf @@ -0,0 +1,36 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.2.0 +# This is an ASDF file +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- +!core/asdf-1.1.0 +int: 42 +history: + extensions: + - software: + name: "asdf" + version: "4.1.0" + manifest_software: + name: "asdf_standard" + version: "1.1.1" + extension_class: "asdf.extension._manifest.ManifestExtension" + extension_uri: "asdf://asdf-format.org/core/extensions/core-1.6.0" +string: "foo" +asdf/library: !core/software-1.0.0 + version: "2.0.0" + name: "ASDF.jl" + author: "Erik Schnetter " + homepage: "https://github.com/JuliaAstro/ASDF.jl" +float: 3.14 +asdf_library: + name: "asdf" + author: "The ASDF Developers" + homepage: "http://github.com/asdf-format/asdf" + version: "4.1.0" +... +#ASDF BLOCK INDEX +%YAML 1.1 +--- +[] +... diff --git a/test/data/asdf-1.6.0/shared.asdf b/test/data/asdf-1.6.0/shared.asdf new file mode 100644 index 0000000000000000000000000000000000000000..175607b333448e69191f92a11760a45941c0201b GIT binary patch literal 943 zcmb_b&2H2%5HA0cUxACOs`RpPcEz%+aw2U?g_;(L?t!XWMc%~OSW4oRJt?fJz6sBO z#GSX`5x8;S1-LLyoMk~m;(#ynPVna40d)7}bX5o1hzKJk7!Q}pB zcow5SBKx!Ogzi${%fNt1B@F&fML-V^@x3p<Wab6fTq>$yZ>(v{hlzdK_0FDppn1B>_RxB9SbJ5Hb^c5)as(Xl7iR1F%41 z!2u9QK(Ozc3vdK>EVuw0=GilDfC?d^NN?u-eBb-?tc8Q_UF72yZaHgkjt0Z!bMIL=3?qqU21@3b$5N-f#MRPNPy}_!Z-w(YZFWs zA+V|W3|hnr)oBB*Z^c&v%1YXV~m|I`~8EZ*3z%PJ#%g3tx^9&^y%8yZ{fQer~Mz_ zt@{OYxM<|Ek(Z3TY~&RquNrwSD>BALzImIUe13m@`0D29+3Ci|*Otw)Sage8y+CNc g|M1QMx_{K|JzjV};E{N8yR)_Fx3*iGPaUZJCwIz3asU7T literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/stream.asdf b/test/data/asdf-1.6.0/stream.asdf new file mode 100644 index 0000000000000000000000000000000000000000..06212fcc8cbe4f4fc64849ab05ae2b1f9fca28a4 GIT binary patch literal 1243 zcmcgqJ!{-R5LG&(Dn$wxDb^Ts5KgN*Fea!;WG+q-J4Jj=2u93Fqmw|=9=jtuBKTM4 zkK`w0R$5(-K)_L9ftU}?oA>suPUG9vl~~G!T==IpPj1)o=T-b=#mx(Do_&pPt~p=& zXY2T*I1vQf$dE}(iCG(g$@0(-fjFiuWQF%wsU2!F7lo>E2V0E9tRUJ&VYn@+Xg4xd^+1#>T?6&`Z0-xuK+V&Z zcrNaFN2!`foLe=Ieo-1ytc1t#J7Hr#8e5Ih_uZsMf(!@-l{|6)Nzw|88A#+FB9j`L zGDk!5#w8y{lE3h+E(bK9r*~^Ogj9MP_7?7!n!aWc_8zQ=ks}#x{FkkNd`JvIlY!0- z46)aN=;EAvn~qTF z4EYDBN}>w6X9}=G-)3)SbMf}hms0vaFRwp&3(xaj@cYcquXh9C)abo;f+xl!hnpH- VI{ei5%HgNRV~3v_UpxH&$KOx9^mYIM literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/structured.asdf b/test/data/asdf-1.6.0/structured.asdf new file mode 100644 index 0000000000000000000000000000000000000000..ae2549f0861d63bf3846c11efd3022a649a0bc2f GIT binary patch literal 928 zcma)5y>8S%5H@tUK(y&5ilQ|3`Ywl}$OVdn0}8nWiMs+6Mb_HhJFjGWPrKv5DN0L8 z%LCBzHYs=qDjtE|wS93wNVwUVouBX9Z#+BEEPe%JG9n{)$Johi9$m!IyI5Zy=*!&? z(fL%{V|RBRy@qW-x(pSnj1!hOAyxSzaDCs0?Mz4(7~j{~y3&|Q$x}(?I_$zqEbt?h zOs8l>xS$u25(uI*oloR-2$QUQc~bP@<8U9*ALY^u+awwTj=Q zYh*&^OwT^$OYc4m4X^Rg>qlv5`tUH@GT-!rOR9fH?!n~B2HLl$wr&Vha>SAikrt(( zIN9&9GK2LOk}9Ys4VU{@E+NFdJD#37Bj@4wtaCSK_~OggAD_40{CxV|`StCI|A;!y lWHLK?oSO6Q$Zi@OPv5>ggVT%nPl literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/structured_roundtrip.asdf b/test/data/asdf-1.6.0/structured_roundtrip.asdf new file mode 100644 index 0000000000000000000000000000000000000000..2cdfa040a4c4b73664c66de29464f086905b8812 GIT binary patch literal 1197 zcma)6&5qMB5H8%v0deK#?P_nt=?_AyXjL^!3SzgrNVEqKLXn#|jUkCEd$Q0M;K-E+ z;L6+Vfp_4AakXJkkRbl@D~K3Xh;%P@Eq8sP|qWSI#~ z%+E?vE2kpoHxAqk#P-Bw@f{ zyJ^ejc15#6thT_VO1Lrsqm#2&R;$~Coe9; V+3US01K)+W!@=?R(K`n{{ssn%SA75g literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/unicode_bmp.asdf b/test/data/asdf-1.6.0/unicode_bmp.asdf new file mode 100644 index 0000000000000000000000000000000000000000..c3fce7a939b0f1e7736daf22180d1f284095cd60 GIT binary patch literal 961 zcmb_b!EVz)5REwc)K8eIs(Ojn35{B0AtVlgY)?F`S);r?0kdAYC)Zh2!HxdOsQ+ HI!60F@njj6 literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/unicode_bmp_roundtrip.asdf b/test/data/asdf-1.6.0/unicode_bmp_roundtrip.asdf new file mode 100644 index 0000000000000000000000000000000000000000..95141c4fc3c8d819c6ee7e9dda5fff1e82475029 GIT binary patch literal 1186 zcmb_c&2G~`5RN$d)JNE+>aAYKRTZ^X6ycJ9N|GXxAb}9FHufZ5u)XQ-x{#;fIob!| z1vqfw%nKmy?ATs6{0Jd|Wy!naZ@zDKzR@0p^U*`%(;n?P2e^*ri|}$3J{_6hF$T#Z z<%*b>6{G^Mcm`yUYQ0f`=P{M6T%}o&Fqu#ZEYT?>B36_OZ|EGp2xn76?QWq{e9UN?jl5GtD{UyXA zY#X~}tWN3*TyksxN|Z7-eMInbt3k*Fq^Tz3T4#Vrm9kCKc1)=Z%=l6=V^yPSyV}a$ z*$F@V!h2k1oP|nD;k6=IY-Ecy3_s0GTfriQdeld74Vl=ODf*3|ac4&lx!u~RYKk%8 z?%d7z+5Qjz-67xb|CK|OQs?9ObkgZ{eth pbvpltrMhLv`Si(y3Awl&ji2w_5qJ^2JUcr*@_YT$qgM{r{v8bMP09cO literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/unicode_spp.asdf b/test/data/asdf-1.6.0/unicode_spp.asdf new file mode 100644 index 0000000000000000000000000000000000000000..b89856af2154c28862256bc553ae0ca7f2f2fd6f GIT binary patch literal 945 zcmbtT&2G~`5DquACmvy{s_G?P$8BkmRUuqlP}Q_ZZ4jWSvNrZ6wq$#)-Ekp8c?_P5 z18=}Xz?E75qyY)F2p{a3+4<)C_FM0Ea2>t^kB|6>ZQD4yo&=X+a1~PVkcvAWf{QW5 z9^07&uVD*NEPR6|&bUl!UzlX>GRJXXD^^;%Hg~9-kT#6+Y^H@?`rWcobNnc@q!wa? z&!WauX&)x3ghojyZ)L73sm%~lRme&#qz}6(V&%K;BEz(v@mLit(Ol_5U=!~S89*rt z>bAr~xTP7HDt$PxYE~@G3@TDWZSoV!(k>dC_0reXsE{ZUfudDOEgO(1&V?~83135a zRERR0ON0EP3r>5IKk&8AS~TsaajV?~S9;+#7OsbyCMOa09IS|*BkFDZo2_>~L{ zmME1@B>e(;hL}sLnN(B+n(p1O9`yWP3Tsf`{ZAB}bN20Id_EWret#Z!>H7O(y84pH z??1l&eBxidJ1*=x3?2>s58YfkI2pfrc@Afn;pzL08%NiQIdF8ePw!{@Pnp&F3mVlG Axc~qF literal 0 HcmV?d00001 diff --git a/test/data/asdf-1.6.0/unicode_spp_roundtrip.asdf b/test/data/asdf-1.6.0/unicode_spp_roundtrip.asdf new file mode 100644 index 0000000000000000000000000000000000000000..3d4ded4f61f9c730d09a36a3350c5e205c58cc1a GIT binary patch literal 1170 zcmbtU&2G~`5Dquw6OS-W)my!etE!?_6g4HKf~G}kf&f*>+SrqL!S+VG>p~7Z2G7NT zH{c=Q%8u=I!;cVBElc*!e)D~^^Nqbbo=u)Y$OdfS?b3TZn~$$18MD?=5OAF?eQkJ|a+NZLJT)1nGSN)PVC3z+8DCB>TsG+1gOOHC^PAGBPW4_slhrdY`fjG#}1 z{Tf8UTU3;0&+DWD;zOexE7r-H08~atF z51R^n2rWR%N+q_B1YT?mN|mCr)#SpM90{p2zHZy@!+}>Dn2Uwt)~Z(3b+wbdv%yYO zV?%2f4HSEq6EUw$CQ0i|iT5x|vJ#C!1w-v#PgVsNIZI?QqJ&HqxaKQz-3{YPn5>&})Qar=!?^%PUW-MQQG zbNwIP-yw4AzZ_zWd0$Vb7rjC6*Xv=s?4Bn#pFXGf?faJ>kD}{W!-Dpq_n>#b?E0?) ir_&ctF5ubK2{o8#lh`{Cedu>aPhT7LrjeM#g1 literal 0 HcmV?d00001 diff --git a/test/blue_upchan_gain.00000000.asdf b/test/data/blue_upchan_gain.00000000.asdf similarity index 100% rename from test/blue_upchan_gain.00000000.asdf rename to test/data/blue_upchan_gain.00000000.asdf diff --git a/test/chunking.asdf b/test/data/chunking.asdf similarity index 100% rename from test/chunking.asdf rename to test/data/chunking.asdf diff --git a/test/test-read.jl b/test/test-read.jl index bd25b6a..6bf89e3 100644 --- a/test/test-read.jl +++ b/test/test-read.jl @@ -1,5 +1,5 @@ @testset "Read ASDF file" begin - asdf = ASDF.load_file("blue_upchan_gain.00000000.asdf") + asdf = ASDF.load_file(joinpath("data", "blue_upchan_gain.00000000.asdf")) println(YAML.write(asdf.metadata)) map_tree(output, asdf.metadata) diff --git a/test/test-read_chunked.jl b/test/test-read_chunked.jl index b7a101c..9efab4c 100644 --- a/test/test-read_chunked.jl +++ b/test/test-read_chunked.jl @@ -1,5 +1,5 @@ @testset "Read ASDF file with chunked arrays" begin - asdf = ASDF.load_file("chunking.asdf") + asdf = ASDF.load_file(joinpath("data", "chunking.asdf")) println(YAML.write(asdf.metadata)) map_tree(output, asdf.metadata) diff --git a/test/test-reference.jl b/test/test-reference.jl index 40acb0d..1ce08d1 100644 --- a/test/test-reference.jl +++ b/test/test-reference.jl @@ -11,30 +11,43 @@ function test_fields(af1, af2) end end -function roundtrip(fpath) - af = ASDF.load_file(fpath; extensions = false, validate_checksum = true) +function roundtrip(fpath; extensions = false, validate_checksum = true) + af = ASDF.load_file(fpath; extensions, validate_checksum) fpath_roundtrip = replace(fpath, ".asdf" => "_roundtrip.asdf") ASDF.write_file(fpath_roundtrip, af.metadata) - af_roundtrip = ASDF.load_file(fpath_roundtrip; extensions = false, validate_checksum = true) + af_roundtrip = ASDF.load_file(fpath_roundtrip; extensions, validate_checksum) return af_roundtrip, af end function test_references(references) for reference in references @testset "$(reference)" begin - af_roundtrip, af = roundtrip(joinpath("data", reference * ".asdf")) + af_roundtrip, af = if reference == "compressed" + # Bug on Python side, see 03 Apr ASDF office hour discussion + roundtrip(joinpath("data", "asdf-1.6.0", reference * ".asdf"); validate_checksum = false) + else + roundtrip(joinpath("data", "asdf-1.6.0", reference * ".asdf")) + end test_fields(af_roundtrip, af) end end end -function yea() - references = [ - "anchor", - "ascii", - "basic", - "complex", - ] - - test_references(references) -end +references = [ + "anchor", + "ascii", + "basic", + "complex", + "compressed", + "endian", + #"exploded", See https://github.com/JuliaAstro/ASDF.jl/issues/31 + "float", + "int", + "scalars", + "shared", + #"stream", See https://github.com/JuliaAstro/ASDF.jl/issues/31 + "structured", + "unicode_bmp", + "unicode_spp", +] +test_references(references) From c04c88ec48d4c44a4a8f9f00348f1d3a3cf79e01 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 4 Apr 2026 02:17:09 -0700 Subject: [PATCH 08/11] docs: up --- src/ASDF.jl | 122 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 116 insertions(+), 6 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index 44047fe..d66a578 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -352,36 +352,120 @@ asdf_datatype_yaml(dt::Datatype) = string(dt) # "float32", "uint8", etc. ################################################################################ +""" + AsciiDatatype(length::Int) + +An ASDF fixed-length ASCII string datatype, corresponding to the `["ascii", N]` form in +the ASDF datatype spec. Each element occupies exactly `length` bytes, one byte per +character (all codepoints < 128). + +The corresponding Julia type is `NTuple{N, UInt8}`. Byte order is irrelevant for this +type since each character is a single byte. + +See also: [`Ucs4Datatype`](@ref), [`parse_asdf_datatype`](@ref). +""" struct AsciiDatatype length::Int # Bytes (1 per char) end +""" + Ucs4Datatype(length::Int) + +An ASDF fixed-length UCS-4 string datatype, corresponding to the `["ucs4", N]` form in +the ASDF datatype spec. Each element occupies exactly `4 * length` bytes, with each +character encoded as a 4-byte UInt32 in the array's declared byte order. + +The corresponding Julia type is `NTuple{N, UInt32}`. + +See also: [`AsciiDatatype`](@ref), [`parse_asdf_datatype`](@ref). +""" struct Ucs4Datatype length::Int # Characters (4 bytes each) end +""" + Base.Type(dt::AsciiDatatype) -> NTuple{N, UInt8} + +Return the Julia `isbitstype` corresponding to an ASDF string datatype, suitable for use +with `reinterpret` and `sizeof`. `N` is the number of characters (`dt.length`). +""" Base.Type(dt::AsciiDatatype) = NTuple{dt.length, UInt8} + +""" + Base.Type(dt::Ucs4Datatype) -> NTuple{N, UInt32} + +Return the Julia `isbitstype` corresponding to an ASDF string datatype, suitable for use +with `reinterpret` and `sizeof`. `N` is the number of characters (`dt.length`). +""" Base.Type(dt::Ucs4Datatype) = NTuple{dt.length, UInt32} +""" + asdf_datatype_yaml(dt) -> Union{String, Vector} + +Serialize an ASDF datatype to its YAML-representable form. +""" +asdf_datatype_yaml + asdf_datatype_yaml(dt::AsciiDatatype) = ["ascii", dt.length] asdf_datatype_yaml(dt::Ucs4Datatype) = ["ucs4", dt.length] ################################################################################ """ -The other variant of [Datatype](@ref). + StructuredField(name::String, datatype::Union{Datatype,AsciiDatatype,Ucs4Datatype}, byteorder::Byteorder) + +A single named field within a [`StructuredDatatype`](@ref). Corresponds to one entry in the ASDF `datatype` list when it takes the dict form: + +```yaml +- {name: x, datatype: float32, byteorder: little} +- {name: label, datatype: [ascii, 8], byteorder: little} +``` + +`byteorder` specifies the byte order for this field specifically, overriding the ndarray's top-level `byteorder`. For [`AsciiDatatype`](@ref) fields, `byteorder` is stored but +ignored during byte-swapping since single bytes have no endianness. """ struct StructuredField name::String datatype::Union{Datatype, AsciiDatatype, Ucs4Datatype} byteorder::Byteorder - #shape::Vector{Int64} end +""" + StructuredDatatype(fields::Vector{StructuredField}) + +An ASDF compound datatype consisting of named fields, corresponding to the list-of-dicts form of the `datatype` key in an ndarray: + +```yaml +datatype: +- {name: x, datatype: float32, byteorder: little} +- {name: y, datatype: float32, byteorder: little} +- {name: label, datatype: [ascii, 4], byteorder: little} +``` + +The corresponding Julia type (returned by `Base.Type`) is a `NamedTuple` with one field per entry, e.g., `@NamedTuple{x::Float32, y::Float32, label::NTuple{4,UInt8}}`. This type is `isbitstype`, so `reinterpret`, `sizeof`, and `bswap` all work correctly on it. + +See also: [`StructuredField`](@ref), [`parse_asdf_datatype`](@ref). +""" struct StructuredDatatype fields::Vector{StructuredField} end +""" + Base.Type(sd::StructuredDatatype) -> NamedTuple type + +Return the `NamedTuple` type corresponding to a structured ASDF datatype. Field names and types are derived from `sd.fields` in order. + +# Example + +``` +sd = StructuredDatatype([ + StructuredField("x", Datatype_float32, Byteorder_little), + StructuredField("label", AsciiDatatype(4), Byteorder_little), +]) + +Base.Type(sd) == @NamedTuple{x::Float32, label::NTuple{4,UInt8}} +``` +""" function Base.Type(sd::StructuredDatatype) names = Tuple(Symbol(f.name) for f in sd.fields) types = Tuple{(Type(f.datatype) for f in sd.fields)...} @@ -397,10 +481,22 @@ end ################################################################################ -# For parsing raw YAML -function parse_asdf_datatype(val::AbstractString) - return Datatype(val) -end +""" + parse_asdf_datatype(val) -> Union{Datatype, AsciiDatatype, Ucs4Datatype, StructuredDatatype} + +Parse a raw YAML datatype value into its corresponding ASDF datatype: + +- `AbstractString` (e.g. `"float32"`) --> [`Datatype`](@ref) +- 2-element `AbstractVector` (e.g. `["ascii", 4]`) --> [`AsciiDatatype`](@ref) or [`Ucs4Datatype`](@ref) +- `AbstractVector` of dicts --> [`StructuredDatatype`](@ref) + +Field-level `datatype` values within a structured dtype are parsed recursively, so `["ascii", N]` is valid as a field type. + +This is the inverse of [`asdf_datatype_yaml`](@ref). +""" +parse_asdf_datatype + +parse_asdf_datatype(val::AbstractString) = Datatype(val) function parse_asdf_datatype(val::AbstractVector) # 2-element [kind, length] form: ["ascii", 4] or ["ucs4", 4] @@ -795,6 +891,20 @@ Base.isempty(blocks::Blocks) = isempty(blocks.arrays) && isempty(blocks.position # This means that `write_file` is not thread-safe. const blocks::Blocks = Blocks() +""" + infer_asdf_datatype(T::Type) --> Union{Datatype, AsciiDatatype, Ucs4Datatype, StructuredDatatype} + +Infer the ASDF datatype from a Julia element type, used when writing arrays: + +- `NTuple{N, UInt8}` --> [`AsciiDatatype(N)`](@ref AsciiDatatype) +- `NTuple{N, UInt32}` --> [`Ucs4Datatype(N)`](@ref Ucs4Datatype) +- `NamedTuple` --> [`StructuredDatatype`](@ref) with fields inferred recursively +- Any other type --> [`Datatype`](@ref) via the existing `type_datatype_dict` lookup + +Errors if `T` is not representable as an ASDF datatype. + +See also: [`asdf_datatype_yaml`](@ref). +""" function infer_asdf_datatype(T::Type)::Union{Datatype, AsciiDatatype, Ucs4Datatype, StructuredDatatype} if T <: NTuple{N, UInt8} where N return AsciiDatatype(fieldcount(T)) From c8f8fb81d3a9a57bc4195ce1a95e7a005f461312 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 10 Apr 2026 21:33:08 -0700 Subject: [PATCH 09/11] test: more tests --- src/ASDF.jl | 2 +- test/test-ndarray.jl | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index d66a578..096676e 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -507,7 +507,7 @@ function parse_asdf_datatype(val::AbstractVector) elseif val[1] == "ucs4" return Ucs4Datatype(n) else - error("Unknown string datatype kind: $(val[1])") + throw(ArgumentError("Unknown string datatype kind: $(val[1])")) end end fields = map(val) do d diff --git a/test/test-ndarray.jl b/test/test-ndarray.jl index 702eb82..c5df43e 100644 --- a/test/test-ndarray.jl +++ b/test/test-ndarray.jl @@ -78,6 +78,11 @@ end "`strides` must have only positive elements."; strides = Int64[0], ) + test_ndarray( + ArgumentError, + "Unknown string datatype kind: utf16"; + datatype = ["utf16", 8], + ) end @testset "getindex" begin @@ -93,8 +98,12 @@ end disk_bytes = collect(reinterpret(UInt8, bswap.(expected))) lbh = ASDF.LazyBlockHeaders() push!(lbh.block_headers, make_block_header(disk_bytes)) + nd = make_ndarray(; lazy_block_headers = lbh, source = Int64(0), data = nothing, byteorder = opposite) @test nd[] == expected + + nd = make_ndarray(; lazy_block_headers = lbh, source = Int64(0), data = nothing, byteorder = opposite, datatype = ASDF.Ucs4Datatype(2), shape = [Int64(1)], strides = [Int64(8)]) + @test nd[] == [(UInt32(0x00000001), UInt32(0x00000002))] end nd = make_ndarray(; strides = Int64[5]) From 7f4ba499b9e6c017980df3f5b7c8d700e146b8f8 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Fri, 10 Apr 2026 23:50:22 -0700 Subject: [PATCH 10/11] style: mark dead branches --- src/ASDF.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index 0d82163..2449a8e 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -847,10 +847,11 @@ function Base.getindex(ndarray::NDArray) data = map(elem -> map(bswap, elem), data) end else - error("Unhandled datatype type: $(typeof(ndarray.datatype))") + # Unreachable branch. All variants of `datatype` covered in `parse_asdf_datatype`. + @assert false end else - # Caught in the constructor for `NDArray`. This branch would imply that + # Unreachable branch. Caught in the constructor for `NDArray`. This branch would imply that # `ndarray` is in invalid state; neither `source` nor `data` is given. @assert false end From 330ac1e4475ec13e95ea9b641ebfcd3e82fc7e33 Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Sat, 11 Apr 2026 01:06:21 -0700 Subject: [PATCH 11/11] refactor: use multiple dispatch to handle byte order corrections --- src/ASDF.jl | 83 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index 2449a8e..49eec9e 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -817,39 +817,7 @@ function Base.getindex(ndarray::NDArray) data = reshape(data, shape[2:end]) # Correct byteorder if necessary. # Do this after imposing the datatype since byteorder depends on the datatype. - if ndarray.datatype isa Datatype - if ndarray.byteorder != host_byteorder - map!(bswap, data, data) - end - elseif ndarray.datatype isa StructuredDatatype - needs_swap = any( - !(f.datatype isa AsciiDatatype) && f.byteorder != host_byteorder - for f in ndarray.datatype.fields - ) - if needs_swap - data = map(data) do elem - swapped = map(ndarray.datatype.fields, Tuple(elem)) do field, val - if field.datatype isa AsciiDatatype - val # bytes have no endianness - elseif field.byteorder != host_byteorder - bswap(val) - else - val - end - end - NT(swapped) - end - end - elseif ndarray.datatype isa AsciiDatatype - # AsciiDatatype: no byteswap needed (single bytes have no endianness) - elseif ndarray.datatype isa Ucs4Datatype - if ndarray.byteorder != host_byteorder - data = map(elem -> map(bswap, elem), data) - end - else - # Unreachable branch. All variants of `datatype` covered in `parse_asdf_datatype`. - @assert false - end + data = correct_byteorder(data, ndarray.datatype, ndarray.byteorder) else # Unreachable branch. Caught in the constructor for `NDArray`. This branch would imply that # `ndarray` is in invalid state; neither `source` nor `data` is given. @@ -866,6 +834,55 @@ function Base.getindex(ndarray::NDArray) return data::AbstractArray end +""" + correct_byteorder(data, dt::Union{Datatype, AsciiDatatype, Ucs4Datatype}, byteorder::Byteorder) + +Applies any necessary byteswap to the `data` within `ndarray` after it has been reinterpreted as `Type(dt)`, where `dt = ndarray.datatype`. + +`byteorder` is the ndarray-level [`Byteorder`](@ref). +""" +correct_byteorder + +function correct_byteorder(data, ::Datatype, byteorder::Byteorder) + if byteorder != host_byteorder + map!(bswap, data, data) + end + return data +end + +function correct_byteorder(data, dt::StructuredDatatype, ::Byteorder) + needs_swap = any( + !(f.datatype isa AsciiDatatype) && f.byteorder != host_byteorder + for f in dt.fields + ) + if needs_swap + data = map(data) do elem + swapped = map(dt.fields, Tuple(elem)) do field, val + if field.datatype isa AsciiDatatype + val # bytes have no endianness + elseif field.byteorder != host_byteorder + bswap(val) + else + val + end + end + Type(dt)(swapped) + end + end + return data +end + +function correct_byteorder(data, ::AsciiDatatype, ::Byteorder) + return data +end + +function correct_byteorder(data, ::Ucs4Datatype, byteorder::Byteorder) + if byteorder != host_byteorder + data = map(elem -> map(bswap, elem), data) + end + return data +end + function YAML._print(io::IO, val::NDArray, level::Int = 0, ignore_level::Bool = false) # TODO: Get compression from underlying header block? YAML._print(io, NDArrayWrapper(val[]; compression = C_None), level, ignore_level)