diff --git a/src/ASDF.jl b/src/ASDF.jl index 822199d..ec032c8 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -481,6 +481,177 @@ Base.show(io::IO, datatype::Datatype) = show(io, string(datatype)) Base.Type(datatype::Datatype) = datatype_type_dict[datatype] Datatype(type::Type) = type_datatype_dict[type] +asdf_datatype_yaml(dt::Datatype) = string(dt) # "float32", "uint8", etc. + +################################################################################ + +""" + AsciiDatatype(length::Int) + +An ASDF fixed-length ASCII string datatype, corresponding to the `["ascii", N]` form in +the ASDF datatype spec. Each element occupies exactly `length` bytes, one byte per +character (all codepoints < 128). + +The corresponding Julia type is `NTuple{N, UInt8}`. Byte order is irrelevant for this +type since each character is a single byte. + +See also: [`Ucs4Datatype`](@ref), [`parse_asdf_datatype`](@ref). +""" +struct AsciiDatatype + length::Int # Bytes (1 per char) +end + +""" + Ucs4Datatype(length::Int) + +An ASDF fixed-length UCS-4 string datatype, corresponding to the `["ucs4", N]` form in +the ASDF datatype spec. Each element occupies exactly `4 * length` bytes, with each +character encoded as a 4-byte UInt32 in the array's declared byte order. + +The corresponding Julia type is `NTuple{N, UInt32}`. + +See also: [`AsciiDatatype`](@ref), [`parse_asdf_datatype`](@ref). +""" +struct Ucs4Datatype + length::Int # Characters (4 bytes each) +end + +""" + Base.Type(dt::AsciiDatatype) -> NTuple{N, UInt8} + +Return the Julia `isbitstype` corresponding to an ASDF string datatype, suitable for use +with `reinterpret` and `sizeof`. `N` is the number of characters (`dt.length`). +""" +Base.Type(dt::AsciiDatatype) = NTuple{dt.length, UInt8} + +""" + Base.Type(dt::Ucs4Datatype) -> NTuple{N, UInt32} + +Return the Julia `isbitstype` corresponding to an ASDF string datatype, suitable for use +with `reinterpret` and `sizeof`. `N` is the number of characters (`dt.length`). +""" +Base.Type(dt::Ucs4Datatype) = NTuple{dt.length, UInt32} + +""" + asdf_datatype_yaml(dt) -> Union{String, Vector} + +Serialize an ASDF datatype to its YAML-representable form. +""" +asdf_datatype_yaml + +asdf_datatype_yaml(dt::AsciiDatatype) = ["ascii", dt.length] +asdf_datatype_yaml(dt::Ucs4Datatype) = ["ucs4", dt.length] + +################################################################################ + +""" + StructuredField(name::String, datatype::Union{Datatype,AsciiDatatype,Ucs4Datatype}, byteorder::Byteorder) + +A single named field within a [`StructuredDatatype`](@ref). Corresponds to one entry in the ASDF `datatype` list when it takes the dict form: + +```yaml +- {name: x, datatype: float32, byteorder: little} +- {name: label, datatype: [ascii, 8], byteorder: little} +``` + +`byteorder` specifies the byte order for this field specifically, overriding the ndarray's top-level `byteorder`. For [`AsciiDatatype`](@ref) fields, `byteorder` is stored but +ignored during byte-swapping since single bytes have no endianness. +""" +struct StructuredField + name::String + datatype::Union{Datatype, AsciiDatatype, Ucs4Datatype} + byteorder::Byteorder +end + +""" + StructuredDatatype(fields::Vector{StructuredField}) + +An ASDF compound datatype consisting of named fields, corresponding to the list-of-dicts form of the `datatype` key in an ndarray: + +```yaml +datatype: +- {name: x, datatype: float32, byteorder: little} +- {name: y, datatype: float32, byteorder: little} +- {name: label, datatype: [ascii, 4], byteorder: little} +``` + +The corresponding Julia type (returned by `Base.Type`) is a `NamedTuple` with one field per entry, e.g., `@NamedTuple{x::Float32, y::Float32, label::NTuple{4,UInt8}}`. This type is `isbitstype`, so `reinterpret`, `sizeof`, and `bswap` all work correctly on it. + +See also: [`StructuredField`](@ref), [`parse_asdf_datatype`](@ref). +""" +struct StructuredDatatype + fields::Vector{StructuredField} +end + +""" + Base.Type(sd::StructuredDatatype) -> NamedTuple type + +Return the `NamedTuple` type corresponding to a structured ASDF datatype. Field names and types are derived from `sd.fields` in order. + +# Example + +``` +sd = StructuredDatatype([ + StructuredField("x", Datatype_float32, Byteorder_little), + StructuredField("label", AsciiDatatype(4), Byteorder_little), +]) + +Base.Type(sd) == @NamedTuple{x::Float32, label::NTuple{4,UInt8}} +``` +""" +function Base.Type(sd::StructuredDatatype) + names = Tuple(Symbol(f.name) for f in sd.fields) + types = Tuple{(Type(f.datatype) for f in sd.fields)...} + return NamedTuple{names, types} +end + +function asdf_datatype_yaml(dt::StructuredDatatype) + return map(dt.fields) do f + # To-do: replace with OrderedDict after https://github.com/JuliaAstro/ASDF.jl/pull/26 + Dict("name" => f.name, "datatype" => asdf_datatype_yaml(f.datatype), "byteorder" => string(f.byteorder)) + end +end + +################################################################################ + +""" + parse_asdf_datatype(val) -> Union{Datatype, AsciiDatatype, Ucs4Datatype, StructuredDatatype} + +Parse a raw YAML datatype value into its corresponding ASDF datatype: + +- `AbstractString` (e.g. `"float32"`) --> [`Datatype`](@ref) +- 2-element `AbstractVector` (e.g. `["ascii", 4]`) --> [`AsciiDatatype`](@ref) or [`Ucs4Datatype`](@ref) +- `AbstractVector` of dicts --> [`StructuredDatatype`](@ref) + +Field-level `datatype` values within a structured dtype are parsed recursively, so `["ascii", N]` is valid as a field type. + +This is the inverse of [`asdf_datatype_yaml`](@ref). +""" +parse_asdf_datatype + +parse_asdf_datatype(val::AbstractString) = Datatype(val) + +function parse_asdf_datatype(val::AbstractVector) + # 2-element [kind, length] form: ["ascii", 4] or ["ucs4", 4] + if length(val) == 2 && val[1] isa AbstractString && val[2] isa Integer + n = Int(val[2]) + if val[1] == "ascii" + return AsciiDatatype(n) + elseif val[1] == "ucs4" + return Ucs4Datatype(n) + else + throw(ArgumentError("Unknown string datatype kind: $(val[1])")) + end + end + fields = map(val) do d + name = d["name"]::String + dt = parse_asdf_datatype(d["datatype"]) + bo = haskey(d, "byteorder") ? Byteorder(d["byteorder"]::String) : host_byteorder + StructuredField(name, dt, bo) + end + return StructuredDatatype(fields) +end + ################################################################################ """ @@ -508,7 +679,7 @@ struct NDArray source::Union{Nothing,Int64,AbstractString} data::Union{Nothing,AbstractArray} shape::Vector{Int64} - datatype::Datatype + datatype::Union{Datatype,StructuredDatatype,AsciiDatatype,Ucs4Datatype} byteorder::Byteorder offset::Int64 strides::Vector{Int64} # stored in ASDF (Python/C) order, not in Julia (Fortran) order @@ -519,7 +690,7 @@ struct NDArray source::Union{Nothing,Int64,AbstractString}, data::Union{Nothing,AbstractArray}, shape::Vector{Int64}, - datatype::Datatype, + datatype::Union{Datatype,StructuredDatatype,AsciiDatatype,Ucs4Datatype}, byteorder::Byteorder, offset::Int64, strides::Vector{Int64}, @@ -559,7 +730,7 @@ function NDArray( source::Union{Nothing,Integer}, data::Union{Nothing,AbstractArray}, shape::AbstractVector{<:Integer}, - datatype::Union{Datatype,AbstractString}, + datatype::Union{Datatype,StructuredDatatype,AsciiDatatype,Ucs4Datatype,AbstractString,AbstractVector}, byteorder::Union{Nothing,Byteorder,AbstractString}, offset::Union{Nothing,Integer}=0, strides::Union{Nothing,<:AbstractVector{<:Integer}}=nothing, @@ -567,8 +738,8 @@ function NDArray( if source isa Integer source = Int64(source) end - if datatype isa AbstractString - datatype = Datatype(datatype) + if datatype isa AbstractString || datatype isa AbstractVector + datatype = parse_asdf_datatype(datatype) end if data !== nothing # Convert arrays of arrays into multi-dimensional arrays @@ -601,7 +772,7 @@ function make_construct_yaml_ndarray(block_headers::LazyBlockHeaders) source = get(mapping, "source", nothing)::Union{Nothing,Integer} data = get(mapping, "data", nothing)::Union{Nothing,AbstractVector} shape = mapping["shape"]::AbstractVector{<:Integer} - datatype = mapping["datatype"]::AbstractString + datatype = mapping["datatype"] # No annotation needed, `parse_asdf_datatype` handles dispatch byteorder = get(mapping, "byteorder", nothing)::Union{Nothing,AbstractString} offset = get(mapping, "offset", nothing)::Union{Nothing,Integer} strides = get(mapping, "strides", nothing)::Union{Nothing,AbstractVector{<:Integer}} @@ -640,16 +811,15 @@ function Base.getindex(ndarray::NDArray) strides = (1, reverse(ndarray.strides)...) data = StridedView(data, Int.(shape), Int.(strides), Int(ndarray.offset)) # Impose datatype - data = reinterpret(Type(ndarray.datatype), data) + NT = Type(ndarray.datatype) + data = reinterpret(NT, data) # Remove the new dimension again data = reshape(data, shape[2:end]) # Correct byteorder if necessary. # Do this after imposing the datatype since byteorder depends on the datatype. - if ndarray.byteorder != host_byteorder - map!(bswap, data, data) - end + data = correct_byteorder(data, ndarray.datatype, ndarray.byteorder) else - # Caught in the constructor for `NDArray`. This branch would imply that + # Unreachable branch. Caught in the constructor for `NDArray`. This branch would imply that # `ndarray` is in invalid state; neither `source` nor `data` is given. @assert false end @@ -664,6 +834,60 @@ function Base.getindex(ndarray::NDArray) return data::AbstractArray end +""" + correct_byteorder(data, dt::Union{Datatype, AsciiDatatype, Ucs4Datatype}, byteorder::Byteorder) + +Applies any necessary byteswap to the `data` within `ndarray` after it has been reinterpreted as `Type(dt)`, where `dt = ndarray.datatype`. + +`byteorder` is the ndarray-level [`Byteorder`](@ref). +""" +correct_byteorder + +function correct_byteorder(data, ::Datatype, byteorder::Byteorder) + if byteorder != host_byteorder + map!(bswap, data, data) + end + return data +end + +function correct_byteorder(data, dt::StructuredDatatype, ::Byteorder) + needs_swap = any( + !(f.datatype isa AsciiDatatype) && f.byteorder != host_byteorder + for f in dt.fields + ) + if needs_swap + data = map(data) do elem + swapped = map(dt.fields, Tuple(elem)) do field, val + if field.datatype isa AsciiDatatype + val # bytes have no endianness + elseif field.byteorder != host_byteorder + bswap(val) + else + val + end + end + Type(dt)(swapped) + end + end + return data +end + +function correct_byteorder(data, ::AsciiDatatype, ::Byteorder) + return data +end + +function correct_byteorder(data, ::Ucs4Datatype, byteorder::Byteorder) + if byteorder != host_byteorder + data = map(elem -> map(bswap, elem), data) + end + return data +end + +function YAML._print(io::IO, val::NDArray, level::Int = 0, ignore_level::Bool = false) + # TODO: Get compression from underlying header block? + YAML._print(io, NDArrayWrapper(val[]; compression = C_None), level, ignore_level) +end + ################################################################################ """ @@ -724,7 +948,7 @@ A logical N-dimensional array assembled from a collection of arbitrarily-positio """ struct ChunkedNDArray shape::Vector{Int64} - datatype::Datatype + datatype::Union{Datatype, StructuredDatatype, AsciiDatatype, Ucs4Datatype} chunks::AbstractVector{NDArrayChunk} function ChunkedNDArray(shape::Vector{Int64}, datatype::Datatype, chunks::Vector{NDArrayChunk}) @@ -751,9 +975,9 @@ struct ChunkedNDArray end function ChunkedNDArray( - shape::AbstractVector{<:Integer}, datatype::Union{Datatype,AbstractString}, chunks::AbstractVector{NDArrayChunk} + shape::AbstractVector{<:Integer}, datatype::Union{Datatype,StructuredDatatype,AsciiDatatype,Ucs4Datatype,AbstractString,AbstractVector}, chunks::AbstractVector{NDArrayChunk} ) - if datatype isa AbstractString + if datatype isa AbstractString || datatype isa AbstractVector datatype = Datatype(datatype) end return ChunkedNDArray(Vector{Int64}(shape), datatype, chunks) @@ -763,7 +987,7 @@ function make_construct_yaml_chunked_ndarray(block_headers::LazyBlockHeaders) function construct_yaml_chunked_ndarray(constructor::YAML.Constructor, node::YAML.Node) mapping = YAML.construct_mapping(constructor, node) shape = mapping["shape"]::AbstractVector{<:Integer} - datatype = mapping["datatype"]::AbstractString + datatype = mapping["datatype"] chunks = mapping["chunks"]::AbstractVector{NDArrayChunk} return ChunkedNDArray(shape, datatype, chunks) end @@ -1100,7 +1324,37 @@ Base.isempty(blocks::Blocks) = isempty(blocks.arrays) && isempty(blocks.position # This means that `write_file` is not thread-safe. const blocks::Blocks = Blocks() +""" + infer_asdf_datatype(T::Type) --> Union{Datatype, AsciiDatatype, Ucs4Datatype, StructuredDatatype} + +Infer the ASDF datatype from a Julia element type, used when writing arrays: + +- `NTuple{N, UInt8}` --> [`AsciiDatatype(N)`](@ref AsciiDatatype) +- `NTuple{N, UInt32}` --> [`Ucs4Datatype(N)`](@ref Ucs4Datatype) +- `NamedTuple` --> [`StructuredDatatype`](@ref) with fields inferred recursively +- Any other type --> [`Datatype`](@ref) via the existing `type_datatype_dict` lookup + +Errors if `T` is not representable as an ASDF datatype. + +See also: [`asdf_datatype_yaml`](@ref). +""" +function infer_asdf_datatype(T::Type)::Union{Datatype, AsciiDatatype, Ucs4Datatype, StructuredDatatype} + if T <: NTuple{N, UInt8} where N + return AsciiDatatype(fieldcount(T)) + elseif T <: NTuple{N, UInt32} where N + return Ucs4Datatype(fieldcount(T)) + elseif T <: NamedTuple + fields = map(fieldnames(T), fieldtypes(T)) do name, FT + StructuredField(string(name), infer_asdf_datatype(FT), host_byteorder) + end + return StructuredDatatype(collect(fields)) + else + return Datatype(T) # existing dict lookup, errors on unknown types + end +end + function YAML._print(io::IO, val::NDArrayWrapper, level::Int=0, ignore_level::Bool=false) + datatype = infer_asdf_datatype(eltype(val.array)) if val.inline data = val.array # Split multidimensional arrays into array-of-arrays @@ -1108,7 +1362,7 @@ function YAML._print(io::IO, val::NDArrayWrapper, level::Int=0, ignore_level::Bo ndarray = OrderedDict( :data => data, :shape => collect(reverse(size(val.array)))::Vector{<:Integer}, - :datatype => string(Datatype(eltype(val.array))), + :datatype => asdf_datatype_yaml(datatype),#string(Datatype(eltype(val.array))), # :offset => 0::Integer, # :strides => ::Vector{Int64}, ) @@ -1120,7 +1374,7 @@ function YAML._print(io::IO, val::NDArrayWrapper, level::Int=0, ignore_level::Bo ndarray = OrderedDict( :source => source::Integer, :shape => collect(reverse(size(val.array)))::Vector{<:Integer}, - :datatype => string(Datatype(eltype(val.array))), + :datatype => asdf_datatype_yaml(datatype), :byteorder => string(host_byteorder::Byteorder), # :offset => 0::Integer, # :strides => ::Vector{Int64}, @@ -1224,7 +1478,6 @@ function write_file(filename::AbstractString, document::AbstractDict) header_size = 48 flags = 0 # not streamed compression = compression_keys[array.compression] - data_size = sizeof(array.array) # Write block # TODO: create function write_block @@ -1237,6 +1490,8 @@ function write_file(filename::AbstractString, document::AbstractDict) # Reinterpret as UInt8 input = reinterpret(UInt8, input) + data_size = UInt64(length(input)) + # TODO: Write directly to file if array.compression == C_None data = input diff --git a/test/data/asdf-1.6.0/anchor.asdf b/test/data/asdf-1.6.0/anchor.asdf new file mode 100644 index 0000000..09ac78e --- /dev/null +++ b/test/data/asdf-1.6.0/anchor.asdf @@ -0,0 +1,17 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.6.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.1.0 +asdf_library: !core/software-1.0.0 {author: The ASDF Developers, homepage: 'http://github.com/asdf-format/asdf', + name: asdf, version: 4.1.0} +history: + extensions: + - !core/extension_metadata-1.0.0 + extension_class: asdf.extension._manifest.ManifestExtension + extension_uri: asdf://asdf-format.org/core/extensions/core-1.6.0 + manifest_software: !core/software-1.0.0 {name: asdf_standard, version: 1.1.1} + software: !core/software-1.0.0 {name: asdf, version: 4.1.0} +a: &id001 {abc: 123} +b: *id001 +... diff --git a/test/data/asdf-1.6.0/anchor_roundtrip.asdf b/test/data/asdf-1.6.0/anchor_roundtrip.asdf new file mode 100644 index 0000000..77858f7 --- /dev/null +++ b/test/data/asdf-1.6.0/anchor_roundtrip.asdf @@ -0,0 +1,37 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.2.0 +# This is an ASDF file +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- +!core/asdf-1.1.0 +asdf_library: + author: "The ASDF Developers" + homepage: "http://github.com/asdf-format/asdf" + name: "asdf" + version: "4.1.0" +history: + extensions: + - extension_class: "asdf.extension._manifest.ManifestExtension" + extension_uri: "asdf://asdf-format.org/core/extensions/core-1.6.0" + manifest_software: + name: "asdf_standard" + version: "1.1.1" + software: + name: "asdf" + version: "4.1.0" +a: + abc: 123 +b: + abc: 123 +asdf/library: !core/software-1.0.0 + name: "ASDF.jl" + author: "Erik Schnetter " + homepage: "https://github.com/JuliaAstro/ASDF.jl" + version: "2.0.0" +... +#ASDF BLOCK INDEX +%YAML 1.1 +--- +[] +... diff --git a/test/data/asdf-1.6.0/ascii.asdf b/test/data/asdf-1.6.0/ascii.asdf new file mode 100644 index 0000000..7509922 Binary files /dev/null and b/test/data/asdf-1.6.0/ascii.asdf differ diff --git a/test/data/asdf-1.6.0/ascii_roundtrip.asdf b/test/data/asdf-1.6.0/ascii_roundtrip.asdf new file mode 100644 index 0000000..48f4071 Binary files /dev/null and b/test/data/asdf-1.6.0/ascii_roundtrip.asdf differ diff --git a/test/data/asdf-1.6.0/basic.asdf b/test/data/asdf-1.6.0/basic.asdf new file mode 100644 index 0000000..b39b0f5 Binary files /dev/null and b/test/data/asdf-1.6.0/basic.asdf differ diff --git a/test/data/asdf-1.6.0/basic_roundtrip.asdf b/test/data/asdf-1.6.0/basic_roundtrip.asdf new file mode 100644 index 0000000..316a9e0 Binary files /dev/null and b/test/data/asdf-1.6.0/basic_roundtrip.asdf differ diff --git a/test/data/asdf-1.6.0/complex.asdf b/test/data/asdf-1.6.0/complex.asdf new file mode 100644 index 0000000..07fa151 Binary files /dev/null and b/test/data/asdf-1.6.0/complex.asdf differ diff --git a/test/data/asdf-1.6.0/complex_roundtrip.asdf b/test/data/asdf-1.6.0/complex_roundtrip.asdf new file mode 100644 index 0000000..fcb49ff Binary files /dev/null and b/test/data/asdf-1.6.0/complex_roundtrip.asdf differ diff --git a/test/data/asdf-1.6.0/compressed.asdf b/test/data/asdf-1.6.0/compressed.asdf new file mode 100644 index 0000000..2480bb5 Binary files /dev/null and b/test/data/asdf-1.6.0/compressed.asdf differ diff --git a/test/data/asdf-1.6.0/compressed_roundtrip.asdf b/test/data/asdf-1.6.0/compressed_roundtrip.asdf new file mode 100644 index 0000000..2530ec2 Binary files /dev/null and b/test/data/asdf-1.6.0/compressed_roundtrip.asdf differ diff --git a/test/data/asdf-1.6.0/endian.asdf b/test/data/asdf-1.6.0/endian.asdf new file mode 100644 index 0000000..1e72f02 Binary files /dev/null and b/test/data/asdf-1.6.0/endian.asdf differ diff --git a/test/data/asdf-1.6.0/endian_roundtrip.asdf b/test/data/asdf-1.6.0/endian_roundtrip.asdf new file mode 100644 index 0000000..68da0e6 Binary files /dev/null and b/test/data/asdf-1.6.0/endian_roundtrip.asdf differ diff --git a/test/data/asdf-1.6.0/exploded.asdf b/test/data/asdf-1.6.0/exploded.asdf new file mode 100644 index 0000000..157f895 --- /dev/null +++ b/test/data/asdf-1.6.0/exploded.asdf @@ -0,0 +1,20 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.6.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.1.0 +asdf_library: !core/software-1.0.0 {author: The ASDF Developers, homepage: 'http://github.com/asdf-format/asdf', + name: asdf, version: 4.1.0} +history: + extensions: + - !core/extension_metadata-1.0.0 + extension_class: asdf.extension._manifest.ManifestExtension + extension_uri: asdf://asdf-format.org/core/extensions/core-1.6.0 + manifest_software: !core/software-1.0.0 {name: asdf_standard, version: 1.1.1} + software: !core/software-1.0.0 {name: asdf, version: 4.1.0} +data: !core/ndarray-1.1.0 + source: exploded0000.asdf + datatype: int64 + byteorder: little + shape: [8] +... diff --git a/test/data/asdf-1.6.0/exploded0000.asdf b/test/data/asdf-1.6.0/exploded0000.asdf new file mode 100644 index 0000000..054f2af Binary files /dev/null and b/test/data/asdf-1.6.0/exploded0000.asdf differ diff --git a/test/data/asdf-1.6.0/float.asdf b/test/data/asdf-1.6.0/float.asdf new file mode 100644 index 0000000..f4c2dc9 Binary files /dev/null and b/test/data/asdf-1.6.0/float.asdf differ diff --git a/test/data/asdf-1.6.0/float_roundtrip.asdf b/test/data/asdf-1.6.0/float_roundtrip.asdf new file mode 100644 index 0000000..71387f5 Binary files /dev/null and b/test/data/asdf-1.6.0/float_roundtrip.asdf differ diff --git a/test/data/asdf-1.6.0/int.asdf b/test/data/asdf-1.6.0/int.asdf new file mode 100644 index 0000000..193a417 Binary files /dev/null and b/test/data/asdf-1.6.0/int.asdf differ diff --git a/test/data/asdf-1.6.0/int_roundtrip.asdf b/test/data/asdf-1.6.0/int_roundtrip.asdf new file mode 100644 index 0000000..aa8ff0e Binary files /dev/null and b/test/data/asdf-1.6.0/int_roundtrip.asdf differ diff --git a/test/data/asdf-1.6.0/scalars.asdf b/test/data/asdf-1.6.0/scalars.asdf new file mode 100644 index 0000000..f18c657 --- /dev/null +++ b/test/data/asdf-1.6.0/scalars.asdf @@ -0,0 +1,18 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.6.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.1.0 +asdf_library: !core/software-1.0.0 {author: The ASDF Developers, homepage: 'http://github.com/asdf-format/asdf', + name: asdf, version: 4.1.0} +history: + extensions: + - !core/extension_metadata-1.0.0 + extension_class: asdf.extension._manifest.ManifestExtension + extension_uri: asdf://asdf-format.org/core/extensions/core-1.6.0 + manifest_software: !core/software-1.0.0 {name: asdf_standard, version: 1.1.1} + software: !core/software-1.0.0 {name: asdf, version: 4.1.0} +float: 3.14 +int: 42 +string: foo +... diff --git a/test/data/asdf-1.6.0/scalars_roundtrip.asdf b/test/data/asdf-1.6.0/scalars_roundtrip.asdf new file mode 100644 index 0000000..0709575 --- /dev/null +++ b/test/data/asdf-1.6.0/scalars_roundtrip.asdf @@ -0,0 +1,36 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.2.0 +# This is an ASDF file +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- +!core/asdf-1.1.0 +asdf_library: + author: "The ASDF Developers" + homepage: "http://github.com/asdf-format/asdf" + name: "asdf" + version: "4.1.0" +history: + extensions: + - extension_class: "asdf.extension._manifest.ManifestExtension" + extension_uri: "asdf://asdf-format.org/core/extensions/core-1.6.0" + manifest_software: + name: "asdf_standard" + version: "1.1.1" + software: + name: "asdf" + version: "4.1.0" +float: 3.14 +int: 42 +string: "foo" +asdf/library: !core/software-1.0.0 + name: "ASDF.jl" + author: "Erik Schnetter " + homepage: "https://github.com/JuliaAstro/ASDF.jl" + version: "2.0.0" +... +#ASDF BLOCK INDEX +%YAML 1.1 +--- +[] +... diff --git a/test/data/asdf-1.6.0/shared.asdf b/test/data/asdf-1.6.0/shared.asdf new file mode 100644 index 0000000..175607b Binary files /dev/null and b/test/data/asdf-1.6.0/shared.asdf differ diff --git a/test/data/asdf-1.6.0/shared_roundtrip.asdf b/test/data/asdf-1.6.0/shared_roundtrip.asdf new file mode 100644 index 0000000..0841e19 Binary files /dev/null and b/test/data/asdf-1.6.0/shared_roundtrip.asdf differ diff --git a/test/data/asdf-1.6.0/stream.asdf b/test/data/asdf-1.6.0/stream.asdf new file mode 100644 index 0000000..06212fc Binary files /dev/null and b/test/data/asdf-1.6.0/stream.asdf differ diff --git a/test/data/asdf-1.6.0/structured.asdf b/test/data/asdf-1.6.0/structured.asdf new file mode 100644 index 0000000..ae2549f Binary files /dev/null and b/test/data/asdf-1.6.0/structured.asdf differ diff --git a/test/data/asdf-1.6.0/structured_roundtrip.asdf b/test/data/asdf-1.6.0/structured_roundtrip.asdf new file mode 100644 index 0000000..b37dd05 Binary files /dev/null and b/test/data/asdf-1.6.0/structured_roundtrip.asdf differ diff --git a/test/data/asdf-1.6.0/unicode_bmp.asdf b/test/data/asdf-1.6.0/unicode_bmp.asdf new file mode 100644 index 0000000..c3fce7a Binary files /dev/null and b/test/data/asdf-1.6.0/unicode_bmp.asdf differ diff --git a/test/data/asdf-1.6.0/unicode_bmp_roundtrip.asdf b/test/data/asdf-1.6.0/unicode_bmp_roundtrip.asdf new file mode 100644 index 0000000..a6f1f30 Binary files /dev/null and b/test/data/asdf-1.6.0/unicode_bmp_roundtrip.asdf differ diff --git a/test/data/asdf-1.6.0/unicode_spp.asdf b/test/data/asdf-1.6.0/unicode_spp.asdf new file mode 100644 index 0000000..b89856a Binary files /dev/null and b/test/data/asdf-1.6.0/unicode_spp.asdf differ diff --git a/test/data/asdf-1.6.0/unicode_spp_roundtrip.asdf b/test/data/asdf-1.6.0/unicode_spp_roundtrip.asdf new file mode 100644 index 0000000..f99c51b Binary files /dev/null and b/test/data/asdf-1.6.0/unicode_spp_roundtrip.asdf differ diff --git a/test/blue_upchan_gain.00000000.asdf b/test/data/blue_upchan_gain.00000000.asdf similarity index 100% rename from test/blue_upchan_gain.00000000.asdf rename to test/data/blue_upchan_gain.00000000.asdf diff --git a/test/chunking.asdf b/test/data/chunking.asdf similarity index 100% rename from test/chunking.asdf rename to test/data/chunking.asdf diff --git a/test/test-ndarray.jl b/test/test-ndarray.jl index 702eb82..c5df43e 100644 --- a/test/test-ndarray.jl +++ b/test/test-ndarray.jl @@ -78,6 +78,11 @@ end "`strides` must have only positive elements."; strides = Int64[0], ) + test_ndarray( + ArgumentError, + "Unknown string datatype kind: utf16"; + datatype = ["utf16", 8], + ) end @testset "getindex" begin @@ -93,8 +98,12 @@ end disk_bytes = collect(reinterpret(UInt8, bswap.(expected))) lbh = ASDF.LazyBlockHeaders() push!(lbh.block_headers, make_block_header(disk_bytes)) + nd = make_ndarray(; lazy_block_headers = lbh, source = Int64(0), data = nothing, byteorder = opposite) @test nd[] == expected + + nd = make_ndarray(; lazy_block_headers = lbh, source = Int64(0), data = nothing, byteorder = opposite, datatype = ASDF.Ucs4Datatype(2), shape = [Int64(1)], strides = [Int64(8)]) + @test nd[] == [(UInt32(0x00000001), UInt32(0x00000002))] end nd = make_ndarray(; strides = Int64[5]) diff --git a/test/test-read.jl b/test/test-read.jl index 88137be..cb8b44c 100644 --- a/test/test-read.jl +++ b/test/test-read.jl @@ -1,5 +1,5 @@ @testset "Read ASDF file" begin - asdf = load("blue_upchan_gain.00000000.asdf") + asdf = load(joinpath("data", "blue_upchan_gain.00000000.asdf")) println(YAML.write(asdf.metadata)) map_tree(output, asdf.metadata) diff --git a/test/test-read_chunked.jl b/test/test-read_chunked.jl index ca0dbef..5818533 100644 --- a/test/test-read_chunked.jl +++ b/test/test-read_chunked.jl @@ -1,5 +1,5 @@ @testset "Read ASDF file with chunked arrays" begin - asdf = load("chunking.asdf") + asdf = load(joinpath("data", "chunking.asdf")) println(YAML.write(asdf.metadata)) map_tree(output, asdf.metadata) diff --git a/test/test-reference.jl b/test/test-reference.jl new file mode 100644 index 0000000..1ce08d1 --- /dev/null +++ b/test/test-reference.jl @@ -0,0 +1,53 @@ +compare(field1::ASDF.NDArray, field2::ASDF.NDArray) = isequal(field1[], field2[]) +compare(field1, field2) = isequal(field1, field2) + +function test_fields(af1, af2) + for ((k1, v1), (k2, v2)) in zip(af1.metadata, af2.metadata) + if occursin("asdf", k1) + # Skip non-data entries + else + @test compare(v1, v2) + end + end +end + +function roundtrip(fpath; extensions = false, validate_checksum = true) + af = ASDF.load_file(fpath; extensions, validate_checksum) + fpath_roundtrip = replace(fpath, ".asdf" => "_roundtrip.asdf") + ASDF.write_file(fpath_roundtrip, af.metadata) + af_roundtrip = ASDF.load_file(fpath_roundtrip; extensions, validate_checksum) + return af_roundtrip, af +end + +function test_references(references) + for reference in references + @testset "$(reference)" begin + af_roundtrip, af = if reference == "compressed" + # Bug on Python side, see 03 Apr ASDF office hour discussion + roundtrip(joinpath("data", "asdf-1.6.0", reference * ".asdf"); validate_checksum = false) + else + roundtrip(joinpath("data", "asdf-1.6.0", reference * ".asdf")) + end + test_fields(af_roundtrip, af) + end + end +end + +references = [ + "anchor", + "ascii", + "basic", + "complex", + "compressed", + "endian", + #"exploded", See https://github.com/JuliaAstro/ASDF.jl/issues/31 + "float", + "int", + "scalars", + "shared", + #"stream", See https://github.com/JuliaAstro/ASDF.jl/issues/31 + "structured", + "unicode_bmp", + "unicode_spp", +] +test_references(references) diff --git a/test/test-show.jl b/test/test-show.jl index eb965b3..f351382 100644 --- a/test/test-show.jl +++ b/test/test-show.jl @@ -1,5 +1,5 @@ @testset "Show method for `ASDF.ASDFFile`" begin - af = load("blue_upchan_gain.00000000.asdf") + af = load(joinpath("data", "blue_upchan_gain.00000000.asdf")) @test occursin("blue_upchan_gain.00000000.asdf\nā”œā”€", sprint(show, MIME"text/plain"(), af))