Skip to content

Fix EnumSerializer.write to accept numpy scalars from NDArray of records #285

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SAY-5 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Fix EnumSerializer.write to accept numpy scalars from NDArray of records #285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
58785a3
Fix enum fields in NDArray of records on the Python numpy serializati…
SAY-5 May 19, 2026
82221ad
Read all record fields via read_numpy in RecordSerializer.read_numpy
SAY-5 Jun 3, 2026
e860ffa
Fix numpy record write/read dispatch and round-trip fixed vectors in …
SAY-5 Jun 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions cpp/test/generated/binary/protocols.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4139,6 +4139,26 @@ void EnumsWriter::WriteRecImpl(test_model::RecordWithEnums const& value) {
test_model::binary::WriteRecordWithEnums(stream_, value);
}

void EnumsWriter::WriteRecArrayImpl(yardl::DynamicNDArray<test_model::RecordWithEnums> const& value) {
yardl::binary::WriteDynamicNDArray<test_model::RecordWithEnums, test_model::binary::WriteRecordWithEnums>(stream_, value);
}

void EnumsWriter::WriteRecWithFixedVectorsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithFixedVectors> const& value) {
yardl::binary::WriteDynamicNDArray<test_model::RecordWithFixedVectors, test_model::binary::WriteRecordWithFixedVectors>(stream_, value);
}

void EnumsWriter::WriteRecWithOptionalFieldsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithOptionalFields> const& value) {
yardl::binary::WriteDynamicNDArray<test_model::RecordWithOptionalFields, test_model::binary::WriteRecordWithOptionalFields>(stream_, value);
}

void EnumsWriter::WriteRecWithVlensArrayImpl(yardl::DynamicNDArray<test_model::RecordWithVlens> const& value) {
yardl::binary::WriteDynamicNDArray<test_model::RecordWithVlens, test_model::binary::WriteRecordWithVlens>(stream_, value);
}

void EnumsWriter::WriteRecWithStringsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithStrings> const& value) {
yardl::binary::WriteDynamicNDArray<test_model::RecordWithStrings, test_model::binary::WriteRecordWithStrings>(stream_, value);
}

void EnumsWriter::Flush() {
stream_.Flush();
}
Expand All @@ -4163,6 +4183,26 @@ void EnumsReader::ReadRecImpl(test_model::RecordWithEnums& value) {
test_model::binary::ReadRecordWithEnums(stream_, value);
}

void EnumsReader::ReadRecArrayImpl(yardl::DynamicNDArray<test_model::RecordWithEnums>& value) {
yardl::binary::ReadDynamicNDArray<test_model::RecordWithEnums, test_model::binary::ReadRecordWithEnums>(stream_, value);
}

void EnumsReader::ReadRecWithFixedVectorsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithFixedVectors>& value) {
yardl::binary::ReadDynamicNDArray<test_model::RecordWithFixedVectors, test_model::binary::ReadRecordWithFixedVectors>(stream_, value);
}

void EnumsReader::ReadRecWithOptionalFieldsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithOptionalFields>& value) {
yardl::binary::ReadDynamicNDArray<test_model::RecordWithOptionalFields, test_model::binary::ReadRecordWithOptionalFields>(stream_, value);
}

void EnumsReader::ReadRecWithVlensArrayImpl(yardl::DynamicNDArray<test_model::RecordWithVlens>& value) {
yardl::binary::ReadDynamicNDArray<test_model::RecordWithVlens, test_model::binary::ReadRecordWithVlens>(stream_, value);
}

void EnumsReader::ReadRecWithStringsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithStrings>& value) {
yardl::binary::ReadDynamicNDArray<test_model::RecordWithStrings, test_model::binary::ReadRecordWithStrings>(stream_, value);
}

void EnumsReader::CloseImpl() {
if (!skip_completed_check_) {
stream_.VerifyFinished();
Expand Down
10 changes: 10 additions & 0 deletions cpp/test/generated/binary/protocols.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1098,6 +1098,11 @@ class EnumsWriter : public test_model::EnumsWriterBase, yardl::binary::BinaryWri
void WriteVecImpl(std::vector<test_model::Fruits> const& value) override;
void WriteSizeImpl(test_model::SizeBasedEnum const& value) override;
void WriteRecImpl(test_model::RecordWithEnums const& value) override;
void WriteRecArrayImpl(yardl::DynamicNDArray<test_model::RecordWithEnums> const& value) override;
void WriteRecWithFixedVectorsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithFixedVectors> const& value) override;
void WriteRecWithOptionalFieldsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithOptionalFields> const& value) override;
void WriteRecWithVlensArrayImpl(yardl::DynamicNDArray<test_model::RecordWithVlens> const& value) override;
void WriteRecWithStringsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithStrings> const& value) override;
void CloseImpl() override;

Version version_;
Expand All @@ -1119,6 +1124,11 @@ class EnumsReader : public test_model::EnumsReaderBase, yardl::binary::BinaryRea
void ReadVecImpl(std::vector<test_model::Fruits>& value) override;
void ReadSizeImpl(test_model::SizeBasedEnum& value) override;
void ReadRecImpl(test_model::RecordWithEnums& value) override;
void ReadRecArrayImpl(yardl::DynamicNDArray<test_model::RecordWithEnums>& value) override;
void ReadRecWithFixedVectorsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithFixedVectors>& value) override;
void ReadRecWithOptionalFieldsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithOptionalFields>& value) override;
void ReadRecWithVlensArrayImpl(yardl::DynamicNDArray<test_model::RecordWithVlens>& value) override;
void ReadRecWithStringsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithStrings>& value) override;
void CloseImpl() override;

Version version_;
Expand Down
40 changes: 40 additions & 0 deletions cpp/test/generated/hdf5/protocols.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3304,6 +3304,26 @@ void EnumsWriter::WriteRecImpl(test_model::RecordWithEnums const& value) {
yardl::hdf5::WriteScalarDataset<test_model::RecordWithEnums, test_model::RecordWithEnums>(group_, "rec", test_model::hdf5::GetRecordWithEnumsHdf5Ddl(), value);
}

void EnumsWriter::WriteRecArrayImpl(yardl::DynamicNDArray<test_model::RecordWithEnums> const& value) {
yardl::hdf5::WriteScalarDataset<yardl::hdf5::InnerDynamicNdArray<test_model::RecordWithEnums, test_model::RecordWithEnums>, yardl::DynamicNDArray<test_model::RecordWithEnums>>(group_, "recArray", yardl::hdf5::DynamicNDArrayDdl<test_model::RecordWithEnums, test_model::RecordWithEnums>(test_model::hdf5::GetRecordWithEnumsHdf5Ddl()), value);
}

void EnumsWriter::WriteRecWithFixedVectorsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithFixedVectors> const& value) {
yardl::hdf5::WriteScalarDataset<yardl::hdf5::InnerDynamicNdArray<test_model::hdf5::_Inner_RecordWithFixedVectors, test_model::RecordWithFixedVectors>, yardl::DynamicNDArray<test_model::RecordWithFixedVectors>>(group_, "recWithFixedVectorsArray", yardl::hdf5::DynamicNDArrayDdl<test_model::hdf5::_Inner_RecordWithFixedVectors, test_model::RecordWithFixedVectors>(test_model::hdf5::GetRecordWithFixedVectorsHdf5Ddl()), value);
}

void EnumsWriter::WriteRecWithOptionalFieldsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithOptionalFields> const& value) {
yardl::hdf5::WriteScalarDataset<yardl::hdf5::InnerDynamicNdArray<test_model::hdf5::_Inner_RecordWithOptionalFields, test_model::RecordWithOptionalFields>, yardl::DynamicNDArray<test_model::RecordWithOptionalFields>>(group_, "recWithOptionalFieldsArray", yardl::hdf5::DynamicNDArrayDdl<test_model::hdf5::_Inner_RecordWithOptionalFields, test_model::RecordWithOptionalFields>(test_model::hdf5::GetRecordWithOptionalFieldsHdf5Ddl()), value);
}

void EnumsWriter::WriteRecWithVlensArrayImpl(yardl::DynamicNDArray<test_model::RecordWithVlens> const& value) {
yardl::hdf5::WriteScalarDataset<yardl::hdf5::InnerDynamicNdArray<test_model::hdf5::_Inner_RecordWithVlens, test_model::RecordWithVlens>, yardl::DynamicNDArray<test_model::RecordWithVlens>>(group_, "recWithVlensArray", yardl::hdf5::DynamicNDArrayDdl<test_model::hdf5::_Inner_RecordWithVlens, test_model::RecordWithVlens>(test_model::hdf5::GetRecordWithVlensHdf5Ddl()), value);
}

void EnumsWriter::WriteRecWithStringsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithStrings> const& value) {
yardl::hdf5::WriteScalarDataset<yardl::hdf5::InnerDynamicNdArray<test_model::hdf5::_Inner_RecordWithStrings, test_model::RecordWithStrings>, yardl::DynamicNDArray<test_model::RecordWithStrings>>(group_, "recWithStringsArray", yardl::hdf5::DynamicNDArrayDdl<test_model::hdf5::_Inner_RecordWithStrings, test_model::RecordWithStrings>(test_model::hdf5::GetRecordWithStringsHdf5Ddl()), value);
}

EnumsReader::EnumsReader(std::string path, bool skip_completed_check)
: test_model::EnumsReaderBase(skip_completed_check), yardl::hdf5::Hdf5Reader::Hdf5Reader(path, "Enums", schema_) {
}
Expand All @@ -3324,6 +3344,26 @@ void EnumsReader::ReadRecImpl(test_model::RecordWithEnums& value) {
yardl::hdf5::ReadScalarDataset<test_model::RecordWithEnums, test_model::RecordWithEnums>(group_, "rec", test_model::hdf5::GetRecordWithEnumsHdf5Ddl(), value);
}

void EnumsReader::ReadRecArrayImpl(yardl::DynamicNDArray<test_model::RecordWithEnums>& value) {
yardl::hdf5::ReadScalarDataset<yardl::hdf5::InnerDynamicNdArray<test_model::RecordWithEnums, test_model::RecordWithEnums>, yardl::DynamicNDArray<test_model::RecordWithEnums>>(group_, "recArray", yardl::hdf5::DynamicNDArrayDdl<test_model::RecordWithEnums, test_model::RecordWithEnums>(test_model::hdf5::GetRecordWithEnumsHdf5Ddl()), value);
}

void EnumsReader::ReadRecWithFixedVectorsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithFixedVectors>& value) {
yardl::hdf5::ReadScalarDataset<yardl::hdf5::InnerDynamicNdArray<test_model::hdf5::_Inner_RecordWithFixedVectors, test_model::RecordWithFixedVectors>, yardl::DynamicNDArray<test_model::RecordWithFixedVectors>>(group_, "recWithFixedVectorsArray", yardl::hdf5::DynamicNDArrayDdl<test_model::hdf5::_Inner_RecordWithFixedVectors, test_model::RecordWithFixedVectors>(test_model::hdf5::GetRecordWithFixedVectorsHdf5Ddl()), value);
}

void EnumsReader::ReadRecWithOptionalFieldsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithOptionalFields>& value) {
yardl::hdf5::ReadScalarDataset<yardl::hdf5::InnerDynamicNdArray<test_model::hdf5::_Inner_RecordWithOptionalFields, test_model::RecordWithOptionalFields>, yardl::DynamicNDArray<test_model::RecordWithOptionalFields>>(group_, "recWithOptionalFieldsArray", yardl::hdf5::DynamicNDArrayDdl<test_model::hdf5::_Inner_RecordWithOptionalFields, test_model::RecordWithOptionalFields>(test_model::hdf5::GetRecordWithOptionalFieldsHdf5Ddl()), value);
}

void EnumsReader::ReadRecWithVlensArrayImpl(yardl::DynamicNDArray<test_model::RecordWithVlens>& value) {
yardl::hdf5::ReadScalarDataset<yardl::hdf5::InnerDynamicNdArray<test_model::hdf5::_Inner_RecordWithVlens, test_model::RecordWithVlens>, yardl::DynamicNDArray<test_model::RecordWithVlens>>(group_, "recWithVlensArray", yardl::hdf5::DynamicNDArrayDdl<test_model::hdf5::_Inner_RecordWithVlens, test_model::RecordWithVlens>(test_model::hdf5::GetRecordWithVlensHdf5Ddl()), value);
}

void EnumsReader::ReadRecWithStringsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithStrings>& value) {
yardl::hdf5::ReadScalarDataset<yardl::hdf5::InnerDynamicNdArray<test_model::hdf5::_Inner_RecordWithStrings, test_model::RecordWithStrings>, yardl::DynamicNDArray<test_model::RecordWithStrings>>(group_, "recWithStringsArray", yardl::hdf5::DynamicNDArrayDdl<test_model::hdf5::_Inner_RecordWithStrings, test_model::RecordWithStrings>(test_model::hdf5::GetRecordWithStringsHdf5Ddl()), value);
}

FlagsWriter::FlagsWriter(std::string path)
: yardl::hdf5::Hdf5Writer::Hdf5Writer(path, "Flags", schema_) {
}
Expand Down
20 changes: 20 additions & 0 deletions cpp/test/generated/hdf5/protocols.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -856,6 +856,16 @@ class EnumsWriter : public test_model::EnumsWriterBase, public yardl::hdf5::Hdf5

void WriteRecImpl(test_model::RecordWithEnums const& value) override;

void WriteRecArrayImpl(yardl::DynamicNDArray<test_model::RecordWithEnums> const& value) override;

void WriteRecWithFixedVectorsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithFixedVectors> const& value) override;

void WriteRecWithOptionalFieldsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithOptionalFields> const& value) override;

void WriteRecWithVlensArrayImpl(yardl::DynamicNDArray<test_model::RecordWithVlens> const& value) override;

void WriteRecWithStringsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithStrings> const& value) override;

private:
};

Expand All @@ -872,6 +882,16 @@ class EnumsReader : public test_model::EnumsReaderBase, public yardl::hdf5::Hdf5

void ReadRecImpl(test_model::RecordWithEnums& value) override;

void ReadRecArrayImpl(yardl::DynamicNDArray<test_model::RecordWithEnums>& value) override;

void ReadRecWithFixedVectorsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithFixedVectors>& value) override;

void ReadRecWithOptionalFieldsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithOptionalFields>& value) override;

void ReadRecWithVlensArrayImpl(yardl::DynamicNDArray<test_model::RecordWithVlens>& value) override;

void ReadRecWithStringsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithStrings>& value) override;

private:
};

Expand Down
120 changes: 120 additions & 0 deletions cpp/test/generated/mocks.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3085,6 +3085,86 @@ class MockEnumsWriter : public EnumsWriterBase {
WriteRecImpl_expected_values_.push(value);
}

void WriteRecArrayImpl (yardl::DynamicNDArray<test_model::RecordWithEnums> const& value) override {
if (WriteRecArrayImpl_expected_values_.empty()) {
throw std::runtime_error("Unexpected call to WriteRecArrayImpl");
}
if (WriteRecArrayImpl_expected_values_.front() != value) {
throw std::runtime_error("Unexpected argument value for call to WriteRecArrayImpl");
}
WriteRecArrayImpl_expected_values_.pop();
}

std::queue<yardl::DynamicNDArray<test_model::RecordWithEnums>> WriteRecArrayImpl_expected_values_;

void ExpectWriteRecArrayImpl (yardl::DynamicNDArray<test_model::RecordWithEnums> const& value) {
WriteRecArrayImpl_expected_values_.push(value);
}

void WriteRecWithFixedVectorsArrayImpl (yardl::DynamicNDArray<test_model::RecordWithFixedVectors> const& value) override {
if (WriteRecWithFixedVectorsArrayImpl_expected_values_.empty()) {
throw std::runtime_error("Unexpected call to WriteRecWithFixedVectorsArrayImpl");
}
if (WriteRecWithFixedVectorsArrayImpl_expected_values_.front() != value) {
throw std::runtime_error("Unexpected argument value for call to WriteRecWithFixedVectorsArrayImpl");
}
WriteRecWithFixedVectorsArrayImpl_expected_values_.pop();
}

std::queue<yardl::DynamicNDArray<test_model::RecordWithFixedVectors>> WriteRecWithFixedVectorsArrayImpl_expected_values_;

void ExpectWriteRecWithFixedVectorsArrayImpl (yardl::DynamicNDArray<test_model::RecordWithFixedVectors> const& value) {
WriteRecWithFixedVectorsArrayImpl_expected_values_.push(value);
}

void WriteRecWithOptionalFieldsArrayImpl (yardl::DynamicNDArray<test_model::RecordWithOptionalFields> const& value) override {
if (WriteRecWithOptionalFieldsArrayImpl_expected_values_.empty()) {
throw std::runtime_error("Unexpected call to WriteRecWithOptionalFieldsArrayImpl");
}
if (WriteRecWithOptionalFieldsArrayImpl_expected_values_.front() != value) {
throw std::runtime_error("Unexpected argument value for call to WriteRecWithOptionalFieldsArrayImpl");
}
WriteRecWithOptionalFieldsArrayImpl_expected_values_.pop();
}

std::queue<yardl::DynamicNDArray<test_model::RecordWithOptionalFields>> WriteRecWithOptionalFieldsArrayImpl_expected_values_;

void ExpectWriteRecWithOptionalFieldsArrayImpl (yardl::DynamicNDArray<test_model::RecordWithOptionalFields> const& value) {
WriteRecWithOptionalFieldsArrayImpl_expected_values_.push(value);
}

void WriteRecWithVlensArrayImpl (yardl::DynamicNDArray<test_model::RecordWithVlens> const& value) override {
if (WriteRecWithVlensArrayImpl_expected_values_.empty()) {
throw std::runtime_error("Unexpected call to WriteRecWithVlensArrayImpl");
}
if (WriteRecWithVlensArrayImpl_expected_values_.front() != value) {
throw std::runtime_error("Unexpected argument value for call to WriteRecWithVlensArrayImpl");
}
WriteRecWithVlensArrayImpl_expected_values_.pop();
}

std::queue<yardl::DynamicNDArray<test_model::RecordWithVlens>> WriteRecWithVlensArrayImpl_expected_values_;

void ExpectWriteRecWithVlensArrayImpl (yardl::DynamicNDArray<test_model::RecordWithVlens> const& value) {
WriteRecWithVlensArrayImpl_expected_values_.push(value);
}

void WriteRecWithStringsArrayImpl (yardl::DynamicNDArray<test_model::RecordWithStrings> const& value) override {
if (WriteRecWithStringsArrayImpl_expected_values_.empty()) {
throw std::runtime_error("Unexpected call to WriteRecWithStringsArrayImpl");
}
if (WriteRecWithStringsArrayImpl_expected_values_.front() != value) {
throw std::runtime_error("Unexpected argument value for call to WriteRecWithStringsArrayImpl");
}
WriteRecWithStringsArrayImpl_expected_values_.pop();
}

std::queue<yardl::DynamicNDArray<test_model::RecordWithStrings>> WriteRecWithStringsArrayImpl_expected_values_;

void ExpectWriteRecWithStringsArrayImpl (yardl::DynamicNDArray<test_model::RecordWithStrings> const& value) {
WriteRecWithStringsArrayImpl_expected_values_.push(value);
}

void Verify() {
if (!WriteSingleImpl_expected_values_.empty()) {
throw std::runtime_error("Expected call to WriteSingleImpl was not received");
Expand All @@ -3098,6 +3178,21 @@ class MockEnumsWriter : public EnumsWriterBase {
if (!WriteRecImpl_expected_values_.empty()) {
throw std::runtime_error("Expected call to WriteRecImpl was not received");
}
if (!WriteRecArrayImpl_expected_values_.empty()) {
throw std::runtime_error("Expected call to WriteRecArrayImpl was not received");
}
if (!WriteRecWithFixedVectorsArrayImpl_expected_values_.empty()) {
throw std::runtime_error("Expected call to WriteRecWithFixedVectorsArrayImpl was not received");
}
if (!WriteRecWithOptionalFieldsArrayImpl_expected_values_.empty()) {
throw std::runtime_error("Expected call to WriteRecWithOptionalFieldsArrayImpl was not received");
}
if (!WriteRecWithVlensArrayImpl_expected_values_.empty()) {
throw std::runtime_error("Expected call to WriteRecWithVlensArrayImpl was not received");
}
if (!WriteRecWithStringsArrayImpl_expected_values_.empty()) {
throw std::runtime_error("Expected call to WriteRecWithStringsArrayImpl was not received");
}
}
};

Expand Down Expand Up @@ -3133,6 +3228,31 @@ class TestEnumsWriterBase : public EnumsWriterBase {
mock_writer_.ExpectWriteRecImpl(value);
}

void WriteRecArrayImpl(yardl::DynamicNDArray<test_model::RecordWithEnums> const& value) override {
writer_->WriteRecArray(value);
mock_writer_.ExpectWriteRecArrayImpl(value);
}

void WriteRecWithFixedVectorsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithFixedVectors> const& value) override {
writer_->WriteRecWithFixedVectorsArray(value);
mock_writer_.ExpectWriteRecWithFixedVectorsArrayImpl(value);
}

void WriteRecWithOptionalFieldsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithOptionalFields> const& value) override {
writer_->WriteRecWithOptionalFieldsArray(value);
mock_writer_.ExpectWriteRecWithOptionalFieldsArrayImpl(value);
}

void WriteRecWithVlensArrayImpl(yardl::DynamicNDArray<test_model::RecordWithVlens> const& value) override {
writer_->WriteRecWithVlensArray(value);
mock_writer_.ExpectWriteRecWithVlensArrayImpl(value);
}

void WriteRecWithStringsArrayImpl(yardl::DynamicNDArray<test_model::RecordWithStrings> const& value) override {
writer_->WriteRecWithStringsArray(value);
mock_writer_.ExpectWriteRecWithStringsArrayImpl(value);
}

void CloseImpl() override {
close_called_ = true;
writer_->Close();
Expand Down
Loading
Loading