feat: add serde support

Cargo.toml

@@ -1,10 +1,13 @@
 [workspace]
 resolver = "2"
-members = [
-    "msgpacker",
-    "msgpacker-bench",
-    "msgpacker-derive"
-]
+members = ["msgpacker", "msgpacker-bench", "msgpacker-derive"]
+
+[workspace.package]
+version = "0.5.0"
+authors = ["Victor Lopez <vhrlopes@gmail.com>"]
+edition = "2021"
+license = "MIT/Apache-2.0"
+repository = "https://github.com/codx-dev/msgpacker"
 
 [profile.bench]
 lto = true

README.md

@@ -16,6 +16,11 @@ It will implement `Packable` and `Unpackable` for Rust atomic types. The traits
 - derive: Enables `MsgPacker` derive convenience macro.
 - strict: Will panic if there is a protocol violation of the size of a buffer; the maximum allowed size is `u32::MAX`.
 - std: Will implement the `Packable` and `Unpackable` for `std` collections.
+- serde: Adds support for [serde](https://crates.io/crates/serde)
+
+## Known issues
+
+- The library, as noted in issue [#18](https://github.com/codx-dev/msgpacker/issues/18), exhibits a stricter approach when importing data generated by external tools due to the support of mixed types in languages like Python for collections. A pertinent instance from that issue involves an array serialization where the initial element is a u64, followed by a f64. While a solution can be devised in Rust using a wrapper that abstracts primitive types, it introduces an undesirable overhead that may not be suitable for typical use cases. Although this feature could potentially be added in future updates, it remains unimplemented at present.
 
 ## Example
 
@@ -60,19 +65,38 @@ println!("deserialized {} bytes", n);
 assert_eq!(city, deserialized);
 ```
 
-## Benchmarks
+## Serde
 
-Results obtained with `Intel(R) Core(TM) i9-9900X CPU @ 3.50GHz`.
+Version `0.5.0` introduces [serde](https://crates.io/crates/serde) support.
 
-The simplicity of the implementation unlocks a performance more than ~10x better than [rmp-serde](https://crates.io/crates/rmp-serde).
+```rust
+use msgpacker::serde;
+use serde_json::{json, Value};
 
+let val = serde_json::json!({"foo": "bar"});
+let ser = serde::to_vec(&val);
+let des: Value = serde::from_slice(&ser).unwrap();
 
-#### Pack 1.000 elements
+assert_eq!(val, des);
+```
 
-![image](https://github.com/codx-dev/msgpacker/assets/8730839/ef69622d-0e2f-4bb1-b47c-6412d89fc19a)
-![image](https://github.com/codx-dev/msgpacker/assets/8730839/ce2de037-252a-4c90-b429-430d131ccf7e)
+While it's important to recognize that `serde`'s performance can be notably slower, this is primarily due to its implementation of a visitor pattern for type serialization, rather than solely relying on the static structure of declarations. However, `serde` is broadly used and having its support is helpful since a plethora of other libraries will be automatically supported just by having this feature enabled.
 
-#### Unpack 1.000 elements
+For more information, refer to `Benchmarks`.
 
-![image](https://github.com/codx-dev/msgpacker/assets/8730839/5576f99d-6f37-4907-89db-5d666b13f9d5)
-![image](https://github.com/codx-dev/msgpacker/assets/8730839/234c31d2-f319-414b-9418-4103e97d0a9c)
+## Benchmarks
+
+Results obtained with `AMD EPYC 7402P 24-Core Processor`.
+
+![Image](https://github.com/user-attachments/assets/4d695e79-59bc-40c9-9e53-5a203c703462)
+![Image](https://github.com/user-attachments/assets/f6a72499-9b5c-4b47-b6ea-ec4acbfea5f3)
+![Image](https://github.com/user-attachments/assets/60809961-f058-4a86-952b-b6f7d7b3c9a5)
+![Image](https://github.com/user-attachments/assets/de1a2be4-50e0-4dac-94c2-e4fb2ca24e2d)
+![Image](https://github.com/user-attachments/assets/f88696f0-0479-43b7-a8f1-8a8ad7dab911)
+![Image](https://github.com/user-attachments/assets/98277148-e2c1-4878-abd0-6b8ab5371317)
+
+To run the benchmarks:
+
+```sh
+cd msgpacker-bench && cargo bench
+```

msgpacker-bench/Cargo.toml

@@ -1,20 +1,21 @@
 [package]
 name = "msgpacker-bench"
 version = "0.0.0"
-authors = ["Victor Lopez <vhrlopes@gmail.com>"]
-edition = "2021"
-repository = "https://github.com/codx-dev/msgpacker"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
 description = "Benchmarks for msgpacker."
 publish = false
 
 [dependencies]
 msgpacker = { path = "../msgpacker" }
-rmp-serde = "1.1"
+rmp-serde = "1.3"
 serde = { version = "1.0", features = ["derive"] }
 rand = "0.8"
 
 [dev-dependencies]
-criterion = { version = "0.5", features = ["html_reports"] }
+criterion = { version = "0.8", features = ["html_reports"] }
 
 [[bench]]
 name = "msgpacker"

msgpacker-bench/benches/msgpacker.rs

@@ -1,4 +1,6 @@
-use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion};
+use std::hint::black_box;
+
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
 use msgpacker_bench::Value;
 use rand::{distributions::Standard, prelude::*};
 use rmp_serde::{decode::Deserializer, encode::Serializer};
@@ -14,12 +16,17 @@ pub fn pack(c: &mut Criterion) {
 
     // preallocate the required bytes
     let mut bufs_msgpacker = Vec::new();
+    let mut bufs_msgpacker_serde = Vec::new();
     let mut bufs_rmps = Vec::new();
     for count in counts {
         let mut buf = Vec::new();
         msgpacker::pack_array(&mut buf, values.iter().take(count));
         bufs_msgpacker.push(buf);
 
+        let mut buf = Vec::new();
+        msgpacker::serde::to_buffer(&mut buf, &values[..count]);
+        bufs_msgpacker_serde.push(buf);
+
         let mut buf = Vec::new();
         let mut serializer = Serializer::new(&mut buf);
         (&values[..count]).serialize(&mut serializer).unwrap();
@@ -41,6 +48,18 @@ pub fn pack(c: &mut Criterion) {
             },
         );
 
+        group.bench_with_input(
+            format!("msgpacker serde {count}"),
+            &(&values[..*count], bufs_msgpacker_serde[i].capacity()),
+            |b, (val, buf)| {
+                b.iter_batched(
+                    || Vec::with_capacity(*buf),
+                    |mut buf| msgpacker::serde::to_buffer(black_box(&mut buf), val),
+                    BatchSize::LargeInput,
+                );
+            },
+        );
+
         group.bench_with_input(
             format!("rmps {count}"),
             &(&values[..*count], bufs_rmps[i].capacity()),
@@ -71,6 +90,14 @@ pub fn pack(c: &mut Criterion) {
             },
         );
 
+        group.bench_with_input(
+            format!("msgpacker serde {count}"),
+            &bufs_msgpacker_serde[i],
+            |b, buf| {
+                b.iter(|| msgpacker::serde::from_slice::<Vec<Value>>(black_box(buf)));
+            },
+        );
+
         group.bench_with_input(format!("rmps {count}"), &bufs_rmps[i], |b, buf| {
             b.iter(|| {
                 <Vec<Value>>::deserialize(&mut Deserializer::new(black_box(&buf[..]))).unwrap()

msgpacker-derive/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "msgpacker-derive"
-version = "0.3.2"
-authors = ["Victor Lopez <vhrlopes@gmail.com>"]
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
 categories = ["compression", "encoding", "parser-implementations"]
-edition = "2021"
 keywords = ["messagepack", "msgpack"]
-license = "MIT/Apache-2.0"
 readme = "README.md"
-repository = "https://github.com/codx-dev/msgpacker"
 description = "Derive macros for the MessagePack protocol implementation for Rust."
 
 [lib]

msgpacker-derive/src/lib.rs

@@ -18,16 +18,15 @@ fn contains_attribute(field: &Field, name: &str) -> bool {
     let name = name.to_string();
     if let Some(attr) = field.attrs.first() {
         if let Meta::List(list) = &attr.meta {
-            if list.path.is_ident("msgpacker") {
-                if list
+            if list.path.is_ident("msgpacker")
+                && list
                     .tokens
                     .clone()
                     .into_iter()
                     .find(|a| a.to_string() == name)
                     .is_some()
-                {
-                    return true;
-                }
+            {
+                return true;
             }
         }
     }
@@ -67,12 +66,12 @@ fn impl_fields_named(name: Ident, f: FieldsNamed) -> impl Into<TokenStream> {
                 let mut is_vec_u8 = false;
 
                 match &ty {
-                    Type::Path(p) if p.path.segments.last().filter(|p| p.ident.to_string() == "Vec").is_some() => {
+                    Type::Path(p) if p.path.segments.last().filter(|p| p.ident == "Vec").is_some() => {
                         is_vec = true;
                         match &p.path.segments.last().unwrap().arguments {
                             PathArguments::AngleBracketed(a) if a.args.len() == 1 => {
                                 if let Some(GenericArgument::Type(Type::Path(p))) = a.args.first() {
-                                    if p.path.segments.last().filter(|p| p.ident.to_string() == "u8").is_some() {
+                                    if p.path.segments.last().filter(|p| p.ident == "u8").is_some() {
                                         is_vec_u8 = true;
                                     }
                                 }

msgpacker/Cargo.toml

@@ -1,29 +1,40 @@
 [package]
 name = "msgpacker"
-version = "0.4.8"
-authors = ["Victor Lopez <vhrlopes@gmail.com>"]
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
 categories = ["compression", "encoding", "parser-implementations"]
-edition = "2021"
 keywords = ["messagepack", "msgpack"]
-license = "MIT/Apache-2.0"
 readme = "README.md"
-repository = "https://github.com/codx-dev/msgpacker"
 description = "MessagePack protocol implementation for Rust."
 
 [dependencies]
-msgpacker-derive = { version = "0.3", optional = true }
+msgpacker-derive = { path = "../msgpacker-derive", optional = true }
+serde = { version = "1.0", default-features = false, optional = true }
 
 [dev-dependencies]
-proptest = "1.2"
-proptest-derive = "0.3"
+arbitrary = "1.4"
+arbitrary-json = "0.1"
+msgpacker-derive.path = "../msgpacker-derive"
+proptest = "1.10"
+proptest-derive = "0.8"
+serde = { version = "1.0", features = ["derive"] }
+serde_bytes = "0.11"
+serde_json = "1.0"
 
 [features]
-default = ["std", "derive"]
-alloc = []
+default = ["derive", "std", "serde"]
+alloc = ["serde?/alloc"]
 derive = ["msgpacker-derive"]
 strict = []
-std = ["alloc"]
+std = ["alloc", "serde?/std"]
 
 [[test]]
 name = "collections"
 required-features = ["derive"]
+
+[[test]]
+name = "serde"
+required-features = ["alloc", "derive", "serde"]

msgpacker/src/error.rs

@@ -15,6 +15,8 @@ pub enum Error {
     UnexpectedFormatTag,
     /// The provided bin length is not valid.
     UnexpectedBinLength,
+    /// Not yet implemented.
+    NotImplemented,
 }
 
 impl fmt::Display for Error {

msgpacker/src/lib.rs

@@ -11,8 +11,11 @@ mod extension;
 mod error;
 mod format;
 mod helpers;
-mod pack;
-mod unpack;
+pub(crate) mod pack;
+pub(crate) mod unpack;
+
+#[cfg(feature = "serde")]
+pub mod serde;
 
 pub use error::Error;
 use format::Format;

msgpacker/src/pack/binary.rs

@@ -1,26 +1,30 @@
 use super::{Format, Packable};
 use core::iter;
 
+pub fn pack_bytes_slice_len<T: Extend<u8>>(buf: &mut T, slice: &[u8]) -> usize {
+    if slice.len() <= u8::MAX as usize {
+        buf.extend(iter::once(Format::BIN8).chain(iter::once(slice.len() as u8)));
+        2
+    } else if slice.len() <= u16::MAX as usize {
+        buf.extend(iter::once(Format::BIN16).chain((slice.len() as u16).to_be_bytes()));
+        3
+    } else if slice.len() <= u32::MAX as usize {
+        buf.extend(iter::once(Format::BIN32).chain((slice.len() as u32).to_be_bytes()));
+        5
+    } else {
+        #[cfg(feature = "strict")]
+        panic!("strict serialization enabled; the buffer is too large");
+        return 0;
+    }
+}
+
 impl Packable for [u8] {
     #[allow(unreachable_code)]
     fn pack<T>(&self, buf: &mut T) -> usize
     where
         T: Extend<u8>,
     {
-        let n = if self.len() <= u8::MAX as usize {
-            buf.extend(iter::once(Format::BIN8).chain(iter::once(self.len() as u8)));
-            2
-        } else if self.len() <= u16::MAX as usize {
-            buf.extend(iter::once(Format::BIN16).chain((self.len() as u16).to_be_bytes()));
-            3
-        } else if self.len() <= u32::MAX as usize {
-            buf.extend(iter::once(Format::BIN32).chain((self.len() as u32).to_be_bytes()));
-            5
-        } else {
-            #[cfg(feature = "strict")]
-            panic!("strict serialization enabled; the buffer is too large");
-            return 0;
-        };
+        let n = pack_bytes_slice_len(buf, self);
         buf.extend(self.iter().copied());
         n + self.len()
     }