From d6c9cafbbd9e5db033b836fce1837f347383f1b7 Mon Sep 17 00:00:00 2001 From: Matthew Hambrecht Date: Thu, 4 Jun 2026 00:04:05 -0400 Subject: [PATCH 1/2] feat: Template out core of project update docs --- .github/workflows/quality.yml | 64 +++++----- .github/workflows/unit.yml | 33 +++-- Cargo.lock | 87 +++++++++++++ Cargo.toml | 3 + docs/architecture/API.md | 16 +-- docs/architecture/FLOW.md | 4 +- docs/architecture/OBJECTS.md | 12 +- src/builder.rs | 181 +++++++++++++++++++++++++++ src/globals.rs | 4 + src/instance.rs | 36 ++++++ src/lib.rs | 18 ++- src/registry.rs | 89 +++++++++++++ src/traits.rs | 39 ++++++ src/types.rs | 9 ++ src/utils/metadatas/mod.rs | 3 + src/utils/metadatas/stub_metadata.rs | 21 ++++ src/utils/mod.rs | 6 + src/utils/readers/mod.rs | 3 + src/utils/readers/stub_reader.rs | 17 +++ src/utils/writers/mod.rs | 3 + src/utils/writers/stub_writer.rs | 21 ++++ src/viewer.rs | 31 +++++ 22 files changed, 626 insertions(+), 74 deletions(-) create mode 100644 src/builder.rs create mode 100644 src/globals.rs create mode 100644 src/instance.rs create mode 100644 src/registry.rs create mode 100644 src/traits.rs create mode 100644 src/types.rs create mode 100644 src/utils/metadatas/mod.rs create mode 100644 src/utils/metadatas/stub_metadata.rs create mode 100644 src/utils/mod.rs create mode 100644 src/utils/readers/mod.rs create mode 100644 src/utils/readers/stub_reader.rs create mode 100644 src/utils/writers/mod.rs create mode 100644 src/utils/writers/stub_writer.rs create mode 100644 src/viewer.rs diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 63a9e55..29b0791 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -18,37 +18,33 @@ jobs: steps: - uses: actions/checkout@v5 - # - uses: dtolnay/rust-toolchain@stable - # with: - # components: clippy - # - # - uses: Swatinem/rust-cache@v2 - # with: - # shared-key: cargo-${{ inputs.runner }} - # - # - name: Install nightly rustfmt - # run: rustup toolchain install nightly --profile minimal --component rustfmt - # - # - name: Rustfmt - # run: | - # cargo +nightly fmt --all - # if ! git diff --quiet; then - # git status --short - # git diff --stat - # exit 1 - # fi - # - # - name: Clippy - # run: cargo clippy --all-features --all-targets -- -D warnings - # - # - name: Feature compile checks - # run: | - # cargo check - # cargo check --tests - # cargo check --features file - # cargo check --features http - # cargo check --features debug - # cargo check --all-features - # - # - name: Docs.rs feature set - # run: RUSTDOCFLAGS='--cfg docsrs' cargo doc --all-features --no-deps + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - uses: Swatinem/rust-cache@v2 + with: + shared-key: cargo-${{ inputs.runner }} + + - name: Install nightly rustfmt + run: rustup toolchain install nightly --profile minimal --component rustfmt + + - name: Rustfmt + run: | + cargo +nightly fmt --all + if ! git diff --quiet; then + git status --short + git diff --stat + exit 1 + fi + + - name: Clippy + run: cargo clippy --all-features --all-targets -- -D warnings + + - name: Feature compile checks + run: | + cargo check + cargo check --tests + + - name: Docs.rs feature set + run: RUSTDOCFLAGS='--cfg docsrs' cargo doc --all-features --no-deps diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 5ce578a..fad277b 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -18,21 +18,18 @@ jobs: steps: - uses: actions/checkout@v5 - # - uses: dtolnay/rust-toolchain@stable - # - # - uses: taiki-e/install-action@v2 - # with: - # tool: nextest - # - # - uses: Swatinem/rust-cache@v2 - # with: - # shared-key: cargo-${{ inputs.runner }} - # - # - name: Unit tests - # run: cargo nextest run --features utils - # - # - name: Debug harness tests - # run: cargo nextest run --features utils,debug - # - # - name: Doc tests - # run: cargo test --doc --all-features + - uses: dtolnay/rust-toolchain@stable + + - uses: taiki-e/install-action@v2 + with: + tool: nextest + + - uses: Swatinem/rust-cache@v2 + with: + shared-key: cargo-${{ inputs.runner }} + + - name: Unit tests (no features) + run: cargo nextest run + + - name: Doc tests + run: cargo test --doc --all-features diff --git a/Cargo.lock b/Cargo.lock index cf881e5..d82b619 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,93 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "bitflags" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + [[package]] name = "sparseio" version = "0.0.1" +dependencies = [ + "bytes", + "parking_lot", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" diff --git a/Cargo.toml b/Cargo.toml index 0e01ea0..87a7ef7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,3 +12,6 @@ keywords = ["storage", "tokio", "fs", "filesystem", "stream"] readme = "README.md" license = "Apache-2.0" +[dependencies] +parking_lot = "0.12" +bytes = "1.11" diff --git a/docs/architecture/API.md b/docs/architecture/API.md index 7fe3f9a..77185aa 100644 --- a/docs/architecture/API.md +++ b/docs/architecture/API.md @@ -7,8 +7,8 @@ a HuggingFace repository, a remote FTP server, or any other distant data source. ability to read explicit byte ranges from the data source. As such the user must provide two functions to produce a functional Reader: -- `func len() -> usize`: The number of bytes in the data source. -- `func read(offset: usize, length: usize) -> io::Result`: Read a byte range from the data source. +- `fn len(&self) -> io::Result`: The number of bytes in the data source. +- `fn read_at(&self, offset: usize, length: usize) -> io::Result`: Read a byte range from the data source. **Note**: It is up to the Developer to ensure access patterns made by the Reader are efficient for the underlying data source (i.e. managing rate limits, connection pooling, etc.). SparseIO will not attempt to optimize access patterns @@ -20,9 +20,9 @@ The Writer component is responsible for writing data to the downstream cache in speeds on future reads. This could be a local disk, a remote cache server, or any other location where data can be written. As such the user is expected to provide three functions to produce a functional Writer: -- `func write(key: &str, offset: usize, data: Bytes)`: Write a byte range to the cache. -- `func read(key: &str, offset: usize, length: usize) -> Result>`: Read a byte range from the cache. -- `func delete(key: &str)`: Delete a key from the cache. +- `fn write(&self, key: &str) -> io::Result<()>`: Write to the cache under `key`. +- `fn read(&self, key: &str) -> io::Result>`: Read bytes from the cache under `key`. +- `fn delete(&self, key: &str) -> io::Result<()>`: Delete a key from the cache. ## Metadata @@ -30,9 +30,9 @@ The Metadata component is responsible for keeping track of the data in the cache individual data sources, and other relevant metadata for the application. As such it is just a generic interface to a key-value store and the user is expected to provide three functions to produce a functional Metadata store: -- `func get(key: &str) -> Result>`: Get a value from the metadata store. -- `func set(key: &str, value: Bytes)`: Set a value in the metadata store. -- `func delete(key: &str)`: Delete a key from the metadata store. +- `fn get(&self, key: &str) -> io::Result>`: Get a value from the metadata store. +- `fn set(&self, key: &str, value: Bytes) -> io::Result<()>`: Set a value in the metadata store. +- `fn delete(&self, key: &str) -> io::Result<()>`: Delete a key from the metadata store. ## Sample User Application Diagram diff --git a/docs/architecture/FLOW.md b/docs/architecture/FLOW.md index 8d376c8..896979d 100644 --- a/docs/architecture/FLOW.md +++ b/docs/architecture/FLOW.md @@ -4,11 +4,11 @@ In this section we will go over the flow of how a read request is processed in S When a user initially constructs a SparseIO object they are required to provide a `Metadata` implementation, a `Writer` implementation, and a `ReaderRegistry` implementation loaded with -the `Reader` implementations they want to use. This is all done through the `SparseIOBuilder` object, which +the `Reader` implementations they want to use. This is all done through the `Builder` object, which provides a nice interface for constructing the `SparseIO` object without having to worry about defaults. Users can then call `SparseIO::open` with a canonicalized path to get back a `Viewer` object. -This looks something like `instance.open(mysite+/path/to/object)`. The main component of the +This looks something like `instance.open("mysite+/path/to/object")`. The main component of the viewer object is the `read_at()` method because it provides the core functionality behind the bytestream, and general reads. diff --git a/docs/architecture/OBJECTS.md b/docs/architecture/OBJECTS.md index c32ce68..061d55a 100644 --- a/docs/architecture/OBJECTS.md +++ b/docs/architecture/OBJECTS.md @@ -6,7 +6,7 @@ The SparseIO object is the main interface for users of the library. It provides backing logic management behind the interaction between the `Reader`, `Writer`, and `Metadata` objects to maintain the abstraction from the user. -When a user creates a SparseIO object, they do it through a SparseIOBuilder, which +When a user creates a SparseIO object, they do it through a `Builder`, which is a [builder pattern](https://www.lurklurk.org/effective-rust/builders.html) making it easy to construct the object without having to worry about defaults. @@ -17,8 +17,8 @@ Three things are required to construct a SparseIO object: - A [ReaderRegistry](#ReaderRegistry) implementation There are some other tunable parameters that can be set in the builder, such as the -`chunk_size`, control over prefetching behavior, etc. However these are explained a -bit further in the [SparseIOBuilder docs](http://docs.rs/sparseio/latest/sparseio/struct.SparseIOBuilder.html). +`chunk_size`. However these are explained a bit further in the +[`Builder` docs](http://docs.rs/sparseio/latest/sparseio/struct.Builder.html). ## ReaderRegistry @@ -50,6 +50,6 @@ When a user attempts to read an object through the `SparseIO` object through `Sp (e.g. `open("mysite+/path/to/object")`) they are actually getting back a `Viewer` object. The Viewer provides a couple of core methods for interacting with the data including: -- `read(offset: usize, length: usize) -> io::Result`: Read a byte range from the object. -- `len() -> usize`: Get the total length of the object in bytes. -- `to_bytestream() -> impl Stream>`: Convert the Viewer to a bytestream for easier integration with async applications. +- `read_at(&self, offset: usize, length: usize) -> io::Result`: Read a byte range from the object. +- `len(&self) -> io::Result`: Get the total length of the object in bytes. +- `bytestream(&self) -> ByteStream`: Convert the Viewer to a bytestream for easier integration with applications. diff --git a/src/builder.rs b/src/builder.rs new file mode 100644 index 0000000..94d7561 --- /dev/null +++ b/src/builder.rs @@ -0,0 +1,181 @@ +//! Builder for constructing [`SparseIO`](crate::SparseIO) instances. + +use std::io; +use std::sync::Arc; + +use crate::{Metadata, ReaderRegistry, SparseIO, Writer}; + +/// Builder for the main SparseIO coordinator. +#[derive(Default)] +pub struct Builder { + writer: Option>, + metadata: Option>, + registry: Option>, + + chunk_size: Option, +} + +impl Builder { + /// Create a new builder with default tunables. + pub fn new() -> Self { + Self::default() + } + + /// Set the cache writer implementation. + pub fn writer(mut self, writer: impl Writer + 'static) -> Self { + self.writer = Some(Arc::new(writer)); + self + } + + /// Set the cache writer implementation from a shared trait object. + pub fn writer_arc(mut self, writer: Arc) -> Self { + self.writer = Some(writer); + self + } + + /// Set the metadata store implementation. + pub fn metadata(mut self, metadata: impl Metadata + 'static) -> Self { + self.metadata = Some(Arc::new(metadata)); + self + } + + /// Set the metadata store implementation from a shared trait object. + pub fn metadata_arc(mut self, metadata: Arc) -> Self { + self.metadata = Some(metadata); + self + } + + /// Set the reader registry object. + pub fn registry(mut self, registry: ReaderRegistry) -> Self { + self.registry = Some(Arc::new(registry)); + self + } + + /// Set the reader registry implementation from a shared object. + pub fn registry_arc(mut self, reader_registry: Arc) -> Self { + self.registry = Some(reader_registry); + self + } + + /// Set the immutable chunk size for this SparseIO instance. + pub fn chunk_size(mut self, chunk_size: usize) -> Self { + self.chunk_size = Some(chunk_size); + self + } + + /// Build a SparseIO instance. + pub fn build(self) -> io::Result { + if self.writer.is_none() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "Writer implementation is required")); + } + + if self.metadata.is_none() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "Metadata implementation is required")); + } + + if self.registry.is_none() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "ReaderRegistry implementation is required")); + } + + Ok(SparseIO { + writer: self.writer.expect("writer should be set"), + metadata: self.metadata.expect("metadata should be set"), + registry: self.registry.expect("registry should be set"), + chunk_size: self.chunk_size.unwrap_or(crate::globals::DEFAULT_CHUNK_SIZE), + }) + } +} + +#[cfg(test)] +mod tests { + use std::io; + use std::sync::Arc; + + use super::Builder; + use crate::ReaderRegistry; + use crate::utils::metadatas::StubMetadata; + use crate::utils::readers::StubReader; + use crate::utils::writers::StubWriter; + + fn writer() -> StubWriter { + StubWriter + } + + fn metadata() -> StubMetadata { + StubMetadata + } + + fn registry() -> ReaderRegistry { + let mut registry = ReaderRegistry::new(); + registry.register("stub", StubReader); + registry + } + + fn assert_invalid_input(result: io::Result, expected_message: &str) { + let error = match result { + Ok(_) => panic!("builder should reject missing required input"), + Err(error) => error, + }; + + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(error.to_string(), expected_message); + } + + #[test] + fn build_requires_writer() { + let result = Builder::new().metadata(metadata()).registry(registry()).build(); + + assert_invalid_input(result, "Writer implementation is required"); + } + + #[test] + fn build_requires_metadata() { + let result = Builder::new().writer(writer()).registry(registry()).build(); + + assert_invalid_input(result, "Metadata implementation is required"); + } + + #[test] + fn build_requires_registry() { + let result = Builder::new().writer(writer()).metadata(metadata()).build(); + + assert_invalid_input(result, "ReaderRegistry implementation is required"); + } + + #[test] + fn build_uses_default_chunk_size() { + let sparseio = Builder::new() + .writer(writer()) + .metadata(metadata()) + .registry(registry()) + .build() + .expect("builder should accept all required inputs"); + + assert_eq!(sparseio.chunk_size, crate::globals::DEFAULT_CHUNK_SIZE); + } + + #[test] + fn build_uses_configured_chunk_size() { + let sparseio = Builder::new() + .writer(writer()) + .metadata(metadata()) + .registry(registry()) + .chunk_size(8192) + .build() + .expect("builder should accept custom chunk size"); + + assert_eq!(sparseio.chunk_size, 8192); + } + + #[test] + fn build_accepts_arc_inputs() { + let sparseio = Builder::new() + .writer_arc(Arc::new(writer())) + .metadata_arc(Arc::new(metadata())) + .registry_arc(Arc::new(registry())) + .build() + .expect("builder should accept shared inputs"); + + assert_eq!(sparseio.chunk_size, crate::globals::DEFAULT_CHUNK_SIZE); + } +} diff --git a/src/globals.rs b/src/globals.rs new file mode 100644 index 0000000..5fb0ae8 --- /dev/null +++ b/src/globals.rs @@ -0,0 +1,4 @@ +//! Global constants used across the crate. + +/// Default chunk size used when the caller does not configure one. +pub const DEFAULT_CHUNK_SIZE: usize = 1 * 1024 * 1024; // 1 MiB diff --git a/src/instance.rs b/src/instance.rs new file mode 100644 index 0000000..af67209 --- /dev/null +++ b/src/instance.rs @@ -0,0 +1,36 @@ +//! Core instance struct and related functions. +//! +//! This module defines the `SparseIO` object, which represents a single +//! instance of our management interface for the entire sparse file read system. + +use std::io; +use std::sync::Arc; + +use crate::viewer::Viewer; +use crate::{Metadata, ReaderRegistry, Writer}; + +/// Coordinator for sparse, chunked reads and cache materialization. +#[allow(dead_code)] +#[derive(Clone)] +pub struct SparseIO { + // World interface components + pub(crate) writer: Arc, + pub(crate) metadata: Arc, + pub(crate) registry: Arc, + + // Tunable States + pub(crate) chunk_size: usize, +} + +impl SparseIO { + /// Create a builder for configuring a SparseIO instance. + pub fn builder() -> crate::builder::Builder { + crate::builder::Builder::new() + } + + /// Open a canonical SparseIO path and return a viewer over the object. + pub fn open(&self, path: impl AsRef) -> io::Result { + let _ = path; + todo!("resolve reader and create viewer") + } +} diff --git a/src/lib.rs b/src/lib.rs index 8a7db22..f8d24c6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,12 @@ -compile_error!( - r" -This crate is currently undergoing an architectural refactor. Please stay patient -and keep track of http://github.com/mhambre/sparseio for development updates. -" -); +mod builder; +mod globals; +mod instance; +mod registry; +mod traits; +mod types; +mod utils; +mod viewer; + +pub use instance::SparseIO; +pub use registry::ReaderRegistry; +pub use traits::{Metadata, Reader, Writer}; diff --git a/src/registry.rs b/src/registry.rs new file mode 100644 index 0000000..9b49ec4 --- /dev/null +++ b/src/registry.rs @@ -0,0 +1,89 @@ +//! Reader registry abstractions. + +use std::collections::HashMap; + +use crate::Reader; + +/// Registry for storing SparseIO reader implementations. +#[derive(Default)] +pub struct ReaderRegistry { + readers: HashMap>, +} + +impl ReaderRegistry { + /// Create a new, empty registry. + pub fn new() -> Self { + Self { + readers: HashMap::new(), + } + } + + /// Register a reader implementation under a canonical name. + pub fn register(&mut self, name: impl Into, reader: impl Reader + 'static) { + self.readers.insert(name.into(), Box::new(reader)); + } + + /// Retrieve a reader implementation by its canonical name. + pub fn get(&self, name: &str) -> Option<&(dyn Reader + Send + Sync)> { + self.readers.get(name).map(|r| r.as_ref()) + } +} + +#[cfg(test)] +mod tests { + use std::io; + + use bytes::Bytes; + + use super::ReaderRegistry; + use crate::Reader; + use crate::utils::readers::StubReader; + + struct LenReader(usize); + + impl Reader for LenReader { + fn len(&self) -> io::Result { + Ok(self.0) + } + + fn read_at(&self, _offset: usize, _length: usize) -> io::Result { + unimplemented!("registry tests should not call reader read_at") + } + } + + #[test] + fn new_starts_empty() { + let registry = ReaderRegistry::new(); + + assert!(registry.get("missing").is_none()); + } + + #[test] + fn default_starts_empty() { + let registry = ReaderRegistry::default(); + + assert!(registry.get("missing").is_none()); + } + + #[test] + fn register_makes_reader_retrievable_by_name() { + let mut registry = ReaderRegistry::new(); + + registry.register("stub", StubReader); + + assert!(registry.get("stub").is_some()); + assert!(registry.get("other").is_none()); + } + + #[test] + fn register_replaces_existing_reader_for_name() { + let mut registry = ReaderRegistry::new(); + + registry.register("source", LenReader(10)); + registry.register("source", LenReader(20)); + + let reader = registry.get("source").expect("replacement reader should be registered"); + + assert_eq!(reader.len().expect("len should be readable"), 20); + } +} diff --git a/src/traits.rs b/src/traits.rs new file mode 100644 index 0000000..4b8749d --- /dev/null +++ b/src/traits.rs @@ -0,0 +1,39 @@ +//! User-facing traits for plugging storage backends into SparseIO. + +use std::io; + +use bytes::Bytes; + +/// Reads explicit byte ranges from an upstream data source. +#[allow(clippy::len_without_is_empty)] +pub trait Reader: Send + Sync { + /// Return the total number of bytes in the source object. + fn len(&self) -> io::Result; + + /// Read `length` bytes beginning at `offset`. + fn read_at(&self, offset: usize, length: usize) -> io::Result; +} + +/// Stores and retrieves cached byte ranges. +pub trait Writer: Send + Sync { + /// Write bytes into the cache under `key`. + fn write(&self, key: &str) -> io::Result<()>; + + /// Read a byte range from the cache. + fn read(&self, key: &str) -> io::Result>; + + /// Delete all cached data associated with `key`. + fn delete(&self, key: &str) -> io::Result<()>; +} + +/// Key-value metadata store used for coverage maps and CAS reference counts. +pub trait Metadata: Send + Sync { + /// Retrieve a value from the metadata store. + fn get(&self, key: &str) -> io::Result>; + + /// Persist a value in the metadata store. + fn set(&self, key: &str, value: Bytes) -> io::Result<()>; + + /// Delete a value from the metadata store. + fn delete(&self, key: &str) -> io::Result<()>; +} diff --git a/src/types.rs b/src/types.rs new file mode 100644 index 0000000..ccbdae3 --- /dev/null +++ b/src/types.rs @@ -0,0 +1,9 @@ +//! Shared public types used by SparseIO traits and objects. + +use std::io; +use std::pin::Pin; + +use bytes::Bytes; + +/// Stream type for chunked byte output from a viewer. +pub type ByteStream = Pin> + Send>>; diff --git a/src/utils/metadatas/mod.rs b/src/utils/metadatas/mod.rs new file mode 100644 index 0000000..f35e21f --- /dev/null +++ b/src/utils/metadatas/mod.rs @@ -0,0 +1,3 @@ +mod stub_metadata; + +pub(crate) use stub_metadata::StubMetadata; diff --git a/src/utils/metadatas/stub_metadata.rs b/src/utils/metadatas/stub_metadata.rs new file mode 100644 index 0000000..ec3d5f9 --- /dev/null +++ b/src/utils/metadatas/stub_metadata.rs @@ -0,0 +1,21 @@ +use std::io; + +use bytes::Bytes; + +use crate::Metadata; + +pub(crate) struct StubMetadata; + +impl Metadata for StubMetadata { + fn get(&self, _key: &str) -> io::Result> { + unimplemented!("stub metadata get should not be called") + } + + fn set(&self, _key: &str, _value: Bytes) -> io::Result<()> { + unimplemented!("stub metadata set should not be called") + } + + fn delete(&self, _key: &str) -> io::Result<()> { + unimplemented!("stub metadata delete should not be called") + } +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs new file mode 100644 index 0000000..2636127 --- /dev/null +++ b/src/utils/mod.rs @@ -0,0 +1,6 @@ +#[cfg(test)] +pub(crate) mod metadatas; +#[cfg(test)] +pub(crate) mod readers; +#[cfg(test)] +pub(crate) mod writers; diff --git a/src/utils/readers/mod.rs b/src/utils/readers/mod.rs new file mode 100644 index 0000000..9764552 --- /dev/null +++ b/src/utils/readers/mod.rs @@ -0,0 +1,3 @@ +mod stub_reader; + +pub(crate) use stub_reader::StubReader; diff --git a/src/utils/readers/stub_reader.rs b/src/utils/readers/stub_reader.rs new file mode 100644 index 0000000..4986fd9 --- /dev/null +++ b/src/utils/readers/stub_reader.rs @@ -0,0 +1,17 @@ +use std::io; + +use bytes::Bytes; + +use crate::Reader; + +pub(crate) struct StubReader; + +impl Reader for StubReader { + fn len(&self) -> io::Result { + unimplemented!("stub reader len should not be called") + } + + fn read_at(&self, _offset: usize, _length: usize) -> io::Result { + unimplemented!("stub reader read_at should not be called") + } +} diff --git a/src/utils/writers/mod.rs b/src/utils/writers/mod.rs new file mode 100644 index 0000000..9c91529 --- /dev/null +++ b/src/utils/writers/mod.rs @@ -0,0 +1,3 @@ +mod stub_writer; + +pub(crate) use stub_writer::StubWriter; diff --git a/src/utils/writers/stub_writer.rs b/src/utils/writers/stub_writer.rs new file mode 100644 index 0000000..c0f3588 --- /dev/null +++ b/src/utils/writers/stub_writer.rs @@ -0,0 +1,21 @@ +use std::io; + +use bytes::Bytes; + +use crate::Writer; + +pub(crate) struct StubWriter; + +impl Writer for StubWriter { + fn write(&self, _key: &str) -> io::Result<()> { + unimplemented!("stub writer write should not be called") + } + + fn read(&self, _key: &str) -> io::Result> { + unimplemented!("stub writer read should not be called") + } + + fn delete(&self, _key: &str) -> io::Result<()> { + unimplemented!("stub writer delete should not be called") + } +} diff --git a/src/viewer.rs b/src/viewer.rs new file mode 100644 index 0000000..40741b0 --- /dev/null +++ b/src/viewer.rs @@ -0,0 +1,31 @@ +//! Object viewer returned by [`SparseIO::open`](crate::SparseIO::open). + +use std::io; + +use bytes::Bytes; + +use crate::types::ByteStream; + +/// Read interface for a single upstream object mediated by SparseIO. +#[allow(dead_code, clippy::len_without_is_empty)] +pub struct Viewer { + /// Stub + _stub: (), +} + +impl Viewer { + /// Alias for offset-based reads. + pub fn read_at(&self, _offset: usize, _length: usize) -> io::Result { + todo!("read bytes at offset from upstream object") + } + + /// Return the total length of the upstream object. + pub fn len(&self) -> io::Result { + todo!("return upstream object length") + } + + /// Convert this viewer into a chunked byte stream. + pub fn bytestream(&self) -> ByteStream { + todo!("convert viewer into a chunked byte stream") + } +} From 91fb7de13e7395b4f195064cf2ec1fab0ebb7260 Mon Sep 17 00:00:00 2001 From: Matthew Hambrecht Date: Thu, 4 Jun 2026 00:12:07 -0400 Subject: [PATCH 2/2] lower chunk size --- src/globals.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/globals.rs b/src/globals.rs index 5fb0ae8..474eb13 100644 --- a/src/globals.rs +++ b/src/globals.rs @@ -1,4 +1,4 @@ //! Global constants used across the crate. /// Default chunk size used when the caller does not configure one. -pub const DEFAULT_CHUNK_SIZE: usize = 1 * 1024 * 1024; // 1 MiB +pub const DEFAULT_CHUNK_SIZE: usize = 64 * 1024; // 64 KiB