Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions .github/workflows/rustdoc.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
name: rustdoc-gate

on:
push:
branches: ["**"]
pull_request:

jobs:
rustdoc:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Setup Rust
uses: dtolnay/rust-toolchain@stable

- name: Cache cargo
uses: Swatinem/rust-cache@v2

- name: Rustdoc warnings as errors (selected crates)
env:
RUSTDOCFLAGS: -D warnings
run: |
cargo doc --no-deps -p ffq-common
cargo doc --no-deps -p ffq-planner
cargo doc --no-deps -p ffq-execution
cargo doc --no-deps -p ffq-storage
cargo doc --no-deps -p ffq-client
cargo doc --no-deps -p ffq-distributed

- name: Missing docs gate - ffq-common
run: cargo rustc -p ffq-common --lib -- -D missing-docs

- name: Missing docs gate - ffq-storage
run: cargo rustc -p ffq-storage --lib -- -D missing-docs

- name: Missing docs gate - ffq-planner
run: cargo rustc -p ffq-planner --lib -- -D missing-docs

- name: Missing docs gate - ffq-execution
run: cargo rustc -p ffq-execution --lib -- -D missing-docs

- name: Missing docs gate - ffq-shuffle
run: cargo rustc -p ffq-shuffle --lib -- -D missing-docs

- name: Missing docs gate - ffq-sql
run: cargo rustc -p ffq-sql --lib -- -D missing-docs

- name: Missing docs gate - ffq-client
run: cargo rustc -p ffq-client --lib -- -D missing-docs

- name: Missing docs gate - ffq-distributed (no grpc)
run: cargo rustc -p ffq-distributed --lib -- -D missing-docs

- name: Missing docs gate - ffq-distributed (grpc)
run: cargo rustc -p ffq-distributed --features grpc --lib -- -D missing-docs
16 changes: 8 additions & 8 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ default-members = ["crates/client"]
[workspace.package]
edition = "2024"
license = "Apache-2.0"
version = "1.0.1"
version = "1.0.2"
repository = "https://example.invalid/ffq" # TODO

[workspace.dependencies]
Expand Down
3 changes: 3 additions & 0 deletions Contributing.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ Open an issue describing:
- Update docs/README if you change usage.
- Be respectful in review discussions.

Source-level Rust documentation standard:
- `docs/dev/rustdoc-style.md`

## Distributed Compose Smoke Test
Use the v1 coordinator + 2 worker topology:

Expand Down
20 changes: 13 additions & 7 deletions crates/client/examples/run_bench_13_3.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use std::collections::{BTreeMap, HashMap};
use std::env;
use std::fs::File;
use std::fs;
use std::fs::File;
#[cfg(feature = "distributed")]
use std::net::{TcpStream, ToSocketAddrs};
use std::path::{Path, PathBuf};
Expand All @@ -12,11 +12,11 @@ use std::time::{Instant, SystemTime, UNIX_EPOCH};
use arrow::array::{Float64Array, Int64Array, StringArray};
use arrow::record_batch::RecordBatch;
use arrow_schema::{DataType, Field, Schema};
use ffq_client::Engine;
use ffq_client::bench_fixtures::{
default_benchmark_fixture_root, generate_default_benchmark_fixtures,
};
use ffq_client::bench_queries::{load_benchmark_query_from_root, BenchmarkQueryId};
use ffq_client::Engine;
use ffq_client::bench_queries::{BenchmarkQueryId, load_benchmark_query_from_root};
use ffq_common::{EngineConfig, FfqError, Result};
use ffq_planner::LiteralValue;
use ffq_storage::{TableDef, TableStats};
Expand Down Expand Up @@ -1262,7 +1262,10 @@ fn maybe_verify_official_tpch_correctness(
if !is_official {
return Ok(());
}
if !matches!(query_id, BenchmarkQueryId::TpchQ1 | BenchmarkQueryId::TpchQ3) {
if !matches!(
query_id,
BenchmarkQueryId::TpchQ1 | BenchmarkQueryId::TpchQ3
) {
return Ok(());
}

Expand Down Expand Up @@ -1417,9 +1420,12 @@ fn read_parquet_batches(path: &Path) -> Result<Vec<RecordBatch>> {
let file = File::open(path)?;
let builder = ParquetRecordBatchReaderBuilder::try_new(file)
.map_err(|e| FfqError::Execution(format!("open parquet {} failed: {e}", path.display())))?;
let reader = builder
.build()
.map_err(|e| FfqError::Execution(format!("build parquet reader {} failed: {e}", path.display())))?;
let reader = builder.build().map_err(|e| {
FfqError::Execution(format!(
"build parquet reader {} failed: {e}",
path.display()
))
})?;
let mut out = Vec::new();
for batch in reader {
out.push(batch.map_err(|e| {
Expand Down
22 changes: 22 additions & 0 deletions crates/client/src/bench_fixtures.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,48 @@ const RAG_DOC_ROWS: i64 = 10_000;
const RAG_EMBED_DIM: i32 = 64;
const RAG_SYNTH_SEED: u64 = 42;

/// Per-file manifest entry for generated benchmark fixtures.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct FixtureFileManifest {
/// File name relative to fixture directory.
pub file: String,
/// Number of rows in the parquet file.
pub rows: i64,
/// Flattened field descriptors in `name:type:nullable` format.
pub schema: Vec<String>,
}

/// Manifest for one generated fixture set.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct FixtureManifest {
/// Fixture identifier.
pub fixture: String,
/// Human-readable fixture description.
pub description: String,
/// Seed that makes generation deterministic.
pub deterministic_seed: u64,
/// Files produced for this fixture.
pub files: Vec<FixtureFileManifest>,
}

/// Top-level index file for all generated fixture sets.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct FixtureIndex {
/// Stable fixture-set id.
pub fixture_set: String,
/// Subdirectories included in this fixture set.
pub fixtures: Vec<String>,
}

/// Generates deterministic default benchmark fixtures under `root`.
///
/// Creates:
/// - `tpch_sf1` (synthetic TPCH-shaped data)
/// - `rag_synth` (synthetic vector data)
/// - `index.json` and per-fixture manifests
///
/// # Errors
/// Returns an error if directory creation, parquet writing, or manifest writes fail.
pub fn generate_default_benchmark_fixtures(root: &Path) -> Result<()> {
std::fs::create_dir_all(root)?;
let tpch_root = root.join("tpch_sf1");
Expand Down Expand Up @@ -350,6 +371,7 @@ fn write_json<T: Serialize>(path: &Path, value: &T, err_prefix: &str) -> Result<
Ok(())
}

/// Returns default directory for benchmark fixtures used by benchmark runners.
pub fn default_benchmark_fixture_root() -> PathBuf {
PathBuf::from("./tests/bench/fixtures")
}
21 changes: 21 additions & 0 deletions crates/client/src/bench_queries.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,21 @@ use std::path::{Path, PathBuf};

use ffq_common::{FfqError, Result};

/// Identifier for canonical benchmark SQL files.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum BenchmarkQueryId {
/// TPC-H Q1 aggregate workload.
TpchQ1,
/// TPC-H Q3 join + filter workload.
TpchQ3,
/// Vector brute-force top-k benchmark query.
RagTopkBruteforce,
/// Optional qdrant-backed vector top-k benchmark query.
RagTopkQdrant,
}

impl BenchmarkQueryId {
/// Stable machine-readable identifier used in result artifacts.
pub fn stable_id(self) -> &'static str {
match self {
Self::TpchQ1 => "tpch_q1",
Expand All @@ -21,6 +27,7 @@ impl BenchmarkQueryId {
}
}

/// Relative SQL file location under the benchmark query root.
pub fn file_name(self) -> &'static str {
match self {
Self::TpchQ1 => "canonical/tpch_q1.sql",
Expand All @@ -31,23 +38,33 @@ impl BenchmarkQueryId {
}
}

/// Ordered list of benchmark queries expected by the benchmark runner.
pub const CANONICAL_BENCHMARK_QUERIES: [BenchmarkQueryId; 4] = [
BenchmarkQueryId::TpchQ1,
BenchmarkQueryId::TpchQ3,
BenchmarkQueryId::RagTopkBruteforce,
BenchmarkQueryId::RagTopkQdrant,
];

/// Returns the default benchmark query directory.
pub fn default_benchmark_query_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../tests/bench/queries")
.to_path_buf()
}

/// Loads one benchmark SQL file from the default query root.
///
/// # Errors
/// Returns an error when file loading fails or query content is empty.
pub fn load_benchmark_query(id: BenchmarkQueryId) -> Result<String> {
load_benchmark_query_from_root(&default_benchmark_query_root(), id)
}

/// Loads one benchmark SQL file from an explicit query root.
///
/// # Errors
/// Returns an error when file loading fails or query content is empty.
pub fn load_benchmark_query_from_root(root: &Path, id: BenchmarkQueryId) -> Result<String> {
let path = root.join(id.file_name());
let query = fs::read_to_string(&path).map_err(|e| {
Expand All @@ -71,6 +88,10 @@ pub fn load_benchmark_query_from_root(root: &Path, id: BenchmarkQueryId) -> Resu
Ok(trimmed)
}

/// Loads all canonical benchmark SQL files.
///
/// # Errors
/// Returns an error if any canonical query fails to load.
pub fn load_all_benchmark_queries() -> Result<Vec<(BenchmarkQueryId, String)>> {
let root = default_benchmark_query_root();
CANONICAL_BENCHMARK_QUERIES
Expand Down
Loading