diff --git a/Cargo.lock b/Cargo.lock index 591b0447045..b6fb2e0a4a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7360,6 +7360,7 @@ dependencies = [ "rand 0.9.2", "reqwest 0.12.23", "rocksdb", + "schemars", "serde", "serde_json", "service-kit", diff --git a/crates/flowctl/src/lib.rs b/crates/flowctl/src/lib.rs index 10075c772dd..f42370b4e85 100644 --- a/crates/flowctl/src/lib.rs +++ b/crates/flowctl/src/lib.rs @@ -124,7 +124,7 @@ pub struct CliContext { config: config::Config, /// Selected output format (table / JSON / YAML) for command results. output: output::Output, - /// Tracks in-flight work units; cloned into preview-next connector drivers. + /// Tracks in-flight work units; cloned into preview connector drivers. registry: service_kit::Registry, } diff --git a/crates/flowctl/src/main.rs b/crates/flowctl/src/main.rs index a21dbe8eab6..02b0b0ca4de 100644 --- a/crates/flowctl/src/main.rs +++ b/crates/flowctl/src/main.rs @@ -15,8 +15,8 @@ fn main() -> Result<(), anyhow::Error> { // Process-global handler registry. Service-kit's trace layer consults this // for per-handler verbosity overrides, and its event layer records opt-in // `event!` breadcrumbs onto registered handlers. Both are inert unless a - // command (today: `raw preview-next --debug-port`) actually registers - // handlers and serves the admin surface that lets an operator flip them on. + // command (today: `preview --debug-port`) actually registers handlers and + // serves the admin surface that lets an operator flip them on. let registry = service_kit::Registry::new(); let env_filter = EnvFilter::builder() diff --git a/crates/flowctl/src/raw/preview_next/capture_driver.rs b/crates/flowctl/src/preview/capture_driver.rs similarity index 85% rename from crates/flowctl/src/raw/preview_next/capture_driver.rs rename to crates/flowctl/src/preview/capture_driver.rs index 6f58da49d44..c6e71e7e8c4 100644 --- a/crates/flowctl/src/raw/preview_next/capture_driver.rs +++ b/crates/flowctl/src/preview/capture_driver.rs @@ -1,4 +1,4 @@ -//! Capture driver for `flowctl preview-next`. +//! Capture driver for `flowctl preview`. //! //! Captures are leaderless: each shard runs its own connector container, //! RocksDB, publish loop, and transaction loop with no cross-shard coordination. @@ -12,7 +12,9 @@ //! interpreting the output as a workload signal requires a range-partitioning //! connector. -use crate::raw::preview_next::services::Run; +use crate::preview::Controls; +use crate::preview::services::Run; +use anyhow::Context; use prost::Message; use proto_flow::{flow, runtime as cruntime}; use runtime_next::proto; @@ -24,10 +26,10 @@ pub async fn run_sessions( run: &Run, spec: &flow::CaptureSpec, session_targets: Vec, + controls: Controls, stop_token: CancellationToken, ) -> anyhow::Result<()> { - let join_shards = - crate::raw::preview_next::shards::build_capture_join_shards(run.n_shards, spec)?; + let join_shards = crate::preview::shards::build_capture_join_shards(run.n_shards, spec)?; let mut handles = Vec::with_capacity(run.n_shards as usize); for i in 0..run.n_shards { @@ -37,16 +39,25 @@ pub async fn run_sessions( // own auto-managed tempdir via RocksDB::open(None). rocksdb_path: (i == 0).then(|| run.rocksdb_path.clone()), network: run.network.clone(), - log_handler: run.log_handler, registry: run.registry.clone(), }; let spec = spec.clone(); let join_shard = join_shards[i as usize].clone(); let session_targets = session_targets.clone(); + let controls = controls.clone(); let stop_token = stop_token.clone(); handles.push(tokio::spawn(async move { - drive_one_shard(run_handle, spec, i, join_shard, session_targets, stop_token).await + drive_one_shard( + run_handle, + spec, + i, + join_shard, + session_targets, + controls, + stop_token, + ) + .await })); } @@ -78,7 +89,6 @@ pub async fn run_sessions( struct RunHandle { rocksdb_path: Option, network: String, - log_handler: fn(&::ops::Log), registry: service_kit::Registry, } @@ -88,23 +98,18 @@ async fn drive_one_shard( shard_index: u32, join_shard: proto::join::Shard, session_targets: Vec, + controls: Controls, stop_token: CancellationToken, ) -> anyhow::Result<()> { let task_name = format!("preview-capture-{shard_index:03}"); - let publisher_factory: gazette::journal::ClientFactory = std::sync::Arc::new({ - move |_authz_sub: String, _authz_obj: String| -> gazette::journal::Client { - unreachable!("live Publisher is not used by preview ({_authz_sub}, {_authz_obj})") - } - }); - let shard_svc = runtime_next::shard::Service::new( cruntime::Plane::Local, run.network, - run.log_handler, None, task_name, - publisher_factory, + controls.publisher_factory.clone(), + controls.logger_factory.clone(), run.registry, None, // No AuthN+AuthZ signer (local loopback). ); @@ -113,6 +118,23 @@ async fn drive_one_shard( let mut response_rx = shard_svc.spawn_capture(UnboundedReceiverStream::new(request_rx)); let spec_bytes: bytes::Bytes = spec.encode_to_vec().into(); + // Seed shard zero's RocksDB with any `--initial-state` before the runtime + // opens it at SessionLoop, so it recovers the state on its first scan. + // Only shard zero carries a tracked `rocksdb_path`. + if let Some(rocksdb_path) = &run.rocksdb_path { + if !controls.initial_state_json.is_empty() { + runtime_next::seed_initial_connector_state( + cruntime::RocksDbDescriptor { + rocksdb_path: rocksdb_path.clone(), + rocksdb_env_memptr: 0, + }, + &controls.initial_state_json, + ) + .await + .context("seeding --initial-state into shard-zero RocksDB")?; + } + } + let rocksdb_descriptor = run.rocksdb_path.map(|p| cruntime::RocksDbDescriptor { rocksdb_path: p, rocksdb_env_memptr: 0, @@ -156,7 +178,6 @@ async fn drive_one_shard( .send(Ok(proto::Capture { task: Some(proto::Task { spec: spec_bytes.clone(), - preview: true, max_transactions: target_txns, sqlite_vfs_uri: String::new(), publisher_id: Default::default(), diff --git a/crates/flowctl/src/raw/preview_next/derive_driver.rs b/crates/flowctl/src/preview/derive_driver.rs similarity index 83% rename from crates/flowctl/src/raw/preview_next/derive_driver.rs rename to crates/flowctl/src/preview/derive_driver.rs index 39be16afe28..e4c6f34bf4e 100644 --- a/crates/flowctl/src/raw/preview_next/derive_driver.rs +++ b/crates/flowctl/src/preview/derive_driver.rs @@ -5,7 +5,9 @@ //! as the `Task.sqlite_vfs_uri` (production supplies a recorded recovery-log //! VFS instead). -use crate::raw::preview_next::services::Run; +use crate::preview::Controls; +use crate::preview::services::Run; +use anyhow::Context; use prost::Message; use proto_flow::{flow, flow::collection_spec::derivation::ConnectorType, runtime as cruntime}; use runtime_next::proto; @@ -17,10 +19,11 @@ pub async fn run_sessions( run: &Run, spec: &flow::CollectionSpec, session_targets: Vec, + fixture_dirs: Vec, + controls: Controls, stop_token: CancellationToken, ) -> anyhow::Result<()> { - let join_shards = - crate::raw::preview_next::shards::build_derive_join_shards(run.n_shards, spec)?; + let join_shards = crate::preview::shards::build_derive_join_shards(run.n_shards, spec)?; // SQLite derivations require a VFS URI; preview supplies a plain tempfile // path (the connector opens it with SQLite's default file VFS). @@ -37,12 +40,13 @@ pub async fn run_sessions( shuffle_log_dir: run.shuffle_log_dir.clone(), rocksdb_path: run.rocksdb_path.clone(), network: run.network.clone(), - log_handler: run.log_handler, registry: run.registry.clone(), }; let spec = spec.clone(); let join_shards = join_shards.clone(); let session_targets = session_targets.clone(); + let fixture_dirs = fixture_dirs.clone(); + let controls = controls.clone(); let stop_token = stop_token.clone(); handles.push(tokio::spawn(async move { @@ -53,6 +57,8 @@ pub async fn run_sessions( is_sqlite, join_shards, session_targets, + fixture_dirs, + controls, stop_token, ) .await @@ -85,7 +91,6 @@ struct RunHandle { shuffle_log_dir: String, rocksdb_path: String, network: String, - log_handler: fn(&::ops::Log), registry: service_kit::Registry, } @@ -96,25 +101,21 @@ async fn drive_one_shard( is_sqlite: bool, join_shards: Vec, session_targets: Vec, + fixture_dirs: Vec, + controls: Controls, stop_token: CancellationToken, ) -> anyhow::Result<()> { let (request_tx, request_rx) = mpsc::unbounded_channel::>(); let task_name = format!("preview-derive-{shard_index:03}"); - let publisher_factory: gazette::journal::ClientFactory = std::sync::Arc::new({ - move |_authz_sub: String, _authz_obj: String| -> gazette::journal::Client { - unreachable!("live Publisher is not used by preview ({_authz_sub}, {_authz_obj})") - } - }); - let shard_svc = runtime_next::shard::Service::new( cruntime::Plane::Local, run.network.clone(), - run.log_handler, None, task_name, - publisher_factory, + controls.publisher_factory.clone(), + controls.logger_factory.clone(), run.registry, None, // No AuthN+AuthZ signer (local loopback). ); @@ -130,6 +131,20 @@ async fn drive_one_shard( String::new() }; + // Seed shard zero's RocksDB with any `--initial-state` before the runtime + // opens it at SessionLoop, so it recovers the state on its first scan. + if shard_index == 0 && !controls.initial_state_json.is_empty() { + runtime_next::seed_initial_connector_state( + cruntime::RocksDbDescriptor { + rocksdb_path: run.rocksdb_path.clone(), + rocksdb_env_memptr: 0, + }, + &controls.initial_state_json, + ) + .await + .context("seeding --initial-state into shard-zero RocksDB")?; + } + let rocksdb_descriptor = if shard_index == 0 { Some(cruntime::RocksDbDescriptor { rocksdb_path: run.rocksdb_path.clone(), @@ -151,13 +166,20 @@ async fn drive_one_shard( } let session_index = idx + 1; + // A fixture preview reads each session from its own directory (fresh + // segments from segment one); live preview shares the run's directory. + let shuffle_directory = fixture_dirs + .get(idx) + .cloned() + .unwrap_or_else(|| run.shuffle_log_dir.clone()); + request_tx .send(Ok(proto::Derive { join: Some(proto::Join { etcd_mod_revision: session_index as i64, shards: join_shards.clone(), shard_index, - shuffle_directory: run.shuffle_log_dir.clone(), + shuffle_directory, shuffle_endpoint: run.peer_endpoint.clone(), leader_endpoint: run.peer_endpoint.clone(), }), @@ -177,10 +199,9 @@ async fn drive_one_shard( .send(Ok(proto::Derive { task: Some(proto::Task { spec: spec_bytes.clone(), - preview: true, max_transactions: target_txns, sqlite_vfs_uri: sqlite_vfs_uri.clone(), - publisher_id: Default::default(), // Unused when `preview`. + publisher_id: Default::default(), // The harness forwards no leader producer. }), ..Default::default() })) diff --git a/crates/flowctl/src/raw/preview_next/driver.rs b/crates/flowctl/src/preview/driver.rs similarity index 82% rename from crates/flowctl/src/raw/preview_next/driver.rs rename to crates/flowctl/src/preview/driver.rs index 0b7d405fcfa..eb455852de4 100644 --- a/crates/flowctl/src/raw/preview_next/driver.rs +++ b/crates/flowctl/src/preview/driver.rs @@ -3,7 +3,9 @@ //! SessionLoop/Join/Task envelopes the controller (Go in production) would //! normally send. -use crate::raw::preview_next::services::Run; +use crate::preview::Controls; +use crate::preview::services::Run; +use anyhow::Context; use prost::Message; use proto_flow::{flow, runtime as cruntime}; use runtime_next::proto; @@ -18,10 +20,11 @@ pub async fn run_sessions( run: &Run, spec: &flow::MaterializationSpec, session_targets: Vec, + fixture_dirs: Vec, + controls: Controls, stop_token: CancellationToken, ) -> anyhow::Result<()> { - let join_shards = - crate::raw::preview_next::shards::build_materialize_join_shards(run.n_shards, spec)?; + let join_shards = crate::preview::shards::build_materialize_join_shards(run.n_shards, spec)?; let mut handles = Vec::with_capacity(run.n_shards as usize); for i in 0..run.n_shards { @@ -30,12 +33,13 @@ pub async fn run_sessions( shuffle_log_dir: run.shuffle_log_dir.clone(), rocksdb_path: run.rocksdb_path.clone(), network: run.network.clone(), - log_handler: run.log_handler, registry: run.registry.clone(), }; let spec = spec.clone(); let join_shards = join_shards.clone(); let session_targets = session_targets.clone(); + let fixture_dirs = fixture_dirs.clone(); + let controls = controls.clone(); let stop_token = stop_token.clone(); handles.push(tokio::spawn(async move { @@ -45,6 +49,8 @@ pub async fn run_sessions( i, join_shards, session_targets, + fixture_dirs, + controls, stop_token, ) .await @@ -82,7 +88,6 @@ struct RunHandle { shuffle_log_dir: String, rocksdb_path: String, network: String, - log_handler: fn(&::ops::Log), registry: service_kit::Registry, } @@ -92,25 +97,21 @@ async fn drive_one_shard( shard_index: u32, join_shards: Vec, session_targets: Vec, + fixture_dirs: Vec, + controls: Controls, stop_token: CancellationToken, ) -> anyhow::Result<()> { let (request_tx, request_rx) = mpsc::unbounded_channel::>(); let task_name = format!("preview-shard-{shard_index:03}"); - let publisher_factory: gazette::journal::ClientFactory = std::sync::Arc::new({ - move |_authz_sub: String, _authz_obj: String| -> gazette::journal::Client { - unreachable!("live Publisher is not used by preview ({_authz_sub}, {_authz_obj})") - } - }); - let shard_svc = runtime_next::shard::Service::new( cruntime::Plane::Local, run.network.clone(), - run.log_handler, None, task_name, - publisher_factory, + controls.publisher_factory.clone(), + controls.logger_factory.clone(), run.registry, None, // No AuthN+AuthZ signer (local loopback). ); @@ -118,6 +119,20 @@ async fn drive_one_shard( let mut response_rx = shard_svc.spawn_materialize(UnboundedReceiverStream::new(request_rx)); let spec_bytes: bytes::Bytes = spec.encode_to_vec().into(); + // Seed shard zero's RocksDB with any `--initial-state` before the runtime + // opens it at SessionLoop, so it recovers the state on its first scan. + if shard_index == 0 && !controls.initial_state_json.is_empty() { + runtime_next::seed_initial_connector_state( + cruntime::RocksDbDescriptor { + rocksdb_path: run.rocksdb_path.clone(), + rocksdb_env_memptr: 0, + }, + &controls.initial_state_json, + ) + .await + .context("seeding --initial-state into shard-zero RocksDB")?; + } + // Open the SessionLoop once. `runtime-next` opens RocksDB here and keeps // the handle live across the repeated Join/Task sessions below. let rocksdb_descriptor = if shard_index == 0 { @@ -141,13 +156,20 @@ async fn drive_one_shard( } let session_index = idx + 1; + // A fixture preview reads each session from its own directory (fresh + // segments from segment one); live preview shares the run's directory. + let shuffle_directory = fixture_dirs + .get(idx) + .cloned() + .unwrap_or_else(|| run.shuffle_log_dir.clone()); + request_tx .send(Ok(proto::Materialize { join: Some(proto::Join { etcd_mod_revision: session_index as i64, shards: join_shards.clone(), shard_index, - shuffle_directory: run.shuffle_log_dir.clone(), + shuffle_directory, shuffle_endpoint: run.peer_endpoint.clone(), leader_endpoint: run.peer_endpoint.clone(), }), @@ -167,10 +189,9 @@ async fn drive_one_shard( .send(Ok(proto::Materialize { task: Some(proto::Task { spec: spec_bytes.clone(), - preview: true, max_transactions: target_txns, sqlite_vfs_uri: String::new(), - publisher_id: Default::default(), // Unused when `preview`. + publisher_id: Default::default(), // The harness forwards no leader producer. }), ..Default::default() })) diff --git a/crates/flowctl/src/preview/fixture.rs b/crates/flowctl/src/preview/fixture.rs new file mode 100644 index 00000000000..b085140d266 --- /dev/null +++ b/crates/flowctl/src/preview/fixture.rs @@ -0,0 +1,1008 @@ +//! Fixture input for `flowctl preview --fixture`. +//! +//! flowctl reads a newline-delimited fixture file and writes its transactions +//! directly as shuffle log segments (via the public [`shuffle::log::Writer`]), +//! producing one synthetic checkpoint [`shuffle::Frontier`] per transaction. +//! Those frontiers are relayed by a fixture [`ShuffleSessionFactory`] (see +//! [`fixture_opener`]), owned by this module: it hands the runtime-next leader +//! one frontier per checkpoint request — so the consumer reads fixture documents +//! exactly as if they came from live journals. All fixture machinery lives here; +//! the shuffle crate is unaware of fixtures and no `shuffle::Service` is +//! constructed. +//! +//! Fixture format (one JSON value per line, matching legacy `flowctl preview`): +//! - a document: `["collection/name", { ...document... }]` +//! - a commit marker: `{"commit": true}` +//! +//! Documents between commit markers form one transaction. Each transaction is +//! written as a single log block, and (paired with a collapsed transaction +//! duration window on the preview task spec) commits as exactly one runtime +//! transaction — preserving the 1:1 transaction boundaries of legacy fixture +//! preview. Empty transactions — consecutive commit markers — are deliberate +//! and preserved: connectors-repo fixtures lead with one to drive an initial +//! empty commit cycle, and apply-only tests use a fixture that is a single +//! bare `{"commit": true}` line. +//! +//! ## Per-session segments +//! +//! The runtime-next consumer's log `Reader` is ephemeral: it restarts at the +//! first segment each session and unlinks segments as it reads them — exactly as +//! in production, where each session re-derives its segments from the (durable) +//! source journals starting at the recovered offset. We mirror that: each +//! `--sessions` iteration gets its own directory holding only that session's +//! transactions, as fresh segments numbered from one. Publication clocks +//! increase globally across sessions so the runtime's recovered frontier doesn't +//! re-admit a prior session's documents, but each session's read barrier +//! (`flushed_lsn`) is session-local. +//! +//! ## Streaming fixtures +//! +//! A FIFO (or stdin `-`) fixture cannot be pre-planned: its transaction count +//! is unknown and reads block on the producing writer. [`start_streaming`] +//! instead runs a single unbounded session fed by a spawned feeder task, which +//! incrementally reads lines, writes each transaction as it commits, and relays +//! its frontier — the producer (e.g. a benchmark generator) paces the run. At +//! stream EOF the feeder sends a [`FixtureItem::Boundary`] whose ack +//! fires once every relayed frontier has been delivered, and only then triggers +//! a graceful stop: stopping any earlier would truncate transactions still +//! queued ahead of the consumer. + +use anyhow::Context; +use proto_gazette::uuid; +use runtime_next::{ShuffleSession, ShuffleSessionFactory}; +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; +use tokio::io::AsyncBufReadExt; +use tokio::sync::{Mutex, mpsc}; + +/// Fixed synthetic producer for all fixture documents (matches legacy preview). +const FIXTURE_PRODUCER: uuid::Producer = uuid::Producer([7, 19, 83, 3, 3, 17]); + +/// One queued item of a fixture-replay opener's channel. +pub enum FixtureItem { + /// A synthetic checkpoint Frontier, relayed to the consumer one per + /// `NextCheckpoint` request. + Frontier(shuffle::Frontier), + /// A session boundary: the current source stops delivering frontiers. + /// Because the channel is FIFO and frontiers relay one-per-request, the + /// boundary is received only after every prior Frontier has been delivered; + /// `reached` (when Some) fires at that moment, letting a streaming producer + /// trigger a graceful consumer stop without truncating queued transactions. + Boundary { + reached: Option>, + }, +} + +/// Build a fixture [`ShuffleSessionFactory`] that replays the [`FixtureItem`]s +/// sent on the returned channel, reading no journals. No `shuffle::Service` is +/// constructed: flowctl writes the log segments itself and feeds the matching +/// checkpoint Frontiers here. flowctl pushes one `Frontier` per fixture +/// transaction, then a `Boundary` per session boundary; dropping the sender +/// signals end-of-fixtures. +pub fn fixture_opener() -> (FixtureOpener, mpsc::UnboundedSender) { + let (frontier_tx, frontier_rx) = mpsc::unbounded_channel::(); + let opener = FixtureOpener { + frontier_rx: Arc::new(Mutex::new(frontier_rx)), + }; + (opener, frontier_tx) +} + +/// A [`ShuffleSessionFactory`] that yields fixture-replay [`ShuffleSession`]s. +/// +/// Preview drives sessions strictly sequentially (one `--sessions` iteration at +/// a time), so the single frontier receiver is shared behind a mutex: each +/// `open` acquires it for that session's lifetime, and the next session blocks +/// until the prior [`ShuffleSession`] is closed (or dropped). The journal-reading +/// Session logic in the shuffle crate is bypassed entirely. +pub struct FixtureOpener { + frontier_rx: Arc>>, +} + +impl ShuffleSessionFactory for FixtureOpener { + type Session = FixtureCheckpoints; + + async fn open( + &self, + _task: shuffle::proto::Task, + _shards: Vec, + _resume: shuffle::Frontier, + ) -> anyhow::Result { + // Acquire the shared frontier stream for this session's lifetime; a + // following session blocks here until the prior source releases it. A + // single-shard fixture trusts its own write cursor, so the task spec, + // topology, and resume Frontier are unused. + let frontier_rx = self.frontier_rx.clone().lock_owned().await; + Ok(FixtureCheckpoints { + frontier_rx, + boundary_reached: false, + }) + } +} + +/// A fixture-replay [`ShuffleSession`]: yields one queued [`FixtureItem:: +/// Frontier`] per checkpoint request. A [`FixtureItem::Boundary`] (or a dropped +/// sender) ends this session's frontiers — the request is left unanswered (the +/// leader stops via its `max_transactions` limit or an external Stop), and every +/// subsequent request parks, so a stopping leader's speculative checkpoint can't +/// pop into the next session's frontiers. +pub struct FixtureCheckpoints { + frontier_rx: tokio::sync::OwnedMutexGuard>, + /// Set once a Boundary (or end-of-fixtures) is observed; latches every + /// further `recv_checkpoint` into an unresolving park. + boundary_reached: bool, +} + +impl ShuffleSession for FixtureCheckpoints { + fn request_checkpoint(&self) { + // No request protocol: `recv_checkpoint` pops the next queued frontier. + } + + async fn recv_checkpoint(&mut self) -> anyhow::Result { + // Once the boundary is reached, never touch the channel again: a + // re-issued request must not pop the next session's first frontier. + if self.boundary_reached { + return std::future::pending().await; + } + match self.frontier_rx.recv().await { + Some(FixtureItem::Frontier(frontier)) => Ok(frontier), + Some(FixtureItem::Boundary { reached }) => { + self.boundary_reached = true; + // Every prior frontier has been delivered (the channel is FIFO); + // tell a streaming producer it may now request a graceful stop. + if let Some(reached) = reached { + let _ = reached.send(()); + } + std::future::pending().await + } + None => { + self.boundary_reached = true; + std::future::pending().await + } + } + } + + async fn close(self) -> anyhow::Result<()> { + // Dropping releases the shared frontier stream for the next session. + Ok(()) + } +} + +/// A parsed fixture transaction: documents and their source collection names. +type Transaction = Vec<(String, serde_json::Value)>; + +/// A materialized fixture, ready to drive a preview run. +pub struct FixturePlan { + /// Per-session transaction budgets, bounded by the available fixtures. + pub session_targets: Vec, + /// Per-session shuffle-log directory; the shard for session `i` reads from + /// `session_dirs[i]` (carried in its `Join.shuffle_directory`). + pub session_dirs: Vec, + /// Per-session checkpoint frontiers, in order; fed one-per-NextCheckpoint. + pub session_frontiers: Vec>, + /// Retained writers/segments: their files are unlinked on drop, so they must + /// outlive the consumer's reads (i.e. the whole preview run). + _keepalive: Keepalive, +} + +struct Keepalive { + _writers: Vec, + _sealed: Vec, +} + +/// Parse `path` and write its transactions as shuffle log segments, one +/// per-session directory under `base_dir`. `requested_targets` are the +/// `--sessions` budgets (`0` = unbounded); the returned plan bounds them by the +/// number of fixture transactions. `task` supplies the binding ↔ collection +/// mapping and shuffle key extractors. +pub fn build( + task: &shuffle::proto::Task, + path: &std::path::Path, + base_dir: &std::path::Path, + requested_targets: &[u32], +) -> anyhow::Result { + let (bindings, mut validators, collection_bindings) = task_bindings(task)?; + + let mut transactions = parse(path)?; + // A session bounded by `max_transactions` can't run zero transactions, so + // an empty fixture file becomes one empty transaction: the session still + // runs the connector's Apply and one empty commit cycle before stopping. + if transactions.is_empty() { + transactions.push(Vec::new()); + } + let session_targets = session_targets(requested_targets, transactions.len()); + + let mut keepalive = Keepalive { + _writers: Vec::new(), + _sealed: Vec::new(), + }; + let mut session_dirs = Vec::with_capacity(session_targets.len()); + let mut session_frontiers = Vec::with_capacity(session_targets.len()); + + // Publication clock and per-(journal, binding) committed offsets advance + // globally across sessions; segment LSNs restart per session. Offsets are + // tracked per binding to mirror live reads, where each binding of a shared + // journal independently observes that journal's (single) offset space. + let mut clock = uuid::Clock::from_unix(1, 0); + let mut journal_offsets: HashMap<(String, u16), i64> = HashMap::new(); + let mut packed_key = bytes::BytesMut::new(); + let mut transactions = transactions.into_iter(); + + for (session_index, &budget) in session_targets.iter().enumerate() { + let dir = base_dir.join(format!("{session_index:03}")); + std::fs::create_dir(&dir) + .with_context(|| format!("creating fixture session directory {dir:?}"))?; + let dir = dir.to_string_lossy().into_owned(); + + let mut writer = shuffle::log::Writer::new(std::path::Path::new(&dir), 0) + .context("opening fixture shuffle-log writer")?; + let mut last_lsn = shuffle::log::Lsn::ZERO; + let mut frontiers = Vec::with_capacity(budget as usize); + + for _ in 0..budget { + let transaction = transactions + .next() + .expect("session_targets are bounded by the transaction count"); + + frontiers.push(write_transaction( + &transaction, + &bindings, + &mut validators, + &collection_bindings, + &mut writer, + &mut keepalive._sealed, + &mut clock, + &mut journal_offsets, + &mut packed_key, + &mut last_lsn, + )?); + } + + keepalive._writers.push(writer); + session_dirs.push(dir); + session_frontiers.push(frontiers); + } + + Ok(FixturePlan { + session_targets, + session_dirs, + session_frontiers, + _keepalive: keepalive, + }) +} + +/// Start a streaming fixture: spawn a feeder task that incrementally reads +/// newline-delimited fixture lines from `path` (or stdin, when `None`), writes +/// each transaction as it commits, and relays its frontier. Returns the single +/// session's shuffle-log directory and the feeder's join handle. +/// +/// Feeder lifecycle: +/// - At stream EOF — or on an error, such as a malformed line — it sends a +/// `Boundary` whose ack fires once every relayed frontier has been delivered +/// to the consumer, and only then cancels `eof_stop`: a graceful stop which +/// cannot truncate still-queued transactions, nor land mid session-startup +/// (where a Stop is a protocol error). An error surfaces when the caller +/// joins the returned handle after the run. +/// - It retains the log writer and sealed segments until `hold` cancels (the +/// run ended): the consumer unlinks segment files as it reads, and the +/// writer/segment drops tolerate NotFound. +pub fn start_streaming( + task: &shuffle::proto::Task, + path: Option, + base_dir: &std::path::Path, + frontier_tx: tokio::sync::mpsc::UnboundedSender, + eof_stop: tokio_util::sync::CancellationToken, + hold: tokio_util::sync::CancellationToken, +) -> anyhow::Result<(String, tokio::task::JoinHandle>)> { + let (bindings, validators, collection_bindings) = task_bindings(task)?; + + // The session reads from its own directory, mirroring the eager per-session + // layout. + let dir = base_dir.join("000"); + std::fs::create_dir(&dir) + .with_context(|| format!("creating fixture session directory {dir:?}"))?; + let dir = dir.to_string_lossy().into_owned(); + + let writer = shuffle::log::Writer::new(std::path::Path::new(&dir), 0) + .context("opening fixture shuffle-log writer")?; + + let handle = tokio::spawn(feed_stream( + bindings, + validators, + collection_bindings, + path, + writer, + frontier_tx, + eof_stop, + hold, + )); + Ok((dir, handle)) +} + +async fn feed_stream( + bindings: Vec, + mut validators: Vec, + collection_bindings: HashMap>, + path: Option, + mut writer: shuffle::log::Writer, + frontier_tx: tokio::sync::mpsc::UnboundedSender, + eof_stop: tokio_util::sync::CancellationToken, + hold: tokio_util::sync::CancellationToken, +) -> anyhow::Result<()> { + let mut sealed = Vec::new(); + let result = feed_lines( + &bindings, + &mut validators, + &collection_bindings, + path, + &mut writer, + &mut sealed, + &frontier_tx, + &hold, + ) + .await; + + // Request the graceful stop only once every relayed frontier has been + // delivered (the Boundary ack): stopping earlier would truncate queued + // transactions, or interrupt session startup (where a Stop is a protocol + // error). This applies to errors too — transactions committed before a + // malformed line still run, and the error surfaces at join time. + let (reached_tx, reached_rx) = tokio::sync::oneshot::channel(); + if frontier_tx + .send(FixtureItem::Boundary { + reached: Some(reached_tx), + }) + .is_ok() + { + tokio::select! { + _ = reached_rx => (), + () = hold.cancelled() => (), // The run ended some other way. + } + } + eof_stop.cancel(); + + // The writer and sealed segments must outlive the consumer's reads. + () = hold.cancelled().await; + result +} + +/// Incrementally read fixture lines, writing each transaction as it commits and +/// relaying its frontier. Returns at stream EOF, when the run ends (`hold` +/// cancels), or on a stream / fixture error. +async fn feed_lines( + bindings: &[shuffle::Binding], + validators: &mut [doc::Validator], + collection_bindings: &HashMap>, + path: Option, + writer: &mut shuffle::log::Writer, + sealed: &mut Vec, + frontier_tx: &tokio::sync::mpsc::UnboundedSender, + hold: &tokio_util::sync::CancellationToken, +) -> anyhow::Result<()> { + // Opening a FIFO blocks until its writer connects (stdin is immediate). + let reader: std::pin::Pin> = match &path { + Some(path) => { + let file = tokio::select! { + biased; + () = hold.cancelled() => return Ok(()), + file = tokio::fs::File::open(path) => { + file.with_context(|| format!("opening fixture stream {path:?}"))? + } + }; + Box::pin(file) + } + None => Box::pin(tokio::io::stdin()), + }; + let mut lines = tokio::io::BufReader::new(reader).lines(); + + let mut clock = uuid::Clock::from_unix(1, 0); + let mut journal_offsets: HashMap<(String, u16), i64> = HashMap::new(); + let mut packed_key = bytes::BytesMut::new(); + let mut last_lsn = shuffle::log::Lsn::ZERO; + + let mut current: Transaction = Vec::new(); + let mut committed = 0usize; + let mut lineno = 0usize; + + loop { + // `hold` cancelling means the run ended out from under us (Ctrl-C or + // timeout): abandon the stream rather than waiting on its writer. + let line = tokio::select! { + biased; + () = hold.cancelled() => return Ok(()), + line = lines.next_line() => line.context("reading fixture stream")?, + }; + let Some(line) = line else { + break; // EOF: the stream's writer closed. + }; + lineno += 1; + + match parse_line(&line, lineno)? { + None => (), + Some(Line::Doc(collection, doc)) => current.push((collection, doc)), + Some(Line::Commit) => { + let frontier = write_transaction( + &std::mem::take(&mut current), + bindings, + validators, + collection_bindings, + writer, + sealed, + &mut clock, + &mut journal_offsets, + &mut packed_key, + &mut last_lsn, + )?; + committed += 1; + if frontier_tx.send(FixtureItem::Frontier(frontier)).is_err() { + return Ok(()); // The consumer went away. + } + } + } + } + + // Trailing documents without a final commit marker form a final + // transaction, and an entirely-empty stream still runs one empty + // transaction (the connector's Apply and one empty commit cycle) — both + // mirroring eager parsing. + if !current.is_empty() || committed == 0 { + let frontier = write_transaction( + ¤t, + bindings, + validators, + collection_bindings, + writer, + sealed, + &mut clock, + &mut journal_offsets, + &mut packed_key, + &mut last_lsn, + )?; + let _ = frontier_tx.send(FixtureItem::Frontier(frontier)); + } + Ok(()) +} + +/// Build shuffle bindings and validators for `task`, plus a map from each +/// source collection name to the binding indices it feeds (a collection may be +/// read by multiple derivation transforms). +fn task_bindings( + task: &shuffle::proto::Task, +) -> anyhow::Result<( + Vec, + Vec, + HashMap>, +)> { + let (bindings, validators) = + shuffle::Binding::from_task(task).context("building shuffle bindings from task")?; + + let mut collection_bindings: HashMap> = HashMap::new(); + for (index, binding) in bindings.iter().enumerate() { + collection_bindings + .entry(binding.collection.to_string()) + .or_default() + .push(index); + } + Ok((bindings, validators, collection_bindings)) +} + +/// Write one transaction as a single log block and return its checkpoint +/// frontier. The block's `flushed_lsn` is the session-local LSN; the producer's +/// `last_commit` is the transaction's max clock so all of its documents are +/// visible at the checkpoint. +fn write_transaction( + transaction: &Transaction, + bindings: &[shuffle::Binding], + validators: &mut [doc::Validator], + collection_bindings: &HashMap>, + writer: &mut shuffle::log::Writer, + sealed: &mut Vec, + clock: &mut uuid::Clock, + journal_offsets: &mut HashMap<(String, u16), i64>, + packed_key: &mut bytes::BytesMut, + last_lsn: &mut shuffle::log::Lsn, +) -> anyhow::Result { + let mut entries: Vec<(shuffle::log::BlockMeta, u32, bytes::Bytes, bytes::Bytes)> = Vec::new(); + let mut block_journals: HashMap = HashMap::new(); + // (journal, binding) => (max committed clock, source bytes this txn). + let mut frontier_acc: BTreeMap<(String, u16), (uuid::Clock, i64)> = BTreeMap::new(); + + for (collection, doc) in transaction { + let Some(binding_indices) = collection_bindings.get(collection.as_str()) else { + continue; // Collection isn't a source of this task. + }; + + for &bi in binding_indices { + let binding = &bindings[bi]; + let journal = fixture_journal(&binding.collection); + let doc_clock = clock.tick(); + + // Inject a synthetic UUID at the collection's UUID pointer. + let mut doc = doc.clone(); + let synthetic_uuid = uuid::build(FIXTURE_PRODUCER, doc_clock, uuid::Flags::OUTSIDE_TXN); + *json::ptr::create_value(&binding.source_uuid_ptr, &mut doc) + .context("creating fixture UUID location in document")? = + serde_json::json!(synthetic_uuid.as_hyphenated().to_string()); + + let alloc = doc::HeapNode::new_allocator(); + let heap = + doc::HeapNode::from_serde(&doc, &alloc).context("allocating fixture document")?; + let archive = heap.to_archive(); + let archived = doc::ArchivedNode::from_archive(archive.as_slice()); + + // Mirror the slice: set the schema-valid flag from validation and + // pack the shuffle key from the archived document. + let mut flags = uuid::Flags::OUTSIDE_TXN.0; + if validators[bi].is_valid(archived) { + flags |= shuffle::FLAGS_SCHEMA_VALID; + } + + packed_key.clear(); + doc::Extractor::extract_all( + archived, + &binding.key_extractors, + doc::Encoding::Packed, + packed_key, + None, + ); + + let doc_bytes = bytes::Bytes::from(archive.to_vec()); + let source_len = doc_bytes.len() as u32; + + let journal_bid = { + let next = block_journals.len() as u16; + *block_journals.entry(journal.clone()).or_insert(next) + }; + + entries.push(( + shuffle::log::BlockMeta { + binding: binding.index, + journal_bid, + producer_bid: 0, + flags, + clock: doc_clock.as_u64(), + }, + source_len, + packed_key.split().freeze(), + doc_bytes, + )); + + let acc = frontier_acc + .entry((journal.clone(), binding.index)) + .or_insert((uuid::Clock::from_u64(0), 0)); + acc.0 = acc.0.max(doc_clock); + acc.1 += source_len as i64; + *journal_offsets.entry((journal, binding.index)).or_insert(0) += source_len as i64; + } + } + + // Write this transaction's documents as a single block (if any), advancing + // the session-local read barrier to its LSN. + if !entries.is_empty() { + let producers: HashMap = [(FIXTURE_PRODUCER, 0)].into(); + let (lsn, rolled) = writer + .append_block(block_journals, producers, entries) + .context("writing fixture log block")?; + if let Some(rolled) = rolled { + sealed.push(rolled); + } + *last_lsn = lsn; + } + + // `frontier_acc` iterates sorted by (journal, binding), satisfying Frontier + // ordering invariants. + let journals: Vec = frontier_acc + .into_iter() + .map(|((journal, binding), (last_commit, bytes_read))| { + let offset = -journal_offsets + .get(&(journal.clone(), binding)) + .copied() + .unwrap_or(0); + shuffle::JournalFrontier { + journal: journal.into(), + binding, + producers: vec![shuffle::ProducerFrontier { + producer: FIXTURE_PRODUCER, + last_commit, + hinted_commit: uuid::Clock::from_u64(0), + offset, + }], + bytes_read_delta: bytes_read, + bytes_behind_delta: 0, + } + }) + .collect(); + + shuffle::Frontier::new(journals, vec![last_lsn.as_u64()]) + .context("building fixture checkpoint frontier") +} + +/// Synthetic journal name for a collection's fixture documents. The runtime-next +/// consumer ignores the journal name during processing; it is carried only in the +/// checkpoint frontier (where it must match the block's journal for visibility). +fn fixture_journal(collection: &models::Collection) -> String { + format!("{}/fixture", collection.as_str()) +} + +/// Bound each requested session's transaction target by the fixtures still +/// unconsumed. An "unbounded" request (`0`) consumes the remainder; sessions past +/// exhaustion are dropped. Each session then ends cleanly via its +/// `max_transactions` limit once its fixtures are processed. +fn session_targets(requested: &[u32], txn_count: usize) -> Vec { + let mut remaining = txn_count; + let mut out = Vec::with_capacity(requested.len()); + for &target in requested { + if remaining == 0 { + break; + } + let take = if target == 0 { + remaining + } else { + (target as usize).min(remaining) + }; + out.push(take as u32); + remaining -= take; + } + out +} + +/// Read and parse a fixture file into transactions. +fn parse(path: &std::path::Path) -> anyhow::Result> { + let content = + std::fs::read_to_string(path).with_context(|| format!("reading fixture file {path:?}"))?; + parse_content(&content) +} + +/// Parse fixture content into transactions, splitting on `{"commit": true}` +/// lines. Trailing documents without a final commit marker form a final +/// transaction. +fn parse_content(content: &str) -> anyhow::Result> { + let mut transactions: Vec = Vec::new(); + let mut current: Transaction = Vec::new(); + + for (lineno, line) in content.lines().enumerate() { + match parse_line(line, lineno + 1)? { + None => continue, + Some(Line::Commit) => transactions.push(std::mem::take(&mut current)), + Some(Line::Doc(collection, doc)) => current.push((collection, doc)), + } + } + + if !current.is_empty() { + transactions.push(current); + } + + Ok(transactions) +} + +/// One parsed fixture line: a transaction boundary or a sourced document. +enum Line { + Commit, + Doc(String, serde_json::Value), +} + +/// Parse a single fixture line (`None` for blank lines); `lineno` is 1-based. +fn parse_line(line: &str, lineno: usize) -> anyhow::Result> { + let line = line.trim(); + if line.is_empty() { + return Ok(None); + } + if is_commit_line(line) { + return Ok(Some(Line::Commit)); + } + let (collection, doc): (String, serde_json::Value) = serde_json::from_str(line) + .with_context(|| format!("fixture line {lineno} is not [collection, document]: {line}"))?; + Ok(Some(Line::Doc(collection, doc))) +} + +/// True if `line` is a `{"commit": true}` transaction boundary marker. +fn is_commit_line(line: &str) -> bool { + serde_json::from_str::(line) + .ok() + .as_ref() + .and_then(|v| v.as_object()) + .and_then(|o| o.get("commit")) + .and_then(|c| c.as_bool()) + .unwrap_or(false) +} + +#[cfg(test)] +mod test { + use super::*; + + /// A checkpoint Frontier carrying `lsn` as its single `flushed_lsn`. + fn frontier(lsn: u64) -> shuffle::Frontier { + shuffle::Frontier::new( + vec![shuffle::JournalFrontier { + journal: "fixture/test/coll".into(), + binding: 0, + producers: vec![shuffle::ProducerFrontier { + producer: uuid::Producer::from_bytes([0x01, 0, 0, 0, 0, 0]), + last_commit: uuid::Clock::from_unix(lsn, 0), + hinted_commit: uuid::Clock::from_u64(0), + offset: -(lsn as i64), + }], + bytes_read_delta: 0, + bytes_behind_delta: 0, + }], + vec![lsn], + ) + .unwrap() + } + + #[tokio::test] + async fn relays_one_frontier_per_checkpoint() { + let (opener, frontier_tx) = fixture_opener(); + + // Open a source for the first session; task/topology/resume are unused. + let mut src = opener + .open(Default::default(), Vec::new(), shuffle::Frontier::default()) + .await + .unwrap(); + + // Each request yields the next queued frontier, in order. + frontier_tx + .send(FixtureItem::Frontier(frontier(1))) + .unwrap(); + frontier_tx + .send(FixtureItem::Frontier(frontier(2))) + .unwrap(); + for expect_lsn in [1u64, 2] { + src.request_checkpoint(); + let frontier = src.recv_checkpoint().await.unwrap(); + assert_eq!(frontier.encode().flushed_lsn, vec![expect_lsn]); + } + + // A Boundary leaves this request unanswered (the leader stops via + // max_transactions). Its `reached` ack fires only now — after both + // frontiers were delivered. + let (reached_tx, reached_rx) = tokio::sync::oneshot::channel(); + frontier_tx + .send(FixtureItem::Boundary { + reached: Some(reached_tx), + }) + .unwrap(); + // A frontier queued *after* the boundary belongs to the next session. + frontier_tx + .send(FixtureItem::Frontier(frontier(3))) + .unwrap(); + + tokio::select! { + _ = src.recv_checkpoint() => panic!("recv_checkpoint must park on a Boundary"), + r = reached_rx => r.expect("boundary ack fires"), + } + + // The boundary latches: a re-issued request parks rather than popping + // the next session's frontier. + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), src.recv_checkpoint()) + .await + .is_err(), + "a post-boundary request must not steal the next session's frontier", + ); + + // Closing releases the shared stream; the next session resumes at the + // frontier queued after the boundary. + src.close().await.unwrap(); + let mut next = opener + .open(Default::default(), Vec::new(), shuffle::Frontier::default()) + .await + .unwrap(); + next.request_checkpoint(); + let frontier = next.recv_checkpoint().await.unwrap(); + assert_eq!(frontier.encode().flushed_lsn, vec![3]); + } + + #[test] + fn test_session_targets() { + // (requested, txn_count) => expected, where 0 means "unbounded". + // Unbounded consumes the remainder. + assert_eq!(session_targets(&[0], 3), vec![3]); + // Bounded sessions pass through when they fit. + assert_eq!(session_targets(&[2, 1], 3), vec![2, 1]); + // A trailing unbounded session takes the remainder; exhausted ones drop. + assert_eq!(session_targets(&[2, 1, 0], 3), vec![2, 1]); + assert_eq!(session_targets(&[1, 0], 3), vec![1, 2]); + // A bounded request larger than what's left is capped. + assert_eq!(session_targets(&[5], 3), vec![3]); + assert_eq!(session_targets(&[2, 5], 3), vec![2, 1]); + // Sessions beyond exhaustion are dropped. + assert_eq!(session_targets(&[1, 1, 1, 1], 2), vec![1, 1]); + // No fixtures: no sessions. + assert_eq!(session_targets(&[0], 0), Vec::::new()); + } + + #[test] + fn test_parse_content() { + let content = "\ +[\"a/coll\", {\"k\": 1}] +[\"b/coll\", {\"k\": 2}] +{\"commit\": true} + +[\"a/coll\", {\"k\": 3}] +{\"commit\": true} +[\"a/coll\", {\"k\": 4}] +"; + let txns = parse_content(content).unwrap(); + // Three transactions: two committed, plus a trailing un-committed one. + assert_eq!(txns.len(), 3); + assert_eq!(txns[0].len(), 2); + assert_eq!(txns[0][0].0, "a/coll"); + assert_eq!(txns[1].len(), 1); + assert_eq!(txns[2].len(), 1); + assert_eq!(txns[2][0].1, serde_json::json!({"k": 4})); + } + + /// A Task with no bindings: fixture documents are skipped (no collection is + /// a source), but the streaming feeder's transaction cadence — frontiers, + /// the EOF Boundary ack, and the graceful stop — is fully exercised. + fn empty_task() -> shuffle::proto::Task { + shuffle::proto::Task { + task: Some(shuffle::proto::task::Task::Materialization( + Default::default(), + )), + } + } + + struct StreamHarness { + _tmp: tempfile::TempDir, + frontier_rx: tokio::sync::mpsc::UnboundedReceiver, + eof_stop: tokio_util::sync::CancellationToken, + hold: tokio_util::sync::CancellationToken, + feeder: tokio::task::JoinHandle>, + } + + fn start_stream_harness(content: &str) -> StreamHarness { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("stream"); + std::fs::write(&path, content).unwrap(); + + let (frontier_tx, frontier_rx) = tokio::sync::mpsc::unbounded_channel(); + let eof_stop = tokio_util::sync::CancellationToken::new(); + let hold = tokio_util::sync::CancellationToken::new(); + + let (dir, feeder) = start_streaming( + &empty_task(), + Some(path), + tmp.path(), + frontier_tx, + eof_stop.clone(), + hold.clone(), + ) + .unwrap(); + assert!(dir.ends_with("000")); + + StreamHarness { + _tmp: tmp, + frontier_rx, + eof_stop, + hold, + feeder, + } + } + + #[tokio::test] + async fn test_streaming_cadence_and_eof_stop() { + let mut h = start_stream_harness( + "[\"a/coll\", {\"k\": 1}]\n{\"commit\": true}\n{\"commit\": true}\n[\"a/coll\", {\"k\": 2}]\n", + ); + + // Two committed transactions, plus the trailing document's final one. + for _ in 0..3 { + let item = h.frontier_rx.recv().await.unwrap(); + assert!(matches!(item, FixtureItem::Frontier(_))); + } + + // EOF: a Boundary whose ack (fired by the fixture source once every + // prior frontier was delivered) triggers the graceful stop. + let Some(FixtureItem::Boundary { reached: Some(ack) }) = h.frontier_rx.recv().await else { + panic!("expected an acked Boundary at EOF"); + }; + assert!(!h.eof_stop.is_cancelled()); + ack.send(()).unwrap(); + h.eof_stop.cancelled().await; + + // Releasing the hold lets the feeder drop its writer and exit cleanly. + h.hold.cancel(); + h.feeder.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn test_streaming_empty_stream() { + let mut h = start_stream_harness(""); + + // An entirely-empty stream still runs one empty transaction. + let item = h.frontier_rx.recv().await.unwrap(); + assert!(matches!(item, FixtureItem::Frontier(_))); + let item = h.frontier_rx.recv().await.unwrap(); + assert!(matches!(item, FixtureItem::Boundary { .. })); + + h.hold.cancel(); + h.feeder.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn test_streaming_malformed_line() { + let mut h = start_stream_harness("not json\n"); + + // An error still performs the Boundary handshake — so the stop can't + // land mid session-startup — and surfaces its parse error when joined. + let Some(FixtureItem::Boundary { reached: Some(ack) }) = h.frontier_rx.recv().await else { + panic!("expected an acked Boundary on error"); + }; + ack.send(()).unwrap(); + h.eof_stop.cancelled().await; + + h.hold.cancel(); + let err = h.feeder.await.unwrap().unwrap_err(); + assert!(format!("{err:#}").contains("fixture line 1"), "{err:#}"); + } + + /// Drive the feeder through a real FIFO: a frontier arrives while the + /// writer still holds the pipe open (proving incremental reads), and + /// closing the writer produces EOF. + #[cfg(unix)] + #[tokio::test] + async fn test_streaming_fifo() { + use tokio::io::AsyncWriteExt; + + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("fifo"); + assert!( + std::process::Command::new("mkfifo") + .arg(&path) + .status() + .unwrap() + .success() + ); + + let (frontier_tx, mut frontier_rx) = tokio::sync::mpsc::unbounded_channel(); + let eof_stop = tokio_util::sync::CancellationToken::new(); + let hold = tokio_util::sync::CancellationToken::new(); + + let (_dir, feeder) = start_streaming( + &empty_task(), + Some(path.clone()), + tmp.path(), + frontier_tx, + eof_stop.clone(), + hold.clone(), + ) + .unwrap(); + + // Opening the write end rendezvouses with the feeder's read-end open. + let mut pipe = tokio::fs::OpenOptions::new() + .write(true) + .open(&path) + .await + .unwrap(); + + pipe.write_all(b"[\"a/coll\", {\"k\": 1}]\n{\"commit\": true}\n") + .await + .unwrap(); + pipe.flush().await.unwrap(); + + // The transaction's frontier arrives while the pipe remains open. + let item = frontier_rx.recv().await.unwrap(); + assert!(matches!(item, FixtureItem::Frontier(_))); + + drop(pipe); // EOF. + + let Some(FixtureItem::Boundary { reached: Some(ack) }) = frontier_rx.recv().await else { + panic!("expected an acked Boundary at EOF"); + }; + ack.send(()).unwrap(); + eof_stop.cancelled().await; + + hold.cancel(); + feeder.await.unwrap().unwrap(); + } + + #[test] + fn test_is_commit_line() { + assert!(is_commit_line(r#"{"commit": true}"#)); + assert!(!is_commit_line(r#"{"commit": false}"#)); + assert!(!is_commit_line(r#"["a/coll", {"commit": true}]"#)); + assert!(!is_commit_line(r#"{"other": true}"#)); + assert!(!is_commit_line("not json")); + } +} diff --git a/crates/flowctl/src/preview/journal_reader.rs b/crates/flowctl/src/preview/journal_reader.rs deleted file mode 100644 index 91c3571f0fb..00000000000 --- a/crates/flowctl/src/preview/journal_reader.rs +++ /dev/null @@ -1,301 +0,0 @@ -use anyhow::Context; -use futures::{SinkExt, StreamExt, TryFutureExt, channel::mpsc}; -use proto_flow::flow; -use proto_gazette::broker; - -/// Reader is a runtime::harness::Reader which performs active reads of live -/// collection journals. -#[derive(Clone)] -pub struct Reader { - rest: flow_client_next::rest::Client, - user_tokens: tokens::PendingWatch, - router: gazette::Router, - delay: std::time::Duration, -} - -/// Source is a common read description across a derivation and materialization. -struct Source { - collection: String, - not_before: Option, - partition_selector: broker::LabelSelector, - read_suffix: String, -} - -impl Reader { - /// Return a new Reader which uses the `control_plane` to identify and read journals from - /// their respective collection data planes. - /// - /// `delay` is an artificial, injected delay between a read and a subsequent checkpoint. - /// It emulates back-pressure and encourages amortized transactions and reductions. - pub fn new(ctx: &crate::CliContext, delay: std::time::Duration) -> Self { - Self { - rest: ctx.rest.clone(), - user_tokens: ctx.user_tokens.clone(), - router: ctx.router.clone(), - delay, - } - } - - fn start( - self, - sources: Vec, - mut resume: proto_gazette::consumer::Checkpoint, - ) -> mpsc::Receiver> { - let reader = coroutines::try_coroutine(move |mut co| async move { - // Concurrently fetch authorizations for all sourced collections. - let sources = futures::future::try_join_all(sources.iter().map(|source| { - crate::dataplane::user_collection_journal( - &self.rest, - &self.user_tokens, - &self.router, - &source.collection, - models::Capability::Read, - ) - .map_ok(move |(_journal_name_prefix, client)| (source, client)) - })) - .await?; - - // Concurrently list the journals of every Source. - let journals: Vec<(&Source, Vec, &gazette::journal::Client)> = - futures::future::try_join_all(sources.iter().map(|(source, client)| { - Self::list_journals(*source, client).map_ok(move |l| (*source, l, client)) - })) - .await?; - - // Flatten into (binding, source, journal, client). - let journals: Vec<(u32, &Source, String, &gazette::journal::Client)> = journals - .iter() - .enumerate() - .flat_map(|(binding, (source, journals, client))| { - journals.into_iter().map(move |journal| { - ( - binding as u32, - *source, - format!("{};{}", journal.name, source.read_suffix), - *client, - ) - }) - }) - .collect(); - - // Map into a stream that yields lines from across all journals, as they're ready. - let mut journals = futures::stream::select_all(journals.iter().map( - |(binding, source, journal, client)| { - let initial_offset = resume - .sources - .get(journal.as_str()) - .map(|s| s.read_through) - .unwrap_or_default(); - Self::read_journal_lines(*binding, client, journal, initial_offset, source) - .boxed() - }, - )); - - // Reset-able timer for delivery of delayed checkpoints. - let deadline = tokio::time::sleep(std::time::Duration::MAX); - tokio::pin!(deadline); - - let mut in_txn = false; // Have we emitted a document that awaits a checkpoint? - - loop { - let step = tokio::select! { - Some(read) = journals.next() => Ok(read?), - () = deadline.as_mut(), if in_txn => Err(()) - }; - - match step { - Ok((binding, doc_json, journal, read_through)) => { - let resume = match resume.sources.get_mut(journal) { - Some(entry) => entry, - None => resume.sources.entry(journal.clone()).or_default(), - }; - resume.read_through = read_through; - - () = co - .yield_(runtime::harness::Read::Document { - binding: binding as u32, - doc: doc_json.into(), - }) - .await; - - // If this is the first Read of this transaction, - // schedule when it will Checkpoint. - if !in_txn { - in_txn = true; - deadline - .as_mut() - .reset(tokio::time::Instant::now() + self.delay); - } - } - Err(()) => { - () = co - .yield_(runtime::harness::Read::Checkpoint(resume.clone())) - .await; - in_txn = false; - } - } - } - }); - - // Dispatch through an mpsc for a modest parallelism improvement. - let (mut tx, rx) = mpsc::channel(runtime::CHANNEL_BUFFER); - - tokio::spawn(async move { - tokio::pin!(reader); - - while let Some(read) = reader.next().await { - if let Err(_) = tx.feed(read).await { - break; // Receiver was dropped. - } - } - }); - - rx - } - - async fn list_journals( - source: &Source, - client: &gazette::journal::Client, - ) -> anyhow::Result> { - let resp = client - .list(broker::ListRequest { - selector: Some(source.partition_selector.clone()), - ..Default::default() - }) - .await - .with_context(|| { - format!( - "failed to list journals for collection {}", - &source.collection - ) - })?; - - let listing = resp - .journals - .into_iter() - .map(|j| j.spec.unwrap()) - .collect::>(); - - if listing.is_empty() { - anyhow::bail!( - "the collection '{}' has not had any data written to it", - &source.collection, - ); - } - Ok(listing) - } - - fn read_journal_lines<'s>( - binding: u32, - client: &gazette::journal::Client, - journal: &'s String, - initial_offset: i64, - source: &Source, - ) -> impl futures::Stream> { - use gazette::journal::ReadJsonLine; - - let mut offset = initial_offset; - - let begin_mod_time = source - .not_before - .as_ref() - .map(|b| b.seconds) - .unwrap_or_default(); - - let mut lines = client.clone().read_json_lines( - broker::ReadRequest { - journal: journal.clone(), - offset, - block: true, - begin_mod_time, - // TODO(johnny): Set `do_not_proxy: true` once cronut is migrated. - ..Default::default() - }, - 1, - ); - - let ser_policy = doc::SerPolicy::noop(); - - coroutines::try_coroutine(move |mut co| async move { - while let Some(line) = lines.next().await { - let (root, next_offset) = match line { - Ok(ReadJsonLine::Doc { root, next_offset }) => (root, next_offset), - Ok(ReadJsonLine::Meta(meta)) => { - offset = meta.offset; - continue; - } - Err(gazette::RetryError { - attempt, - inner: err, - }) if err.is_transient() => { - tracing::warn!(?err, %attempt, %journal, %binding, "transient error reading journal (will retry)"); - continue; - } - Err(gazette::RetryError { inner, .. }) => return Err(inner), - }; - - // TODO(johnny): plumb through OwnedArchivedNode end-to-end. - let doc_json = serde_json::to_string(&ser_policy.on(root.get())).unwrap(); - - // TODO(johnny): This is pretty janky. - if doc_json.starts_with("{\"_meta\":{\"ack\":true,") { - continue; - } - - () = co.yield_((binding, doc_json, journal, offset)).await; - offset = next_offset; - } - Ok(()) - }) - } -} - -impl runtime::harness::Reader for Reader { - type Stream = mpsc::Receiver>; - - fn start_for_derivation( - self, - derivation: &flow::CollectionSpec, - resume: proto_gazette::consumer::Checkpoint, - ) -> Self::Stream { - let transforms = &derivation.derivation.as_ref().unwrap().transforms; - - let sources = transforms - .iter() - .map(|t| { - let collection = t.collection.as_ref().unwrap(); - - Source { - collection: collection.name.clone(), - not_before: t.not_before.clone(), - partition_selector: t.partition_selector.clone().unwrap(), - read_suffix: t.journal_read_suffix.clone(), - } - }) - .collect(); - - self.start(sources, resume) - } - - fn start_for_materialization( - self, - materialization: &flow::MaterializationSpec, - resume: proto_gazette::consumer::Checkpoint, - ) -> Self::Stream { - let sources = materialization - .bindings - .iter() - .map(|b| { - let collection = b.collection.as_ref().unwrap(); - Source { - collection: collection.name.clone(), - not_before: b.not_before.clone(), - partition_selector: b.partition_selector.clone().unwrap(), - read_suffix: b.journal_read_suffix.clone(), - } - }) - .collect(); - - self.start(sources, resume) - } -} diff --git a/crates/flowctl/src/preview/logger.rs b/crates/flowctl/src/preview/logger.rs new file mode 100644 index 00000000000..f4b9f56e8c3 --- /dev/null +++ b/crates/flowctl/src/preview/logger.rs @@ -0,0 +1,269 @@ +//! Preview-rendering logger for `flowctl preview`. +//! +//! Installs into runtime-next through its [`runtime_next::LoggerFactory`] +//! seam — the user-facing event channel. It is the observation half of +//! preview's output (the document half is [`super::publish`]): +//! +//! - [`Logger::log`](runtime_next::Logger::log) forwards the task's log +//! stream to the chosen `flowctl` log handler (stderr JSON or tracing). +//! - [`Logger::event`](runtime_next::Logger::event) intercepts the two +//! events preview renders to stdout: [`runtime_next::LogEvent::Applied`] becomes +//! the legacy `--output-apply` line, and [`runtime_next::LogEvent::Persist`] +//! becomes the legacy `--output-state` line(s), decoding the connector-state +//! delta from runtime-next's tab-framed patch wire format. +//! +//! All other events (and these two, when their flag is off) flatten through +//! their canonical [`runtime_next::LogEvent::to_log`] rendering into the same log +//! handler, alongside the connector's own logs. +//! +//! `--output-state` / `--output-apply` (and `--fixture`) require `--shards 1`, +//! so there is exactly one logger writing to stdout when these render; its +//! whole-line atomic `write_all`s never splice with the publisher's document +//! lines, which runtime-next flushes before each committing `persist`. + +use bytes::Bytes; +use std::io::Write as _; + +/// [`runtime_next::LoggerFactory`] producing preview-rendering loggers. The +/// `log_handler` sinks connector logs; `emit_state` / `emit_apply` gate the +/// `--output-state` / `--output-apply` lines. +#[derive(Clone)] +pub struct PreviewLoggerFactory { + log_handler: fn(&::ops::Log), + emit_state: bool, + emit_apply: bool, +} + +impl PreviewLoggerFactory { + pub fn new(log_handler: fn(&::ops::Log), emit_state: bool, emit_apply: bool) -> Self { + Self { + log_handler, + emit_state, + emit_apply, + } + } +} + +impl runtime_next::LoggerFactory for PreviewLoggerFactory { + type Logger = PreviewLogger; + + fn open(&self, _task_name: &str) -> PreviewLogger { + PreviewLogger { + log_handler: self.log_handler, + emit_state: self.emit_state, + emit_apply: self.emit_apply, + } + } +} + +/// Per-session preview logger. Cheap to clone (the connector log pump holds +/// its own handle); all fields are `Copy`. +#[derive(Clone)] +pub struct PreviewLogger { + log_handler: fn(&::ops::Log), + emit_state: bool, + emit_apply: bool, +} + +impl runtime_next::Logger for PreviewLogger { + fn log(&self, log: &::ops::Log) { + (self.log_handler)(log) + } + + fn event(&self, event: runtime_next::LogEvent<'_>) { + match event { + runtime_next::LogEvent::Applied { + action_description, .. + } if self.emit_apply => { + write_line(&applied_line(action_description)); + } + runtime_next::LogEvent::Persist { persist, .. } if self.emit_state => { + if let Some(line) = persist_state_line(persist) { + write_line(&line); + } + } + // Everything else — including Applied / Persist when their flag is + // off — flattens to a log of the task's log stream. + event => { + if let Some(log) = event.to_log() { + self.log(&log); + } + } + } + } +} + +/// Decide the `--output-state` stdout line for a `Persist`, or `None` to suppress +/// it. This reproduces legacy `flowctl preview`'s per-commit cadence: +/// +/// - A non-empty connector-state delta renders the `{"updated":…}` line. +/// - A *committing* transaction (`committed_frontier` set) whose delta is empty +/// renders legacy's `["connectorState",{}]` filler. Remote-authoritative +/// connectors (materialize postgres/sqlite, derive-sqlite, …) keep their +/// checkpoint in the endpoint, so their per-transaction delta is empty; the +/// filler keeps stdout at one line per commit for consumers that count commit +/// boundaries (e.g. the materialize benchmark `run.sh`). +/// - A non-committing (hint) persist — and every capture persist, which never +/// sets `committed_frontier` — with an empty delta renders nothing. The +/// blackbox capture harness ignores empty-`updated` lines regardless. +fn persist_state_line(persist: &runtime_next::proto::Persist) -> Option> { + if !persist.connector_patches_json.is_empty() { + Some(connector_state_line(&persist.connector_patches_json)) + } else if persist.committed_frontier.is_some() { + Some(connector_state_kv(&[], false)) + } else { + None + } +} + +/// Encode a `--output-state` line from runtime-next's tab-framed connector-state +/// patch payload: `["connectorState",{"updated":,"mergePatch":}]\n`. +/// +/// The wire form is a JSON array of merge patches; a leading `null` patch marks +/// a full state replacement. The common single-merge-patch case embeds the +/// connector's update document verbatim (byte-for-byte the legacy `flowctl +/// preview` serialization, which connector snapshots pin). A replacement or a +/// reduced multi-patch transaction is rendered as the reduced update document. +fn connector_state_line(connector_patches_json: &Bytes) -> Vec { + let patches = + runtime_next::patches::split_state_patches(connector_patches_json).unwrap_or_default(); + + match patches.as_slice() { + // Common case: a single merge patch is the connector's update document. + [single] if single.as_ref() != b"null" => connector_state_kv(single, true), + _ => { + // Replacement (leading `null`) or multiple reduced patches: apply the + // patches to an empty base to recover the effective update document. + let is_replace = patches + .first() + .map(|p| p.as_ref() == b"null") + .unwrap_or(false); + let reduced = + runtime_next::patches::apply_state_patches(&Bytes::new(), connector_patches_json) + .unwrap_or_else(|_| Bytes::from_static(b"{}")); + connector_state_kv(&reduced, !is_replace) + } + } +} + +/// Frame an `updated` document (verbatim bytes) into a `connectorState` line. +/// `` is the legacy `flow::ConnectorState` serialization — +/// `{"updated":,"mergePatch":true}` with default-valued fields omitted, so +/// an absent update encodes as `{}`. The update bytes are embedded verbatim. +fn connector_state_kv(updated_json: &[u8], merge_patch: bool) -> Vec { + let mut line = Vec::with_capacity(updated_json.len() + 64); + line.extend_from_slice(b"[\"connectorState\",{"); + if !updated_json.is_empty() { + line.extend_from_slice(b"\"updated\":"); + line.extend_from_slice(updated_json); + } + if merge_patch { + if !updated_json.is_empty() { + line.push(b','); + } + line.extend_from_slice(b"\"mergePatch\":true"); + } + line.extend_from_slice(b"}]\n"); + line +} + +/// Emit the run's final reduced connector state as the legacy `--output-state` +/// final line: `["connectorState",{"updated":}]` (`mergePatch:false`, +/// since this is the whole reduced document, not a patch). An empty / absent +/// final state renders as `["connectorState",{}]`. Called once at run end, after +/// the session loop closes the runtime's RocksDB and flowctl re-reads it. +pub fn emit_final_state(state_json: &[u8]) { + write_line(&connector_state_kv(state_json, false)); +} + +/// Encode a `--output-apply` line: +/// `["applied.actionDescription", ""]\n` — byte-for-byte the legacy +/// `flowctl preview` format, including the space after the comma and Rust +/// `{:?}` escaping of the description text. +fn applied_line(action_description: &str) -> Vec { + format!("[\"applied.actionDescription\", {action_description:?}]\n").into_bytes() +} + +/// Write a complete, newline-terminated output line to stdout as a single atomic +/// `write_all` under the stdout lock, so lines never splice together. +fn write_line(line: &[u8]) { + std::io::stdout().write_all(line).unwrap(); +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn preview_lines_match_legacy_serialization() { + // `connector_state_kv` must reproduce the legacy `flowctl preview` + // serialization byte-for-byte: serde of `flow::ConnectorState` with + // default fields omitted, framed as `["connectorState",]\n`. + for (updated, merge_patch) in [ + (br#"{"cursor":"abc"}"#.as_slice(), true), + (br#"{"cursor":"abc"}"#.as_slice(), false), + (b"".as_slice(), false), + (b"".as_slice(), true), + ] { + let state = proto_flow::flow::ConnectorState { + updated_json: bytes::Bytes::copy_from_slice(updated), + merge_patch, + }; + let legacy = format!( + "[\"connectorState\",{}]\n", + serde_json::to_string(&state).unwrap() + ); + assert_eq!( + String::from_utf8(connector_state_kv(updated, merge_patch)).unwrap(), + legacy, + ); + } + + // A single merge patch (the canonical wire form `[{patch}\t]`) embeds the + // connector's update document verbatim with `mergePatch:true`. + assert_eq!( + String::from_utf8(connector_state_line(&Bytes::from_static( + b"[{\"cursor\":\"abc\"}\t]" + ))) + .unwrap(), + "[\"connectorState\",{\"updated\":{\"cursor\":\"abc\"},\"mergePatch\":true}]\n", + ); + + // `applied_line` matches the legacy format, space after comma included. + assert_eq!( + String::from_utf8(applied_line("create table \"foo\"")).unwrap(), + "[\"applied.actionDescription\", \"create table \\\"foo\\\"\"]\n", + ); + } + + #[test] + fn persist_state_line_reproduces_legacy_per_commit_cadence() { + use runtime_next::proto; + + // Non-empty delta (single merge-patch wire form) -> the `{"updated":…}` line. + let persist = proto::Persist { + connector_patches_json: Bytes::from_static(b"[{\"cursor\":\"abc\"}\t]"), + ..Default::default() + }; + assert_eq!( + String::from_utf8(persist_state_line(&persist).unwrap()).unwrap(), + "[\"connectorState\",{\"updated\":{\"cursor\":\"abc\"},\"mergePatch\":true}]\n", + ); + + // Empty delta on a *committing* persist -> legacy per-commit `{}` filler. + // This is the remote-authoritative case (materialize-postgres/sqlite) that + // the benchmark relies on for one commit-boundary marker per transaction. + let persist = proto::Persist { + committed_frontier: Some(Default::default()), + ..Default::default() + }; + assert_eq!( + String::from_utf8(persist_state_line(&persist).unwrap()).unwrap(), + "[\"connectorState\",{}]\n", + ); + + // Empty delta on a non-committing (hint) persist -> suppressed. Capture + // persists never set `committed_frontier`, so they land here too. + assert!(persist_state_line(&proto::Persist::default()).is_none()); + } +} diff --git a/crates/flowctl/src/preview/mod.rs b/crates/flowctl/src/preview/mod.rs index fa458def29a..32e5ad7acad 100644 --- a/crates/flowctl/src/preview/mod.rs +++ b/crates/flowctl/src/preview/mod.rs @@ -1,9 +1,22 @@ +//! `flowctl preview` on top of the runtime-next + shuffle stack. +//! +//! Spawns one in-process tonic server hosting both `runtime_next::Service` +//! and `shuffle::Service` on a single ephemeral 127.0.0.1 port, then runs +//! N synthetic shards as tokio tasks each driving one long-lived SessionLoop. +//! Source documents come from real Gazette journal reads (authed via the +//! user's flowctl token); endpoint mutations go to the connector container +//! as in production. use crate::local_specs; use anyhow::Context; -use futures::TryStreamExt; -use proto_flow::{capture, derive, flow, materialize}; -mod journal_reader; +mod capture_driver; +mod derive_driver; +mod driver; +mod fixture; +mod logger; +mod publish; +mod services; +mod shards; #[derive(Debug, clap::Args)] #[clap(rename_all = "kebab-case")] @@ -15,12 +28,15 @@ pub struct Preview { /// Required if there are multiple tasks in --source specifications. #[clap(long)] name: Option, - /// Optional, artificial delay between transactions to simulate back-pressure - /// and encourage reductions. The default is no delay. - #[clap(long)] - delay: Option, - /// How long can the task produce no data before this command stops? - /// The default is that there is no timeout. + /// Number of synthetic shards to drive in parallel. Default 1. + /// For captures, N shards fan out as N independent connector instances. + /// Many connectors ignore the key range and will duplicate output; + /// use N=1 unless the connector partitions by key_begin/key_end. + #[clap(long, default_value = "1")] + shards: u32, + /// How long should preview run before gracefully stopping? + /// The default is no timeout. Note that the task may finish active + /// transaction activity even after this timeout is reached. #[clap(long)] timeout: Option, /// How many connector sessions should be run, and what is the target number @@ -41,33 +57,71 @@ pub struct Preview { /// The default is a single session with an unbounded number of transactions. #[clap(long, value_parser, value_delimiter = ',')] sessions: Option>, - /// Path to a transactions fixture to use, instead of reading live collections. - /// Fixtures are used only for derivations and materializations. - /// The fixture format is newline-delimited JSON where each line is either a - /// document ["collection/name", {...document...}] or a commit marker {"commit": true} - /// denoting a transaction boundary. + /// Path to a transactions fixture to feed in place of live collection data. + /// Newline-delimited JSON: documents `["collection/name", {...}]` separated + /// by `{"commit": true}` transaction markers. Fixtures are only for + /// derivations and materializations, and require `--shards 1`. + /// + /// A regular file is read eagerly and may be split across `--sessions`. + /// A named pipe (FIFO), or `-` for stdin, streams: transactions are fed + /// incrementally as the producing writer emits them, in a single unbounded + /// session which stops gracefully at stream EOF. #[clap(long)] fixture: Option, + /// Artificial delay between transactions, simulating back-pressure and + /// encouraging reductions. The delay raises the task's minimum transaction + /// duration, so each transaction batches at least `delay` of live input. + /// Cannot be combined with `--fixture` (whose transaction boundaries are + /// fixed by the fixture's own commit markers). + #[clap(long)] + delay: Option, /// Docker network to run connector images. #[clap(long, default_value = "bridge")] network: String, - /// Initial JSON connector state to use, which defaults to an empty object. + /// Initial JSON connector state to seed the run with. /// When developing a connector, you may want to use --initial-state to pass /// in crafted state configurations you expect the connector to resume from. - #[clap(long, default_value = "{}")] - initial_state: String, - - /// Output state updates + /// It seeds only the very first session of a run: once any connector state + /// is persisted, later sessions resume from it instead. + #[clap(long)] + initial_state: Option, + /// Output state updates. + /// Each committed connector state update is printed to stdout as a + /// `["connectorState",]` line. Requires `--shards 1`. #[clap(long, action)] output_state: bool, - - /// Output apply RPC description + /// Output apply RPC description. + /// Each connector Applied response is printed to stdout as a + /// `["applied.actionDescription",""]` line. Requires `--shards 1`. #[clap(long, action)] output_apply: bool, - - /// Output task logs in JSON format to stderr + /// Output task logs in JSON format to stderr. #[clap(long, action)] log_json: bool, + /// Loopback HTTP port hosting the service-kit admin dashboard + /// (handler inventory, per-handler trace overrides, /metrics). + #[clap(long)] + debug_port: Option, +} + +/// Harness controls threaded into each driver: the `--initial-state` seed (used +/// to pre-seed shard zero's RocksDB), the output-capturing publisher factory, +/// and the preview-rendering logger factory installed on the shard (and, for +/// materializations / derivations, the leader) Service. The logger carries the +/// `--output-state` / `--output-apply` behavior the legacy `flowctl preview` +/// flags expressed; the publisher captures captured / derived documents. +#[derive(Clone)] +struct Controls { + initial_state_json: bytes::Bytes, + publisher_factory: publish::PreviewPublisherFactory, + logger_factory: logger::PreviewLoggerFactory, +} + +/// Resolved task selected from the source specifications. +enum TaskSpec { + Capture(proto_flow::flow::CaptureSpec), + Materialization(proto_flow::flow::MaterializationSpec), + Derivation(proto_flow::flow::CollectionSpec), } impl Preview { @@ -75,358 +129,609 @@ impl Preview { let Self { source, name, - delay, + shards, timeout, sessions, fixture, + delay, network, initial_state, output_state, output_apply, log_json, + debug_port, } = self; - let source = build::arg_source_to_url(source, false)?; - - let log_handler: fn(&ops::Log) = if *log_json { - ops::stderr_log_handler - } else { - ops::tracing_log_handler - }; + let fixture = fixture.as_deref(); + let delay: Option = delay.map(|d| d.into()); - // TODO(johnny): validate only `name`, if presented. - let (_sources, _live, validations) = - local_specs::load_and_validate_full(ctx, source.as_str(), &network, log_handler) - .await?; + if fixture.is_some() && *shards != 1 { + anyhow::bail!("--fixture requires --shards 1"); + } + if (*output_state || *output_apply) && *shards != 1 { + anyhow::bail!("--output-state and --output-apply require --shards 1"); + } + if fixture.is_some() && delay.is_some() { + anyhow::bail!("--delay cannot be combined with --fixture"); + } - let runtime = runtime::Runtime::new( - runtime::Plane::Local, - network.clone(), - log_handler, - None, - "preview".to_string(), - ); + let source_url = build::arg_source_to_url(source, false)?; - // Default to no delay. - let delay = delay - .map(|i| i.clone().into()) - .unwrap_or(std::time::Duration::ZERO); - - // Default to no timeout. - let timeout = timeout - .map(|i| i.clone().into()) - .unwrap_or(std::time::Duration::MAX); - - // Negative sessions mean "unlimited transactions", and default to a single unlimited session. - let sessions = if let Some(sessions) = sessions { - sessions - .iter() - .map(|i| usize::try_from(*i).unwrap_or(usize::MAX)) - .collect() + let log_handler: fn(&::ops::Log) = if *log_json { + ::ops::stderr_log_handler } else { - vec![usize::MAX] + ::ops::tracing_log_handler }; - // Parse a provided data fixture. - let fixture_reader = - fixture.as_ref().map( - |fixture| runtime::harness::streaming_fixture::StreamingReader { - path: std::path::PathBuf::from(fixture), - }, - ); - let journal_reader = journal_reader::Reader::new(ctx, delay); - - let initial_state = - models::RawValue::from_str(initial_state).context("initial state is not valid JSON")?; - let state_dir = tempfile::tempdir().unwrap(); - - let num_tasks = validations.built_captures.len() - + validations.built_materializations.len() - + validations - .built_collections - .iter() - .filter(|c| { - c.spec - .as_ref() - .map(|s| s.derivation.is_some()) - .unwrap_or_default() - }) - .count(); - - if num_tasks == 0 { - anyhow::bail!( - "sourced specification files do not contain any tasks (captures, derivations, or materializations)" - ); - } else if num_tasks > 1 && name.is_none() { - anyhow::bail!( - "sourced specification files contain multiple tasks (captures, derivations, or materializations). Use --name to identify a specific task" - ); - } - - for row in validations.built_captures.iter() { - if !matches!(name, Some(n) if n == row.capture.as_str()) && name.is_some() { - continue; + // An explicit --initial-state value seeds shard zero's first session. + let initial_state_json = match initial_state { + None => bytes::Bytes::new(), + Some(initial_state) => { + let initial_state = models::RawValue::from_str(initial_state) + .context("initial state is not valid JSON")?; + bytes::Bytes::from(initial_state.get().to_string()) } - let Some(spec) = &row.spec else { - continue; - }; - - return preview_capture( - delay, - runtime, - sessions, - spec.clone(), - initial_state, - state_dir.path(), - timeout, + }; + let controls = Controls { + initial_state_json, + publisher_factory: publish::PreviewPublisherFactory, + logger_factory: logger::PreviewLoggerFactory::new( + log_handler, *output_state, *output_apply, - ) - .await; - } + ), + }; - for row in validations.built_collections.iter() { - if !matches!(name, Some(n) if n == row.collection.as_str()) && name.is_some() { - continue; - } - let Some(spec) = &row.spec else { - continue; - }; + let (_sources, _live, validations) = + local_specs::load_and_validate_full(ctx, source_url.as_str(), network, log_handler) + .await?; - if spec.derivation.is_none() && name.is_some() { - anyhow::bail!("{} is not a derivation", name.as_ref().unwrap()); - } else if spec.derivation.is_none() { - continue; - } + let task = resolve_task(&validations, name.as_deref())?; - if let Some(reader) = fixture_reader { - return preview_derivation( - reader, - runtime, - sessions, - spec.clone(), - initial_state, - state_dir.path(), - timeout, - *output_state, - ) - .await; - } else { - return preview_derivation( - journal_reader, - runtime, - sessions, - spec.clone(), - initial_state, - state_dir.path(), - timeout, - *output_state, + let timeout = timeout.map(|i| i.into()); + + let session_targets: Vec = if let Some(s) = sessions { + s.iter() + .map(|i| { + if *i < 0 { + Ok(0) + } else { + u32::try_from(*i).context("--sessions values must fit in uint32") + } + }) + .collect::>()? + } else { + vec![0] + }; + + let stop_token = tokio_util::sync::CancellationToken::new(); + + let result: anyhow::Result<()> = match task { + TaskSpec::Capture(mut spec) => { + anyhow::ensure!( + fixture.is_none(), + "--fixture is only supported for derivations and materializations", + ); + if let Some(delay) = delay { + set_min_txn_duration(spec.shard_template.as_mut(), delay); + } + let run = services::Run::start_capture( + network.clone(), + *shards, + *debug_port, + ctx.registry.clone(), ) - .await; + .await?; + let session_loop = capture_driver::run_sessions( + &run, + &spec, + session_targets, + controls.clone(), + stop_token.clone(), + ); + tokio::pin!(session_loop); + let result = run_with_timeout(session_loop, timeout, &stop_token).await; + finish_output_state(&run, *output_state, result).await } - } + TaskSpec::Materialization(mut spec) => { + let run = services::Run::start_with_shuffle_leader( + ctx, + network.clone(), + *shards, + *debug_port, + ctx.registry.clone(), + fixture.is_some(), + controls.publisher_factory.clone(), + controls.logger_factory.clone(), + ) + .await?; - for row in validations.built_materializations.iter() { - if !matches!(name, Some(n) if n == row.materialization.as_str()) && name.is_some() { - continue; + // Hold the fixture keepalive (writers/segments) for the life of + // the session loop so its segment files stay readable. + let (session_targets, fixture_dirs, session_stop, fixture_keepalive) = + prepare_sessions( + &run, + &mut spec, + |spec| spec.shard_template.as_mut(), + |spec| shuffle::proto::Task { + task: Some(shuffle::proto::task::Task::Materialization(spec.clone())), + }, + fixture, + delay, + session_targets, + &stop_token, + )?; + + let session_loop = driver::run_sessions( + &run, + &spec, + session_targets, + fixture_dirs, + controls.clone(), + session_stop, + ); + tokio::pin!(session_loop); + let result = run_with_timeout(session_loop, timeout, &stop_token).await; + let result = finish_fixtures(result, fixture_keepalive).await; + finish_output_state(&run, *output_state, result).await } - let Some(spec) = &row.spec else { - continue; - }; - - if let Some(reader) = fixture_reader { - return preview_materialization( - reader, - runtime, - sessions, - spec.clone(), - initial_state, - state_dir.path(), - timeout, - *output_state, - *output_apply, + TaskSpec::Derivation(mut spec) => { + let run = services::Run::start_with_shuffle_leader( + ctx, + network.clone(), + *shards, + *debug_port, + ctx.registry.clone(), + fixture.is_some(), + controls.publisher_factory.clone(), + controls.logger_factory.clone(), ) - .await; - } else { - return preview_materialization( - journal_reader, - runtime, - sessions, - spec.clone(), - initial_state, - state_dir.path(), - timeout, - *output_state, - *output_apply, - ) - .await; + .await?; + + let (session_targets, fixture_dirs, session_stop, fixture_keepalive) = + prepare_sessions( + &run, + &mut spec, + |spec| { + spec.derivation + .as_mut() + .and_then(|d| d.shard_template.as_mut()) + }, + |spec| shuffle::proto::Task { + task: Some(shuffle::proto::task::Task::Derivation(spec.clone())), + }, + fixture, + delay, + session_targets, + &stop_token, + )?; + + let session_loop = derive_driver::run_sessions( + &run, + &spec, + session_targets, + fixture_dirs, + controls.clone(), + session_stop, + ); + tokio::pin!(session_loop); + let result = run_with_timeout(session_loop, timeout, &stop_token).await; + let result = finish_fixtures(result, fixture_keepalive).await; + finish_output_state(&run, *output_state, result).await } + }; + + // `run` drops here, aborting the tonic server and removing the + // RocksDB / shuffle-log tempdirs. + result + } +} + +/// Fixture state held for the life of the session loop. Both variants keep +/// shuffle-log writers/segments alive while the consumer reads (and unlinks) +/// them; `Streaming` also carries the feeder task, joined after the run to +/// surface stream errors. +enum FixtureKeepalive { + Eager { + _plan: fixture::FixturePlan, + }, + Streaming { + // Dropping the guard cancels the feeder's hold token: it releases its + // writer/segments and exits with the stream's result. + _hold: tokio_util::sync::DropGuard, + feeder: tokio::task::JoinHandle>, + }, +} + +/// Prepare per-session inputs for a derivation or materialization preview. +/// +/// With `--fixture`: force 1:1 fixture-transaction-to-runtime-transaction +/// boundaries, then either materialize a regular file into per-session shuffle +/// segments and start the frontier feeder (eager), or — for a FIFO or stdin — +/// stream it through a feeder task driving a single unbounded session that's +/// stopped at EOF via the returned session stop token (a child of `stop_token`, +/// so Ctrl-C and `--timeout` behave identically). Without a fixture: apply +/// `--delay` (if any) as the task's minimum transaction duration, batching live +/// reads into fewer, larger transactions. +fn prepare_sessions( + run: &services::Run, + spec: &mut S, + shard_template: impl FnOnce(&mut S) -> Option<&mut proto_gazette::consumer::ShardSpec>, + build_task: impl FnOnce(&S) -> shuffle::proto::Task, + fixture: Option<&str>, + delay: Option, + session_targets: Vec, + stop_token: &tokio_util::sync::CancellationToken, +) -> anyhow::Result<( + Vec, + Vec, + tokio_util::sync::CancellationToken, + Option, +)> { + let Some(path) = fixture else { + if let Some(delay) = delay { + set_min_txn_duration(shard_template(spec), delay); } + return Ok((session_targets, Vec::new(), stop_token.clone(), None)); + }; + + force_single_transaction(shard_template(spec)); + let task = build_task(spec); + + if is_streaming_fixture(path)? { + anyhow::ensure!( + session_targets == [0], + "a streaming --fixture (FIFO or stdin) runs exactly one unbounded session; omit --sessions or pass `--sessions -1`", + ); + let frontier_tx = run + .frontier_tx + .clone() + .expect("fixture run was started with a frontier sender"); + + let session_stop = stop_token.child_token(); + let hold = tokio_util::sync::CancellationToken::new(); + let source = (path != "-").then(|| std::path::PathBuf::from(path)); - anyhow::bail!("could not find task {}", name.as_ref().unwrap()); + let (dir, feeder) = fixture::start_streaming( + &task, + source, + std::path::Path::new(&run.shuffle_log_dir), + frontier_tx, + session_stop.clone(), + hold.clone(), + )?; + return Ok(( + vec![0], + vec![dir], + session_stop, + Some(FixtureKeepalive::Streaming { + _hold: hold.drop_guard(), + feeder, + }), + )); } + + let (targets, dirs, plan) = start_fixtures(run, task, path, session_targets)?; + Ok(( + targets, + dirs, + stop_token.clone(), + Some(FixtureKeepalive::Eager { _plan: plan }), + )) } -async fn preview_capture( - delay: std::time::Duration, - runtime: runtime::Runtime, - sessions: Vec, - spec: flow::CaptureSpec, - state: models::RawValue, - state_dir: &std::path::Path, - timeout: std::time::Duration, +/// A streaming fixture is stdin (`-`) or a named pipe: its transactions are fed +/// incrementally as they're produced, rather than eagerly pre-planned. +fn is_streaming_fixture(path: &str) -> anyhow::Result { + if path == "-" { + return Ok(true); + } + let meta = + std::fs::metadata(path).with_context(|| format!("inspecting fixture path {path:?}"))?; + + #[cfg(unix)] + { + use std::os::unix::fs::FileTypeExt; + Ok(meta.file_type().is_fifo()) + } + #[cfg(not(unix))] + { + let _ = meta; + Ok(false) + } +} + +/// Conclude fixture feeding once the session loop has ended: release the +/// streaming feeder's segment keepalive and join it, surfacing a stream error +/// (e.g. a malformed fixture line) that otherwise manifests only as a +/// gracefully-stopped run. +async fn finish_fixtures( + result: anyhow::Result<()>, + keepalive: Option, +) -> anyhow::Result<()> { + let Some(FixtureKeepalive::Streaming { _hold, feeder }) = keepalive else { + return result; + }; + drop(_hold); // Cancels the feeder's hold token. + let feeder_result = feeder + .await + .unwrap_or_else(|panic| Err(anyhow::anyhow!("fixture feeder panic: {panic}"))); + result.and(feeder_result) +} + +/// On a successful `--output-state` run, emit the final reduced connector state. +/// A successful session result is replaced by a final-state read error; a failed +/// session result passes through unchanged (skip the final-state read). +async fn finish_output_state( + run: &services::Run, output_state: bool, - output_apply: bool, + result: anyhow::Result<()>, ) -> anyhow::Result<()> { - let responses_rx = runtime::harness::run_capture( - delay, - runtime, - sessions, - &spec, - state, - state_dir, - timeout, - output_apply, - ); - tokio::pin!(responses_rx); - - while let Some(response) = responses_rx.try_next().await? { - let internal = response - .get_internal() - .context("failed to decode internal runtime.CaptureResponseExt")?; - - if let Some(capture::response::Captured { binding, doc_json }) = response.captured { - let proto_flow::runtime::capture_response_ext::Captured { - key_packed, - partitions_packed, - } = internal.captured.unwrap_or_default(); - - tracing::trace!(?key_packed, ?partitions_packed, "captured"); - - let collection = &spec.bindings[binding as usize] - .collection - .as_ref() - .unwrap() - .name; + if !output_state || result.is_err() { + return result; + } + emit_final_connector_state(run).await?; + result +} - print!("[{collection:?},{}]\n", str::from_utf8(&doc_json).unwrap()); - } else if let Some(capture::response::Checkpoint { state }) = response.checkpoint { - let proto_flow::runtime::capture_response_ext::Checkpoint { stats, .. } = - internal.checkpoint.unwrap_or_default(); +/// Re-open shard zero's RocksDB and emit its final reduced connector state as a +/// `--output-state` line. Safe to open directly: the runtime's shard serve loop +/// drops its `RocksDB` handle (releasing the exclusive lock) when its request +/// stream ends, which is strictly before its response stream reaches EOF — and +/// the session loop only returns once the driver has drained that EOF. +async fn emit_final_connector_state(run: &services::Run) -> anyhow::Result<()> { + let state = runtime_next::read_final_connector_state(proto_flow::runtime::RocksDbDescriptor { + rocksdb_path: run.rocksdb_path.clone(), + rocksdb_env_memptr: 0, + }) + .await + .context("reading final connector state for --output-state")?; + + logger::emit_final_state(&state); + Ok(()) +} - let collection = "connectorState"; - let state_json = state - .as_ref() - .map(|s| serde_json::to_string(s)) - .transpose()? - .unwrap_or("{}".to_string()); +/// Apply `--delay` to a live preview by raising the task's minimum transaction +/// duration: the leader holds each transaction open for at least `delay`, +/// batching source output into fewer, larger transactions. This is the +/// runtime-next analog of legacy preview's sleep between transaction polls. +fn set_min_txn_duration( + shard_template: Option<&mut proto_gazette::consumer::ShardSpec>, + delay: std::time::Duration, +) { + let Some(shard_template) = shard_template else { + return; + }; + let min = pbjson_types::Duration { + seconds: delay.as_secs() as i64, + nanos: delay.subsec_nanos() as i32, + }; + // Keep the close-policy band well-formed if the template's configured + // maximum is below the requested minimum. + if shard_template + .max_txn_duration + .as_ref() + .map_or(true, |max| { + (max.seconds, max.nanos) < (min.seconds, min.nanos) + }) + { + shard_template.max_txn_duration = Some(min.clone()); + } + shard_template.min_txn_duration = Some(min); +} - if output_state { - print!("[{collection:?},{state_json}]\n"); +/// Force one-transaction-per-checkpoint in the leader by collapsing the task's +/// transaction-duration window, so each fixture transaction commits as exactly +/// one runtime transaction (legacy fixture preview's 1:1 boundaries). +/// +/// A literal `max_txn_duration` of zero would deadlock the leader: `HeadIdle` +/// gates the first checkpoint load on `open_age < max_txn_duration`, and a fresh +/// transaction's `open_age` is zero. The smallest positive duration loads one +/// checkpoint, after which the Load round's IO advances the clock past the bound +/// and the transaction closes. Applied only to fixture preview. +fn force_single_transaction(shard_template: Option<&mut proto_gazette::consumer::ShardSpec>) { + if let Some(shard_template) = shard_template { + shard_template.min_txn_duration = Some(pbjson_types::Duration { + seconds: 0, + nanos: 0, + }); + shard_template.max_txn_duration = Some(pbjson_types::Duration { + seconds: 0, + nanos: 1, + }); + } +} + +/// Materialize a fixture into per-session shuffle log segments and spawn the +/// task that feeds its checkpoint frontiers to the fixture CheckpointOpener. +/// Returns fixture-bounded session targets, the per-session shuffle directories +/// (for the drivers' `Join`s), and the plan to keep alive for the run. +fn start_fixtures( + run: &services::Run, + task: shuffle::proto::Task, + fixture_path: &str, + requested_targets: Vec, +) -> anyhow::Result<(Vec, Vec, fixture::FixturePlan)> { + let mut plan = fixture::build( + &task, + std::path::Path::new(fixture_path), + std::path::Path::new(&run.shuffle_log_dir), + &requested_targets, + )?; + let session_targets = plan.session_targets.clone(); + let session_dirs = plan.session_dirs.clone(); + let session_frontiers = std::mem::take(&mut plan.session_frontiers); + + let frontier_tx = run + .frontier_tx + .clone() + .expect("fixture run was started with a frontier sender"); + + // Feed each session its frontiers, then a Boundary marker. The marker + // bounds a session's consumption so a stopping leader's speculative + // checkpoint consumes the marker rather than stealing the next session's + // first frontier. + tokio::spawn(async move { + for frontiers in session_frontiers { + for frontier in frontiers { + if frontier_tx + .send(fixture::FixtureItem::Frontier(frontier)) + .is_err() + { + return; // The consumer went away. + } + } + if frontier_tx + .send(fixture::FixtureItem::Boundary { reached: None }) + .is_err() + { + return; } - tracing::debug!(stats=?ops::DebugJson(stats), state=?ops::DebugJson(state), "checkpoint"); } - } + // Dropping `frontier_tx` here signals end-of-fixtures to the replay + // Session (relevant only once `run.frontier_tx` is also dropped). + }); - Ok(()) + Ok((session_targets, session_dirs, plan)) } -async fn preview_derivation( - reader: impl runtime::harness::Reader, - runtime: runtime::Runtime, - sessions: Vec, - spec: flow::CollectionSpec, - state: models::RawValue, - state_dir: &std::path::Path, - timeout: std::time::Duration, - output_state: bool, -) -> anyhow::Result<()> { - let responses_rx = - runtime::harness::run_derive(reader, runtime, sessions, &spec, state, state_dir, timeout); - tokio::pin!(responses_rx); - - while let Some(response) = responses_rx.try_next().await? { - let internal = response - .get_internal() - .context("failed to decode internal runtime.DeriveResponseExt")?; - - if let Some(derive::response::Published { doc_json }) = response.published { - let proto_flow::runtime::derive_response_ext::Published { - max_clock, - key_packed, - partitions_packed, - } = internal.published.unwrap_or_default(); - - tracing::trace!(?max_clock, ?key_packed, ?partitions_packed, "published"); - - print!("{}\n", str::from_utf8(&doc_json).unwrap()); - } else if let Some(derive::response::Flushed { .. }) = response.flushed { - let proto_flow::runtime::derive_response_ext::Flushed { stats } = - internal.flushed.unwrap_or_default(); - tracing::debug!(stats=?ops::DebugJson(stats), "flushed"); - } else if let Some(derive::response::StartedCommit { state }) = response.started_commit { - let collection = "connectorState"; - let state_json = state +async fn run_with_timeout( + mut session_loop: std::pin::Pin<&mut F>, + timeout: Option, + stop_token: &tokio_util::sync::CancellationToken, +) -> anyhow::Result<()> +where + F: std::future::Future>, +{ + let timeout = timeout.unwrap_or_else(|| std::time::Duration::MAX); + + tokio::select! { + result = &mut session_loop => result, + _ = tokio::signal::ctrl_c() => { + tracing::info!("Ctrl-C received; stopping active session"); + stop_token.cancel(); + session_loop.await + } + _ = tokio::time::sleep(timeout) => { + tracing::info!(?timeout, "preview --timeout reached; stopping active session"); + stop_token.cancel(); + session_loop.await + } + } +} + +fn resolve_task(validations: &tables::Validations, name: Option<&str>) -> anyhow::Result { + let derivations_count = validations + .built_collections + .iter() + .filter(|c| { + c.spec .as_ref() - .map(|s| serde_json::to_string(s)) - .transpose()? - .unwrap_or("{}".to_string()); - if output_state { - print!("[{collection:?},{state_json}]\n"); + .map(|s| s.derivation.is_some()) + .unwrap_or_default() + }) + .count(); + let num_tasks = validations.built_captures.len() + + validations.built_materializations.len() + + derivations_count; + + if num_tasks == 0 { + anyhow::bail!( + "sourced specification files do not contain any tasks (captures, derivations, or materializations)", + ); + } + if num_tasks > 1 && name.is_none() { + anyhow::bail!( + "sourced specification files contain multiple tasks; use --name to identify the task", + ); + } + + for row in validations.built_captures.iter() { + if let Some(target) = name { + if row.capture.as_str() != target { + continue; } - tracing::debug!(state=?ops::DebugJson(state), "started commit"); } + let Some(spec) = &row.spec else { continue }; + return Ok(TaskSpec::Capture(spec.clone())); } - Ok(()) + for row in validations.built_materializations.iter() { + if let Some(target) = name { + if row.materialization.as_str() != target { + continue; + } + } + let Some(spec) = &row.spec else { continue }; + return Ok(TaskSpec::Materialization(spec.clone())); + } + + for row in validations.built_collections.iter() { + if let Some(target) = name { + if row.collection.as_str() != target { + continue; + } + } + let Some(spec) = &row.spec else { continue }; + if spec.derivation.is_some() { + return Ok(TaskSpec::Derivation(spec.clone())); + } + } + + if let Some(target) = name { + anyhow::bail!("could not find capture, materialization, or derivation {target}"); + } + anyhow::bail!("no capture, materialization, or derivation found in source"); } -async fn preview_materialization( - reader: impl runtime::harness::Reader, - runtime: runtime::Runtime, - sessions: Vec, - spec: flow::MaterializationSpec, - state: models::RawValue, - state_dir: &std::path::Path, - timeout: std::time::Duration, - output_state: bool, - output_apply: bool, -) -> anyhow::Result<()> { - let responses_rx = runtime::harness::run_materialize( - reader, - runtime, - sessions, - &spec, - state, - state_dir, - timeout, - output_apply, - ); - tokio::pin!(responses_rx); - - while let Some(response) = responses_rx.try_next().await? { - let internal = response - .get_internal() - .context("failed to decode internal runtime.MaterializeResponseExt")?; - - if let Some(materialize::response::Flushed { .. }) = response.flushed { - let proto_flow::runtime::materialize_response_ext::Flushed { stats } = - internal.flushed.unwrap_or_default(); - tracing::debug!(stats=?ops::DebugJson(stats), "flushed"); - } else if let Some(materialize::response::StartedCommit { state }) = response.started_commit +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_is_streaming_fixture() { + assert!(is_streaming_fixture("-").unwrap()); + + let file = tempfile::NamedTempFile::new().unwrap(); + assert!(!is_streaming_fixture(file.path().to_str().unwrap()).unwrap()); + + assert!(is_streaming_fixture("/does/not/exist").is_err()); + + #[cfg(unix)] { - let collection = "connectorState"; - let state_json = state - .as_ref() - .map(|s| serde_json::to_string(s)) - .transpose()? - .unwrap_or("{}".to_string()); - if output_state { - print!("[{collection:?},{state_json}]\n"); - } - tracing::debug!(state=?ops::DebugJson(state), "started commit"); + let dir = tempfile::tempdir().unwrap(); + let fifo = dir.path().join("fifo"); + assert!( + std::process::Command::new("mkfifo") + .arg(&fifo) + .status() + .unwrap() + .success() + ); + assert!(is_streaming_fixture(fifo.to_str().unwrap()).unwrap()); } } - Ok(()) + #[test] + fn test_set_min_txn_duration() { + let dur = |seconds, nanos| pbjson_types::Duration { seconds, nanos }; + + // An unset maximum is raised alongside the minimum. + let mut template = proto_gazette::consumer::ShardSpec::default(); + set_min_txn_duration(Some(&mut template), std::time::Duration::from_secs(10)); + assert_eq!(template.min_txn_duration, Some(dur(10, 0))); + assert_eq!(template.max_txn_duration, Some(dur(10, 0))); + + // A maximum above the delay is left alone. + template.max_txn_duration = Some(dur(30, 0)); + set_min_txn_duration(Some(&mut template), std::time::Duration::from_secs(10)); + assert_eq!(template.min_txn_duration, Some(dur(10, 0))); + assert_eq!(template.max_txn_duration, Some(dur(30, 0))); + + // A maximum below the delay is raised to keep the band well-formed. + template.max_txn_duration = Some(dur(5, 0)); + set_min_txn_duration(Some(&mut template), std::time::Duration::from_secs(10)); + assert_eq!(template.min_txn_duration, Some(dur(10, 0))); + assert_eq!(template.max_txn_duration, Some(dur(10, 0))); + } } diff --git a/crates/flowctl/src/preview/publish.rs b/crates/flowctl/src/preview/publish.rs new file mode 100644 index 00000000000..9b0c5000114 --- /dev/null +++ b/crates/flowctl/src/preview/publish.rs @@ -0,0 +1,144 @@ +//! Output-capturing publisher for `flowctl preview`. +//! +//! Installs into runtime-next through its [`runtime_next::PublisherFactory`] +//! seam: instead of Gazette journal IO, captured / derived documents are +//! written to stdout as NDJSON (`["",]` lines). runtime-next is +//! unaware this is "preview"; it simply publishes through the factory, and the +//! monomorphized factory decides what that means. +//! +//! This is the document half of preview's output. The observation half — +//! connector-state updates (`--output-state`) and Apply actions +//! (`--output-apply`) — flows through the separate +//! [`Logger`](runtime_next::Logger) seam in [`super::logger`]. + +use bytes::Bytes; +use std::collections::BTreeMap; +use std::io::Write as _; + +/// Flush [`PreviewPublisher`]'s `line_buf` to stdout once it reaches this many +/// bytes. Sized to amortize the stdout lock + `write(2)` across many documents +/// while bounding buffered memory. +const FLUSH_THRESHOLD: usize = 32 * 1024; + +/// [`runtime_next::PublisherFactory`] that captures output to stdout instead of +/// publishing to journals. Stateless: captured / derived documents always print, +/// so there's nothing to configure (the `--output-state` / `--output-apply` +/// gating lives on the preview [`Logger`](super::logger::PreviewLogger)). +#[derive(Clone)] +pub struct PreviewPublisherFactory; + +impl runtime_next::PublisherFactory for PreviewPublisherFactory { + type Publisher = PreviewPublisher; + + fn open( + &self, + _authz_subject: String, + _producer: proto_gazette::uuid::Producer, + _stats_journal: &str, + collection_specs: &[&proto_flow::flow::CollectionSpec], + ) -> anyhow::Result { + Ok(PreviewPublisher { + collection_names: collection_specs.iter().map(|s| s.name.clone()).collect(), + line_buf: Vec::new(), + }) + } +} + +/// [`runtime_next::Publisher`] that performs no journal IO: captured / derived +/// documents are buffered as stdout lines. +pub struct PreviewPublisher { + /// Collection names indexed by binding, for the `["",]` framing. + /// Empty for a leader's stats-only publisher (it publishes no documents). + collection_names: Vec, + /// Accumulates complete lines, flushed to stdout as a single atomic + /// `write_all` once it crosses [`FLUSH_THRESHOLD`] or on `flush`. One + /// publisher exists per shard, all writing the process-global stdout; + /// flushing whole lines under the stdout lock keeps shards' output from + /// splicing together. + line_buf: Vec, +} + +impl runtime_next::Publisher for PreviewPublisher { + fn update_clock(&mut self) { + // No journal IO: there are no document UUIDs to stamp. + } + + async fn publish_stats(&mut self, stats: ops::proto::Stats) -> tonic::Result<()> { + tracing::info!(stats = ?ops::DebugJson(stats), "transaction stats"); + Ok(()) + } + + async fn publish_doc( + &mut self, + binding_index: usize, + doc: doc::OwnedNode, + _uuid_ptr: &json::Pointer, + ) -> tonic::Result { + let collection_name = &self.collection_names[binding_index]; + write!(self.line_buf, "[{collection_name:?},").unwrap(); + + // Serialize the body directly into the line buffer, sampling its length + // to report body bytes (excluding framing). Serializing a valid + // OwnedNode cannot fail, so the body can never be left partially written + // ahead of a flush. + let body_start = self.line_buf.len(); + serde_json::to_writer(&mut self.line_buf, &doc::SerPolicy::noop().on_owned(&doc)).unwrap(); + let body_len = self.line_buf.len() - body_start; + + self.line_buf.extend_from_slice(b"]\n"); + self.maybe_flush(); + Ok(body_len) + } + + async fn flush(&mut self) -> tonic::Result<()> { + if !self.line_buf.is_empty() { + std::io::stdout().write_all(&self.line_buf).unwrap(); + self.line_buf.clear(); + } + Ok(()) + } + + fn commit_intents( + &mut self, + ) -> Option<( + proto_gazette::uuid::Producer, + proto_gazette::uuid::Clock, + Vec, + )> { + // No real publishes happened, so there are no commit positions. + None + } + + async fn write_intents( + &mut self, + journal_intents: BTreeMap, + ) -> tonic::Result<()> { + debug_assert!( + journal_intents.is_empty(), + "PreviewPublisher received non-empty ACK intents", + ); + Ok(()) + } + + fn take_throttle_samples(&mut self) -> Vec> { + // No journal IO happens in preview, so there is no append back-pressure + // to sample and no auto-splitting to drive. + Vec::new() + } + + fn split_partition( + &self, + _journal: &str, + ) -> Option>> { + None + } +} + +impl PreviewPublisher { + fn maybe_flush(&mut self) { + if self.line_buf.len() >= FLUSH_THRESHOLD { + std::io::stdout().write_all(&self.line_buf).unwrap(); + self.line_buf.clear(); + } + } +} diff --git a/crates/flowctl/src/raw/preview_next/services.rs b/crates/flowctl/src/preview/services.rs similarity index 50% rename from crates/flowctl/src/raw/preview_next/services.rs rename to crates/flowctl/src/preview/services.rs index 0469ee8d61f..a48ffa2513a 100644 --- a/crates/flowctl/src/raw/preview_next/services.rs +++ b/crates/flowctl/src/preview/services.rs @@ -1,15 +1,19 @@ //! Run-scoped state for `flowctl preview`. //! -//! For materializations: one tonic server hosting both `runtime_next::Service` -//! and `shuffle::Service` on a single ephemeral 127.0.0.1 port, plus tempdirs -//! for shard-zero RocksDB and shuffle log segments. The preview driver keeps -//! SessionLoop streams open across `--sessions` iterations so RocksDB is reused -//! without closing. +//! For materializations: one tonic server hosting `runtime_next::Service` (plus +//! `shuffle::Service`, for live previews that read journals) on a single +//! ephemeral 127.0.0.1 port, plus tempdirs for shard-zero RocksDB and shuffle +//! log segments. A fixture preview instead hands the runtime a fixture +//! [`ShuffleSessionFactory`](runtime_next::ShuffleSessionFactory) and hosts no +//! shuffle service. The preview driver keeps SessionLoop streams open across +//! `--sessions` iterations so RocksDB is reused without closing. //! //! For captures: just a RocksDB tempdir and (optionally) the admin surface. //! Captures are leaderless and don't read journals, so the in-process Leader / //! Shuffle services and the journal-reading auth token are not constructed. use anyhow::Context; +use runtime_next::{ShuffleSession, ShuffleSessionFactory}; +use tokio::sync::mpsc; use tokio_stream::wrappers::TcpListenerStream; /// Run-scoped resources for one `flowctl preview` invocation. Field order @@ -24,13 +28,18 @@ pub struct Run { /// Empty for capture; the materialize driver dials this peer for Leader and /// Shuffle RPCs. pub peer_endpoint: String, - pub log_handler: fn(&::ops::Log), pub network: String, pub rocksdb_path: String, /// Empty for capture. pub shuffle_log_dir: String, pub n_shards: u32, pub registry: service_kit::Registry, + /// `Some` only for a fixture preview: the channel into the fixture + /// [`ShuffleSessionFactory`](runtime_next::ShuffleSessionFactory). flowctl + /// pushes one synthetic checkpoint Frontier per fixture transaction, then a + /// `Boundary` per session boundary; dropping the sender signals + /// end-of-fixtures. + pub frontier_tx: Option>, // Triggered on `Run::drop` (via the channel closing) to stop the admin // surface gracefully alongside the tonic server. _shutdown_tx: tokio::sync::broadcast::Sender<()>, @@ -42,14 +51,11 @@ impl Run { /// in-process Leader / Shuffle services and a logged-in token are not /// required. pub async fn start_capture( - log_json: bool, network: String, n_shards: u32, debug_port: Option, registry: service_kit::Registry, ) -> anyhow::Result { - let log_handler = pick_log_handler(log_json); - // `rocksdb_path` is the inspectable shard-0 tempdir; shards >=1 each // get their own auto-managed tempdir via `RocksDB::open(None)`. let _rocksdb_tmp = tempfile::tempdir().context("creating preview RocksDB tempdir")?; @@ -71,71 +77,93 @@ impl Run { _rocksdb_tmp, _shuffle_log_tmp: None, peer_endpoint: String::new(), - log_handler, network, rocksdb_path, shuffle_log_dir: String::new(), n_shards, registry, + frontier_tx: None, _shutdown_tx: shutdown_tx, }) } /// Resources for previewing a materialization or derivation: the capture - /// set plus an ephemeral tonic server hosting `runtime_next::Service` + - /// `shuffle::Service` and the shuffle log tempdir. Requires a logged-in - /// flowctl token to read source-journal documents. Both task types drive - /// the same in-process Leader + Shuffle stack. + /// set plus an ephemeral tonic server hosting `runtime_next::Service` and the + /// shuffle log tempdir. A live preview additionally hosts a loopback + /// `shuffle::Service` and requires a logged-in flowctl token to read + /// source-journal documents; a fixture preview hands the runtime a fixture + /// [`ShuffleSessionFactory`](runtime_next::ShuffleSessionFactory) and reads + /// no journals. pub async fn start_with_shuffle_leader( ctx: &mut crate::CliContext, network: String, - log_json: bool, n_shards: u32, debug_port: Option, registry: service_kit::Registry, + fixture: bool, + publisher_factory: super::publish::PreviewPublisherFactory, + logger_factory: super::logger::PreviewLoggerFactory, ) -> anyhow::Result { - let log_handler = pick_log_handler(log_json); - - anyhow::ensure!( - ctx.access_token().is_some(), - "you must be logged in to preview. Try `flowctl auth login`" - ); - - // Share the live, auto-refreshing user-token watch so a long-lived - // preview re-mints collection authorizations with a currently-valid - // access token and survives rotation of both token layers. - let user_tokens = ctx.user_tokens.clone(); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .context("binding ephemeral preview listener")?; let local_addr = listener.local_addr()?; let peer_endpoint = format!("http://{local_addr}"); - let factory = flow_client_next::workflows::user_collection_auth::new_journal_client_factory( - ctx.rest.clone(), - models::Capability::Read, - ctx.router.clone(), - user_tokens, - ); - let shuffle_svc = - shuffle::Service::new_loopback(peer_endpoint.clone(), factory, registry.clone()); - - let publisher_factory: gazette::journal::ClientFactory = std::sync::Arc::new({ - move |_authz_sub: String, _authz_obj: String| -> gazette::journal::Client { - unreachable!("live Publisher is not used by preview ({_authz_sub}, {_authz_obj})") - } - }); + // A fixture preview reads no journals (flowctl feeds synthetic frontiers + // and writes the segments itself), so it constructs no shuffle Service — + // just a fixture shuffle factory — and needs neither a logged-in token + // nor a journal client factory. A live preview authenticates and reads + // source collections from real journals via a loopback shuffle Service. + let (shuffle_factory, shuffle_svc, frontier_tx): ( + PreviewShuffleFactory, + Option, + Option>, + ) = if fixture { + let (opener, tx) = super::fixture::fixture_opener(); + (PreviewShuffleFactory::Fixture(opener), None, Some(tx)) + } else { + anyhow::ensure!( + ctx.access_token().is_some(), + "you must be logged in to preview. Try `flowctl auth login`" + ); + + // Share the live, auto-refreshing user-token watch so a long-lived + // preview re-mints collection authorizations with a currently-valid + // access token and survives rotation of both token layers. + let user_tokens = ctx.user_tokens.clone(); + let factory = + flow_client_next::workflows::user_collection_auth::new_journal_client_factory( + ctx.rest.clone(), + models::Capability::Read, + ctx.router.clone(), + user_tokens, + ); + let svc = + shuffle::Service::new_loopback(peer_endpoint.clone(), factory, registry.clone()); + ( + PreviewShuffleFactory::Live(runtime_next::ShuffleServiceFactory::new(svc.clone())), + Some(svc), + None, + ) + }; + let runtime_svc = runtime_next::Service::new( - shuffle_svc.clone(), + shuffle_factory, publisher_factory, + logger_factory, registry.clone(), true, // Disarm AuthN+AuthZ (local loopback). ); - let router = tonic::transport::Server::builder() - .add_service(runtime_svc.into_tonic_service()) - .add_service(shuffle_svc.into_tonic_service()); + // Only a live preview serves peer Slice/Log RPCs; a fixture preview reads + // its pre-written segments from disk and never dials shuffle. + let router = + tonic::transport::Server::builder().add_service(runtime_svc.into_tonic_service()); + let router = match shuffle_svc { + Some(svc) => router.add_service(svc.into_tonic_service()), + None => router, + }; // `_shutdown_tx` lives on `Run` so the channel closes on drop, which // resolves the admin surface's graceful-shutdown future. @@ -167,22 +195,70 @@ impl Run { _rocksdb_tmp, _shuffle_log_tmp: Some(_shuffle_log_tmp), peer_endpoint, - log_handler, network, rocksdb_path, shuffle_log_dir, n_shards, registry, + frontier_tx, _shutdown_tx: shutdown_tx, }) } } -fn pick_log_handler(log_json: bool) -> fn(&::ops::Log) { - if log_json { - ::ops::stderr_log_handler - } else { - ::ops::tracing_log_handler +/// Shuffle-session factory for a preview leader. The new +/// [`ShuffleSessionFactory`] seam is monomorphized (`open` / `recv_checkpoint` +/// are `-> impl Future` and `close` takes `self`, so it is not object-safe); +/// this enum lets one leader `Service` host either source — a fixture replay +/// (`--fixture`) or a live in-process journal-reading shuffle Session — chosen +/// per run. +pub(crate) enum PreviewShuffleFactory { + Fixture(super::fixture::FixtureOpener), + Live(runtime_next::ShuffleServiceFactory), +} + +impl ShuffleSessionFactory for PreviewShuffleFactory { + type Session = PreviewShuffleSession; + + async fn open( + &self, + task: shuffle::proto::Task, + shards: Vec, + resume: shuffle::Frontier, + ) -> anyhow::Result { + Ok(match self { + Self::Fixture(f) => PreviewShuffleSession::Fixture(f.open(task, shards, resume).await?), + Self::Live(f) => PreviewShuffleSession::Live(f.open(task, shards, resume).await?), + }) + } +} + +/// Per-session shuffle source opened by [`PreviewShuffleFactory`]. +pub(crate) enum PreviewShuffleSession { + Fixture(super::fixture::FixtureCheckpoints), + Live(shuffle::SessionClient), +} + +impl ShuffleSession for PreviewShuffleSession { + fn request_checkpoint(&self) { + match self { + Self::Fixture(s) => s.request_checkpoint(), + Self::Live(s) => s.request_checkpoint(), + } + } + + async fn recv_checkpoint(&mut self) -> anyhow::Result { + match self { + Self::Fixture(s) => s.recv_checkpoint().await, + Self::Live(s) => s.recv_checkpoint().await, + } + } + + async fn close(self) -> anyhow::Result<()> { + match self { + Self::Fixture(s) => s.close().await, + Self::Live(s) => s.close().await, + } } } diff --git a/crates/flowctl/src/raw/preview_next/shards.rs b/crates/flowctl/src/preview/shards.rs similarity index 100% rename from crates/flowctl/src/raw/preview_next/shards.rs rename to crates/flowctl/src/preview/shards.rs diff --git a/crates/flowctl/src/raw/mod.rs b/crates/flowctl/src/raw/mod.rs index 3f5321e4186..a4b6b5e6f97 100644 --- a/crates/flowctl/src/raw/mod.rs +++ b/crates/flowctl/src/raw/mod.rs @@ -15,7 +15,6 @@ mod alerts; mod discover; mod materialize_fixture; mod oauth; -mod preview_next; mod shards; mod spec; mod split_shards; @@ -78,9 +77,6 @@ pub enum Command { /// Print environment variables for working with a given data-plane /// and prefix using Gazette's `gazctl`. GazctlEnv(GazctlEnv), - /// Locally run and preview a capture, derivation, or materialization using - /// the V2 runtime. - PreviewNext(preview_next::Preview), } #[derive(Debug, clap::Args)] @@ -233,7 +229,6 @@ impl Advanced { Command::ListShards(selector) => shards::do_list_shards(ctx, selector).await, Command::SplitShards(split) => split_shards::do_split(ctx, split).await, Command::GazctlEnv(gazctl_env) => gazctl_env.run(ctx).await, - Command::PreviewNext(preview) => preview.run(ctx).await, } } } diff --git a/crates/flowctl/src/raw/preview_next/mod.rs b/crates/flowctl/src/raw/preview_next/mod.rs deleted file mode 100644 index b1d89ad76c3..00000000000 --- a/crates/flowctl/src/raw/preview_next/mod.rs +++ /dev/null @@ -1,261 +0,0 @@ -//! `flowctl preview` on top of the runtime-next + shuffle stack. -//! -//! Spawns one in-process tonic server hosting both `runtime_next::Service` -//! and `shuffle::Service` on a single ephemeral 127.0.0.1 port, then runs -//! N synthetic shards as tokio tasks each driving one long-lived SessionLoop. -//! Source documents come from real Gazette journal reads (authed via the -//! user's flowctl token); endpoint mutations go to the connector container -//! as in production. -use crate::local_specs; -use anyhow::Context; - -mod capture_driver; -mod derive_driver; -mod driver; -mod services; -mod shards; - -#[derive(Debug, clap::Args)] -#[clap(rename_all = "kebab-case")] -pub struct Preview { - /// Path or URL to a Flow specification file. - #[clap(long)] - source: String, - /// Name of the task to preview within the Flow specification file. - /// Required if there are multiple tasks in --source specifications. - #[clap(long)] - name: Option, - /// Number of synthetic shards to drive in parallel. Default 1. - /// For captures, N shards fan out as N independent connector instances. - /// Many connectors ignore the key range and will duplicate output; - /// use N=1 unless the connector partitions by key_begin/key_end. - #[clap(long, default_value = "1")] - shards: u32, - /// How long should preview run before gracefully stopping? - /// The default is no timeout. Note that the task may finish active - /// transaction activity even after this timeout is reached. - #[clap(long)] - timeout: Option, - /// How many connector sessions should be run, and what is the target number - /// of transactions for each session? - /// - /// Sessions are specified as a comma-separated list of the number of - /// transactions for the ordered session. For a given session, a value less - /// than zero means "unlimited transactions", though the session will still - /// end upon a connector exit / EOF (when a capture) or timeout. - /// - /// For example, to run three sessions consisting of two transactions, - /// then one transaction, and then unlimited transactions, - /// use argument `--sessions 2,1,-1`. - /// - /// A session is stopped and the next started upon reaching the target number - /// of transactions, or upon a timeout, or if the connector exits. - /// - /// The default is a single session with an unbounded number of transactions. - #[clap(long, value_parser, value_delimiter = ',')] - sessions: Option>, - /// Docker network to run connector images. - #[clap(long, default_value = "bridge")] - network: String, - /// Output task logs in JSON format to stderr. - #[clap(long, action)] - log_json: bool, - /// Loopback HTTP port hosting the service-kit admin dashboard - /// (handler inventory, per-handler trace overrides, /metrics). - #[clap(long)] - debug_port: Option, -} - -/// Resolved task selected from the source specifications. -enum TaskSpec { - Capture(proto_flow::flow::CaptureSpec), - Materialization(proto_flow::flow::MaterializationSpec), - Derivation(proto_flow::flow::CollectionSpec), -} - -impl Preview { - pub async fn run(&self, ctx: &mut crate::CliContext) -> anyhow::Result<()> { - let Self { - source, - name, - shards, - timeout, - sessions, - network, - log_json, - debug_port, - } = self; - - let source_url = build::arg_source_to_url(source, false)?; - - let log_handler: fn(&::ops::Log) = if *log_json { - ::ops::stderr_log_handler - } else { - ::ops::tracing_log_handler - }; - - let (_sources, _live, validations) = - local_specs::load_and_validate_full(ctx, source_url.as_str(), network, log_handler) - .await?; - - let task = resolve_task(&validations, name.as_deref())?; - - let timeout = timeout.map(|i| i.into()); - - let session_targets: Vec = if let Some(s) = sessions { - s.iter() - .map(|i| { - if *i < 0 { - Ok(0) - } else { - u32::try_from(*i).context("--sessions values must fit in uint32") - } - }) - .collect::>()? - } else { - vec![0] - }; - - let stop_token = tokio_util::sync::CancellationToken::new(); - - let result: anyhow::Result<()> = match task { - TaskSpec::Capture(spec) => { - let run = services::Run::start_capture( - *log_json, - network.clone(), - *shards, - *debug_port, - ctx.registry.clone(), - ) - .await?; - let session_loop = - capture_driver::run_sessions(&run, &spec, session_targets, stop_token.clone()); - tokio::pin!(session_loop); - run_with_timeout(session_loop, timeout, &stop_token).await - } - TaskSpec::Materialization(spec) => { - let run = services::Run::start_with_shuffle_leader( - ctx, - network.clone(), - *log_json, - *shards, - *debug_port, - ctx.registry.clone(), - ) - .await?; - let session_loop = - driver::run_sessions(&run, &spec, session_targets, stop_token.clone()); - tokio::pin!(session_loop); - run_with_timeout(session_loop, timeout, &stop_token).await - } - TaskSpec::Derivation(spec) => { - let run = services::Run::start_with_shuffle_leader( - ctx, - network.clone(), - *log_json, - *shards, - *debug_port, - ctx.registry.clone(), - ) - .await?; - let session_loop = - derive_driver::run_sessions(&run, &spec, session_targets, stop_token.clone()); - tokio::pin!(session_loop); - run_with_timeout(session_loop, timeout, &stop_token).await - } - }; - - // `run` drops here, aborting the tonic server and removing the - // RocksDB / shuffle-log tempdirs. - result - } -} - -async fn run_with_timeout( - mut session_loop: std::pin::Pin<&mut F>, - timeout: Option, - stop_token: &tokio_util::sync::CancellationToken, -) -> anyhow::Result<()> -where - F: std::future::Future>, -{ - let timeout = timeout.unwrap_or_else(|| std::time::Duration::MAX); - - tokio::select! { - result = &mut session_loop => result, - _ = tokio::signal::ctrl_c() => { - tracing::info!("Ctrl-C received; stopping active session"); - stop_token.cancel(); - session_loop.await - } - _ = tokio::time::sleep(timeout) => { - tracing::info!(?timeout, "preview --timeout reached; stopping active session"); - stop_token.cancel(); - session_loop.await - } - } -} - -fn resolve_task(validations: &tables::Validations, name: Option<&str>) -> anyhow::Result { - let derivations_count = validations - .built_collections - .iter() - .filter(|c| { - c.spec - .as_ref() - .map(|s| s.derivation.is_some()) - .unwrap_or_default() - }) - .count(); - let num_tasks = validations.built_captures.len() - + validations.built_materializations.len() - + derivations_count; - - if num_tasks == 0 { - anyhow::bail!( - "sourced specification files do not contain any tasks (captures, derivations, or materializations)", - ); - } - if num_tasks > 1 && name.is_none() { - anyhow::bail!( - "sourced specification files contain multiple tasks; use --name to identify the task", - ); - } - - for row in validations.built_captures.iter() { - if let Some(target) = name { - if row.capture.as_str() != target { - continue; - } - } - let Some(spec) = &row.spec else { continue }; - return Ok(TaskSpec::Capture(spec.clone())); - } - - for row in validations.built_materializations.iter() { - if let Some(target) = name { - if row.materialization.as_str() != target { - continue; - } - } - let Some(spec) = &row.spec else { continue }; - return Ok(TaskSpec::Materialization(spec.clone())); - } - - for row in validations.built_collections.iter() { - if let Some(target) = name { - if row.collection.as_str() != target { - continue; - } - } - let Some(spec) = &row.spec else { continue }; - if spec.derivation.is_some() { - return Ok(TaskSpec::Derivation(spec.clone())); - } - } - - if let Some(target) = name { - anyhow::bail!("could not find capture, materialization, or derivation {target}"); - } - anyhow::bail!("no capture, materialization, or derivation found in source"); -} diff --git a/crates/flowctl/src/shuffle_read.rs b/crates/flowctl/src/shuffle_read.rs index aeb2411ad1c..7d212a79925 100644 --- a/crates/flowctl/src/shuffle_read.rs +++ b/crates/flowctl/src/shuffle_read.rs @@ -4,8 +4,8 @@ //! `raw stats`) by hosting an ephemeral, single-shard `shuffle::Service` on a //! loopback tonic server and draining a `shuffle::proto::Task` — invoking a //! caller callback for each committed, non-ACK document and, after each -//! checkpoint, an optional checkpoint callback. (`raw preview-next` also uses -//! the `shuffle` crate, but drives its own Session directly rather than through +//! checkpoint, an optional checkpoint callback. (`preview` also uses the +//! `shuffle` crate, but drives its own Session directly rather than through //! this module.) //! //! Reads are non-blocking by default. The Session always tails its journals, so diff --git a/crates/gazette/src/lib.rs b/crates/gazette/src/lib.rs index 31b201c58d2..e4c07052a5a 100644 --- a/crates/gazette/src/lib.rs +++ b/crates/gazette/src/lib.rs @@ -94,6 +94,21 @@ impl Error { true } + // A broker aborts an Append RPC when its client fails to sustain + // the minimum data rate (gazette's MinAppendRate flow-control + // policing). This sheds a slow or contended client so others may + // proceed; the append rolled back cleanly and is safe to retry. + // Gazette surfaces it as an Unknown status wrapping + // ErrFlowControlUnderflow, so match on the stable core phrase + // rather than the wrapped-context prefix. + tonic::Code::Unknown + if status + .message() + .contains("didn't sustain the minimum append data rate") => + { + true + } + _ => false, // Others are not. }, @@ -198,3 +213,23 @@ fn backoff(attempt: usize) -> std::time::Duration { _ => std::time::Duration::from_secs(5), } } + +#[cfg(test)] +mod test { + use super::Error; + + #[test] + fn test_flow_control_underflow_is_transient() { + // A broker aborts a too-slow Append RPC by returning ErrFlowControlUnderflow + // wrapped with "append stream" context, delivered as an Unknown-coded gRPC + // status. It must be retried, not treated as a hard failure. + let err = Error::Grpc(tonic::Status::unknown( + "append stream: client stream didn't sustain the minimum append data rate", + )); + assert!(err.is_transient()); + + // Other Unknown-coded statuses remain fatal, so we don't retry genuine + // server-side errors indefinitely. + assert!(!Error::Grpc(tonic::Status::unknown("some other failure")).is_transient()); + } +} diff --git a/crates/proto-flow/src/runtime.rs b/crates/proto-flow/src/runtime.rs index ccb0cad67a3..01eff51fca1 100644 --- a/crates/proto-flow/src/runtime.rs +++ b/crates/proto-flow/src/runtime.rs @@ -490,10 +490,8 @@ pub struct Task { /// Task specification (protobuf-encoded bytes). #[prost(bytes = "bytes", tag = "1")] pub spec: ::prost::bytes::Bytes, - /// When true, documents and stats are written to output and not directed to collections. - #[prost(bool, tag = "2")] - pub preview: bool, - /// Preview / harness control. Zero means unlimited. + /// Maximum number of transactions to run before exiting. Zero means unlimited. + /// Used by "preview" workflows. #[prost(uint32, tag = "3")] pub max_transactions: u32, /// URL of a SQLite VFS the shard threads to a SQLite derive connector via diff --git a/crates/proto-flow/src/runtime.serde.rs b/crates/proto-flow/src/runtime.serde.rs index b2c3f226bb2..01aafe78827 100644 --- a/crates/proto-flow/src/runtime.serde.rs +++ b/crates/proto-flow/src/runtime.serde.rs @@ -10116,9 +10116,6 @@ impl serde::Serialize for Task { if !self.spec.is_empty() { len += 1; } - if self.preview { - len += 1; - } if self.max_transactions != 0 { len += 1; } @@ -10134,9 +10131,6 @@ impl serde::Serialize for Task { #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("spec", pbjson::private::base64::encode(&self.spec).as_str())?; } - if self.preview { - struct_ser.serialize_field("preview", &self.preview)?; - } if self.max_transactions != 0 { struct_ser.serialize_field("maxTransactions", &self.max_transactions)?; } @@ -10159,7 +10153,6 @@ impl<'de> serde::Deserialize<'de> for Task { { const FIELDS: &[&str] = &[ "spec", - "preview", "max_transactions", "maxTransactions", "sqlite_vfs_uri", @@ -10171,7 +10164,6 @@ impl<'de> serde::Deserialize<'de> for Task { #[allow(clippy::enum_variant_names)] enum GeneratedField { Spec, - Preview, MaxTransactions, SqliteVfsUri, PublisherId, @@ -10198,7 +10190,6 @@ impl<'de> serde::Deserialize<'de> for Task { { match value { "spec" => Ok(GeneratedField::Spec), - "preview" => Ok(GeneratedField::Preview), "maxTransactions" | "max_transactions" => Ok(GeneratedField::MaxTransactions), "sqliteVfsUri" | "sqlite_vfs_uri" => Ok(GeneratedField::SqliteVfsUri), "publisherId" | "publisher_id" => Ok(GeneratedField::PublisherId), @@ -10222,7 +10213,6 @@ impl<'de> serde::Deserialize<'de> for Task { V: serde::de::MapAccess<'de>, { let mut spec__ = None; - let mut preview__ = None; let mut max_transactions__ = None; let mut sqlite_vfs_uri__ = None; let mut publisher_id__ = None; @@ -10236,12 +10226,6 @@ impl<'de> serde::Deserialize<'de> for Task { Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) ; } - GeneratedField::Preview => { - if preview__.is_some() { - return Err(serde::de::Error::duplicate_field("preview")); - } - preview__ = Some(map_.next_value()?); - } GeneratedField::MaxTransactions => { if max_transactions__.is_some() { return Err(serde::de::Error::duplicate_field("maxTransactions")); @@ -10271,7 +10255,6 @@ impl<'de> serde::Deserialize<'de> for Task { } Ok(Task { spec: spec__.unwrap_or_default(), - preview: preview__.unwrap_or_default(), max_transactions: max_transactions__.unwrap_or_default(), sqlite_vfs_uri: sqlite_vfs_uri__.unwrap_or_default(), publisher_id: publisher_id__.unwrap_or_default(), diff --git a/crates/runtime-next/Cargo.toml b/crates/runtime-next/Cargo.toml index 680d14b729e..20e145c11c9 100644 --- a/crates/runtime-next/Cargo.toml +++ b/crates/runtime-next/Cargo.toml @@ -66,6 +66,7 @@ rand = { workspace = true } url = { workspace = true } reqwest = { workspace = true } rocksdb = { workspace = true } +schemars = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } diff --git a/crates/runtime-next/README.md b/crates/runtime-next/README.md index 54e16d2db4c..5b8ac279bec 100644 --- a/crates/runtime-next/README.md +++ b/crates/runtime-next/README.md @@ -58,14 +58,25 @@ communicate solely by gRPC; no shared memory. ``` src/ -├── lib.rs # crate root, shared helpers (Verify, LogHandler, Accumulator) +├── lib.rs # crate root, shared helpers (Verify, Accumulator) ├── task_service.rs # CGO entry point: binds UDS, serves Shard service -├── publish.rs # Rust journal publishing (used by both leader and shard) +├── publish.rs # Publisher / PublisherFactory traits + JournalPublisher +│ # (journal-IO) impls; leader & shard are monomorphized over +│ # the factory (preview installs its own from flowctl, so this +│ # crate is preview-agnostic) +├── logger.rs # Logger / LoggerFactory traits: the task's log + event stream +│ # (connector log sink + structured Events — persist / applied / +│ # inferred-schema / container lifecycle — which flatten to logs +│ # via LogEvent::to_log). Production shards install FnLoggerFactory +│ # (→ task-log file); leaders & tests install TracingLogger ├── patches.rs # wire format for connector-state patch streams │ ├── leader/ # sidecar Leader service │ ├── service.rs # gRPC entry, per-task Join rendezvous │ ├── join.rs # protocol primitives for joining shards into a session +│ ├── shuffle.rs # ShuffleSession / ShuffleSessionFactory traits + ShuffleServiceFactory +│ │ # (journal-reading Session) impl; leader is monomorphized over the +│ │ # factory (preview installs its own fixture replay from flowctl) │ └── materialize/ │ ├── handler.rs # gRPC stream handler, dispatches to startup/actor │ ├── startup.rs # Recover / Open / Apply / Recovered phase diff --git a/crates/runtime-next/src/container.rs b/crates/runtime-next/src/container.rs index 08a0d02678c..4d417db767a 100644 --- a/crates/runtime-next/src/container.rs +++ b/crates/runtime-next/src/container.rs @@ -17,8 +17,10 @@ const USAGE_RATE_LABEL: &str = "dev.estuary.usage-rate"; const PORT_PUBLIC_LABEL_PREFIX: &str = "dev.estuary.port-public."; const PORT_PROTO_LABEL_PREFIX: &str = "dev.estuary.port-proto."; +// `flow-connector-init` is extracted from this image when a locally-built copy +// isn't found by `locate_bin` (dev/CI builds place one alongside the executable). // TODO(johnny): Consider better packaging and versioning of `flow-connector-init`. -const CONNECTOR_INIT_IMAGE: &str = "ghcr.io/estuary/flow:v0.5.24-30-ga3eba41f95"; +const CONNECTOR_INIT_IMAGE: &str = "ghcr.io/estuary/reactor:v0.6.10-62-g8b6aeec1cd3"; const CONNECTOR_INIT_IMAGE_PATH: &str = "/usr/local/bin/flow-connector-init"; /// Determines the protocol of an image. If the image has a `FLOW_RUNTIME_PROTOCOL` label, @@ -30,7 +32,11 @@ pub async fn flow_runtime_protocol(image: &str) -> anyhow::Result anyhow::Result( image: &str, - log_handler: impl crate::LogHandler, + logger: L, log_level: ops::LogLevel, network: &str, task_name: &str, @@ -60,7 +67,7 @@ pub async fn start( ) -> anyhow::Result<( runtime::Container, tonic::transport::Channel, - Guard, + Guard, connector_init::Codec, )> { validate_connector_image(image, plane)?; @@ -92,7 +99,7 @@ pub async fn start( // and parsing its advertised network ports. let ((), (image_inspection, codec)) = futures::try_join!( find_connector_init_and_copy(tmp_connector_init.path()), - inspect_image_and_copy(image, tmp_docker_inspect.path()), + inspect_image_and_copy(image, tmp_docker_inspect.path(), &logger), )?; // Close our open files but retain a deletion guard. @@ -194,9 +201,10 @@ pub async fn start( // our inner flow-connector-init process to produce its startup log. let (ready_tx, ready_rx) = oneshot::channel::<()>(); - // Service process stderr by decoding ops::Logs and sending to our handler. + // Service process stderr by decoding ops::Logs and reporting to the logger. let stderr = process.stderr.take().unwrap(); let quoted_task_name: bytes::Bytes = format!("\"{task_name}\"").into(); + let pump_logger = logger.clone(); tokio::spawn(async move { let mut stderr = tokio::io::BufReader::new(stderr); let mut line = String::new(); @@ -226,7 +234,7 @@ pub async fn start( let (log, consume) = decoder.line_to_log(&line, stderr.buffer()); stderr.consume(consume); let sanitized = sanitize_event_type("ed_task_name, log); - log_handler.log(&sanitized); + pump_logger.log(&sanitized); } }); @@ -258,7 +266,9 @@ pub async fn start( format!("failed to connect to container connector-init at {init_address}") })?; - tracing::info!( + // Low-level network / codec detail stays at debug; the user-facing "started" + // event is reported to the logger below (whose default logs it at info). + tracing::debug!( %image, %init_address, %ip_addr, @@ -268,23 +278,31 @@ pub async fn start( ?codec, %task_name, ?task_type, - "started connector container" + "dialed connector container" ); let usage_rate = image_inspection.usage_rate; let network_ports = image_inspection.network_ports; + let container = runtime::Container { + ip_addr: format!("{ip_addr}"), + network_ports, + usage_rate, + mapped_host_ports, + }; + logger.event(crate::LogEvent::ContainerStarted { + image, + container: &container, + }); + Ok(( - runtime::Container { - ip_addr: format!("{ip_addr}"), - network_ports, - usage_rate, - mapped_host_ports, - }, + container, channel, Guard { _tmp_connector_init: tmp_connector_init, _tmp_docker_inspect: tmp_docker_inspect, _process: process, + image: image.to_string(), + logger, }, codec, )) @@ -346,10 +364,20 @@ fn sanitize_event_type(quoted_task_name: &bytes::Bytes, mut log: ops::Log) -> op /// Guard contains a running image container instance, /// which will be stopped and cleaned up when the Guard is dropped. -pub struct Guard { +/// Its drop reports a `container_stopped` event through the logger. +pub struct Guard { _tmp_connector_init: tempfile::TempPath, _tmp_docker_inspect: tempfile::TempPath, _process: async_process::Child, + image: String, + logger: L, +} + +impl Drop for Guard { + fn drop(&mut self) { + self.logger + .event(crate::LogEvent::ContainerStopped { image: &self.image }); + } } fn unique_container_name() -> String { @@ -396,7 +424,7 @@ where Ok(output.stdout) } -async fn docker_pull(image: &str) -> anyhow::Result<()> { +async fn docker_pull(image: &str, logger: &impl crate::Logger) -> anyhow::Result<()> { const MAX_RETRIES: u32 = 3; const RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(2); @@ -412,13 +440,11 @@ async fn docker_pull(image: &str) -> anyhow::Result<()> { || err_str.contains("unexpected EOF"); if is_transient && attempt < MAX_RETRIES { - tracing::warn!( - %image, + logger.event(crate::LogEvent::ImagePullRetry { + image, attempt, - max_retries = MAX_RETRIES, - error = %err, - "transient error pulling image (will retry)" - ); + error: &err_str, + }); tokio::time::sleep(RETRY_DELAY).await; } else { return Err(err); @@ -670,9 +696,10 @@ async fn find_connector_init_and_copy(tmp_path: &std::path::Path) -> anyhow::Res async fn inspect_image_and_copy( image: &str, tmp_path: &std::path::Path, + logger: &impl crate::Logger, ) -> anyhow::Result<(ImageInspection, connector_init::Codec)> { if !image.ends_with(":local") { - docker_pull(image).await.context("pulling image")?; + docker_pull(image, logger).await.context("pulling image")?; } let inspect_content = docker_cmd(&["inspect", image]) @@ -711,7 +738,7 @@ mod test { let (container, channel, _guard, _codec) = start( "ghcr.io/estuary/source-http-ingest:dev", - ops::tracing_log_handler, + crate::TracingLogger, ops::LogLevel::Debug, "", "a-task-name", @@ -772,7 +799,7 @@ mod test { let Err(err) = start( "alpine", // Not a connector. - ops::tracing_log_handler, + crate::TracingLogger, ops::LogLevel::Debug, "", "a-task-name", diff --git a/crates/runtime-next/src/image_connector.rs b/crates/runtime-next/src/image_connector.rs index 6f0ae86b5f9..e42828fccd4 100644 --- a/crates/runtime-next/src/image_connector.rs +++ b/crates/runtime-next/src/image_connector.rs @@ -8,9 +8,9 @@ pub type StartRpcFuture = /// Serve an image-based connector by starting a container, dialing connector-init, /// and then starting a gRPC request. -pub async fn serve( +pub async fn serve( image: String, // Container image to run. - log_handler: L, // Handler for connector logs. + logger: L, // Logger for connector logs and lifecycle. log_level: ops::LogLevel, // Log-level of the connector, if known. network: &str, // Container network to use. request_rx: mpsc::Receiver, // Caller's input request stream. @@ -31,13 +31,7 @@ where + 'static, { let (container, channel, guard, codec) = container::start( - &image, - log_handler.clone(), - log_level, - &network, - &task_name, - task_type, - plane, + &image, logger, log_level, &network, &task_name, task_type, plane, ) .await?; diff --git a/crates/runtime-next/src/leader/capture/fsm.rs b/crates/runtime-next/src/leader/capture/fsm.rs index 0f28252dd82..28383542e59 100644 --- a/crates/runtime-next/src/leader/capture/fsm.rs +++ b/crates/runtime-next/src/leader/capture/fsm.rs @@ -688,7 +688,7 @@ fn build_stats_doc(task: &Task, extents: &Extents) -> ops::proto::Stats { ops::proto::Stats { meta: Some(ops::proto::Meta { - uuid: String::new(), // Stamped by Publisher::enqueue() + uuid: String::new(), // Stamped by publisher::Publisher::enqueue() }), shard: Some(task.shard_ref.clone()), timestamp: extents.open.to_pb_json_timestamp(), diff --git a/crates/runtime-next/src/leader/derive/actor.rs b/crates/runtime-next/src/leader/derive/actor.rs index 1aff9ee2c6d..6cf49b42b0c 100644 --- a/crates/runtime-next/src/leader/derive/actor.rs +++ b/crates/runtime-next/src/leader/derive/actor.rs @@ -10,31 +10,33 @@ use std::time::Duration; use tokio::sync::mpsc; /// Actor leads transactions of an established derivation task session. -pub struct Actor { +pub struct Actor { // Future for an in-flight ACK intents write, if any. - intents_write_fut: Option>>, + intents_write_fut: Option>>, // Optional full Frontier and Checkpoint, used for V1 rollback support. legacy_checkpoint: Option<(shuffle::Frontier, consumer::Checkpoint)>, // Per-task metrics counters. metrics: super::Metrics, + // Logger of task-centric state changes and events. + logger: L, // Publisher for stats and ACK intents, parked while no async operation is in-flight. - parked_publisher: Option, + parked_publisher: Option

, // ACK intents to persist and append at later transaction stages. pending_ack_intents: BTreeMap, // One channel to each shard for synchronously sending it messages. shard_tx: Vec>>, // Future for an in-flight stats flush, if any, yielding ACK intents. - stats_write_fut: - Option)>>>, + stats_write_fut: Option)>>>, // Task being executed by this actor. task: Task, } -impl Actor { +impl Actor { pub fn new( legacy_checkpoint: Option, metrics: super::Metrics, - publisher: crate::Publisher, + logger: L, + publisher: P, shard_tx: Vec>>, task: Task, ) -> Self { @@ -42,6 +44,7 @@ impl Actor { intents_write_fut: None, legacy_checkpoint: legacy_checkpoint.map(|f| (f, consumer::Checkpoint::default())), metrics, + logger, parked_publisher: Some(publisher), pending_ack_intents: BTreeMap::new(), shard_tx, @@ -51,11 +54,11 @@ impl Actor { } #[tracing::instrument(level = "debug", err(Debug, level = "warn"), skip_all)] - pub async fn serve( + pub async fn serve( &mut self, mut head: fsm::Head, mut tail: fsm::Tail, - mut session: shuffle::SessionClient, + mut session: S, shard_rx: Vec>>, ) -> anyhow::Result<()> { service_kit::event!( @@ -355,6 +358,9 @@ impl Actor { } fsm::Action::Persist { persist } => { + self.logger + .event(crate::LogEvent::Persist { persist: &persist }); + service_kit::event!(tracing::Level::DEBUG, "shard", "sending L:Persist"); let _ = self.shard_tx[0].send(Ok(proto::Derive { persist: Some(persist), diff --git a/crates/runtime-next/src/leader/derive/fsm.rs b/crates/runtime-next/src/leader/derive/fsm.rs index b943f7f4245..c7b2c21da9c 100644 --- a/crates/runtime-next/src/leader/derive/fsm.rs +++ b/crates/runtime-next/src/leader/derive/fsm.rs @@ -892,7 +892,7 @@ fn build_stats_doc( Ok(ops::proto::Stats { meta: Some(ops::proto::Meta { - uuid: String::new(), // Stamped by Publisher::enqueue() + uuid: String::new(), // Stamped by publisher::Publisher::enqueue() }), shard: Some(task.shard_ref.clone()), timestamp: extents.open.to_pb_json_timestamp(), diff --git a/crates/runtime-next/src/leader/derive/handler.rs b/crates/runtime-next/src/leader/derive/handler.rs index 0641d108f7a..fe425e86d73 100644 --- a/crates/runtime-next/src/leader/derive/handler.rs +++ b/crates/runtime-next/src/leader/derive/handler.rs @@ -4,8 +4,13 @@ use futures::StreamExt; use tokio::sync::mpsc; use tracing::Instrument; -pub(crate) async fn serve( - service: crate::Service, +pub(crate) async fn serve< + R, + S: crate::ShuffleSessionFactory, + P: crate::PublisherFactory, + L: crate::LoggerFactory, +>( + service: crate::Service, authz: proto_grpc::Authorizer, request_rx: R, response_tx: mpsc::UnboundedSender>, @@ -20,8 +25,13 @@ where .await } -async fn serve_inner( - service: crate::Service, +async fn serve_inner< + R, + S: crate::ShuffleSessionFactory, + P: crate::PublisherFactory, + L: crate::LoggerFactory, +>( + service: crate::Service, authz: proto_grpc::Authorizer, mut request_rx: R, response_tx: mpsc::UnboundedSender>, @@ -180,6 +190,7 @@ where let startup::Startup { committed_close, committed_frontier, + logger, pending_ack_intents, publisher, session, @@ -211,7 +222,14 @@ where Some(committed_frontier) }; - let mut actor = actor::Actor::new(legacy_checkpoint, metrics, publisher, shard_tx, task); + let mut actor = actor::Actor::new( + legacy_checkpoint, + metrics, + logger, + publisher, + shard_tx, + task, + ); handler.set_phase("running"); actor.serve(head, tail, session, shard_rx).await } diff --git a/crates/runtime-next/src/leader/derive/startup.rs b/crates/runtime-next/src/leader/derive/startup.rs index f8af236a29a..57bc0e69cf2 100644 --- a/crates/runtime-next/src/leader/derive/startup.rs +++ b/crates/runtime-next/src/leader/derive/startup.rs @@ -11,17 +11,19 @@ use std::collections::BTreeMap; use tokio::sync::mpsc; /// Outcomes of the leader protocol startup phase. -pub(super) struct Startup { +pub(super) struct Startup { // Clock at which the last-committed transaction closed. pub committed_close: uuid::Clock, // Fully committed Frontier. pub committed_frontier: shuffle::Frontier, + // Logger of task-centric state changes and events. + pub logger: L, // Recovered ACK intents of the last transaction. pub pending_ack_intents: BTreeMap, // Publisher for writing stats and ACK intents. - pub publisher: crate::Publisher, + pub publisher: P, // Initiated shuffle session for the task and topology. - pub session: shuffle::SessionClient, + pub session: S, // Task definition. pub task: Task, } @@ -32,17 +34,21 @@ pub(super) struct Startup { skip_all, fields(shard_zero = %shard_ids[0], shards = shard_ids.len()) )] -pub(super) async fn run( +pub(super) async fn run< + S: crate::ShuffleSessionFactory, + P: crate::PublisherFactory, + L: crate::LoggerFactory, +>( build: String, drop_v1_rollback: bool, ops_stats_journal: String, reactors: Vec, shard_rx: &mut Vec>>, shard_tx: &Vec>>, - service: &crate::Service, + service: &crate::Service, shard_ids: Vec, shard_shuffles: Vec, -) -> anyhow::Result { +) -> anyhow::Result> { let n_shards = reactors.len(); assert_eq!(n_shards, shard_rx.len()); assert_eq!(n_shards, shard_tx.len()); @@ -76,7 +82,6 @@ pub(super) async fn run( // Build task definition. let proto::Task { - preview, max_transactions, spec: spec_bytes, sqlite_vfs_uri: _, @@ -89,19 +94,19 @@ pub(super) async fn run( .await .context("building task definition")?; - // Initialize publisher. - let publisher = if preview { - crate::Publisher::new_preview([]) - } else { - crate::Publisher::new_real( + // Open a Logger for runtime events, bound to the task. + let logger = service.logger_factory.open(&task.shard_ref.name); + + // Open a publisher for stats and ACK intents (no collection bindings). + let publisher = service + .publisher_factory + .open( shard_ids[0].clone(), // Shard zero is AuthZ subject. crate::publish::producer_from_bytes(&publisher_id)?, - &service.publisher_factory, &ops_stats_journal, - [], // No additional bindings. + &[], ) - .context("creating publisher")? - }; + .context("opening publisher")?; // Receive Recover fan-in. let proto::Recover { @@ -301,18 +306,16 @@ pub(super) async fn run( let shuffle_task = shuffle::proto::Task { task: Some(shuffle::proto::task::Task::Derivation(spec)), }; - let session = shuffle::SessionClient::open( - &service.shuffle_service, - shuffle_task, - shard_shuffles, - resume_frontier, - ) - .await - .context("opening shuffle Session")?; + let session = service + .shuffle_factory + .open(shuffle_task, shard_shuffles, resume_frontier) + .await + .context("opening shuffle Session")?; Ok(Startup { committed_close, committed_frontier, + logger, pending_ack_intents, publisher, session, diff --git a/crates/runtime-next/src/leader/materialize/actor.rs b/crates/runtime-next/src/leader/materialize/actor.rs index d570ae29dab..a296eb92159 100644 --- a/crates/runtime-next/src/leader/materialize/actor.rs +++ b/crates/runtime-next/src/leader/materialize/actor.rs @@ -10,17 +10,19 @@ use std::time::Duration; use tokio::sync::mpsc; /// Actor leads transactions of an established materialization task session. -pub struct Actor { +pub struct Actor { // Client used for trigger dispatch. http_client: reqwest::Client, // Future for an in-flight ACK intents write, if any. - intents_write_fut: Option>>, + intents_write_fut: Option>>, // Optional full Frontier and Checkpoint, used for V1 rollback support. legacy_checkpoint: Option<(shuffle::Frontier, consumer::Checkpoint)>, // Per-task metrics counters and gauges. metrics: super::Metrics, + // Logger of task-centric state changes and events. + logger: L, // Publisher for stats and ACK intents, parked while no async operation is in-flight. - parked_publisher: Option, + parked_publisher: Option

, // ACK intents to persist and append at later transaction stages. pending_ack_intents: BTreeMap, // One channel to each shard for synchronously sending it messages. @@ -28,20 +30,20 @@ pub struct Actor { // it follows a strict request/response pattern. shard_tx: Vec>>, // Future for an in-flight stats flush, if any, yielding ACK intents. - stats_write_fut: - Option)>>>, + stats_write_fut: Option)>>>, // Task being executed by this actor. task: Task, // Future for an in-flight trigger dispatch, if any. trigger_fut: Option>>, } -impl Actor { +impl Actor { pub fn new( http_client: reqwest::Client, legacy_checkpoint: Option, metrics: super::Metrics, - publisher: crate::Publisher, + logger: L, + publisher: P, shard_tx: Vec>>, task: Task, ) -> Self { @@ -50,6 +52,7 @@ impl Actor { intents_write_fut: None, legacy_checkpoint: legacy_checkpoint.map(|f| (f, consumer::Checkpoint::default())), metrics, + logger, parked_publisher: Some(publisher), pending_ack_intents: BTreeMap::new(), shard_tx, @@ -60,11 +63,11 @@ impl Actor { } #[tracing::instrument(level = "debug", err(Debug, level = "warn"), skip_all)] - pub async fn serve( + pub async fn serve( &mut self, mut head: fsm::Head, mut tail: fsm::Tail, - mut session: shuffle::SessionClient, + mut session: S, shard_rx: Vec>>, ) -> anyhow::Result<()> { service_kit::event!( @@ -389,6 +392,9 @@ impl Actor { } fsm::Action::Persist { persist } => { + self.logger + .event(crate::LogEvent::Persist { persist: &persist }); + service_kit::event!(tracing::Level::DEBUG, "shard", "sending L:Persist"); let _ = self.shard_tx[0].send(Ok(proto::Materialize { persist: Some(persist), @@ -575,7 +581,7 @@ mod tests { fn mk_actor( n_shards: usize, ) -> ( - Actor, + Actor, Vec>>, ) { let mut shard_tx = Vec::with_capacity(n_shards); @@ -600,7 +606,8 @@ mod tests { reqwest::Client::new(), None, super::super::Metrics::new("test/task/shard"), - crate::Publisher::new_preview([]), + crate::TracingLogger, + crate::publish::NoopPublisher, shard_tx, task, ); diff --git a/crates/runtime-next/src/leader/materialize/fsm.rs b/crates/runtime-next/src/leader/materialize/fsm.rs index 700f6a331bd..af8ac8019ec 100644 --- a/crates/runtime-next/src/leader/materialize/fsm.rs +++ b/crates/runtime-next/src/leader/materialize/fsm.rs @@ -266,7 +266,7 @@ impl HeadIdle { // Termination condition: stop at a clean transaction boundary. if stopping && !is_open && tail_done { - return (Action::Idle, Head::Stop); + return (Action::PollAgain, Head::Stop); } // Clear stale close_requested from after prior transaction close. if !is_open { @@ -841,7 +841,7 @@ impl HeadStartCommit { // (commit) without starting any post-commit activity: that's left // for the next session, which will recover our commit state and // resume from Tail::Begin. - (Action::Idle, Head::Stop) + (Action::PollAgain, Head::Stop) } else { // Rotate to begin a next transaction. `idempotent_replay` // is one-shot — only the first transaction of a session may replay @@ -1158,7 +1158,7 @@ fn build_stats_doc( Ok(ops::proto::Stats { meta: Some(ops::proto::Meta { - uuid: String::new(), // Stamped by Publisher::enqueue() + uuid: String::new(), // Stamped by publisher::Publisher::enqueue() }), shard: Some(task.shard_ref.clone()), timestamp: extents.open.to_pb_json_timestamp(), @@ -1727,11 +1727,13 @@ mod tests { } // Final Persisted under stopping: HeadStartCommit chained - // (next_action, next_state) = (Idle, Head::Stop) — no Rotate. + // (next_action, next_state) = (PollAgain, Head::Stop) — no Rotate. + // PollAgain (not Idle) lets the actor loop exit `while !Head::Stop` + // immediately rather than parking for a 60s ACTOR_TICK_INTERVAL. ctx.shard_rx = Some(mk_head_persisted(&head)); let (action, h) = ctx.step_head(head, &mut tail); head = h; - assert!(matches!(action, Action::Idle)); + assert!(matches!(action, Action::PollAgain)); assert!(matches!(head, Head::Stop)); assert!(matches!(tail, Tail::Done(_))); } diff --git a/crates/runtime-next/src/leader/materialize/handler.rs b/crates/runtime-next/src/leader/materialize/handler.rs index 34c49e2e5e9..93f3e226ef5 100644 --- a/crates/runtime-next/src/leader/materialize/handler.rs +++ b/crates/runtime-next/src/leader/materialize/handler.rs @@ -4,8 +4,13 @@ use futures::StreamExt; use tokio::sync::mpsc; use tracing::Instrument; -pub(crate) async fn serve( - service: crate::Service, +pub(crate) async fn serve< + R, + S: crate::ShuffleSessionFactory, + P: crate::PublisherFactory, + L: crate::LoggerFactory, +>( + service: crate::Service, authz: proto_grpc::Authorizer, request_rx: R, response_tx: mpsc::UnboundedSender>, @@ -23,8 +28,13 @@ where .await } -async fn serve_inner( - service: crate::Service, +async fn serve_inner< + R, + S: crate::ShuffleSessionFactory, + P: crate::PublisherFactory, + L: crate::LoggerFactory, +>( + service: crate::Service, authz: proto_grpc::Authorizer, mut request_rx: R, response_tx: mpsc::UnboundedSender>, @@ -187,6 +197,7 @@ where committed_close, committed_frontier, idempotent_replay, + logger, pending_ack_intents, pending_trigger_params, publisher, @@ -230,6 +241,7 @@ where service.http_client.clone(), legacy_checkpoint, metrics, + logger, publisher, shard_tx, task, diff --git a/crates/runtime-next/src/leader/materialize/startup.rs b/crates/runtime-next/src/leader/materialize/startup.rs index 2bad6c7ef25..5687dabf8a5 100644 --- a/crates/runtime-next/src/leader/materialize/startup.rs +++ b/crates/runtime-next/src/leader/materialize/startup.rs @@ -11,21 +11,23 @@ use std::collections::BTreeMap; use tokio::sync::mpsc; /// Outcomes of the leader protocol startup phase. -pub(super) struct Startup { +pub(super) struct Startup { // Clock at which the last-committed transaction closed. pub committed_close: uuid::Clock, // Fully committed Frontier. pub committed_frontier: shuffle::Frontier, // Is the first transaction an idempotent replay of a recovered hinted Frontier? pub idempotent_replay: bool, + // Logger of task-centric state changes and events. + pub logger: L, // Recovered ACK intents of the last transaction. pub pending_ack_intents: BTreeMap, // Recovered variables for the task. pub pending_trigger_params: Bytes, // Publisher for writing stats and ACK intents. - pub publisher: crate::Publisher, + pub publisher: P, // Initiated shuffle session for the task and topology. - pub session: shuffle::SessionClient, + pub session: S, // Task definition. pub task: Task, } @@ -36,17 +38,21 @@ pub(super) struct Startup { skip_all, fields(shard_zero = %shard_ids[0], shards = shard_ids.len()) )] -pub(super) async fn run( +pub(super) async fn run< + S: crate::ShuffleSessionFactory, + P: crate::PublisherFactory, + L: crate::LoggerFactory, +>( build: String, drop_v1_rollback: bool, ops_stats_journal: String, reactors: Vec, shard_rx: &mut Vec>>, shard_tx: &Vec>>, - service: &crate::Service, + service: &crate::Service, shard_ids: Vec, shard_shuffles: Vec, -) -> anyhow::Result { +) -> anyhow::Result> { let n_shards = reactors.len(); assert_eq!(n_shards, shard_rx.len()); assert_eq!(n_shards, shard_tx.len()); @@ -80,7 +86,6 @@ pub(super) async fn run( // Build task definition. let proto::Task { - preview, max_transactions, spec: spec_bytes, sqlite_vfs_uri: _, @@ -93,19 +98,19 @@ pub(super) async fn run( .await .context("building task definition")?; - // Initialize publisher. - let publisher = if preview { - crate::Publisher::new_preview([]) - } else { - crate::Publisher::new_real( + // Open a Logger for runtime events, bound to the task. + let logger = service.logger_factory.open(&task.shard_ref.name); + + // Open a publisher for stats and ACK intents (no collection bindings). + let publisher = service + .publisher_factory + .open( shard_ids[0].clone(), // Shard zero is AuthZ subject. crate::publish::producer_from_bytes(&publisher_id)?, - &service.publisher_factory, &ops_stats_journal, - [], // No additional bindings. + &[], ) - .context("creating publisher")? - }; + .context("opening publisher")?; // Receive Recover fan-in. let proto::Recover { @@ -153,6 +158,7 @@ pub(super) async fn run( &spec_bytes, &task.shard_ref.build, &mut connector_state_json, + &logger, ) .await?; @@ -335,19 +341,17 @@ pub(super) async fn run( let shuffle_task = shuffle::proto::Task { task: Some(shuffle::proto::task::Task::Materialization(spec)), }; - let session = shuffle::SessionClient::open( - &service.shuffle_service, - shuffle_task, - shard_shuffles, - resume_frontier, - ) - .await - .context("opening shuffle Session")?; + let session = service + .shuffle_factory + .open(shuffle_task, shard_shuffles, resume_frontier) + .await + .context("opening shuffle Session")?; Ok(Startup { committed_close, committed_frontier, idempotent_replay, + logger, pending_ack_intents, pending_trigger_params, publisher, @@ -411,7 +415,7 @@ async fn send_persist( // with the OLD `last_applied` against the partially-advanced state, // requiring the connector's Apply to be idempotent across repeated // invocations of the same target spec. -async fn apply_loop( +async fn apply_loop( rx: &mut BoxStream<'static, tonic::Result>, tx: &mpsc::UnboundedSender>, peer: &str, @@ -419,6 +423,7 @@ async fn apply_loop( next_applied: &Bytes, next_version: &str, connector_state_json: &mut Bytes, + logger: &L, ) -> anyhow::Result<()> { let verify_applied = crate::verify("Materialize", "Applied", peer); let last_version = if last_applied.is_empty() { @@ -455,13 +460,16 @@ async fn apply_loop( }), .. } => { - let patches_clone: bytes::Bytes = connector_patches_json.clone(); + logger.event(crate::LogEvent::Applied { + action_description: &action_description, + }); + service_kit::event!( tracing::Level::INFO, "leader", iteration, - action_description = action_description.clone(), - patches = service_kit::event::debug(patches_clone), + action_description, + patches = service_kit::event::debug(connector_patches_json.clone()), "connector Apply completed", ); connector_patches_json @@ -501,18 +509,14 @@ async fn apply_loop( *connector_state_json = crate::patches::apply_state_patches(connector_state_json, &applied_patches_json)?; - // Persist the iteration's patches to shard zero. - send_persist( - rx, - tx, - peer, - proto::Persist { - seq_no: iteration, // End-of-sequence. - connector_patches_json: applied_patches_json, - ..Default::default() - }, - ) - .await?; + // Persist the iteration's patches to shard zero, observing the delta. + let persist = proto::Persist { + seq_no: iteration, // End-of-sequence. + connector_patches_json: applied_patches_json, + ..Default::default() + }; + logger.event(crate::LogEvent::Persist { persist: &persist }); + send_persist(rx, tx, peer, persist).await?; } anyhow::bail!( @@ -603,9 +607,18 @@ mod tests { let same = Bytes::new(); let mut state = Bytes::from_static(b"{\"k\":1}"); - apply_loop(&mut rx, &leader_tx, "p", &same, &same, "v1", &mut state) - .await - .unwrap(); + apply_loop( + &mut rx, + &leader_tx, + "p", + &same, + &same, + "v1", + &mut state, + &crate::TracingLogger, + ) + .await + .unwrap(); let m = leader_rx.try_recv().unwrap().unwrap(); let apply = m.apply.expect("Apply was sent"); @@ -629,9 +642,18 @@ mod tests { let last = Bytes::new(); let next = Bytes::from_static(b"new-spec-bytes"); let mut state = Bytes::from_static(b"{}"); - apply_loop(&mut rx, &leader_tx, "p", &last, &next, "v2", &mut state) - .await - .unwrap(); + apply_loop( + &mut rx, + &leader_tx, + "p", + &last, + &next, + "v2", + &mut state, + &crate::TracingLogger, + ) + .await + .unwrap(); let m1 = leader_rx.try_recv().unwrap().unwrap(); let apply = m1.apply.unwrap(); @@ -668,9 +690,18 @@ mod tests { let last = Bytes::new(); let next = Bytes::from_static(b"spec"); let mut state = Bytes::from_static(br#"{"nested":{"a":0},"keep":"v0","drop":"x"}"#); - apply_loop(&mut rx, &leader_tx, "p", &last, &next, "v2", &mut state) - .await - .unwrap(); + apply_loop( + &mut rx, + &leader_tx, + "p", + &last, + &next, + "v2", + &mut state, + &crate::TracingLogger, + ) + .await + .unwrap(); // Apply (iter 1) — connector observes the original state. let apply1 = leader_rx.try_recv().unwrap().unwrap().apply.unwrap(); @@ -775,9 +806,18 @@ mod tests { let last = Bytes::new(); let next = Bytes::from_static(b"spec"); let mut state = Bytes::from_static(b"{}"); - let err = apply_loop(&mut rx, &leader_tx, "p", &last, &next, "v2", &mut state) - .await - .unwrap_err(); + let err = apply_loop( + &mut rx, + &leader_tx, + "p", + &last, + &next, + "v2", + &mut state, + &crate::TracingLogger, + ) + .await + .unwrap_err(); let s = format!("{err:?}"); assert!( s.contains(case.expect), diff --git a/crates/runtime-next/src/leader/mod.rs b/crates/runtime-next/src/leader/mod.rs index e057d6af536..641f7d9c410 100644 --- a/crates/runtime-next/src/leader/mod.rs +++ b/crates/runtime-next/src/leader/mod.rs @@ -5,6 +5,7 @@ use join::{JoinOutcome, JoinSlot, PendingJoin, validate as validate_join}; pub mod close_policy; pub mod frontier_mapping; mod service; +mod shuffle; // Task-specific handling. pub mod capture; // `pub` because it's directly used by shard actor. @@ -12,6 +13,7 @@ mod derive; mod materialize; pub use service::Service; +pub use shuffle::{ShuffleServiceFactory, ShuffleSession, ShuffleSessionFactory}; /// Shard-label feature flag (under the `estuary.dev/flag/` prefix) that, when /// set to `"true"`, tells the leader to drop V1 rollback support for the task. diff --git a/crates/runtime-next/src/leader/service.rs b/crates/runtime-next/src/leader/service.rs index 27785970a7d..81ffb335191 100644 --- a/crates/runtime-next/src/leader/service.rs +++ b/crates/runtime-next/src/leader/service.rs @@ -4,20 +4,38 @@ use std::sync::Arc; use tokio::sync::mpsc; /// Service is the implementation of the Leader gRPC service trait. -#[derive(Clone)] -pub struct Service(Arc); +pub struct Service< + S: crate::ShuffleSessionFactory, + P: crate::PublisherFactory, + L: crate::LoggerFactory, +>(Arc>); + +impl Clone + for Service +{ + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} /// ServiceImpl holds shared implementation state for the Leader gRPC service. -pub struct ServiceImpl { +pub struct ServiceImpl< + S: crate::ShuffleSessionFactory, + P: crate::PublisherFactory, + L: crate::LoggerFactory, +> { /// In-progress Derive session Joins, keyed by task name. pub(crate) derive_joins: std::sync::Mutex>>, /// In-progress Materialize session Joins, keyed by task name. pub(crate) materialize_joins: std::sync::Mutex>>, - /// Service used by leader sessions to open shuffle Sessions. - pub(crate) shuffle_service: shuffle::Service, - /// Factory for building Gazette clients for publish operations. - pub(crate) publisher_factory: gazette::journal::ClientFactory, + /// Factory used by leader sessions to open a [`ShuffleSession`](crate::ShuffleSession). + pub(crate) shuffle_factory: S, + /// Factory used by leader sessions to open a [`Publisher`](crate::Publisher) of stats and ACK intents. + pub(crate) publisher_factory: P, + /// Factory used by leader sessions to open a [`Logger`](crate::Logger) + /// of task-centric state changes and events. + pub(crate) logger_factory: L, /// Process-wide HTTP client used by the actor to deliver trigger webhooks. pub(crate) http_client: reqwest::Client, /// Registry of in-flight Leader session handlers, for the admin surface. @@ -26,18 +44,22 @@ pub struct ServiceImpl { pub(crate) disarm_auth: bool, } -impl Service { +impl + Service +{ pub fn new( - shuffle_service: shuffle::Service, - publisher_factory: gazette::journal::ClientFactory, + shuffle_factory: S, + publisher_factory: P, + logger_factory: L, registry: service_kit::Registry, disarm_auth: bool, ) -> Self { Self(Arc::new(ServiceImpl { derive_joins: std::sync::Mutex::new(HashMap::new()), materialize_joins: std::sync::Mutex::new(HashMap::new()), - shuffle_service, + shuffle_factory, publisher_factory, + logger_factory, http_client: reqwest::Client::new(), registry, disarm_auth, @@ -95,8 +117,10 @@ impl Service { } } -impl std::ops::Deref for Service { - type Target = ServiceImpl; +impl + std::ops::Deref for Service +{ + type Target = ServiceImpl; fn deref(&self) -> &Self::Target { &self.0 @@ -104,7 +128,9 @@ impl std::ops::Deref for Service { } #[tonic::async_trait] -impl proto_grpc::runtime::leader_server::Leader for Service { +impl + proto_grpc::runtime::leader_server::Leader for Service +{ type DeriveStream = tokio_stream::wrappers::UnboundedReceiverStream>; type MaterializeStream = diff --git a/crates/runtime-next/src/leader/shuffle.rs b/crates/runtime-next/src/leader/shuffle.rs new file mode 100644 index 00000000000..2e8ee8a6be7 --- /dev/null +++ b/crates/runtime-next/src/leader/shuffle.rs @@ -0,0 +1,86 @@ +//! Checkpoint sourcing for leader sessions. +//! +//! A leader obtains a sequence of checkpoint [`shuffle::Frontier`]s and, for +//! each, reads documents from already-written shuffle log segments up to that +//! Frontier (`shard/*/scan.rs`). Where those Frontiers come from is encapsulated +//! by [`ShuffleSession`]: production and live `flowctl preview` read source +//! journals via an in-process shuffle [`shuffle::SessionClient`]. +//! +//! ## Durability contract +//! +//! A Frontier yielded by [`ShuffleSession::recv_checkpoint`] must only reference +//! log content already durably written to the shard directories that the +//! runtime's shard scanners consume. The journal-reading session upholds this +//! (it completes segment flush IO before reporting progress); a fixture must +//! write its segment before feeding the matching Frontier. + +/// A source of checkpoint [`shuffle::Frontier`]s for one leader session. +pub trait ShuffleSession: Send + 'static { + /// Request the next checkpoint without awaiting it. At most one request is + /// outstanding at a time; pair with [`Self::recv_checkpoint`]. + fn request_checkpoint(&self); + + /// Await the Frontier responding to a prior [`Self::request_checkpoint`]. + /// + /// Cancel-safe: dropping the returned future before it resolves loses no + /// checkpoint, so it may be re-awaited across `select!` iterations. + fn recv_checkpoint( + &mut self, + ) -> impl std::future::Future> + Send; + + /// Cleanly close the session, draining any underlying topology to EOF. + fn close(self) -> impl std::future::Future> + Send; +} + +/// Opens a [`ShuffleSession`] for each leader session. +pub trait ShuffleSessionFactory: Send + Sync + 'static { + /// Concrete per-session shuffle session this factory produces. + type Session: ShuffleSession; + + fn open( + &self, + task: shuffle::proto::Task, + shards: Vec, + resume: shuffle::Frontier, + ) -> impl std::future::Future> + Send; +} + +/// The standard [`ShuffleSessionFactory`]: opens in-process journal-reading +/// shuffle sessions from a [`shuffle::Service`]. +pub struct ShuffleServiceFactory { + service: shuffle::Service, +} + +impl ShuffleServiceFactory { + pub fn new(service: shuffle::Service) -> Self { + Self { service } + } +} + +impl ShuffleSessionFactory for ShuffleServiceFactory { + type Session = shuffle::SessionClient; + + async fn open( + &self, + task: shuffle::proto::Task, + shards: Vec, + resume: shuffle::Frontier, + ) -> anyhow::Result { + shuffle::SessionClient::open(&self.service, task, shards, resume).await + } +} + +// A journal-reading Session is the canonical ShuffleSession. +impl ShuffleSession for shuffle::SessionClient { + fn request_checkpoint(&self) { + shuffle::SessionClient::request_checkpoint(self) + } + fn recv_checkpoint( + &mut self, + ) -> impl std::future::Future> + Send { + shuffle::SessionClient::recv_checkpoint(self) + } + fn close(self) -> impl std::future::Future> + Send { + shuffle::SessionClient::close(self) + } +} diff --git a/crates/runtime-next/src/lib.rs b/crates/runtime-next/src/lib.rs index e8a71e9353b..8b780ec8bfd 100644 --- a/crates/runtime-next/src/lib.rs +++ b/crates/runtime-next/src/lib.rs @@ -35,6 +35,7 @@ mod local_connector; mod tokio_context; pub mod leader; +pub mod logger; pub mod patches; pub mod publish; pub mod shard; @@ -42,8 +43,13 @@ mod task_service; pub use container::flow_runtime_protocol; -pub use leader::Service; -pub use publish::{Publisher, new_producer}; +pub use leader::{Service, ShuffleServiceFactory, ShuffleSession, ShuffleSessionFactory}; +pub use logger::{ + FnLogger, FnLoggerFactory, LogEvent, Logger, LoggerFactory, TracingLogger, TracingLoggerFactory, +}; +pub use publish::{ + JournalPublisher, JournalPublisherFactory, Publisher, PublisherFactory, new_producer, +}; pub use task_service::TaskService; pub use tokio_context::TokioContext; @@ -129,18 +135,44 @@ pub fn status_to_anyhow(status: tonic::Status) -> anyhow::Error { } } -pub trait LogHandler: Send + Sync + Clone + 'static { - fn log(&self, log: &ops::Log); - - fn as_fn(self) -> impl Fn(&ops::Log) + Send + Sync + 'static { - move |log| self.log(log) - } +/// Seed shard zero's RocksDB at `descriptor` with an initial connector state, +/// then close it. The `flowctl preview --initial-state` harness calls this +/// before handing the same path to the runtime via a `SessionLoop`, so the +/// runtime recovers the seeded state on its first scan exactly as if a prior +/// connector session had persisted it. Production never calls this. +/// +/// Kept as a free function rather than inlined into flowctl: doing so would +/// require exposing the whole crate-private [`shard::rocksdb::RocksDB`] API +/// (`open` / `scan` / `persist`) and its `DecodeError`. This narrow seam takes +/// only a `descriptor` and keeps that machinery encapsulated. +pub async fn seed_initial_connector_state( + descriptor: proto_flow::runtime::RocksDbDescriptor, + initial_state_json: &[u8], +) -> anyhow::Result<()> { + let db = shard::rocksdb::RocksDB::open(Some(descriptor)).await?; + let _db = db.put_connector_state_base(initial_state_json).await?; + Ok(()) } -impl LogHandler for T { - fn log(&self, log: &ops::Log) { - self(log) - } +/// Re-open shard zero's RocksDB at `descriptor` and return its reduced connector +/// state — the exact `Recover.connector_state_json` the runtime itself would +/// recover (empty if none was ever persisted). The `flowctl preview +/// --output-state` harness calls this after the session loop has closed the +/// runtime's own handle, to emit the run's final reduced state. Reuses the +/// normal recovery `scan`, so it stays consistent with how the runtime reads +/// state; production never calls this. +/// +/// Kept as a free function for the same reason as +/// [`seed_initial_connector_state`]: it narrows a full `scan` — which returns +/// the DB handle and a whole `Recover`, and can fail with the crate-private +/// `DecodeError` — down to just the reduced connector-state bytes flowctl +/// wants, without exposing `RocksDB` to flowctl. +pub async fn read_final_connector_state( + descriptor: proto_flow::runtime::RocksDbDescriptor, +) -> anyhow::Result { + let db = shard::rocksdb::RocksDB::open(Some(descriptor)).await?; + let (_db, recover) = db.scan(Vec::new()).await?; + Ok(recover.connector_state_json) } struct Accumulator(doc::combine::Accumulator, simd_doc::Parser); diff --git a/crates/runtime-next/src/local_connector.rs b/crates/runtime-next/src/local_connector.rs index c7c0871b2f6..4c48a5a454c 100644 --- a/crates/runtime-next/src/local_connector.rs +++ b/crates/runtime-next/src/local_connector.rs @@ -6,7 +6,7 @@ use tokio::sync::mpsc; pub fn serve( command: Vec, // Connector to run. env: BTreeMap, // Environment variables. - log_handler: impl crate::LogHandler, // Handler for connector logs. + logger: impl crate::Logger, // Logger for connector logs. log_level: ops::LogLevel, // Log-level of the container, if known. codec: connector_init::Codec, // Codec spoken by the connector. request_rx: mpsc::Receiver, // Caller's input request stream. @@ -22,11 +22,15 @@ where connector.env("LOG_FORMAT", "json"); connector.env("LOG_LEVEL", log_level.or(ops::LogLevel::Info).as_str_name()); + // Local connectors have no container lifecycle; the logger is used only as + // the connector log sink. + let log_sink = move |log: &ops::Log| logger.log(log); + let container_rx = connector_init::rpc::bidi::( connector, codec, tokio_stream::wrappers::ReceiverStream::new(request_rx).map(Result::Ok), - log_handler.clone().as_fn(), + log_sink, )?; Ok(container_rx) diff --git a/crates/runtime-next/src/logger.rs b/crates/runtime-next/src/logger.rs new file mode 100644 index 00000000000..568f777af67 --- /dev/null +++ b/crates/runtime-next/src/logger.rs @@ -0,0 +1,428 @@ +//! Out-of-band reporting seam: the task's log and event stream. +//! +//! This is the third host-facing seam of the crate, alongside [`Publisher`] +//! (document output) and [`ShuffleSession`] (checkpoint input). It carries the +//! task's *log stream* — everything that is eligible for surfacing to a human. +//! That stream has two sources: the connector's own logs ([`Logger::log`]), +//! and structured runtime [`LogEvent`]s ([`Logger::event`]) which flatten into +//! log records via their canonical [`LogEvent::to_log`] rendering. +//! +//! The membership test for adding a [`LogEvent`] variant is: *would this be a +//! line in the task's ops-log journal in production?* Events are structured +//! ops-log records that haven't been flattened yet — a host may intercept them +//! structurally (with their typed, verbatim payloads) before they degrade into +//! rendered log lines. Host-specific rendering (e.g. `flowctl preview`'s +//! `--output-state` / `--output-apply` lines) lives entirely in the host. +//! +//! The seam is deliberately distinct from two adjacent surfaces: +//! +//! - From `service_kit` (the [`Registry`](service_kit::Registry) + `event!` +//! macro), which is the *operator/admin* surface — in-flight handler phases, +//! breadcrumb rings, an admin dashboard. That never reaches user task logs; +//! this seam is the one that does. Likewise ad-hoc `tracing::*` diagnostics: +//! anything that need *not* reach the user's task logs stays plain tracing. +//! - From [`Publisher`], which emits the task's *data* (captured / derived +//! collection documents). A [`Logger`] reports *about* the runtime, not +//! the data it moves. +//! +//! The seam is generic, not dynamic: each leader / shard `Service` is +//! monomorphized over its concrete [`LoggerFactory`]. Every real installation +//! is an `ops::Log` sink, differing only in where logs go: +//! +//! - Production shards install an [`FnLoggerFactory`] wrapping the task's +//! encoded-JSON log writer: connector logs and flattened events both land in +//! the task-log file, which the Go runtime forwards to the task's ops-log +//! journal. +//! - Leaders and unit tests install [`TracingLoggerFactory`], which renders +//! logs as tracing events (sidecar stderr / journald). The leader sidecar's +//! tracing is *not* yet forwarded to task ops-logs; bridging that gap is an +//! Logger whose [`log`](Logger::log) publishes to the task's ops journal +//! asynchronously — a later change that needs only that one method. +//! - `flowctl preview` installs a Logger that intercepts the events it +//! renders to stdout ([`LogEvent::Persist`], [`LogEvent::Applied`]) and flattens +//! the rest to its chosen log handler. +//! +//! [`Publisher`]: crate::Publisher +//! [`ShuffleSession`]: crate::ShuffleSession + +use crate::proto; + +/// Per-session logger: the sink for the task's log and event stream. The +/// leader and shards obtain one from a [`LoggerFactory`] at the start of +/// each session. Cheap to clone (the connector log pump holds its own handle). +/// +/// Every method is synchronous and off the hot path; any async publication is +/// the implementation's internal concern (a background drain), never an `await` +/// at the call site. +pub trait Logger: Clone + Send + Sync + 'static { + /// Sink one log of the task's log stream: a connector log line, or a + /// runtime [`LogEvent`] flattened through [`LogEvent::to_log`]. Required (no + /// default) so no installer silently drops the stream. + fn log(&self, log: &ops::Log); + + /// Report a structured runtime [`LogEvent`]. The default flattens it into its + /// canonical log record and sinks it through [`log`](Logger::log) — + /// override only to intercept events structurally, and delegate unhandled + /// events to [`LogEvent::to_log`] to preserve their log-line rendering. + fn event(&self, event: LogEvent<'_>) { + if let Some(log) = event.to_log() { + self.log(&log); + } + } +} + +/// A structured runtime event: an ops-log record that hasn't been flattened +/// yet. Variants carry borrowed, verbatim payloads so a host can intercept +/// them structurally; [`LogEvent::to_log`] is their canonical log rendering. +/// +/// The enum and its variants are `#[non_exhaustive]`: new events and new +/// fields of existing events are non-breaking, so hosts must match with a +/// wildcard arm (delegating to [`LogEvent::to_log`]) and bind fields with `..`. +#[derive(Debug)] +#[non_exhaustive] +pub enum LogEvent<'a> { + /// A connector-state [`proto::Persist`] at the point it's emitted: the + /// leader's committing transaction and its Apply loop (derive / + /// materialize), and the capture shard's committing transaction and its + /// Apply loop. + #[non_exhaustive] + Persist { persist: &'a proto::Persist }, + + /// A connector Apply action description, once per Apply iteration as the + /// Apply loop converges (before any session + /// [`Publisher`](crate::Publisher) exists). + #[non_exhaustive] + Applied { action_description: &'a str }, + + /// A collection's inferred write-schema widened this transaction. + /// `binding` is the source binding index for captures (multiple bindings + /// per task) and `None` for derivations (a single derived collection). + /// `schema` is the representative JSON Schema of the widened write-shape, + /// as produced by [`doc::shape::schema::to_schema`]. + #[non_exhaustive] + InferredSchema { + collection_name: &'a str, + binding: Option, + schema: &'a schemars::Schema, + }, + + /// A connector container started and is dialed. Lower-level network / + /// codec detail is logged separately at debug by `container::start`. + #[non_exhaustive] + ContainerStarted { + image: &'a str, + container: &'a proto::Container, + }, + + /// A connector container is being torn down (its [`Guard`] was dropped at + /// session end or on error). + /// + /// [`Guard`]: crate::container::Guard + #[non_exhaustive] + ContainerStopped { image: &'a str }, + + /// A transient image-pull failure that will be retried. + #[non_exhaustive] + ImagePullRetry { + image: &'a str, + attempt: u32, + error: &'a str, + }, +} + +impl LogEvent<'_> { + /// Flatten this event into its canonical [`ops::Log`] — the single place + /// events render as task-log lines, matching the legacy runtime's lines. + /// Returns `None` when the event surfaces nothing (a [`LogEvent::Persist`] + /// carrying no connector-state delta: idempotent replays, ACK-only + /// persists, startup checkpoint reconciliation). + pub fn to_log(&self) -> Option { + let mut fields: Vec<(&str, bytes::Bytes)> = Vec::new(); + + let (level, message) = match self { + LogEvent::Persist { persist, .. } => { + if persist.connector_patches_json.is_empty() { + return None; + } + // The patch payload is valid JSON (a tab-delimited JSON array; + // see `crate::patches`), so it embeds verbatim. + fields.push(("patches", persist.connector_patches_json.clone())); + (ops::LogLevel::Debug, "persisted connector-state delta") + } + LogEvent::Applied { + action_description, .. + } => { + fields.push(("actionDescription", json_field(action_description))); + (ops::LogLevel::Info, "connector applied") + } + LogEvent::InferredSchema { + collection_name, + binding, + schema, + .. + } => { + fields.push(("collection", json_field(collection_name))); + if let Some(binding) = binding { + fields.push(("binding", json_field(binding))); + } + fields.push(("schema", json_field(schema))); + (ops::LogLevel::Info, "inferred schema updated") + } + LogEvent::ContainerStarted { + image, container, .. + } => { + fields.push(("image", json_field(image))); + fields.push(("container", json_field(container))); + (ops::LogLevel::Info, "started connector container") + } + LogEvent::ContainerStopped { image, .. } => { + fields.push(("image", json_field(image))); + (ops::LogLevel::Debug, "stopped connector container") + } + LogEvent::ImagePullRetry { + image, + attempt, + error, + .. + } => { + fields.push(("image", json_field(image))); + fields.push(("attempt", json_field(attempt))); + fields.push(("error", json_field(error))); + ( + ops::LogLevel::Warn, + "transient error pulling image (will retry)", + ) + } + }; + + Some(ops::Log { + meta: None, + shard: None, + timestamp: Some(proto_flow::as_timestamp(std::time::SystemTime::now())), + level: level as i32, + message: message.to_string(), + fields_json_map: fields + .into_iter() + .map(|(key, value)| (key.to_string(), value)) + .collect(), + spans: Vec::new(), + }) + } +} + +fn json_field(value: &impl serde::Serialize) -> bytes::Bytes { + serde_json::to_vec(value) + .expect("event field always serializes") + .into() +} + +/// Opens a [`Logger`] for each leader / shard session. Held by the leader +/// [`Service`](crate::leader::Service) and shard [`Service`](crate::shard::Service), +/// which are monomorphized over it. +pub trait LoggerFactory: Clone + Send + Sync + 'static { + /// Concrete per-session logger this factory produces. + type Logger: Logger; + + /// Open a [`Logger`] bound to the given task. `task_name` identifies the + /// task whose logs (and, in the future, ops-log journal) the logger + /// sinks; the `Fn` and tracing loggers ignore it. + fn open(&self, task_name: &str) -> Self::Logger; +} + +/// [`Logger`] whose [`log`](Logger::log) forwards to a `Fn(&ops::Log)`. +/// Events flatten through the default [`event`](Logger::event) into the same +/// `Fn`. The production shard install: the `Fn` is the task's encoded-JSON +/// log writer, so connector logs and runtime events both reach the task-log +/// file. +#[derive(Clone)] +pub struct FnLogger(F); + +impl Logger for FnLogger { + fn log(&self, log: &ops::Log) { + (self.0)(log) + } +} + +/// [`LoggerFactory`] producing [`FnLogger`]s. Each session's logger is a +/// clone of the wrapped log handler; the handler is shared, the per-session +/// logger is a cheap clone. +#[derive(Clone)] +pub struct FnLoggerFactory(F); + +impl FnLoggerFactory { + pub fn new(log_handler: F) -> Self { + Self(log_handler) + } +} + +impl LoggerFactory for FnLoggerFactory { + type Logger = FnLogger; + + fn open(&self, _task_name: &str) -> FnLogger { + FnLogger(self.0.clone()) + } +} + +/// [`Logger`] rendering the log stream as tracing events +/// ([`ops::tracing_log_handler`]). Installed by leaders — whose logs surface on +/// sidecar stderr until the async ops-log publishing Logger exists — and by +/// unit tests. +#[derive(Clone)] +pub struct TracingLogger; + +impl Logger for TracingLogger { + fn log(&self, log: &ops::Log) { + ops::tracing_log_handler(log); + } +} + +/// [`LoggerFactory`] opening [`TracingLogger`]s. The default install for +/// the leader `Service`. +#[derive(Clone)] +pub struct TracingLoggerFactory; + +impl LoggerFactory for TracingLoggerFactory { + type Logger = TracingLogger; + + fn open(&self, _task_name: &str) -> TracingLogger { + TracingLogger + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn event_to_log_renderings() { + // A Persist carrying no connector-state delta surfaces nothing. + assert!( + LogEvent::Persist { + persist: &proto::Persist::default() + } + .to_log() + .is_none() + ); + + let persist = proto::Persist { + connector_patches_json: b"[{\"cursor\":\"abc\"}\t,{\"cursor\":\"def\"}\t]" + .as_slice() + .into(), + ..Default::default() + }; + let schema: schemars::Schema = + serde_json::from_value(serde_json::json!({"type": "object"})).unwrap(); + let container = proto::Container { + ip_addr: "10.0.0.2".to_string(), + ..Default::default() + }; + let image = "ghcr.io/estuary/source-hello-world:dev"; + + let logs: Vec = [ + LogEvent::Persist { persist: &persist }, + LogEvent::Applied { + action_description: "create table \"foo\"", + }, + LogEvent::InferredSchema { + collection_name: "acmeCo/collection", + binding: Some(2), + schema: &schema, + }, + LogEvent::InferredSchema { + collection_name: "acmeCo/collection", + binding: None, + schema: &schema, + }, + LogEvent::ContainerStarted { + image, + container: &container, + }, + LogEvent::ContainerStopped { image }, + LogEvent::ImagePullRetry { + image, + attempt: 2, + error: "TLS handshake timeout", + }, + ] + .iter() + .map(|event| { + let mut log = event.to_log().unwrap(); + log.timestamp = None; // Stabilize the snapshot. + serde_json::to_value(&log).unwrap() + }) + .collect(); + + insta::assert_json_snapshot!(logs, @r#" + [ + { + "fields": { + "patches": [ + { + "cursor": "abc" + }, + { + "cursor": "def" + } + ] + }, + "level": "debug", + "message": "persisted connector-state delta" + }, + { + "fields": { + "actionDescription": "create table \"foo\"" + }, + "level": "info", + "message": "connector applied" + }, + { + "fields": { + "binding": 2, + "collection": "acmeCo/collection", + "schema": { + "type": "object" + } + }, + "level": "info", + "message": "inferred schema updated" + }, + { + "fields": { + "collection": "acmeCo/collection", + "schema": { + "type": "object" + } + }, + "level": "info", + "message": "inferred schema updated" + }, + { + "fields": { + "container": { + "ipAddr": "10.0.0.2" + }, + "image": "ghcr.io/estuary/source-hello-world:dev" + }, + "level": "info", + "message": "started connector container" + }, + { + "fields": { + "image": "ghcr.io/estuary/source-hello-world:dev" + }, + "level": "debug", + "message": "stopped connector container" + }, + { + "fields": { + "attempt": 2, + "error": "TLS handshake timeout", + "image": "ghcr.io/estuary/source-hello-world:dev" + }, + "level": "warn", + "message": "transient error pulling image (will retry)" + } + ] + "#); + } +} diff --git a/crates/runtime-next/src/publish.rs b/crates/runtime-next/src/publish.rs index c921f5e578b..e7efc7f8ff2 100644 --- a/crates/runtime-next/src/publish.rs +++ b/crates/runtime-next/src/publish.rs @@ -1,69 +1,135 @@ -//! Publishing surface used by leader actors. +//! Output publisher used by leader and shard actors. //! -//! `Publisher` is the unified entry point. Two variants: +//! Two traits form the seam between runtime-next and how a given deployment +//! emits its output documents. Production installs [`JournalPublisherFactory`], +//! which performs Gazette journal IO. An in-process harness (`flowctl preview`) +//! installs its own factory that writes documents to stdout. This crate is +//! unaware of either context: it always [`PublisherFactory::open`]s a +//! [`Publisher`], and the installed implementation decides what that means. //! -//! - `Publisher::Real` wraps a real `publisher::Publisher` and performs -//! Gazette journal IO (stats / logs / ACK intents / future capture & -//! derive collection writes). -//! - `Publisher::Preview` performs no journal IO. Stats and log documents -//! are emitted as `tracing::info!` events. Captured documents are written -//! as NDJSON to stdout, one JSON object per line. +//! The seam is generic, not dynamic: a leader / shard `Service` is monomorphized +//! over its concrete [`PublisherFactory`] (`JournalPublisherFactory` in +//! production), so the hot `publish_doc` path is a static call with no vtable or +//! boxed future. The associated [`PublisherFactory::Publisher`] type carries the +//! concrete `Publisher` through actors and drains. //! -//! Construction is decided in `startup::run` based on the `preview` flag in -//! `L:Task`: `false` ⇒ `Real`, `true` ⇒ `Preview`. The leader actor parks the -//! `Publisher` across IO futures. +//! - [`Publisher`] is the per-session output publisher. The leader publishes +//! stats and ACK intents through it; shards additionally publish captured / +//! derived collection documents. +//! - [`PublisherFactory`] is the long-lived object held by each leader / shard +//! `Service`. It opens a [`Publisher`] per session. +//! +//! Runtime *events* (connector-state persists, Apply actions, ...) are reported +//! through the separate [`Logger`](crate::logger) seam, not this one. use bytes::Bytes; use proto_gazette::uuid; use std::collections::BTreeMap; -use std::io::Write as _; -/// Publishing entity used by leader sessions and runtime-next shards. -/// `crate::publish::Publisher` is the operative publisher from the -/// perspective of the `leader` and `runtime-next` crates; the inner -/// `publisher::Publisher` is an implementation detail of the `Real` -/// variant. -/// -/// Methods mirror the subset of `publisher::Publisher` the leader actor -/// uses, plus `publish_stats` which factors out the actor's -/// stats-enqueue-then-flush idiom. The `Real` arm delegates to -/// `publisher::Publisher`; the `Preview` arm performs no IO. -pub enum Publisher { - Real(publisher::Publisher), - Preview { - /// Collection names indexed by binding. - collection_names: Vec, - /// Accumulates complete `["",]\n` lines. Flushed to stdout - /// with a single atomic `write_all` once it crosses [`PREVIEW_FLUSH_THRESHOLD`] - /// or at transaction commit. Preview spawns one publisher per shard, all - /// writing the process-global stdout `LineWriter`; flushing whole lines - /// under the stdout lock keeps shards' output from splicing together. - line_buf: Vec, - }, +/// Per-session output publisher. The leader and shards obtain one from a +/// [`PublisherFactory`] at the start of each session, and park it across IO +/// futures. Async methods are return-position `impl Future + Send` (not +/// `dyn`-dispatched), so a concrete `Publisher` is called statically. +pub trait Publisher: Send + 'static { + /// Advance the publisher's clock to the current wall-clock time. + /// + /// Called once at the start of each transaction's stream of published + /// documents (the shard drain, or the leader stats + ACK write) so that + /// stamped UUIDs reflect the transaction's write time and then tick up + /// minimally between documents. The clock is monotonic, so this never + /// regresses below a prior written clock. + fn update_clock(&mut self); + + /// Enqueue and flush a single stats document as a `CONTINUE_TXN`. + fn publish_stats( + &mut self, + stats: ops::proto::Stats, + ) -> impl std::future::Future> + Send; + + /// Enqueue one captured or derived collection document. `binding_index` + /// is zero-based within the task bindings. Returns the serialized document + /// byte length, excluding any framing bytes. + fn publish_doc( + &mut self, + binding_index: usize, + doc: doc::OwnedNode, + uuid_ptr: &json::Pointer, + ) -> impl std::future::Future> + Send; + + /// Flush all currently buffered documents. + fn flush(&mut self) -> impl std::future::Future> + Send; + + /// Snapshot this publisher's contribution to the current transaction's + /// ACK intents, or `None` when no real publishes happened (so there are no + /// commit positions to encode). + fn commit_intents(&mut self) -> Option<(uuid::Producer, uuid::Clock, Vec)>; + + /// Write per-journal ACK intent documents to their journals. + fn write_intents( + &mut self, + journal_intents: BTreeMap, + ) -> impl std::future::Future> + Send; + + /// Take accumulated per-journal append-throttle samples since the last call. + fn take_throttle_samples(&mut self) -> Vec>; + + /// Build a detached future which attempts to split partition `journal` at + /// its key-range midpoint. Returns `None` when `journal` is not a partition + /// of any Mapped binding (e.g. the fixed ops-stats journal) — such journals + /// can never be split — or when the publisher performs no real journal IO. + fn split_partition( + &self, + journal: &str, + ) -> Option>>; } -/// Flush `Publisher::Preview`'s `line_buf` to stdout once it reaches this many -/// bytes. Sized to amortize the stdout lock + `write(2)` across many documents -/// while bounding buffered memory. -const PREVIEW_FLUSH_THRESHOLD: usize = 32 * 1024; +/// Opens a [`Publisher`] for each leader / shard session. Held by the leader +/// [`Service`](crate::leader::Service) and shard [`Service`](crate::shard::Service), +/// which are monomorphized over it. Production installs [`JournalPublisherFactory`]. +pub trait PublisherFactory: Clone + Send + Sync + 'static { + /// Concrete per-session publisher this factory produces. + type Publisher: Publisher; + + /// Open a [`Publisher`] for the given task bindings. `collection_specs` are + /// the capture / derive collection bindings (empty for a leader's + /// stats-only publisher); `stats_journal` is the fixed ops-stats binding. + /// `authz_subject` and `producer` identify the publisher. + fn open( + &self, + authz_subject: String, + producer: uuid::Producer, + stats_journal: &str, + collection_specs: &[&proto_flow::flow::CollectionSpec], + ) -> anyhow::Result; +} + +/// Production [`PublisherFactory`]: opens [`JournalPublisher`]s that perform +/// Gazette journal IO. +#[derive(Clone)] +pub struct JournalPublisherFactory { + client_factory: gazette::journal::ClientFactory, +} -impl Publisher { - /// Build a real `Publisher` backed by a `publisher::Publisher` for the - /// pre-created `ops_stats_journal` plus any additional supplied collection - /// specs. `producer` is chosen by the caller (see [`new_producer`]). - pub fn new_real<'a, I>( +impl JournalPublisherFactory { + pub fn new(client_factory: gazette::journal::ClientFactory) -> Self { + Self { client_factory } + } +} + +impl PublisherFactory for JournalPublisherFactory { + type Publisher = JournalPublisher; + + fn open( + &self, authz_subject: String, producer: uuid::Producer, - client_factory: &gazette::journal::ClientFactory, - ops_stats_journal: &str, - collection_specs: I, - ) -> anyhow::Result - where - I: IntoIterator, - { - let mut bindings = Vec::new(); + stats_journal: &str, + collection_specs: &[&proto_flow::flow::CollectionSpec], + ) -> anyhow::Result { + let mut bindings = Vec::with_capacity(collection_specs.len() + 1); - bindings.push(publisher::Binding::for_fixed_journal(ops_stats_journal)); + // Binding zero is the fixed ops-stats journal. + bindings.push(publisher::Binding::for_fixed_journal(stats_journal)); for spec in collection_specs { bindings.push(publisher::Binding::from_collection_spec(spec)?); @@ -72,221 +138,135 @@ impl Publisher { let mut publisher = publisher::Publisher::new( authz_subject, bindings, - client_factory.clone(), + self.client_factory.clone(), producer, uuid::Clock::zero(), ); publisher.update_clock(); - Ok(Self::Real(publisher)) + Ok(JournalPublisher(publisher)) } +} - /// Build a preview publisher that performs no journal IO. Stats are emitted - /// to `tracing::info!`; captured documents are written to stdout. - pub fn new_preview<'a, I>(collection_specs: I) -> Self - where - I: IntoIterator, - { - Self::Preview { - collection_names: collection_specs - .into_iter() - .map(|s| s.name.clone()) - .collect(), - line_buf: Vec::new(), - } - } - - /// Build a real `Publisher` over clients of an unreachable local endpoint - #[cfg(test)] - pub(crate) fn new_test_real<'a, I>(collection_specs: I) -> Self - where - I: IntoIterator, - { - let fragment_client = gazette::journal::Client::new_fragment_client(); - let factory: gazette::journal::ClientFactory = - std::sync::Arc::new(move |_subject, _object| { - gazette::journal::Client::new( - "http://localhost:0".to_string(), - fragment_client.clone(), - proto_grpc::Metadata::new(), - gazette::Router::new("local"), - ) - }); - Self::new_real( - "test".to_string(), - new_producer(), - &factory, - "test/ops/stats", - collection_specs, - ) - .unwrap() +/// Production [`Publisher`]: wraps a [`publisher::Publisher`] and performs +/// Gazette journal IO. The inner `publisher::Publisher` is an implementation +/// detail; from the leader / shard perspective the operative publisher is the +/// [`Publisher`] trait. +pub struct JournalPublisher(publisher::Publisher); + +impl JournalPublisher { + /// Access the wrapped [`publisher::Publisher`] for low-level enqueues. + /// Used only by the `split_e2e` integration test, which drives raw appends + /// against a live broker; not part of the leader / shard hot path. + #[doc(hidden)] + pub fn inner_mut(&mut self) -> &mut publisher::Publisher { + &mut self.0 } +} - /// Advance the publisher's clock to the current wall-clock time. - /// - /// Called once at the start of each transaction's stream of published - /// documents (the shard drain, or the leader stats + ACK write) so that - /// stamped UUIDs reflect the transaction's write time and then tick up - /// minimally between documents. The clock is monotonic, so this never - /// regresses below a prior written clock. No-op in preview mode. - pub fn update_clock(&mut self) { - match self { - Self::Real(p) => p.update_clock(), - Self::Preview { .. } => {} - } +impl Publisher for JournalPublisher { + fn update_clock(&mut self) { + self.0.update_clock() } - /// Enqueue and flush a single stats document as a `CONTINUE_TXN`. - /// - /// This consolidates the leader actor's prior "enqueue stats, then - /// flush" pattern into one method so the parking pattern stays - /// symmetric across `Real` and `Preview` arms. - pub async fn publish_stats(&mut self, mut stats: ops::proto::Stats) -> tonic::Result<()> { - match self { - Self::Real(p) => { - p.enqueue( - |uuid| { - // Binding index 0 is the fixed ops_stats journal. - let meta = stats.meta.as_mut().ok_or_else(|| { - tonic::Status::internal("stats document is missing required `meta`") - })?; - meta.uuid = uuid.to_string(); - - let value = serde_json::to_value(&stats).map_err(|err| { - tonic::Status::internal(format!("serializing stats document: {err}")) - })?; - Ok((0, value)) - }, - uuid::Flags::CONTINUE_TXN, - ) - .await?; - p.flush().await - } - Self::Preview { .. } => { - tracing::info!(stats = ?ops::DebugJson(stats), "transaction stats"); - Ok(()) - } - } + async fn publish_stats(&mut self, mut stats: ops::proto::Stats) -> tonic::Result<()> { + self.0 + .enqueue( + |uuid| { + // Binding index 0 is the fixed ops_stats journal. + let meta = stats.meta.as_mut().ok_or_else(|| { + tonic::Status::internal("stats document is missing required `meta`") + })?; + meta.uuid = uuid.to_string(); + + let value = serde_json::to_value(&stats).map_err(|err| { + tonic::Status::internal(format!("serializing stats document: {err}")) + })?; + Ok((0, value)) + }, + uuid::Flags::CONTINUE_TXN, + ) + .await?; + self.0.flush().await } - /// Enqueue one captured or derived collection document. `binding_index` - /// is zero-based within the task bindings; binding zero of the underlying - /// publisher is reserved for the fixed ops stats journal. Returns the - /// serialized document byte length, excluding the framing bytes. - pub async fn publish_doc( + async fn publish_doc( &mut self, binding_index: usize, mut doc: doc::OwnedNode, uuid_ptr: &json::Pointer, ) -> tonic::Result { - match self { - Self::Real(p) => { - let publisher_binding = binding_index + 1; - let (_, bytes_written) = p - .enqueue_owned( - |uuid| { - patch_document_uuid(&mut doc, uuid_ptr, uuid)?; - Ok((publisher_binding, doc)) - }, - uuid::Flags::CONTINUE_TXN, - ) - .await?; - Ok(bytes_written) - } - Self::Preview { - collection_names, - line_buf, - } => { - let collection_name = &collection_names[binding_index]; - write!(line_buf, "[{collection_name:?},").unwrap(); - - // Serialize the body directly into the line buffer, sampling its - // length to report body bytes (excluding framing). Serializing a - // valid OwnedNode cannot fail, so the body can never be left - // partially written ahead of a flush. - let body_start = line_buf.len(); - serde_json::to_writer(&mut *line_buf, &doc::SerPolicy::noop().on_owned(&doc)) - .unwrap(); - let body_len = line_buf.len() - body_start; - - line_buf.extend_from_slice(b"]\n"); - - // Flush whole lines under the stdout lock in a single atomic - // write_all so concurrent shards' output can't splice together. - if line_buf.len() >= PREVIEW_FLUSH_THRESHOLD { - std::io::stdout().write_all(line_buf).unwrap(); - line_buf.clear(); - } - Ok(body_len) - } - } + // Publisher binding zero is reserved for the fixed ops stats journal. + let publisher_binding = binding_index + 1; + let (_, bytes_written) = self + .0 + .enqueue_owned( + |uuid| { + patch_document_uuid(&mut doc, uuid_ptr, uuid)?; + Ok((publisher_binding, doc)) + }, + uuid::Flags::CONTINUE_TXN, + ) + .await?; + Ok(bytes_written) } - /// Flush all currently buffered documents. - pub async fn flush(&mut self) -> tonic::Result<()> { - match self { - Self::Real(p) => p.flush().await, - Self::Preview { line_buf, .. } => { - if !line_buf.is_empty() { - std::io::stdout().write_all(line_buf).unwrap(); - line_buf.clear(); - } - Ok(()) - } - } + async fn flush(&mut self) -> tonic::Result<()> { + self.0.flush().await } - /// Take accumulated per-journal append-throttle samples since the last call. - pub fn take_throttle_samples(&mut self) -> Vec> { - match self { - Self::Real(p) => p.take_throttle_samples(), - Self::Preview { .. } => Vec::new(), - } + fn commit_intents(&mut self) -> Option<(uuid::Producer, uuid::Clock, Vec)> { + Some(self.0.commit_intents()) } - /// Build a detached future which attempts to split partition `journal` at - /// its key-range midpoint. Delegates to [`publisher::Publisher`], which - /// owns the bindings and journal clients; see its `split_partition`. - /// - /// Returns None in preview mode, or when `journal` is not a partition of - /// any Mapped binding (e.g. the fixed ops-stats journal) — such journals - /// can never be split. - pub fn split_partition( + fn take_throttle_samples(&mut self) -> Vec> { + self.0.take_throttle_samples() + } + + fn split_partition( &self, journal: &str, ) -> Option>> { - match self { - Self::Real(p) => p.split_partition(journal), - Self::Preview { .. } => None, - } + self.0.split_partition(journal) } - /// Snapshot this producer's contribution to the current transaction's - /// ACK intents. In preview mode, returns an empty list — no real - /// publishes happened, so there are no commit positions to encode. - pub fn commit_intents(&mut self) -> Option<(uuid::Producer, uuid::Clock, Vec)> { - match self { - Self::Real(p) => Some(p.commit_intents()), - Self::Preview { .. } => None, - } - } - - /// Write per-journal ACK intent documents to their journals. - /// No-op in preview mode (intents are necessarily empty). - pub async fn write_intents( + async fn write_intents( &mut self, journal_intents: BTreeMap, ) -> tonic::Result<()> { - match self { - Self::Real(p) => p.write_intents(journal_intents).await, - Self::Preview { .. } => { - debug_assert!( - journal_intents.is_empty(), - "Publisher::Preview received non-empty ACK intents", - ); - Ok(()) - } - } + self.0.write_intents(journal_intents).await + } +} + +#[cfg(test)] +impl JournalPublisher { + /// Build a real `JournalPublisher` over clients of an unreachable local + /// endpoint, for tests that exercise real publisher plumbing (e.g. partition + /// splitting) without a live Gazette. + pub(crate) fn new_test_real<'a, I>(collection_specs: I) -> Self + where + I: IntoIterator, + { + let fragment_client = gazette::journal::Client::new_fragment_client(); + let factory: gazette::journal::ClientFactory = + std::sync::Arc::new(move |_subject, _object| { + gazette::journal::Client::new( + "http://localhost:0".to_string(), + fragment_client.clone(), + proto_grpc::Metadata::new(), + gazette::Router::new("local"), + ) + }); + let collection_specs: Vec<&proto_flow::flow::CollectionSpec> = + collection_specs.into_iter().collect(); + JournalPublisherFactory::new(factory) + .open( + "test".to_string(), + new_producer(), + "test/ops/stats", + &collection_specs, + ) + .unwrap() } } @@ -364,16 +344,64 @@ fn missing_uuid_placeholder(uuid_ptr: &json::Pointer) -> tonic::Status { )) } +/// Test [`Publisher`] performing no journal IO: the in-crate analogue of the +/// preview harness's publisher, letting actor tests run without Gazette. +#[cfg(test)] +pub(crate) struct NoopPublisher; + +#[cfg(test)] +impl Publisher for NoopPublisher { + fn update_clock(&mut self) {} + + async fn publish_stats(&mut self, _stats: ops::proto::Stats) -> tonic::Result<()> { + Ok(()) + } + + async fn publish_doc( + &mut self, + _binding_index: usize, + _doc: doc::OwnedNode, + _uuid_ptr: &json::Pointer, + ) -> tonic::Result { + Ok(0) + } + + async fn flush(&mut self) -> tonic::Result<()> { + Ok(()) + } + + fn commit_intents(&mut self) -> Option<(uuid::Producer, uuid::Clock, Vec)> { + None + } + + async fn write_intents( + &mut self, + _journal_intents: BTreeMap, + ) -> tonic::Result<()> { + Ok(()) + } + + fn take_throttle_samples(&mut self) -> Vec> { + Vec::new() + } + + fn split_partition( + &self, + _journal: &str, + ) -> Option>> { + None + } +} + #[cfg(test)] mod test { use super::*; #[test] - fn preview_take_throttle_samples_is_empty() { - // The auto-split signal path stays inert in preview mode: no journal IO - // happens, so there are no throttle samples to surface. - let mut publisher = - Publisher::new_preview(std::iter::empty::<&proto_flow::flow::CollectionSpec>()); + fn noop_take_throttle_samples_is_empty() { + // The auto-split signal path stays inert without journal IO: the + // NoopPublisher performs no appends, so there are no throttle samples. + let mut publisher = NoopPublisher; assert!(publisher.take_throttle_samples().is_empty()); } diff --git a/crates/runtime-next/src/shard/capture/actor.rs b/crates/runtime-next/src/shard/capture/actor.rs index a35be94495f..6cfa977ceea 100644 --- a/crates/runtime-next/src/shard/capture/actor.rs +++ b/crates/runtime-next/src/shard/capture/actor.rs @@ -23,18 +23,20 @@ use tokio::sync::mpsc; /// resource (`db`, `publisher`, ...) is `None` exactly while its future runs, /// and is restored when that future completes — the "parking" pattern shared /// with the materialize leader actor. -pub(super) struct Actor { +pub(super) struct Actor { // --- Task and IO endpoints, fixed for the session. --- // `task` is shared (Arc) so the drain future can hold its own handle. task: std::sync::Arc, connector_tx: mpsc::Sender, // Per-session metrics counters. metrics: super::Metrics, + // Logger of task-centric state changes and events. + logger: L, // --- Parked resources: `Some` unless borrowed by an in-flight future. --- // RocksDB is parked with its per-binding state keys. db: Option<(crate::shard::RocksDB, Vec)>, - publisher: Option, + publisher: Option

, // Inferred per-binding write-shapes. Seeded from prior sessions at // construction, parked into the drain future, handed back at session end. shapes: Option>, @@ -46,12 +48,11 @@ pub(super) struct Actor { // --- In-flight IO futures; `None` when idle. --- acknowledge_fut: Option>>, - drain_fut: Option>>, - intents_write_fut: Option>>, + drain_fut: Option>>>, + intents_write_fut: Option>>, persist_fut: Option)>>>, split_fut: Option, - stats_write_fut: - Option)>>>, + stats_write_fut: Option)>>>, // --- Hand-offs staged between FSM steps. --- // Drain output, staged for `TailDrain`. @@ -67,13 +68,14 @@ struct DrainInput { parser: simd_doc::Parser, } -impl Actor { +impl Actor { pub fn new( binding_state_keys: Vec, connector_tx: mpsc::Sender, db: crate::shard::RocksDB, metrics: super::Metrics, - publisher: crate::Publisher, + logger: L, + publisher: P, shapes: Vec, task: std::sync::Arc, ) -> Self { @@ -81,6 +83,7 @@ impl Actor { task, connector_tx, metrics, + logger, db: Some((db, binding_state_keys)), publisher: Some(publisher), shapes: Some(shapes), @@ -98,16 +101,16 @@ impl Actor { } #[tracing::instrument(level = "debug", err(Debug, level = "warn"), skip_all)] - pub async fn serve( + pub async fn serve( mut self, - connector_rx: C, - controller_rx: &mut R, + connector_rx: Conn, + controller_rx: &mut Ctrl, mut head: fsm::Head, mut tail: fsm::Tail, ) -> anyhow::Result<(crate::shard::RocksDB, Vec)> where - R: futures::Stream> + Send + Unpin + 'static, - C: futures::Stream> + Send + Unpin + 'static, + Ctrl: futures::Stream> + Send + Unpin + 'static, + Conn: futures::Stream> + Send + Unpin + 'static, { let mut connector_rx = std::pin::pin!(connector_rx); @@ -231,7 +234,7 @@ impl Actor { // Prioritize completions of Tail IO first. Some(result) = maybe_fut(&mut self.drain_fut) => { - let output : drain::Output = result?; + let output: drain::Output

= result?; accumulator_idle = Some(output.accumulator); self.publisher = Some(output.publisher); self.shapes = Some(output.shapes); @@ -399,6 +402,7 @@ impl Actor { let shapes = self.shapes.take().context("missing capture shape state")?; let task = std::sync::Arc::clone(&self.task); let metrics = self.metrics.clone(); + let logger = self.logger.clone(); self.drain_fut = Some( async move { drain::drain_and_publish( @@ -409,6 +413,7 @@ impl Actor { sourced_schemas, shapes, metrics, + logger, ) .await } @@ -441,6 +446,9 @@ impl Actor { } fsm::Action::Persist { persist } => { + self.logger + .event(crate::LogEvent::Persist { persist: &persist }); + let (db, binding_state_keys) = self.db.take().context("Persist while RocksDB is busy")?; self.persist_fut = Some( @@ -677,7 +685,7 @@ mod tests { } /// Drive `Actor::serve` end-to-end over mpsc channels standing in for the - /// connector and controller, with a real RocksDB and a preview Publisher. + /// connector and controller, with a real RocksDB. /// /// The connector emits two Captured documents (into distinct bindings) and a /// Checkpoint carrying connector state. The actor accumulates them, closes @@ -699,16 +707,6 @@ mod tests { mpsc::unbounded_channel::>(); let task = std::sync::Arc::new(mk_task(true)); - // Preview only reads each spec's `name`; a minimal spec per binding suffices. - let collection_specs: Vec = task - .bindings - .iter() - .map(|b| flow::CollectionSpec { - name: b.collection_name.clone(), - ..Default::default() - }) - .collect(); - let publisher = crate::Publisher::new_preview(collection_specs.iter()); let shapes = task.binding_shapes_by_index(Default::default()); let actor = Actor::new( @@ -716,7 +714,8 @@ mod tests { connector_tx, crate::shard::RocksDB::open(None).await.unwrap(), super::super::Metrics::new("test/shard"), - publisher, + crate::TracingLogger, + crate::publish::NoopPublisher, shapes, task, ); @@ -792,7 +791,7 @@ mod tests { }), ..Default::default() }; - let publisher = crate::Publisher::new_test_real([&spec]); + let publisher = crate::JournalPublisher::new_test_real([&spec]); let shapes = task.binding_shapes_by_index(Default::default()); let mut actor = Actor::new( @@ -800,6 +799,7 @@ mod tests { connector_tx, crate::shard::RocksDB::open(None).await.unwrap(), super::super::Metrics::new("test/shard"), + crate::TracingLogger, publisher, shapes, task, diff --git a/crates/runtime-next/src/shard/capture/connector.rs b/crates/runtime-next/src/shard/capture/connector.rs index c9c1213bb8e..0a7f4be2375 100644 --- a/crates/runtime-next/src/shard/capture/connector.rs +++ b/crates/runtime-next/src/shard/capture/connector.rs @@ -9,8 +9,9 @@ use tokio_stream::wrappers::ReceiverStream; use unseal; use zeroize::Zeroize; -pub async fn start( - service: &crate::shard::Service, +pub async fn start( + service: &crate::shard::Service, + logger: &L::Logger, log_level: ops::LogLevel, mut initial: Request, ) -> anyhow::Result<( @@ -44,7 +45,7 @@ pub async fn start( // Captures don't have conditional JSON fields, so _codec is unused. let (rx, container, _codec) = crate::image_connector::serve( image, - service.log_handler.clone(), + logger.clone(), log_level, &service.container_network, connector_rx, @@ -78,7 +79,7 @@ pub async fn start( let rx = crate::local_connector::serve( command, env, - service.log_handler.clone(), + logger.clone(), log_level, codec, connector_rx, diff --git a/crates/runtime-next/src/shard/capture/drain.rs b/crates/runtime-next/src/shard/capture/drain.rs index 070971bf4e4..2047ae41443 100644 --- a/crates/runtime-next/src/shard/capture/drain.rs +++ b/crates/runtime-next/src/shard/capture/drain.rs @@ -24,28 +24,29 @@ use std::collections::{BTreeMap, BTreeSet}; const SOURCED_SCHEMA_COMPLEXITY_LIMIT: usize = 10_000; /// Resources and results handed back to the actor when a drain completes. -pub(super) struct Output { +pub(super) struct Output { /// The drained combiner, recycled as the next transaction's `idle_accumulator`. pub(super) accumulator: crate::Accumulator, /// Per-transaction connector patches and stats, staged for the TailFSM. pub(super) drained: fsm::DrainedCapture, /// The publisher, borrowed for the drain's journal appends. - pub(super) publisher: crate::Publisher, + pub(super) publisher: P, /// Per-binding inferred write-shapes, carried across sessions of the shard. pub(super) shapes: Vec, } /// Drain a rotated combiner: apply sourced schemas to inference, publish each /// captured document, and accumulate the connector-state patch stream. -pub(super) async fn drain_and_publish( +pub(super) async fn drain_and_publish( mut drainer: doc::combine::Drainer, parser: simd_doc::Parser, - mut publisher: crate::Publisher, + mut publisher: P, task: std::sync::Arc, sourced_schemas: BTreeMap, mut shapes: Vec, metrics: super::Metrics, -) -> anyhow::Result { + logger: L, +) -> anyhow::Result> { // Resync the publisher clock to wall-clock time at the start of this // transaction's stream of published documents. Each `publish_doc` and the // closing `commit_intents` then tick it up by a single microsecond, so @@ -125,13 +126,12 @@ pub(super) async fn drain_and_publish( // `to_schema` emits the shape's annotations, including the // `x-complexity-limit` set by `apply_sourced_schemas` or the // per-session default seeded by `Task::binding_shapes_by_index`. - let serialized = doc::shape::schema::to_schema(shapes[*binding].clone()); - tracing::info!( - schema = ?ops::DebugJson(serialized), - collection_name = %task.bindings[*binding].collection_name, - binding, - "inferred schema updated" - ); + let schema = doc::shape::schema::to_schema(shapes[*binding].clone()); + logger.event(crate::LogEvent::InferredSchema { + collection_name: &task.bindings[*binding].collection_name, + binding: Some(*binding), + schema: &schema, + }); metrics.inferred_schema_updates.increment(1); } diff --git a/crates/runtime-next/src/shard/capture/handler.rs b/crates/runtime-next/src/shard/capture/handler.rs index 5c00015ccbd..c5cd0ca11fa 100644 --- a/crates/runtime-next/src/shard/capture/handler.rs +++ b/crates/runtime-next/src/shard/capture/handler.rs @@ -1,4 +1,5 @@ use super::connector; +use crate::Logger as _; use crate::leader::capture::fsm; use crate::proto; use anyhow::Context; @@ -9,8 +10,8 @@ use std::collections::BTreeMap; use tokio::sync::mpsc; use tracing::Instrument; -pub(crate) async fn serve( - service: crate::shard::Service, +pub(crate) async fn serve( + service: crate::shard::Service, mut controller_rx: R, controller_tx: mpsc::UnboundedSender>, ) -> anyhow::Result<()> @@ -100,16 +101,18 @@ where Ok(()) } -async fn serve_unary( - service: &crate::shard::Service, +async fn serve_unary( + service: &crate::shard::Service, request: capture::Request, log_level: ops::LogLevel, ) -> anyhow::Result { let is_spec = request.spec.is_some(); let is_discover = request.discover.is_some(); let is_validate = request.validate.is_some(); + + let logger = service.logger_factory.open(&service.task_name); let (connector_tx, mut connector_rx, _container) = - connector::start(service, log_level, request).await?; + connector::start(service, &logger, log_level, request).await?; std::mem::drop(connector_tx); let verify = crate::verify("Capture", "unary response", "connector"); @@ -140,8 +143,8 @@ async fn serve_unary( Ok(response) } -async fn serve_session_loop( - service: &crate::shard::Service, +async fn serve_session_loop( + service: &crate::shard::Service, controller_rx: &mut R, controller_tx: &mpsc::UnboundedSender>, session_loop: proto::SessionLoop, @@ -184,8 +187,8 @@ where Ok(()) } -async fn serve_session( - service: &crate::shard::Service, +async fn serve_session( + service: &crate::shard::Service, controller_rx: &mut R, controller_tx: &mpsc::UnboundedSender>, db: crate::shard::RocksDB, @@ -215,8 +218,8 @@ where .await } -async fn serve_session_inner( - service: &crate::shard::Service, +async fn serve_session_inner( + service: &crate::shard::Service, controller_rx: &mut R, controller_tx: &mpsc::UnboundedSender>, db: crate::shard::RocksDB, @@ -253,6 +256,7 @@ where handler.set_field("etcd_mod_revision", join.etcd_mod_revision); handler.set_phase("joined"); + let logger = service.logger_factory.open(&service.task_name); let metrics = super::Metrics::new(&shard_id); _ = controller_tx.send(Ok(proto::Capture { @@ -267,7 +271,6 @@ where let verify = crate::verify("Capture", "Task", "controller"); let proto::Task { spec, - preview, max_transactions, sqlite_vfs_uri: _, publisher_id: _, // Captures are leaderless; the shard's own producer is used. @@ -302,7 +305,7 @@ where db = db.seed_connector_state(&mut recover).await?; let proto::Recover { ack_intents, - connector_state_json, + mut connector_state_json, last_applied, .. } = recover; @@ -314,9 +317,9 @@ where // unchanged-spec short-circuit compares like for like — independent of how // the controller (Go gogoproto) happened to frame `Task.spec`. let next_applied = bytes::Bytes::from(spec.encode_to_vec()); - let mut connector_state_json = connector_state_json; db = apply_loop( service, + &logger, db, &binding_state_keys, &last_applied, @@ -337,7 +340,7 @@ where ..Default::default() }; let (connector_tx, mut connector_rx, container) = - connector::start(service, log_level, open.clone()).await?; + connector::start(service, &logger, log_level, open.clone()).await?; let verify = crate::verify("Capture", "Opened", "connector"); let opened = match verify.not_eof(connector_rx.next().await)? { capture::Response { @@ -355,19 +358,20 @@ where max_transactions, )?); - let collection_specs = spec.bindings.iter().filter_map(|b| b.collection.as_ref()); - let publisher = if preview { - crate::Publisher::new_preview(collection_specs) - } else { - crate::Publisher::new_real( + let collection_specs: Vec<&flow::CollectionSpec> = spec + .bindings + .iter() + .filter_map(|b| b.collection.as_ref()) + .collect(); + let publisher = service + .publisher_factory + .open( shard_id, producer, - &service.publisher_factory, &labeling.stats_journal, - collection_specs, + &collection_specs, ) - .context("creating publisher")? - }; + .context("opening publisher")?; _ = controller_tx.send(Ok(proto::Capture { opened: Some(proto::capture::Opened { container }), @@ -395,6 +399,7 @@ where connector_tx, db, metrics, + logger, publisher, shapes, task.clone(), @@ -423,8 +428,9 @@ where /// against partially-advanced state — the connector's Apply must be idempotent /// across repeated invocations of the same target spec (see the `C:Apply` proto /// comment). -async fn apply_loop( - service: &crate::shard::Service, +async fn apply_loop( + service: &crate::shard::Service, + logger: &L::Logger, mut db: crate::shard::RocksDB, binding_state_keys: &[String], last_applied: &bytes::Bytes, @@ -465,6 +471,7 @@ async fn apply_loop( let (connector_tx, mut connector_rx, _container) = connector::start( service, + logger, log_level, capture::Request { apply: Some(apply), @@ -492,6 +499,10 @@ async fn apply_loop( }; verify.eof(connector_rx.next().await)?; + logger.event(crate::LogEvent::Applied { + action_description: &action_description, + }); + service_kit::event!( tracing::Level::INFO, "shard", @@ -522,14 +533,14 @@ async fn apply_loop( *connector_state_json = crate::patches::apply_state_patches(connector_state_json, &applied_patches_json)?; + // Persist the iteration's patches, observing the delta as it's emitted. + let persist = proto::Persist { + connector_patches_json: applied_patches_json, + ..Default::default() + }; + logger.event(crate::LogEvent::Persist { persist: &persist }); db = db - .persist( - &proto::Persist { - connector_patches_json: applied_patches_json, - ..Default::default() - }, - binding_state_keys, - ) + .persist(&persist, binding_state_keys) .await .context("persisting capture Apply connector patches")?; } diff --git a/crates/runtime-next/src/shard/derive/actor.rs b/crates/runtime-next/src/shard/derive/actor.rs index 63a0b3b70ba..2854d44aee7 100644 --- a/crates/runtime-next/src/shard/derive/actor.rs +++ b/crates/runtime-next/src/shard/derive/actor.rs @@ -20,7 +20,7 @@ enum Phase { } /// Shard-side derivation reactor for one joined leader session. -pub(super) struct Actor { +pub(super) struct Actor { // FIFO of outbound connector requests, drained head-first into // `connector_tx` as channel capacity permits. connector_pending: Vec, @@ -34,13 +34,16 @@ pub(super) struct Actor { db_persist_fut: Option>>, // Output-combiner drain + publish future, when in flight. - drain_fut: Option>>, + drain_fut: Option>>>, // Channel for sending to the leader. leader_tx: mpsc::UnboundedSender, // Per-session metrics counters. metrics: super::Metrics, + // Logger through which the drain reports inferred-schema updates. Cloned + // into each drain future. + logger: L, // Publisher for derived documents; parked while a drain borrows it. - publisher: Option, + publisher: Option

, // C:Published measures of the open transaction (reset at each L:Store). published_docs: u64, published_bytes: u64, @@ -56,14 +59,15 @@ pub(super) struct Actor { write_shape: Option, } -impl Actor { +impl Actor { pub fn new( codec: connector_init::Codec, connector_tx: mpsc::Sender, db: crate::shard::RocksDB, leader_tx: mpsc::UnboundedSender, metrics: super::Metrics, - publisher: crate::Publisher, + logger: L, + publisher: P, task: Arc, write_shape: doc::Shape, ) -> Self { @@ -76,6 +80,7 @@ impl Actor { drain_fut: None, leader_tx, metrics, + logger, publisher: Some(publisher), published_docs: 0, published_bytes: 0, @@ -89,18 +94,18 @@ impl Actor { } #[tracing::instrument(level = "debug", err(Debug, level = "warn"), skip_all)] - pub async fn serve( + pub async fn serve( mut self, accumulator: crate::Accumulator, - connector_rx: &mut C, - controller_rx: &mut R, - leader_rx: &mut L, + connector_rx: &mut Conn, + controller_rx: &mut Ctrl, + leader_rx: &mut Ldr, shuffle_reader: shuffle::log::Reader, ) -> anyhow::Result where - R: futures::Stream> + Send + Unpin + 'static, - C: futures::Stream> + Send + Unpin + 'static, - L: futures::Stream> + Send + Unpin + 'static, + Ctrl: futures::Stream> + Send + Unpin + 'static, + Conn: futures::Stream> + Send + Unpin + 'static, + Ldr: futures::Stream> + Send + Unpin + 'static, { // Source-document validators, indexed by transform. Built once and lent // to each `Scanner::step` to re-validate documents that the shuffle read @@ -396,10 +401,19 @@ impl Actor { let task = Arc::clone(&self.task); let metrics = self.metrics.clone(); + let logger = self.logger.clone(); self.drain_fut = Some( async move { - drain::drain_and_publish(drainer, parser, publisher, task, write_shape, metrics) - .await + drain::drain_and_publish( + drainer, + parser, + publisher, + task, + write_shape, + metrics, + logger, + ) + .await } .boxed(), ); @@ -555,7 +569,6 @@ mod tests { use super::super::task::Transform; use super::*; use proto_flow::derive::response; - use proto_flow::flow; use tokio_stream::wrappers::{ReceiverStream, UnboundedReceiverStream}; fn test_task() -> Task { @@ -585,7 +598,7 @@ mod tests { let (actor_to_leader_tx, _leader_rx) = mpsc::unbounded_channel::(); let task = Arc::new(test_task()); - let spec = flow::CollectionSpec { + let spec = proto_flow::flow::CollectionSpec { name: task.collection_name.clone(), partition_template: Some(proto_gazette::broker::JournalSpec { name: "test/derived/v1".to_string(), @@ -593,7 +606,7 @@ mod tests { }), ..Default::default() }; - let publisher = crate::Publisher::new_test_real([&spec]); + let publisher = crate::JournalPublisher::new_test_real([&spec]); let write_shape = task.write_shape.clone(); let mut actor = Actor::new( @@ -602,6 +615,7 @@ mod tests { crate::shard::RocksDB::open(None).await.unwrap(), actor_to_leader_tx, super::super::Metrics::new("test/shard"), + crate::TracingLogger, publisher, task, write_shape, @@ -681,10 +695,6 @@ mod tests { let task = Arc::new(test_task()); let accumulator = crate::Accumulator::new(task.combine_spec().unwrap()).unwrap(); - let publisher = crate::Publisher::new_preview([&flow::CollectionSpec { - name: task.collection_name.clone(), - ..Default::default() - }]); let write_shape = task.write_shape.clone(); let db = crate::shard::RocksDB::open(None).await.unwrap(); let shuffle_dir = tempfile::tempdir().unwrap(); @@ -696,7 +706,8 @@ mod tests { db, actor_to_leader_tx, super::super::Metrics::new("test/shard"), - publisher, + crate::TracingLogger, + crate::publish::NoopPublisher, task, write_shape, ); diff --git a/crates/runtime-next/src/shard/derive/connector.rs b/crates/runtime-next/src/shard/derive/connector.rs index 5e53e38e46e..165ddaa5766 100644 --- a/crates/runtime-next/src/shard/derive/connector.rs +++ b/crates/runtime-next/src/shard/derive/connector.rs @@ -12,8 +12,9 @@ use unseal; /// Unlike the materialize / capture connector starts, derivations don't perform /// an IAM token-exchange Spec pre-dance (no derive connector uses IAM today), /// and they support an in-process `Sqlite` connector alongside image / local. -pub async fn start( - service: &crate::shard::Service, +pub async fn start( + service: &crate::shard::Service, + logger: &L::Logger, log_level: ops::LogLevel, mut initial: derive::Request, ) -> anyhow::Result<( @@ -52,7 +53,7 @@ pub async fn start( let (rx, container, codec) = crate::image_connector::serve( image, - service.log_handler.clone(), + logger.clone(), log_level, &service.container_network, connector_rx, @@ -87,7 +88,7 @@ pub async fn start( let rx = crate::local_connector::serve( command, env, - service.log_handler.clone(), + logger.clone(), log_level, codec, connector_rx, diff --git a/crates/runtime-next/src/shard/derive/drain.rs b/crates/runtime-next/src/shard/derive/drain.rs index a9aee0da8f7..ed89685a672 100644 --- a/crates/runtime-next/src/shard/derive/drain.rs +++ b/crates/runtime-next/src/shard/derive/drain.rs @@ -12,7 +12,7 @@ use anyhow::Context; use bytes::Bytes; /// Resources and results handed back to the actor when a drain completes. -pub(super) struct Output { +pub(super) struct Output { /// The drained combiner, recycled as the next transaction's accumulator. pub accumulator: crate::Accumulator, /// Documents drained from the combiner and published this transaction. @@ -22,21 +22,22 @@ pub(super) struct Output { /// This shard's publisher commit, or None when nothing was published. pub publisher_commit: Option, /// The publisher, borrowed for the drain's journal appends. - pub publisher: crate::Publisher, + pub publisher: P, /// The collection's inferred write shape, possibly widened this transaction. pub write_shape: doc::Shape, } /// Drain the rotated output combiner: publish each derived document, fold it /// into inference, then flush and snapshot the publisher commit. -pub(super) async fn drain_and_publish( +pub(super) async fn drain_and_publish( drainer: doc::combine::Drainer, parser: simd_doc::Parser, - mut publisher: crate::Publisher, + mut publisher: P, task: std::sync::Arc, mut write_shape: doc::Shape, metrics: super::Metrics, -) -> anyhow::Result { + logger: L, +) -> anyhow::Result> { // Resync the publisher clock to wall-clock time at the start of this // transaction's stream of published documents. Each `publish_doc` and the // closing `commit_intents` then tick it up by a single microsecond, so @@ -93,12 +94,12 @@ pub(super) async fn drain_and_publish( ); if updated_inference { - let serialized = doc::shape::schema::to_schema(write_shape.clone()); - tracing::info!( - schema = ?ops::DebugJson(serialized), - collection_name = %task.collection_name, - "inferred schema updated", - ); + let schema = doc::shape::schema::to_schema(write_shape.clone()); + logger.event(crate::LogEvent::InferredSchema { + collection_name: &task.collection_name, + binding: None, + schema: &schema, + }); metrics.inferred_schema_updates.increment(1); } diff --git a/crates/runtime-next/src/shard/derive/handler.rs b/crates/runtime-next/src/shard/derive/handler.rs index ccd5966facb..af64b5f942f 100644 --- a/crates/runtime-next/src/shard/derive/handler.rs +++ b/crates/runtime-next/src/shard/derive/handler.rs @@ -6,8 +6,8 @@ use proto_flow::derive; use tokio::sync::mpsc; use tracing::Instrument; -pub(crate) async fn serve( - service: crate::shard::Service, +pub(crate) async fn serve( + service: crate::shard::Service, mut controller_rx: R, controller_tx: mpsc::UnboundedSender>, ) -> anyhow::Result<()> @@ -76,16 +76,17 @@ where Ok(()) } -pub async fn serve_unary( - service: &crate::shard::Service, +pub async fn serve_unary( + service: &crate::shard::Service, request: derive::Request, log_level: ops::LogLevel, ) -> anyhow::Result { let is_spec = request.spec.is_some(); let is_validate = request.validate.is_some(); + let logger = service.logger_factory.open(&service.task_name); let (connector_tx, mut connector_rx, _container, _codec) = - connector::start(service, log_level, request).await?; + connector::start(service, &logger, log_level, request).await?; std::mem::drop(connector_tx); // Send EOF. let verify = crate::verify("Derive", "unary response", "connector"); @@ -111,8 +112,8 @@ pub async fn serve_unary( Ok(response) } -async fn serve_session_loop( - service: &crate::shard::Service, +async fn serve_session_loop( + service: &crate::shard::Service, controller_rx: &mut R, controller_tx: &mpsc::UnboundedSender>, session_loop: proto::SessionLoop, @@ -154,8 +155,8 @@ where Ok(()) } -async fn serve_session( - service: &crate::shard::Service, +async fn serve_session( + service: &crate::shard::Service, controller_rx: &mut R, controller_tx: &mpsc::UnboundedSender>, db: crate::shard::RocksDB, @@ -182,8 +183,8 @@ where .await } -async fn serve_session_inner( - service: &crate::shard::Service, +async fn serve_session_inner( + service: &crate::shard::Service, controller_rx: &mut R, controller_tx: &mpsc::UnboundedSender>, db: crate::shard::RocksDB, @@ -218,6 +219,7 @@ where handler.set_field("etcd_mod_revision", join.etcd_mod_revision); handler.set_phase("joining"); + let logger = service.logger_factory.open(&service.task_name); let metrics = super::Metrics::new(&shard_id); service_kit::event!( @@ -269,6 +271,7 @@ where leader_rx, leader_tx, log_level, + &logger, service, shard_id, shard_index, @@ -285,6 +288,7 @@ where db, leader_tx, metrics, + logger, publisher, std::sync::Arc::new(task), write_shape, diff --git a/crates/runtime-next/src/shard/derive/startup.rs b/crates/runtime-next/src/shard/derive/startup.rs index 7b637ad4165..deb4a1db31a 100644 --- a/crates/runtime-next/src/shard/derive/startup.rs +++ b/crates/runtime-next/src/shard/derive/startup.rs @@ -61,7 +61,7 @@ pub async fn dial_and_join( } } -pub(super) struct Startup { +pub(super) struct Startup { pub accumulator: crate::Accumulator, pub codec: connector_init::Codec, pub connector_rx: BoxStream<'static, tonic::Result>, @@ -69,13 +69,13 @@ pub(super) struct Startup { pub db: crate::shard::RocksDB, pub leader_rx: tonic::Streaming, pub leader_tx: mpsc::UnboundedSender, - pub publisher: crate::Publisher, + pub publisher: P, pub shuffle_reader: shuffle::log::Reader, pub task: Task, pub write_shape: doc::Shape, } -pub(super) async fn run( +pub(super) async fn run( controller_rx: &mut R, controller_tx: &mpsc::UnboundedSender>, db: crate::shard::RocksDB, @@ -84,12 +84,13 @@ pub(super) async fn run( mut leader_rx: tonic::Streaming, leader_tx: mpsc::UnboundedSender, log_level: ops::LogLevel, - service: &crate::shard::Service, + logger: &L::Logger, + service: &crate::shard::Service, shard_id: String, shard_index: u32, shard_producer: proto_gazette::uuid::Producer, shuffle_directory: String, -) -> anyhow::Result +) -> anyhow::Result> where R: futures::Stream> + Send + Unpin + 'static, { @@ -114,7 +115,6 @@ where let proto::Task { max_transactions: _, - preview, spec: spec_bytes, sqlite_vfs_uri, publisher_id: _, // Consumed above; the leader, not this shard, uses it. @@ -127,18 +127,15 @@ where // The derived collection is the single additional publisher binding; // publisher binding zero is the fixed ops-stats journal. - let publisher = if preview { - crate::Publisher::new_preview([&spec]) - } else { - crate::Publisher::new_real( + let publisher = service + .publisher_factory + .open( shard_id, // Shard ID is AuthZ subject. shard_producer, - &service.publisher_factory, &labeling.stats_journal, - [&spec], + &[&spec], ) - .context("creating publisher")? - }; + .context("opening publisher")?; // Scan and send L:Recover state from RocksDB. Derivations have no max-keys // (connector state is a singleton), but the committed/hinted frontier is @@ -223,7 +220,7 @@ where } let (connector_tx, mut connector_rx, container, codec) = - super::connector::start(service, log_level, initial).await?; + super::connector::start(service, logger, log_level, initial).await?; let verify = crate::verify("Derive", "Opened", "connector"); let opened = match verify.not_eof(connector_rx.next().await)? { diff --git a/crates/runtime-next/src/shard/materialize/actor.rs b/crates/runtime-next/src/shard/materialize/actor.rs index fe029a3d956..bc3dbf52fd4 100644 --- a/crates/runtime-next/src/shard/materialize/actor.rs +++ b/crates/runtime-next/src/shard/materialize/actor.rs @@ -86,18 +86,18 @@ impl Actor { } #[tracing::instrument(level = "debug", err(Debug, level = "warn"), skip_all)] - pub async fn serve( + pub async fn serve( mut self, accumulator: crate::Accumulator, - connector_rx: &mut C, - controller_rx: &mut R, - leader_rx: &mut L, + connector_rx: &mut Conn, + controller_rx: &mut Ctrl, + leader_rx: &mut Ldr, shuffle_reader: shuffle::log::Reader, ) -> anyhow::Result where - R: futures::Stream> + Send + Unpin + 'static, - C: futures::Stream> + Send + Unpin + 'static, - L: futures::Stream> + Send + Unpin + 'static, + Ctrl: futures::Stream> + Send + Unpin + 'static, + Conn: futures::Stream> + Send + Unpin + 'static, + Ldr: futures::Stream> + Send + Unpin + 'static, { let mut phase = Phase::Idle { accumulator, diff --git a/crates/runtime-next/src/shard/materialize/connector.rs b/crates/runtime-next/src/shard/materialize/connector.rs index 99cbf5841c8..4a48ae80bac 100644 --- a/crates/runtime-next/src/shard/materialize/connector.rs +++ b/crates/runtime-next/src/shard/materialize/connector.rs @@ -9,8 +9,9 @@ use zeroize::Zeroize; // Start a materialization connector as indicated by the `initial` Request. // Returns a pair of Streams for sending Requests and receiving Responses, // plus OpenExtras with decrypted trigger configs and connector metadata. -pub async fn start( - service: &crate::shard::Service, +pub async fn start( + service: &crate::shard::Service, + logger: &L::Logger, log_level: ops::LogLevel, mut initial: materialize::Request, ) -> anyhow::Result<( @@ -45,7 +46,7 @@ pub async fn start( let (rx, container, codec) = crate::image_connector::serve( image.clone(), - service.log_handler.clone(), + logger.clone(), log_level, &service.container_network, connector_rx, @@ -82,7 +83,7 @@ pub async fn start( let rx = crate::local_connector::serve( command, env, - service.log_handler.clone(), + logger.clone(), log_level, codec, connector_rx, diff --git a/crates/runtime-next/src/shard/materialize/handler.rs b/crates/runtime-next/src/shard/materialize/handler.rs index aac9be2c59f..b3a7edd2505 100644 --- a/crates/runtime-next/src/shard/materialize/handler.rs +++ b/crates/runtime-next/src/shard/materialize/handler.rs @@ -6,8 +6,8 @@ use proto_flow::materialize; use tokio::sync::mpsc; use tracing::Instrument; -pub(crate) async fn serve( - service: crate::shard::Service, +pub(crate) async fn serve( + service: crate::shard::Service, mut controller_rx: R, controller_tx: mpsc::UnboundedSender>, ) -> anyhow::Result<()> @@ -72,8 +72,8 @@ where Ok(()) } -pub async fn serve_unary( - service: &crate::shard::Service, +pub async fn serve_unary( + service: &crate::shard::Service, request: materialize::Request, log_level: ops::LogLevel, ) -> anyhow::Result { @@ -81,8 +81,9 @@ pub async fn serve_unary( let is_validate = request.validate.is_some(); let is_apply = request.apply.is_some(); + let logger = service.logger_factory.open(&service.task_name); let (connector_tx, mut connector_rx, _container, _codec) = - connector::start(service, log_level, request).await?; + connector::start(service, &logger, log_level, request).await?; std::mem::drop(connector_tx); // Send EOF. // Read connector response, and verify it matches the request type. @@ -124,8 +125,8 @@ pub async fn serve_unary( Ok(response) } -async fn serve_session_loop( - service: &crate::shard::Service, +async fn serve_session_loop( + service: &crate::shard::Service, controller_rx: &mut R, controller_tx: &mpsc::UnboundedSender>, session_loop: proto::SessionLoop, @@ -164,8 +165,8 @@ where Ok(()) } -async fn serve_session( - service: &crate::shard::Service, +async fn serve_session( + service: &crate::shard::Service, controller_rx: &mut R, controller_tx: &mpsc::UnboundedSender>, db: crate::shard::RocksDB, @@ -193,8 +194,8 @@ where .await } -async fn serve_session_inner( - service: &crate::shard::Service, +async fn serve_session_inner( + service: &crate::shard::Service, controller_rx: &mut R, controller_tx: &mpsc::UnboundedSender>, db: crate::shard::RocksDB, @@ -228,6 +229,7 @@ where handler.set_field("etcd_mod_revision", join.etcd_mod_revision); handler.set_phase("joining"); + let logger = service.logger_factory.open(&service.task_name); let metrics = super::Metrics::new(&shard_id); service_kit::event!( @@ -281,6 +283,7 @@ where leader_rx, leader_tx, log_level, + &logger, service, shard_index, shuffle_directory, diff --git a/crates/runtime-next/src/shard/materialize/startup.rs b/crates/runtime-next/src/shard/materialize/startup.rs index 101304edd3c..9502ccb5cb5 100644 --- a/crates/runtime-next/src/shard/materialize/startup.rs +++ b/crates/runtime-next/src/shard/materialize/startup.rs @@ -78,7 +78,7 @@ pub(super) struct Startup { pub shuffle_reader: shuffle::log::Reader, } -pub(super) async fn run( +pub(super) async fn run( controller_rx: &mut R, controller_tx: &mpsc::UnboundedSender>, db: crate::shard::RocksDB, @@ -87,7 +87,8 @@ pub(super) async fn run( mut leader_rx: tonic::Streaming, leader_tx: mpsc::UnboundedSender, log_level: ops::LogLevel, - service: &crate::shard::Service, + logger: &L::Logger, + service: &crate::shard::Service, shard_index: u32, shuffle_directory: String, ) -> anyhow::Result @@ -116,7 +117,6 @@ where let proto::Task { max_transactions: _, - preview: _, spec: spec_bytes, sqlite_vfs_uri: _, publisher_id: _, @@ -245,7 +245,7 @@ where ..Default::default() }; let (connector_tx, mut connector_rx, container, codec) = - super::connector::start(service, log_level, initial).await?; + super::connector::start(service, logger, log_level, initial).await?; // Read C:Opened from the connector. let verify = crate::verify("Materialize", "Opened", "connector"); diff --git a/crates/runtime-next/src/shard/mod.rs b/crates/runtime-next/src/shard/mod.rs index 592cbf855c6..0bf2bef6839 100644 --- a/crates/runtime-next/src/shard/mod.rs +++ b/crates/runtime-next/src/shard/mod.rs @@ -2,7 +2,7 @@ pub mod capture; pub mod derive; pub mod materialize; pub(crate) mod recovery; -mod rocksdb; +pub(crate) mod rocksdb; mod service; pub mod split_policy; @@ -41,9 +41,9 @@ pub type SplitFuture = /// A dispatched journal stays "due" until its outcome lands: the actor's /// single-flight parking of the returned future is what prevents duplicate /// dispatch, and an RPC error leaves a still-hot journal due for retry. -pub fn start_due_split( +pub fn start_due_split( policy: &mut crate::shard::split_policy::SplitPolicy, - publisher: &crate::Publisher, + publisher: &P, now: std::time::Instant, ) -> Option { use futures::FutureExt; @@ -258,8 +258,7 @@ mod tests { /// re-evaluated forever. #[test] fn start_due_split_terminally_ignores_unsplittable_journals() { - let publisher = - crate::Publisher::new_preview(std::iter::empty::<&proto_flow::flow::CollectionSpec>()); + let publisher = crate::publish::NoopPublisher; let (mut policy, now) = due_policy(); assert!(start_due_split(&mut policy, &publisher, now).is_none()); @@ -282,7 +281,7 @@ mod tests { }), ..Default::default() }; - let publisher = crate::Publisher::new_test_real([&spec]); + let publisher = crate::JournalPublisher::new_test_real([&spec]); let (mut policy, now) = due_policy(); let j2 = "test/collection/v1/pivot=80"; diff --git a/crates/runtime-next/src/shard/rocksdb.rs b/crates/runtime-next/src/shard/rocksdb.rs index ef129de9625..b25e8841981 100644 --- a/crates/runtime-next/src/shard/rocksdb.rs +++ b/crates/runtime-next/src/shard/rocksdb.rs @@ -111,6 +111,28 @@ impl RocksDB { .context("RocksDB Persist write") } + /// Durably (`set_sync`) `Put` `base` as the connector-state document at + /// [`recovery::KEY_CONNECTOR_STATE`], replacing any prior value. A `Put` + /// (not `Merge`) establishes the exact base document, so a connector's later + /// state patches merge atop it, and it's read back verbatim as + /// `Recover.connector_state_json`. + /// + /// Shared by [`Self::seed_connector_state`], which seeds `{}` during the + /// runtime's own recovery scan, and by the `flowctl preview + /// --initial-state` harness, which seeds an arbitrary base by opening shard + /// zero's RocksDB by path before any Recover scan observes it. + pub(crate) async fn put_connector_state_base(self, base: &[u8]) -> anyhow::Result { + let mut wb = rocksdb::WriteBatch::default(); + wb.put(recovery::KEY_CONNECTOR_STATE, base); + + let mut wo = rocksdb::WriteOptions::new(); + wo.set_sync(true); + + self.write_opt(wb, wo) + .await + .context("RocksDB connector-state base write") + } + /// Scan the entire DB into a [`proto::Recover`] using a blocking /// background thread. Returns `(self, Recover)`. /// @@ -219,16 +241,7 @@ impl RocksDB { return Ok(self); } - let mut wb = rocksdb::WriteBatch::default(); - wb.put(recovery::KEY_CONNECTOR_STATE, b"{}"); - let mut wo = rocksdb::WriteOptions::new(); - wo.set_sync(true); - - let db = self - .write_opt(wb, wo) - .await - .context("seeding initial connector state")?; - + let db = self.put_connector_state_base(b"{}").await?; recover.connector_state_json = bytes::Bytes::from_static(b"{}"); Ok(db) } diff --git a/crates/runtime-next/src/shard/service.rs b/crates/runtime-next/src/shard/service.rs index c8fd243df99..cd94f88ad9f 100644 --- a/crates/runtime-next/src/shard/service.rs +++ b/crates/runtime-next/src/shard/service.rs @@ -6,6 +6,9 @@ //! preview`, or a unit-test harness. From this crate's perspective the //! controller is just the peer of the bidi stream that commands the //! runtime and bounds its lifecycle. +//! +//! `Service` is monomorphized over its [`PublisherFactory`](crate::PublisherFactory) +//! `P`, so the publish path is statically dispatched. use crate::proto; use futures::Stream; @@ -15,43 +18,44 @@ use tokio_stream::wrappers; /// Service is the implementation of the controller-facing `Shard` gRPC /// service trait, hosting one shard's transaction loop. #[derive(Clone)] -pub struct Service { +pub struct Service { pub plane: crate::Plane, pub container_network: String, - pub log_handler: L, pub set_log_level: Option>, pub task_name: String, - pub publisher_factory: gazette::journal::ClientFactory, + pub publisher_factory: P, + pub logger_factory: L, pub registry: service_kit::Registry, pub data_plane_signer: Option, } -impl Service { +impl Service { /// Build a new Shard Service. /// - `plane`: the type of data plane in which this Service is operating. /// - `container_network`: the Docker container network used for connector containers. - /// - `log_handler`: handler to which connector logs are dispatched. /// - `set_log_level`: callback for adjusting the log level implied by runtime requests. /// - `task_name`: name which is used to label any started connector containers. - /// - `publisher_factory`: client factory for creating and appending to collection partitions. + /// - `publisher_factory`: opens publishers for appending to collection partitions. + /// - `logger_factory`: opens a Logger per session, which sinks connector + /// logs and reports runtime events. /// - `registry`: in-flight handler registry, shared with any co-hosted admin surface. pub fn new( plane: crate::Plane, container_network: String, - log_handler: L, set_log_level: Option>, task_name: String, - publisher_factory: gazette::journal::ClientFactory, + publisher_factory: P, + logger_factory: L, registry: service_kit::Registry, data_plane_signer: Option, ) -> Self { Self { plane, container_network, - log_handler, set_log_level, task_name, publisher_factory, + logger_factory, registry, data_plane_signer, } @@ -137,7 +141,9 @@ impl Service { } #[tonic::async_trait] -impl proto_grpc::runtime::shard_server::Shard for Service { +impl proto_grpc::runtime::shard_server::Shard + for Service +{ type CaptureStream = wrappers::UnboundedReceiverStream>; type DeriveStream = wrappers::UnboundedReceiverStream>; type MaterializeStream = wrappers::UnboundedReceiverStream>; diff --git a/crates/runtime-next/src/task_service.rs b/crates/runtime-next/src/task_service.rs index bfd055042a6..8c70c906ecf 100644 --- a/crates/runtime-next/src/task_service.rs +++ b/crates/runtime-next/src/task_service.rs @@ -61,10 +61,14 @@ impl TaskService { let shard_svc = shard::Service::new( crate::proto::Plane::try_from(plane).context("invalid TaskServiceConfig.plane")?, container_network, - log_handler, Some(tokio_context.set_log_level_fn()), task_name, - publisher_factory, + crate::JournalPublisherFactory::new(publisher_factory), + // Each session's Logger sinks the task's log stream — connector + // logs and flattened runtime Events alike — to the task-log file + // (the same encoded-JSON handler the runtime's own tracing uses), + // which the Go runtime forwards to the task's ops-log journal. + crate::FnLoggerFactory::new(log_handler), // Inert registry: TaskService is the CGO entry point and does not // serve an admin surface; event! tracks still capture per-handler. service_kit::Registry::default(), diff --git a/crates/runtime-next/tests/split_e2e.rs b/crates/runtime-next/tests/split_e2e.rs index 5dd680d9b80..c69d20307c0 100644 --- a/crates/runtime-next/tests/split_e2e.rs +++ b/crates/runtime-next/tests/split_e2e.rs @@ -20,7 +20,7 @@ //! partitioned collection. use runtime_next::shard::split_policy::SplitPolicy; -use runtime_next::{Publisher, shard}; +use runtime_next::{JournalPublisher, JournalPublisherFactory, Publisher, PublisherFactory, shard}; use proto_gazette::{broker, uuid}; use publisher::SplitOutcome; @@ -99,8 +99,26 @@ impl KeyPool { } } +/// Open a real single-collection publisher over `factory`, mirroring how a +/// shard opens its [`JournalPublisher`] in production. Binding zero is the +/// fixed ops-stats journal (never appended to here); the collection is binding +/// one. +fn open_publisher( + factory: &gazette::journal::ClientFactory, + spec: &proto_flow::flow::CollectionSpec, +) -> JournalPublisher { + JournalPublisherFactory::new(factory.clone()) + .open( + "test".to_string(), + runtime_next::new_producer(), + "testing/ops/stats", + &[spec], + ) + .expect("open JournalPublisher") +} + /// Publish one transaction of `ids` as documents. -async fn publish_txn(publisher: &mut Publisher, ids: &[String], payload: &str) { +async fn publish_txn(publisher: &mut JournalPublisher, ids: &[String], payload: &str) { let docs = ids .iter() .map(|id| serde_json::json!({"id": id, "payload": payload})) @@ -110,10 +128,8 @@ async fn publish_txn(publisher: &mut Publisher, ids: &[String], payload: &str) { /// Publish one transaction of arbitrary collection documents, stamping each /// with its assigned UUID under `/_meta/uuid`. -async fn publish_txn_docs(publisher: &mut Publisher, docs: Vec) { - let Publisher::Real(inner) = publisher else { - unreachable!("test publisher is Real"); - }; +async fn publish_txn_docs(publisher: &mut JournalPublisher, docs: Vec) { + let inner = publisher.inner_mut(); for mut doc in docs { inner .enqueue( @@ -133,7 +149,7 @@ async fn publish_txn_docs(publisher: &mut Publisher, docs: Vec Vec<(String, bool)> { +fn take_samples(publisher: &mut JournalPublisher) -> Vec<(String, bool)> { publisher .take_throttle_samples() .into_iter() @@ -161,7 +177,7 @@ fn observe(policy: &mut SplitPolicy, samples: &[(String, bool)], now: Instant) { /// the loop simply re-evaluates against the refreshed watch. async fn dispatch_one_split( policy: &mut SplitPolicy, - publisher: &Publisher, + publisher: &JournalPublisher, now: Instant, split_count: &mut usize, at_floor: &mut BTreeSet, @@ -371,14 +387,7 @@ async fn journal_auto_split_converges_to_floor() { spec.partition_template.as_mut().unwrap().max_append_rate = TEST_MAX_APPEND_RATE; let partitions_prefix = format!("{}/", spec.partition_template.as_ref().unwrap().name); - let mut publisher = Publisher::new_real( - "test".to_string(), - runtime_next::new_producer(), - &factory, - "testing/ops/stats", // Fixed binding zero; never appended to. - [&spec], - ) - .expect("Publisher::new_real"); + let mut publisher = open_publisher(&factory, &spec); let mut policy = SplitPolicy::with_config(aggressive_config()); @@ -546,22 +555,8 @@ async fn concurrent_journal_split_loses_cas_race() { // Two publishers stand in for two shards of one task. Each holds its own // partition watch and split policy. - let mut pub_a = Publisher::new_real( - "test".to_string(), - runtime_next::new_producer(), - &factory, - "testing/ops/stats", - [&spec], - ) - .expect("Publisher::new_real"); - let mut pub_b = Publisher::new_real( - "test".to_string(), - runtime_next::new_producer(), - &factory, - "testing/ops/stats", - [&spec], - ) - .expect("Publisher::new_real"); + let mut pub_a = open_publisher(&factory, &spec); + let mut pub_b = open_publisher(&factory, &spec); // Any observed journal is instantly due, and the cooldown is long enough // that a completed attempt is observable as suppression. @@ -702,14 +697,7 @@ async fn journal_split_lost_cas_is_non_destructive() { let (data_plane, spec, factory) = start_fixture("testing/autosplit").await; let partitions_prefix = format!("{}/", spec.partition_template.as_ref().unwrap().name); - let mut publisher = Publisher::new_real( - "test".to_string(), - runtime_next::new_producer(), - &factory, - "testing/ops/stats", - [&spec], - ) - .expect("Publisher::new_real"); + let mut publisher = open_publisher(&factory, &spec); // One document creates the full-range partition. let payload: String = "x".repeat(256); @@ -805,14 +793,7 @@ async fn journal_auto_split_preserves_document_completeness() { spec.partition_template.as_mut().unwrap().max_append_rate = TEST_MAX_APPEND_RATE; let partitions_prefix = format!("{}/", spec.partition_template.as_ref().unwrap().name); - let mut publisher = Publisher::new_real( - "test".to_string(), - runtime_next::new_producer(), - &factory, - "testing/ops/stats", - [&spec], - ) - .expect("Publisher::new_real"); + let mut publisher = open_publisher(&factory, &spec); let mut policy = SplitPolicy::with_config(aggressive_config()); let payload: String = "x".repeat(PAYLOAD_LEN); @@ -929,14 +910,7 @@ async fn journal_auto_split_respects_logical_partitions() { spec.partition_template.as_mut().unwrap().max_append_rate = TEST_MAX_APPEND_RATE; let partitions_prefix = format!("{}/", spec.partition_template.as_ref().unwrap().name); - let mut publisher = Publisher::new_real( - "test".to_string(), - runtime_next::new_producer(), - &factory, - "testing/ops/stats", - [&spec], - ) - .expect("Publisher::new_real"); + let mut publisher = open_publisher(&factory, &spec); let mut policy = SplitPolicy::with_config(aggressive_config()); let payload: String = "x".repeat(PAYLOAD_LEN); diff --git a/crates/runtime-sidecar/src/lib.rs b/crates/runtime-sidecar/src/lib.rs index 324c3cc6ce6..5d2f4d021e4 100644 --- a/crates/runtime-sidecar/src/lib.rs +++ b/crates/runtime-sidecar/src/lib.rs @@ -113,8 +113,11 @@ pub async fn run(args: Args, registry: service_kit::Registry) -> anyhow::Result< Some(shuffle_signer), ); let runtime_svc = runtime_next::Service::new( - shuffle_svc.clone(), - publisher_factory, + runtime_next::leader::ShuffleServiceFactory::new(shuffle_svc.clone()), + runtime_next::JournalPublisherFactory::new(publisher_factory), + // Events render as sidecar tracing until the Logger that publishes + // the task's log stream (async) to its ops-log journal exists. + runtime_next::TracingLoggerFactory, registry.clone(), false, // Don't disarm, enforce AuthN+AuthZ. ); diff --git a/crates/runtime/src/container.rs b/crates/runtime/src/container.rs index 5e40a494fdb..8774c5a23c7 100644 --- a/crates/runtime/src/container.rs +++ b/crates/runtime/src/container.rs @@ -17,8 +17,10 @@ const USAGE_RATE_LABEL: &str = "dev.estuary.usage-rate"; const PORT_PUBLIC_LABEL_PREFIX: &str = "dev.estuary.port-public."; const PORT_PROTO_LABEL_PREFIX: &str = "dev.estuary.port-proto."; +// `flow-connector-init` is extracted from this image when a locally-built copy +// isn't found by `locate_bin` (dev/CI builds place one alongside the executable). // TODO(johnny): Consider better packaging and versioning of `flow-connector-init`. -const CONNECTOR_INIT_IMAGE: &str = "ghcr.io/estuary/flow:v0.5.24-30-ga3eba41f95"; +const CONNECTOR_INIT_IMAGE: &str = "ghcr.io/estuary/reactor:v0.6.10-62-g8b6aeec1cd3"; const CONNECTOR_INIT_IMAGE_PATH: &str = "/usr/local/bin/flow-connector-init"; /// Determines the protocol of an image. If the image has a `FLOW_RUNTIME_PROTOCOL` label, diff --git a/crates/runtime/src/harness/capture.rs b/crates/runtime/src/harness/capture.rs deleted file mode 100644 index e75a645b750..00000000000 --- a/crates/runtime/src/harness/capture.rs +++ /dev/null @@ -1,273 +0,0 @@ -use crate::capture::ResponseStream; -use crate::{LogHandler, Runtime, rocksdb::RocksDB, verify}; -use anyhow::Context; -use futures::{TryStreamExt, channel::mpsc}; -use proto_flow::capture::{Request, Response, request, response}; -use proto_flow::flow; -use proto_flow::runtime::{ - self, CaptureResponseExt, capture_request_ext, - capture_response_ext::{self, PollResult}, -}; -use proto_gazette::consumer; -use std::pin::Pin; - -pub fn run_capture( - delay: std::time::Duration, - runtime: Runtime, - sessions: Vec, - spec: &flow::CaptureSpec, - mut state: models::RawValue, - state_dir: &std::path::Path, - timeout: std::time::Duration, - output_apply: bool, -) -> impl ResponseStream { - let spec = spec.clone(); - let state_dir = state_dir.to_owned(); - - coroutines::try_coroutine(move |mut co| async move { - let (mut request_tx, request_rx) = mpsc::channel(crate::CHANNEL_BUFFER); - let response_rx = runtime.serve_capture(request_rx); - tokio::pin!(response_rx); - - let state_dir = state_dir.to_str().context("tempdir is not utf8")?; - let rocksdb_desc = runtime::RocksDbDescriptor { - rocksdb_env_memptr: 0, - rocksdb_path: state_dir.to_owned(), - }; - - let sessions_len = sessions.len(); - for (sessions_index, target_transactions) in sessions.into_iter().enumerate() { - () = run_session( - &mut co, - delay, - &mut request_tx, - &mut response_rx, - &rocksdb_desc, - sessions_index, - sessions_len, - &spec, - &mut state, - target_transactions, - timeout, - output_apply, - ) - .await?; - } - - std::mem::drop(request_tx); - verify("runtime", "EOF").is_eof(response_rx.try_next().await?)?; - - // Re-open RocksDB. - let rocksdb = RocksDB::open(Some(rocksdb_desc)).await?; - - let checkpoint = rocksdb.load_checkpoint().await?; - tracing::debug!(checkpoint = ?::ops::DebugJson(checkpoint), "final runtime checkpoint"); - - // Extract and yield the final connector state - let state = rocksdb - .load_connector_state(models::RawValue::default()) - .await?; - - () = co - .yield_(Response { - checkpoint: Some(response::Checkpoint { - state: Some(flow::ConnectorState { - updated_json: state.into(), - merge_patch: false, - }), - }), - ..Default::default() - }) - .await; - - anyhow::Result::Ok(()) - }) -} - -async fn run_session( - co: &mut coroutines::Suspend, - delay: std::time::Duration, - request_tx: &mut mpsc::Sender>, - response_rx: &mut Pin<&mut impl ResponseStream>, - rocksdb_desc: &runtime::RocksDbDescriptor, - sessions_index: usize, - sessions_len: usize, - spec: &flow::CaptureSpec, - state: &mut models::RawValue, - target_transactions: usize, - timeout: std::time::Duration, - output_apply: bool, -) -> anyhow::Result<()> { - let labeling = crate::parse_shard_labeling(spec.shard_template.as_ref())?; - - // Send Apply. - let apply = Request { - apply: Some(request::Apply { - capture: Some(spec.clone()), - version: labeling.build.clone(), - last_capture: None, - last_version: String::new(), - state_json: bytes::Bytes::new(), - }), - ..Default::default() - } - .with_internal(|internal| { - if sessions_index == 0 { - internal.rocksdb_descriptor = Some(rocksdb_desc.clone()); - } - internal.set_log_level(labeling.log_level()); - }); - request_tx.try_send(Ok(apply)).expect("sender is empty"); - - // Receive Applied. - match response_rx.try_next().await? { - Some(applied) if applied.applied.is_some() => { - if output_apply { - print!( - "[\"applied.actionDescription\", {:?}]\n", - applied.applied.as_ref().unwrap().action_description - ); - } - () = co.yield_(applied).await; - } - response => return verify("runtime", "Applied").fail(response), - } - - // Send Open. - let open = Request { - open: Some(request::Open { - capture: Some(spec.clone()), - version: labeling.build.clone(), - range: Some(flow::RangeSpec { - key_begin: 0, - key_end: u32::MAX, - r_clock_begin: 0, - r_clock_end: u32::MAX, - }), - state_json: std::mem::take(state).into(), - }), - ..Default::default() - } - .with_internal(|internal| internal.set_log_level(labeling.log_level())); - request_tx.try_send(Ok(open)).expect("sender is empty"); - - // Receive Opened. - match response_rx.try_next().await? { - Some(opened) if opened.opened.is_some() => { - () = co.yield_(opened).await; - } - response => return verify("runtime", "Opened").fail(response), - } - - // Reset-able timer for assessing `timeout` between transactions. - let mut deadline = tokio::time::sleep(timeout); - let mut transaction = 0; - - while transaction != target_transactions { - // Future which sleeps for `delay` and then sends a poll request. - let send_poll = async { - if !delay.is_zero() { - () = tokio::time::sleep(delay).await; - } - request_tx - .try_send(Ok(Request { - acknowledge: Some(request::Acknowledge { checkpoints: 0 }), - ..Default::default() - })) - .expect("sender is empty"); - - Ok(()) - }; - - // Join over sending a poll request and reading its result. - let ((), poll_response) = futures::try_join!(send_poll, response_rx.try_next())?; - - let ready = { - let verify = verify("runtime", "Poll Result"); - let poll_response = verify.not_eof(poll_response)?; - let CaptureResponseExt { - checkpoint: - Some(capture_response_ext::Checkpoint { - stats: None, - poll_result, - }), - .. - } = poll_response.get_internal()? - else { - return verify.fail(poll_response); - }; - - let poll_result = PollResult::try_from(poll_result).context("invalid PollResult")?; - tracing::debug!(?poll_result, "polled capture"); - - match poll_result { - PollResult::Invalid => return verify.fail(poll_response), - PollResult::Ready => true, - PollResult::CoolOff if sessions_index + 1 == sessions_len => break, - PollResult::CoolOff | PollResult::NotReady => false, - PollResult::Restart => break, - } - }; - - if !ready && !timeout.is_zero() && deadline.is_elapsed() { - break; - } else if !ready { - continue; // Poll again. - } - - // Receive Captured and Checkpoint. - let mut done = false; - while !done { - let verify = verify("runtime", "Captured or Checkpoint"); - let response = verify.not_eof(response_rx.try_next().await?)?; - - done = match &response { - Response { - captured: Some(_), .. - } => false, - Response { - checkpoint: Some(response::Checkpoint { state }), - .. - } => state.is_none(), // Final Checkpoint (only) has no `state`. - _ => return verify.fail(response), - }; - () = co.yield_(response).await; - } - - // Send a StartCommit with a synthetic checkpoint that reflects our current poll. - request_tx - .try_send(Ok(Request::default().with_internal(|internal| { - internal.start_commit = Some(capture_request_ext::StartCommit { - runtime_checkpoint: Some(consumer::Checkpoint { - sources: [( - format!("test/transactions"), - consumer::checkpoint::Source { - read_through: 1 + transaction as i64, - ..Default::default() - }, - )] - .into(), - ack_intents: Default::default(), - }), - }); - }))) - .expect("sender is empty"); - - // Receive StartedCommit. - match response_rx.try_next().await? { - Some(Response { - checkpoint: Some(response::Checkpoint { state: None }), - .. - }) => (), - response => return verify("runtime", "StartedCommit").fail(response), - } - - transaction += 1; - - if timeout != std::time::Duration::MAX { - deadline = tokio::time::sleep(timeout); - } - } - - Ok(()) -} diff --git a/crates/runtime/src/harness/derive.rs b/crates/runtime/src/harness/derive.rs deleted file mode 100644 index 648d5528fd1..00000000000 --- a/crates/runtime/src/harness/derive.rs +++ /dev/null @@ -1,240 +0,0 @@ -use super::{Read, Reader}; -use crate::derive::ResponseStream; -use crate::{LogHandler, Runtime, rocksdb::RocksDB, verify}; -use anyhow::Context; -use futures::{TryStreamExt, channel::mpsc}; -use proto_flow::derive::{Request, Response, request, response}; -use proto_flow::flow; -use proto_flow::runtime::{self, derive_request_ext}; -use std::pin::Pin; - -pub fn run_derive( - reader: impl Reader, - runtime: Runtime, - sessions: Vec, - spec: &flow::CollectionSpec, - mut state: models::RawValue, - state_dir: &std::path::Path, - timeout: std::time::Duration, -) -> impl ResponseStream { - let spec = spec.clone(); - let state_dir = state_dir.to_owned(); - - coroutines::try_coroutine(move |mut co| async move { - let (mut request_tx, request_rx) = mpsc::channel(crate::CHANNEL_BUFFER); - let response_rx = runtime.serve_derive(request_rx); - tokio::pin!(response_rx); - - let state_dir = state_dir.to_str().context("tempdir is not utf8")?; - let rocksdb_desc = runtime::RocksDbDescriptor { - rocksdb_env_memptr: 0, - rocksdb_path: state_dir.to_owned(), - }; - - for (sessions_index, target_transactions) in sessions.into_iter().enumerate() { - () = run_session( - &mut co, - reader.clone(), - &mut request_tx, - &mut response_rx, - &rocksdb_desc, - sessions_index, - &spec, - &mut state, - target_transactions, - timeout, - ) - .await?; - } - - std::mem::drop(request_tx); - verify("runtime", "EOF").is_eof(response_rx.try_next().await?)?; - - // Re-open RocksDB. - let rocksdb = RocksDB::open(Some(rocksdb_desc)).await?; - - let checkpoint = rocksdb.load_checkpoint().await?; - tracing::debug!(checkpoint = ?::ops::DebugJson(checkpoint), "final runtime checkpoint"); - - // Extract and yield the final connector state - let state = rocksdb - .load_connector_state(models::RawValue::default()) - .await?; - - () = co - .yield_(Response { - started_commit: Some(response::StartedCommit { - state: Some(flow::ConnectorState { - updated_json: state.into(), - merge_patch: false, - }), - }), - ..Default::default() - }) - .await; - - Ok(()) - }) -} - -async fn run_session( - co: &mut coroutines::Suspend, - reader: impl Reader, - request_tx: &mut mpsc::Sender>, - response_rx: &mut Pin<&mut impl ResponseStream>, - rocksdb_desc: &runtime::RocksDbDescriptor, - sessions_index: usize, - spec: &flow::CollectionSpec, - state: &mut models::RawValue, - target_transactions: usize, - timeout: std::time::Duration, -) -> anyhow::Result<()> { - let labeling = crate::parse_shard_labeling( - spec.derivation - .as_ref() - .and_then(|d| d.shard_template.as_ref()), - )?; - - // Send Open. - let open = Request { - open: Some(request::Open { - collection: Some(spec.clone()), - version: labeling.build.clone(), - range: Some(flow::RangeSpec { - key_begin: 0, - key_end: u32::MAX, - r_clock_begin: 0, - r_clock_end: u32::MAX, - }), - state_json: std::mem::take(state).into(), - }), - ..Default::default() - } - .with_internal(|internal| { - if sessions_index == 0 { - internal.rocksdb_descriptor = Some(rocksdb_desc.clone()); - } - internal.open = Some(derive_request_ext::Open { - sqlite_vfs_uri: format!("file://{}/sqlite.db", &rocksdb_desc.rocksdb_path), - }); - internal.set_log_level(labeling.log_level()); - }); - request_tx.try_send(Ok(open)).expect("sender is empty"); - - // Receive Opened. - let checkpoint = match response_rx.try_next().await? { - Some(opened) if opened.opened.is_some() => { - let checkpoint = opened - .opened - .as_ref() - .and_then(|opened| opened.runtime_checkpoint.clone()) - .unwrap_or_default(); - () = co.yield_(opened).await; - checkpoint - } - response => return verify("runtime", "Opened").fail(response), - }; - - let read_rx = reader.start_for_derivation(&spec, checkpoint); - tokio::pin!(read_rx); - - for _transaction in 0..target_transactions { - let deadline = tokio::time::sleep(timeout); - tokio::pin!(deadline); - - let mut started = false; - - // Read documents until a checkpoint. - let checkpoint = loop { - let read = tokio::select! { - read = read_rx.try_next() => read?, - () = deadline.as_mut(), if !started => { - tracing::info!(?timeout, "session ending upon reaching timeout"); - return Ok(()); - }, - }; - started = true; - - let (forward, checkpoint) = match read { - None => { - tracing::info!("session ending because reader returned EOF"); - return Ok(()); - } - // Forward a Read document to the runtime. - Some(Read::Document { binding, doc }) => ( - Request { - read: Some(request::Read { - doc_json: doc, - transform: binding, - ..Default::default() - }), - ..Default::default() - }, - None, - ), - // Forward a Flush to the runtime, then go on to commit a checkpoint. - Some(Read::Checkpoint(checkpoint)) => ( - Request { - flush: Some(request::Flush::default()), - ..Default::default() - }, - Some(checkpoint), - ), - }; - - () = crate::exchange(Ok(forward), request_tx, response_rx) - .try_for_each( - |response| async move { verify("runtime", "no response").fail(response) }, - ) - .await?; - - if let Some(checkpoint) = checkpoint { - break checkpoint; - } - }; - - // Receive Published and then Flushed. - let mut done = false; - while !done { - let verify = verify("runtime", "Published or Flushed"); - let response = verify.not_eof(response_rx.try_next().await?)?; - - done = match &response { - Response { - published: Some(_), .. - } => false, - Response { - flushed: Some(response::Flushed { .. }), - .. - } => true, - _ => return verify.fail(response), - }; - () = co.yield_(response).await; - } - - // Send StartCommit. - request_tx - .try_send(Ok(Request { - start_commit: Some(request::StartCommit { - runtime_checkpoint: Some(checkpoint), - }), - ..Default::default() - })) - .expect("sender is empty"); - - // Receive StartedCommit. - match response_rx.try_next().await? { - Some( - started_commit @ Response { - started_commit: Some(_), - .. - }, - ) => { - () = co.yield_(started_commit).await; - } - response => return verify("runtime", "StartedCommit").fail(response), - } - } - - Ok(()) -} diff --git a/crates/runtime/src/harness/materialize.rs b/crates/runtime/src/harness/materialize.rs deleted file mode 100644 index bf7a6f6b3e7..00000000000 --- a/crates/runtime/src/harness/materialize.rs +++ /dev/null @@ -1,286 +0,0 @@ -use super::{Read, Reader}; -use crate::materialize::ResponseStream; -use crate::{LogHandler, Runtime, rocksdb::RocksDB, verify}; -use anyhow::Context; -use futures::{TryStreamExt, channel::mpsc}; -use proto_flow::flow; -use proto_flow::materialize::{Request, Response, request, response}; -use proto_flow::runtime; -use std::pin::Pin; - -pub fn run_materialize( - reader: impl Reader, - runtime: Runtime, - sessions: Vec, - spec: &flow::MaterializationSpec, - mut state: models::RawValue, - state_dir: &std::path::Path, - timeout: std::time::Duration, - output_apply: bool, -) -> impl ResponseStream { - let spec = spec.clone(); - let state_dir = state_dir.to_owned(); - - coroutines::try_coroutine(move |mut co| async move { - let (mut request_tx, request_rx) = mpsc::channel(crate::CHANNEL_BUFFER); - let response_rx = runtime.serve_materialize(request_rx); - tokio::pin!(response_rx); - - let state_dir = state_dir.to_str().context("tempdir is not utf8")?; - let rocksdb_desc = runtime::RocksDbDescriptor { - rocksdb_env_memptr: 0, - rocksdb_path: state_dir.to_owned(), - }; - - for (sessions_index, target_transactions) in sessions.into_iter().enumerate() { - () = run_session( - &mut co, - reader.clone(), - &mut request_tx, - &mut response_rx, - &rocksdb_desc, - sessions_index, - &spec, - &mut state, - target_transactions, - timeout, - output_apply, - ) - .await?; - } - - std::mem::drop(request_tx); - verify("runtime", "EOF").is_eof(response_rx.try_next().await?)?; - - // Re-open RocksDB. - let rocksdb = RocksDB::open(Some(rocksdb_desc)).await?; - - let checkpoint = rocksdb.load_checkpoint().await?; - tracing::debug!(checkpoint = ?::ops::DebugJson(checkpoint), "final runtime checkpoint"); - - // Extract and yield the final connector state - let state = rocksdb - .load_connector_state(models::RawValue::default()) - .await?; - - () = co - .yield_(Response { - started_commit: Some(response::StartedCommit { - state: Some(flow::ConnectorState { - updated_json: state.into(), - merge_patch: false, - }), - }), - ..Default::default() - }) - .await; - - Ok(()) - }) -} - -async fn run_session( - co: &mut coroutines::Suspend, - reader: impl Reader, - request_tx: &mut mpsc::Sender>, - response_rx: &mut Pin<&mut impl ResponseStream>, - rocksdb_desc: &runtime::RocksDbDescriptor, - sessions_index: usize, - spec: &flow::MaterializationSpec, - state: &mut models::RawValue, - target_transactions: usize, - timeout: std::time::Duration, - output_apply: bool, -) -> anyhow::Result<()> { - let labeling = crate::parse_shard_labeling(spec.shard_template.as_ref())?; - - // Send Apply. - let apply = Request { - apply: Some(request::Apply { - materialization: Some(spec.clone()), - version: labeling.build.clone(), - last_materialization: None, - last_version: String::new(), - state_json: bytes::Bytes::new(), - }), - ..Default::default() - } - .with_internal(|internal| { - if sessions_index == 0 { - internal.rocksdb_descriptor = Some(rocksdb_desc.clone()); - } - internal.set_log_level(labeling.log_level()); - }); - request_tx.try_send(Ok(apply)).expect("sender is empty"); - - // Receive Applied. - match response_rx.try_next().await? { - Some(applied) if applied.applied.is_some() => { - if output_apply { - print!( - "[\"applied.actionDescription\", {:?}]\n", - applied.applied.as_ref().unwrap().action_description - ); - } - () = co.yield_(applied).await; - } - response => return verify("runtime", "Applied").fail(response), - } - - // Send Open. - let open = Request { - open: Some(request::Open { - materialization: Some(spec.clone()), - version: labeling.build.clone(), - range: Some(flow::RangeSpec { - key_begin: 0, - key_end: u32::MAX, - r_clock_begin: 0, - r_clock_end: u32::MAX, - }), - state_json: std::mem::take(state).into(), - }), - ..Default::default() - } - .with_internal(|internal| internal.set_log_level(labeling.log_level())); - request_tx.try_send(Ok(open)).expect("sender is empty"); - - // Receive Opened. - let verify_opened = verify("runtime", "Opened"); - let opened = verify_opened.not_eof(response_rx.try_next().await?)?; - let Response { - opened: Some(response::Opened { - runtime_checkpoint, .. - }), - .. - } = &opened - else { - return verify_opened.fail(opened); - }; - - let checkpoint = runtime_checkpoint.clone().unwrap_or_default(); - () = co.yield_(opened).await; - - // Send initial Acknowledge of the session. - request_tx - .try_send(Ok(Request { - acknowledge: Some(request::Acknowledge { - state_patches_json: bytes::Bytes::new(), // Not implemented. - }), - ..Default::default() - })) - .expect("sender is empty"); - - let read_rx = reader.start_for_materialization(&spec, checkpoint); - tokio::pin!(read_rx); - - for _transaction in 0..target_transactions { - let deadline = tokio::time::sleep(timeout); - tokio::pin!(deadline); - - let mut started = false; - let mut saw_acknowledged = false; - - // Read documents until a checkpoint. - let checkpoint = loop { - let read = tokio::select! { - read = read_rx.try_next() => read?, - () = deadline.as_mut(), if !started => { - tracing::info!(?timeout, "session ending upon reaching timeout"); - return Ok(()); - }, - }; - started = true; - - match read { - None => { - tracing::info!("session ending because reader returned EOF"); - return Ok(()); - } - Some(Read::Checkpoint(checkpoint)) => break checkpoint, // Commit below. - Some(Read::Document { binding, doc }) => { - // Forward to the runtime as a Load document. - let request = Request { - load: Some(request::Load { - binding, - key_json: doc, - ..Default::default() - }), - ..Default::default() - }; - - () = crate::exchange(Ok(request), request_tx, response_rx) - .try_for_each(|response| { - futures::future::ready(if response.acknowledged.is_some() { - saw_acknowledged = true; - Ok(()) - } else { - verify("runtime", "Acknowledged").fail(response) - }) - }) - .await?; - - continue; - } - }; - }; - - // Receive Acknowledged, if we haven't already. - if !saw_acknowledged { - match response_rx.try_next().await? { - Some(response) if response.acknowledged.is_some() => (), - response => return verify("runtime", "Acknowledged").fail(response), - } - } - - // Send Flush. - let flush = Request { - flush: Some(request::Flush { - state_patches_json: bytes::Bytes::new(), // Not implemented. - }), - ..Default::default() - }; - () = crate::exchange(Ok(flush), request_tx, response_rx) - .try_for_each(|response| async { verify("runtime", "no response").fail(response) }) - .await?; - - // Receive Flushed. - match response_rx.try_next().await? { - Some(response) if response.flushed.is_some() => { - () = co.yield_(response).await; - } - response => return verify("runtime", "Flushed").fail(response), - } - - // Send StartCommit. - request_tx - .try_send(Ok(Request { - start_commit: Some(request::StartCommit { - runtime_checkpoint: Some(checkpoint), - state_patches_json: bytes::Bytes::new(), // Not implemented. - }), - ..Default::default() - })) - .expect("sender is empty"); - - // Receive StartedCommit. - match response_rx.try_next().await? { - Some(response) if response.started_commit.is_some() => { - () = co.yield_(response).await; - } - response => return verify("runtime", "StartedCommit").fail(response), - } - - // Send Acknowledge. - request_tx - .try_send(Ok(Request { - acknowledge: Some(request::Acknowledge { - state_patches_json: bytes::Bytes::new(), // Not implemented. - }), - ..Default::default() - })) - .expect("sender is empty"); - } - - Ok(()) -} diff --git a/crates/runtime/src/harness/mod.rs b/crates/runtime/src/harness/mod.rs deleted file mode 100644 index 9f3d3529bdc..00000000000 --- a/crates/runtime/src/harness/mod.rs +++ /dev/null @@ -1,44 +0,0 @@ -use proto_flow::flow; -use proto_gazette::consumer; - -mod capture; -mod derive; -mod materialize; -pub mod streaming_fixture; - -// Routines for building test harness of captures, derivations, -// and materializations. All test harnesses have the same basic -// shape: -// * `sessions` is the number of times the underlying connector should be re-opened, -// exercising state & checkpoint recovery and resumption, and the target number -// of transactions for each session. -// * `delay` is artificial delay added between transactions, simulating back-pressure. -// * `timeout` is how long the task may produce no data before its current session ends, -// though a next may then start. -pub use capture::run_capture; -pub use derive::run_derive; -pub use materialize::run_materialize; - -pub enum Read { - Document { binding: u32, doc: bytes::Bytes }, - Checkpoint(consumer::Checkpoint), -} - -/// Reader is used for derivation and materialization test harnesses. -/// It builds a stream of read collection documents, which may come -/// from a data fixture or represent live journal data. -pub trait Reader: Clone + Send + Sync + 'static { - type Stream: futures::Stream> + Send + 'static; - - fn start_for_derivation( - self, - derivation: &flow::CollectionSpec, - resume: consumer::Checkpoint, - ) -> Self::Stream; - - fn start_for_materialization( - self, - materialization: &flow::MaterializationSpec, - resume: consumer::Checkpoint, - ) -> Self::Stream; -} diff --git a/crates/runtime/src/harness/streaming_fixture.rs b/crates/runtime/src/harness/streaming_fixture.rs deleted file mode 100644 index ee48724759e..00000000000 --- a/crates/runtime/src/harness/streaming_fixture.rs +++ /dev/null @@ -1,243 +0,0 @@ -use super::Read; -use anyhow::Context; -use futures::{StreamExt, stream::BoxStream}; -use proto_flow::flow; -use proto_gazette::consumer; -use std::collections::HashMap; -use tokio::io::{AsyncBufReadExt, BufReader}; - -// StreamingReader reads fixture documents line-by-line from a file. -// Each line is either: -// - A document: ["collection/name", {...document...}] -// - A commit marker: {"commit": true} -// -// The commit marker denotes a transaction boundary. All documents between -// two commit markers (or between the start and first commit) belong to the same transaction. -#[derive(Clone)] -pub struct StreamingReader { - pub path: std::path::PathBuf, -} - -impl super::Reader for StreamingReader { - type Stream = BoxStream<'static, anyhow::Result>; - - fn start_for_derivation( - self, - derivation: &flow::CollectionSpec, - resume: consumer::Checkpoint, - ) -> Self::Stream { - let transforms = &derivation.derivation.as_ref().unwrap().transforms; - - let index = transforms - .iter() - .enumerate() - .map(|(index, t)| { - let collection = t.collection.as_ref().unwrap(); - ( - collection.name.clone(), - (index, json::Pointer::from_str(&collection.uuid_ptr)), - ) - }) - .fold( - HashMap::>::new(), - |mut acc, item| { - if let Some(existing) = acc.get_mut(&item.0) { - existing.push(item.1); - } else { - acc.insert(item.0, vec![item.1]); - } - - acc - }, - ); - - self.start(index, resume) - } - - fn start_for_materialization( - self, - materialization: &flow::MaterializationSpec, - resume: consumer::Checkpoint, - ) -> Self::Stream { - let index = materialization - .bindings - .iter() - .enumerate() - .map(|(index, t)| { - let collection = t.collection.as_ref().unwrap(); - ( - collection.name.clone(), - (index, json::Pointer::from_str(&collection.uuid_ptr)), - ) - }) - .fold( - HashMap::>::new(), - |mut acc, item| { - if let Some(existing) = acc.get_mut(&item.0) { - existing.push(item.1); - } else { - acc.insert(item.0, vec![item.1]); - } - - acc - }, - ); - - self.start(index, resume) - } -} - -impl StreamingReader { - fn start( - self, - index: HashMap>, - resume: consumer::Checkpoint, - ) -> BoxStream<'static, anyhow::Result> { - let skip = resume - .sources - .get("fixture") - .as_ref() - .map(|source| source.read_through as usize) - .unwrap_or_default(); - - let producer = crate::uuid::Producer([7, 19, 83, 3, 3, 17]); - let path = self.path.clone(); - - coroutines::try_coroutine(move |mut co| async move { - let file = tokio::fs::File::open(&path) - .await - .context(format!("couldn't open streaming fixture file: {:?}", path))?; - - let reader = BufReader::new(file); - let mut lines = reader.lines(); - - let mut txn: usize = 0; - let mut offset: usize = 0; - let mut line_number: usize = 0; - - // Skip transactions we've already processed - let mut skipped = 0; - while skipped < skip { - line_number += 1; - let line = match lines.next_line().await? { - Some(line) => line, - None => return Ok(()), // Reached end of file - }; - - if is_commit_line(&line)? { - skipped += 1; - } - } - - loop { - line_number += 1; - let line = match lines.next_line().await? { - Some(line) => line, - None => break, // End of file - }; - let line = line.trim(); - if line.is_empty() { - continue; - } - - if is_commit_line(&line)? { - tracing::info!(line_number, txn, "detected commit, emitting checkpoint"); - // Emit a checkpoint for the completed transaction - () = co - .yield_(Read::Checkpoint(consumer::Checkpoint { - sources: [( - "fixture".to_string(), - consumer::checkpoint::Source { - read_through: 1 + txn as i64, - producers: Vec::new(), - }, - )] - .into(), - ack_intents: Default::default(), - })) - .await; - - txn += 1; - offset = 0; - continue; - } - - // Parse as document: [collection, doc] - let (collection, mut doc): (models::Collection, serde_json::Value) = - serde_json::from_str(&line).context(format!( - "couldn't parse fixture line {} as [collection, document]: '{}'", - line_number, line - ))?; - let Some(bindings) = index.get(collection.as_str()) else { - offset += 1; - continue; - }; - - for (binding, ptr) in bindings { - // Add a UUID fixture with a synthetic publication time. - let seconds = 3600 * txn + offset; // Synthetic timestamp of the document. - let uuid = crate::uuid::build( - producer, - crate::uuid::Clock::from_unix(seconds as u64, 0), - crate::uuid::Flags(0), - ); - - *json::ptr::create_value(ptr, &mut doc).expect("able to create fixture UUID") = - serde_json::json!(uuid.as_hyphenated()); - - () = co - .yield_(Read::Document { - binding: *binding as u32, - doc: doc.to_string().into(), - }) - .await; - } - - offset += 1; - - if line_number % 1000000 == 0 { - tracing::info!( - line_number, - txn, - offset, - "processed streaming fixture lines" - ); - } - } - - // If there are any remaining documents without a final ack, - // emit a checkpoint for the last transaction - if offset > 0 { - () = co - .yield_(Read::Checkpoint(consumer::Checkpoint { - sources: [( - "fixture".to_string(), - consumer::checkpoint::Source { - read_through: 1 + txn as i64, - producers: Vec::new(), - }, - )] - .into(), - ack_intents: Default::default(), - })) - .await; - } - - Ok(()) - }) - .boxed() - } -} - -// Helper function to check if a line is an commit marker -fn is_commit_line(line: &str) -> anyhow::Result { - // Try to parse as JSON object with "commit" field - if let Ok(val) = serde_json::from_str::(line) { - if let Some(obj) = val.as_object() { - if let Some(commit) = obj.get("commit") { - return Ok(commit.as_bool().unwrap_or(false)); - } - } - } - Ok(false) -} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 677bbf1aefc..9b2c42b6546 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -6,7 +6,6 @@ mod capture; mod combine; mod container; mod derive; -pub mod harness; mod image_connector; mod local_connector; mod materialize; @@ -310,47 +309,6 @@ impl Verify { } } -/// exchange is a combinator for avoiding deadlocks. It sends into a request -/// Stream while concurrently polling and yielding responses of a corresponding -/// response Stream. It returns a stream which completes once the send has -/// completed. -/// -/// `exchange` mitigates an extremely common deadlock mistake, of sending into -/// a receiver without consideration for whether the receiver may be unable to -/// receive because it's output channel is stuffed and is not being serviced. -/// This is a generalized problem -- in no way unique to Rust -- but the polled -/// nature of Futures and Streams accentuate it because receiving from a Stream -/// is *also* polling it, allowing it to perform other important activity even -/// if it cannot immediately yield an item. -fn exchange<'s, Request, Tx, Response, Rx>( - request: Request, - tx: &'s mut Tx, - rx: &'s mut Rx, -) -> impl futures::Stream + 's -where - Request: 'static, - Tx: futures::Sink + Unpin + 's, - Rx: futures::Stream + Unpin + 's, -{ - use futures::{SinkExt, StreamExt}; - - futures::stream::unfold((tx.feed(request), rx), move |(mut feed, rx)| async move { - tokio::select! { - biased; - - // We suppress a `feed` error, which represents a disconnection / reset, - // because a better and causal error will invariably be surfaced by `rx`. - _result = &mut feed => None, - - response = rx.next() => if let Some(response) = response { - Some((response, (feed, rx))) - } else { - None - }, - } - }) -} - fn truncate_chars(s: &str, max_chars: usize) -> &str { match s.char_indices().nth(max_chars) { None => s, diff --git a/crates/shuffle/src/log/mod.rs b/crates/shuffle/src/log/mod.rs index 8b289fcaed3..f2b58485c51 100644 --- a/crates/shuffle/src/log/mod.rs +++ b/crates/shuffle/src/log/mod.rs @@ -83,6 +83,7 @@ pub mod reader; mod state; pub mod writer; +pub use block::BlockMeta; pub use reader::{FrontierScan, Reader, Remainder}; pub use writer::Writer; diff --git a/go/protocols/runtime/runtime.pb.go b/go/protocols/runtime/runtime.pb.go index 570ec17ac51..2c9e254eab5 100644 --- a/go/protocols/runtime/runtime.pb.go +++ b/go/protocols/runtime/runtime.pb.go @@ -1515,9 +1515,8 @@ var xxx_messageInfo_Joined proto.InternalMessageInfo type Task struct { // Task specification (protobuf-encoded bytes). Spec []byte `protobuf:"bytes,1,opt,name=spec,proto3" json:"spec,omitempty"` - // When true, documents and stats are written to output and not directed to collections. - Preview bool `protobuf:"varint,2,opt,name=preview,proto3" json:"preview,omitempty"` - // Preview / harness control. Zero means unlimited. + // Maximum number of transactions to run before exiting. Zero means unlimited. + // Used by "preview" workflows. MaxTransactions uint32 `protobuf:"varint,3,opt,name=max_transactions,json=maxTransactions,proto3" json:"max_transactions,omitempty"` // URL of a SQLite VFS the shard threads to a SQLite derive connector via // `DeriveRequestExt.open.sqlite_vfs_uri` on C:Open. Set by the controller @@ -3691,276 +3690,276 @@ func init() { } var fileDescriptor_73af6e0737ce390c = []byte{ - // 4303 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5b, 0x4d, 0x6c, 0x1c, 0x47, - 0x76, 0xf6, 0xfc, 0xcf, 0xbc, 0x19, 0x92, 0xc3, 0x12, 0x49, 0xb5, 0x46, 0xb6, 0x48, 0x8f, 0xed, - 0x98, 0x16, 0xa9, 0x21, 0x4d, 0xdb, 0xbb, 0xb6, 0xe2, 0x3f, 0xfe, 0x69, 0x4d, 0x2d, 0x25, 0x71, - 0x8b, 0x94, 0x90, 0xe4, 0xd2, 0x68, 0x76, 0x15, 0x87, 0x2d, 0xf6, 0x74, 0xb5, 0xbb, 0x7b, 0x48, - 0x71, 0x4f, 0x01, 0x72, 0x09, 0x90, 0x43, 0x2e, 0xb9, 0x04, 0xb9, 0x24, 0xa7, 0xfc, 0x00, 0x39, - 0xe4, 0x14, 0x60, 0x0f, 0x39, 0xe5, 0x60, 0xec, 0x29, 0xc9, 0x21, 0x48, 0x2e, 0x02, 0xb2, 0xb9, - 0xe6, 0xb6, 0x09, 0x90, 0x08, 0x39, 0x04, 0xf5, 0xd3, 0xbf, 0xd3, 0x43, 0x51, 0xb4, 0x11, 0x18, - 0x8b, 0x3d, 0x48, 0xd3, 0xf5, 0xde, 0xf7, 0xaa, 0x5e, 0x55, 0xbd, 0x57, 0xf5, 0x55, 0x75, 0x13, - 0xba, 0x7d, 0xb6, 0xe2, 0x7a, 0x2c, 0x60, 0x26, 0xb3, 0xfd, 0x15, 0x6f, 0xe8, 0x04, 0xd6, 0x80, - 0x86, 0xbf, 0x3d, 0xa1, 0x41, 0x35, 0x55, 0xec, 0xdc, 0x3a, 0xf4, 0xd8, 0x09, 0xf5, 0x22, 0x83, - 0xe8, 0x41, 0x02, 0x3b, 0x0b, 0x26, 0x73, 0xfc, 0xe1, 0xe0, 0x02, 0x44, 0xba, 0x39, 0xd3, 0x70, - 0x83, 0xa1, 0x47, 0xc3, 0xdf, 0xb0, 0x96, 0x14, 0x86, 0x50, 0xcf, 0x3a, 0xa5, 0xea, 0x47, 0x21, - 0x5e, 0x4f, 0x21, 0x8e, 0x6c, 0x76, 0x26, 0xfe, 0x53, 0xda, 0xdb, 0x29, 0xed, 0xc0, 0x08, 0xa8, - 0x67, 0x19, 0xb6, 0xf5, 0x53, 0x9a, 0x7c, 0x56, 0xd8, 0x4e, 0x0a, 0xcb, 0x5c, 0xf1, 0x2f, 0xd7, - 0x57, 0xff, 0x78, 0x78, 0x74, 0x64, 0xd3, 0xf0, 0x57, 0x61, 0x66, 0xfa, 0xac, 0xcf, 0xc4, 0xe3, - 0x0a, 0x7f, 0x92, 0xd2, 0xee, 0xdf, 0x15, 0x60, 0xfa, 0xc0, 0xf0, 0x4f, 0xf6, 0xa9, 0x77, 0x6a, - 0x99, 0x74, 0x93, 0x39, 0x47, 0x56, 0x1f, 0xdd, 0x82, 0xa6, 0xcd, 0xfa, 0xfa, 0x91, 0x65, 0x53, + // 4299 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5b, 0x4b, 0x6c, 0x1c, 0x47, + 0x7a, 0xf6, 0xbc, 0x67, 0xfe, 0x19, 0x92, 0xc3, 0x12, 0x49, 0xb5, 0x46, 0xb6, 0x48, 0x8f, 0xed, + 0x98, 0x16, 0xa9, 0x21, 0x4d, 0xdb, 0xbb, 0xb6, 0x62, 0xd9, 0xe2, 0x4b, 0x6b, 0x6a, 0x29, 0x89, + 0x5b, 0xa4, 0x84, 0x24, 0x97, 0x46, 0xb3, 0xab, 0x38, 0x6c, 0xb1, 0xa7, 0xab, 0xdd, 0xdd, 0x43, + 0x89, 0x7b, 0x0a, 0x90, 0x4b, 0x80, 0x1c, 0x72, 0xd9, 0x4b, 0x90, 0x4b, 0x72, 0x0a, 0x12, 0x20, + 0x87, 0x9c, 0x02, 0xec, 0x21, 0xa7, 0x1c, 0x8c, 0x3d, 0x25, 0x39, 0x04, 0xc9, 0x45, 0x40, 0x36, + 0xd7, 0xdc, 0x36, 0x01, 0x12, 0x21, 0x87, 0xa0, 0x1e, 0xfd, 0x9c, 0x1e, 0x8a, 0xa2, 0x8d, 0xc0, + 0x58, 0xec, 0x41, 0x9a, 0xae, 0xff, 0x51, 0xf5, 0x57, 0xd5, 0xff, 0xf8, 0xaa, 0xba, 0x09, 0xdd, + 0x3e, 0x5b, 0x71, 0x3d, 0x16, 0x30, 0x93, 0xd9, 0xfe, 0x8a, 0x37, 0x74, 0x02, 0x6b, 0x40, 0xc3, + 0xdf, 0x9e, 0xe0, 0xa0, 0x9a, 0x6a, 0x76, 0x6e, 0x1c, 0x7a, 0xec, 0x84, 0x7a, 0x91, 0x42, 0xf4, + 0x20, 0x05, 0x3b, 0x0b, 0x26, 0x73, 0xfc, 0xe1, 0xe0, 0x1c, 0x89, 0xf4, 0x70, 0xa6, 0xe1, 0x06, + 0x43, 0x8f, 0x86, 0xbf, 0x61, 0x2f, 0x29, 0x19, 0x42, 0x3d, 0xeb, 0x94, 0xaa, 0x1f, 0x25, 0xf1, + 0x66, 0x4a, 0xe2, 0xc8, 0x66, 0xcf, 0xc4, 0x7f, 0x8a, 0x7b, 0x33, 0xc5, 0x1d, 0x18, 0x01, 0xf5, + 0x2c, 0xc3, 0xb6, 0x7e, 0x4a, 0x93, 0xcf, 0x4a, 0xb6, 0x93, 0x92, 0x65, 0xae, 0xf8, 0x97, 0x6b, + 0xab, 0x7f, 0x3c, 0x3c, 0x3a, 0xb2, 0x69, 0xf8, 0xab, 0x64, 0x66, 0xfa, 0xac, 0xcf, 0xc4, 0xe3, + 0x0a, 0x7f, 0x92, 0xd4, 0xee, 0xdf, 0x15, 0x60, 0xfa, 0xc0, 0xf0, 0x4f, 0xf6, 0xa9, 0x77, 0x6a, + 0x99, 0x74, 0x93, 0x39, 0x47, 0x56, 0x1f, 0xdd, 0x80, 0xa6, 0xcd, 0xfa, 0xfa, 0x91, 0x65, 0x53, 0xfd, 0x88, 0x68, 0x85, 0x85, 0xc2, 0x62, 0x05, 0x37, 0x6c, 0xd6, 0xbf, 0x67, 0xd9, 0xf4, 0x1e, - 0x41, 0x37, 0xa1, 0x11, 0x18, 0xfe, 0x89, 0xee, 0x18, 0x03, 0xaa, 0x15, 0x17, 0x0a, 0x8b, 0x0d, - 0x5c, 0xe7, 0x82, 0x87, 0xc6, 0x80, 0xa2, 0x1b, 0x50, 0x1f, 0x12, 0x5f, 0x77, 0x8d, 0xe0, 0x58, - 0x2b, 0x09, 0x5d, 0x6d, 0x48, 0xfc, 0x3d, 0x23, 0x38, 0x46, 0x4b, 0x30, 0x6d, 0x32, 0x27, 0x30, - 0x2c, 0x87, 0x7a, 0xba, 0x43, 0x83, 0x33, 0xe6, 0x9d, 0x68, 0x65, 0x81, 0x69, 0x47, 0x8a, 0x87, - 0x52, 0x8e, 0xde, 0x86, 0x8a, 0x6b, 0x1b, 0x0e, 0xd5, 0xaa, 0x0b, 0x85, 0xc5, 0xc9, 0xb5, 0xc9, - 0x5e, 0x38, 0xd5, 0x7b, 0x5c, 0x8a, 0xa5, 0xb2, 0xfb, 0x3f, 0x65, 0x98, 0xdc, 0x97, 0x1d, 0xc5, - 0xf4, 0xeb, 0x21, 0xf5, 0x03, 0xb4, 0x03, 0xb5, 0xa7, 0x6c, 0xe8, 0x39, 0x86, 0x2d, 0x3c, 0x6f, - 0x6c, 0xac, 0xbc, 0x78, 0x3e, 0xbf, 0xd4, 0x67, 0xbd, 0xbe, 0xf1, 0x53, 0x1a, 0x04, 0xb4, 0x47, - 0xe8, 0xe9, 0x8a, 0xc9, 0x3c, 0xba, 0x92, 0x09, 0x92, 0xde, 0x7d, 0x69, 0x86, 0x43, 0x7b, 0x34, - 0x07, 0x55, 0x8f, 0xba, 0xb6, 0x71, 0x2e, 0x7a, 0x59, 0xc7, 0xaa, 0xc4, 0xfb, 0x78, 0x38, 0xb4, - 0x6c, 0xa2, 0x5b, 0x24, 0xec, 0xa3, 0x28, 0xef, 0x10, 0x74, 0x0f, 0xaa, 0xec, 0xe8, 0xc8, 0xa7, - 0x81, 0xe8, 0x58, 0x69, 0xa3, 0xf7, 0xe2, 0xf9, 0xfc, 0xed, 0xcb, 0x34, 0xfe, 0x48, 0x58, 0x61, - 0x65, 0x8d, 0x1e, 0x00, 0x50, 0x87, 0xe8, 0xaa, 0xae, 0xca, 0x95, 0xea, 0x6a, 0x50, 0x87, 0xc8, - 0x47, 0xb4, 0x04, 0x15, 0xcf, 0x70, 0xfa, 0x72, 0x34, 0x9b, 0x6b, 0x53, 0x3d, 0x11, 0x86, 0x98, - 0x8b, 0xf6, 0x5d, 0x6a, 0x6e, 0x94, 0xbf, 0x79, 0x3e, 0xff, 0x1a, 0x96, 0x18, 0xb4, 0x0f, 0x4d, - 0x93, 0x31, 0x8f, 0x58, 0x8e, 0x11, 0x30, 0x4f, 0xab, 0x89, 0x51, 0x7c, 0xff, 0xc5, 0xf3, 0xf9, - 0x3b, 0x79, 0x8d, 0x8f, 0xa4, 0x52, 0x6f, 0xff, 0xd8, 0xf0, 0xc8, 0xce, 0x16, 0x4e, 0xd6, 0x82, - 0x56, 0x01, 0x3c, 0xea, 0x33, 0x7b, 0x18, 0x58, 0xcc, 0xd1, 0xea, 0xc2, 0x8d, 0x76, 0x2f, 0xb2, - 0xf9, 0x8a, 0x1a, 0x84, 0x7a, 0x38, 0x81, 0x41, 0x6f, 0xc1, 0x84, 0x8a, 0x61, 0xdd, 0x72, 0x08, - 0x7d, 0xa6, 0x35, 0x16, 0x0a, 0x8b, 0x13, 0xb8, 0xa5, 0x84, 0x3b, 0x5c, 0x86, 0x3e, 0x04, 0x10, - 0x19, 0x67, 0x88, 0x6a, 0x41, 0x54, 0x3b, 0x23, 0x7b, 0xb7, 0xc9, 0x6c, 0x9b, 0x9a, 0x5c, 0xce, - 0xbb, 0x88, 0x13, 0x38, 0xb4, 0x09, 0x53, 0x71, 0x8a, 0x49, 0xd3, 0xa6, 0x30, 0xbd, 0x21, 0x4d, - 0x1f, 0xa4, 0x95, 0xc2, 0x3e, 0x6b, 0xd1, 0xfd, 0xa7, 0x32, 0x4c, 0x45, 0xb1, 0xe7, 0xbb, 0xcc, - 0xf1, 0x29, 0x5a, 0x84, 0xaa, 0x1f, 0x18, 0xc1, 0xd0, 0x17, 0xb1, 0x37, 0xb9, 0xd6, 0xee, 0x85, - 0xc3, 0xd3, 0xdb, 0x17, 0x72, 0xac, 0xf4, 0x1c, 0x79, 0x2c, 0xfa, 0x2c, 0x62, 0x2b, 0x6f, 0x2c, - 0x94, 0x1e, 0xbd, 0x03, 0x93, 0x01, 0xf5, 0x06, 0x96, 0x63, 0xd8, 0x3a, 0xf5, 0x3c, 0xe6, 0xa9, - 0x98, 0x9b, 0x08, 0xa5, 0xdb, 0x5c, 0x88, 0x7e, 0x02, 0x2d, 0x8f, 0x1a, 0x44, 0x0f, 0x8e, 0x3d, - 0x36, 0xec, 0x1f, 0x5f, 0x31, 0xfe, 0x9a, 0xbc, 0x8e, 0x03, 0x59, 0x05, 0x0f, 0xc2, 0x33, 0xcf, - 0x0a, 0xa8, 0xce, 0x3d, 0xb9, 0x6a, 0x10, 0x8a, 0x1a, 0x78, 0x97, 0xd0, 0x0e, 0x54, 0x0c, 0x8f, - 0x3a, 0x86, 0x08, 0xc2, 0xd6, 0xc6, 0x07, 0x2f, 0x9e, 0xcf, 0xaf, 0xf4, 0xad, 0xe0, 0x78, 0x78, - 0xd8, 0x33, 0xd9, 0x60, 0x85, 0xfa, 0xc1, 0xd0, 0xf0, 0xce, 0xe5, 0x32, 0x39, 0xb2, 0x70, 0xf6, - 0xd6, 0xb9, 0x29, 0x96, 0x35, 0xa0, 0x77, 0xa0, 0x4c, 0x98, 0xe9, 0x6b, 0xb5, 0x85, 0xd2, 0x62, - 0x73, 0xad, 0x29, 0x67, 0x6d, 0xdf, 0xb6, 0x4c, 0xaa, 0x42, 0x59, 0xa8, 0xd1, 0x57, 0x50, 0x93, - 0x19, 0xe4, 0x6b, 0xf5, 0x85, 0xd2, 0x15, 0xbc, 0x0f, 0xcd, 0x79, 0x9c, 0x0d, 0x87, 0x16, 0xd1, - 0x5d, 0xc3, 0x0b, 0x7c, 0xad, 0x21, 0x9a, 0x55, 0x59, 0xf4, 0xf8, 0xf1, 0xce, 0xd6, 0x1e, 0x17, - 0xab, 0xa6, 0x1b, 0x1c, 0x28, 0x04, 0x3c, 0xe8, 0x5d, 0xc3, 0x3c, 0xa1, 0x44, 0x3f, 0xa1, 0xe7, - 0x1a, 0x8c, 0x73, 0xb6, 0x21, 0x41, 0x3f, 0xa6, 0xe7, 0x5d, 0x02, 0xd3, 0x98, 0x99, 0x27, 0xfe, - 0xd6, 0xc6, 0x16, 0xf5, 0x4d, 0xcf, 0x72, 0x79, 0xee, 0x2c, 0x03, 0xf2, 0xb8, 0x90, 0x1c, 0xea, - 0xd4, 0x39, 0xd5, 0x07, 0x74, 0xe0, 0x06, 0x9e, 0x88, 0xb0, 0x2a, 0x6e, 0x2b, 0xcd, 0xb6, 0x73, - 0xfa, 0x40, 0xc8, 0xd1, 0x9b, 0xd0, 0x0a, 0xd1, 0x62, 0x15, 0x96, 0x2b, 0x74, 0x53, 0xc9, 0xf8, - 0x4a, 0xdc, 0xfd, 0xa3, 0x22, 0x34, 0x36, 0xc3, 0x15, 0x17, 0x5d, 0x87, 0x9a, 0xe5, 0xea, 0x06, - 0x21, 0xb2, 0xce, 0x06, 0xae, 0x5a, 0xee, 0x3a, 0x21, 0x1e, 0xfa, 0x01, 0x4c, 0xa8, 0x65, 0x5a, - 0x77, 0x19, 0xef, 0x77, 0x51, 0xf4, 0x60, 0x5a, 0xf6, 0x40, 0xad, 0xd4, 0x7b, 0xcc, 0x0b, 0x70, - 0xcb, 0x89, 0x0b, 0x3e, 0xda, 0x87, 0xe9, 0x81, 0xe1, 0xba, 0x94, 0xe8, 0xc7, 0xcc, 0x0f, 0x94, - 0x6d, 0x49, 0xd8, 0xbe, 0x1b, 0xad, 0xe3, 0x51, 0xfb, 0xbd, 0x07, 0x02, 0xfb, 0x15, 0xf3, 0x03, - 0x61, 0xbe, 0xed, 0x04, 0xde, 0x39, 0x4f, 0xb7, 0x94, 0x14, 0xbd, 0x01, 0x30, 0xf4, 0x8d, 0x3e, - 0xd5, 0x3d, 0x23, 0xa0, 0x22, 0xba, 0x8b, 0xb8, 0x21, 0x24, 0xd8, 0x08, 0x68, 0x67, 0x03, 0x66, - 0xf2, 0xea, 0x41, 0x6d, 0x28, 0xf1, 0xb1, 0x2f, 0x88, 0xb5, 0x83, 0x3f, 0xa2, 0x19, 0xa8, 0x9c, - 0x1a, 0xf6, 0x30, 0xdc, 0xba, 0x64, 0xe1, 0x6e, 0xf1, 0xe3, 0x42, 0xf7, 0xaf, 0x8a, 0x30, 0xbd, - 0x29, 0xb7, 0x78, 0xb5, 0x9b, 0x6c, 0x3f, 0xe3, 0x6b, 0x27, 0xdf, 0xfb, 0x74, 0x9b, 0x9e, 0x52, - 0x5b, 0xa5, 0xf5, 0x64, 0x8f, 0xef, 0xbe, 0xbb, 0xac, 0xdf, 0xdb, 0xe5, 0x52, 0x5c, 0xb7, 0x59, - 0x5f, 0x3c, 0xa1, 0x9d, 0x78, 0xaa, 0x48, 0x34, 0x81, 0x2a, 0xc5, 0x3b, 0x51, 0xdf, 0x47, 0xa6, - 0x18, 0x4f, 0x2b, 0xab, 0xc4, 0xac, 0xef, 0x40, 0xcb, 0x0f, 0x0c, 0x2f, 0xd0, 0x4d, 0x36, 0x18, - 0x58, 0x81, 0xc8, 0xfa, 0xe6, 0xda, 0x6f, 0xc4, 0x03, 0x98, 0xf5, 0x94, 0x2f, 0x31, 0x5e, 0xb0, - 0x29, 0xd0, 0xb8, 0xe9, 0xc7, 0x85, 0x0e, 0x86, 0x66, 0x42, 0x87, 0x36, 0x01, 0xa9, 0x4a, 0x74, - 0xf3, 0x98, 0x9a, 0x27, 0x2e, 0xb3, 0x9c, 0x40, 0x74, 0x8d, 0x2f, 0x9e, 0xd1, 0x8a, 0xb5, 0x19, - 0xe9, 0xf0, 0xb4, 0xc2, 0xc7, 0xa2, 0xee, 0xff, 0x96, 0x01, 0x45, 0x2e, 0xc8, 0xe5, 0x8f, 0x8f, - 0xd6, 0x2a, 0x34, 0xa2, 0xbd, 0x5c, 0x55, 0x89, 0x46, 0xe7, 0x1c, 0xc7, 0x20, 0x74, 0x17, 0xaa, - 0xcc, 0xa5, 0x0e, 0x25, 0x6a, 0x98, 0xba, 0xa3, 0x3d, 0x8c, 0xaa, 0xef, 0x3d, 0x12, 0x48, 0xac, - 0x2c, 0xd0, 0x97, 0x50, 0x57, 0x9c, 0x8c, 0xa8, 0xf1, 0x79, 0xfb, 0x22, 0x6b, 0x25, 0x22, 0x38, - 0xb2, 0x42, 0xf7, 0x00, 0x12, 0x63, 0x50, 0x1e, 0x37, 0xc6, 0x89, 0x3a, 0xe2, 0x51, 0x49, 0x58, - 0x76, 0x1e, 0x40, 0x55, 0xfa, 0xf6, 0x9d, 0x8c, 0x6e, 0xe7, 0x09, 0xd4, 0x43, 0x67, 0x79, 0xe4, - 0x9f, 0xd0, 0x73, 0x5d, 0x2e, 0x12, 0xa2, 0xa2, 0x16, 0x6e, 0x9c, 0xd0, 0xf3, 0x3d, 0x21, 0xe0, - 0xb4, 0x8a, 0xaf, 0x4a, 0x16, 0xdf, 0x94, 0xfc, 0x10, 0x55, 0x14, 0xa8, 0x76, 0xac, 0x90, 0xe0, - 0xce, 0x19, 0x40, 0xdc, 0x0a, 0x5a, 0x80, 0x0a, 0xdf, 0x8e, 0x7c, 0xe5, 0x1d, 0x88, 0xb0, 0xe6, - 0x1b, 0x95, 0x8f, 0xa5, 0x02, 0xfd, 0x08, 0x9a, 0x2e, 0xb3, 0x6d, 0xdd, 0xa3, 0xfe, 0xd0, 0x0e, - 0x44, 0xb5, 0x93, 0x17, 0x8f, 0xcf, 0x1e, 0xb3, 0x6d, 0x2c, 0xd0, 0x18, 0xdc, 0xe8, 0xb9, 0xfb, - 0x10, 0x20, 0xd6, 0xa0, 0x26, 0xd4, 0x76, 0x1e, 0x3e, 0x59, 0xdf, 0xdd, 0xd9, 0x6a, 0xbf, 0x86, - 0x1a, 0x50, 0xc1, 0xdb, 0xeb, 0x5b, 0xbf, 0xdd, 0x2e, 0xa0, 0x09, 0x68, 0x3c, 0x7c, 0x74, 0xa0, - 0xcb, 0x62, 0x11, 0xb5, 0xa0, 0xbe, 0xf9, 0xe8, 0xd1, 0xae, 0xfe, 0xe8, 0xde, 0xbd, 0x76, 0x89, - 0x1b, 0xe1, 0xed, 0xfd, 0x83, 0x75, 0x7c, 0xd0, 0x2e, 0x77, 0xff, 0xa3, 0x00, 0xed, 0x2d, 0xc1, - 0xb5, 0xbf, 0x07, 0xa9, 0xba, 0x06, 0x65, 0x1e, 0x90, 0x2a, 0x04, 0x6f, 0x45, 0xc6, 0x59, 0x07, - 0x45, 0xf8, 0x62, 0x81, 0xed, 0x2c, 0x43, 0x99, 0x97, 0xd0, 0xdb, 0x30, 0xe9, 0x7f, 0x6d, 0xf3, - 0x5d, 0xf6, 0xf4, 0xc8, 0xd7, 0x87, 0x9e, 0xa5, 0x16, 0xe1, 0x96, 0x94, 0x3e, 0x39, 0xf2, 0x1f, - 0x7b, 0x56, 0xf7, 0x3f, 0x4b, 0x30, 0x1d, 0xd6, 0xf6, 0x6d, 0x92, 0xed, 0x93, 0x4c, 0xb2, 0xbd, - 0x39, 0xe2, 0xeb, 0xd8, 0x5c, 0xdb, 0x80, 0x86, 0x3b, 0x3c, 0xb4, 0x2d, 0xff, 0x38, 0x27, 0xd9, - 0x46, 0xad, 0xf7, 0x42, 0x2c, 0x8e, 0xcd, 0xd0, 0xa7, 0x50, 0x3b, 0xb2, 0x87, 0xa2, 0x86, 0x72, - 0x26, 0xd9, 0x47, 0x6b, 0xb8, 0x27, 0x91, 0x38, 0x34, 0xf9, 0xae, 0x73, 0x2c, 0x80, 0x46, 0xe4, - 0x24, 0x3f, 0xd4, 0x0c, 0x8c, 0x67, 0xba, 0x69, 0x33, 0xf3, 0x44, 0x6d, 0xad, 0xf5, 0x81, 0xf1, - 0x6c, 0x93, 0x97, 0x33, 0x19, 0x58, 0xbc, 0x54, 0x06, 0x96, 0xc6, 0x64, 0xe0, 0x12, 0xd4, 0x54, - 0xc7, 0x5e, 0x9e, 0x7e, 0xdd, 0x3f, 0x2c, 0xc0, 0x6c, 0x4c, 0x46, 0xbf, 0x07, 0xa1, 0xde, 0xfd, - 0x59, 0x01, 0xe6, 0x52, 0x1e, 0x7d, 0x9b, 0x68, 0x5c, 0x8f, 0xc3, 0x41, 0x3a, 0x13, 0xd3, 0x83, - 0xfc, 0x36, 0x46, 0x63, 0xe2, 0x95, 0x86, 0xf3, 0x67, 0x65, 0x98, 0xdc, 0x64, 0x83, 0x43, 0xcb, - 0x89, 0x8e, 0x8b, 0xab, 0x2a, 0x75, 0xa5, 0xcd, 0xeb, 0x09, 0x7f, 0x93, 0xb0, 0x44, 0xe2, 0xa2, - 0x3b, 0x50, 0x32, 0x48, 0xe8, 0xf0, 0xcd, 0x71, 0x06, 0xeb, 0x84, 0x60, 0x8e, 0xeb, 0xfc, 0x73, - 0x51, 0x25, 0xfa, 0x97, 0x50, 0x3f, 0xb4, 0x1c, 0x62, 0x39, 0x7d, 0xee, 0x61, 0x29, 0xbd, 0x57, - 0x8d, 0xb6, 0xd6, 0xdb, 0x90, 0x60, 0x1c, 0x59, 0x75, 0xfe, 0xa0, 0x08, 0x35, 0x25, 0x45, 0x08, - 0xca, 0x47, 0x43, 0x5b, 0x4e, 0x7d, 0x1d, 0x8b, 0xe7, 0x90, 0xeb, 0x70, 0x96, 0xd6, 0x90, 0x5c, - 0xe7, 0x63, 0x68, 0xba, 0x1e, 0x7b, 0x2a, 0x8f, 0x41, 0x21, 0x07, 0x6b, 0x4b, 0xfe, 0xb6, 0x17, - 0x29, 0x14, 0x0d, 0x4d, 0x42, 0xd1, 0x67, 0xd0, 0xf4, 0xcd, 0x63, 0x3a, 0x30, 0xf4, 0xa7, 0x3e, - 0x73, 0x44, 0xb6, 0xb6, 0x36, 0x5e, 0x7f, 0xf1, 0x7c, 0x5e, 0xa3, 0x8e, 0xc9, 0xb8, 0x0b, 0x2b, - 0x5c, 0xd1, 0xc3, 0xc6, 0xd9, 0x03, 0xea, 0x0b, 0x1a, 0x06, 0xd2, 0xe0, 0xbe, 0xcf, 0x1c, 0xd4, - 0x03, 0xf0, 0xa9, 0xa7, 0xbb, 0xcc, 0xb6, 0xcc, 0x73, 0x71, 0x74, 0x88, 0xf8, 0xf2, 0x3e, 0xf5, - 0xf6, 0x84, 0x18, 0x37, 0xfc, 0xf0, 0x51, 0x5c, 0x1b, 0x08, 0x7e, 0x1d, 0x78, 0xe2, 0x78, 0xd0, - 0xc0, 0x35, 0x41, 0xa3, 0x03, 0x8f, 0x9f, 0xc2, 0x05, 0x45, 0x93, 0x6c, 0xbf, 0x81, 0x55, 0xa9, - 0xe3, 0x40, 0x69, 0x9d, 0x10, 0xa4, 0x41, 0x4d, 0x0d, 0x90, 0x22, 0x79, 0x61, 0x11, 0xfd, 0x10, - 0xea, 0x84, 0x99, 0xd2, 0xff, 0xe2, 0x25, 0xfc, 0xaf, 0x11, 0x66, 0x0a, 0xe7, 0x67, 0xa0, 0x72, - 0xe4, 0x31, 0x47, 0x52, 0xae, 0x3a, 0x96, 0x85, 0xee, 0xbf, 0x14, 0x60, 0x2a, 0x9a, 0x27, 0x75, - 0xde, 0x1b, 0xdf, 0xb8, 0x06, 0x35, 0x42, 0x6d, 0x1a, 0xa8, 0xd0, 0xae, 0xe3, 0xb0, 0x98, 0x72, - 0xab, 0x74, 0x25, 0xb7, 0xca, 0x09, 0xb7, 0x32, 0x6b, 0x53, 0x25, 0xbb, 0x36, 0xbd, 0x05, 0x13, - 0x72, 0xbc, 0x42, 0x84, 0x38, 0x7c, 0xe1, 0x96, 0x14, 0x4a, 0x50, 0xf7, 0x3a, 0xcc, 0x6e, 0x32, - 0xc7, 0xa1, 0x66, 0xc0, 0xbc, 0x3d, 0x8f, 0x3d, 0x3b, 0x57, 0x81, 0xd8, 0xfd, 0x93, 0x02, 0xcc, - 0x65, 0x35, 0xaa, 0xeb, 0xf7, 0xa1, 0xc6, 0x8f, 0x0c, 0xd4, 0xf7, 0xd5, 0x3d, 0xcb, 0xea, 0x8b, - 0xe7, 0xf3, 0xcb, 0x97, 0x39, 0x5b, 0x6d, 0x3b, 0x44, 0xae, 0xc9, 0x61, 0x05, 0x7c, 0xf6, 0x5d, - 0x5e, 0xb9, 0x6e, 0x11, 0xc5, 0xca, 0x6b, 0xa2, 0xbc, 0x43, 0x50, 0x07, 0x4a, 0x36, 0xeb, 0xab, - 0xfd, 0xa6, 0x1e, 0xae, 0x70, 0x98, 0x0b, 0xbb, 0x7f, 0x53, 0x82, 0xf2, 0x7d, 0x66, 0x39, 0xe8, - 0x36, 0x4c, 0xd3, 0xc0, 0x24, 0xfa, 0x80, 0x11, 0xdd, 0xa3, 0xa7, 0x96, 0xcf, 0x4f, 0xf4, 0xdc, - 0xab, 0x12, 0x9e, 0xe2, 0x8a, 0x07, 0x8c, 0x60, 0x25, 0x46, 0x4b, 0x50, 0xf5, 0x8f, 0x0d, 0x8f, - 0x84, 0xa7, 0x99, 0x6b, 0x51, 0x12, 0xf2, 0xaa, 0xe4, 0xe5, 0x05, 0x56, 0x10, 0x34, 0x0f, 0x4d, - 0xf1, 0xa4, 0x6e, 0x20, 0x4a, 0x62, 0x8e, 0x41, 0x88, 0xe4, 0xfd, 0xc3, 0x12, 0x4c, 0x87, 0x97, - 0x14, 0xc4, 0xf2, 0xc4, 0x30, 0x9d, 0x87, 0x77, 0x5a, 0x4a, 0xb1, 0x15, 0xca, 0xd1, 0x7b, 0x10, - 0xca, 0x74, 0xaa, 0xc6, 0x40, 0x4c, 0x58, 0x03, 0x4f, 0x29, 0x79, 0x38, 0x34, 0xe8, 0x5d, 0x98, - 0xb2, 0xc5, 0xf1, 0x3f, 0x46, 0xca, 0xb4, 0x98, 0x94, 0xe2, 0x10, 0xd8, 0xf9, 0xeb, 0x02, 0x54, - 0x84, 0xcf, 0x68, 0x12, 0x8a, 0x16, 0x51, 0xe4, 0xa1, 0x68, 0x11, 0xd4, 0x83, 0xba, 0x6d, 0x1c, - 0x52, 0x9b, 0x07, 0x67, 0x51, 0xad, 0xc6, 0x62, 0x45, 0xe4, 0xe8, 0x5d, 0xa5, 0xc1, 0x11, 0x06, - 0xad, 0x41, 0xcd, 0xa3, 0x06, 0xf7, 0x54, 0x8d, 0xb6, 0x16, 0x5f, 0x49, 0xec, 0x79, 0xcc, 0xa4, - 0xbe, 0xbf, 0xef, 0x52, 0xb3, 0xb7, 0xb3, 0x85, 0x43, 0x20, 0x5a, 0x85, 0x19, 0x31, 0xf0, 0xa6, - 0x47, 0x8d, 0x80, 0xc6, 0x63, 0x2f, 0x2e, 0x1f, 0x30, 0xe2, 0xba, 0x4d, 0xa1, 0x0a, 0x87, 0xbf, - 0xfb, 0x21, 0x54, 0xf9, 0x38, 0x53, 0xc2, 0x27, 0x8d, 0xef, 0xb8, 0xc2, 0x3e, 0x3b, 0x69, 0x03, - 0xe3, 0xd9, 0x76, 0x60, 0x46, 0x93, 0xd6, 0xfd, 0x8b, 0x02, 0x94, 0x0f, 0x0c, 0xff, 0x84, 0x2f, - 0x7b, 0xbe, 0x4b, 0x4d, 0xc5, 0x82, 0xc5, 0x33, 0x4f, 0x35, 0x97, 0x57, 0x40, 0xcf, 0xc2, 0x54, - 0x53, 0x45, 0x3e, 0xe0, 0xbc, 0x89, 0xc0, 0x33, 0x1c, 0xdf, 0x88, 0xd6, 0x40, 0x3e, 0x87, 0xbc, - 0x85, 0x83, 0x84, 0x38, 0x87, 0x86, 0x95, 0x47, 0x69, 0x18, 0x3f, 0x5b, 0x87, 0x64, 0xc6, 0xe3, - 0xc1, 0x2a, 0xd3, 0xad, 0x19, 0xc9, 0x76, 0x48, 0xf7, 0xcf, 0x2a, 0x50, 0xc3, 0xd4, 0x64, 0xa7, - 0x62, 0x7f, 0x6b, 0x1a, 0xe6, 0x89, 0x6e, 0x39, 0x01, 0x75, 0x82, 0x70, 0xd5, 0x5f, 0x88, 0x37, - 0x5c, 0x09, 0xeb, 0xad, 0x9b, 0x27, 0x3b, 0x12, 0x22, 0xcf, 0xbe, 0x60, 0x44, 0x02, 0xb4, 0x06, - 0xb3, 0xf2, 0xfc, 0x17, 0x50, 0xc2, 0xd9, 0x89, 0x4f, 0x15, 0x47, 0x29, 0x0a, 0x8e, 0x72, 0x2d, - 0x52, 0x6e, 0x72, 0x9d, 0xa4, 0x2b, 0x5f, 0x02, 0x8a, 0x6d, 0xc4, 0x2a, 0x61, 0xd1, 0x70, 0x52, - 0xa7, 0x7b, 0xe1, 0xc5, 0xf0, 0x3d, 0xa5, 0xc0, 0xd3, 0x11, 0x38, 0x14, 0xa1, 0x65, 0x98, 0x31, - 0xc3, 0xb4, 0xd7, 0xf9, 0xde, 0x49, 0x13, 0xdb, 0x00, 0x9e, 0x8c, 0x74, 0x7c, 0x77, 0xa5, 0x68, - 0x19, 0xd0, 0x31, 0xef, 0x63, 0xda, 0xc1, 0x8a, 0xbc, 0x9f, 0x90, 0x9a, 0x84, 0x77, 0x77, 0x61, - 0x4a, 0xa1, 0x23, 0xd7, 0xaa, 0xe3, 0x5c, 0x9b, 0x94, 0xc8, 0xc8, 0xaf, 0x37, 0xa1, 0x65, 0x1b, - 0x7e, 0xa0, 0x1b, 0xae, 0x6b, 0x5b, 0x94, 0x88, 0xbb, 0xc9, 0x16, 0x6e, 0x72, 0xd9, 0xba, 0x14, - 0xa1, 0x75, 0x98, 0xb6, 0x69, 0xdf, 0x30, 0xcf, 0x93, 0xcc, 0xb0, 0x7e, 0x01, 0x33, 0x6c, 0x4b, - 0x78, 0xe2, 0x58, 0xf4, 0x31, 0x70, 0xea, 0xa7, 0x9f, 0xd0, 0xf3, 0xf0, 0xaa, 0xe7, 0x8d, 0x91, - 0x39, 0x7b, 0x60, 0x3c, 0xfb, 0x31, 0x3d, 0x57, 0x13, 0x56, 0x1b, 0xc8, 0x12, 0xba, 0x0d, 0xd7, - 0x02, 0xcf, 0xea, 0xf7, 0xf9, 0xd6, 0x67, 0x78, 0xc6, 0xc0, 0x97, 0xc3, 0x06, 0xc2, 0xcd, 0x09, - 0xa5, 0xda, 0x13, 0x9a, 0xce, 0x67, 0x30, 0x95, 0x99, 0xf8, 0xe4, 0x65, 0x45, 0x23, 0xe7, 0xb2, - 0xa2, 0x95, 0xb8, 0xac, 0xe8, 0xdc, 0x85, 0x56, 0xd2, 0x87, 0x97, 0x5d, 0x74, 0x24, 0x6d, 0xbb, - 0x3f, 0xaf, 0x41, 0x6d, 0x8f, 0x7a, 0xbe, 0xe5, 0x07, 0x68, 0x16, 0xaa, 0x3e, 0xfd, 0x5a, 0x77, - 0x98, 0x30, 0x2d, 0xe3, 0x8a, 0x4f, 0xbf, 0x7e, 0xc8, 0xf8, 0x9c, 0xca, 0x0d, 0x4b, 0x4f, 0x46, - 0xb0, 0xcc, 0xaf, 0xb6, 0xd4, 0xc4, 0xde, 0x67, 0x03, 0xbd, 0x94, 0x09, 0x74, 0xd5, 0xd6, 0xd5, - 0x02, 0xbd, 0x3c, 0x3e, 0xd0, 0xef, 0xc2, 0x0d, 0xe5, 0x64, 0x4e, 0xbc, 0x57, 0x84, 0xaf, 0xd7, - 0x25, 0x60, 0x73, 0x24, 0xc4, 0xf3, 0x93, 0xa4, 0xfa, 0x0a, 0x49, 0xb2, 0x0a, 0x73, 0x71, 0x92, - 0xb8, 0x46, 0x60, 0x1e, 0x53, 0x35, 0xdf, 0x32, 0x2c, 0xdb, 0x91, 0x76, 0x4f, 0x2a, 0xc7, 0x24, - 0x4a, 0x7d, 0x4c, 0xa2, 0x7c, 0x08, 0x73, 0xaa, 0x77, 0xd9, 0x7c, 0x69, 0x88, 0xae, 0xcd, 0x48, - 0xed, 0x57, 0xe9, 0x14, 0xc9, 0x49, 0x2f, 0xb8, 0x6a, 0x7a, 0x35, 0x47, 0xd3, 0xeb, 0x63, 0xd0, - 0x94, 0x53, 0xa3, 0x59, 0xd6, 0x12, 0x6e, 0x29, 0xa7, 0x77, 0xb3, 0x59, 0x95, 0x9b, 0x98, 0x13, - 0x57, 0x4e, 0xcc, 0xc9, 0x4c, 0x62, 0x86, 0x31, 0x96, 0x9f, 0x98, 0x6b, 0x30, 0xab, 0xdc, 0x4e, - 0xe7, 0xa7, 0x36, 0x25, 0x7c, 0xbe, 0x26, 0x95, 0x07, 0xc9, 0x04, 0x1d, 0x97, 0xcc, 0xed, 0xef, - 0x59, 0x32, 0x77, 0xa1, 0xa1, 0xfa, 0x4e, 0xc9, 0x98, 0x6c, 0xee, 0xfe, 0x79, 0x01, 0x2a, 0x7c, - 0x06, 0xcf, 0xc7, 0x6d, 0xa0, 0xa7, 0xbc, 0x06, 0xc5, 0x93, 0x1b, 0x38, 0x2c, 0xf2, 0x53, 0xb1, - 0x08, 0x08, 0x61, 0x22, 0x17, 0xff, 0x3a, 0x17, 0x70, 0x22, 0x10, 0x45, 0x4b, 0x68, 0x2b, 0xa9, - 0x8c, 0x88, 0x96, 0x27, 0xca, 0x7e, 0x75, 0xcc, 0x3e, 0x22, 0x49, 0x28, 0x4a, 0xef, 0x23, 0x9c, - 0xe4, 0x76, 0x9f, 0x42, 0x2d, 0x0c, 0xb5, 0x3b, 0x80, 0xe4, 0xee, 0x1c, 0x1d, 0x5a, 0x43, 0x86, - 0xd0, 0xc0, 0xd3, 0x52, 0xb3, 0x15, 0x2b, 0x2e, 0x48, 0xc7, 0x62, 0x7e, 0x3a, 0x76, 0x7f, 0x59, - 0x50, 0x47, 0xb3, 0x57, 0x1b, 0x94, 0x77, 0xc2, 0x97, 0x69, 0xa5, 0xdc, 0x97, 0x69, 0xe1, 0x6b, - 0xb4, 0xb7, 0x2e, 0xdc, 0x43, 0xc5, 0x89, 0x94, 0xa2, 0x8f, 0x12, 0x11, 0x5d, 0x11, 0x11, 0x1d, - 0x9f, 0xc7, 0xc5, 0x29, 0x30, 0x37, 0x9c, 0xbf, 0x55, 0xbc, 0x00, 0xd4, 0xc5, 0x22, 0xf3, 0x90, - 0x9d, 0x75, 0xab, 0x50, 0xde, 0x0f, 0x98, 0xdb, 0x6d, 0x40, 0x8d, 0xff, 0xba, 0x94, 0x74, 0x7f, - 0x0b, 0x9a, 0xfb, 0xd4, 0xe7, 0x1d, 0xdd, 0x65, 0xcc, 0x1d, 0x73, 0x75, 0x50, 0xb8, 0xca, 0xd5, - 0xc1, 0x1f, 0x57, 0xa1, 0xa6, 0x2e, 0x0c, 0xd1, 0x7b, 0x89, 0x11, 0x6f, 0xae, 0xcd, 0xf6, 0xc2, - 0x37, 0xeb, 0xe1, 0x09, 0x58, 0x0c, 0xa4, 0x9c, 0x88, 0xdf, 0x84, 0x09, 0xfe, 0xab, 0x7b, 0xea, - 0xe4, 0xa1, 0xc8, 0xec, 0x5c, 0xc2, 0x46, 0x2a, 0xa4, 0x51, 0x8b, 0x83, 0xa3, 0x53, 0xca, 0x47, - 0x50, 0x27, 0x96, 0x2f, 0xb6, 0x6c, 0x35, 0x5d, 0x37, 0x46, 0xda, 0xda, 0x52, 0x00, 0x1c, 0x41, - 0xd1, 0xa7, 0x00, 0xe1, 0x73, 0x74, 0x55, 0xf5, 0xfa, 0x68, 0x83, 0x5b, 0x11, 0x06, 0x27, 0xf0, - 0xbc, 0xd1, 0x53, 0xc3, 0xb6, 0x88, 0x11, 0x50, 0x75, 0xf4, 0x1d, 0x6d, 0xf4, 0x89, 0x02, 0xe0, - 0x08, 0x8a, 0x3e, 0x81, 0x46, 0xf8, 0x4c, 0xd4, 0x46, 0x74, 0x73, 0xb4, 0xcd, 0xd0, 0x90, 0xe0, - 0x18, 0x9d, 0xbe, 0x0d, 0x6a, 0xbc, 0xe4, 0x36, 0xe8, 0x87, 0xd0, 0xf2, 0xe5, 0x0c, 0xeb, 0x36, - 0x63, 0xae, 0x36, 0xa3, 0xd6, 0xe0, 0x70, 0x32, 0x13, 0xd3, 0x8f, 0x9b, 0x7e, 0x22, 0x16, 0xde, - 0x84, 0xf2, 0x53, 0x66, 0x39, 0xda, 0xac, 0x30, 0x98, 0x48, 0x1d, 0x9c, 0xb0, 0x50, 0xa1, 0x77, - 0xa1, 0xfa, 0x54, 0xd0, 0x7b, 0x6d, 0x4e, 0x25, 0x47, 0x12, 0x44, 0x09, 0x56, 0x6a, 0x5e, 0x57, - 0x60, 0xf8, 0x27, 0xda, 0xf5, 0x4c, 0x5d, 0x9c, 0xe5, 0x63, 0xa1, 0x42, 0x2b, 0xd1, 0x5d, 0xa5, - 0x26, 0x40, 0xd7, 0xb3, 0xd7, 0xce, 0xd9, 0x1b, 0xca, 0x1e, 0x34, 0xe4, 0xbe, 0xea, 0xb0, 0x33, - 0xed, 0x86, 0xda, 0xf4, 0x22, 0x1b, 0x15, 0xf3, 0xb8, 0x6e, 0xaa, 0x27, 0xee, 0x83, 0x1f, 0x30, - 0x57, 0xeb, 0x64, 0x7c, 0xe0, 0xa9, 0x80, 0x85, 0x0a, 0xdd, 0x86, 0x9a, 0x2f, 0x13, 0x43, 0xbb, - 0xa9, 0xde, 0xd3, 0x26, 0x51, 0x2e, 0x25, 0x38, 0x04, 0x74, 0xee, 0x46, 0xd7, 0x93, 0xaf, 0x7c, - 0x13, 0xd6, 0xfd, 0xd7, 0x39, 0x68, 0x26, 0xae, 0xbc, 0xd0, 0x9d, 0x54, 0x7e, 0xdc, 0xe8, 0x25, - 0xbf, 0x08, 0xc9, 0xc9, 0x91, 0x2f, 0xf2, 0x73, 0xa4, 0x93, 0xb1, 0x1b, 0x9f, 0x27, 0x9f, 0x24, - 0x42, 0x56, 0xe6, 0xc9, 0x1b, 0xb9, 0x6d, 0xe6, 0x84, 0xed, 0x67, 0xc9, 0xb0, 0x95, 0xa9, 0x32, - 0x9f, 0xdf, 0xee, 0xcb, 0x43, 0xb7, 0xf2, 0x2b, 0x12, 0xba, 0xb7, 0xf9, 0x59, 0x5a, 0xae, 0x3a, - 0x5a, 0x26, 0x6c, 0xd4, 0x01, 0x02, 0x87, 0x00, 0xf4, 0x36, 0x54, 0x38, 0xdf, 0x3a, 0x57, 0x11, - 0x1b, 0x7f, 0xe9, 0x22, 0x36, 0x6c, 0x2c, 0x95, 0xbc, 0xc6, 0x90, 0x95, 0x75, 0x32, 0x35, 0xaa, - 0xfd, 0x12, 0x87, 0x00, 0xee, 0xa0, 0xb8, 0xd3, 0xbc, 0x99, 0x71, 0x30, 0x71, 0x89, 0xf9, 0x41, - 0x94, 0x5b, 0xaf, 0x67, 0xee, 0x31, 0x13, 0x51, 0x98, 0xcd, 0xaf, 0x3b, 0x50, 0xb6, 0x99, 0x41, - 0xb4, 0x45, 0x15, 0x94, 0x79, 0x26, 0xbb, 0xcc, 0x20, 0x58, 0xc0, 0x78, 0x1b, 0xfc, 0x97, 0x12, - 0xed, 0xbd, 0x0b, 0xda, 0xd8, 0x15, 0x10, 0xac, 0xa0, 0x68, 0x15, 0x2a, 0xe2, 0x6a, 0x57, 0xbb, - 0x9d, 0xd9, 0x62, 0x92, 0x36, 0xe2, 0xc6, 0x17, 0x4b, 0x20, 0xfa, 0x41, 0x7c, 0x89, 0xbc, 0x94, - 0xb9, 0xc4, 0x1d, 0xb1, 0x49, 0xdc, 0x1c, 0xf3, 0x96, 0xfc, 0x80, 0x79, 0x54, 0x5b, 0xbe, 0xa0, - 0xa5, 0x7d, 0x8e, 0xc0, 0x12, 0xc8, 0x3b, 0x24, 0x1e, 0x88, 0x76, 0xe7, 0x82, 0x0e, 0x09, 0x13, - 0x82, 0x15, 0x14, 0x6d, 0x66, 0x5e, 0xe3, 0xf6, 0x84, 0xe9, 0xc2, 0x18, 0xd3, 0xfc, 0x17, 0xb8, - 0x68, 0x07, 0x26, 0x45, 0x91, 0x9f, 0x1c, 0x64, 0x35, 0x2b, 0x99, 0xd7, 0x27, 0x23, 0xd5, 0x50, - 0xa2, 0x2a, 0x9a, 0xf0, 0x93, 0x45, 0xb4, 0x21, 0x8e, 0x6a, 0x0e, 0x3b, 0xb3, 0x29, 0xe9, 0x53, - 0x6d, 0xf5, 0x02, 0x77, 0xd6, 0x63, 0x1c, 0x4e, 0x1a, 0xa1, 0x6d, 0x68, 0x25, 0x8a, 0x44, 0x7b, - 0x3f, 0xf3, 0x2e, 0x69, 0x4c, 0x25, 0x04, 0xa7, 0xcc, 0x78, 0x4c, 0xbb, 0x92, 0xb9, 0x6a, 0x6b, - 0x99, 0x98, 0x56, 0x8c, 0x16, 0x87, 0x00, 0xbe, 0xa4, 0xba, 0x21, 0xcb, 0xd5, 0x3e, 0xc8, 0x2c, - 0xa9, 0x11, 0xff, 0xc5, 0x31, 0x28, 0xbd, 0x1b, 0x7c, 0x78, 0xf9, 0xdd, 0xe0, 0xd3, 0x4b, 0xed, - 0x06, 0x9f, 0xbd, 0x6c, 0x37, 0xf8, 0xbd, 0xc2, 0xd5, 0xb7, 0x03, 0xf4, 0xa3, 0x24, 0x77, 0x4c, - 0x1c, 0x97, 0x8a, 0x17, 0x1c, 0x97, 0xae, 0x45, 0x16, 0x89, 0x77, 0x5c, 0x1f, 0x41, 0x99, 0x27, - 0x18, 0xba, 0x03, 0xf5, 0xe8, 0x38, 0x58, 0x18, 0x77, 0x1c, 0x8c, 0x20, 0x9d, 0x5f, 0x16, 0xa1, - 0x2a, 0x13, 0x13, 0x7d, 0x31, 0xf2, 0xda, 0xe2, 0xad, 0x0b, 0xf2, 0x78, 0xf4, 0xad, 0x85, 0x3c, - 0x03, 0x88, 0x6b, 0x73, 0x4f, 0x97, 0x5f, 0x70, 0x1c, 0x9e, 0x07, 0x54, 0xde, 0x25, 0x94, 0xf9, - 0x19, 0x40, 0xea, 0x1e, 0x73, 0xd5, 0x06, 0xd7, 0x74, 0xfe, 0xab, 0x10, 0xbf, 0xe7, 0x98, 0x81, - 0x8a, 0xbc, 0x7b, 0x95, 0xdc, 0x56, 0x16, 0xd0, 0x22, 0xb4, 0x07, 0x96, 0xa3, 0xfb, 0x6c, 0xe8, - 0x99, 0xe9, 0x0b, 0xb1, 0xc9, 0x81, 0xe5, 0xec, 0x0b, 0xb1, 0x3c, 0x44, 0x2f, 0xca, 0x2b, 0xc0, - 0x14, 0xb2, 0xa4, 0x90, 0xc6, 0xb3, 0x24, 0x72, 0x19, 0x90, 0x44, 0x11, 0x9d, 0x30, 0xd3, 0xd7, - 0x03, 0x16, 0x18, 0xb6, 0xd8, 0xd0, 0xca, 0xb8, 0xad, 0x34, 0x5b, 0xcc, 0xf4, 0x0f, 0xb8, 0x1c, - 0xf5, 0xe0, 0x5a, 0x88, 0x16, 0xdd, 0x51, 0xf0, 0x8a, 0x80, 0x4f, 0x2b, 0x95, 0xe8, 0x8e, 0xc4, - 0x77, 0x61, 0x42, 0x11, 0x7d, 0x9d, 0x50, 0x3b, 0x50, 0x1f, 0x41, 0xe1, 0xa6, 0x64, 0xf4, 0x5b, - 0x5c, 0xd4, 0xf9, 0x04, 0x2a, 0x62, 0x95, 0xba, 0xe0, 0x28, 0x53, 0xc8, 0x3f, 0xca, 0x74, 0xfe, - 0xbb, 0x10, 0xbf, 0x07, 0xbb, 0xe8, 0x45, 0x53, 0xce, 0x8a, 0x98, 0x3b, 0x65, 0xaf, 0x78, 0x94, - 0xea, 0x9c, 0xbf, 0x6c, 0xc6, 0x6e, 0xc3, 0xb4, 0x5c, 0xe1, 0x93, 0x83, 0x2b, 0x43, 0x60, 0x4a, - 0x2a, 0xe2, 0xb1, 0x5d, 0x06, 0xa4, 0xb0, 0xc9, 0xa1, 0x2d, 0xc9, 0x99, 0x90, 0x9a, 0x78, 0x64, - 0x3b, 0x35, 0xa8, 0x88, 0x25, 0xb7, 0xf3, 0xf7, 0x05, 0xa8, 0xca, 0xc5, 0xf7, 0xd2, 0x41, 0x2b, - 0xe1, 0x39, 0xaf, 0xda, 0x2e, 0xd3, 0x1f, 0xb9, 0xc0, 0xe7, 0xf4, 0x47, 0x2a, 0x52, 0xfd, 0x51, - 0xd8, 0x9c, 0xfe, 0x48, 0x4d, 0xa2, 0x3f, 0xbf, 0x5f, 0x48, 0x7f, 0xad, 0xf3, 0xca, 0xc1, 0xf0, - 0xdd, 0xad, 0x1e, 0xeb, 0x30, 0x91, 0xda, 0x4b, 0xae, 0x10, 0x98, 0x5f, 0x40, 0x33, 0xb1, 0x03, - 0x5c, 0xa1, 0x82, 0x2f, 0xa1, 0x95, 0xdc, 0x42, 0x5e, 0xbd, 0x86, 0xee, 0xdf, 0x22, 0xa8, 0xca, - 0xaf, 0x0b, 0xd0, 0x62, 0x8a, 0x56, 0xcf, 0xf4, 0xd4, 0xd7, 0xda, 0x39, 0x8c, 0xfa, 0x6e, 0x3e, - 0xa3, 0x9e, 0x8d, 0x4d, 0xc6, 0x93, 0xe9, 0x0f, 0x47, 0xc8, 0xb4, 0x96, 0x6d, 0x29, 0x87, 0x47, - 0x7f, 0x3c, 0xca, 0xa3, 0x3b, 0x23, 0xad, 0xfd, 0x9a, 0x42, 0xe7, 0x51, 0xe8, 0x4b, 0x10, 0xde, - 0x5e, 0x86, 0xf0, 0xce, 0x65, 0x3e, 0x3c, 0xc9, 0x72, 0xdd, 0xc5, 0x14, 0xd7, 0x9d, 0xc9, 0xa2, - 0x13, 0x34, 0xb7, 0x97, 0xa1, 0xb9, 0x73, 0x79, 0xd8, 0x04, 0xc3, 0x5d, 0x4a, 0x33, 0xdc, 0xd9, - 0x2c, 0x3c, 0x45, 0x6e, 0xdf, 0xcf, 0x92, 0xdb, 0xeb, 0xb9, 0xf0, 0x24, 0xaf, 0x5d, 0x4a, 0xf3, - 0xda, 0x91, 0xfa, 0x53, 0x94, 0xb6, 0x97, 0xa1, 0xb4, 0x73, 0xb9, 0xe8, 0x98, 0xcd, 0x7e, 0x9e, - 0xcb, 0x66, 0x6f, 0x8e, 0x5a, 0x8d, 0x21, 0xb2, 0x5b, 0x63, 0x88, 0xec, 0x1b, 0xb9, 0x35, 0x8c, - 0xe3, 0xb0, 0xbf, 0x26, 0x8e, 0xdf, 0x57, 0xe2, 0xf8, 0xf3, 0x98, 0x38, 0xde, 0x1d, 0xd9, 0x83, - 0x6f, 0xe5, 0x67, 0xc6, 0x77, 0xc2, 0x19, 0xff, 0xf1, 0x57, 0x8f, 0x33, 0x7e, 0x1b, 0x3e, 0xf8, - 0x28, 0xa6, 0x83, 0xaf, 0xce, 0x1f, 0x10, 0x94, 0x07, 0x7c, 0x01, 0x91, 0x6f, 0xfb, 0xc4, 0x73, - 0xcc, 0xb2, 0x7e, 0xb7, 0x14, 0xb1, 0xac, 0x55, 0x98, 0x89, 0x3e, 0xed, 0x4b, 0xf6, 0x5f, 0xbe, - 0x7a, 0x40, 0x91, 0x2e, 0x1e, 0x81, 0x35, 0x98, 0x8d, 0x2d, 0x92, 0x63, 0x20, 0x27, 0xf6, 0x5a, - 0xa4, 0x4c, 0x30, 0xe7, 0x65, 0x40, 0xc4, 0xe3, 0xd1, 0x9d, 0x6a, 0x43, 0xb1, 0x27, 0xa5, 0x49, - 0x8d, 0x71, 0x88, 0x4e, 0xd6, 0x2f, 0xa7, 0x64, 0x5a, 0xa9, 0x12, 0xb5, 0xff, 0x04, 0xda, 0xf1, - 0x1b, 0x7d, 0xb5, 0x24, 0x55, 0x32, 0x5f, 0x01, 0xa7, 0x96, 0xc2, 0xe8, 0xc3, 0x46, 0x4f, 0xad, - 0x4d, 0x53, 0x6e, 0x5a, 0xd0, 0xd1, 0x61, 0x2a, 0x83, 0x41, 0x1d, 0xf1, 0x81, 0x0b, 0x19, 0x9a, - 0x2a, 0x8b, 0x5a, 0x38, 0x2a, 0xf3, 0x68, 0x4d, 0x06, 0xa3, 0x2c, 0x70, 0x0b, 0xf5, 0x67, 0x48, - 0xf2, 0x75, 0x6a, 0x03, 0x47, 0xe5, 0xce, 0x93, 0x34, 0x41, 0x1c, 0x97, 0xf3, 0x85, 0x57, 0xcd, - 0xf9, 0xa9, 0x0c, 0xdd, 0xbb, 0xbd, 0x04, 0x15, 0xf1, 0xe7, 0x56, 0x08, 0xa0, 0xba, 0xf7, 0x78, - 0x63, 0x77, 0x67, 0xb3, 0xfd, 0x1a, 0x6a, 0x42, 0x6d, 0x0f, 0xef, 0x3c, 0x59, 0x3f, 0xd8, 0x6e, - 0x17, 0x50, 0x03, 0x2a, 0xbb, 0x8f, 0x36, 0xd7, 0x77, 0xdb, 0xc5, 0xb5, 0xfb, 0x50, 0x57, 0x7f, - 0x0e, 0xe3, 0xa1, 0xcf, 0xa1, 0xa6, 0x9e, 0x51, 0xbc, 0x61, 0xa5, 0xff, 0x50, 0xab, 0xa3, 0x8d, - 0x2a, 0x24, 0xc9, 0x59, 0x2d, 0xac, 0xed, 0x42, 0x5d, 0x7d, 0x6a, 0xe5, 0xa1, 0x2f, 0xa1, 0xa6, - 0x9e, 0x13, 0x75, 0xa5, 0x3f, 0x98, 0x4b, 0xd4, 0x95, 0xf9, 0x42, 0x6b, 0xb1, 0xb0, 0x5a, 0x58, - 0x3b, 0x86, 0xc9, 0xf4, 0x47, 0x4c, 0xe8, 0x09, 0x4c, 0x89, 0x87, 0x48, 0xec, 0xa3, 0x5b, 0xc9, - 0x85, 0x75, 0xf4, 0x53, 0xa8, 0xce, 0xfc, 0x58, 0x7d, 0xa2, 0xa5, 0x33, 0xa8, 0xee, 0xca, 0xbf, - 0xda, 0xe9, 0x45, 0x9c, 0x73, 0x2a, 0x13, 0x47, 0x9d, 0xac, 0x80, 0x5b, 0xa2, 0xcf, 0xd2, 0xf7, - 0xbf, 0x33, 0x79, 0xc7, 0x95, 0x4e, 0xae, 0x54, 0x34, 0xfc, 0x97, 0xd1, 0x67, 0x40, 0xef, 0xc7, - 0x2f, 0x59, 0xda, 0xd9, 0x0b, 0xf3, 0xce, 0x88, 0x44, 0xb4, 0xfd, 0xff, 0xeb, 0xeb, 0xc6, 0xe7, - 0xdf, 0xfc, 0xdb, 0xad, 0xd7, 0xbe, 0xf9, 0xc5, 0xad, 0xc2, 0x3f, 0xfc, 0xe2, 0x56, 0xe1, 0x4f, - 0xff, 0xfd, 0x56, 0xe1, 0x77, 0x96, 0x2f, 0xf5, 0x67, 0x40, 0xaa, 0xbe, 0xc3, 0xaa, 0x10, 0x7d, - 0xf0, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xc0, 0xdd, 0xc1, 0x81, 0x13, 0x3a, 0x00, 0x00, + 0x41, 0xd7, 0xa1, 0x11, 0x18, 0xfe, 0x89, 0xee, 0x18, 0x03, 0xaa, 0x15, 0x17, 0x0a, 0x8b, 0x0d, + 0x5c, 0xe7, 0x84, 0x87, 0xc6, 0x80, 0xa2, 0x6b, 0x50, 0x1f, 0x12, 0x5f, 0x77, 0x8d, 0xe0, 0x58, + 0x2b, 0x09, 0x5e, 0x6d, 0x48, 0xfc, 0x3d, 0x23, 0x38, 0x46, 0x4b, 0x30, 0x6d, 0x32, 0x27, 0x30, + 0x2c, 0x87, 0x7a, 0xba, 0x43, 0x83, 0x67, 0xcc, 0x3b, 0xd1, 0xca, 0x42, 0xa6, 0x1d, 0x31, 0x1e, + 0x4a, 0x3a, 0x7a, 0x17, 0x2a, 0xae, 0x6d, 0x38, 0x54, 0xab, 0x2e, 0x14, 0x16, 0x27, 0xd7, 0x26, + 0x7b, 0xe1, 0x56, 0xef, 0x71, 0x2a, 0x96, 0xcc, 0xee, 0xff, 0x94, 0x61, 0x72, 0x5f, 0x4e, 0x14, + 0xd3, 0xaf, 0x87, 0xd4, 0x0f, 0xd0, 0x0e, 0xd4, 0x9e, 0xb2, 0xa1, 0xe7, 0x18, 0xb6, 0xb0, 0xbc, + 0xb1, 0xb1, 0xf2, 0xf2, 0xc5, 0xfc, 0x52, 0x9f, 0xf5, 0xfa, 0xc6, 0x4f, 0x69, 0x10, 0xd0, 0x1e, + 0xa1, 0xa7, 0x2b, 0x26, 0xf3, 0xe8, 0x4a, 0xc6, 0x49, 0x7a, 0xf7, 0xa5, 0x1a, 0x0e, 0xf5, 0xd1, + 0x1c, 0x54, 0x3d, 0xea, 0xda, 0xc6, 0x99, 0x98, 0x65, 0x1d, 0xab, 0x16, 0x9f, 0xe3, 0xe1, 0xd0, + 0xb2, 0x89, 0x6e, 0x91, 0x70, 0x8e, 0xa2, 0xbd, 0x43, 0xd0, 0x3d, 0xa8, 0xb2, 0xa3, 0x23, 0x9f, + 0x06, 0x62, 0x62, 0xa5, 0x8d, 0xde, 0xcb, 0x17, 0xf3, 0x37, 0x2f, 0x32, 0xf8, 0x23, 0xa1, 0x85, + 0x95, 0x36, 0x7a, 0x00, 0x40, 0x1d, 0xa2, 0xab, 0xbe, 0x2a, 0x97, 0xea, 0xab, 0x41, 0x1d, 0x22, + 0x1f, 0xd1, 0x12, 0x54, 0x3c, 0xc3, 0xe9, 0xcb, 0xd5, 0x6c, 0xae, 0x4d, 0xf5, 0x84, 0x1b, 0x62, + 0x4e, 0xda, 0x77, 0xa9, 0xb9, 0x51, 0xfe, 0xe6, 0xc5, 0xfc, 0x1b, 0x58, 0xca, 0xa0, 0x7d, 0x68, + 0x9a, 0x8c, 0x79, 0xc4, 0x72, 0x8c, 0x80, 0x79, 0x5a, 0x4d, 0xac, 0xe2, 0x87, 0x2f, 0x5f, 0xcc, + 0xdf, 0xca, 0x1b, 0x7c, 0x24, 0x94, 0x7a, 0xfb, 0xc7, 0x86, 0x47, 0x76, 0xb6, 0x70, 0xb2, 0x17, + 0xb4, 0x0a, 0xe0, 0x51, 0x9f, 0xd9, 0xc3, 0xc0, 0x62, 0x8e, 0x56, 0x17, 0x66, 0xb4, 0x7b, 0x91, + 0xce, 0x57, 0xd4, 0x20, 0xd4, 0xc3, 0x09, 0x19, 0xf4, 0x0e, 0x4c, 0x28, 0x1f, 0xd6, 0x2d, 0x87, + 0xd0, 0xe7, 0x5a, 0x63, 0xa1, 0xb0, 0x38, 0x81, 0x5b, 0x8a, 0xb8, 0xc3, 0x69, 0xe8, 0x63, 0x00, + 0x11, 0x71, 0x86, 0xe8, 0x16, 0x44, 0xb7, 0x33, 0x72, 0x76, 0x9b, 0xcc, 0xb6, 0xa9, 0xc9, 0xe9, + 0x7c, 0x8a, 0x38, 0x21, 0x87, 0x36, 0x61, 0x2a, 0x0e, 0x31, 0xa9, 0xda, 0x14, 0xaa, 0xd7, 0xa4, + 0xea, 0x83, 0x34, 0x53, 0xe8, 0x67, 0x35, 0xba, 0xff, 0x54, 0x86, 0xa9, 0xc8, 0xf7, 0x7c, 0x97, + 0x39, 0x3e, 0x45, 0x8b, 0x50, 0xf5, 0x03, 0x23, 0x18, 0xfa, 0xc2, 0xf7, 0x26, 0xd7, 0xda, 0xbd, + 0x70, 0x79, 0x7a, 0xfb, 0x82, 0x8e, 0x15, 0x9f, 0x4b, 0x1e, 0x8b, 0x39, 0x0b, 0xdf, 0xca, 0x5b, + 0x0b, 0xc5, 0x47, 0xef, 0xc1, 0x64, 0x40, 0xbd, 0x81, 0xe5, 0x18, 0xb6, 0x4e, 0x3d, 0x8f, 0x79, + 0xca, 0xe7, 0x26, 0x42, 0xea, 0x36, 0x27, 0xa2, 0x9f, 0x40, 0xcb, 0xa3, 0x06, 0xd1, 0x83, 0x63, + 0x8f, 0x0d, 0xfb, 0xc7, 0x97, 0xf4, 0xbf, 0x26, 0xef, 0xe3, 0x40, 0x76, 0xc1, 0x9d, 0xf0, 0x99, + 0x67, 0x05, 0x54, 0xe7, 0x96, 0x5c, 0xd6, 0x09, 0x45, 0x0f, 0x7c, 0x4a, 0x68, 0x07, 0x2a, 0x86, + 0x47, 0x1d, 0x43, 0x38, 0x61, 0x6b, 0xe3, 0xa3, 0x97, 0x2f, 0xe6, 0x57, 0xfa, 0x56, 0x70, 0x3c, + 0x3c, 0xec, 0x99, 0x6c, 0xb0, 0x42, 0xfd, 0x60, 0x68, 0x78, 0x67, 0x32, 0x4d, 0x8e, 0x24, 0xce, + 0xde, 0x3a, 0x57, 0xc5, 0xb2, 0x07, 0xf4, 0x1e, 0x94, 0x09, 0x33, 0x7d, 0xad, 0xb6, 0x50, 0x5a, + 0x6c, 0xae, 0x35, 0xe5, 0xae, 0xed, 0xdb, 0x96, 0x49, 0x95, 0x2b, 0x0b, 0x36, 0xfa, 0x0a, 0x6a, + 0x32, 0x82, 0x7c, 0xad, 0xbe, 0x50, 0xba, 0x84, 0xf5, 0xa1, 0x3a, 0xf7, 0xb3, 0xe1, 0xd0, 0x22, + 0xba, 0x6b, 0x78, 0x81, 0xaf, 0x35, 0xc4, 0xb0, 0x2a, 0x8a, 0x1e, 0x3f, 0xde, 0xd9, 0xda, 0xe3, + 0x64, 0x35, 0x74, 0x83, 0x0b, 0x0a, 0x02, 0x77, 0x7a, 0xd7, 0x30, 0x4f, 0x28, 0xd1, 0x4f, 0xe8, + 0x99, 0x06, 0xe3, 0x8c, 0x6d, 0x48, 0xa1, 0x1f, 0xd3, 0xb3, 0x2e, 0x81, 0x69, 0xcc, 0xcc, 0x13, + 0x7f, 0x6b, 0x63, 0x8b, 0xfa, 0xa6, 0x67, 0xb9, 0x3c, 0x76, 0x96, 0x01, 0x79, 0x9c, 0x48, 0x0e, + 0x75, 0xea, 0x9c, 0xea, 0x03, 0x3a, 0x70, 0x03, 0x4f, 0x78, 0x58, 0x15, 0xb7, 0x15, 0x67, 0xdb, + 0x39, 0x7d, 0x20, 0xe8, 0xe8, 0x6d, 0x68, 0x85, 0xd2, 0x22, 0x0b, 0xcb, 0x0c, 0xdd, 0x54, 0x34, + 0x9e, 0x89, 0xbb, 0x3f, 0x2b, 0x42, 0x63, 0x33, 0xcc, 0xb8, 0xe8, 0x2a, 0xd4, 0x2c, 0x57, 0x37, + 0x08, 0x91, 0x7d, 0x36, 0x70, 0xd5, 0x72, 0xd7, 0x09, 0xf1, 0xd0, 0x0f, 0x60, 0x42, 0xa5, 0x69, + 0xdd, 0x65, 0x7c, 0xde, 0x45, 0x31, 0x83, 0x69, 0x39, 0x03, 0x95, 0xa9, 0xf7, 0x98, 0x17, 0xe0, + 0x96, 0x13, 0x37, 0x7c, 0xb4, 0x0f, 0xd3, 0x03, 0xc3, 0x75, 0x29, 0xd1, 0x8f, 0x99, 0x1f, 0x28, + 0xdd, 0x92, 0xd0, 0x7d, 0x3f, 0xca, 0xe3, 0xd1, 0xf8, 0xbd, 0x07, 0x42, 0xf6, 0x2b, 0xe6, 0x07, + 0x42, 0x7d, 0xdb, 0x09, 0xbc, 0x33, 0x1e, 0x6e, 0x29, 0x2a, 0x7a, 0x0b, 0x60, 0xe8, 0x1b, 0x7d, + 0xaa, 0x7b, 0x46, 0x40, 0x85, 0x77, 0x17, 0x71, 0x43, 0x50, 0xb0, 0x11, 0xd0, 0xce, 0x06, 0xcc, + 0xe4, 0xf5, 0x83, 0xda, 0x50, 0xe2, 0x6b, 0x5f, 0x10, 0xb9, 0x83, 0x3f, 0xa2, 0x19, 0xa8, 0x9c, + 0x1a, 0xf6, 0x30, 0x2c, 0x5d, 0xb2, 0x71, 0xbb, 0xf8, 0x69, 0xa1, 0xfb, 0x57, 0x45, 0x98, 0xde, + 0x94, 0x25, 0x5e, 0x55, 0x93, 0xed, 0xe7, 0x3c, 0x77, 0xf2, 0xda, 0xa7, 0xdb, 0xf4, 0x94, 0xda, + 0x2a, 0xac, 0x27, 0x7b, 0xbc, 0xfa, 0xee, 0xb2, 0x7e, 0x6f, 0x97, 0x53, 0x71, 0xdd, 0x66, 0x7d, + 0xf1, 0x84, 0x76, 0xe2, 0xad, 0x22, 0xd1, 0x06, 0xaa, 0x10, 0xef, 0x44, 0x73, 0x1f, 0xd9, 0x62, + 0x3c, 0xad, 0xb4, 0x12, 0xbb, 0xbe, 0x03, 0x2d, 0x3f, 0x30, 0xbc, 0x40, 0x37, 0xd9, 0x60, 0x60, + 0x05, 0x22, 0xea, 0x9b, 0x6b, 0xbf, 0x15, 0x2f, 0x60, 0xd6, 0x52, 0x9e, 0x62, 0xbc, 0x60, 0x53, + 0x48, 0xe3, 0xa6, 0x1f, 0x37, 0x3a, 0x18, 0x9a, 0x09, 0x1e, 0xda, 0x04, 0xa4, 0x3a, 0xd1, 0xcd, + 0x63, 0x6a, 0x9e, 0xb8, 0xcc, 0x72, 0x02, 0x31, 0x35, 0x9e, 0x3c, 0xa3, 0x8c, 0xb5, 0x19, 0xf1, + 0xf0, 0xb4, 0x92, 0x8f, 0x49, 0xdd, 0xff, 0x2d, 0x03, 0x8a, 0x4c, 0x90, 0xe9, 0x8f, 0xaf, 0xd6, + 0x2a, 0x34, 0xa2, 0x5a, 0xae, 0xba, 0x44, 0xa3, 0x7b, 0x8e, 0x63, 0x21, 0x74, 0x1b, 0xaa, 0xcc, + 0xa5, 0x0e, 0x25, 0x6a, 0x99, 0xba, 0xa3, 0x33, 0x8c, 0xba, 0xef, 0x3d, 0x12, 0x92, 0x58, 0x69, + 0xa0, 0xbb, 0x50, 0x57, 0x98, 0x8c, 0xa8, 0xf5, 0x79, 0xf7, 0x3c, 0x6d, 0x45, 0x22, 0x38, 0xd2, + 0x42, 0xf7, 0x00, 0x12, 0x6b, 0x50, 0x1e, 0xb7, 0xc6, 0x89, 0x3e, 0xe2, 0x55, 0x49, 0x68, 0x76, + 0x1e, 0x40, 0x55, 0xda, 0xf6, 0x9d, 0xac, 0x6e, 0xe7, 0x09, 0xd4, 0x43, 0x63, 0xb9, 0xe7, 0x9f, + 0xd0, 0x33, 0x5d, 0x26, 0x09, 0xd1, 0x51, 0x0b, 0x37, 0x4e, 0xe8, 0xd9, 0x9e, 0x20, 0x70, 0x58, + 0xc5, 0xb3, 0x92, 0xc5, 0x8b, 0x92, 0x1f, 0x4a, 0x15, 0x85, 0x54, 0x3b, 0x66, 0x48, 0xe1, 0xce, + 0x33, 0x80, 0x78, 0x14, 0xb4, 0x00, 0x15, 0x5e, 0x8e, 0x7c, 0x65, 0x1d, 0x08, 0xb7, 0xe6, 0x85, + 0xca, 0xc7, 0x92, 0x81, 0x7e, 0x04, 0x4d, 0x97, 0xd9, 0xb6, 0xee, 0x51, 0x7f, 0x68, 0x07, 0xa2, + 0xdb, 0xc9, 0xf3, 0xd7, 0x67, 0x8f, 0xd9, 0x36, 0x16, 0xd2, 0x18, 0xdc, 0xe8, 0xb9, 0xfb, 0x10, + 0x20, 0xe6, 0xa0, 0x26, 0xd4, 0x76, 0x1e, 0x3e, 0x59, 0xdf, 0xdd, 0xd9, 0x6a, 0xbf, 0x81, 0x1a, + 0x50, 0xc1, 0xdb, 0xeb, 0x5b, 0xbf, 0xdb, 0x2e, 0xa0, 0x09, 0x68, 0x3c, 0x7c, 0x74, 0xa0, 0xcb, + 0x66, 0x11, 0xb5, 0xa0, 0xbe, 0xf9, 0xe8, 0xd1, 0xae, 0xfe, 0xe8, 0xde, 0xbd, 0x76, 0x89, 0x2b, + 0xe1, 0xed, 0xfd, 0x83, 0x75, 0x7c, 0xd0, 0x2e, 0x77, 0xff, 0xa3, 0x00, 0xed, 0x2d, 0x81, 0xb5, + 0xbf, 0x07, 0xa1, 0xba, 0x06, 0x65, 0xee, 0x90, 0xca, 0x05, 0x6f, 0x44, 0xca, 0x59, 0x03, 0x85, + 0xfb, 0x62, 0x21, 0xdb, 0x59, 0x86, 0x32, 0x6f, 0xa1, 0x77, 0x61, 0xd2, 0xff, 0xda, 0xe6, 0x55, + 0xf6, 0xf4, 0xc8, 0xd7, 0x87, 0x9e, 0xa5, 0x92, 0x70, 0x4b, 0x52, 0x9f, 0x1c, 0xf9, 0x8f, 0x3d, + 0xab, 0xfb, 0x9f, 0x25, 0x98, 0x0e, 0x7b, 0xfb, 0x36, 0xc1, 0xf6, 0x59, 0x26, 0xd8, 0xde, 0x1e, + 0xb1, 0x75, 0x6c, 0xac, 0x6d, 0x40, 0xc3, 0x1d, 0x1e, 0xda, 0x96, 0x7f, 0x9c, 0x13, 0x6c, 0xa3, + 0xda, 0x7b, 0xa1, 0x2c, 0x8e, 0xd5, 0xd0, 0xe7, 0x50, 0x3b, 0xb2, 0x87, 0xa2, 0x87, 0x72, 0x26, + 0xd8, 0x47, 0x7b, 0xb8, 0x27, 0x25, 0x71, 0xa8, 0xf2, 0x5d, 0xc7, 0x58, 0x00, 0x8d, 0xc8, 0x48, + 0x7e, 0xa8, 0x19, 0x18, 0xcf, 0x75, 0xd3, 0x66, 0xe6, 0x89, 0x2a, 0xad, 0xf5, 0x81, 0xf1, 0x7c, + 0x93, 0xb7, 0x33, 0x11, 0x58, 0xbc, 0x50, 0x04, 0x96, 0xc6, 0x44, 0xe0, 0x12, 0xd4, 0xd4, 0xc4, + 0x5e, 0x1d, 0x7e, 0xdd, 0x3f, 0x2e, 0xc0, 0x6c, 0x0c, 0x46, 0xbf, 0x07, 0xae, 0xde, 0xfd, 0x79, + 0x01, 0xe6, 0x52, 0x16, 0x7d, 0x1b, 0x6f, 0x5c, 0x8f, 0xdd, 0x41, 0x1a, 0x13, 0xc3, 0x83, 0xfc, + 0x31, 0x46, 0x7d, 0xe2, 0xb5, 0x96, 0xf3, 0xe7, 0x65, 0x98, 0xdc, 0x64, 0x83, 0x43, 0xcb, 0x89, + 0x8e, 0x8b, 0xab, 0x2a, 0x74, 0xa5, 0xce, 0x9b, 0x09, 0x7b, 0x93, 0x62, 0x89, 0xc0, 0x45, 0xb7, + 0xa0, 0x64, 0x90, 0xd0, 0xe0, 0xeb, 0xe3, 0x14, 0xd6, 0x09, 0xc1, 0x5c, 0xae, 0xf3, 0xcf, 0x45, + 0x15, 0xe8, 0x77, 0xa1, 0x7e, 0x68, 0x39, 0xc4, 0x72, 0xfa, 0xdc, 0xc2, 0x52, 0xba, 0x56, 0x8d, + 0x8e, 0xd6, 0xdb, 0x90, 0xc2, 0x38, 0xd2, 0xea, 0xfc, 0x51, 0x11, 0x6a, 0x8a, 0x8a, 0x10, 0x94, + 0x8f, 0x86, 0xb6, 0xdc, 0xfa, 0x3a, 0x16, 0xcf, 0x21, 0xd6, 0xe1, 0x28, 0xad, 0x21, 0xb1, 0xce, + 0xa7, 0xd0, 0x74, 0x3d, 0xf6, 0x54, 0x1e, 0x83, 0x42, 0x0c, 0xd6, 0x96, 0xf8, 0x6d, 0x2f, 0x62, + 0x28, 0x18, 0x9a, 0x14, 0x45, 0x77, 0xa0, 0xe9, 0x9b, 0xc7, 0x74, 0x60, 0xe8, 0x4f, 0x7d, 0xe6, + 0x88, 0x68, 0x6d, 0x6d, 0xbc, 0xf9, 0xf2, 0xc5, 0xbc, 0x46, 0x1d, 0x93, 0x71, 0x13, 0x56, 0x38, + 0xa3, 0x87, 0x8d, 0x67, 0x0f, 0xa8, 0x2f, 0x60, 0x18, 0x48, 0x85, 0xfb, 0x3e, 0x73, 0x50, 0x0f, + 0xc0, 0xa7, 0x9e, 0xee, 0x32, 0xdb, 0x32, 0xcf, 0xc4, 0xd1, 0x21, 0xc2, 0xcb, 0xfb, 0xd4, 0xdb, + 0x13, 0x64, 0xdc, 0xf0, 0xc3, 0x47, 0x71, 0x6d, 0x20, 0xf0, 0x75, 0xe0, 0x89, 0xe3, 0x41, 0x03, + 0xd7, 0x04, 0x8c, 0x0e, 0x3c, 0x7e, 0x0a, 0x17, 0x10, 0x4d, 0xa2, 0xfd, 0x06, 0x56, 0xad, 0x8e, + 0x03, 0xa5, 0x75, 0x42, 0x90, 0x06, 0x35, 0xb5, 0x40, 0x0a, 0xe4, 0x85, 0x4d, 0xf4, 0x43, 0xa8, + 0x13, 0x66, 0x4a, 0xfb, 0x8b, 0x17, 0xb0, 0xbf, 0x46, 0x98, 0x29, 0x8c, 0x9f, 0x81, 0xca, 0x91, + 0xc7, 0x1c, 0x09, 0xb9, 0xea, 0x58, 0x36, 0xba, 0xff, 0x52, 0x80, 0xa9, 0x68, 0x9f, 0xd4, 0x79, + 0x6f, 0xfc, 0xe0, 0x1a, 0xd4, 0x08, 0xb5, 0x69, 0xa0, 0x5c, 0xbb, 0x8e, 0xc3, 0x66, 0xca, 0xac, + 0xd2, 0xa5, 0xcc, 0x2a, 0x27, 0xcc, 0xca, 0xe4, 0xa6, 0x4a, 0x36, 0x37, 0xbd, 0x03, 0x13, 0x72, + 0xbd, 0x42, 0x09, 0x71, 0xf8, 0xc2, 0x2d, 0x49, 0x94, 0x42, 0xdd, 0xab, 0x30, 0xbb, 0xc9, 0x1c, + 0x87, 0x9a, 0x01, 0xf3, 0xf6, 0x3c, 0xf6, 0xfc, 0x4c, 0x39, 0x62, 0xf7, 0x4f, 0x0b, 0x30, 0x97, + 0xe5, 0xa8, 0xa9, 0xdf, 0x87, 0x1a, 0x3f, 0x32, 0x50, 0xdf, 0x57, 0xf7, 0x2c, 0xab, 0x2f, 0x5f, + 0xcc, 0x2f, 0x5f, 0xe4, 0x6c, 0xb5, 0xed, 0x10, 0x99, 0x93, 0xc3, 0x0e, 0xf8, 0xee, 0xbb, 0xbc, + 0x73, 0xdd, 0x22, 0x0a, 0x95, 0xd7, 0x44, 0x7b, 0x87, 0xa0, 0x0e, 0x94, 0x6c, 0xd6, 0x57, 0xf5, + 0xa6, 0x1e, 0x66, 0x38, 0xcc, 0x89, 0xdd, 0xbf, 0x29, 0x41, 0xf9, 0x3e, 0xb3, 0x1c, 0x74, 0x13, + 0xa6, 0x69, 0x60, 0x12, 0x7d, 0xc0, 0x88, 0xee, 0xd1, 0x53, 0xcb, 0xe7, 0x27, 0x7a, 0x6e, 0x55, + 0x09, 0x4f, 0x71, 0xc6, 0x03, 0x46, 0xb0, 0x22, 0xa3, 0x25, 0xa8, 0xfa, 0xc7, 0x86, 0x47, 0xc2, + 0xd3, 0xcc, 0x95, 0x28, 0x08, 0x79, 0x57, 0xf2, 0xf2, 0x02, 0x2b, 0x11, 0x34, 0x0f, 0x4d, 0xf1, + 0xa4, 0x6e, 0x20, 0x4a, 0x62, 0x8f, 0x41, 0x90, 0xe4, 0xfd, 0xc3, 0x12, 0x4c, 0x87, 0x97, 0x14, + 0xc4, 0xf2, 0xc4, 0x32, 0x9d, 0x85, 0x77, 0x5a, 0x8a, 0xb1, 0x15, 0xd2, 0xd1, 0x07, 0x10, 0xd2, + 0x74, 0xaa, 0xd6, 0x40, 0x6c, 0x58, 0x03, 0x4f, 0x29, 0x7a, 0xb8, 0x34, 0xe8, 0x7d, 0x98, 0xb2, + 0xc5, 0xf1, 0x3f, 0x96, 0x94, 0x61, 0x31, 0x29, 0xc9, 0xa1, 0x60, 0xe7, 0xaf, 0x0b, 0x50, 0x11, + 0x36, 0xa3, 0x49, 0x28, 0x5a, 0x44, 0x81, 0x87, 0xa2, 0x45, 0x50, 0x0f, 0xea, 0xb6, 0x71, 0x48, + 0x6d, 0xee, 0x9c, 0x45, 0x95, 0x8d, 0x45, 0x46, 0xe4, 0xd2, 0xbb, 0x8a, 0x83, 0x23, 0x19, 0xb4, + 0x06, 0x35, 0x8f, 0x1a, 0xdc, 0x52, 0xb5, 0xda, 0x5a, 0x7c, 0x25, 0xb1, 0xe7, 0x31, 0x93, 0xfa, + 0xfe, 0xbe, 0x4b, 0xcd, 0xde, 0xce, 0x16, 0x0e, 0x05, 0xd1, 0x2a, 0xcc, 0x88, 0x85, 0x37, 0x3d, + 0x6a, 0x04, 0x34, 0x5e, 0x7b, 0x71, 0xf9, 0x80, 0x11, 0xe7, 0x6d, 0x0a, 0x56, 0xb8, 0xfc, 0xdd, + 0x8f, 0xa1, 0xca, 0xd7, 0x99, 0x12, 0xbe, 0x69, 0xbc, 0xe2, 0x0a, 0xfd, 0xec, 0xa6, 0x0d, 0x8c, + 0xe7, 0xdb, 0x81, 0x19, 0x6d, 0x5a, 0xf7, 0x67, 0x05, 0x28, 0x1f, 0x18, 0xfe, 0x09, 0x4f, 0x7b, + 0xbe, 0x4b, 0x4d, 0x85, 0x82, 0xc5, 0x33, 0x5f, 0x56, 0xde, 0x51, 0xe0, 0x19, 0x8e, 0x6f, 0x44, + 0x99, 0x8e, 0xef, 0x14, 0xef, 0xe7, 0x20, 0x41, 0xce, 0x01, 0x5b, 0xe5, 0x51, 0xb0, 0xc5, 0x4f, + 0xd0, 0x21, 0x64, 0xf1, 0xb8, 0x4b, 0xca, 0xa0, 0x6a, 0x46, 0xb4, 0x1d, 0x72, 0xbf, 0x5c, 0x2f, + 0xb6, 0x4b, 0xdd, 0x3f, 0xaf, 0x40, 0x0d, 0x53, 0x93, 0x9d, 0x8a, 0x5a, 0xd6, 0x34, 0xcc, 0x13, + 0xdd, 0x72, 0x02, 0xea, 0x04, 0x61, 0x86, 0x5f, 0x88, 0x8b, 0xab, 0x14, 0xeb, 0xad, 0x9b, 0x27, + 0x3b, 0x52, 0x44, 0x9e, 0x73, 0xc1, 0x88, 0x08, 0x68, 0x0d, 0x66, 0xe5, 0x59, 0x2f, 0xa0, 0x84, + 0x23, 0x11, 0x9f, 0x2a, 0x3c, 0x52, 0x14, 0x78, 0xe4, 0x4a, 0xc4, 0xdc, 0xe4, 0x3c, 0x09, 0x4d, + 0xee, 0x02, 0x8a, 0x75, 0x44, 0x46, 0xb0, 0x68, 0xb8, 0x81, 0xd3, 0xbd, 0xf0, 0x12, 0xf8, 0x9e, + 0x62, 0xe0, 0xe9, 0x48, 0x38, 0x24, 0xa1, 0x65, 0x98, 0x31, 0xc3, 0x10, 0xd7, 0x79, 0x9d, 0xa4, + 0x89, 0x94, 0x8f, 0x27, 0x23, 0x1e, 0xaf, 0xa4, 0x14, 0x2d, 0x03, 0x3a, 0xe6, 0x73, 0x4c, 0x1b, + 0x58, 0x91, 0x77, 0x11, 0x92, 0x93, 0xb0, 0xee, 0x36, 0x4c, 0x29, 0xe9, 0xc8, 0xb4, 0xea, 0x38, + 0xd3, 0x26, 0xa5, 0x64, 0x64, 0xd7, 0xdb, 0xd0, 0xb2, 0x0d, 0x3f, 0xd0, 0x0d, 0xd7, 0xb5, 0x2d, + 0x4a, 0xc4, 0x3d, 0x64, 0x0b, 0x37, 0x39, 0x6d, 0x5d, 0x92, 0xd0, 0x3a, 0x4c, 0xdb, 0xb4, 0x6f, + 0x98, 0x67, 0x49, 0x14, 0x58, 0x3f, 0x07, 0x05, 0xb6, 0xa5, 0x78, 0xe2, 0x08, 0xf4, 0x29, 0x70, + 0x98, 0xa7, 0x9f, 0xd0, 0xb3, 0xf0, 0x5a, 0xe7, 0xad, 0x91, 0x3d, 0x7b, 0x60, 0x3c, 0xff, 0x31, + 0x3d, 0x53, 0x1b, 0x56, 0x1b, 0xc8, 0x16, 0xba, 0x09, 0x57, 0x02, 0xcf, 0xea, 0xf7, 0x79, 0x99, + 0x33, 0x3c, 0x63, 0xe0, 0xcb, 0x65, 0x03, 0x61, 0xe6, 0x84, 0x62, 0xed, 0x09, 0x4e, 0xe7, 0x0e, + 0x4c, 0x65, 0x36, 0x3e, 0x79, 0x31, 0xd1, 0xc8, 0xb9, 0x98, 0x68, 0x25, 0x2e, 0x26, 0x3a, 0xb7, + 0xa1, 0x95, 0xb4, 0xe1, 0x55, 0x97, 0x1a, 0x49, 0xdd, 0xee, 0x2f, 0x6a, 0x50, 0xdb, 0xa3, 0x9e, + 0x6f, 0xf9, 0x01, 0x9a, 0x85, 0xaa, 0x4f, 0xbf, 0xd6, 0x1d, 0x26, 0x54, 0xcb, 0xb8, 0xe2, 0xd3, + 0xaf, 0x1f, 0x32, 0xbe, 0xa7, 0xb2, 0x38, 0xe9, 0x49, 0x0f, 0x96, 0x65, 0xab, 0x2d, 0x39, 0xb1, + 0xf5, 0x59, 0x47, 0x2f, 0x65, 0x1c, 0x5d, 0x8d, 0x75, 0x39, 0x47, 0x2f, 0x8f, 0x77, 0xf4, 0xdb, + 0x70, 0x4d, 0x19, 0x99, 0xe3, 0xef, 0x15, 0x61, 0xeb, 0x55, 0x29, 0xb0, 0x39, 0xe2, 0xe2, 0xf9, + 0x41, 0x52, 0x7d, 0x8d, 0x20, 0x59, 0x85, 0xb9, 0x38, 0x48, 0x5c, 0x23, 0x30, 0x8f, 0xa9, 0xda, + 0x6f, 0xe9, 0x96, 0xed, 0x88, 0xbb, 0x27, 0x99, 0x63, 0x02, 0xa5, 0x3e, 0x26, 0x50, 0x3e, 0x86, + 0x39, 0x35, 0xbb, 0x6c, 0xbc, 0x34, 0xc4, 0xd4, 0x66, 0x24, 0xf7, 0xab, 0x74, 0x88, 0xe4, 0x84, + 0x17, 0x5c, 0x36, 0xbc, 0x9a, 0xa3, 0xe1, 0xf5, 0x29, 0x68, 0xca, 0xa8, 0xd1, 0x28, 0x6b, 0x09, + 0xb3, 0x94, 0xd1, 0xbb, 0xd9, 0xa8, 0xca, 0x0d, 0xcc, 0x89, 0x4b, 0x07, 0xe6, 0x64, 0x26, 0x30, + 0x43, 0x1f, 0xcb, 0x0f, 0xcc, 0x35, 0x98, 0x55, 0x66, 0xa7, 0xe3, 0x53, 0x9b, 0x12, 0x36, 0x5f, + 0x91, 0xcc, 0x83, 0x64, 0x80, 0x8e, 0x0b, 0xe6, 0xf6, 0xf7, 0x2c, 0x98, 0xbb, 0xd0, 0x50, 0x73, + 0xa7, 0x64, 0x4c, 0x34, 0x77, 0xff, 0xa2, 0x00, 0x15, 0xbe, 0x83, 0x67, 0xb9, 0xc5, 0x52, 0x83, + 0xda, 0x29, 0xef, 0x41, 0x61, 0xe2, 0x06, 0x0e, 0x9b, 0xfc, 0x04, 0x2c, 0x1c, 0x42, 0xa8, 0xc8, + 0xe4, 0x5f, 0xe7, 0x04, 0x5e, 0xf4, 0x23, 0x6f, 0x09, 0x75, 0x25, 0x6c, 0x11, 0xde, 0xf2, 0x44, + 0xe9, 0xaf, 0x8e, 0xa9, 0x23, 0x12, 0x70, 0xa2, 0x74, 0x1d, 0xe1, 0x80, 0xb6, 0xfb, 0x14, 0x6a, + 0xa1, 0xab, 0xdd, 0x02, 0x24, 0x6b, 0x74, 0x74, 0x40, 0x0d, 0xd1, 0x40, 0x03, 0x4f, 0x4b, 0xce, + 0x56, 0xcc, 0x38, 0x27, 0x1c, 0x8b, 0xf9, 0xe1, 0xd8, 0xfd, 0x55, 0x41, 0x1d, 0xc3, 0x5e, 0x6f, + 0x51, 0xde, 0x0b, 0x5f, 0x9c, 0x95, 0x72, 0x5f, 0x9c, 0x85, 0xaf, 0xcc, 0xde, 0x39, 0xb7, 0x86, + 0x8a, 0xd3, 0x27, 0x45, 0x9f, 0x24, 0x3c, 0xba, 0x22, 0x3c, 0x3a, 0x3e, 0x7b, 0x8b, 0x13, 0x5f, + 0xae, 0x3b, 0x7f, 0x2b, 0x7f, 0x01, 0xa8, 0x8b, 0x24, 0xf3, 0x90, 0x3d, 0xeb, 0x56, 0xa1, 0xbc, + 0x1f, 0x30, 0xb7, 0xdb, 0x80, 0x1a, 0xff, 0x75, 0x29, 0xe9, 0xfe, 0x0e, 0x34, 0xf7, 0xa9, 0xcf, + 0x27, 0xba, 0xcb, 0x98, 0x3b, 0xe6, 0x9a, 0xa0, 0x70, 0x99, 0x6b, 0x82, 0x3f, 0xa9, 0x42, 0x4d, + 0x5d, 0x0e, 0xa2, 0x0f, 0x12, 0x2b, 0xde, 0x5c, 0x9b, 0xed, 0x85, 0x6f, 0xd1, 0xc3, 0xd3, 0xae, + 0x58, 0x48, 0xb9, 0x11, 0xbf, 0x0d, 0x13, 0xfc, 0x57, 0xf7, 0xd4, 0x29, 0x43, 0x01, 0xd7, 0xb9, + 0x84, 0x8e, 0x64, 0x48, 0xa5, 0x16, 0x17, 0x8e, 0x4e, 0x24, 0x9f, 0x40, 0x9d, 0x58, 0xbe, 0x28, + 0xd9, 0x6a, 0xbb, 0xae, 0x8d, 0x8c, 0xb5, 0xa5, 0x04, 0x70, 0x24, 0x8a, 0x3e, 0x07, 0x08, 0x9f, + 0xa3, 0x6b, 0xa9, 0x37, 0x47, 0x07, 0xdc, 0x8a, 0x64, 0x70, 0x42, 0x9e, 0x0f, 0x7a, 0x6a, 0xd8, + 0x16, 0x31, 0x02, 0xaa, 0x8e, 0xb9, 0xa3, 0x83, 0x3e, 0x51, 0x02, 0x38, 0x12, 0x45, 0x9f, 0x41, + 0x23, 0x7c, 0x26, 0xaa, 0x10, 0x5d, 0x1f, 0x1d, 0x33, 0x54, 0x24, 0x38, 0x96, 0x4e, 0xdf, 0xfc, + 0x34, 0x5e, 0x71, 0xf3, 0xf3, 0x43, 0x68, 0xf9, 0x72, 0x87, 0x75, 0x9b, 0x31, 0x57, 0x9b, 0x51, + 0x39, 0x38, 0xdc, 0xcc, 0xc4, 0xf6, 0xe3, 0xa6, 0x9f, 0xf0, 0x85, 0xb7, 0xa1, 0xfc, 0x94, 0x59, + 0x8e, 0x36, 0x2b, 0x14, 0x26, 0x52, 0x87, 0x24, 0x2c, 0x58, 0xe8, 0x7d, 0xa8, 0x3e, 0x15, 0x50, + 0x5e, 0x9b, 0x53, 0xc1, 0x91, 0x14, 0xa2, 0x04, 0x2b, 0x36, 0xef, 0x2b, 0x30, 0xfc, 0x13, 0xed, + 0x6a, 0xa6, 0x2f, 0x8e, 0xe8, 0xb1, 0x60, 0xa1, 0x95, 0xe8, 0x5e, 0x52, 0x13, 0x42, 0x57, 0xb3, + 0x57, 0xcc, 0xd9, 0xdb, 0xc8, 0x1e, 0x34, 0x64, 0x5d, 0x75, 0xd8, 0x33, 0xed, 0x9a, 0x2a, 0x7a, + 0x91, 0x8e, 0xf2, 0x79, 0x5c, 0x37, 0xd5, 0x13, 0xb7, 0xc1, 0x0f, 0x98, 0xab, 0x75, 0x32, 0x36, + 0xf0, 0x50, 0xc0, 0x82, 0x85, 0x6e, 0x42, 0xcd, 0x97, 0x81, 0xa1, 0x5d, 0x57, 0xef, 0x64, 0x93, + 0x52, 0x2e, 0x25, 0x38, 0x14, 0xe8, 0xdc, 0x8e, 0xae, 0x22, 0x5f, 0xfb, 0xd6, 0xab, 0xfb, 0xaf, + 0x73, 0xd0, 0x4c, 0x5c, 0x6f, 0xa1, 0x5b, 0xa9, 0xf8, 0xb8, 0xd6, 0x4b, 0x7e, 0xfd, 0x91, 0x13, + 0x23, 0x5f, 0xe6, 0xc7, 0x48, 0x27, 0xa3, 0x37, 0x3e, 0x4e, 0x3e, 0x4b, 0xb8, 0xac, 0x8c, 0x93, + 0xb7, 0x72, 0xc7, 0xcc, 0x71, 0xdb, 0x3b, 0x49, 0xb7, 0x95, 0xa1, 0x32, 0x9f, 0x3f, 0xee, 0xab, + 0x5d, 0xb7, 0xf2, 0x6b, 0xe2, 0xba, 0x37, 0xf9, 0xb9, 0x59, 0x66, 0x1d, 0x2d, 0xe3, 0x36, 0xea, + 0x00, 0x81, 0x43, 0x01, 0xf4, 0x2e, 0x54, 0x38, 0xde, 0x3a, 0x53, 0x1e, 0x1b, 0x7f, 0xd5, 0x22, + 0x0a, 0x36, 0x96, 0x4c, 0xde, 0x63, 0x88, 0xca, 0x3a, 0x99, 0x1e, 0x55, 0xbd, 0xc4, 0xa1, 0x00, + 0x37, 0x50, 0xdc, 0x5f, 0x5e, 0xcf, 0x18, 0x98, 0xb8, 0xb0, 0xfc, 0x28, 0x8a, 0xad, 0x37, 0x33, + 0x77, 0x96, 0x09, 0x2f, 0xcc, 0xc6, 0xd7, 0x2d, 0x28, 0xdb, 0xcc, 0x20, 0xda, 0xa2, 0x72, 0xca, + 0x3c, 0x95, 0x5d, 0x66, 0x10, 0x2c, 0xc4, 0xf8, 0x18, 0xfc, 0x97, 0x12, 0xed, 0x83, 0x73, 0xc6, + 0xd8, 0x15, 0x22, 0x58, 0x89, 0xa2, 0x55, 0xa8, 0x88, 0x6b, 0x5c, 0xed, 0x66, 0xa6, 0xc4, 0x24, + 0x75, 0xc4, 0xed, 0x2e, 0x96, 0x82, 0xe8, 0x07, 0xf1, 0x85, 0xf1, 0x52, 0xe6, 0xc2, 0x76, 0x44, + 0x27, 0x71, 0x4b, 0xcc, 0x47, 0xf2, 0x03, 0xe6, 0x51, 0x6d, 0xf9, 0x9c, 0x91, 0xf6, 0xb9, 0x04, + 0x96, 0x82, 0x7c, 0x42, 0xe2, 0x81, 0x68, 0xb7, 0xce, 0x99, 0x90, 0x50, 0x21, 0x58, 0x89, 0xa2, + 0xcd, 0xcc, 0x2b, 0xdb, 0x9e, 0x50, 0x5d, 0x18, 0xa3, 0x9a, 0xff, 0xb2, 0x16, 0xed, 0xc0, 0xa4, + 0x68, 0xf2, 0x93, 0x83, 0xec, 0x66, 0x25, 0xf3, 0xaa, 0x64, 0xa4, 0x1b, 0x4a, 0x54, 0x47, 0x13, + 0x7e, 0xb2, 0x89, 0x36, 0xc4, 0x51, 0xcd, 0x61, 0xcf, 0x6c, 0x4a, 0xfa, 0x54, 0x5b, 0x3d, 0xc7, + 0x9c, 0xf5, 0x58, 0x0e, 0x27, 0x95, 0xd0, 0x36, 0xb4, 0x12, 0x4d, 0xa2, 0x7d, 0x98, 0x79, 0x6f, + 0x34, 0xa6, 0x13, 0x82, 0x53, 0x6a, 0xdc, 0xa7, 0x5d, 0x89, 0x5c, 0xb5, 0xb5, 0x8c, 0x4f, 0x2b, + 0x44, 0x8b, 0x43, 0x01, 0x9e, 0x52, 0xdd, 0x10, 0xe5, 0x6a, 0x1f, 0x65, 0x52, 0x6a, 0x84, 0x7f, + 0x71, 0x2c, 0x94, 0xae, 0x06, 0x1f, 0x5f, 0xbc, 0x1a, 0x7c, 0x7e, 0xa1, 0x6a, 0x70, 0xe7, 0x55, + 0xd5, 0xe0, 0x0f, 0x0a, 0x97, 0x2f, 0x07, 0xe8, 0x47, 0x49, 0xec, 0x98, 0x38, 0x2e, 0x15, 0xcf, + 0x39, 0x2e, 0x5d, 0x89, 0x34, 0x12, 0xef, 0xb3, 0x3e, 0x81, 0x32, 0x0f, 0x30, 0x74, 0x0b, 0xea, + 0xd1, 0x71, 0xb0, 0x30, 0xee, 0x38, 0x18, 0x89, 0x74, 0x7e, 0x55, 0x84, 0xaa, 0x0c, 0x4c, 0xf4, + 0xe5, 0xc8, 0x2b, 0x8a, 0x77, 0xce, 0x89, 0xe3, 0xd1, 0x37, 0x14, 0xf2, 0x0c, 0x20, 0xae, 0xc8, + 0x3d, 0x5d, 0x7e, 0xad, 0x71, 0x78, 0x16, 0x50, 0x79, 0x97, 0x50, 0xe6, 0x67, 0x00, 0xc9, 0x7b, + 0xcc, 0x59, 0x1b, 0x9c, 0xd3, 0xf9, 0xaf, 0x42, 0xfc, 0x4e, 0x63, 0x06, 0x2a, 0xf2, 0x9e, 0x55, + 0x62, 0x5b, 0xd9, 0x40, 0x8b, 0xd0, 0x1e, 0x58, 0x8e, 0xee, 0xb3, 0xa1, 0x67, 0xa6, 0x2f, 0xc4, + 0x26, 0x07, 0x96, 0xb3, 0x2f, 0xc8, 0xf2, 0x10, 0xbd, 0x28, 0x2f, 0x02, 0x53, 0x92, 0x25, 0x25, + 0x69, 0x3c, 0x4f, 0x4a, 0x2e, 0x03, 0x92, 0x52, 0x44, 0x27, 0xcc, 0xf4, 0xf5, 0x80, 0x05, 0x86, + 0x2d, 0x0a, 0x5a, 0x19, 0xb7, 0x15, 0x67, 0x8b, 0x99, 0xfe, 0x01, 0xa7, 0xa3, 0x1e, 0x5c, 0x09, + 0xa5, 0xc5, 0x74, 0x94, 0x78, 0x45, 0x88, 0x4f, 0x2b, 0x96, 0x98, 0x8e, 0x94, 0xef, 0xc2, 0x84, + 0x02, 0xfa, 0x3a, 0xa1, 0x76, 0xa0, 0x3e, 0x78, 0xc2, 0x4d, 0x89, 0xe8, 0xb7, 0x38, 0xa9, 0xf3, + 0x19, 0x54, 0x44, 0x96, 0x3a, 0xe7, 0x28, 0x53, 0xc8, 0x3f, 0xca, 0x74, 0xfe, 0xbb, 0x10, 0xbf, + 0xf3, 0x3a, 0xef, 0xa5, 0x52, 0x4e, 0x46, 0xcc, 0xdd, 0xb2, 0xd7, 0x3c, 0x4a, 0x75, 0xce, 0x5e, + 0xb5, 0x63, 0x37, 0x61, 0x5a, 0x66, 0xf8, 0xe4, 0xe2, 0x4a, 0x17, 0x98, 0x92, 0x8c, 0x78, 0x6d, + 0x97, 0x01, 0x29, 0xd9, 0xe4, 0xd2, 0x96, 0xe4, 0x4e, 0x48, 0x4e, 0xbc, 0xb2, 0x9d, 0x1a, 0x54, + 0x44, 0xca, 0xed, 0xfc, 0x7d, 0x01, 0xaa, 0x32, 0xf9, 0x5e, 0xd8, 0x69, 0xa5, 0x78, 0xce, 0x6b, + 0xb5, 0x8b, 0xcc, 0x47, 0x26, 0xf8, 0x9c, 0xf9, 0x48, 0x46, 0x6a, 0x3e, 0x4a, 0x36, 0x67, 0x3e, + 0x92, 0x93, 0x98, 0xcf, 0x1f, 0x16, 0xd2, 0x5f, 0xe6, 0xbc, 0xb6, 0x33, 0x7c, 0x77, 0xd9, 0x63, + 0x1d, 0x26, 0x52, 0xb5, 0xe4, 0x12, 0x8e, 0xf9, 0x25, 0x34, 0x13, 0x15, 0xe0, 0x12, 0x1d, 0xdc, + 0x85, 0x56, 0xb2, 0x84, 0xbc, 0x7e, 0x0f, 0xdd, 0xbf, 0x45, 0x50, 0x95, 0x5f, 0x12, 0xa0, 0xc5, + 0x14, 0xac, 0x9e, 0xe9, 0xa9, 0x2f, 0xb3, 0x73, 0x10, 0xf5, 0xed, 0x7c, 0x44, 0x3d, 0x1b, 0xab, + 0x8c, 0x07, 0xd3, 0x1f, 0x8f, 0x80, 0x69, 0x2d, 0x3b, 0x52, 0x0e, 0x8e, 0xfe, 0x74, 0x14, 0x47, + 0x77, 0x46, 0x46, 0xfb, 0x0d, 0x84, 0xce, 0x83, 0xd0, 0x17, 0x00, 0xbc, 0xbd, 0x0c, 0xe0, 0x9d, + 0xcb, 0x7c, 0x64, 0x92, 0xc5, 0xba, 0x8b, 0x29, 0xac, 0x3b, 0x93, 0x95, 0x4e, 0xc0, 0xdc, 0x5e, + 0x06, 0xe6, 0xce, 0xe5, 0xc9, 0x26, 0x10, 0xee, 0x52, 0x1a, 0xe1, 0xce, 0x66, 0xc5, 0x53, 0xe0, + 0xf6, 0xc3, 0x2c, 0xb8, 0xbd, 0x9a, 0x2b, 0x9e, 0xc4, 0xb5, 0x4b, 0x69, 0x5c, 0x3b, 0xd2, 0x7f, + 0x0a, 0xd2, 0xf6, 0x32, 0x90, 0x76, 0x2e, 0x57, 0x3a, 0x46, 0xb3, 0x5f, 0xe4, 0xa2, 0xd9, 0xeb, + 0xa3, 0x5a, 0x63, 0x80, 0xec, 0xd6, 0x18, 0x20, 0xfb, 0x56, 0x6e, 0x0f, 0xe3, 0x30, 0xec, 0x6f, + 0x80, 0xe3, 0xf7, 0x15, 0x38, 0xfe, 0x22, 0x06, 0x8e, 0xb7, 0x47, 0x6a, 0xf0, 0x8d, 0xfc, 0xc8, + 0xf8, 0x4e, 0x30, 0xe3, 0x3f, 0xfe, 0xfa, 0x61, 0xc6, 0x6f, 0x83, 0x07, 0x1f, 0xc5, 0x70, 0xf0, + 0xf5, 0xf1, 0x03, 0x82, 0xf2, 0x80, 0x27, 0x10, 0xf9, 0xb6, 0x4f, 0x3c, 0xc7, 0x28, 0xeb, 0xf7, + 0x4b, 0x11, 0xca, 0x5a, 0x85, 0x99, 0xe8, 0x33, 0xbe, 0xe4, 0xfc, 0xe5, 0xab, 0x07, 0x14, 0xf1, + 0xe2, 0x15, 0x58, 0x83, 0xd9, 0x58, 0x23, 0xb9, 0x06, 0x72, 0x63, 0xaf, 0x44, 0xcc, 0x04, 0x72, + 0x5e, 0x06, 0x44, 0x3c, 0xee, 0xdd, 0xa9, 0x31, 0x14, 0x7a, 0x52, 0x9c, 0xd4, 0x1a, 0x87, 0xd2, + 0xc9, 0xfe, 0xe5, 0x96, 0x4c, 0x2b, 0x56, 0xa2, 0xf7, 0x9f, 0x40, 0x3b, 0x7e, 0xaf, 0xaf, 0x52, + 0x52, 0x25, 0xf3, 0xc5, 0x6f, 0x2a, 0x15, 0x46, 0x1f, 0x31, 0x7a, 0x2a, 0x37, 0x4d, 0xb9, 0x69, + 0x42, 0x47, 0x87, 0xa9, 0x8c, 0x0c, 0xea, 0x88, 0x8f, 0x59, 0xc8, 0xd0, 0x54, 0x51, 0xd4, 0xc2, + 0x51, 0x9b, 0x7b, 0x6b, 0xd2, 0x19, 0x65, 0x83, 0x6b, 0xa8, 0x3f, 0x39, 0x92, 0xaf, 0x53, 0x1b, + 0x38, 0x6a, 0x77, 0x9e, 0xa4, 0x01, 0xe2, 0xb8, 0x98, 0x2f, 0xbc, 0x6e, 0xcc, 0x4f, 0x65, 0xe0, + 0xde, 0xcd, 0x25, 0xa8, 0x88, 0x3f, 0xad, 0x42, 0x00, 0xd5, 0xbd, 0xc7, 0x1b, 0xbb, 0x3b, 0x9b, + 0xed, 0x37, 0x50, 0x13, 0x6a, 0x7b, 0x78, 0xe7, 0xc9, 0xfa, 0xc1, 0x76, 0xbb, 0x80, 0x1a, 0x50, + 0xd9, 0x7d, 0xb4, 0xb9, 0xbe, 0xdb, 0x2e, 0xae, 0xdd, 0x87, 0xba, 0xfa, 0xd3, 0x17, 0x0f, 0x7d, + 0x01, 0x35, 0xf5, 0x8c, 0xe2, 0x82, 0x95, 0xfe, 0xa3, 0xac, 0x8e, 0x36, 0xca, 0x90, 0x20, 0x67, + 0xb5, 0xb0, 0xb6, 0x0b, 0x75, 0xf5, 0x59, 0x95, 0x87, 0xee, 0x42, 0x4d, 0x3d, 0x27, 0xfa, 0x4a, + 0x7f, 0x1c, 0x97, 0xe8, 0x2b, 0xf3, 0x35, 0xd6, 0x62, 0x61, 0xb5, 0xb0, 0x76, 0x0c, 0x93, 0xe9, + 0x0f, 0x96, 0xd0, 0x13, 0x98, 0x12, 0x0f, 0x11, 0xd9, 0x47, 0x37, 0x92, 0x89, 0x75, 0xf4, 0xb3, + 0xa7, 0xce, 0xfc, 0x58, 0x7e, 0x62, 0xa4, 0x67, 0x50, 0xdd, 0x95, 0x7f, 0xa1, 0xd3, 0x8b, 0x30, + 0xe7, 0x54, 0xc6, 0x8f, 0x3a, 0x59, 0x02, 0xd7, 0x44, 0x77, 0xd2, 0xf7, 0xbf, 0x33, 0x79, 0xc7, + 0x95, 0x4e, 0x2e, 0x55, 0x0c, 0xfc, 0x97, 0xd1, 0x27, 0x3f, 0x1f, 0xc6, 0x2f, 0x59, 0xda, 0xd9, + 0x0b, 0xf3, 0xce, 0x08, 0x45, 0x8c, 0xfd, 0xff, 0x6b, 0xeb, 0xc6, 0x17, 0xdf, 0xfc, 0xdb, 0x8d, + 0x37, 0xbe, 0xf9, 0xe5, 0x8d, 0xc2, 0x3f, 0xfc, 0xf2, 0x46, 0xe1, 0xcf, 0xfe, 0xfd, 0x46, 0xe1, + 0xf7, 0x96, 0x2f, 0xf4, 0x27, 0x3f, 0xaa, 0xbf, 0xc3, 0xaa, 0x20, 0x7d, 0xf4, 0x7f, 0x01, 0x00, + 0x00, 0xff, 0xff, 0x1c, 0xa9, 0x09, 0x29, 0xff, 0x39, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -6424,16 +6423,6 @@ func (m *Task) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x18 } - if m.Preview { - i-- - if m.Preview { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x10 - } if len(m.Spec) > 0 { i -= len(m.Spec) copy(dAtA[i:], m.Spec) @@ -9948,9 +9937,6 @@ func (m *Task) ProtoSize() (n int) { if l > 0 { n += 1 + l + sovRuntime(uint64(l)) } - if m.Preview { - n += 2 - } if m.MaxTransactions != 0 { n += 1 + sovRuntime(uint64(m.MaxTransactions)) } @@ -15676,26 +15662,6 @@ func (m *Task) Unmarshal(dAtA []byte) error { m.Spec = []byte{} } iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Preview", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRuntime - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.Preview = bool(v != 0) case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field MaxTransactions", wireType) diff --git a/go/protocols/runtime/runtime.proto b/go/protocols/runtime/runtime.proto index 7758224ff75..026a3842666 100644 --- a/go/protocols/runtime/runtime.proto +++ b/go/protocols/runtime/runtime.proto @@ -470,11 +470,12 @@ message Joined { // Sent from Controller to Shard, and from Shard zero (only) to Leader // after Joined. Other shards do not forward Task. message Task { + reserved 2; + // Task specification (protobuf-encoded bytes). bytes spec = 1; - // When true, documents and stats are written to output and not directed to collections. - bool preview = 2; - // Preview / harness control. Zero means unlimited. + // Maximum number of transactions to run before exiting. Zero means unlimited. + // Used by "preview" workflows. uint32 max_transactions = 3; // URL of a SQLite VFS the shard threads to a SQLite derive connector via // `DeriveRequestExt.open.sqlite_vfs_uri` on C:Open. Set by the controller diff --git a/go/runtime/capture_v2.go b/go/runtime/capture_v2.go index f4ecf5ac497..6b803049842 100644 --- a/go/runtime/capture_v2.go +++ b/go/runtime/capture_v2.go @@ -149,7 +149,6 @@ func (c *captureAppV2) runOneSession(shard consumer.Shard, ch chan<- consumer.En _ = c.client.Send(&pr.Capture{ Task: &pr.Task{ Spec: specBytes, - Preview: false, MaxTransactions: 0, }, }) diff --git a/go/runtime/derive_v2.go b/go/runtime/derive_v2.go index 0e22d1928f7..a796f4abd89 100644 --- a/go/runtime/derive_v2.go +++ b/go/runtime/derive_v2.go @@ -258,7 +258,6 @@ func (m *deriveAppV2) runOneSession(shard consumer.Shard, ch chan<- consumer.Env _ = m.client.Send(&pr.Derive{ Task: &pr.Task{ Spec: specBytes, - Preview: false, MaxTransactions: 0, SqliteVfsUri: sqliteVfsUri, }, diff --git a/go/runtime/materialize_v2.go b/go/runtime/materialize_v2.go index 50aac1e5459..428b1fb4141 100644 --- a/go/runtime/materialize_v2.go +++ b/go/runtime/materialize_v2.go @@ -221,7 +221,6 @@ func (m *materializeAppV2) runOneSession(shard consumer.Shard, ch chan<- consume _ = m.client.Send(&pr.Materialize{ Task: &pr.Task{ Spec: specBytes, - Preview: false, MaxTransactions: 0, }, }) diff --git a/plans/runtime-v2/preview-harness.md b/plans/runtime-v2/preview-harness.md deleted file mode 100644 index f50d61b66ac..00000000000 --- a/plans/runtime-v2/preview-harness.md +++ /dev/null @@ -1,327 +0,0 @@ -# `flowctl raw preview-next` as a runtime-next E2E harness - -This is a hands-on guide for using `flowctl raw preview-next` as a -repeatable end-to-end test of the `runtime-next` + `leader` + `shuffle` -stack against a local Postgres database. It assumes the runtime-v2 -branch is checked out and built. - -`flowctl preview` is the legacy harness against the existing runtime -crate; the runtime-next harness lives under `raw` while the new stack -is in development. They share most flags and the same test spec format. - -**Scope.** `preview-next` runs `runtime-next` + `leader` + `shuffle` -*in-process inside flowctl* (its own tonic server, sharing flowctl's -process-global `service_kit::Registry`). It does **not** go through the -reactor or the runtime-sidecar process. Pass `--debug-port ` to -mount the same handler dashboard the sidecar exposes on `--admin-port`, -served loopback-only on `127.0.0.1:/`. Captures, -materializations, and derivations are all supported. -To exercise the full reactor path (including the sidecar -admin surface), publish a task to a local data plane with -`shards: { flags: { enable-runtime-v2: "true" } }` (the -`estuary.dev/flag/enable-runtime-v2` shard label that `useRuntimeV2` in -`go/runtime/flow_consumer.go` checks); the sidecar's handler dashboard -is then at `http://127.0.0.1:/`. Note connectors on the -local stack run on the `supabase_network_flow` Docker network — see -`local/README.md` for the endpoint-address implications. - -Capture runtime-v2 inside the **reactor** process registers handlers on -the per-shard task service rather than on the Rust runtime sidecar -admin surface; the sidecar dashboard surfaces only Shuffle Leader / -Shuffle handlers. `preview-next --debug-port` is the one place where -capture handlers (kind `shard.capture`) show up on a service-kit -dashboard. - -Minimal local capture shape: - -```yaml -captures: - acmeCo/hello-world: - endpoint: - connector: - image: ghcr.io/estuary/source-hello-world:dev - config: - rate: 2 - interval: 5s - shards: - logLevel: info - flags: - enable-runtime-v2: "true" - bindings: - - resource: - name: greetings - prefix: "Hello {}!" - target: acmeCo/events - -collections: - acmeCo/events: - schema: - $schema: "http://json-schema.org/draft/2020-12/schema" - type: object - properties: - ts: - type: string - format: date-time - message: - type: string - required: [ts, message] - key: [/ts] -``` - -Useful validation loop after publishing: - -```bash -cargo run -p flowctl -- --profile local catalog publish --source ./capture.flow.yaml - -journalctl --user -u flow-reactor@local-cluster-8099.service --since '10 min ago' --no-pager \ - | rg -i 'acmeCo/hello-world|created partition|servePrimary failed|not authorized|transaction|stats' - -set -a -. ~/flow-local/env/reactor-local-cluster-8099.env -set +a - -~/go/bin/flowctl-go journals list \ - --selector 'name:prefix=acmeCo/events/' \ - --format table - -timeout 15 ~/go/bin/flowctl-go journals read \ - --selector 'name=acmeCo/events/14d588f6a580018e/pivot=00' \ - --output - \ - --file-root ~/flow-local/fragments -``` - -**Exactly-once probe.** `source-hello-world` emits `"Hello N!"` for increasing -`N` and persists `N` as connector state, so the counter **resumes rather than -restarting at 0** across a reactor restart, spec update, or disable + re-enable. -Read the collection back and assert `N` has no gaps or duplicates — a cheap -check that runtime-v2 carries connector checkpoint state across sessions exactly -once. (Also reproducible in-process via `preview-next --sessions 2,2,2` against -a persistent shard-zero RocksDB tempdir.) - -## One-time setup - -Done once per workstation. Skip the steps you've already completed. - -### 1. Local Postgres - -A Postgres reachable at `localhost:5432` with `postgres / postgres` -credentials (this matches the dev `supabase` instance the repo already -ships with). Quick verify: - -```bash -psql postgresql://postgres:postgres@localhost:5432/postgres -c 'SELECT 1;' -``` - -### 2. Build a native materialize-postgres binary - -The published `ghcr.io/estuary/materialize-postgres:dev` image only -ships `linux/amd64`. On ARM hosts you can build the connector natively -from the sibling `connectors` repo. We use `local:` mode in the spec to -drive the binary directly, avoiding container plumbing entirely: - -```bash -cd /home/johnny/estuary/connectors/materialize-postgres -go build -o /tmp/materialize-postgres . -``` - -Re-run when the connector source changes. Any other `materialize-*` -connector under `connectors/` works the same way. - -### 3. Build flowctl - -From this repo: - -```bash -cd /home/johnny/estuary/flow -cargo build -p flowctl --bin flowctl -``` - -The resulting binary is at `/home/johnny/cargo-target/debug/flowctl`. - -## Repeatable E2E run - -### 1. The harness spec - -Live at `/tmp/preview-test/local.flow.yaml` (or wherever you keep it). -The `local:` endpoint plus `protobuf: true` skips Docker, runs the -connector as a child process. - -```yaml -materializations: - test/preview/wiki: - endpoint: - local: - command: - - /tmp/materialize-postgres - config: - address: localhost:5432 - user: postgres - credentials: - auth_type: UserPassword - password: postgres - database: postgres - schema: public - protobuf: true - shards: - logLevel: info - bindings: - - source: demo/wikipedia/recentchange-sampled - resource: - table: preview_wiki - schema: public -``` - -Notes: -- The `--name` you'll pass to `flowctl raw preview-next` is the - materialization name — `test/preview/wiki` here. -- Source is a real production collection. flowctl auths reads via your - flowctl token (`~/.flowctl/config-default.yaml`). Materializations require - a logged-in token; captures don't (they don't read journals, and the - in-process Leader / Shuffle stack is not constructed for the capture path). -- Pick a `resource.table` name that's unique per scenario you're - exercising — leftover state from prior runs (the checkpoint table - `flow_checkpoints_v1` and the per-binding table) will block - re-validation. See **Reset Postgres state** below. - -### 2. Reset Postgres state - -Each fresh run requires the bindings table absent (the connector -refuses to bind a new materialization onto a pre-existing table) and -the materialization checkpoint table clean (otherwise `Apply` is a -re-attach against stale state): - -```bash -psql postgresql://postgres:postgres@localhost:5432/postgres -c ' - DROP TABLE IF EXISTS public.preview_wiki, - public.flow_checkpoints_v1 - CASCADE;' -``` - -If you change the binding's `table` in the spec, drop the *old* table -too — the connector will create the new one but refuses to overwrite -either. - -### 3. Run the harness - -The minimal invocation: - -```bash -cd /tmp/preview-test -RUST_BACKTRACE=1 RUST_LOG=h2=info,info /home/johnny/cargo-target/debug/flowctl raw preview-next \ - --source ./local.flow.yaml \ - --name test/preview/wiki \ - --sessions=-1 \ - --timeout 60s \ - 2> preview.stderr -``` - -Flags worth knowing: -- `--sessions=-1` — one unbounded session (default is also one - unbounded session). Use `--sessions 2,2,2` to exercise cross-session - recovery: three sessions of two transactions each, against a single - persistent shard-zero RocksDB tempdir. -- `--shards N` — synthetic N-shard topology. For materializations N=1 - (default) hits the fast-path Join consensus and N≥2 exercises full - multi-shard rendezvous. The `materialize-postgres` spec above is not a - valid N>1 materialization workload: each shard drives an independent - connector transaction against the same table. Use a connector/spec - designed for multi-shard materialization before treating N>1 results as - runtime signal. Captures with N≥2 fan out N independent shards (own - connector, RocksDB, publisher) — runtime-correct, but the connector - must honor `Open.range.key_begin/key_end` to split work meaningfully. - Most capture connectors (including `source-hello-world`) ignore the - range and will simply duplicate output across shards. -- `--timeout 60s` — graceful stop trigger. Set high enough that the - close-policy can fire on whatever your source produces. -- `--log-json` — JSON ops logs to stderr. Off by default; useful when - feeding the run into log tooling. -- `--debug-port 9999` — loopback HTTP port hosting the service-kit - admin dashboard. Off by default. Mirror of the runtime-sidecar's - `--admin-port`: handler inventory at `/`, JSON snapshot at - `/debug/handlers.json`, per-handler trace overrides via - `POST /debug/handlers/{id}/level/{level}`, Prometheus scrape at - `/metrics`. Per-handler trace overrides take effect because flowctl - installs the service-kit `layer_filter` + `event::layer` against a - process-global registry (see `crates/flowctl/src/main.rs`). - -Per-transaction observability is via `tracing` to stderr (see the -`Publisher::Preview` arm in `crates/runtime-next/src/publish.rs`): - -- `connector applied` — emitted by the leader's apply loop with the - connector's `action_description`, the iteration number, and any - applied connector-state patches (one per loop iteration). -- `transaction stats` — emitted once per committed transaction with the - full `ops::Stats` document (per-binding docs/bytes counts, etc). - -These events are at info level. Filter further with -`RUST_LOG=runtime_next::publish=info` if you want only these and nothing -else. - -### 4. Inspect what landed in Postgres - -Standard psql against the dev DB. The connector creates a checkpoint -metadata table alongside the binding table: - -```bash -# All flow tables -psql postgresql://postgres:postgres@localhost:5432/postgres \ - -c '\dt public.flow_*' \ - -c '\dt public.preview_wiki' - -# Row count + sample rows -psql postgresql://postgres:postgres@localhost:5432/postgres \ - -c 'SELECT count(*) FROM public.preview_wiki;' - -psql postgresql://postgres:postgres@localhost:5432/postgres \ - -c 'SELECT title, "user", wiki, type, timestamp - FROM public.preview_wiki - ORDER BY timestamp DESC - LIMIT 10;' - -# Per-binding committed checkpoint position (one row per shard) -psql postgresql://postgres:postgres@localhost:5432/postgres \ - -c 'SELECT * FROM public.flow_checkpoints_v1;' -``` - -A passing run leaves you with: -- `preview_wiki` populated with N rows where N = shuffled documents - combined per transaction × number of committed transactions. -- `flow_checkpoints_v1` with one row containing this shard's last - committed Frontier. - -## What's exercised by each scenario - -| Scenario | Validates | -|--------------------------------------------|---------------------------------------------------------------------------| -| `--sessions=-1 --timeout 60s` | Single open session, transactions close on `maxTxnDuration` / data volume | -| `--shards 4 --sessions=-1 --timeout 60s` | Multi-shard Join consensus, fan-out shuffle, leader cross-shard reduce with a multi-shard-safe connector/spec | -| `--sessions 2,2,2` | Cross-session recovery — sessions 2 and 3 see non-empty `L:Recover` | -| (Ctrl-C mid-session) | Clean tonic-server shutdown, tempdirs removed, no port left bound | -| `--name ` against a capture spec | Capture session runs; `Joined → Opened → … → Stopped` ladder | -| `--debug-port 9999` | Loopback dashboard at `http://127.0.0.1:9999/` lists live handlers; `POST /debug/handlers/{id}/level/trace` flips one handler to TRACE | -| `--name ` (SQLite) | In-process derive-sqlite (no Docker); remote-authoritative — single-shard only | -| `--name ` (TypeScript/Python) | Image connector via Docker (`derive-typescript:dev` / `derive-python:dev`); codegen + Deno compile; needs the musl `flow-connector-init` (see Known issues) | -| `--name --shards 2` | Multi-shard Join consensus + fan-out shuffle + leader cross-shard reduce (TypeScript; SQLite is rejected as single-shard) | -| `--name --sessions 2,2,2` | Cross-session exactly-once recovery: RocksDB-authoritative (TS) or remote-authoritative checkpoint (SQLite); read frontier resumes without gaps | - -## Known issues / current state - -- A single connector log line at startup renders as nested ANSI — it - comes from the legacy `runtime` crate's build-time validation path, - which doesn't set `LOG_FORMAT=json`. All runtime-next per-shard - connector logs render cleanly. - -- **Image connectors (TypeScript/Python derivations, image captures & - materializations) need a compatible `flow-connector-init`.** The - container runtime injects the host's `flow-connector-init` as the - entrypoint (`crates/runtime/src/container.rs`, via - `locate_bin::locate` — which prefers the binary *alongside* the - flowctl/reactor executable, then `$PATH`). If that build is - dynamically linked against a newer glibc than the connector image's - base, the container dies on exec with - `/flow-connector-init: ... GLIBC_2.3x not found`. Fix: build / - place the statically-linked **musl** `flow-connector-init` - (`cargo-target/x86_64-unknown-linux-musl/debug/flow-connector-init`, - `static-pie linked`) where `locate` finds it — e.g. copy it over - `cargo-target/debug/flow-connector-init`. In-process derive-sqlite is - unaffected (no container). \ No newline at end of file diff --git a/tests/soak/README.md b/tests/soak/README.md index af89e3366ea..a9d0f73b42c 100644 --- a/tests/soak/README.md +++ b/tests/soak/README.md @@ -31,7 +31,7 @@ surfaces as a downstream contradiction. Each document carries (see An account `id` is an integer in `[key_begin, key_begin + idRange)`, where `key_begin` is the shard's owned key range (from shard labels when published; an even u32 split under -`preview-next --shards N`). Transfers stay within a shard's window, so each shard's +`preview --shards N`). Transfers stay within a shard's window, so each shard's conservation is self-contained. On a shard **split** both children fork the parent's state but own disjoint windows: the low child keeps `key_begin` and its accounts; the high child gets a fresh window and (after pruning inherited out-of-window ids on `Open`) @@ -52,7 +52,7 @@ requires reading both at a causally-consistent cut within one transaction. A tor One file per task, each under a component directory that also homes that task's connector. Every file imports just the upstream specs it sources from, so each can be fed -to `flowctl raw preview-next` alone, building only what it needs. Full chain: `source` +to `flowctl preview` alone, building only what it needs. Full chain: `source` (capture) → `events/{alpha,beta,gamma}` → `accounts` (derivation) → `{views, ledger}` (materializations); every task carries `enable-runtime-v2`, and top-level `flow.yaml` imports all of them for a whole-chain publish. @@ -257,7 +257,7 @@ Supabase Docker network"). ## Running -### In-process (`flowctl raw preview-next`) +### In-process (`flowctl preview`) Fastest loop: `runtime-next` in-process, no reactor/publish/auth (see `plans/runtime-v2/preview-harness.md`). @@ -266,20 +266,20 @@ Fastest loop: `runtime-next` in-process, no reactor/publish/auth (see FLOWCTL=~/cargo-target/debug/flowctl # Single unbounded session, stop after 4s. -"$FLOWCTL" raw preview-next --source tests/soak/capture/flow.yaml \ +"$FLOWCTL" preview --source tests/soak/capture/flow.yaml \ --name test/soak/source --sessions=-1 --timeout 4s # Cross-session exactly-once: 3 sessions × 3 txns over one persistent RocksDB tempdir. -"$FLOWCTL" raw preview-next --source tests/soak/capture/flow.yaml \ +"$FLOWCTL" preview --source tests/soak/capture/flow.yaml \ --name test/soak/source --sessions 3,3,3 --timeout 30s # Shard scale-out: N synthetic shards split the u32 space into disjoint windows. # (Lower `idRange` in the config to force seq>0 reuse.) -"$FLOWCTL" raw preview-next --source tests/soak/capture/flow.yaml \ +"$FLOWCTL" preview --source tests/soak/capture/flow.yaml \ --name test/soak/source --shards 4 --sessions=-1 --timeout 4s ``` -The `accounts` derivation and `ledger` also run under preview-next, with two extra +The `accounts` derivation and `ledger` also run under preview, with two extra requirements. They are **image connectors** (derive-typescript needs Docker/Deno/a musl `flow-connector-init` entrypoint — see the preview-harness doc), and — unlike the capture — they **read source journals from the local stack**, so the upstream tasks must already @@ -292,12 +292,12 @@ export SSL_CERT_FILE=~/flow-local/ca.crt # Derivation — union, in-order, oracle, conservation. Add --shards N for real cross-shard # Flush scatter/gather; --sessions 3,3,3 for the cross-session exactly-once probe. -"$FLOWCTL" --profile local raw preview-next --source tests/soak/derivation/flow.yaml \ +"$FLOWCTL" --profile local preview --source tests/soak/derivation/flow.yaml \ --name test/soak/accounts --sessions=-1 --timeout 8s # Ledger — Loads, max-keys/exists, oracle integrity, StartedCommit→Acknowledge # conservation. Same --shards / --sessions variations apply. -"$FLOWCTL" --profile local raw preview-next --source tests/soak/materialization/ledger.flow.yaml \ +"$FLOWCTL" --profile local preview --source tests/soak/materialization/ledger.flow.yaml \ --name test/soak/ledger --sessions=-1 --timeout 8s ``` diff --git a/tests/soak/materialization/views.flow.yaml b/tests/soak/materialization/views.flow.yaml index f76b92509c5..203262d7330 100644 --- a/tests/soak/materialization/views.flow.yaml +++ b/tests/soak/materialization/views.flow.yaml @@ -25,7 +25,7 @@ materializations: test/soak/views: endpoint: connector: - image: ghcr.io/estuary/materialize-postgres:13946b2 + image: ghcr.io/estuary/materialize-postgres:dev config: address: supabase_db_flow:5432 database: postgres