From f06424df4660965c632b872f4d5cd03977150d94 Mon Sep 17 00:00:00 2001 From: pfried Date: Mon, 20 Nov 2017 10:34:33 -0500 Subject: [PATCH 01/73] fix how status readers/writers are created and passed around when initializing event streams --- flo-server/src/engine/controller/mod.rs | 10 ++-- flo-server/src/engine/event_stream/mod.rs | 38 ++++++++++----- .../src/engine/event_stream/partition/mod.rs | 47 +++++++++++++++---- 3 files changed, 68 insertions(+), 27 deletions(-) diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 028d33a..ca468d6 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -32,7 +32,6 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::sync::atomic::AtomicUsize; - use atomics::AtomicBoolWriter; debug!("Starting Flo Controller with: {:?}", options); @@ -42,18 +41,15 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul // Once we start work on clustering, the system stream will be used exclusively for cluster communication // and other event stream(s) will be used for application data - // There's only one machine, so all partitions will always be primary. Again, this is just temporary - let status_writer = AtomicBoolWriter::with_value(true); - let system_stream_dir = storage_dir.join(&default_stream_options.name); let event_stream_ref = if system_stream_dir.exists() { - init_existing_event_stream(system_stream_dir, default_stream_options, status_writer.reader(), remote)? + init_existing_event_stream(system_stream_dir, default_stream_options, remote)? } else { - init_new_event_stream(system_stream_dir, default_stream_options, status_writer.reader(), remote)? + init_new_event_stream(system_stream_dir, default_stream_options, remote)? }; let mut streams = HashMap::with_capacity(1); - streams.insert(system_stream_name(), event_stream_ref); + streams.insert(system_stream_name(), event_stream_ref.clone_ref()); let engine = EngineRef { current_connection_id: Arc::new(AtomicUsize::new(0)), diff --git a/flo-server/src/engine/event_stream/mod.rs b/flo-server/src/engine/event_stream/mod.rs index a836587..88330f1 100644 --- a/flo-server/src/engine/event_stream/mod.rs +++ b/flo-server/src/engine/event_stream/mod.rs @@ -9,8 +9,7 @@ use futures::{Sink, Async, AsyncSink, StartSend, Poll}; use chrono::Duration; use event::ActorId; -use self::partition::{PartitionRef, initialize_existing_partition, initialize_new_partition}; -use atomics::AtomicBoolReader; +use self::partition::{PartitionRef, PartitionRefMut, initialize_existing_partition, initialize_new_partition}; pub use self::highest_counter::HighestCounter; @@ -44,7 +43,7 @@ impl EventStreamOptions { -pub fn init_existing_event_stream(event_stream_storage_dir: PathBuf, options: EventStreamOptions, status_reader: AtomicBoolReader, remote: Remote) -> Result { +pub fn init_existing_event_stream(event_stream_storage_dir: PathBuf, options: EventStreamOptions, remote: Remote) -> Result { debug!("Starting initialization of existing event stream with: {:?}", &options); let partition_numbers = determine_existing_partition_dirs(&event_stream_storage_dir)?; @@ -54,34 +53,34 @@ pub fn init_existing_event_stream(event_stream_storage_dir: PathBuf, options: Ev let mut partition_refs = Vec::with_capacity(partition_numbers.len()); for partition_num in partition_numbers { - let partition_ref = initialize_existing_partition(partition_num, &event_stream_storage_dir, &options, status_reader.clone(), highest_counter.clone())?; + let partition_ref = initialize_existing_partition(partition_num, &event_stream_storage_dir, &options, highest_counter.clone())?; partition_refs.push(partition_ref); } partition_refs.sort_by_key(|part| part.partition_num()); let tick_interval = options.get_tick_interval(); - let event_stream = EventStreamRef { + let event_stream = EventStreamRefMut { name: options.name, partitions: partition_refs, }; - start_tick_timer(remote, event_stream.clone(), tick_interval); + start_tick_timer(remote, event_stream.clone_ref(), tick_interval); Ok(event_stream) } -pub fn init_new_event_stream(event_stream_storage_dir: PathBuf, options: EventStreamOptions, status_reader: AtomicBoolReader, remote: Remote) -> Result { +pub fn init_new_event_stream(event_stream_storage_dir: PathBuf, options: EventStreamOptions, remote: Remote) -> Result { debug!("Starting initialization of new event stream with: {:?}", &options); let partition_count = options.num_partitions; ::std::fs::create_dir_all(&event_stream_storage_dir)?; - let mut partition_refs: Vec = Vec::with_capacity(partition_count as usize); + let mut partition_refs: Vec = Vec::with_capacity(partition_count as usize); let highest_counter = HighestCounter::zero(); for i in 0..partition_count { let partition_num: ActorId = i + 1; - let partition_ref = initialize_new_partition(partition_num, &event_stream_storage_dir, &options, status_reader.clone(), highest_counter.clone())?; + let partition_ref = initialize_new_partition(partition_num, &event_stream_storage_dir, &options, highest_counter.clone())?; // We're appending these in order so that they can be indexed up by partition number later partition_refs.push(partition_ref); @@ -90,11 +89,11 @@ pub fn init_new_event_stream(event_stream_storage_dir: PathBuf, options: EventSt let tick_interval = options.get_tick_interval(); let EventStreamOptions{name, ..} = options; debug!("Finished initializing {} partitions for event stream: '{}'", partition_count, &name); - let event_stream = EventStreamRef { + let event_stream = EventStreamRefMut { name: name, partitions: partition_refs, }; - start_tick_timer(remote, event_stream.clone(), tick_interval); + start_tick_timer(remote, event_stream.clone_ref(), tick_interval); Ok(event_stream) } @@ -124,6 +123,23 @@ fn determine_existing_partition_dirs(event_stream_dir: &Path) -> io::Result +} + +impl EventStreamRefMut { + pub fn clone_ref(&self) -> EventStreamRef { + let name = self.name.clone(); + let partitions = self.partitions.iter().map(|part| part.clone_ref()).collect::>(); + + EventStreamRef { + name, + partitions + } + } +} #[derive(Clone, Debug)] pub struct EventStreamRef { diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index e9c5142..de8cca3 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -11,7 +11,7 @@ use std::sync::{Arc, RwLock}; use std::thread; use std::io; -use atomics::{AtomicCounterReader, AtomicBoolReader}; +use atomics::{AtomicCounterReader, AtomicBoolReader, AtomicBoolWriter}; use engine::ConnectionId; use engine::event_stream::{EventStreamOptions, HighestCounter}; use protocol::{ProduceEvent}; @@ -157,6 +157,22 @@ impl SharedReaderRefs { pub type AsyncProduceResult = Result; pub type AsyncConsumeResult = Result; +#[derive(Debug)] +pub struct PartitionRefMut { + status_writer: AtomicBoolWriter, + partition_ref: PartitionRef, +} + +impl PartitionRefMut { + pub fn partition_num(&self) -> ActorId { + self.partition_ref.partition_num() + } + + pub fn clone_ref(&self) -> PartitionRef { + self.partition_ref.clone() + } +} + #[derive(Clone, Debug)] pub struct PartitionRef { event_stream_name: String, @@ -223,23 +239,36 @@ impl PartitionRef { pub fn initialize_existing_partition(partition_num: ActorId, event_stream_data_dir: &Path, event_stream_options: &EventStreamOptions, - status_reader: AtomicBoolReader, - highest_counter: HighestCounter) -> io::Result { + highest_counter: HighestCounter) -> io::Result { + let status_writer = AtomicBoolWriter::with_value(true); let partition_data_dir = get_partition_data_dir(event_stream_data_dir, partition_num); - let partition_impl = PartitionImpl::init_existing(partition_num, partition_data_dir, event_stream_options, status_reader, highest_counter)?; - run_partition(partition_impl) + let partition_impl = PartitionImpl::init_existing(partition_num, partition_data_dir, event_stream_options, status_writer.reader(), highest_counter)?; + + run_partition(partition_impl).map(|partition_ref| { + PartitionRefMut { + status_writer, + partition_ref, + } + }) } pub fn initialize_new_partition(partition_num: ActorId, event_stream_data_dir: &Path, event_stream_options: &EventStreamOptions, - status_reader: AtomicBoolReader, - highest_counter: HighestCounter) -> io::Result { + highest_counter: HighestCounter) -> io::Result { + // TODO: for now we are starting every partition as primary. This will need to change once we have a raft implementation + let status_writer = AtomicBoolWriter::with_value(true); let partition_data_dir = get_partition_data_dir(event_stream_data_dir, partition_num); - let partition_impl = PartitionImpl::init_new(partition_num, partition_data_dir, &event_stream_options, status_reader, highest_counter)?; - run_partition(partition_impl) + let partition_impl = PartitionImpl::init_new(partition_num, partition_data_dir, &event_stream_options, status_writer.reader(), highest_counter)?; + run_partition(partition_impl).map(|partition_ref| { + PartitionRefMut { + status_writer, + partition_ref, + } + }) + } pub fn run_partition(partition_impl: PartitionImpl) -> io::Result { From d63a2bd9933f515064c839fb7ea0d5d7c0779cc1 Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 21 Nov 2017 21:47:19 -0500 Subject: [PATCH 02/73] initialize FloController on startup, though it doesn't actually do anything yet --- .../connection_handler/connection_state.rs | 2 + .../src/engine/connection_handler/mod.rs | 11 +- flo-server/src/engine/controller/mod.rs | 209 +++++++++++++++--- .../src/engine/controller/system_stream.rs | 31 +++ flo-server/src/engine/event_stream/mod.rs | 6 +- flo-server/src/engine/mod.rs | 21 +- 6 files changed, 240 insertions(+), 40 deletions(-) create mode 100644 flo-server/src/engine/controller/system_stream.rs diff --git a/flo-server/src/engine/connection_handler/connection_state.rs b/flo-server/src/engine/connection_handler/connection_state.rs index 11ab64d..b5b30f9 100644 --- a/flo-server/src/engine/connection_handler/connection_state.rs +++ b/flo-server/src/engine/connection_handler/connection_state.rs @@ -25,6 +25,8 @@ pub struct ConnectionState { impl ConnectionState { pub fn new(connection_id: ConnectionId, client_sender: ClientSender, engine: EngineRef, reactor: Handle) -> ConnectionState { let event_stream = engine.get_default_stream(); + debug!("Starting connection_id: {} with event_stream: {}", connection_id, event_stream.name()); + ConnectionState { client_name: None, connection_id, diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 49269cc..c1ba723 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -124,6 +124,7 @@ impl Debug for ConnectionHandler { #[cfg(test)] mod test { use std::collections::HashMap; + use std::sync::{Arc, Mutex}; use tokio_core::reactor::Core; use super::*; @@ -132,6 +133,7 @@ mod test { use engine::event_stream::EventStreamRef; use engine::event_stream::partition::*; use engine::ClientReceiver; + use engine::controller::SystemStreamRef; use atomics::{AtomicCounterWriter, AtomicBoolWriter}; struct Fixture { @@ -156,10 +158,11 @@ mod test { counter_writer.reader(), primary.reader(), tx); - let stream = EventStreamRef::new(system_stream_name(), vec![part_ref]); - let mut streams = HashMap::new(); - streams.insert(system_stream_name(), stream); - let engine = EngineRef::new(streams); + + let system_stream = SystemStreamRef::new(part_ref); + + let streams = Arc::new(Mutex::new(HashMap::new())); + let engine = EngineRef::new(system_stream, streams); let subject = ConnectionHandler::new(456, client_sender, engine.clone(), reactor.handle()); diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index ca468d6..72a7d13 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -1,17 +1,34 @@ +mod system_stream; -use std::path::PathBuf; -use std::sync::Arc; +use std::path::{PathBuf, Path}; +use std::sync::{Arc, Mutex}; use std::collections::HashMap; +use std::fs::DirEntry; use std::io; use tokio_core::reactor::Remote; -use engine::{EngineRef, system_stream_name}; +use engine::{EngineRef, system_stream_name, SYSTEM_STREAM_NAME}; use engine::event_stream::{EventStreamRef, + EventStreamRefMut, EventStreamOptions, init_existing_event_stream, init_new_event_stream}; +use engine::event_stream::partition::{PartitionSender, + PartitionReceiver, + PartitionRef, + Operation, + create_partition_channels}; +use engine::event_stream::partition::controller::PartitionImpl; +use atomics::{AtomicBoolReader, AtomicBoolWriter, AtomicCounterReader}; + + +pub use self::system_stream::SystemStreamRef; + + +/// Options passed to the controller on startup that determine how this instance will start and behave. +/// These options will come from the command line if this is a standalone server. #[derive(Debug, PartialEq)] pub struct ControllerOptions { pub storage_dir: PathBuf, @@ -21,39 +38,179 @@ pub struct ControllerOptions { /// A specialized event stream that always has exactly one partition and manages the cluster state and consensus /// Of course there is no cluster state and thus no consensus at the moment, but we'll just leave this here... -#[allow(dead_code)] // TODO: implement raft lol +#[allow(dead_code)] pub struct FloController { - engine_ref: Arc, - event_streams: HashMap, + /// Shared references to all event streams in the system + shared_event_stream_refs: Arc>>, + + /// Unique mutable references to every event stream in the system + event_streams: HashMap, + + /// used as defaults when creating new event streams + default_stream_options: EventStreamOptions, + + /// directory in which all event stream data is stored + storage_dir: PathBuf, + + /// the partition that persists system events. Used as the RAFT log + system_partition: PartitionImpl, + + /// used to set the status of the system stream. There is only ever at most one instance in a cluster + /// where this variable is true ...if things actually work correctly ;) + system_primary_status_writer: AtomicBoolWriter, } +impl FloController { + pub fn new(system_partition: PartitionImpl, + system_primary_setter: AtomicBoolWriter, + event_streams: HashMap, + storage_dir: PathBuf, + default_stream_options: EventStreamOptions) -> FloController { + let stream_refs = to_stream_refs(&event_streams); -pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Result { - use std::collections::HashMap; - use std::sync::{Arc, Mutex}; - use std::sync::atomic::AtomicUsize; + FloController { + shared_event_stream_refs: Arc::new(Mutex::new(stream_refs)), + event_streams, + system_partition, + storage_dir, + default_stream_options, + system_primary_status_writer: system_primary_setter, + } + } - debug!("Starting Flo Controller with: {:?}", options); + fn process(&mut self, _operation: Operation) { + unimplemented!() + } + fn shutdown(&mut self) { + info!("Shutting down FloController"); + } + + fn get_shared_streams(&self) -> Arc>> { + self.shared_event_stream_refs.clone() + } +} + +fn to_stream_refs(mut_refs: &HashMap) -> HashMap { + mut_refs.iter().map(|(k, v)| { + (k.to_owned(), v.clone_ref()) + }).collect::>() +} + + +pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Result { + debug!("Starting Flo Controller with: {:?}", options); let ControllerOptions{storage_dir, default_stream_options} = options; - // for now, we'll just create a default "system" stream. This is temporary. - // Once we start work on clustering, the system stream will be used exclusively for cluster communication - // and other event stream(s) will be used for application data + // TODO: initialize system primary status to false once clustering works + let system_primary_writer = AtomicBoolWriter::with_value(true); + let partition_result = init_system_partition(&storage_dir, + system_primary_writer.reader(), + &default_stream_options); + + partition_result.and_then(|system_partition| { + debug!("Initialized system partition"); + init_user_streams(&storage_dir, &default_stream_options, &remote).map(|user_streams| { + debug!("Initialized all {} user event streams", user_streams.len()); + + let system_primary_reader = system_primary_writer.reader(); + let system_highest_counter = system_partition.event_counter_reader(); - let system_stream_dir = storage_dir.join(&default_stream_options.name); - let event_stream_ref = if system_stream_dir.exists() { - init_existing_event_stream(system_stream_dir, default_stream_options, remote)? + let flo_controller = FloController::new(system_partition, + system_primary_writer, + user_streams, + storage_dir, + default_stream_options); + + let (system_partition_tx, system_partition_rx) = create_partition_channels(); + let engine_ref = create_engine_ref(&flo_controller, system_highest_counter, system_primary_reader, system_partition_tx); + + run_controller_impl(flo_controller, system_partition_rx); + engine_ref + }) + }) +} + +fn init_system_partition(storage_dir: &Path, system_primary_reader: AtomicBoolReader, default_stream_options: &EventStreamOptions) -> io::Result { + use engine::event_stream::HighestCounter; + + let mut system_partition_dir: PathBuf = storage_dir.join(system_stream_name()); + system_partition_dir.push("1"); + + if system_partition_dir.is_dir() { + PartitionImpl::init_existing(1, + system_partition_dir, + default_stream_options, + system_primary_reader, + HighestCounter::zero()) } else { - init_new_event_stream(system_stream_dir, default_stream_options, remote)? - }; + PartitionImpl::init_new(1, + system_partition_dir, + default_stream_options, + system_primary_reader, + HighestCounter::zero()) + } +} + +fn create_engine_ref(controller: &FloController, + system_highest_counter: AtomicCounterReader, + system_primary_reader: AtomicBoolReader, + system_sender: PartitionSender) -> EngineRef { - let mut streams = HashMap::with_capacity(1); - streams.insert(system_stream_name(), event_stream_ref.clone_ref()); + let system_partition_ref = PartitionRef::new(system_stream_name(), + 1, + system_highest_counter, + system_primary_reader, + system_sender); - let engine = EngineRef { - current_connection_id: Arc::new(AtomicUsize::new(0)), - event_streams: Arc::new(Mutex::new(streams)), - }; - Ok(engine) + let system_stream_ref = SystemStreamRef::new(system_partition_ref); + let shared_stream_refs = controller.get_shared_streams(); + EngineRef::new(system_stream_ref, shared_stream_refs) } + + +fn init_user_streams(storage_dir: &Path, options: &EventStreamOptions, remote: &Remote) -> io::Result> { + let mut user_streams = HashMap::new(); + for file_result in ::std::fs::read_dir(storage_dir)? { + let entry = file_result?; + if is_user_event_stream(&entry)? { + let stream_storage = entry.path(); + debug!("attempting to initialize user stream at path: {:?}", stream_storage); + let stream = init_existing_event_stream( + stream_storage, + options.clone(), + remote.clone())?; + user_streams.insert(stream.get_name().to_owned(), stream); + } + } + + if !user_streams.contains_key(&options.name) { + let new_stream_dir = storage_dir.join(&options.name); + let new_stream = init_new_event_stream( + new_stream_dir, + options.clone(), + remote.clone())?; + + user_streams.insert(options.name.clone(), new_stream); + } + + Ok(user_streams) +} + + +fn is_user_event_stream(dir_entry: &DirEntry) -> io::Result { + let is_dir = dir_entry.file_type()?.is_dir(); + Ok(is_dir && SYSTEM_STREAM_NAME != &dir_entry.file_name()) +} + + +fn run_controller_impl(mut controller: FloController, system_partition_rx: PartitionReceiver) { + ::std::thread::spawn(move || { + debug!("Starting FloController processing"); + while let Ok(operation) = system_partition_rx.recv() { + controller.process(operation); + } + controller.shutdown(); + }); +} + diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs new file mode 100644 index 0000000..5cb1a06 --- /dev/null +++ b/flo-server/src/engine/controller/system_stream.rs @@ -0,0 +1,31 @@ + +use engine::event_stream::partition::{PartitionRef}; +use engine::event_stream::EventStreamRef; + + + +#[derive(Clone, Debug)] +pub struct SystemStreamRef { + inner: PartitionRef, +} + +impl SystemStreamRef { + + pub fn new(partition_ref: PartitionRef) -> SystemStreamRef { + SystemStreamRef { + inner: partition_ref, + } + } + + pub fn to_event_stream(&self) -> EventStreamRef { + let name = self.inner.event_stream_name().to_owned(); + let partition_ref = vec![self.inner.clone()]; + EventStreamRef::new(name, partition_ref) + } +} + + + + + + diff --git a/flo-server/src/engine/event_stream/mod.rs b/flo-server/src/engine/event_stream/mod.rs index 88330f1..82e49e0 100644 --- a/flo-server/src/engine/event_stream/mod.rs +++ b/flo-server/src/engine/event_stream/mod.rs @@ -13,7 +13,7 @@ use self::partition::{PartitionRef, PartitionRefMut, initialize_existing_partiti pub use self::highest_counter::HighestCounter; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct EventStreamOptions { pub name: String, pub num_partitions: u16, @@ -130,6 +130,10 @@ pub struct EventStreamRefMut { } impl EventStreamRefMut { + pub fn get_name(&self) -> &str { + &self.name + } + pub fn clone_ref(&self) -> EventStreamRef { let name = self.name.clone(); let partitions = self.partitions.iter().map(|part| part.clone_ref()).collect::>(); diff --git a/flo-server/src/engine/mod.rs b/flo-server/src/engine/mod.rs index c3b4c00..7b3eaa0 100644 --- a/flo-server/src/engine/mod.rs +++ b/flo-server/src/engine/mod.rs @@ -11,7 +11,7 @@ use protocol::ProtocolMessage; use event::OwnedFloEvent; use self::event_stream::EventStreamRef; -pub use self::controller::{ControllerOptions, start_controller}; +pub use self::controller::{ControllerOptions, SystemStreamRef, start_controller}; pub use self::connection_handler::{ConnectionHandler, ConnectionHandlerResult}; pub type ConnectionId = usize; @@ -40,6 +40,7 @@ pub fn system_stream_name() -> String { #[derive(Clone, Debug)] pub struct EngineRef { current_connection_id: Arc, + system_stream: SystemStreamRef, event_streams: Arc>> } @@ -50,14 +51,11 @@ pub enum ConnectError { } impl EngineRef { - pub fn new(streams: HashMap) -> EngineRef { - if !streams.contains_key(SYSTEM_STREAM_NAME) { - panic!("Cannot create engine ref without a default stream"); - } - + pub fn new(system_stream: SystemStreamRef, event_streams: Arc>>) -> EngineRef { EngineRef { current_connection_id: Arc::new(AtomicUsize::new(0)), - event_streams: Arc::new(Mutex::new(streams)) + system_stream, + event_streams } } @@ -76,8 +74,13 @@ impl EngineRef { } pub fn get_default_stream(&self) -> EventStreamRef { - let guard = self.event_streams.lock().unwrap(); - guard.get(SYSTEM_STREAM_NAME).unwrap().clone() + let stream = { + let guard = self.event_streams.lock().unwrap(); + guard.values().next().map(|stream| stream.clone()) + }; + stream.unwrap_or_else(|| { + self.system_stream.to_event_stream() + }) } } From 5ff88c0c6c8e687717d082492cb42d54a52702a4 Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 21 Nov 2017 22:42:13 -0500 Subject: [PATCH 03/73] move engine initialization into a separate module --- .../src/engine/controller/initialization.rs | 146 ++++++++++++++++++ flo-server/src/engine/controller/mod.rs | 146 +----------------- 2 files changed, 153 insertions(+), 139 deletions(-) create mode 100644 flo-server/src/engine/controller/initialization.rs diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs new file mode 100644 index 0000000..b553c7c --- /dev/null +++ b/flo-server/src/engine/controller/initialization.rs @@ -0,0 +1,146 @@ +use std::path::{PathBuf, Path}; +use std::collections::HashMap; +use std::fs::DirEntry; +use std::io; + +use tokio_core::reactor::Remote; + +use engine::{EngineRef, system_stream_name, SYSTEM_STREAM_NAME}; +use engine::event_stream::{EventStreamRefMut, + EventStreamOptions, + init_existing_event_stream, + init_new_event_stream}; + +use engine::event_stream::partition::{PartitionSender, + PartitionReceiver, + PartitionRef, + create_partition_channels}; +use engine::event_stream::partition::controller::PartitionImpl; +use atomics::{AtomicBoolReader, AtomicBoolWriter, AtomicCounterReader}; + +use super::{FloController, SystemStreamRef}; + + +/// Options passed to the controller on startup that determine how this instance will start and behave. +/// These options will come from the command line if this is a standalone server. +#[derive(Debug, PartialEq)] +pub struct ControllerOptions { + pub storage_dir: PathBuf, + pub default_stream_options: EventStreamOptions, +} + +pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Result { + debug!("Starting Flo Controller with: {:?}", options); + let ControllerOptions{storage_dir, default_stream_options} = options; + + // TODO: initialize system primary status to false once clustering works + let system_primary_writer = AtomicBoolWriter::with_value(true); + let partition_result = init_system_partition(&storage_dir, + system_primary_writer.reader(), + &default_stream_options); + + partition_result.and_then(|system_partition| { + debug!("Initialized system partition"); + init_user_streams(&storage_dir, &default_stream_options, &remote).map(|user_streams| { + debug!("Initialized all {} user event streams", user_streams.len()); + + let system_primary_reader = system_primary_writer.reader(); + let system_highest_counter = system_partition.event_counter_reader(); + + let flo_controller = FloController::new(system_partition, + system_primary_writer, + user_streams, + storage_dir, + default_stream_options); + + let (system_partition_tx, system_partition_rx) = create_partition_channels(); + let engine_ref = create_engine_ref(&flo_controller, system_highest_counter, system_primary_reader, system_partition_tx); + + run_controller_impl(flo_controller, system_partition_rx); + engine_ref + }) + }) +} + +fn init_system_partition(storage_dir: &Path, system_primary_reader: AtomicBoolReader, default_stream_options: &EventStreamOptions) -> io::Result { + use engine::event_stream::HighestCounter; + + let mut system_partition_dir: PathBuf = storage_dir.join(system_stream_name()); + system_partition_dir.push("1"); + + if system_partition_dir.is_dir() { + PartitionImpl::init_existing(1, + system_partition_dir, + default_stream_options, + system_primary_reader, + HighestCounter::zero()) + } else { + PartitionImpl::init_new(1, + system_partition_dir, + default_stream_options, + system_primary_reader, + HighestCounter::zero()) + } +} + +fn create_engine_ref(controller: &FloController, + system_highest_counter: AtomicCounterReader, + system_primary_reader: AtomicBoolReader, + system_sender: PartitionSender) -> EngineRef { + + let system_partition_ref = PartitionRef::new(system_stream_name(), + 1, + system_highest_counter, + system_primary_reader, + system_sender); + + let system_stream_ref = SystemStreamRef::new(system_partition_ref); + let shared_stream_refs = controller.get_shared_streams(); + EngineRef::new(system_stream_ref, shared_stream_refs) +} + + +fn init_user_streams(storage_dir: &Path, options: &EventStreamOptions, remote: &Remote) -> io::Result> { + let mut user_streams = HashMap::new(); + for file_result in ::std::fs::read_dir(storage_dir)? { + let entry = file_result?; + if is_user_event_stream(&entry)? { + let stream_storage = entry.path(); + debug!("attempting to initialize user stream at path: {:?}", stream_storage); + let stream = init_existing_event_stream( + stream_storage, + options.clone(), + remote.clone())?; + user_streams.insert(stream.get_name().to_owned(), stream); + } + } + + if !user_streams.contains_key(&options.name) { + let new_stream_dir = storage_dir.join(&options.name); + let new_stream = init_new_event_stream( + new_stream_dir, + options.clone(), + remote.clone())?; + + user_streams.insert(options.name.clone(), new_stream); + } + + Ok(user_streams) +} + + +fn is_user_event_stream(dir_entry: &DirEntry) -> io::Result { + let is_dir = dir_entry.file_type()?.is_dir(); + Ok(is_dir && SYSTEM_STREAM_NAME != &dir_entry.file_name()) +} + + +fn run_controller_impl(mut controller: FloController, system_partition_rx: PartitionReceiver) { + ::std::thread::spawn(move || { + debug!("Starting FloController processing"); + while let Ok(operation) = system_partition_rx.recv() { + controller.process(operation); + } + controller.shutdown(); + }); +} diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 72a7d13..c9bace4 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -1,41 +1,24 @@ mod system_stream; +mod initialization; -use std::path::{PathBuf, Path}; +use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::collections::HashMap; -use std::fs::DirEntry; -use std::io; -use tokio_core::reactor::Remote; -use engine::{EngineRef, system_stream_name, SYSTEM_STREAM_NAME}; use engine::event_stream::{EventStreamRef, EventStreamRefMut, - EventStreamOptions, - init_existing_event_stream, - init_new_event_stream}; - -use engine::event_stream::partition::{PartitionSender, - PartitionReceiver, - PartitionRef, - Operation, - create_partition_channels}; + EventStreamOptions}; + +use engine::event_stream::partition::Operation; use engine::event_stream::partition::controller::PartitionImpl; -use atomics::{AtomicBoolReader, AtomicBoolWriter, AtomicCounterReader}; +use atomics::AtomicBoolWriter; +pub use self::initialization::{start_controller, ControllerOptions}; pub use self::system_stream::SystemStreamRef; -/// Options passed to the controller on startup that determine how this instance will start and behave. -/// These options will come from the command line if this is a standalone server. -#[derive(Debug, PartialEq)] -pub struct ControllerOptions { - pub storage_dir: PathBuf, - pub default_stream_options: EventStreamOptions, -} - - /// A specialized event stream that always has exactly one partition and manages the cluster state and consensus /// Of course there is no cluster state and thus no consensus at the moment, but we'll just leave this here... #[allow(dead_code)] @@ -98,119 +81,4 @@ fn to_stream_refs(mut_refs: &HashMap) -> HashMap io::Result { - debug!("Starting Flo Controller with: {:?}", options); - let ControllerOptions{storage_dir, default_stream_options} = options; - - // TODO: initialize system primary status to false once clustering works - let system_primary_writer = AtomicBoolWriter::with_value(true); - let partition_result = init_system_partition(&storage_dir, - system_primary_writer.reader(), - &default_stream_options); - - partition_result.and_then(|system_partition| { - debug!("Initialized system partition"); - init_user_streams(&storage_dir, &default_stream_options, &remote).map(|user_streams| { - debug!("Initialized all {} user event streams", user_streams.len()); - - let system_primary_reader = system_primary_writer.reader(); - let system_highest_counter = system_partition.event_counter_reader(); - - let flo_controller = FloController::new(system_partition, - system_primary_writer, - user_streams, - storage_dir, - default_stream_options); - - let (system_partition_tx, system_partition_rx) = create_partition_channels(); - let engine_ref = create_engine_ref(&flo_controller, system_highest_counter, system_primary_reader, system_partition_tx); - - run_controller_impl(flo_controller, system_partition_rx); - engine_ref - }) - }) -} - -fn init_system_partition(storage_dir: &Path, system_primary_reader: AtomicBoolReader, default_stream_options: &EventStreamOptions) -> io::Result { - use engine::event_stream::HighestCounter; - - let mut system_partition_dir: PathBuf = storage_dir.join(system_stream_name()); - system_partition_dir.push("1"); - - if system_partition_dir.is_dir() { - PartitionImpl::init_existing(1, - system_partition_dir, - default_stream_options, - system_primary_reader, - HighestCounter::zero()) - } else { - PartitionImpl::init_new(1, - system_partition_dir, - default_stream_options, - system_primary_reader, - HighestCounter::zero()) - } -} - -fn create_engine_ref(controller: &FloController, - system_highest_counter: AtomicCounterReader, - system_primary_reader: AtomicBoolReader, - system_sender: PartitionSender) -> EngineRef { - - let system_partition_ref = PartitionRef::new(system_stream_name(), - 1, - system_highest_counter, - system_primary_reader, - system_sender); - - let system_stream_ref = SystemStreamRef::new(system_partition_ref); - let shared_stream_refs = controller.get_shared_streams(); - EngineRef::new(system_stream_ref, shared_stream_refs) -} - - -fn init_user_streams(storage_dir: &Path, options: &EventStreamOptions, remote: &Remote) -> io::Result> { - let mut user_streams = HashMap::new(); - for file_result in ::std::fs::read_dir(storage_dir)? { - let entry = file_result?; - if is_user_event_stream(&entry)? { - let stream_storage = entry.path(); - debug!("attempting to initialize user stream at path: {:?}", stream_storage); - let stream = init_existing_event_stream( - stream_storage, - options.clone(), - remote.clone())?; - user_streams.insert(stream.get_name().to_owned(), stream); - } - } - - if !user_streams.contains_key(&options.name) { - let new_stream_dir = storage_dir.join(&options.name); - let new_stream = init_new_event_stream( - new_stream_dir, - options.clone(), - remote.clone())?; - - user_streams.insert(options.name.clone(), new_stream); - } - - Ok(user_streams) -} - - -fn is_user_event_stream(dir_entry: &DirEntry) -> io::Result { - let is_dir = dir_entry.file_type()?.is_dir(); - Ok(is_dir && SYSTEM_STREAM_NAME != &dir_entry.file_name()) -} - - -fn run_controller_impl(mut controller: FloController, system_partition_rx: PartitionReceiver) { - ::std::thread::spawn(move || { - debug!("Starting FloController processing"); - while let Ok(operation) = system_partition_rx.recv() { - controller.process(operation); - } - controller.shutdown(); - }); -} From 56d136b0e9fda3c88c15a0b8e80ad553c855dbf0 Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 21 Nov 2017 22:45:36 -0500 Subject: [PATCH 04/73] only execute OSX builds with stable rust since they take longer --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index f639c1d..43ae6f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,4 +13,9 @@ os: matrix: allow_failures: - rust: nightly + exclude: + - rust: beta + os: osx + - rust: nightly + os: osx script: cargo test --all --no-fail-fast From 57b11bf450756c3f7d7744a9555cee69bf05c5a1 Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 21 Nov 2017 23:05:36 -0500 Subject: [PATCH 05/73] set exit code to 1 when running server fails with error --- flo-server/src/engine/controller/mod.rs | 11 ++++------- flo-server/src/main.rs | 1 + 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index c9bace4..0029eb6 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -49,7 +49,10 @@ impl FloController { event_streams: HashMap, storage_dir: PathBuf, default_stream_options: EventStreamOptions) -> FloController { - let stream_refs = to_stream_refs(&event_streams); + + let stream_refs = event_streams.iter().map(|(k, v)| { + (k.to_owned(), v.clone_ref()) + }).collect::>(); FloController { shared_event_stream_refs: Arc::new(Mutex::new(stream_refs)), @@ -74,11 +77,5 @@ impl FloController { } } -fn to_stream_refs(mut_refs: &HashMap) -> HashMap { - mut_refs.iter().map(|(k, v)| { - (k.to_owned(), v.clone_ref()) - }).collect::>() -} - diff --git a/flo-server/src/main.rs b/flo-server/src/main.rs index cc21463..c856bb1 100644 --- a/flo-server/src/main.rs +++ b/flo-server/src/main.rs @@ -144,6 +144,7 @@ fn main() { let run_finished = server::run(server_options); if let Some(err) = run_finished.err() { error!("IO Error: {}", err); + ::std::process::exit(1); } info!("Shutdown server"); } From 151bdc8aae156c747330b3267d13bf219a53ea06 Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 21 Nov 2017 23:09:18 -0500 Subject: [PATCH 06/73] increase timeout on expiration test to account for really slow OSX machines on travis --- flo-server/tests/embedded_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flo-server/tests/embedded_tests.rs b/flo-server/tests/embedded_tests.rs index 5d34c8a..027c6a6 100644 --- a/flo-server/tests/embedded_tests.rs +++ b/flo-server/tests/embedded_tests.rs @@ -123,7 +123,7 @@ fn oldest_events_are_dropped_from_beginning_of_stream_after_time_based_expiratio } let start_time = ::std::time::Instant::now(); - let until_all_expired = (retention_duration + segment_duration + chrono::Duration::milliseconds(250)).to_std().unwrap(); + let until_all_expired = (retention_duration + segment_duration + chrono::Duration::milliseconds(750)).to_std().unwrap(); let mut first_event_id = FloEventId::new(1, 0); let mut vv = VersionVector::new(); vv.set(first_event_id); From 5373124f10afb486aabe8daa92cc3ebc98f7dfc6 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 26 Nov 2017 10:25:22 -0500 Subject: [PATCH 07/73] WIP on cluster communication --- Cargo.lock | 3 + flo-protocol/src/client.rs | 56 +++++++-- flo-server/Cargo.toml | 3 + flo-server/src/embedded/mod.rs | 1 + .../connection_handler/connection_state.rs | 61 ++++++---- .../src/engine/connection_handler/mod.rs | 15 ++- .../src/engine/connection_handler/peer.rs | 53 +++++++++ .../engine/controller/cluster_state/mod.rs | 14 +++ .../src/engine/controller/initialization.rs | 11 +- flo-server/src/engine/controller/mod.rs | 5 +- .../src/engine/controller/outgoing_io.rs | 107 ++++++++++++++++++ .../src/engine/controller/system_stream.rs | 10 +- .../event_stream/partition/controller/mod.rs | 4 + .../src/engine/event_stream/partition/mod.rs | 2 +- .../src/engine/event_stream/partition/ops.rs | 44 +++---- flo-server/src/engine/mod.rs | 11 +- .../engine/system_stream/flo_instance_id.rs | 35 ++++++ flo-server/src/engine/system_stream/mod.rs | 31 +++++ .../flo_io/client_message_stream.rs | 0 flo-server/src/{server => }/flo_io/mod.rs | 0 .../flo_io/server_message_stream.rs | 0 flo-server/src/lib.rs | 4 +- flo-server/src/main.rs | 17 ++- flo-server/src/server/mod.rs | 18 ++- flo-server/src/server/server_options.rs | 2 +- flo-server/tests/embedded_tests.rs | 2 + 26 files changed, 442 insertions(+), 67 deletions(-) create mode 100644 flo-server/src/engine/connection_handler/peer.rs create mode 100644 flo-server/src/engine/controller/cluster_state/mod.rs create mode 100644 flo-server/src/engine/controller/outgoing_io.rs create mode 100644 flo-server/src/engine/system_stream/flo_instance_id.rs create mode 100644 flo-server/src/engine/system_stream/mod.rs rename flo-server/src/{server => }/flo_io/client_message_stream.rs (100%) rename flo-server/src/{server => }/flo_io/mod.rs (100%) rename flo-server/src/{server => }/flo_io/server_message_stream.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 63531b2..60501c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -267,11 +267,14 @@ dependencies = [ "flo-protocol 0.2.0", "futures 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "log4rs 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "memmap 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/flo-protocol/src/client.rs b/flo-protocol/src/client.rs index f773281..fc9c144 100644 --- a/flo-protocol/src/client.rs +++ b/flo-protocol/src/client.rs @@ -242,7 +242,9 @@ pub struct ClientAnnounce { pub enum ProtocolMessage { /// Always the first message sent by the client to the server Announce(ClientAnnounce), - /// Contains basic information about the status of an event stream + /// Always the first message sent by a server to another server + PeerAnnounce(EventStreamStatus), + /// Sent in response to an Announce message. Contains basic information about the status of an event stream StreamStatus(EventStreamStatus), /// Set the event stream that the client will work with SetEventStream(SetEventStream), @@ -301,15 +303,35 @@ named!{parse_partition_status, named!{parse_event_stream_status>, chain!( _tag: tag!(&[EVENT_STREAM_STATUS]) ~ + status: parse_event_stream_status_struct, + || { + ProtocolMessage::StreamStatus(status) + } + ) +} + +named!{parse_peer_announce>, + chain!( + _tag: tag!(&[PEER_ANNOUNCE]) ~ + status: parse_event_stream_status_struct, + || { + ProtocolMessage::PeerAnnounce(status) + } + ) +} + + +named!{parse_event_stream_status_struct, + chain!( op_id: be_u32 ~ name: parse_str ~ partitions: length_count!(be_u16, parse_partition_status), || { - ProtocolMessage::StreamStatus(EventStreamStatus { + EventStreamStatus { op_id: op_id, name: name, partitions: partitions, - }) + } } ) } @@ -524,7 +546,8 @@ named!{pub parse_any>, alt!( parse_new_start_consuming | parse_set_event_stream | parse_event_stream_status | - parse_client_announce + parse_client_announce | + parse_peer_announce )} fn serialize_new_produce_header(header: &ProduceEvent, buf: &mut [u8]) -> usize { @@ -571,9 +594,9 @@ fn serialize_receive_event_header(event: &E, buf: &mut [u8]) -> usi .finish() } -fn serialize_event_stream_status(status: &EventStreamStatus, buf: &mut [u8]) -> usize { +fn serialize_event_stream_status(header: u8, status: &EventStreamStatus, buf: &mut [u8]) -> usize { Serializer::new(buf) - .write_u8(EVENT_STREAM_STATUS) + .write_u8(header) .write_u32(status.op_id) .write_string(&status.name) .write_u16(status.partitions.len() as u16) @@ -599,8 +622,11 @@ impl ProtocolMessage { .write_u32(announce.consume_batch_size.unwrap_or(0)) .finish() } + ProtocolMessage::PeerAnnounce(ref status) => { + serialize_event_stream_status(PEER_ANNOUNCE, status, buf) + } ProtocolMessage::StreamStatus(ref status) => { - serialize_event_stream_status(status, buf) + serialize_event_stream_status(EVENT_STREAM_STATUS, status, buf) } ProtocolMessage::SetEventStream(ref set_stream) => { Serializer::new(buf) @@ -744,6 +770,22 @@ mod test { test_serialize_then_deserialize(&ProtocolMessage::Announce(announce)); } + #[test] + fn peer_announce() { + let status = EventStreamStatus { + op_id: 1, + name: "system".to_owned(), + partitions: vec![ + PartitionStatus { + partition_num: 1, + head: 638, + primary: true, + } + ], + }; + test_serialize_then_deserialize(&ProtocolMessage::PeerAnnounce(status)); + } + #[test] fn serde_event_stream_status() { let status = EventStreamStatus { diff --git a/flo-server/Cargo.toml b/flo-server/Cargo.toml index 8ae0418..e1f5ed3 100644 --- a/flo-server/Cargo.toml +++ b/flo-server/Cargo.toml @@ -18,6 +18,9 @@ futures = "0.1.16" glob = "0.2" chrono = "^0.2" memmap = "0.5.2" +libc = "0.2.33" +serde = "0.9.15" +serde_json = "0.9.10" [dev-dependencies] env_logger = "*" diff --git a/flo-server/src/embedded/mod.rs b/flo-server/src/embedded/mod.rs index 23ed4f1..efe418b 100644 --- a/flo-server/src/embedded/mod.rs +++ b/flo-server/src/embedded/mod.rs @@ -66,6 +66,7 @@ fn message_to_owned(server_msg: SendProtocolMessage) -> ClientProtocolMessage { ProtocolMessage::CursorCreated(op) => ProtocolMessage::CursorCreated(op), ProtocolMessage::Announce(op) => ProtocolMessage::Announce(op), ProtocolMessage::SetEventStream(op) => ProtocolMessage::SetEventStream(op), + ProtocolMessage::PeerAnnounce(op) => ProtocolMessage::PeerAnnounce(op), } } diff --git a/flo-server/src/engine/connection_handler/connection_state.rs b/flo-server/src/engine/connection_handler/connection_state.rs index b5b30f9..0cf3c8c 100644 --- a/flo-server/src/engine/connection_handler/connection_state.rs +++ b/flo-server/src/engine/connection_handler/connection_state.rs @@ -5,6 +5,8 @@ use protocol::*; use engine::{ConnectionId, ClientSender, EngineRef, SendProtocolMessage}; use engine::event_stream::EventStreamRef; +use engine::system_stream::SystemOp; +use engine::controller::SystemStreamRef; use super::ConnectionHandlerResult; @@ -51,7 +53,7 @@ impl ConnectionState { } pub fn send_stream_status(&mut self, op_id: u32) -> ConnectionHandlerResult { - let status = create_stream_status(op_id, &self.event_stream); + let status = self.get_current_stream_status(op_id); self.send_to_client(ProtocolMessage::StreamStatus(status)).map_err(|err| { format!("Error sending message to client: {:?}", err) }) @@ -63,7 +65,7 @@ impl ConnectionState { match self.engine.get_stream(&name) { Ok(new_stream) => { debug!("Setting event stream to '{}' for {:?}", new_stream.name(), self); - let stream_status = create_stream_status(op_id, &new_stream); + let stream_status = self.get_current_stream_status(op_id); self.event_stream = new_stream; self.send_to_client(ProtocolMessage::StreamStatus(stream_status)) } @@ -86,32 +88,47 @@ impl ConnectionState { } } + pub fn set_to_system_stream(&mut self) { + debug!("Setting connection_id: {} to system stream", self.connection_id); + if self.event_stream.name() != ::engine::SYSTEM_STREAM_NAME { + let system_stream = self.engine.get_system_stream().to_event_stream(); + self.event_stream = system_stream; + } + } + + pub fn get_current_stream_status(&self, op_id: u32) -> EventStreamStatus { + let stream_ref = &self.event_stream; + let mut partition_statuses = Vec::with_capacity(stream_ref.get_partition_count() as usize); + + for partition in stream_ref.partitions() { + let num = partition.partition_num(); + let head = partition.get_highest_event_counter(); + let primary = partition.is_primary(); + let part_status = PartitionStatus { + partition_num: num, + head: head, + primary: primary, + }; + partition_statuses.push(part_status); + } + + EventStreamStatus { + op_id: op_id, + name: stream_ref.name().to_owned(), + partitions: partition_statuses, + } + } + pub fn send_to_client(&self, message: SendProtocolMessage) -> ConnectionHandlerResult { + trace!("Sending to connection_id: {}, message: {:?}", self.connection_id, message); self.client_sender.unbounded_send(message).map_err(|e| { format!("Error sending outgoing message for connection_id: {}, message: {:?}", self.connection_id, e.into_inner()) }) } -} - -fn create_stream_status(op_id: u32, stream_ref: &EventStreamRef) -> EventStreamStatus { - let mut partition_statuses = Vec::with_capacity(stream_ref.get_partition_count() as usize); - - for partition in stream_ref.partitions() { - let num = partition.partition_num(); - let head = partition.get_highest_event_counter(); - let primary = partition.is_primary(); - let part_status = PartitionStatus { - partition_num: num, - head: head, - primary: primary, - }; - partition_statuses.push(part_status); - } - EventStreamStatus { - op_id: op_id, - name: stream_ref.name().to_owned(), - partitions: partition_statuses, + pub fn get_system_stream(&mut self) -> &mut SystemStreamRef { + self.engine.system_stream() } } + diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index c1ba723..0bdd768 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -1,6 +1,7 @@ pub mod connection_state; mod consumer; mod producer; +mod peer; use std::fmt::{self, Debug}; use std::io; @@ -14,12 +15,14 @@ use engine::{ConnectionId, ClientSender, EngineRef, ReceivedProtocolMessage}; use self::connection_state::ConnectionState; use self::consumer::ConsumerConnectionState; use self::producer::ProducerConnectionState; +use self::peer::PeerConnectionState; pub struct ConnectionHandler { common_state: ConnectionState, consumer_state: ConsumerConnectionState, producer_state: ProducerConnectionState, + peer_state: PeerConnectionState, } @@ -31,9 +34,17 @@ impl ConnectionHandler { common_state: ConnectionState::new(connection, client_sender, engine, handle), consumer_state: ConsumerConnectionState::new(), producer_state: ProducerConnectionState::new(), + peer_state: PeerConnectionState::Init, } } + pub fn upgrade_to_outgoing_peer(&mut self) { + debug!("upgrading connection_id: {} to outgoing peer connection", self.common_state.connection_id); + let ConnectionHandler{ref mut common_state, ref mut peer_state, .. } = *self; + + peer_state.initiate_outgoing_peer_connection(common_state); + } + pub fn can_process(&self, _message: &ReceivedProtocolMessage) -> bool { !self.producer_state.requires_poll_complete() && !self.consumer_state.requires_poll_complete() } @@ -41,7 +52,7 @@ impl ConnectionHandler { pub fn handle_incoming_message(&mut self, message: ReceivedProtocolMessage) -> ConnectionHandlerResult { trace!("client: {:?}, received message: {:?}", self.common_state, message); - let ConnectionHandler{ref mut common_state, ref mut consumer_state, ref mut producer_state } = *self; + let ConnectionHandler{ref mut common_state, ref mut consumer_state, ref mut producer_state, .. } = *self; match message { ProtocolMessage::SetEventStream(SetEventStream{op_id, name}) => { @@ -87,7 +98,7 @@ impl Sink for ConnectionHandler { } fn poll_complete(&mut self) -> Poll<(), Self::SinkError> { - let ConnectionHandler {ref mut common_state, ref mut consumer_state, ref mut producer_state} = *self; + let ConnectionHandler {ref mut common_state, ref mut consumer_state, ref mut producer_state, ..} = *self; if producer_state.requires_poll_complete() { producer_state.poll_produce_complete(common_state) diff --git a/flo-server/src/engine/connection_handler/peer.rs b/flo-server/src/engine/connection_handler/peer.rs new file mode 100644 index 0000000..6b4e2c4 --- /dev/null +++ b/flo-server/src/engine/connection_handler/peer.rs @@ -0,0 +1,53 @@ + +use protocol::{ProtocolMessage, EventStreamStatus}; +use engine::{ReceivedProtocolMessage, ConnectionId}; +use super::connection_state::ConnectionState; +use super::ConnectionHandlerResult; + + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum PeerConnectionState { + Init, + AwaitingPeerResponse, + Peer, +} + + +impl PeerConnectionState { + pub fn initiate_outgoing_peer_connection(&mut self, state: &mut ConnectionState) { + assert_eq!(PeerConnectionState::Init, *self); + state.set_to_system_stream(); + let system_stream_state = state.get_current_stream_status(1); + let peer_announce = ProtocolMessage::PeerAnnounce(system_stream_state); + state.send_to_client(peer_announce).expect("failed to send peer announce when establishing outgoing connection"); + + *self = PeerConnectionState::AwaitingPeerResponse; + } + + + pub fn message_received(&mut self, message: ReceivedProtocolMessage, state: &mut ConnectionState) -> ConnectionHandlerResult { + + match message { + ProtocolMessage::StreamStatus(status) => { + self.stream_status_received(state, status); + } + other @ _ => { + error!("Unhandled protocol message from peer: {:?}", other); + } + } + + Ok(()) + } + + fn stream_status_received(&mut self, state: &mut ConnectionState, status: EventStreamStatus) { + self.set_state(state.connection_id, PeerConnectionState::Peer); + + + + } + + fn set_state(&mut self, connection_id: ConnectionId, new_state: PeerConnectionState) { + debug!("Transitioning connection_id: {} from {:?} to {:?}", connection_id, self, new_state); + *self = new_state; + } +} diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs new file mode 100644 index 0000000..cf1ba5b --- /dev/null +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -0,0 +1,14 @@ + +use std::net::SocketAddr; +use std::time::{Instant, Duration}; + +use engine::controller::outgoing_io::OutgoingConnectionCreator; + +pub struct ClusterState { + +} + +struct Peer { + peer_addr: SocketAddr, + +} diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index b553c7c..51fa9a3 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -2,6 +2,7 @@ use std::path::{PathBuf, Path}; use std::collections::HashMap; use std::fs::DirEntry; use std::io; +use std::net::SocketAddr; use tokio_core::reactor::Remote; @@ -27,11 +28,18 @@ use super::{FloController, SystemStreamRef}; pub struct ControllerOptions { pub storage_dir: PathBuf, pub default_stream_options: EventStreamOptions, + pub cluster_options: Option, +} + +#[derive(Debug, PartialEq)] +pub struct ClusterOptions { + pub this_instance_address: SocketAddr, + pub peer_addresses: Vec, } pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Result { debug!("Starting Flo Controller with: {:?}", options); - let ControllerOptions{storage_dir, default_stream_options} = options; + let ControllerOptions{storage_dir, default_stream_options, cluster_options} = options; // TODO: initialize system primary status to false once clustering works let system_primary_writer = AtomicBoolWriter::with_value(true); @@ -51,6 +59,7 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul system_primary_writer, user_streams, storage_dir, + cluster_options, default_stream_options); let (system_partition_tx, system_partition_rx) = create_partition_channels(); diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 0029eb6..faf3209 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -1,3 +1,5 @@ +mod cluster_state; +mod outgoing_io; mod system_stream; mod initialization; @@ -15,7 +17,7 @@ use engine::event_stream::partition::controller::PartitionImpl; use atomics::AtomicBoolWriter; -pub use self::initialization::{start_controller, ControllerOptions}; +pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; pub use self::system_stream::SystemStreamRef; @@ -48,6 +50,7 @@ impl FloController { system_primary_setter: AtomicBoolWriter, event_streams: HashMap, storage_dir: PathBuf, + cluster_options: Option, default_stream_options: EventStreamOptions) -> FloController { let stream_refs = event_streams.iter().map(|(k, v)| { diff --git a/flo-server/src/engine/controller/outgoing_io.rs b/flo-server/src/engine/controller/outgoing_io.rs new file mode 100644 index 0000000..98d1b57 --- /dev/null +++ b/flo-server/src/engine/controller/outgoing_io.rs @@ -0,0 +1,107 @@ +use std::net::SocketAddr; +use std::io; + +use tokio_core::reactor::Handle; +use tokio_core::net::TcpStream; + +#[allow(deprecated)] +use tokio_core::io::Io; +use futures::{Future, Stream, Sink}; + +use event_loops::LoopHandles; +use engine::{EngineRef, create_client_channels, ReceivedProtocolMessage}; + +pub struct CreateOutgoingConnection(SocketAddr); + +impl CreateOutgoingConnection { + pub fn new(addr: SocketAddr) -> CreateOutgoingConnection { + CreateOutgoingConnection(addr) + } + + fn addr(&self) -> &SocketAddr { + &self.0 + } +} + +pub struct OutgoingConnectionCreator(::futures::sync::mpsc::UnboundedSender); + + +pub fn start_outgoing_io(mut event_loops: LoopHandles, engine_ref: EngineRef) -> OutgoingConnectionCreator { + let (tx, rx) = ::futures::sync::mpsc::unbounded::(); + + let remote = event_loops.next_handle(); + let system_stream_ref = engine_ref.get_system_stream(); + + let future = rx.for_each( move |create_outgoing| { + + let engine = engine_ref.clone(); + event_loops.next_handle().spawn(|handle| { + handle_outgoing_connection(handle.clone(), create_outgoing, engine) + }); + Ok(()) + }); + + remote.spawn(|handle| { + future + }); + + OutgoingConnectionCreator(tx) +} + + +fn handle_outgoing_connection(handle: Handle, create_outgoing: CreateOutgoingConnection, engine_ref: EngineRef) -> Box> { + use flo_io::{ProtocolMessageStream, ServerMessageStream}; + use engine::ConnectionHandler; + + // These copies are to work around the ownership rules, since we want to have slightly different error handling for + // connection failures depending on when it fails + let client_addr = *create_outgoing.addr(); + let client_addr_copy = client_addr.clone(); + + let mut system_stream = engine_ref.get_system_stream(); + let mut system_stream_copy = system_stream.clone(); + + let future = TcpStream::connect(create_outgoing.addr(), &handle).map_err( move |io_err| { + + error!("Failed to create outgoing connection to address: {:?}: {:?}", client_addr_copy, io_err); + system_stream.outgoing_connection_failed(0, client_addr_copy); + + }).and_then( move |tcp_stream| { + let connection_id = engine_ref.next_connection_id(); + debug!("Established connection to {:?} with connection_id: {}", client_addr, connection_id); + + if let Err(io_err) = tcp_stream.set_nodelay(true) { + error!("Error setting NODELAY for connection_id: {}. Nagle yet lives!: {:?}", connection_id, io_err); + } + + let (client_tx, client_rx) = create_client_channels(); + + #[allow(deprecated)] + let (read, write) = tcp_stream.split(); + + let server_to_client = ServerMessageStream::new(connection_id, client_rx, write); + let client_message_stream = ProtocolMessageStream::new(connection_id, read); + + let connection_handler = ConnectionHandler::new( + connection_id, + client_tx.clone(), + engine_ref, + handle.clone()); + + let client_to_server = connection_handler + .send_all(client_message_stream) + .map(|_| ()); + + client_to_server.select(server_to_client).then(move |res| { + if let Err((err, _)) = res { + warn!("Closing outgoing connection: {} due to err: {:?}", connection_id, err); + system_stream_copy.outgoing_connection_failed(connection_id, client_addr); + } + info!("Closed connection_id: {} to address: {}", connection_id, client_addr); + Ok(()) + }) + + }); + + Box::new(future) +} diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index 5cb1a06..f49c684 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -1,6 +1,8 @@ +use std::net::SocketAddr; -use engine::event_stream::partition::{PartitionRef}; +use engine::event_stream::partition::{PartitionRef, Operation}; use engine::event_stream::EventStreamRef; +use engine::ConnectionId; @@ -22,6 +24,12 @@ impl SystemStreamRef { let partition_ref = vec![self.inner.clone()]; EventStreamRef::new(name, partition_ref) } + + pub fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, socket_addr: SocketAddr) { + let op = Operation::outgoing_connection_failed(connection_id, socket_addr); + self.inner.send(op).expect("System Stream has shutdown"); + } + } diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index c9028d1..e703acb 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -152,6 +152,10 @@ impl PartitionImpl { self.expire_old_events(); Ok(()) } + other @ _ => { + warn!("received unexpected OpType: {:?}", other); + Ok(()) + } } } diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index de8cca3..e13c63e 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -227,7 +227,7 @@ impl PartitionRef { self.send(Operation::tick()) } - fn send(&mut self, op: Operation) -> PartitionSendResult { + pub fn send(&mut self, op: Operation) -> PartitionSendResult { self.sender.send(op).map_err(|err| { PartitionSendError(err.0) }) diff --git a/flo-server/src/engine/event_stream/partition/ops.rs b/flo-server/src/engine/event_stream/partition/ops.rs index 69b84e7..808f019 100644 --- a/flo-server/src/engine/event_stream/partition/ops.rs +++ b/flo-server/src/engine/event_stream/partition/ops.rs @@ -2,11 +2,13 @@ use std::io; use std::fmt::{self, Debug}; use std::time::Instant; +use std::net::SocketAddr; use futures::sync::oneshot; use engine::event_stream::partition::{EventFilter, PartitionReader}; use engine::ConnectionId; +use engine::system_stream::SystemOp; use protocol::ProduceEvent; use event::{FloEventId, EventCounter}; @@ -59,6 +61,7 @@ pub enum OpType { Consume(ConsumeOperation), StopConsumer, Tick, + System(SystemOp) } @@ -70,6 +73,19 @@ pub struct Operation { } impl Operation { + + fn new(connection_id: ConnectionId, op_type: OpType) -> Operation { + Operation { + connection_id, + client_message_recv_time: Instant::now(), + op_type + } + } + + fn system(connection_id: ConnectionId, system_op: SystemOp) -> Operation { + Operation::new(connection_id, OpType::System(system_op)) + } + pub fn consume(connection_id: ConnectionId, notifier: Box, filter: EventFilter, start_exclusive: EventCounter) -> (Operation, ConsumeResponseReceiver) { let (tx, rx) = oneshot::channel(); let consume = ConsumeOperation { @@ -78,20 +94,12 @@ impl Operation { start_exclusive: start_exclusive, notifier: notifier, }; - let op = Operation { - connection_id: connection_id, - client_message_recv_time: Instant::now(), - op_type: OpType::Consume(consume) - }; + let op = Operation::new(connection_id, OpType::Consume(consume)); (op, rx) } pub fn stop_consumer(connection_id: ConnectionId) -> Operation { - Operation { - connection_id: connection_id, - client_message_recv_time: Instant::now(), - op_type: OpType::StopConsumer - } + Operation::new(connection_id, OpType::StopConsumer) } pub fn produce(connection_id: ConnectionId, op_id: u32, events: Vec) -> (Operation, ProduceResponseReceiver) { @@ -101,20 +109,16 @@ impl Operation { op_id: op_id, events: events }; - let op = Operation { - connection_id: connection_id, - client_message_recv_time: Instant::now(), - op_type: OpType::Produce(produce), - }; + let op = Operation::new(connection_id, OpType::Produce(produce)); (op, rx) } pub fn tick() -> Operation { - Operation { - connection_id: 0, - client_message_recv_time: Instant::now(), - op_type: OpType::Tick, - } + Operation::new(0, OpType::Tick) + } + + pub fn outgoing_connection_failed(connection_id: ConnectionId, socket_addr: SocketAddr) -> Operation { + Operation::system(connection_id, SystemOp::OutgoingConnectionFailed(socket_addr)) } } diff --git a/flo-server/src/engine/mod.rs b/flo-server/src/engine/mod.rs index 7b3eaa0..fbc93ef 100644 --- a/flo-server/src/engine/mod.rs +++ b/flo-server/src/engine/mod.rs @@ -1,4 +1,5 @@ pub mod event_stream; +pub mod system_stream; mod controller; mod connection_handler; @@ -11,7 +12,7 @@ use protocol::ProtocolMessage; use event::OwnedFloEvent; use self::event_stream::EventStreamRef; -pub use self::controller::{ControllerOptions, SystemStreamRef, start_controller}; +pub use self::controller::{ControllerOptions, ClusterOptions, SystemStreamRef, start_controller}; pub use self::connection_handler::{ConnectionHandler, ConnectionHandlerResult}; pub type ConnectionId = usize; @@ -59,6 +60,10 @@ impl EngineRef { } } + pub fn system_stream(&mut self) -> &mut SystemStreamRef { + &mut self.system_stream + } + pub fn next_connection_id(&self) -> ConnectionId { let old = self.current_connection_id.fetch_add(1, ::std::sync::atomic::Ordering::SeqCst); old + 1 @@ -82,6 +87,10 @@ impl EngineRef { self.system_stream.to_event_stream() }) } + + pub fn get_system_stream(&self) -> SystemStreamRef { + self.system_stream.clone() + } } diff --git a/flo-server/src/engine/system_stream/flo_instance_id.rs b/flo-server/src/engine/system_stream/flo_instance_id.rs new file mode 100644 index 0000000..d2b7fc4 --- /dev/null +++ b/flo-server/src/engine/system_stream/flo_instance_id.rs @@ -0,0 +1,35 @@ + +use std::fmt::{self, Display}; + + +/// An opaque identifier used to uniquely identify flo instances. +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub struct FloInstanceId(u64); + +impl FloInstanceId { + + pub fn as_u64(&self) -> u64 { + self.0 + } + + pub fn from_u64(int_value: u64) -> FloInstanceId { + FloInstanceId(int_value) + } + + pub fn generate() -> FloInstanceId { + let num = { + let time = ::event::time::now(); + let low = time.timestamp_subsec_nanos(); + let high = time.num_seconds_from_unix_epoch(); + (high as u64) << 32 & low as u64 + }; + FloInstanceId(num) + } +} + +impl Display for FloInstanceId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Display will just look the same as Debug for now + write!(f, "{:?}", self) + } +} diff --git a/flo-server/src/engine/system_stream/mod.rs b/flo-server/src/engine/system_stream/mod.rs new file mode 100644 index 0000000..811cf32 --- /dev/null +++ b/flo-server/src/engine/system_stream/mod.rs @@ -0,0 +1,31 @@ +mod flo_instance_id; + +use std::net::SocketAddr; + +use event::EventCounter; +use engine::ConnectionId; +pub use self::flo_instance_id::FloInstanceId; + + +#[derive(Debug, PartialEq)] +pub struct FloServer { + pub id: FloInstanceId, + pub address: SocketAddr, +} + +#[derive(Debug, PartialEq)] +pub struct ServerStatus { + peer_id: FloInstanceId, + leader_id: FloInstanceId, + system_event_counter: EventCounter, + current_term: u64, + all_members: Vec, +} + +#[derive(Debug)] +pub enum SystemOp { + OutgoingConnectionFailed(SocketAddr), + PeerConnectionEstablished(ConnectionId, SocketAddr), +} + + diff --git a/flo-server/src/server/flo_io/client_message_stream.rs b/flo-server/src/flo_io/client_message_stream.rs similarity index 100% rename from flo-server/src/server/flo_io/client_message_stream.rs rename to flo-server/src/flo_io/client_message_stream.rs diff --git a/flo-server/src/server/flo_io/mod.rs b/flo-server/src/flo_io/mod.rs similarity index 100% rename from flo-server/src/server/flo_io/mod.rs rename to flo-server/src/flo_io/mod.rs diff --git a/flo-server/src/server/flo_io/server_message_stream.rs b/flo-server/src/flo_io/server_message_stream.rs similarity index 100% rename from flo-server/src/server/flo_io/server_message_stream.rs rename to flo-server/src/flo_io/server_message_stream.rs diff --git a/flo-server/src/lib.rs b/flo-server/src/lib.rs index ddd66ba..c6f19b7 100644 --- a/flo-server/src/lib.rs +++ b/flo-server/src/lib.rs @@ -16,7 +16,8 @@ extern crate clap; extern crate log4rs; extern crate num_cpus; extern crate byteorder; - +extern crate serde; +extern crate serde_json; #[cfg(test)] extern crate env_logger; @@ -24,6 +25,7 @@ extern crate env_logger; extern crate tempdir; +pub mod flo_io; pub mod logging; pub mod server; pub mod embedded; diff --git a/flo-server/src/main.rs b/flo-server/src/main.rs index c856bb1..b8dbbcf 100644 --- a/flo-server/src/main.rs +++ b/flo-server/src/main.rs @@ -77,17 +77,18 @@ fn app_args() -> App<'static, 'static> { .default_value("512") .help("Maximum amount of memory in megabytes to use for the event cache")) .arg(Arg::with_name("join-cluster-address") - .requires("actor-id") + .requires("server-addr") .long("peer-addr") .short("P") .multiple(true) .value_name("HOST:PORT") .help("address of another Flo instance to join a cluster; this argument may be supplied multiple times")) - .arg(Arg::with_name("actor-id") - .long("actor-id") + .arg(Arg::with_name("server-addr") + .requires("join-cluster-address") + .long("server-addr") .short("A") .takes_value(true) - .help("The id to assign to this node in the cluster. This MUST be unique within the cluster. Will be removed once cluster support doesn't suck so bad")) + .help("The socket address that this server is reachable at. Will be removed once cluster support doesn't suck so bad")) .arg(Arg::with_name("max-io-threads") .long("max-io-threads") .takes_value(true) @@ -105,7 +106,11 @@ fn main() { let data_dir = PathBuf::from(args.value_of("data-dir").unwrap_or(".")); let max_cache_memory = get_max_cache_mem_amount(&args); let cluster_addresses = get_cluster_addresses(&args); - let actor_id = args.value_of("actor-id").unwrap_or("1").parse::().expect("ActorId must be an unsigned 16 bit integer"); + let this_address = args.value_of("server-addr").map(|addr_string| { + SocketAddr::from_str(addr_string).map_err(|err| { + format!("Cannot parse server-addr argument: {}", err) + }).or_bail() + }); let max_io_threads = args.value_of("max-io-threads").map(|value| { value.parse::().map_err(|_| { format!("Invalid max-io-threads argument: '{}' value must be a positive integer", value) @@ -134,8 +139,8 @@ fn main() { port: port, data_dir: data_dir, max_cache_memory: max_cache_memory, + this_instance_address: this_address, cluster_addresses: cluster_addresses, - actor_id: actor_id, max_io_threads: max_io_threads, }; diff --git a/flo-server/src/server/mod.rs b/flo-server/src/server/mod.rs index bd76155..e6d5ab3 100644 --- a/flo-server/src/server/mod.rs +++ b/flo-server/src/server/mod.rs @@ -1,4 +1,3 @@ -mod flo_io; mod server_options; use futures::{Stream, Sink, Future}; @@ -13,21 +12,33 @@ pub use self::server_options::{ServerOptions, MemoryLimit, MemoryUnit}; -pub fn run(options: ServerOptions) -> io::Result<()> { +pub fn run(mut options: ServerOptions) -> io::Result<()> { #[allow(deprecated)] use tokio_core::io::Io; use engine::{ControllerOptions, + ClusterOptions, start_controller, system_stream_name, create_client_channels, ConnectionHandler}; use engine::event_stream::EventStreamOptions; - use self::flo_io::{ProtocolMessageStream, ServerMessageStream}; + use flo_io::{ProtocolMessageStream, ServerMessageStream}; const ONE_GB: usize = 1024 * 1024 * 1024; let (join_handle, mut event_loop_handles) = event_loops::spawn_event_loop_threads(options.max_io_threads).unwrap(); + let this_address = options.this_instance_address.take(); + let peer_addresses = options.cluster_addresses.take(); + + let cluster_options = this_address.map(|server_addr| { + let peers = peer_addresses.unwrap(); + ClusterOptions { + this_instance_address: server_addr, + peer_addresses: peers, + } + }); + let controller_options = ControllerOptions { storage_dir: options.data_dir.clone(), default_stream_options: EventStreamOptions{ @@ -37,6 +48,7 @@ pub fn run(options: ServerOptions) -> io::Result<()> { max_segment_duration: options.event_eviction_period, segment_max_size_bytes: ONE_GB, }, + cluster_options, }; let engine_ref = start_controller(controller_options, event_loop_handles.next_handle())?; diff --git a/flo-server/src/server/server_options.rs b/flo-server/src/server/server_options.rs index 13695dd..d08b5f3 100644 --- a/flo-server/src/server/server_options.rs +++ b/flo-server/src/server/server_options.rs @@ -45,8 +45,8 @@ pub struct ServerOptions { pub event_retention_duration: Duration, pub event_eviction_period: Duration, pub max_cache_memory: MemoryLimit, + pub this_instance_address: Option, pub cluster_addresses: Option>, - pub actor_id: ActorId, pub max_io_threads: Option, } diff --git a/flo-server/tests/embedded_tests.rs b/flo-server/tests/embedded_tests.rs index 027c6a6..011a07e 100644 --- a/flo-server/tests/embedded_tests.rs +++ b/flo-server/tests/embedded_tests.rs @@ -12,6 +12,7 @@ extern crate log; use std::fmt::Debug; use std::thread; use std::time::Duration; +use std::net::SocketAddr; use tokio_core::reactor::Core; use futures::{Stream, Future}; @@ -40,6 +41,7 @@ fn integration_test(test_name: &'static str, stream_opts: EventStreamOptions, let controller_options = ControllerOptions { storage_dir: tmp_dir.path().to_owned(), default_stream_options: stream_opts, + cluster_options: None, }; let reactor = Core::new().expect("failed to create reactor"); let embedded_server = run_embedded_server(controller_options, reactor.remote()).expect("failed to run embedded server"); From 0c906d785671473512109f003c52976644b0232c Mon Sep 17 00:00:00 2001 From: Phil Fried Date: Sun, 26 Nov 2017 16:32:50 -0500 Subject: [PATCH 08/73] more WIP on cluster communication --- .../engine/controller/cluster_state/mod.rs | 18 +++- .../src/engine/controller/outgoing_io.rs | 19 +++- .../engine/system_stream/flo_instance_id.rs | 25 ++---- flo-server/src/engine/system_stream/mod.rs | 89 ++++++++++++++++++- .../engine/system_stream/replication_plan.rs | 19 ++++ 5 files changed, 142 insertions(+), 28 deletions(-) create mode 100644 flo-server/src/engine/system_stream/replication_plan.rs diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index cf1ba5b..18d6d99 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -2,13 +2,29 @@ use std::net::SocketAddr; use std::time::{Instant, Duration}; +use event::EventCounter; use engine::controller::outgoing_io::OutgoingConnectionCreator; +use engine::system_stream::FloInstanceId; -pub struct ClusterState { +#[derive(Debug, PartialEq, Clone)] +struct PersistentClusterState { + current_term: u64, + voted_for: Option, +} + +pub struct ClusterState { + persistent: PersistentClusterState, + last_applied: EventCounter, } struct Peer { + id: FloInstanceId, + leader: bool, peer_addr: SocketAddr, + last_ackgnowledged_entry: EventCounter, +} +enum PeerState { + Connecting(Instant), } diff --git a/flo-server/src/engine/controller/outgoing_io.rs b/flo-server/src/engine/controller/outgoing_io.rs index 98d1b57..b0421f0 100644 --- a/flo-server/src/engine/controller/outgoing_io.rs +++ b/flo-server/src/engine/controller/outgoing_io.rs @@ -11,6 +11,7 @@ use futures::{Future, Stream, Sink}; use event_loops::LoopHandles; use engine::{EngineRef, create_client_channels, ReceivedProtocolMessage}; +#[derive(Debug, Copy, Clone, PartialEq)] pub struct CreateOutgoingConnection(SocketAddr); impl CreateOutgoingConnection { @@ -23,10 +24,20 @@ impl CreateOutgoingConnection { } } -pub struct OutgoingConnectionCreator(::futures::sync::mpsc::UnboundedSender); +pub trait OutgoingConnectionCreator { + fn establish_connection(&mut self, create: CreateOutgoingConnection); +} + +pub struct OutgoingConnectionCreatorImpl(::futures::sync::mpsc::UnboundedSender); + +impl OutgoingConnectionCreator for OutgoingConnectionCreatorImpl { + fn establish_connection(&mut self, create: CreateOutgoingConnection) { + self.0.unbounded_send(create).expect("Failed to send message to OutgoingConnectionCreator channel"); + } +} -pub fn start_outgoing_io(mut event_loops: LoopHandles, engine_ref: EngineRef) -> OutgoingConnectionCreator { +pub fn start_outgoing_io(mut event_loops: LoopHandles, engine_ref: EngineRef) -> OutgoingConnectionCreatorImpl { let (tx, rx) = ::futures::sync::mpsc::unbounded::(); let remote = event_loops.next_handle(); @@ -35,7 +46,7 @@ pub fn start_outgoing_io(mut event_loops: LoopHandles, engine_ref: EngineRef) -> let future = rx.for_each( move |create_outgoing| { let engine = engine_ref.clone(); - event_loops.next_handle().spawn(|handle| { + event_loops.next_handle().spawn(move |handle| { handle_outgoing_connection(handle.clone(), create_outgoing, engine) }); Ok(()) @@ -45,7 +56,7 @@ pub fn start_outgoing_io(mut event_loops: LoopHandles, engine_ref: EngineRef) -> future }); - OutgoingConnectionCreator(tx) + OutgoingConnectionCreatorImpl(tx) } diff --git a/flo-server/src/engine/system_stream/flo_instance_id.rs b/flo-server/src/engine/system_stream/flo_instance_id.rs index d2b7fc4..1cbbe17 100644 --- a/flo-server/src/engine/system_stream/flo_instance_id.rs +++ b/flo-server/src/engine/system_stream/flo_instance_id.rs @@ -1,35 +1,22 @@ +use std::net::SocketAddr; use std::fmt::{self, Display}; /// An opaque identifier used to uniquely identify flo instances. -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub struct FloInstanceId(u64); +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub struct FloInstanceId(SocketAddr); impl FloInstanceId { - pub fn as_u64(&self) -> u64 { - self.0 - } - - pub fn from_u64(int_value: u64) -> FloInstanceId { - FloInstanceId(int_value) - } - - pub fn generate() -> FloInstanceId { - let num = { - let time = ::event::time::now(); - let low = time.timestamp_subsec_nanos(); - let high = time.num_seconds_from_unix_epoch(); - (high as u64) << 32 & low as u64 - }; - FloInstanceId(num) + pub fn new(addr: SocketAddr) -> Self { + FloInstanceId(addr) } } impl Display for FloInstanceId { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // Display will just look the same as Debug for now - write!(f, "{:?}", self) + write!(f, "{}", self.0) } } diff --git a/flo-server/src/engine/system_stream/mod.rs b/flo-server/src/engine/system_stream/mod.rs index 811cf32..ee9557d 100644 --- a/flo-server/src/engine/system_stream/mod.rs +++ b/flo-server/src/engine/system_stream/mod.rs @@ -1,31 +1,112 @@ mod flo_instance_id; +mod replication_plan; use std::net::SocketAddr; use event::EventCounter; use engine::ConnectionId; +use engine::event_stream::EventStreamOptions; + pub use self::flo_instance_id::FloInstanceId; +pub use self::replication_plan::{SystemReplicationPlan, StreamReplicationPlan, PartitionReplicationPlan}; + +pub type Term = u64; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct FloServer { pub id: FloInstanceId, pub address: SocketAddr, } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct ServerStatus { peer_id: FloInstanceId, leader_id: FloInstanceId, system_event_counter: EventCounter, - current_term: u64, + current_term: Term, all_members: Vec, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum SystemOp { OutgoingConnectionFailed(SocketAddr), PeerConnectionEstablished(ConnectionId, SocketAddr), + AppendResponse(AppendEntriesResponse), + VoteResponse(RequestVoteResponse), +} + + + +#[derive(Debug, PartialEq, Clone)] +pub struct AppendEntriesCall { + pub leader_id: FloInstanceId, + pub term: Term, + pub prev_entry_term: Term, + pub prev_entry_index: EventCounter, + pub leader_commit_index: EventCounter, + pub entries: Vec, } +impl AppendEntriesCall { + pub fn heartbeat(leader_id: FloInstanceId, + term: Term, + prev_entry_term: Term, + prev_entry_index: EventCounter, + leader_commit_index: EventCounter) -> AppendEntriesCall { + + AppendEntriesCall { + leader_id, + term, + prev_entry_term, + prev_entry_index, + leader_commit_index, + entries: Vec::new(), + } + } + + pub fn is_heartbeat(&self) -> bool { + self.entries.is_empty() + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct AppendEntriesResponse { + pub from_peer: FloInstanceId, + pub term: Term, + pub success: bool, +} +#[derive(Debug, PartialEq, Clone)] +pub struct RequestVoteCall { + pub term: Term, + pub candidate_id: FloInstanceId, + pub last_log_index: EventCounter, + pub last_log_term: Term, +} + + +#[derive(Debug, PartialEq, Clone)] +pub struct RequestVoteResponse { + pub from_peer: FloInstanceId, + pub term: Term, + pub vote_granted: bool, +} + + +#[derive(Debug, Clone, PartialEq)] +pub enum SystemLogEntry { + /// Communicates which server should be the primary for each partition in each event stream + SetReplicationPlan(SystemReplicationPlan), + + /// Command to create a new event stream. Each follower should create the stream, but wait for + /// a replication plan before allowing any writes to it + CreateEventStream(EventStreamOptions), + + /// First phase of adding a new member to the cluster. New members are NOT eligible to become + /// primary until an `ActivateClusterMember` command with its id has been committed to the log. + AddClusterMember(FloServer), + + /// Once this entry has been committed, the given server is now eligible to be elected primary + ActivateClusterMember(FloInstanceId), +} diff --git a/flo-server/src/engine/system_stream/replication_plan.rs b/flo-server/src/engine/system_stream/replication_plan.rs new file mode 100644 index 0000000..8a5fc07 --- /dev/null +++ b/flo-server/src/engine/system_stream/replication_plan.rs @@ -0,0 +1,19 @@ + +use super::FloInstanceId; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct SystemReplicationPlan { + pub streams: Vec, +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct StreamReplicationPlan { + pub name: String, + pub partitions: Vec, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub struct PartitionReplicationPlan { + pub primary: FloInstanceId, + // TODO: Add ability to limit replication to only a subset of secondary nodes to reduce network traffic +} From 13dd6aed3e560e5d0a2526324e8ac29d47cf64b0 Mon Sep 17 00:00:00 2001 From: pfried Date: Mon, 27 Nov 2017 23:27:31 -0500 Subject: [PATCH 09/73] create separate submodules for each message in flo-protocol --- flo-protocol/Cargo.toml | 2 +- flo-protocol/src/client.rs | 979 ------------------ flo-protocol/src/lib.rs | 6 +- flo-protocol/src/messages/client_announce.rs | 44 + flo-protocol/src/messages/consume_start.rs | 54 + flo-protocol/src/messages/cursor_info.rs | 39 + flo-protocol/src/messages/error.rs | 94 ++ flo-protocol/src/messages/event_ack.rs | 43 + .../src/messages/event_stream_status.rs | 81 ++ flo-protocol/src/messages/mod.rs | 557 ++++++++++ flo-protocol/src/messages/peer_announce.rs | 40 + flo-protocol/src/messages/produce_event.rs | 67 ++ flo-protocol/src/messages/receive_event.rs | 40 + flo-protocol/src/messages/set_event_stream.rs | 36 + flo-protocol/src/serializer.rs | 14 + 15 files changed, 1113 insertions(+), 983 deletions(-) delete mode 100644 flo-protocol/src/client.rs create mode 100644 flo-protocol/src/messages/client_announce.rs create mode 100644 flo-protocol/src/messages/consume_start.rs create mode 100644 flo-protocol/src/messages/cursor_info.rs create mode 100644 flo-protocol/src/messages/error.rs create mode 100644 flo-protocol/src/messages/event_ack.rs create mode 100644 flo-protocol/src/messages/event_stream_status.rs create mode 100644 flo-protocol/src/messages/mod.rs create mode 100644 flo-protocol/src/messages/peer_announce.rs create mode 100644 flo-protocol/src/messages/produce_event.rs create mode 100644 flo-protocol/src/messages/receive_event.rs create mode 100644 flo-protocol/src/messages/set_event_stream.rs diff --git a/flo-protocol/Cargo.toml b/flo-protocol/Cargo.toml index 3722e46..9e34579 100644 --- a/flo-protocol/Cargo.toml +++ b/flo-protocol/Cargo.toml @@ -6,6 +6,6 @@ authors = ["pfried "] [dependencies] flo-event = { path = "../flo-event" } log = "0.3" -nom = "2.0" +nom = "2.2.1" byteorder = "1" diff --git a/flo-protocol/src/client.rs b/flo-protocol/src/client.rs deleted file mode 100644 index fc9c144..0000000 --- a/flo-protocol/src/client.rs +++ /dev/null @@ -1,979 +0,0 @@ -//! This is the wire protocol used to communicate between the server and client. Communication is done by sending and -//! receiving series' of distinct messages. Each message begins with an 8 byte header that identifies the type of message. -//! This is rather wasteful, but useful for the early stages when there's still a fair bit of debugging via manual inspection -//! of buffers. Messages are parsed using nom parser combinators, and serialized using simple a wrapper around a writer. -//! -//! The special cases in the protocol are for sending/receiving the events themselves. Since events can be quite large, they -//! are not actually implemented as a single message in the protocol, but rather as just a header. The header has all the basic -//! information as well as the length of the data portion (the body of the event). The event is read by first reading the -//! header and then reading however many bytes are indicated by the header for the body of the event. -//! -//! All numbers use big endian byte order. -//! All Strings are newline terminated. -use nom::{be_u64, be_u32, be_u16}; -use event::{time, OwnedFloEvent, FloEvent, FloEventId, ActorId, EventCounter, Timestamp}; -use serializer::Serializer; -use std::net::SocketAddr; - -pub mod headers { - pub const CLIENT_AUTH: u8 = 1; - pub const PRODUCE_EVENT: u8 = 2; - pub const RECEIVE_EVENT: u8 = 3; - pub const UPDATE_MARKER: u8 = 4; - pub const START_CONSUMING: u8 = 5; - pub const AWAITING_EVENTS: u8 = 6; - pub const PEER_ANNOUNCE: u8 = 7; - pub const PEER_UPDATE: u8 = 8; - pub const ACK_HEADER: u8 = 9; - pub const ERROR_HEADER: u8 = 10; - pub const CLUSTER_STATE: u8 = 11; - pub const SET_BATCH_SIZE: u8 = 12; - pub const NEXT_BATCH: u8 = 13; - pub const END_OF_BATCH: u8 = 14; - pub const STOP_CONSUMING: u8 = 15; - pub const CURSOR_CREATED: u8 = 16; - pub const NEW_START_CONSUMING: u8 = 17; - pub const SET_EVENT_STREAM: u8 = 18; - pub const EVENT_STREAM_STATUS: u8 = 19; - pub const CLIENT_ANNOUNCE: u8 = 170; -} - -use self::headers::*; - -pub const ERROR_INVALID_NAMESPACE: u8 = 15; -pub const ERROR_INVALID_CONSUMER_STATE: u8 = 16; -pub const ERROR_INVALID_VERSION_VECTOR: u8 = 17; -pub const ERROR_STORAGE_ENGINE_IO: u8 = 18; -pub const ERROR_NO_STREAM: u8 = 19; - -/// Describes the type of error. This gets serialized a u8 -#[derive(Debug, PartialEq, Clone)] -pub enum ErrorKind { - /// Indicates that the namespace provided by a consumer was an invalid glob pattern - InvalidNamespaceGlob, - /// Indicates that the client connection was in an invalid state when it attempted some consumer operation - InvalidConsumerState, - /// Indicates that the provided version vector was invalid (contained more than one entry for at least one actor id) - InvalidVersionVector, - /// Unable to read or write to events file - StorageEngineError, - /// Requested event stream does not exist - NoSuchStream, -} - -/// Represents a response to any request that results in an error -#[derive(Debug, PartialEq, Clone)] -pub struct ErrorMessage { - /// The op_id of the request to make it easier to correlate request/response pairs - pub op_id: u32, - - /// The type of error - pub kind: ErrorKind, - - /// A human-readable description of the error - pub description: String, -} - -impl ErrorKind { - /// Converts from the serialized u8 to an ErrorKind - pub fn from_u8(byte: u8) -> Result { - match byte { - ERROR_INVALID_NAMESPACE => Ok(ErrorKind::InvalidNamespaceGlob), - ERROR_INVALID_CONSUMER_STATE => Ok(ErrorKind::InvalidConsumerState), - ERROR_INVALID_VERSION_VECTOR => Ok(ErrorKind::InvalidVersionVector), - ERROR_STORAGE_ENGINE_IO => Ok(ErrorKind::StorageEngineError), - ERROR_NO_STREAM => Ok(ErrorKind::NoSuchStream), - other => Err(other) - } - } - - /// Converts the ErrorKind to it's serialized u8 value - pub fn u8_value(&self) -> u8 { - match self { - &ErrorKind::InvalidNamespaceGlob => ERROR_INVALID_NAMESPACE, - &ErrorKind::InvalidConsumerState => ERROR_INVALID_CONSUMER_STATE, - &ErrorKind::InvalidVersionVector => ERROR_INVALID_VERSION_VECTOR, - &ErrorKind::StorageEngineError => ERROR_STORAGE_ENGINE_IO, - &ErrorKind::NoSuchStream => ERROR_NO_STREAM, - } - } -} - -/// The body of a ProduceEvent `ProtocolMessage`. This is sent from a client producer to the server, and the server will -/// respond with either an `EventAck` or an `ErrorMessage` to indicate success or failure respectively. Although the flo -/// protocol is pipelined, this message includes an `op_id` field to aid in correlation of requests and responses. -#[derive(Debug, PartialEq, Clone)] -pub struct ProduceEvent { - /// This is an arbritrary number, assigned by the client, to aid in correlation of requests and responses. Clients may - /// choose to just set it to the same value for every operation if they wish. - pub op_id: u32, - /// The partition to produce the event onto - pub partition: ActorId, - /// The namespace to produce the event to. See the `namespace` documentation on `FloEvent` for more information on - /// namespaces in general. As far as the protocol is concerned, it's just serialized as a utf-8 string. - pub namespace: String, - /// The parent_id of the new event. This is typically set to the id of whatever event a consumer is responding to. - /// The parent id is optional. On the wire, a null parent_id is serialized as an event id where both the counter and the - /// actor are set to 0. - pub parent_id: Option, - /// The event payload. As far as the flo server is concerned, this is just an opaque byte array. Note that events with - /// 0-length bodies are perfectly fine. - pub data: Vec, -} - -/// Sent by the server to the producer of an event to acknowledge that the event was successfully persisted to the stream. -#[derive(Debug, PartialEq, Clone)] -pub struct EventAck { - /// This will be set to the `op_id` that was sent in the `ProduceEventHeader` - pub op_id: u32, - - /// The id that was assigned to the event. This id is immutable and must be the same across all servers in a flo cluster. - pub event_id: FloEventId, -} - -/// Sent by a client to the server to begin reading events from the stream. -#[derive(Debug, PartialEq, Clone)] -pub struct ConsumerStart { - /// Operation id that is generated by the client and used to correlate the response with the request - pub op_id: u32, - - /// The maximum number of events to consume. Set to `u64::MAX` if you want unlimited. - pub max_events: u64, - - /// The namespace to consume from. This can be any valid glob pattern, to allow reading from multiple namespaces. - pub namespace: String, -} - -pub const CONSUME_UNLIMITED: u64 = 0; - -/// New message sent from client to server to begin reading events from the stream -#[derive(Debug, PartialEq, Clone)] -pub struct NewConsumerStart { - pub op_id: u32, - pub version_vector: Vec, - pub max_events: u64, - pub namespace: String, -} - - -/// Represents information known about a member of the flo cluster from the perspective of whichever member sent the -/// ClusterState message. -#[derive(Debug, PartialEq, Clone)] -pub struct ClusterMember { - /// the address of the cluster member. The peer should be reachable at this address without having to modify or fix it up - pub addr: SocketAddr, - - /// The actor id of the peer - pub actor_id: ActorId, - - /// Whether the peer is currently connected to the sender of the ClusterState message - pub connected: bool, -} - -/// Represents the known state of the cluster from the point of view of _one_ of it's members. -/// Keep in mind that each member of a given cluster may have a different record of what the state of the cluster is. -/// This message represents the point of view of the actor referred to by the `actor_id` field. -#[derive(Debug, PartialEq, Clone)] -pub struct ClusterState { - /// The id of whichever actor has sent this message - pub actor_id: ActorId, - - /// The port number that this actor is listening on. This is not a complete address because of the fact that it's not - /// always possible for a server to know the correct address for connecting to itself. - pub actor_port: u16, - - /// The current version vector of this actor - pub version_vector: Vec, - - /// Information on all the other known members of the cluster. This list will not include duplicated information about - /// the actor who sent the message - pub other_members: Vec, -} - -/// Sent in a CursorCreated message from the server to a client to indicate that a cursor was successfully created. -/// Currently, this message only contains the batch size, but more fields may be added as they become necessary. -#[derive(Debug, PartialEq, Clone)] -pub struct CursorInfo { - /// The operation id from the StartConsuming message that created this cursor. - pub op_id: u32, - - /// The actual batch size that will be used by the server for sending events. Note that this value _may_ differ from the - /// batch size that was explicitly set by the consumer, depending on server settings. This behavior is not currently - /// implemented by the server, but it's definitely possible to change in the near future. - pub batch_size: u32, -} - - -/// Information on the status of a partition. Included as part of `EventStreamStatus` -#[derive(Debug, PartialEq, Clone)] -pub struct PartitionStatus { - pub partition_num: ActorId, - pub head: EventCounter, - pub primary: bool, -} - -/// Contains some basic information on an event stream. Sent in response to a `SetEventStream` -#[derive(Debug, PartialEq, Clone)] -pub struct EventStreamStatus { - pub op_id: u32, - pub name: String, - pub partitions: Vec, -} - -/// Sent by a client to tell the server which event stream to use for all future operations -#[derive(Debug, PartialEq, Clone)] -pub struct SetEventStream{ - pub op_id: u32, - pub name: String, -} - -/// Sent by the client as the very first message to the server. The server will respond with an `EventStreamStatus` for the current (default) stream -#[derive(Debug, PartialEq, Clone)] -pub struct ClientAnnounce { - pub protocol_version: u32, - pub op_id: u32, - pub client_name: String, - pub consume_batch_size: Option, -} - - -/// Defines all the distinct messages that can be sent over the wire between client and server. -#[derive(Debug, PartialEq, Clone)] -pub enum ProtocolMessage { - /// Always the first message sent by the client to the server - Announce(ClientAnnounce), - /// Always the first message sent by a server to another server - PeerAnnounce(EventStreamStatus), - /// Sent in response to an Announce message. Contains basic information about the status of an event stream - StreamStatus(EventStreamStatus), - /// Set the event stream that the client will work with - SetEventStream(SetEventStream), - /// Signals a client's intent to publish a new event. The server will respond with either an `EventAck` or an `ErrorMessage` - ProduceEvent(ProduceEvent), - /// This is a complete event as serialized over the wire. This message is sent to to both consumers as well as other servers - ReceiveEvent(E), - /// Sent from the server to client to acknowledge that an event was persisted successfully. - AckEvent(EventAck), - /// New message sent by a client to start reading events from the stream - NewStartConsuming(NewConsumerStart), - /// send by the server to a client in response to a StartConsuming message to indicate the start of a series of events - CursorCreated(CursorInfo), - /// sent by a client to a server to tell the server to stop sending events. This is required in order to reuse the connection for multiple queries - StopConsuming(u32), - /// Sent by the client to set the batch size to use for consuming. It is an error to send this message while consuming. - SetBatchSize(u32), - /// Sent by the client to tell the server that it is ready for the next batch - NextBatch, - /// Sent by the server to notify a consumer that it has reached the end of a batch and that more events can be sent - /// upon receipt of a `NextBatch` message by the server. - EndOfBatch, - /// Sent by the server to an active consumer to indicate that it has reached the end of the stream. The server will - /// continue to send events as more come in, but this just lets the client know that it may be some time before more - /// events are available. This message will only be sent at most once to a given consumer. - AwaitingEvents, - /// Represents an error response to any other message - Error(ErrorMessage), -} - -named!{pub parse_str, - map_res!( - length_data!(be_u16), - |res| { - ::std::str::from_utf8(res).map(|val| val.to_owned()) - } - ) -} - -named!{parse_partition_status, - chain!( - partition_num: be_u16 ~ - head: be_u64 ~ - status_num: be_u16, - || { - PartitionStatus { - partition_num: partition_num, - head: head, - primary: status_num == 1, - } - } - - ) -} - -named!{parse_event_stream_status>, - chain!( - _tag: tag!(&[EVENT_STREAM_STATUS]) ~ - status: parse_event_stream_status_struct, - || { - ProtocolMessage::StreamStatus(status) - } - ) -} - -named!{parse_peer_announce>, - chain!( - _tag: tag!(&[PEER_ANNOUNCE]) ~ - status: parse_event_stream_status_struct, - || { - ProtocolMessage::PeerAnnounce(status) - } - ) -} - - -named!{parse_event_stream_status_struct, - chain!( - op_id: be_u32 ~ - name: parse_str ~ - partitions: length_count!(be_u16, parse_partition_status), - || { - EventStreamStatus { - op_id: op_id, - name: name, - partitions: partitions, - } - } - ) -} - -fn require_event_id(id: Option) -> Result { - id.ok_or("EventId must not be all zeros") -} - -named!{parse_non_zero_event_id, - map_res!(parse_event_id, require_event_id) -} - -named!{pub parse_zeroable_event_id, - chain!( - counter: be_u64 ~ - actor: be_u16, - || { - FloEventId::new(actor, counter) - } - ) -} - -named!{pub parse_event_id>, - chain!( - counter: be_u64 ~ - actor: be_u16, - || { - if counter > 0 { - Some(FloEventId::new(actor, counter)) - } else { - None - } - } - ) -} - -named!{pub parse_new_producer_event>, - chain!( - _tag: tag!(&[PRODUCE_EVENT]) ~ - namespace: parse_str ~ - parent_id: parse_event_id ~ - op_id: be_u32 ~ - partition: be_u16 ~ - data_len: be_u32, - || { - ProtocolMessage::ProduceEvent(ProduceEvent{ - namespace: namespace.to_owned(), - parent_id: parent_id, - op_id: op_id, - partition: partition, - data: Vec::with_capacity(data_len as usize), - }) - } - ) -} - -named!{parse_timestamp, - map!(be_u64, time::from_millis_since_epoch) -} - -named!{parse_receive_event_header>, - chain!( - _tag: tag!(&[RECEIVE_EVENT]) ~ - id: parse_non_zero_event_id ~ - parent_id: parse_event_id ~ - timestamp: parse_timestamp ~ - namespace: parse_str ~ - data: length_data!(be_u32), - || { - ProtocolMessage::ReceiveEvent(OwnedFloEvent { - id: id, - parent_id: parent_id, - namespace: namespace, - timestamp: timestamp, - data: data.to_vec(), - }) - } - ) -} - -named!{parse_event_ack>, - chain!( - _tag: tag!(&[ACK_HEADER]) ~ - op_id: be_u32 ~ - counter: be_u64 ~ - actor_id: be_u16, - || { - ProtocolMessage::AckEvent(EventAck { - op_id: op_id, - event_id: FloEventId::new(actor_id, counter) - }) - } - ) -} - -named!{parse_new_start_consuming>, - chain!( - _tag: tag!(&[NEW_START_CONSUMING]) ~ - op_id: be_u32 ~ - version_vec: parse_version_vec ~ - max_events: be_u64 ~ - namespace: parse_str, - || { - ProtocolMessage::NewStartConsuming(NewConsumerStart { - op_id: op_id, - version_vector: version_vec, - max_events: max_events, - namespace: namespace, - }) - } - ) -} - -named!{parse_set_event_stream>, - chain!( - _tag: tag!(&[SET_EVENT_STREAM]) ~ - op_id: be_u32 ~ - name: parse_str, - || { - ProtocolMessage::SetEventStream(SetEventStream { - op_id: op_id, - name: name, - }) - } - ) -} - -named!{parse_version_vec>, - length_count!(be_u16, parse_zeroable_event_id) -} - - -named!{parse_error_message>, - chain!( - _tag: tag!(&[ERROR_HEADER]) ~ - op_id: be_u32 ~ - kind: map_res!(take!(1), |res: &[u8]| { - ErrorKind::from_u8(res[0]) - }) ~ - description: parse_str, - || { - ProtocolMessage::Error(ErrorMessage { - op_id: op_id, - kind: kind, - description: description, - }) - } - ) -} - -named!{parse_awaiting_events>, map!(tag!(&[AWAITING_EVENTS]), |_| {ProtocolMessage::AwaitingEvents})} - -named!{parse_set_batch_size>, chain!( - _tag: tag!(&[SET_BATCH_SIZE]) ~ - batch_size: be_u32, - || { - ProtocolMessage::SetBatchSize(batch_size) - } -)} - -named!{parse_next_batch>, map!(tag!(&[NEXT_BATCH]), |_| {ProtocolMessage::NextBatch})} -named!{parse_end_of_batch>, map!(tag!(&[END_OF_BATCH]), |_| {ProtocolMessage::EndOfBatch})} -named!{parse_stop_consuming>, chain!( - _tag: tag!(&[STOP_CONSUMING]) ~ - op_id: be_u32, - || { - ProtocolMessage::StopConsuming(op_id) - } -)} - -named!{parse_cursor_created>, chain!( - _tag: tag!(&[headers::CURSOR_CREATED]) ~ - op_id: be_u32 ~ - batch_size: be_u32, - || { - ProtocolMessage::CursorCreated(CursorInfo{ - op_id: op_id, - batch_size: batch_size - }) - } -)} - -named!{parse_client_announce>, chain!( - _tag: tag!(&[CLIENT_ANNOUNCE]) ~ - protocol_version: be_u32 ~ - op_id: be_u32 ~ - client_name: parse_str ~ - batch_size: be_u32, - || { - let batch = if batch_size > 0 { Some(batch_size) } else { None }; - - ProtocolMessage::Announce(ClientAnnounce{ - protocol_version: protocol_version, - op_id: op_id, - client_name: client_name, - consume_batch_size: batch - }) - } -)} - -named!{pub parse_any>, alt!( - parse_event_ack | - parse_receive_event_header | - parse_error_message | - parse_awaiting_events | - parse_new_producer_event | - parse_set_batch_size | - parse_next_batch | - parse_end_of_batch | - parse_stop_consuming | - parse_cursor_created | - parse_new_start_consuming | - parse_set_event_stream | - parse_event_stream_status | - parse_client_announce | - parse_peer_announce -)} - -fn serialize_new_produce_header(header: &ProduceEvent, buf: &mut [u8]) -> usize { - let (counter, actor) = header.parent_id.map(|id| { - (id.event_counter, id.actor) - }).unwrap_or((0, 0)); - - Serializer::new(buf).write_u8(PRODUCE_EVENT) - .write_string(&header.namespace) - .write_u64(counter) - .write_u16(actor) - .write_u32(header.op_id) - .write_u16(header.partition) - .write_u32(header.data.len() as u32) - .finish() -} - -fn serialize_event_ack(ack: &EventAck, buf: &mut [u8]) -> usize { - Serializer::new(buf).write_u8(ACK_HEADER) - .write_u32(ack.op_id) - .write_u64(ack.event_id.event_counter) - .write_u16(ack.event_id.actor) - .finish() -} - -fn serialize_error_message(err: &ErrorMessage, buf: &mut [u8]) -> usize { - Serializer::new(buf).write_u8(ERROR_HEADER) - .write_u32(err.op_id) - .write_u8(err.kind.u8_value()) - .write_string(&err.description) - .finish() -} - -fn serialize_receive_event_header(event: &E, buf: &mut [u8]) -> usize { - Serializer::new(buf) - .write_u8(::client::headers::RECEIVE_EVENT) - .write_u64(event.id().event_counter) - .write_u16(event.id().actor) - .write_u64(event.parent_id().map(|id| id.event_counter).unwrap_or(0)) - .write_u16(event.parent_id().map(|id| id.actor).unwrap_or(0)) - .write_u64(time::millis_since_epoch(event.timestamp())) - .write_string(event.namespace()) - .write_u32(event.data_len()) - .finish() -} - -fn serialize_event_stream_status(header: u8, status: &EventStreamStatus, buf: &mut [u8]) -> usize { - Serializer::new(buf) - .write_u8(header) - .write_u32(status.op_id) - .write_string(&status.name) - .write_u16(status.partitions.len() as u16) - .write_many(status.partitions.iter(), |ser, partition| { - let status: u16 = if partition.primary { 1 } else { 0 }; - ser.write_u16(partition.partition_num) - .write_u64(partition.head) - .write_u16(status) - }) - .finish() -} - -impl ProtocolMessage { - - pub fn serialize(&self, buf: &mut [u8]) -> usize { - match *self { - ProtocolMessage::Announce(ref announce) => { - Serializer::new(buf) - .write_u8(CLIENT_ANNOUNCE) - .write_u32(announce.protocol_version) - .write_u32(announce.op_id) - .write_string(&announce.client_name) - .write_u32(announce.consume_batch_size.unwrap_or(0)) - .finish() - } - ProtocolMessage::PeerAnnounce(ref status) => { - serialize_event_stream_status(PEER_ANNOUNCE, status, buf) - } - ProtocolMessage::StreamStatus(ref status) => { - serialize_event_stream_status(EVENT_STREAM_STATUS, status, buf) - } - ProtocolMessage::SetEventStream(ref set_stream) => { - Serializer::new(buf) - .write_u8(SET_EVENT_STREAM) - .write_u32(set_stream.op_id) - .write_string(&set_stream.name) - .finish() - } - ProtocolMessage::ReceiveEvent(ref event) => { - serialize_receive_event_header(event, buf) - } - ProtocolMessage::CursorCreated(ref info) => { - Serializer::new(buf).write_u8(headers::CURSOR_CREATED) - .write_u32(info.op_id) - .write_u32(info.batch_size) - .finish() - } - ProtocolMessage::AwaitingEvents => { - Serializer::new(buf).write_u8(AWAITING_EVENTS).finish() - } - ProtocolMessage::StopConsuming(op_id) => { - Serializer::new(buf) - .write_u8(headers::STOP_CONSUMING) - .write_u32(op_id) - .finish() - } - ProtocolMessage::ProduceEvent(ref header) => { - serialize_new_produce_header(header, buf) - } - ProtocolMessage::NewStartConsuming(NewConsumerStart{ref op_id, ref version_vector, ref max_events, ref namespace}) => { - let mut serializer = Serializer::new(buf).write_u8(NEW_START_CONSUMING) - .write_u32(*op_id) - .write_u16(version_vector.len() as u16); - - for id in version_vector.iter() { - serializer = serializer.write_u64(id.event_counter).write_u16(id.actor); - } - serializer.write_u64(*max_events) - .write_string(namespace).finish() - } - ProtocolMessage::AckEvent(ref ack) => { - serialize_event_ack(ack, buf) - } - ProtocolMessage::Error(ref err_message) => { - serialize_error_message(err_message, buf) - } - ProtocolMessage::SetBatchSize(batch_size) => { - Serializer::new(buf).write_u8(SET_BATCH_SIZE) - .write_u32(batch_size) - .finish() - } - ProtocolMessage::NextBatch => { - buf[0] = NEXT_BATCH; - 1 - } - ProtocolMessage::EndOfBatch => { - buf[0] = END_OF_BATCH; - 1 - } - } - } - - pub fn get_body(&self) -> Option<&[u8]> { - match *self { - ProtocolMessage::ProduceEvent(ref produce) => { - Some(produce.data.as_slice()) - } - ProtocolMessage::ReceiveEvent(ref event) => { - Some(event.data()) - } - _ => None - } - } - - pub fn get_op_id(&self) -> u32 { - match *self { - ProtocolMessage::Announce(ref ann) => ann.op_id, - ProtocolMessage::ProduceEvent(ref prod) => prod.op_id, - ProtocolMessage::CursorCreated(ref info) => info.op_id, - ProtocolMessage::Error(ref err) => err.op_id, - ProtocolMessage::AckEvent(ref ack) => ack.op_id, - ProtocolMessage::StreamStatus(ref status) => status.op_id, - ProtocolMessage::SetEventStream(ref set) => set.op_id, - ProtocolMessage::StopConsuming(ref op_id) => *op_id, - _ => 0 - } - } -} - -#[cfg(test)] -mod test { - use super::*; - use nom::{IResult, Needed}; - use event::{OwnedFloEvent, time, FloEventId}; - - fn test_serialize_then_deserialize(message: &ProtocolMessage) { - let result = ser_de(message); - assert_eq!(*message, result); - } - - fn ser_de(message: &ProtocolMessage) -> ProtocolMessage { - serde_with_body(message, false) - } - - fn serde_with_body(message: &ProtocolMessage, include_body: bool) -> ProtocolMessage { - let mut buffer = [0; 1024]; - - let mut len = message.serialize(&mut buffer[..]); - if include_body { - if let Some(body) = message.get_body() { - (&mut buffer[len..(len + body.len())]).copy_from_slice(body); - len += body.len(); - } - } - (&mut buffer[len..(len + 4)]).copy_from_slice(&[4, 3, 2, 1]); // extra bytes at the end of the buffer - println!("buffer: {:?}", &buffer[..(len + 4)]); - - match parse_any(&buffer) { - IResult::Done(remaining, result) => { - assert!(remaining.starts_with(&[4, 3, 2, 1])); - result - } - IResult::Error(err) => { - panic!("Got parse error: {:?}", err) - } - IResult::Incomplete(need) => { - panic!("Got incomplete: {:?}", need) - } - } - - } - - #[test] - fn serde_client_announce() { - let announce = ClientAnnounce { - protocol_version: 1, - op_id: 765, - client_name: "nathan".to_owned(), - consume_batch_size: Some(456), - }; - test_serialize_then_deserialize(&ProtocolMessage::Announce(announce)); - } - - #[test] - fn peer_announce() { - let status = EventStreamStatus { - op_id: 1, - name: "system".to_owned(), - partitions: vec![ - PartitionStatus { - partition_num: 1, - head: 638, - primary: true, - } - ], - }; - test_serialize_then_deserialize(&ProtocolMessage::PeerAnnounce(status)); - } - - #[test] - fn serde_event_stream_status() { - let status = EventStreamStatus { - op_id: 6425, - name: "foo".to_owned(), - partitions: vec![ - PartitionStatus { - partition_num: 1, - head: 638, - primary: true, - }, - PartitionStatus { - partition_num: 2, - head: 0, - primary: false, - }, - PartitionStatus { - partition_num: 3, - head: 638, - primary: true, - }, - ], - }; - test_serialize_then_deserialize(&ProtocolMessage::StreamStatus(status)); - - let status = EventStreamStatus { - op_id: 0, - name: "".to_owned(), - partitions: Vec::new() - }; - test_serialize_then_deserialize(&ProtocolMessage::StreamStatus(status)); - } - - #[test] - fn serde_set_event_stream() { - let set_stream = SetEventStream { - op_id: 7264, - name: "foo".to_owned() - }; - test_serialize_then_deserialize(&ProtocolMessage::SetEventStream(set_stream)); - } - - #[test] - fn serde_new_start_consuming() { - let version_vec = vec![ - FloEventId::new(1, 5), - FloEventId::new(3, 8), - FloEventId::new(8, 5) - ]; - test_serialize_then_deserialize(&ProtocolMessage::NewStartConsuming(NewConsumerStart{ - op_id: 321, - version_vector: version_vec, - max_events: 987, - namespace: "/foo/bar/*".to_owned(), - })); - } - - #[test] - fn serde_new_start_consuming_with_one_event() { - let vv = vec![FloEventId::new(1, 0)]; - let msg = ProtocolMessage::NewStartConsuming(NewConsumerStart { - op_id: 3, - version_vector: vv, - max_events: 1, - namespace: "/foo/*".to_owned(), - }); - test_serialize_then_deserialize(&msg); - } - - #[test] - fn serde_receive_event() { - let event = OwnedFloEvent { - id: FloEventId::new(4, 5), - timestamp: time::from_millis_since_epoch(99), - parent_id: Some(FloEventId::new(4, 3)), - namespace: "/foo/bar".to_owned(), - data: vec![9; 99], - }; - let message = ProtocolMessage::ReceiveEvent(event.clone()); - let result = serde_with_body(&message, true); - assert_eq!(message, result); - } - - #[test] - fn stop_consuming_is_serialized_and_parsed() { - test_serialize_then_deserialize(&ProtocolMessage::StopConsuming(345)); - } - - #[test] - fn cursor_created_is_serialized_and_parsed() { - test_serialize_then_deserialize(&ProtocolMessage::CursorCreated(CursorInfo{op_id: 543, batch_size: 78910})); - } - - #[test] - fn next_batch_is_serialized_and_parsed() { - test_serialize_then_deserialize(&ProtocolMessage::NextBatch); - } - - #[test] - fn end_of_batch_is_serialized_and_parsed() { - test_serialize_then_deserialize(&ProtocolMessage::EndOfBatch); - } - - #[test] - fn set_batch_size_is_serialized_and_parsed() { - test_serialize_then_deserialize(&ProtocolMessage::SetBatchSize(1234567)); - } - - #[test] - fn awaiting_events_message_is_serialized_and_parsed() { - test_serialize_then_deserialize(&mut ProtocolMessage::AwaitingEvents); - } - - #[test] - fn error_message_is_parsed() { - let error = ErrorMessage { - op_id: 12345, - kind: ErrorKind::InvalidNamespaceGlob, - description: "some shit happened".to_owned(), - }; - test_serialize_then_deserialize(&mut ProtocolMessage::Error(error)); - } - - #[test] - fn acknowledge_event_message_is_parsed() { - test_serialize_then_deserialize(&mut ProtocolMessage::AckEvent(EventAck{ - op_id: 2345667, - event_id: FloEventId::new(123, 456), - })); - } - - #[test] - fn parse_producer_event_parses_the_header_but_not_the_data() { - let input = ProduceEvent { - namespace: "/the/namespace".to_owned(), - parent_id: Some(FloEventId::new(123, 456)), - op_id: 9, - partition: 7, - data: vec![9; 5] - }; - let mut message_input = ProtocolMessage::ProduceEvent(input.clone()); - let message_result = ser_de(&mut message_input); - - if let ProtocolMessage::ProduceEvent(result) = message_result { - assert_eq!(input.namespace, result.namespace); - assert_eq!(input.parent_id, result.parent_id); - assert_eq!(input.op_id, result.op_id); - assert_eq!(input.partition, result.partition); - - // The vector must be allocated with the correct capacity, but we haven't actually read all the data - assert_eq!(input.data.len(), result.data.capacity()); - } else { - panic!("got the wrong fucking message. Just quit now"); - } - } - - #[test] - fn parse_string_returns_empty_string_string_length_is_0() { - let input = vec![0, 0, 110, 4, 5, 6, 7]; - let (remaining, result) = parse_str(&input).unwrap(); - assert_eq!("".to_owned(), result); - assert_eq!(&vec![110, 4, 5, 6, 7], &remaining); - } - - #[test] - fn string_is_serialized_and_parsed() { - let input = "hello\n\tmoar bytes"; - let mut buffer = [0; 64]; - - let n_bytes = Serializer::new(&mut buffer).write_string(input).finish(); - assert_eq!(19, n_bytes); - - let (_, result) = parse_str(&buffer[0..19]).unwrap(); - assert_eq!(input.to_owned(), result); - } - - #[test] - fn this_works_how_i_think_it_does() { - let input = vec![ - 3, - 0, 0, 0, 0, 0, 0, 1, 34, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 93, 77, 45, 214, 26, - 47, 101, 118, 101 - ]; - - let result = parse_any(&input); - let expected = IResult::Incomplete(Needed::Size(12164)); - assert_eq!(expected, result); - } -} diff --git a/flo-protocol/src/lib.rs b/flo-protocol/src/lib.rs index 99c825b..6bcc9d9 100644 --- a/flo-protocol/src/lib.rs +++ b/flo-protocol/src/lib.rs @@ -8,13 +8,13 @@ extern crate flo_event as event; extern crate byteorder; pub mod serializer; -mod client; +mod messages; use std::io::{self, Read, Write}; use std::cmp; use std::fmt::{self, Debug}; -pub use self::client::*; +pub use self::messages::*; use event::{FloEvent, OwnedFloEvent}; pub const BUFFER_LENGTH: usize = 8 * 1024; @@ -180,7 +180,7 @@ impl MessageStream where T: Read { read_buffer.fill(io)? }; let buffer_start_length = bytes.len(); - match self::client::parse_any(bytes) { + match self::messages::parse_any(bytes) { IResult::Done(remaining, message) => { bytes_consumed += buffer_start_length - remaining.len(); trace!("Successful parse used {} bytes; got message: {:?}", bytes_consumed, message); diff --git a/flo-protocol/src/messages/client_announce.rs b/flo-protocol/src/messages/client_announce.rs new file mode 100644 index 0000000..8794590 --- /dev/null +++ b/flo-protocol/src/messages/client_announce.rs @@ -0,0 +1,44 @@ +use nom::be_u32; + +use event::OwnedFloEvent; +use serializer::Serializer; +use super::{parse_str, ProtocolMessage}; + +pub const CLIENT_ANNOUNCE: u8 = 170; + +/// Sent by the client as the very first message to the server. The server will respond with an `EventStreamStatus` for the current (default) stream +#[derive(Debug, PartialEq, Clone)] +pub struct ClientAnnounce { + pub protocol_version: u32, + pub op_id: u32, + pub client_name: String, + pub consume_batch_size: Option, +} + +named!{pub parse_client_announce>, chain!( + _tag: tag!(&[CLIENT_ANNOUNCE]) ~ + protocol_version: be_u32 ~ + op_id: be_u32 ~ + client_name: parse_str ~ + batch_size: be_u32, + || { + let batch = if batch_size > 0 { Some(batch_size) } else { None }; + + ProtocolMessage::Announce(ClientAnnounce{ + protocol_version: protocol_version, + op_id: op_id, + client_name: client_name, + consume_batch_size: batch + }) + } +)} + +pub fn serialize_client_announce(announce: &ClientAnnounce, buf: &mut [u8]) -> usize { + Serializer::new(buf) + .write_u8(CLIENT_ANNOUNCE) + .write_u32(announce.protocol_version) + .write_u32(announce.op_id) + .write_string(&announce.client_name) + .write_u32(announce.consume_batch_size.unwrap_or(0)) + .finish() +} diff --git a/flo-protocol/src/messages/consume_start.rs b/flo-protocol/src/messages/consume_start.rs new file mode 100644 index 0000000..2ee8d42 --- /dev/null +++ b/flo-protocol/src/messages/consume_start.rs @@ -0,0 +1,54 @@ + +use nom::{be_u64, be_u32}; + +use event::{OwnedFloEvent, FloEventId}; +use serializer::Serializer; +use super::{ProtocolMessage, parse_str, parse_version_vec}; + + +pub const NEW_START_CONSUMING: u8 = 17; + +pub const CONSUME_UNLIMITED: u64 = 0; + +/// New message sent from client to server to begin reading events from the stream +#[derive(Debug, PartialEq, Clone)] +pub struct NewConsumerStart { + pub op_id: u32, + pub version_vector: Vec, + pub max_events: u64, + pub namespace: String, +} + +named!{pub parse_new_start_consuming>, + chain!( + _tag: tag!(&[NEW_START_CONSUMING]) ~ + op_id: be_u32 ~ + version_vec: parse_version_vec ~ + max_events: be_u64 ~ + namespace: parse_str, + || { + ProtocolMessage::NewStartConsuming(NewConsumerStart { + op_id: op_id, + version_vector: version_vec, + max_events: max_events, + namespace: namespace, + }) + } + ) +} + +pub fn serialize_consumer_start(start: &NewConsumerStart, buf: &mut [u8]) -> usize { + let mut serializer = Serializer::new(buf).write_u8(NEW_START_CONSUMING) + .write_u32(start.op_id) + .write_u16(start.version_vector.len() as u16); + + for id in start.version_vector.iter() { + serializer = serializer.write_u64(id.event_counter).write_u16(id.actor); + } + serializer.write_u64(start.max_events) + .write_string(&start.namespace).finish() +} + + + + diff --git a/flo-protocol/src/messages/cursor_info.rs b/flo-protocol/src/messages/cursor_info.rs new file mode 100644 index 0000000..c78cc33 --- /dev/null +++ b/flo-protocol/src/messages/cursor_info.rs @@ -0,0 +1,39 @@ + +use nom::be_u32; +use serializer::Serializer; +use event::OwnedFloEvent; +use super::ProtocolMessage; + +pub const HEADER: u8 = 16; + +/// Sent in a CursorCreated message from the server to a client to indicate that a cursor was successfully created. +/// Currently, this message only contains the batch size, but more fields may be added as they become necessary. +#[derive(Debug, PartialEq, Clone)] +pub struct CursorInfo { + /// The operation id from the StartConsuming message that created this cursor. + pub op_id: u32, + + /// The actual batch size that will be used by the server for sending events. Note that this value _may_ differ from the + /// batch size that was explicitly set by the consumer, depending on server settings. This behavior is not currently + /// implemented by the server, but it's definitely possible to change in the near future. + pub batch_size: u32, +} + +named!{pub parse_cursor_created>, chain!( + _tag: tag!(&[HEADER]) ~ + op_id: be_u32 ~ + batch_size: be_u32, + || { + ProtocolMessage::CursorCreated(CursorInfo{ + op_id: op_id, + batch_size: batch_size + }) + } +)} + +pub fn serialize_cursor_created(info: &CursorInfo, buf: &mut [u8]) -> usize { + Serializer::new(buf).write_u8(HEADER) + .write_u32(info.op_id) + .write_u32(info.batch_size) + .finish() +} diff --git a/flo-protocol/src/messages/error.rs b/flo-protocol/src/messages/error.rs new file mode 100644 index 0000000..521e5f6 --- /dev/null +++ b/flo-protocol/src/messages/error.rs @@ -0,0 +1,94 @@ + +use nom::be_u32; +use serializer::Serializer; +use event::OwnedFloEvent; +use super::{ProtocolMessage, parse_str}; + +pub const ERROR_HEADER: u8 = 10; + + +pub const ERROR_INVALID_NAMESPACE: u8 = 15; +pub const ERROR_INVALID_CONSUMER_STATE: u8 = 16; +pub const ERROR_INVALID_VERSION_VECTOR: u8 = 17; +pub const ERROR_STORAGE_ENGINE_IO: u8 = 18; +pub const ERROR_NO_STREAM: u8 = 19; + +/// Describes the type of error. This gets serialized a u8 +#[derive(Debug, PartialEq, Clone)] +pub enum ErrorKind { + /// Indicates that the namespace provided by a consumer was an invalid glob pattern + InvalidNamespaceGlob, + /// Indicates that the client connection was in an invalid state when it attempted some consumer operation + InvalidConsumerState, + /// Indicates that the provided version vector was invalid (contained more than one entry for at least one actor id) + InvalidVersionVector, + /// Unable to read or write to events file + StorageEngineError, + /// Requested event stream does not exist + NoSuchStream, +} + +/// Represents a response to any request that results in an error +#[derive(Debug, PartialEq, Clone)] +pub struct ErrorMessage { + /// The op_id of the request to make it easier to correlate request/response pairs + pub op_id: u32, + + /// The type of error + pub kind: ErrorKind, + + /// A human-readable description of the error + pub description: String, +} + +impl ErrorKind { + /// Converts from the serialized u8 to an ErrorKind + pub fn from_u8(byte: u8) -> Result { + match byte { + ERROR_INVALID_NAMESPACE => Ok(ErrorKind::InvalidNamespaceGlob), + ERROR_INVALID_CONSUMER_STATE => Ok(ErrorKind::InvalidConsumerState), + ERROR_INVALID_VERSION_VECTOR => Ok(ErrorKind::InvalidVersionVector), + ERROR_STORAGE_ENGINE_IO => Ok(ErrorKind::StorageEngineError), + ERROR_NO_STREAM => Ok(ErrorKind::NoSuchStream), + other => Err(other) + } + } + + /// Converts the ErrorKind to it's serialized u8 value + pub fn u8_value(&self) -> u8 { + match self { + &ErrorKind::InvalidNamespaceGlob => ERROR_INVALID_NAMESPACE, + &ErrorKind::InvalidConsumerState => ERROR_INVALID_CONSUMER_STATE, + &ErrorKind::InvalidVersionVector => ERROR_INVALID_VERSION_VECTOR, + &ErrorKind::StorageEngineError => ERROR_STORAGE_ENGINE_IO, + &ErrorKind::NoSuchStream => ERROR_NO_STREAM, + } + } +} + + +named!{pub parse_error_message>, + chain!( + _tag: tag!(&[ERROR_HEADER]) ~ + op_id: be_u32 ~ + kind: map_res!(take!(1), |res: &[u8]| { + ErrorKind::from_u8(res[0]) + }) ~ + description: parse_str, + || { + ProtocolMessage::Error(ErrorMessage { + op_id: op_id, + kind: kind, + description: description, + }) + } + ) +} + +pub fn serialize_error_message(err: &ErrorMessage, buf: &mut [u8]) -> usize { + Serializer::new(buf).write_u8(ERROR_HEADER) + .write_u32(err.op_id) + .write_u8(err.kind.u8_value()) + .write_string(&err.description) + .finish() +} diff --git a/flo-protocol/src/messages/event_ack.rs b/flo-protocol/src/messages/event_ack.rs new file mode 100644 index 0000000..7328bba --- /dev/null +++ b/flo-protocol/src/messages/event_ack.rs @@ -0,0 +1,43 @@ + +use nom::{be_u64, be_u32, be_u16}; +use serializer::Serializer; +use event::{FloEventId, OwnedFloEvent}; +use super::ProtocolMessage; + +pub const ACK_HEADER: u8 = 9; + + +/// Sent by the server to the producer of an event to acknowledge that the event was successfully persisted to the stream. +#[derive(Debug, PartialEq, Clone)] +pub struct EventAck { + /// This will be set to the `op_id` that was sent in the `ProduceEventHeader` + pub op_id: u32, + + /// The id that was assigned to the event. This id is immutable and must be the same across all servers in a flo cluster. + pub event_id: FloEventId, +} + +named!{pub parse_event_ack>, + chain!( + _tag: tag!(&[ACK_HEADER]) ~ + op_id: be_u32 ~ + counter: be_u64 ~ + actor_id: be_u16, + || { + ProtocolMessage::AckEvent(EventAck { + op_id: op_id, + event_id: FloEventId::new(actor_id, counter) + }) + } + ) +} + + +pub fn serialize_event_ack(ack: &EventAck, buf: &mut [u8]) -> usize { + Serializer::new(buf).write_u8(ACK_HEADER) + .write_u32(ack.op_id) + .write_u64(ack.event_id.event_counter) + .write_u16(ack.event_id.actor) + .finish() +} + diff --git a/flo-protocol/src/messages/event_stream_status.rs b/flo-protocol/src/messages/event_stream_status.rs new file mode 100644 index 0000000..954af8f --- /dev/null +++ b/flo-protocol/src/messages/event_stream_status.rs @@ -0,0 +1,81 @@ +use nom::{be_u64, be_u32, be_u16}; + +use event::{EventCounter, ActorId, OwnedFloEvent}; +use serializer::Serializer; +use super::{parse_str, ProtocolMessage}; + +pub const EVENT_STREAM_STATUS: u8 = 19; + +/// Information on the status of a partition. Included as part of `EventStreamStatus` +#[derive(Debug, PartialEq, Clone)] +pub struct PartitionStatus { + pub partition_num: ActorId, + pub head: EventCounter, + pub primary: bool, +} + +/// Contains some basic information on an event stream. Sent in response to a `SetEventStream` +#[derive(Debug, PartialEq, Clone)] +pub struct EventStreamStatus { + pub op_id: u32, + pub name: String, + pub partitions: Vec, +} + + +named!{parse_partition_status, + chain!( + partition_num: be_u16 ~ + head: be_u64 ~ + status_num: be_u16, + || { + PartitionStatus { + partition_num: partition_num, + head: head, + primary: status_num == 1, + } + } + + ) +} + +named!{pub parse_event_stream_status>, + chain!( + _tag: tag!(&[EVENT_STREAM_STATUS]) ~ + status: parse_event_stream_status_struct, + || { + ProtocolMessage::StreamStatus(status) + } + ) +} + +named!{parse_event_stream_status_struct, + chain!( + op_id: be_u32 ~ + name: parse_str ~ + partitions: length_count!(be_u16, parse_partition_status), + || { + EventStreamStatus { + op_id: op_id, + name: name, + partitions: partitions, + } + } + ) +} + + +pub fn serialize_event_stream_status(status: &EventStreamStatus, buf: &mut [u8]) -> usize { + Serializer::new(buf) + .write_u8(EVENT_STREAM_STATUS) + .write_u32(status.op_id) + .write_string(&status.name) + .write_u16(status.partitions.len() as u16) + .write_many(status.partitions.iter(), |ser, partition| { + let status: u16 = if partition.primary { 1 } else { 0 }; + ser.write_u16(partition.partition_num) + .write_u64(partition.head) + .write_u16(status) + }) + .finish() +} diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs new file mode 100644 index 0000000..3d0ab1a --- /dev/null +++ b/flo-protocol/src/messages/mod.rs @@ -0,0 +1,557 @@ +//! This is the wire protocol used to communicate between the server and client. Communication is done by sending and +//! receiving series' of distinct messages. Each message begins with an 8 byte header that identifies the type of message. +//! This is rather wasteful, but useful for the early stages when there's still a fair bit of debugging via manual inspection +//! of buffers. Messages are parsed using nom parser combinators, and serialized using simple a wrapper around a writer. +//! +//! The special cases in the protocol are for sending/receiving the events themselves. Since events can be quite large, they +//! are not actually implemented as a single message in the protocol, but rather as just a header. The header has all the basic +//! information as well as the length of the data portion (the body of the event). The event is read by first reading the +//! header and then reading however many bytes are indicated by the header for the body of the event. +//! +//! All numbers use big endian byte order. +//! All Strings are newline terminated. +mod client_announce; +mod peer_announce; +mod error; +mod produce_event; +mod event_ack; +mod consume_start; +mod cursor_info; +mod event_stream_status; +mod set_event_stream; +mod receive_event; + +use nom::{be_u64, be_u32, be_u16, be_u8}; +use event::{time, OwnedFloEvent, FloEvent, FloEventId, ActorId, EventCounter, Timestamp}; +use serializer::Serializer; +use std::net::SocketAddr; + +use self::client_announce::{parse_client_announce, serialize_client_announce}; +use self::peer_announce::{parse_peer_announce, serialize_peer_announce}; +use self::error::{parse_error_message, serialize_error_message}; +use self::produce_event::{parse_new_producer_event, serialize_new_produce_header}; +use self::event_ack::{parse_event_ack, serialize_event_ack}; +use self::consume_start::{parse_new_start_consuming, serialize_consumer_start}; +use self::cursor_info::{parse_cursor_created, serialize_cursor_created}; +use self::event_stream_status::{parse_event_stream_status, serialize_event_stream_status}; +use self::set_event_stream::{serialize_set_event_stream, parse_set_event_stream}; +use self::receive_event::{serialize_receive_event_header, parse_receive_event_header}; + +pub use self::client_announce::ClientAnnounce; +pub use self::peer_announce::PeerAnnounce; +pub use self::error::{ErrorMessage, ErrorKind}; +pub use self::produce_event::ProduceEvent; +pub use self::event_ack::EventAck; +pub use self::consume_start::NewConsumerStart; +pub use self::cursor_info::CursorInfo; +pub use self::event_stream_status::{EventStreamStatus, PartitionStatus}; +pub use self::set_event_stream::SetEventStream; + +pub mod headers { + pub const CLIENT_AUTH: u8 = 1; + pub const UPDATE_MARKER: u8 = 4; + pub const START_CONSUMING: u8 = 5; + pub const AWAITING_EVENTS: u8 = 6; + pub const PEER_UPDATE: u8 = 8; + pub const CLUSTER_STATE: u8 = 11; + pub const SET_BATCH_SIZE: u8 = 12; + pub const NEXT_BATCH: u8 = 13; + pub const END_OF_BATCH: u8 = 14; + pub const STOP_CONSUMING: u8 = 15; +} + +use self::headers::*; + +/// Defines all the distinct messages that can be sent over the wire between client and server. +#[derive(Debug, PartialEq, Clone)] +pub enum ProtocolMessage { + /// Always the first message sent by the client to the server + Announce(ClientAnnounce), + /// Always the first message sent by a server to another server + PeerAnnounce(PeerAnnounce), + /// Sent in response to an Announce message. Contains basic information about the status of an event stream + StreamStatus(EventStreamStatus), + /// Set the event stream that the client will work with + SetEventStream(SetEventStream), + /// Signals a client's intent to publish a new event. The server will respond with either an `EventAck` or an `ErrorMessage` + ProduceEvent(ProduceEvent), + /// This is a complete event as serialized over the wire. This message is sent to to both consumers as well as other servers + ReceiveEvent(E), + /// Sent from the server to client to acknowledge that an event was persisted successfully. + AckEvent(EventAck), + /// New message sent by a client to start reading events from the stream + NewStartConsuming(NewConsumerStart), + /// send by the server to a client in response to a StartConsuming message to indicate the start of a series of events + CursorCreated(CursorInfo), + /// sent by a client to a server to tell the server to stop sending events. This is required in order to reuse the connection for multiple queries + StopConsuming(u32), + /// Sent by the client to set the batch size to use for consuming. It is an error to send this message while consuming. + SetBatchSize(u32), + /// Sent by the client to tell the server that it is ready for the next batch + NextBatch, + /// Sent by the server to notify a consumer that it has reached the end of a batch and that more events can be sent + /// upon receipt of a `NextBatch` message by the server. + EndOfBatch, + /// Sent by the server to an active consumer to indicate that it has reached the end of the stream. The server will + /// continue to send events as more come in, but this just lets the client know that it may be some time before more + /// events are available. This message will only be sent at most once to a given consumer. + AwaitingEvents, + /// Represents an error response to any other message + Error(ErrorMessage), +} + +named!{parse_str, + map_res!( + length_data!(be_u16), + |res| { + ::std::str::from_utf8(res).map(|val| val.to_owned()) + } + ) +} + + + +named!{parse_socket_addr, alt!(parse_socket_addr_v4)} + +named!{parse_socket_addr_v4, + chain!( + _tag: tag!(&[4u8]) ~ + one: be_u8 ~ + two: be_u8 ~ + three: be_u8 ~ + four: be_u8 ~ + port: be_u16, + || { + let ip = ::std::net::Ipv4Addr::new(one, two, three, four); + let addr = ::std::net::SocketAddrV4::new(ip, port); + SocketAddr::V4(addr) + } + ) +} + +fn require_event_id(id: Option) -> Result { + id.ok_or("EventId must not be all zeros") +} + +named!{parse_non_zero_event_id, + map_res!(parse_event_id, require_event_id) +} + +named!{parse_zeroable_event_id, + chain!( + counter: be_u64 ~ + actor: be_u16, + || { + FloEventId::new(actor, counter) + } + ) +} + +named!{parse_event_id>, + chain!( + counter: be_u64 ~ + actor: be_u16, + || { + if counter > 0 { + Some(FloEventId::new(actor, counter)) + } else { + None + } + } + ) +} + +named!{parse_timestamp, + map!(be_u64, time::from_millis_since_epoch) +} + + +named!{parse_version_vec>, + length_count!(be_u16, parse_zeroable_event_id) +} + + +named!{parse_awaiting_events>, map!(tag!(&[AWAITING_EVENTS]), |_| {ProtocolMessage::AwaitingEvents})} + +named!{parse_set_batch_size>, chain!( + _tag: tag!(&[SET_BATCH_SIZE]) ~ + batch_size: be_u32, + || { + ProtocolMessage::SetBatchSize(batch_size) + } +)} + +named!{parse_next_batch>, map!(tag!(&[NEXT_BATCH]), |_| {ProtocolMessage::NextBatch})} + +named!{parse_end_of_batch>, map!(tag!(&[END_OF_BATCH]), |_| {ProtocolMessage::EndOfBatch})} + +named!{parse_stop_consuming>, chain!( + _tag: tag!(&[STOP_CONSUMING]) ~ + op_id: be_u32, + || { + ProtocolMessage::StopConsuming(op_id) + } +)} + + +named!{pub parse_any>, alt!( + parse_event_ack | + parse_receive_event_header | + parse_error_message | + parse_awaiting_events | + parse_new_producer_event | + parse_set_batch_size | + parse_next_batch | + parse_end_of_batch | + parse_stop_consuming | + parse_cursor_created | + parse_new_start_consuming | + parse_set_event_stream | + parse_event_stream_status | + parse_client_announce | + parse_peer_announce +)} + + +impl ProtocolMessage { + + pub fn serialize(&self, buf: &mut [u8]) -> usize { + match *self { + ProtocolMessage::Announce(ref announce) => { + serialize_client_announce(announce, buf) + } + ProtocolMessage::PeerAnnounce(ref announce) => { + serialize_peer_announce(announce, buf) + } + ProtocolMessage::StreamStatus(ref status) => { + serialize_event_stream_status(status, buf) + } + ProtocolMessage::SetEventStream(ref set_stream) => { + serialize_set_event_stream(set_stream, buf) + } + ProtocolMessage::ReceiveEvent(ref event) => { + serialize_receive_event_header(event, buf) + } + ProtocolMessage::CursorCreated(ref info) => { + serialize_cursor_created(info, buf) + } + ProtocolMessage::AwaitingEvents => { + Serializer::new(buf).write_u8(AWAITING_EVENTS).finish() + } + ProtocolMessage::StopConsuming(op_id) => { + Serializer::new(buf) + .write_u8(headers::STOP_CONSUMING) + .write_u32(op_id) + .finish() + } + ProtocolMessage::ProduceEvent(ref header) => { + serialize_new_produce_header(header, buf) + } + ProtocolMessage::NewStartConsuming(ref start) => { + serialize_consumer_start(start, buf) + } + ProtocolMessage::AckEvent(ref ack) => { + serialize_event_ack(ack, buf) + } + ProtocolMessage::Error(ref err_message) => { + serialize_error_message(err_message, buf) + } + ProtocolMessage::SetBatchSize(batch_size) => { + Serializer::new(buf).write_u8(SET_BATCH_SIZE) + .write_u32(batch_size) + .finish() + } + ProtocolMessage::NextBatch => { + buf[0] = NEXT_BATCH; + 1 + } + ProtocolMessage::EndOfBatch => { + buf[0] = END_OF_BATCH; + 1 + } + } + } + + pub fn get_body(&self) -> Option<&[u8]> { + match *self { + ProtocolMessage::ProduceEvent(ref produce) => { + Some(produce.data.as_slice()) + } + ProtocolMessage::ReceiveEvent(ref event) => { + Some(event.data()) + } + _ => None + } + } + + pub fn get_op_id(&self) -> u32 { + match *self { + ProtocolMessage::Announce(ref ann) => ann.op_id, + ProtocolMessage::ProduceEvent(ref prod) => prod.op_id, + ProtocolMessage::CursorCreated(ref info) => info.op_id, + ProtocolMessage::Error(ref err) => err.op_id, + ProtocolMessage::AckEvent(ref ack) => ack.op_id, + ProtocolMessage::StreamStatus(ref status) => status.op_id, + ProtocolMessage::SetEventStream(ref set) => set.op_id, + ProtocolMessage::StopConsuming(ref op_id) => *op_id, + _ => 0 + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use nom::{IResult, Needed}; + use event::{OwnedFloEvent, time, FloEventId}; + + fn test_serialize_then_deserialize(message: &ProtocolMessage) { + let result = ser_de(message); + assert_eq!(*message, result); + } + + fn ser_de(message: &ProtocolMessage) -> ProtocolMessage { + serde_with_body(message, false) + } + + fn serde_with_body(message: &ProtocolMessage, include_body: bool) -> ProtocolMessage { + let mut buffer = [0; 1024]; + + let mut len = message.serialize(&mut buffer[..]); + if include_body { + if let Some(body) = message.get_body() { + (&mut buffer[len..(len + body.len())]).copy_from_slice(body); + len += body.len(); + } + } + (&mut buffer[len..(len + 4)]).copy_from_slice(&[4, 3, 2, 1]); // extra bytes at the end of the buffer + println!("buffer: {:?}", &buffer[..(len + 4)]); + + match parse_any(&buffer) { + IResult::Done(remaining, result) => { + assert!(remaining.starts_with(&[4, 3, 2, 1])); + result + } + IResult::Error(err) => { + panic!("Got parse error: {:?}", err) + } + IResult::Incomplete(need) => { + panic!("Got incomplete: {:?}", need) + } + } + + } + + #[test] + fn serde_client_announce() { + let announce = ClientAnnounce { + protocol_version: 1, + op_id: 765, + client_name: "nathan".to_owned(), + consume_batch_size: Some(456), + }; + test_serialize_then_deserialize(&ProtocolMessage::Announce(announce)); + } + + #[test] + fn serde_peer_announce() { + let addr = ::std::str::FromStr::from_str("123.234.12.1:4321").unwrap(); + let announce = PeerAnnounce { + protocol_version: 9, + peer_address: addr, + op_id: 6543, + }; + test_serialize_then_deserialize(&ProtocolMessage::PeerAnnounce(announce)); + } + + #[test] + fn serde_event_stream_status() { + let status = EventStreamStatus { + op_id: 6425, + name: "foo".to_owned(), + partitions: vec![ + PartitionStatus { + partition_num: 1, + head: 638, + primary: true, + }, + PartitionStatus { + partition_num: 2, + head: 0, + primary: false, + }, + PartitionStatus { + partition_num: 3, + head: 638, + primary: true, + }, + ], + }; + test_serialize_then_deserialize(&ProtocolMessage::StreamStatus(status)); + + let status = EventStreamStatus { + op_id: 0, + name: "".to_owned(), + partitions: Vec::new() + }; + test_serialize_then_deserialize(&ProtocolMessage::StreamStatus(status)); + } + + #[test] + fn serde_set_event_stream() { + let set_stream = SetEventStream { + op_id: 7264, + name: "foo".to_owned() + }; + test_serialize_then_deserialize(&ProtocolMessage::SetEventStream(set_stream)); + } + + #[test] + fn serde_new_start_consuming() { + let version_vec = vec![ + FloEventId::new(1, 5), + FloEventId::new(3, 8), + FloEventId::new(8, 5) + ]; + test_serialize_then_deserialize(&ProtocolMessage::NewStartConsuming(NewConsumerStart{ + op_id: 321, + version_vector: version_vec, + max_events: 987, + namespace: "/foo/bar/*".to_owned(), + })); + } + + #[test] + fn serde_new_start_consuming_with_one_event() { + let vv = vec![FloEventId::new(1, 0)]; + let msg = ProtocolMessage::NewStartConsuming(NewConsumerStart { + op_id: 3, + version_vector: vv, + max_events: 1, + namespace: "/foo/*".to_owned(), + }); + test_serialize_then_deserialize(&msg); + } + + #[test] + fn serde_receive_event() { + let event = OwnedFloEvent { + id: FloEventId::new(4, 5), + timestamp: time::from_millis_since_epoch(99), + parent_id: Some(FloEventId::new(4, 3)), + namespace: "/foo/bar".to_owned(), + data: vec![9; 99], + }; + let message = ProtocolMessage::ReceiveEvent(event.clone()); + let result = serde_with_body(&message, true); + assert_eq!(message, result); + } + + #[test] + fn stop_consuming_is_serialized_and_parsed() { + test_serialize_then_deserialize(&ProtocolMessage::StopConsuming(345)); + } + + #[test] + fn cursor_created_is_serialized_and_parsed() { + test_serialize_then_deserialize(&ProtocolMessage::CursorCreated(CursorInfo{op_id: 543, batch_size: 78910})); + } + + #[test] + fn next_batch_is_serialized_and_parsed() { + test_serialize_then_deserialize(&ProtocolMessage::NextBatch); + } + + #[test] + fn end_of_batch_is_serialized_and_parsed() { + test_serialize_then_deserialize(&ProtocolMessage::EndOfBatch); + } + + #[test] + fn set_batch_size_is_serialized_and_parsed() { + test_serialize_then_deserialize(&ProtocolMessage::SetBatchSize(1234567)); + } + + #[test] + fn awaiting_events_message_is_serialized_and_parsed() { + test_serialize_then_deserialize(&mut ProtocolMessage::AwaitingEvents); + } + + #[test] + fn error_message_is_parsed() { + let error = ErrorMessage { + op_id: 12345, + kind: ErrorKind::InvalidNamespaceGlob, + description: "some shit happened".to_owned(), + }; + test_serialize_then_deserialize(&mut ProtocolMessage::Error(error)); + } + + #[test] + fn acknowledge_event_message_is_parsed() { + test_serialize_then_deserialize(&mut ProtocolMessage::AckEvent(EventAck{ + op_id: 2345667, + event_id: FloEventId::new(123, 456), + })); + } + + #[test] + fn parse_producer_event_parses_the_header_but_not_the_data() { + let input = ProduceEvent { + namespace: "/the/namespace".to_owned(), + parent_id: Some(FloEventId::new(123, 456)), + op_id: 9, + partition: 7, + data: vec![9; 5] + }; + let mut message_input = ProtocolMessage::ProduceEvent(input.clone()); + let message_result = ser_de(&mut message_input); + + if let ProtocolMessage::ProduceEvent(result) = message_result { + assert_eq!(input.namespace, result.namespace); + assert_eq!(input.parent_id, result.parent_id); + assert_eq!(input.op_id, result.op_id); + assert_eq!(input.partition, result.partition); + + // The vector must be allocated with the correct capacity, but we haven't actually read all the data + assert_eq!(input.data.len(), result.data.capacity()); + } else { + panic!("got the wrong fucking message. Just quit now"); + } + } + + #[test] + fn parse_string_returns_empty_string_string_length_is_0() { + let input = vec![0, 0, 110, 4, 5, 6, 7]; + let (remaining, result) = parse_str(&input).unwrap(); + assert_eq!("".to_owned(), result); + assert_eq!(&vec![110, 4, 5, 6, 7], &remaining); + } + + #[test] + fn string_is_serialized_and_parsed() { + let input = "hello\n\tmoar bytes"; + let mut buffer = [0; 64]; + + let n_bytes = Serializer::new(&mut buffer).write_string(input).finish(); + assert_eq!(19, n_bytes); + + let (_, result) = parse_str(&buffer[0..19]).unwrap(); + assert_eq!(input.to_owned(), result); + } + + #[test] + fn this_works_how_i_think_it_does() { + let input = vec![ + 3, + 0, 0, 0, 0, 0, 0, 1, 34, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 93, 77, 45, 214, 26, + 47, 101, 118, 101 + ]; + + let result = parse_any(&input); + let expected = IResult::Incomplete(Needed::Size(12164)); + assert_eq!(expected, result); + } +} diff --git a/flo-protocol/src/messages/peer_announce.rs b/flo-protocol/src/messages/peer_announce.rs new file mode 100644 index 0000000..097c390 --- /dev/null +++ b/flo-protocol/src/messages/peer_announce.rs @@ -0,0 +1,40 @@ + +use std::net::SocketAddr; +use nom::be_u32; +use serializer::Serializer; +use event::OwnedFloEvent; +use super::{parse_socket_addr, ProtocolMessage}; + +pub const PEER_ANNOUNCE: u8 = 7; + +/// Sent by one server to another as the very first message to announce its presence +#[derive(Debug, PartialEq, Clone)] +pub struct PeerAnnounce { + pub protocol_version: u32, + pub peer_address: SocketAddr, + pub op_id: u32, +} + +named!{pub parse_peer_announce>, + chain!( + _tag: tag!(&[PEER_ANNOUNCE]) ~ + protocol_version: be_u32 ~ + op_id: be_u32 ~ + peer_address: parse_socket_addr, + || { + ProtocolMessage::PeerAnnounce(PeerAnnounce { + protocol_version, + op_id, + peer_address + }) + } + ) +} + +pub fn serialize_peer_announce(announce: &PeerAnnounce, buf: &mut [u8]) -> usize { + Serializer::new(buf) + .write_u8(PEER_ANNOUNCE) + .write_socket_addr(announce.peer_address) + .write_u32(announce.op_id) + .finish() +} diff --git a/flo-protocol/src/messages/produce_event.rs b/flo-protocol/src/messages/produce_event.rs new file mode 100644 index 0000000..1c13ca8 --- /dev/null +++ b/flo-protocol/src/messages/produce_event.rs @@ -0,0 +1,67 @@ + +use nom::{be_u64, be_u32, be_u16}; +use event::{OwnedFloEvent, FloEventId, ActorId}; +use serializer::Serializer; +use super::{ProtocolMessage, parse_str, parse_event_id}; + + +pub const PRODUCE_EVENT: u8 = 2; + + +/// The body of a ProduceEvent `ProtocolMessage`. This is sent from a client producer to the server, and the server will +/// respond with either an `EventAck` or an `ErrorMessage` to indicate success or failure respectively. Although the flo +/// protocol is pipelined, this message includes an `op_id` field to aid in correlation of requests and responses. +#[derive(Debug, PartialEq, Clone)] +pub struct ProduceEvent { + /// This is an arbritrary number, assigned by the client, to aid in correlation of requests and responses. Clients may + /// choose to just set it to the same value for every operation if they wish. + pub op_id: u32, + /// The partition to produce the event onto + pub partition: ActorId, + /// The namespace to produce the event to. See the `namespace` documentation on `FloEvent` for more information on + /// namespaces in general. As far as the protocol is concerned, it's just serialized as a utf-8 string. + pub namespace: String, + /// The parent_id of the new event. This is typically set to the id of whatever event a consumer is responding to. + /// The parent id is optional. On the wire, a null parent_id is serialized as an event id where both the counter and the + /// actor are set to 0. + pub parent_id: Option, + /// The event payload. As far as the flo server is concerned, this is just an opaque byte array. Note that events with + /// 0-length bodies are perfectly fine. + pub data: Vec, +} + + +named!{pub parse_new_producer_event>, + chain!( + _tag: tag!(&[PRODUCE_EVENT]) ~ + namespace: parse_str ~ + parent_id: parse_event_id ~ + op_id: be_u32 ~ + partition: be_u16 ~ + data_len: be_u32, + || { + ProtocolMessage::ProduceEvent(ProduceEvent{ + namespace: namespace.to_owned(), + parent_id: parent_id, + op_id: op_id, + partition: partition, + data: Vec::with_capacity(data_len as usize), + }) + } + ) +} + +pub fn serialize_new_produce_header(header: &ProduceEvent, buf: &mut [u8]) -> usize { + let (counter, actor) = header.parent_id.map(|id| { + (id.event_counter, id.actor) + }).unwrap_or((0, 0)); + + Serializer::new(buf).write_u8(PRODUCE_EVENT) + .write_string(&header.namespace) + .write_u64(counter) + .write_u16(actor) + .write_u32(header.op_id) + .write_u16(header.partition) + .write_u32(header.data.len() as u32) + .finish() +} diff --git a/flo-protocol/src/messages/receive_event.rs b/flo-protocol/src/messages/receive_event.rs new file mode 100644 index 0000000..01ae4b9 --- /dev/null +++ b/flo-protocol/src/messages/receive_event.rs @@ -0,0 +1,40 @@ + +use nom::be_u32; +use event::{OwnedFloEvent, FloEvent, time}; +use serializer::Serializer; +use super::{ProtocolMessage, parse_non_zero_event_id, parse_event_id, parse_timestamp, parse_str}; + +pub const RECEIVE_EVENT: u8 = 3; + +named!{pub parse_receive_event_header>, + chain!( + _tag: tag!(&[RECEIVE_EVENT]) ~ + id: parse_non_zero_event_id ~ + parent_id: parse_event_id ~ + timestamp: parse_timestamp ~ + namespace: parse_str ~ + data: length_data!(be_u32), + || { + ProtocolMessage::ReceiveEvent(OwnedFloEvent { + id: id, + parent_id: parent_id, + namespace: namespace, + timestamp: timestamp, + data: data.to_vec(), + }) + } + ) +} + +pub fn serialize_receive_event_header(event: &E, buf: &mut [u8]) -> usize { + Serializer::new(buf) + .write_u8(RECEIVE_EVENT) + .write_u64(event.id().event_counter) + .write_u16(event.id().actor) + .write_u64(event.parent_id().map(|id| id.event_counter).unwrap_or(0)) + .write_u16(event.parent_id().map(|id| id.actor).unwrap_or(0)) + .write_u64(time::millis_since_epoch(event.timestamp())) + .write_string(event.namespace()) + .write_u32(event.data_len()) + .finish() +} diff --git a/flo-protocol/src/messages/set_event_stream.rs b/flo-protocol/src/messages/set_event_stream.rs new file mode 100644 index 0000000..d77acb7 --- /dev/null +++ b/flo-protocol/src/messages/set_event_stream.rs @@ -0,0 +1,36 @@ + +use nom::be_u32; +use serializer::Serializer; +use event::OwnedFloEvent; +use super::{ProtocolMessage, parse_str}; + +pub const SET_EVENT_STREAM: u8 = 18; + +/// Sent by a client to tell the server which event stream to use for all future operations +#[derive(Debug, PartialEq, Clone)] +pub struct SetEventStream{ + pub op_id: u32, + pub name: String, +} + +named!{pub parse_set_event_stream>, + chain!( + _tag: tag!(&[SET_EVENT_STREAM]) ~ + op_id: be_u32 ~ + name: parse_str, + || { + ProtocolMessage::SetEventStream(SetEventStream { + op_id: op_id, + name: name, + }) + } + ) +} + +pub fn serialize_set_event_stream(set_stream: &SetEventStream, buf: &mut [u8]) -> usize { + Serializer::new(buf) + .write_u8(SET_EVENT_STREAM) + .write_u32(set_stream.op_id) + .write_string(&set_stream.name) + .finish() +} diff --git a/flo-protocol/src/serializer.rs b/flo-protocol/src/serializer.rs index 4d110c1..a4c400f 100644 --- a/flo-protocol/src/serializer.rs +++ b/flo-protocol/src/serializer.rs @@ -1,3 +1,4 @@ +use std::net::{SocketAddr, SocketAddrV4}; use byteorder::{ByteOrder, BigEndian}; pub struct Serializer<'a> { @@ -73,6 +74,19 @@ impl <'a> Serializer<'a> { serializer } + pub fn write_socket_addr(mut self, addr: SocketAddr) -> Self { + match addr { + SocketAddr::V4(v4) => { + self.write_u8(4u8) + .write_bytes(v4.ip().octets()) + .write_u16(v4.port()) + } + SocketAddr::V6(v6) => { + unimplemented!() + } + } + } + pub fn finish(self) -> usize { self.position } From d291cc1e96c09d71b346ae42aecb686c0f7e4733 Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 28 Nov 2017 00:20:41 -0500 Subject: [PATCH 10/73] cleanup compiler warnings in protocol and add tests for serializing socket addresses --- flo-protocol/src/messages/consume_start.rs | 1 - flo-protocol/src/messages/mod.rs | 2 +- flo-protocol/src/messages/peer_announce.rs | 3 +- flo-protocol/src/messages/produce_event.rs | 2 +- flo-protocol/src/serializer.rs | 36 ++++++++++++++++++++-- 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/flo-protocol/src/messages/consume_start.rs b/flo-protocol/src/messages/consume_start.rs index 2ee8d42..ab38984 100644 --- a/flo-protocol/src/messages/consume_start.rs +++ b/flo-protocol/src/messages/consume_start.rs @@ -8,7 +8,6 @@ use super::{ProtocolMessage, parse_str, parse_version_vec}; pub const NEW_START_CONSUMING: u8 = 17; -pub const CONSUME_UNLIMITED: u64 = 0; /// New message sent from client to server to begin reading events from the stream #[derive(Debug, PartialEq, Clone)] diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index 3d0ab1a..3eb531b 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -22,7 +22,7 @@ mod set_event_stream; mod receive_event; use nom::{be_u64, be_u32, be_u16, be_u8}; -use event::{time, OwnedFloEvent, FloEvent, FloEventId, ActorId, EventCounter, Timestamp}; +use event::{time, OwnedFloEvent, FloEvent, FloEventId, Timestamp}; use serializer::Serializer; use std::net::SocketAddr; diff --git a/flo-protocol/src/messages/peer_announce.rs b/flo-protocol/src/messages/peer_announce.rs index 097c390..d4e3c1e 100644 --- a/flo-protocol/src/messages/peer_announce.rs +++ b/flo-protocol/src/messages/peer_announce.rs @@ -34,7 +34,8 @@ named!{pub parse_peer_announce>, pub fn serialize_peer_announce(announce: &PeerAnnounce, buf: &mut [u8]) -> usize { Serializer::new(buf) .write_u8(PEER_ANNOUNCE) - .write_socket_addr(announce.peer_address) + .write_u32(announce.protocol_version) .write_u32(announce.op_id) + .write_socket_addr(announce.peer_address) .finish() } diff --git a/flo-protocol/src/messages/produce_event.rs b/flo-protocol/src/messages/produce_event.rs index 1c13ca8..c5cd6a6 100644 --- a/flo-protocol/src/messages/produce_event.rs +++ b/flo-protocol/src/messages/produce_event.rs @@ -1,5 +1,5 @@ -use nom::{be_u64, be_u32, be_u16}; +use nom::{be_u32, be_u16}; use event::{OwnedFloEvent, FloEventId, ActorId}; use serializer::Serializer; use super::{ProtocolMessage, parse_str, parse_event_id}; diff --git a/flo-protocol/src/serializer.rs b/flo-protocol/src/serializer.rs index a4c400f..a10dad9 100644 --- a/flo-protocol/src/serializer.rs +++ b/flo-protocol/src/serializer.rs @@ -1,4 +1,4 @@ -use std::net::{SocketAddr, SocketAddrV4}; +use std::net::SocketAddr; use byteorder::{ByteOrder, BigEndian}; pub struct Serializer<'a> { @@ -74,7 +74,7 @@ impl <'a> Serializer<'a> { serializer } - pub fn write_socket_addr(mut self, addr: SocketAddr) -> Self { + pub fn write_socket_addr(self, addr: SocketAddr) -> Self { match addr { SocketAddr::V4(v4) => { self.write_u8(4u8) @@ -82,7 +82,10 @@ impl <'a> Serializer<'a> { .write_u16(v4.port()) } SocketAddr::V6(v6) => { - unimplemented!() + // TODO: would it be correct to also serialize flowinfo and scope_id for ipv6 addresses? + self.write_u8(6) + .write_bytes(v6.ip().octets()) + .write_u16(v6.port()) } } } @@ -182,4 +185,31 @@ mod test { let result = subject.write_u16(987).write_u64(23).write_string("bacon").finish(); assert_eq!(17, result); } + + fn address(addr: &str) -> SocketAddr { + ::std::str::FromStr::from_str(addr).unwrap() + } + + #[test] + fn ipv4_socket_address_is_written() { + let mut buffer = [0; 64]; + let result = { + let subject = Serializer::new(&mut buffer[..]); + subject.write_socket_addr(address("123.45.67.8:80")).finish() + }; + let expected = [4, 123, 45, 67, 8, 0, 80]; + assert_eq!(&expected, &buffer[..result]); + } + + #[test] + fn ipv6_address_is_written() { + let mut buffer = [0; 64]; + let result = { + let subject = Serializer::new(&mut buffer[..]); + let addr = address("[2001:db8::1]:8080"); + subject.write_socket_addr(addr).finish() + }; + let expected = [6, 32,1, 13,184, 0,0, 0,0, 0,0, 0,0, 0,0, 0,1, 31,144]; + assert_eq!(&expected, &buffer[..result]); + } } From abce6836f379a6e073eccc229c47d971942666ef Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 28 Nov 2017 19:16:07 -0500 Subject: [PATCH 11/73] remove unused SetBatchSize protocol message --- flo-protocol/src/messages/mod.rs | 21 --------------------- flo-server/src/embedded/mod.rs | 1 - 2 files changed, 22 deletions(-) diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index 3eb531b..2d791c0 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -85,8 +85,6 @@ pub enum ProtocolMessage { CursorCreated(CursorInfo), /// sent by a client to a server to tell the server to stop sending events. This is required in order to reuse the connection for multiple queries StopConsuming(u32), - /// Sent by the client to set the batch size to use for consuming. It is an error to send this message while consuming. - SetBatchSize(u32), /// Sent by the client to tell the server that it is ready for the next batch NextBatch, /// Sent by the server to notify a consumer that it has reached the end of a batch and that more events can be sent @@ -173,14 +171,6 @@ named!{parse_version_vec>, named!{parse_awaiting_events>, map!(tag!(&[AWAITING_EVENTS]), |_| {ProtocolMessage::AwaitingEvents})} -named!{parse_set_batch_size>, chain!( - _tag: tag!(&[SET_BATCH_SIZE]) ~ - batch_size: be_u32, - || { - ProtocolMessage::SetBatchSize(batch_size) - } -)} - named!{parse_next_batch>, map!(tag!(&[NEXT_BATCH]), |_| {ProtocolMessage::NextBatch})} named!{parse_end_of_batch>, map!(tag!(&[END_OF_BATCH]), |_| {ProtocolMessage::EndOfBatch})} @@ -200,7 +190,6 @@ named!{pub parse_any>, alt!( parse_error_message | parse_awaiting_events | parse_new_producer_event | - parse_set_batch_size | parse_next_batch | parse_end_of_batch | parse_stop_consuming | @@ -256,11 +245,6 @@ impl ProtocolMessage { ProtocolMessage::Error(ref err_message) => { serialize_error_message(err_message, buf) } - ProtocolMessage::SetBatchSize(batch_size) => { - Serializer::new(buf).write_u8(SET_BATCH_SIZE) - .write_u32(batch_size) - .finish() - } ProtocolMessage::NextBatch => { buf[0] = NEXT_BATCH; 1 @@ -467,11 +451,6 @@ mod test { test_serialize_then_deserialize(&ProtocolMessage::EndOfBatch); } - #[test] - fn set_batch_size_is_serialized_and_parsed() { - test_serialize_then_deserialize(&ProtocolMessage::SetBatchSize(1234567)); - } - #[test] fn awaiting_events_message_is_serialized_and_parsed() { test_serialize_then_deserialize(&mut ProtocolMessage::AwaitingEvents); diff --git a/flo-server/src/embedded/mod.rs b/flo-server/src/embedded/mod.rs index efe418b..ce3cc08 100644 --- a/flo-server/src/embedded/mod.rs +++ b/flo-server/src/embedded/mod.rs @@ -61,7 +61,6 @@ fn message_to_owned(server_msg: SendProtocolMessage) -> ClientProtocolMessage { ProtocolMessage::ProduceEvent(op) => ProtocolMessage::ProduceEvent(op), ProtocolMessage::NextBatch => ProtocolMessage::NextBatch, ProtocolMessage::EndOfBatch => ProtocolMessage::EndOfBatch, - ProtocolMessage::SetBatchSize(op) => ProtocolMessage::SetBatchSize(op), ProtocolMessage::NewStartConsuming(op) => ProtocolMessage::NewStartConsuming(op), ProtocolMessage::CursorCreated(op) => ProtocolMessage::CursorCreated(op), ProtocolMessage::Announce(op) => ProtocolMessage::Announce(op), From caee3637a7436b84682f97fbd2b3c1e9cd2b948e Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 2 Dec 2017 10:38:12 -0500 Subject: [PATCH 12/73] protocol level implementation of append entries --- Cargo.lock | 2 + flo-protocol/Cargo.toml | 2 + flo-protocol/src/lib.rs | 2 + flo-protocol/src/messages/append_entries.rs | 120 +++++++++++++++++ flo-protocol/src/messages/consume_start.rs | 1 + flo-protocol/src/messages/flo_instance_id.rs | 45 +++++++ flo-protocol/src/messages/mod.rs | 123 ++++++++++++++---- flo-protocol/src/serializer.rs | 9 +- .../engine/system_stream/flo_instance_id.rs | 22 ---- .../engine/system_stream/replication_plan.rs | 19 --- 10 files changed, 277 insertions(+), 68 deletions(-) create mode 100644 flo-protocol/src/messages/append_entries.rs create mode 100644 flo-protocol/src/messages/flo_instance_id.rs delete mode 100644 flo-server/src/engine/system_stream/flo_instance_id.rs delete mode 100644 flo-server/src/engine/system_stream/replication_plan.rs diff --git a/Cargo.lock b/Cargo.lock index 60501c6..0bbe0d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -249,9 +249,11 @@ name = "flo-protocol" version = "0.2.0" dependencies = [ "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)", "flo-event 0.2.0", "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/flo-protocol/Cargo.toml b/flo-protocol/Cargo.toml index 9e34579..3fd224b 100644 --- a/flo-protocol/Cargo.toml +++ b/flo-protocol/Cargo.toml @@ -8,4 +8,6 @@ flo-event = { path = "../flo-event" } log = "0.3" nom = "2.2.1" byteorder = "1" +chrono = "^0.2" +rand = "0.3" diff --git a/flo-protocol/src/lib.rs b/flo-protocol/src/lib.rs index 6bcc9d9..50a5480 100644 --- a/flo-protocol/src/lib.rs +++ b/flo-protocol/src/lib.rs @@ -6,6 +6,8 @@ extern crate log; extern crate flo_event as event; extern crate byteorder; +extern crate chrono; +extern crate rand; pub mod serializer; mod messages; diff --git a/flo-protocol/src/messages/append_entries.rs b/flo-protocol/src/messages/append_entries.rs new file mode 100644 index 0000000..14a6f29 --- /dev/null +++ b/flo-protocol/src/messages/append_entries.rs @@ -0,0 +1,120 @@ + +use std::net::SocketAddr; + +use nom::{be_u64, be_u32, be_u8}; +use serializer::Serializer; +use event::{EventCounter, OwnedFloEvent}; +use super::{FloInstanceId, ProtocolMessage}; +use super::flo_instance_id::parse_flo_instance_id; + +pub type Term = u64; + +pub const APPEND_ENTRIES_CALL_HEADER: u8 = 30; +pub const APPEND_ENTRIES_RESPONSE_HEADER: u8 = 31; + +#[derive(Debug, PartialEq, Clone)] +pub struct FloServer { + pub id: FloInstanceId, + pub address: SocketAddr, +} + +#[derive(Debug, PartialEq, Clone)] +pub struct AppendEntriesCall { + pub op_id: u32, + pub leader_id: FloInstanceId, + pub term: Term, + pub prev_entry_term: Term, + pub prev_entry_index: EventCounter, + pub leader_commit_index: EventCounter, + pub entry_count: u32, +} + +impl AppendEntriesCall { + pub fn heartbeat(op_id: u32, + leader_id: FloInstanceId, + term: Term, + prev_entry_term: Term, + prev_entry_index: EventCounter, + leader_commit_index: EventCounter) -> AppendEntriesCall { + + AppendEntriesCall { + op_id, + leader_id, + term, + prev_entry_term, + prev_entry_index, + leader_commit_index, + entry_count: 0, + } + } + + pub fn is_heartbeat(&self) -> bool { + self.entry_count == 0 + } +} + + +#[derive(Debug, PartialEq, Clone)] +pub struct AppendEntriesResponse { + pub op_id: u32, + pub from_peer: FloInstanceId, + pub term: Term, + pub success: bool, +} + +pub fn serialize_append_response(response: &AppendEntriesResponse, buf: &mut [u8]) -> usize { + Serializer::new(buf) + .write_u8(APPEND_ENTRIES_RESPONSE_HEADER) + .write_u32(response.op_id) + .write(&response.from_peer) + .write_u64(response.term) + .write_bool(response.success) + .finish() +} + +named!{pub parse_append_entries_response>, do_parse!( + tag!(&[APPEND_ENTRIES_RESPONSE_HEADER]) >> + op_id: be_u32 >> + from_peer: parse_flo_instance_id >> + term: be_u64 >> + success: map!(be_u8, |val| { val == 1 }) >> + + ( ProtocolMessage::SystemAppendResponse(AppendEntriesResponse { + op_id, from_peer, term, success, + }) ) +)} + +pub fn serialize_append_entries(append: &AppendEntriesCall, buf: &mut [u8]) -> usize { + Serializer::new(buf) + .write_u8(APPEND_ENTRIES_CALL_HEADER) + .write_u32(append.op_id) + .write(&append.leader_id) + .write_u64(append.term) + .write_u64(append.prev_entry_term) + .write_u64(append.prev_entry_index) + .write_u64(append.leader_commit_index) + .write_u32(append.entry_count) + .finish() +} + + +named!{pub parse_append_entries_call>, do_parse!( + tag!(&[APPEND_ENTRIES_CALL_HEADER]) >> + op_id: be_u32 >> + leader_id: parse_flo_instance_id >> + term: be_u64 >> + prev_entry_term: be_u64 >> + prev_entry_index: be_u64 >> + leader_commit_index: be_u64 >> + entry_count: be_u32 >> + + ( ProtocolMessage::SystemAppendCall(AppendEntriesCall { + op_id, + leader_id, + term, + prev_entry_term, + prev_entry_index, + leader_commit_index, + entry_count + }) ) +)} diff --git a/flo-protocol/src/messages/consume_start.rs b/flo-protocol/src/messages/consume_start.rs index ab38984..2ee8d42 100644 --- a/flo-protocol/src/messages/consume_start.rs +++ b/flo-protocol/src/messages/consume_start.rs @@ -8,6 +8,7 @@ use super::{ProtocolMessage, parse_str, parse_version_vec}; pub const NEW_START_CONSUMING: u8 = 17; +pub const CONSUME_UNLIMITED: u64 = 0; /// New message sent from client to server to begin reading events from the stream #[derive(Debug, PartialEq, Clone)] diff --git a/flo-protocol/src/messages/flo_instance_id.rs b/flo-protocol/src/messages/flo_instance_id.rs new file mode 100644 index 0000000..3eb846a --- /dev/null +++ b/flo-protocol/src/messages/flo_instance_id.rs @@ -0,0 +1,45 @@ + +use std::fmt::{self, Display}; +use serializer::{Serializer, FloSerialize}; + +/// An opaque identifier used to uniquely identify flo instances. +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub struct FloInstanceId(u64); + +impl FloInstanceId { + + pub fn generate_new() -> FloInstanceId { + FloInstanceId(::rand::random()) + } + + pub fn as_bytes(&self) -> [u8; 8] { + use ::byteorder::{ByteOrder, BigEndian}; + + let mut bytes = [0; 8]; + BigEndian::write_u64(&mut bytes[..], self.0); + bytes + } + + pub fn from_bytes(bytes: &[u8]) -> FloInstanceId { + use ::byteorder::{ByteOrder, BigEndian}; + + let b = &bytes[0..8]; + let num = BigEndian::read_u64(b); + FloInstanceId(num) + } +} + +impl Display for FloInstanceId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "FloInstanceId({})", self.0) + } +} + + +impl FloSerialize for FloInstanceId { + fn serialize<'a>(&'a self, serializer: Serializer<'a>) -> Serializer<'a> { + serializer.write_u64(self.0) + } +} + +named!{pub parse_flo_instance_id, map!(::nom::be_u64, |val| {FloInstanceId(val) } )} diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index 2d791c0..5d0f14d 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -20,6 +20,8 @@ mod cursor_info; mod event_stream_status; mod set_event_stream; mod receive_event; +mod append_entries; +mod flo_instance_id; use nom::{be_u64, be_u32, be_u16, be_u8}; use event::{time, OwnedFloEvent, FloEvent, FloEventId, Timestamp}; @@ -36,16 +38,19 @@ use self::cursor_info::{parse_cursor_created, serialize_cursor_created}; use self::event_stream_status::{parse_event_stream_status, serialize_event_stream_status}; use self::set_event_stream::{serialize_set_event_stream, parse_set_event_stream}; use self::receive_event::{serialize_receive_event_header, parse_receive_event_header}; +use self::append_entries::{serialize_append_entries, parse_append_entries_call, serialize_append_response, parse_append_entries_response}; pub use self::client_announce::ClientAnnounce; pub use self::peer_announce::PeerAnnounce; pub use self::error::{ErrorMessage, ErrorKind}; pub use self::produce_event::ProduceEvent; pub use self::event_ack::EventAck; -pub use self::consume_start::NewConsumerStart; +pub use self::consume_start::{NewConsumerStart, CONSUME_UNLIMITED}; pub use self::cursor_info::CursorInfo; pub use self::event_stream_status::{EventStreamStatus, PartitionStatus}; pub use self::set_event_stream::SetEventStream; +pub use self::append_entries::{AppendEntriesCall, AppendEntriesResponse}; +pub use self::flo_instance_id::FloInstanceId; pub mod headers { pub const CLIENT_AUTH: u8 = 1; @@ -65,37 +70,39 @@ use self::headers::*; /// Defines all the distinct messages that can be sent over the wire between client and server. #[derive(Debug, PartialEq, Clone)] pub enum ProtocolMessage { - /// Always the first message sent by the client to the server - Announce(ClientAnnounce), - /// Always the first message sent by a server to another server - PeerAnnounce(PeerAnnounce), - /// Sent in response to an Announce message. Contains basic information about the status of an event stream - StreamStatus(EventStreamStatus), - /// Set the event stream that the client will work with - SetEventStream(SetEventStream), - /// Signals a client's intent to publish a new event. The server will respond with either an `EventAck` or an `ErrorMessage` - ProduceEvent(ProduceEvent), - /// This is a complete event as serialized over the wire. This message is sent to to both consumers as well as other servers - ReceiveEvent(E), /// Sent from the server to client to acknowledge that an event was persisted successfully. AckEvent(EventAck), - /// New message sent by a client to start reading events from the stream - NewStartConsuming(NewConsumerStart), + /// Sent by the server to an active consumer to indicate that it has reached the end of the stream. The server will + /// continue to send events as more come in, but this just lets the client know that it may be some time before more + /// events are available. This message will only be sent at most once to a given consumer. + AwaitingEvents, + /// Always the first message sent by the client to the server + Announce(ClientAnnounce), /// send by the server to a client in response to a StartConsuming message to indicate the start of a series of events CursorCreated(CursorInfo), - /// sent by a client to a server to tell the server to stop sending events. This is required in order to reuse the connection for multiple queries - StopConsuming(u32), - /// Sent by the client to tell the server that it is ready for the next batch - NextBatch, /// Sent by the server to notify a consumer that it has reached the end of a batch and that more events can be sent /// upon receipt of a `NextBatch` message by the server. EndOfBatch, - /// Sent by the server to an active consumer to indicate that it has reached the end of the stream. The server will - /// continue to send events as more come in, but this just lets the client know that it may be some time before more - /// events are available. This message will only be sent at most once to a given consumer. - AwaitingEvents, /// Represents an error response to any other message Error(ErrorMessage), + /// New message sent by a client to start reading events from the stream + NewStartConsuming(NewConsumerStart), + /// Sent by the client to tell the server that it is ready for the next batch + NextBatch, + /// Signals a client's intent to publish a new event. The server will respond with either an `EventAck` or an `ErrorMessage` + ProduceEvent(ProduceEvent), + /// Always the first message sent by a server to another server + PeerAnnounce(PeerAnnounce), + /// This is a complete event as serialized over the wire. This message is sent to to both consumers as well as other servers + ReceiveEvent(E), + /// Set the event stream that the client will work with + SetEventStream(SetEventStream), + /// Sent in response to an Announce message. Contains basic information about the status of an event stream + StreamStatus(EventStreamStatus), + /// sent by a client to a server to tell the server to stop sending events. This is required in order to reuse the connection for multiple queries + StopConsuming(u32), + SystemAppendCall(AppendEntriesCall), + SystemAppendResponse(AppendEntriesResponse), } named!{parse_str, @@ -109,7 +116,7 @@ named!{parse_str, -named!{parse_socket_addr, alt!(parse_socket_addr_v4)} +named!{parse_socket_addr, alt!(parse_socket_addr_v4 | parse_socket_addr_v6)} named!{parse_socket_addr_v4, chain!( @@ -127,6 +134,25 @@ named!{parse_socket_addr_v4, ) } +named!{parse_socket_addr_v6, + chain!( + _tag: tag!(&[6u8]) ~ + a: be_u16 ~ + b: be_u16 ~ + c: be_u16 ~ + d: be_u16 ~ + e: be_u16 ~ + f: be_u16 ~ + g: be_u16 ~ + h: be_u16 ~ + port: be_u16, + || { + let ip = ::std::net::Ipv6Addr::new(a, b, c, d, e, f, g, h); + ::std::net::SocketAddrV6::new(ip, port, 0, 0).into() + } + ) +} + fn require_event_id(id: Option) -> Result { id.ok_or("EventId must not be all zeros") } @@ -198,7 +224,9 @@ named!{pub parse_any>, alt!( parse_set_event_stream | parse_event_stream_status | parse_client_announce | - parse_peer_announce + parse_peer_announce | + parse_append_entries_call | + parse_append_entries_response )} @@ -253,6 +281,12 @@ impl ProtocolMessage { buf[0] = END_OF_BATCH; 1 } + ProtocolMessage::SystemAppendCall(ref append) => { + serialize_append_entries(append, buf) + } + ProtocolMessage::SystemAppendResponse(ref response) => { + serialize_append_response(response, buf) + } } } @@ -326,6 +360,32 @@ mod test { } + #[test] + fn serde_append_entries_response() { + let response = AppendEntriesResponse { + op_id: 45, + from_peer: FloInstanceId::generate_new(), + term: 345, + success: false, + }; + test_serialize_then_deserialize(&ProtocolMessage::SystemAppendResponse(response)); + } + + #[test] + fn serde_append_entries_call() { + let append = AppendEntriesCall { + op_id: 345, + leader_id: FloInstanceId::generate_new(), + term: 987, + prev_entry_term: 986, + prev_entry_index: 134, + leader_commit_index: 131, + entry_count: 567, + }; + + test_serialize_then_deserialize(&ProtocolMessage::SystemAppendCall(append)); + } + #[test] fn serde_client_announce() { let announce = ClientAnnounce { @@ -338,7 +398,18 @@ mod test { } #[test] - fn serde_peer_announce() { + fn serde_peer_announce_with_ipv6_address() { + let addr = ::std::str::FromStr::from_str("[1:3:5::2]:4321").unwrap(); + let announce = PeerAnnounce { + protocol_version: 9, + peer_address: addr, + op_id: 6543, + }; + test_serialize_then_deserialize(&ProtocolMessage::PeerAnnounce(announce)); + } + + #[test] + fn serde_peer_announce_with_ipv4_address() { let addr = ::std::str::FromStr::from_str("123.234.12.1:4321").unwrap(); let announce = PeerAnnounce { protocol_version: 9, diff --git a/flo-protocol/src/serializer.rs b/flo-protocol/src/serializer.rs index a10dad9..c06c08a 100644 --- a/flo-protocol/src/serializer.rs +++ b/flo-protocol/src/serializer.rs @@ -1,6 +1,10 @@ use std::net::SocketAddr; use byteorder::{ByteOrder, BigEndian}; +pub trait FloSerialize { + fn serialize<'a>(&'a self, serializer: Serializer<'a>) -> Serializer<'a>; +} + pub struct Serializer<'a> { buffer: &'a mut [u8], position: usize, @@ -15,6 +19,10 @@ impl <'a> Serializer<'a> { } } + pub fn write(self, value: &'a T) -> Self { + value.serialize(self) + } + pub fn write_bool(self, value: bool) -> Self { let b = if value { 1 } else { 0 }; self.write_u8(b) @@ -82,7 +90,6 @@ impl <'a> Serializer<'a> { .write_u16(v4.port()) } SocketAddr::V6(v6) => { - // TODO: would it be correct to also serialize flowinfo and scope_id for ipv6 addresses? self.write_u8(6) .write_bytes(v6.ip().octets()) .write_u16(v6.port()) diff --git a/flo-server/src/engine/system_stream/flo_instance_id.rs b/flo-server/src/engine/system_stream/flo_instance_id.rs deleted file mode 100644 index 1cbbe17..0000000 --- a/flo-server/src/engine/system_stream/flo_instance_id.rs +++ /dev/null @@ -1,22 +0,0 @@ - -use std::net::SocketAddr; -use std::fmt::{self, Display}; - - -/// An opaque identifier used to uniquely identify flo instances. -#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] -pub struct FloInstanceId(SocketAddr); - -impl FloInstanceId { - - pub fn new(addr: SocketAddr) -> Self { - FloInstanceId(addr) - } -} - -impl Display for FloInstanceId { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // Display will just look the same as Debug for now - write!(f, "{}", self.0) - } -} diff --git a/flo-server/src/engine/system_stream/replication_plan.rs b/flo-server/src/engine/system_stream/replication_plan.rs deleted file mode 100644 index 8a5fc07..0000000 --- a/flo-server/src/engine/system_stream/replication_plan.rs +++ /dev/null @@ -1,19 +0,0 @@ - -use super::FloInstanceId; - -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct SystemReplicationPlan { - pub streams: Vec, -} - -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct StreamReplicationPlan { - pub name: String, - pub partitions: Vec, -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub struct PartitionReplicationPlan { - pub primary: FloInstanceId, - // TODO: Add ability to limit replication to only a subset of secondary nodes to reduce network traffic -} From e4c410684468b463e9382160c0ba93d068f492ef Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 2 Dec 2017 11:21:41 -0500 Subject: [PATCH 13/73] add request vote messages to protocol --- flo-protocol/src/messages/append_entries.rs | 3 +- flo-protocol/src/messages/mod.rs | 40 ++++++++++- flo-protocol/src/messages/request_vote.rs | 75 +++++++++++++++++++++ 3 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 flo-protocol/src/messages/request_vote.rs diff --git a/flo-protocol/src/messages/append_entries.rs b/flo-protocol/src/messages/append_entries.rs index 14a6f29..e180574 100644 --- a/flo-protocol/src/messages/append_entries.rs +++ b/flo-protocol/src/messages/append_entries.rs @@ -4,10 +4,9 @@ use std::net::SocketAddr; use nom::{be_u64, be_u32, be_u8}; use serializer::Serializer; use event::{EventCounter, OwnedFloEvent}; -use super::{FloInstanceId, ProtocolMessage}; +use super::{FloInstanceId, ProtocolMessage, Term}; use super::flo_instance_id::parse_flo_instance_id; -pub type Term = u64; pub const APPEND_ENTRIES_CALL_HEADER: u8 = 30; pub const APPEND_ENTRIES_RESPONSE_HEADER: u8 = 31; diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index 5d0f14d..de9c33d 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -21,6 +21,7 @@ mod event_stream_status; mod set_event_stream; mod receive_event; mod append_entries; +mod request_vote; mod flo_instance_id; use nom::{be_u64, be_u32, be_u16, be_u8}; @@ -39,6 +40,7 @@ use self::event_stream_status::{parse_event_stream_status, serialize_event_strea use self::set_event_stream::{serialize_set_event_stream, parse_set_event_stream}; use self::receive_event::{serialize_receive_event_header, parse_receive_event_header}; use self::append_entries::{serialize_append_entries, parse_append_entries_call, serialize_append_response, parse_append_entries_response}; +use self::request_vote::{serialize_request_vote_call, parse_request_vote_call, serialize_request_vote_response, parse_request_vote_response}; pub use self::client_announce::ClientAnnounce; pub use self::peer_announce::PeerAnnounce; @@ -50,8 +52,11 @@ pub use self::cursor_info::CursorInfo; pub use self::event_stream_status::{EventStreamStatus, PartitionStatus}; pub use self::set_event_stream::SetEventStream; pub use self::append_entries::{AppendEntriesCall, AppendEntriesResponse}; +pub use self::request_vote::{RequestVoteCall, RequestVoteResponse}; pub use self::flo_instance_id::FloInstanceId; +pub type Term = u64; + pub mod headers { pub const CLIENT_AUTH: u8 = 1; pub const UPDATE_MARKER: u8 = 4; @@ -103,6 +108,8 @@ pub enum ProtocolMessage { StopConsuming(u32), SystemAppendCall(AppendEntriesCall), SystemAppendResponse(AppendEntriesResponse), + RequestVote(RequestVoteCall), + VoteResponse(RequestVoteResponse), } named!{parse_str, @@ -194,6 +201,7 @@ named!{parse_version_vec>, length_count!(be_u16, parse_zeroable_event_id) } +named!{parse_bool, map!(::nom::be_u8, |val| { val == 1 } )} named!{parse_awaiting_events>, map!(tag!(&[AWAITING_EVENTS]), |_| {ProtocolMessage::AwaitingEvents})} @@ -226,7 +234,9 @@ named!{pub parse_any>, alt!( parse_client_announce | parse_peer_announce | parse_append_entries_call | - parse_append_entries_response + parse_append_entries_response | + parse_request_vote_call | + parse_request_vote_response )} @@ -287,6 +297,12 @@ impl ProtocolMessage { ProtocolMessage::SystemAppendResponse(ref response) => { serialize_append_response(response, buf) } + ProtocolMessage::RequestVote(ref request) => { + serialize_request_vote_call(request, buf) + } + ProtocolMessage::VoteResponse(ref response) => { + serialize_request_vote_response(response, buf) + } } } @@ -360,6 +376,28 @@ mod test { } + #[test] + fn serde_request_vote_response() { + let response = RequestVoteResponse { + op_id: 456, + term: 577, + vote_granted: true, + }; + test_serialize_then_deserialize(&ProtocolMessage::VoteResponse(response)); + } + + #[test] + fn serde_request_vote_call() { + let request = RequestVoteCall { + op_id: 3, + term: 4, + candidate_id: FloInstanceId::generate_new(), + last_log_index: 567, + last_log_term: 8910, + }; + test_serialize_then_deserialize(&ProtocolMessage::RequestVote(request)); + } + #[test] fn serde_append_entries_response() { let response = AppendEntriesResponse { diff --git a/flo-protocol/src/messages/request_vote.rs b/flo-protocol/src/messages/request_vote.rs new file mode 100644 index 0000000..33de0e7 --- /dev/null +++ b/flo-protocol/src/messages/request_vote.rs @@ -0,0 +1,75 @@ +use nom::{be_u32, be_u64}; + +use event::{EventCounter, OwnedFloEvent}; +use super::{Term, FloInstanceId, ProtocolMessage, parse_bool}; +use super::flo_instance_id::parse_flo_instance_id; +use serializer::Serializer; + + +pub const REQUEST_VOTE_CALL_HEADER: u8 = 32; +pub const REQUEST_VOTE_RESPONSE_HEADER: u8 = 33; + +#[derive(Debug, Clone, PartialEq)] +pub struct RequestVoteCall { + pub op_id: u32, + pub term: Term, + pub candidate_id: FloInstanceId, + pub last_log_index: EventCounter, + pub last_log_term: Term, +} + + +#[derive(Debug, Clone, PartialEq)] +pub struct RequestVoteResponse { + pub op_id: u32, + pub term: Term, + pub vote_granted: bool, +} + +pub fn serialize_request_vote_response(response: &RequestVoteResponse, buf: &mut [u8]) -> usize { + Serializer::new(buf) + .write_u8(REQUEST_VOTE_RESPONSE_HEADER) + .write_u32(response.op_id) + .write_u64(response.term) + .write_bool(response.vote_granted) + .finish() +} + +named!{pub parse_request_vote_response>, do_parse!( + tag!(&[REQUEST_VOTE_RESPONSE_HEADER]) >> + op_id: be_u32 >> + term: be_u64 >> + vote_granted: parse_bool >> + + ( ProtocolMessage::VoteResponse(RequestVoteResponse { + op_id, term, vote_granted, + }) ) +)} + +pub fn serialize_request_vote_call(req: &RequestVoteCall, buf: &mut [u8]) -> usize { + Serializer::new(buf) + .write_u8(REQUEST_VOTE_CALL_HEADER) + .write_u32(req.op_id) + .write_u64(req.term) + .write(&req.candidate_id) + .write_u64(req.last_log_index) + .write_u64(req.last_log_term) + .finish() +} + +named!{pub parse_request_vote_call>, do_parse!( + tag!(&[REQUEST_VOTE_CALL_HEADER]) >> + op_id: be_u32 >> + term: be_u64 >> + candidate_id: parse_flo_instance_id >> + last_log_index: be_u64 >> + last_log_term: be_u64 >> + + ( ProtocolMessage::RequestVote(RequestVoteCall{ + op_id, + term, + candidate_id, + last_log_index, + last_log_term, + }) ) +)} From 092a1a91c0cd4b908d34e9b5633d815bec722b07 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 3 Dec 2017 12:19:26 -0500 Subject: [PATCH 14/73] keep a reference to the socket address of the current primary server --- .../src/async/current_stream_state.rs | 5 +- .../src/messages/event_stream_status.rs | 51 +++++-- flo-protocol/src/messages/mod.rs | 9 +- flo-protocol/src/messages/peer_announce.rs | 2 + flo-server/src/embedded/mod.rs | 4 + .../connection_handler/connection_state.rs | 5 +- .../src/engine/connection_handler/mod.rs | 15 +- .../src/engine/connection_handler/peer.rs | 21 +-- .../engine/controller/cluster_state/mod.rs | 5 +- .../src/engine/controller/initialization.rs | 13 +- flo-server/src/engine/controller/mod.rs | 18 ++- .../src/engine/controller/outgoing_io.rs | 118 ---------------- .../engine/controller/peer_connection/mod.rs | 41 ++++++ .../controller/peer_connection/replication.rs | 15 ++ .../controller/peer_connection/system.rs | 129 ++++++++++++++++++ .../src/engine/event_stream/partition/mod.rs | 39 ++++-- .../src/engine/event_stream/partition/ops.rs | 8 +- flo-server/src/engine/mod.rs | 12 +- flo-server/src/engine/system_stream/mod.rs | 112 --------------- flo-server/src/server/server_options.rs | 2 - 20 files changed, 344 insertions(+), 280 deletions(-) delete mode 100644 flo-server/src/engine/controller/outgoing_io.rs create mode 100644 flo-server/src/engine/controller/peer_connection/mod.rs create mode 100644 flo-server/src/engine/controller/peer_connection/replication.rs create mode 100644 flo-server/src/engine/controller/peer_connection/system.rs delete mode 100644 flo-server/src/engine/system_stream/mod.rs diff --git a/flo-client-lib/src/async/current_stream_state.rs b/flo-client-lib/src/async/current_stream_state.rs index 46ebd05..9024e05 100644 --- a/flo-client-lib/src/async/current_stream_state.rs +++ b/flo-client-lib/src/async/current_stream_state.rs @@ -1,3 +1,4 @@ +use std::net::SocketAddr; use event::{ActorId, EventCounter}; @@ -6,6 +7,7 @@ pub struct PartitionState { pub partition_num: ActorId, pub head: EventCounter, pub writable: bool, + pub primary_server_addr: Option, } #[derive(Debug, PartialEq, Clone)] @@ -16,12 +18,13 @@ pub struct CurrentStreamState { impl From<::protocol::PartitionStatus> for PartitionState { fn from(status: ::protocol::PartitionStatus) -> Self { - let ::protocol::PartitionStatus { partition_num, head, primary } = status; + let ::protocol::PartitionStatus { partition_num, head, primary, primary_server_address } = status; PartitionState { partition_num: partition_num, head: head, writable: primary, + primary_server_addr: primary_server_address, } } } diff --git a/flo-protocol/src/messages/event_stream_status.rs b/flo-protocol/src/messages/event_stream_status.rs index 954af8f..5f30a5a 100644 --- a/flo-protocol/src/messages/event_stream_status.rs +++ b/flo-protocol/src/messages/event_stream_status.rs @@ -1,8 +1,10 @@ +use std::net::SocketAddr; + use nom::{be_u64, be_u32, be_u16}; use event::{EventCounter, ActorId, OwnedFloEvent}; -use serializer::Serializer; -use super::{parse_str, ProtocolMessage}; +use serializer::{Serializer, FloSerialize}; +use super::{parse_str, parse_bool, parse_socket_addr, ProtocolMessage}; pub const EVENT_STREAM_STATUS: u8 = 19; @@ -12,6 +14,7 @@ pub struct PartitionStatus { pub partition_num: ActorId, pub head: EventCounter, pub primary: bool, + pub primary_server_address: Option, } /// Contains some basic information on an event stream. Sent in response to a `SetEventStream` @@ -22,17 +25,32 @@ pub struct EventStreamStatus { pub partitions: Vec, } +named!{parse_opt_socket_addr>, alt!( + do_parse!( + tag!(&[0]) >> + + ( None ) + ) | + do_parse!( + tag!(&[1]) >> + addr: parse_socket_addr >> + + ( Some(addr) ) + ) +)} named!{parse_partition_status, chain!( partition_num: be_u16 ~ head: be_u64 ~ - status_num: be_u16, + primary: parse_bool ~ + primary_server_address: parse_opt_socket_addr, || { PartitionStatus { - partition_num: partition_num, - head: head, - primary: status_num == 1, + partition_num, + head, + primary, + primary_server_address } } @@ -64,6 +82,22 @@ named!{parse_event_stream_status_struct, ) } +impl FloSerialize for PartitionStatus { + fn serialize<'a>(&'a self, serializer: Serializer<'a>) -> Serializer<'a> { + let ser = serializer.write_u16(self.partition_num) + .write_u64(self.head) + .write_bool(self.primary); + + match self.primary_server_address.as_ref() { + Some(addr) => { + ser.write_u8(1).write_socket_addr(*addr) + } + None => { + ser.write_u8(0) + } + } + } +} pub fn serialize_event_stream_status(status: &EventStreamStatus, buf: &mut [u8]) -> usize { Serializer::new(buf) @@ -72,10 +106,7 @@ pub fn serialize_event_stream_status(status: &EventStreamStatus, buf: &mut [u8]) .write_string(&status.name) .write_u16(status.partitions.len() as u16) .write_many(status.partitions.iter(), |ser, partition| { - let status: u16 = if partition.primary { 1 } else { 0 }; - ser.write_u16(partition.partition_num) - .write_u64(partition.head) - .write_u16(status) + ser.write(partition) }) .finish() } diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index de9c33d..9710532 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -96,7 +96,7 @@ pub enum ProtocolMessage { NextBatch, /// Signals a client's intent to publish a new event. The server will respond with either an `EventAck` or an `ErrorMessage` ProduceEvent(ProduceEvent), - /// Always the first message sent by a server to another server + /// Always the first message sent by a server to another server for a system stream connection PeerAnnounce(PeerAnnounce), /// This is a complete event as serialized over the wire. This message is sent to to both consumers as well as other servers ReceiveEvent(E), @@ -376,6 +376,10 @@ mod test { } + fn addr(addr_string: &str) -> SocketAddr { + ::std::str::FromStr::from_str(addr_string).unwrap() + } + #[test] fn serde_request_vote_response() { let response = RequestVoteResponse { @@ -467,16 +471,19 @@ mod test { partition_num: 1, head: 638, primary: true, + primary_server_address: Some(addr("173.255.32.4:876")), }, PartitionStatus { partition_num: 2, head: 0, primary: false, + primary_server_address: Some(addr("[56:78::1]:9001")), }, PartitionStatus { partition_num: 3, head: 638, primary: true, + primary_server_address: None, }, ], }; diff --git a/flo-protocol/src/messages/peer_announce.rs b/flo-protocol/src/messages/peer_announce.rs index d4e3c1e..0adba62 100644 --- a/flo-protocol/src/messages/peer_announce.rs +++ b/flo-protocol/src/messages/peer_announce.rs @@ -15,6 +15,8 @@ pub struct PeerAnnounce { pub op_id: u32, } + + named!{pub parse_peer_announce>, chain!( _tag: tag!(&[PEER_ANNOUNCE]) ~ diff --git a/flo-server/src/embedded/mod.rs b/flo-server/src/embedded/mod.rs index ce3cc08..667a36a 100644 --- a/flo-server/src/embedded/mod.rs +++ b/flo-server/src/embedded/mod.rs @@ -66,6 +66,10 @@ fn message_to_owned(server_msg: SendProtocolMessage) -> ClientProtocolMessage { ProtocolMessage::Announce(op) => ProtocolMessage::Announce(op), ProtocolMessage::SetEventStream(op) => ProtocolMessage::SetEventStream(op), ProtocolMessage::PeerAnnounce(op) => ProtocolMessage::PeerAnnounce(op), + ProtocolMessage::SystemAppendCall(op) => ProtocolMessage::SystemAppendCall(op), + ProtocolMessage::SystemAppendResponse(op) => ProtocolMessage::SystemAppendResponse(op), + ProtocolMessage::RequestVote(op) => ProtocolMessage::RequestVote(op), + ProtocolMessage::VoteResponse(op) => ProtocolMessage::VoteResponse(op), } } diff --git a/flo-server/src/engine/connection_handler/connection_state.rs b/flo-server/src/engine/connection_handler/connection_state.rs index 0cf3c8c..4af9d60 100644 --- a/flo-server/src/engine/connection_handler/connection_state.rs +++ b/flo-server/src/engine/connection_handler/connection_state.rs @@ -5,7 +5,6 @@ use protocol::*; use engine::{ConnectionId, ClientSender, EngineRef, SendProtocolMessage}; use engine::event_stream::EventStreamRef; -use engine::system_stream::SystemOp; use engine::controller::SystemStreamRef; use super::ConnectionHandlerResult; @@ -65,8 +64,8 @@ impl ConnectionState { match self.engine.get_stream(&name) { Ok(new_stream) => { debug!("Setting event stream to '{}' for {:?}", new_stream.name(), self); - let stream_status = self.get_current_stream_status(op_id); self.event_stream = new_stream; + let stream_status = self.get_current_stream_status(op_id); self.send_to_client(ProtocolMessage::StreamStatus(stream_status)) } Err(ConnectError::NoStream) => { @@ -104,10 +103,12 @@ impl ConnectionState { let num = partition.partition_num(); let head = partition.get_highest_event_counter(); let primary = partition.is_primary(); + let primary_address = partition.get_primary_server_addr(); let part_status = PartitionStatus { partition_num: num, head: head, primary: primary, + primary_server_address: primary_address }; partition_statuses.push(part_status); } diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 0bdd768..8b4199e 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -135,7 +135,7 @@ impl Debug for ConnectionHandler { #[cfg(test)] mod test { use std::collections::HashMap; - use std::sync::{Arc, Mutex}; + use std::sync::{Arc, Mutex, RwLock}; use tokio_core::reactor::Core; use super::*; @@ -162,18 +162,20 @@ mod test { let (client_sender, client_rx) = ::futures::sync::mpsc::unbounded(); let counter_writer = AtomicCounterWriter::zero(); let primary = AtomicBoolWriter::with_value(true); + let primary_addr = Arc::new(RwLock::new(None)); let (tx, rx) = create_partition_channels(); let part_ref = PartitionRef::new(system_stream_name(), 1, counter_writer.reader(), primary.reader(), - tx); + tx, + primary_addr); let system_stream = SystemStreamRef::new(part_ref); let streams = Arc::new(Mutex::new(HashMap::new())); - let engine = EngineRef::new(system_stream, streams); + let engine = EngineRef::new(None, system_stream, streams); let subject = ConnectionHandler::new(456, client_sender, engine.clone(), reactor.handle()); @@ -202,12 +204,14 @@ mod test { let partition_num = i + 1; let counter_writer = AtomicCounterWriter::zero(); let primary = AtomicBoolWriter::with_value(true); + let primary_addr = Arc::new(RwLock::new(None)); let (tx, rx) = create_partition_channels(); let part_ref = PartitionRef::new(name.to_owned(), partition_num, counter_writer.reader(), primary.reader(), - tx); + tx, + primary_addr); partition_refs.push(part_ref); self.partition_receivers.insert((name.to_owned(), partition_num), rx); } @@ -274,16 +278,19 @@ mod test { partition_num: 1, head: 0, primary: true, + primary_server_address: None, }, PartitionStatus { partition_num: 2, head: 0, primary: true, + primary_server_address: None, }, PartitionStatus { partition_num: 3, head: 0, primary: true, + primary_server_address: None, }, ], }; diff --git a/flo-server/src/engine/connection_handler/peer.rs b/flo-server/src/engine/connection_handler/peer.rs index 6b4e2c4..11d9d80 100644 --- a/flo-server/src/engine/connection_handler/peer.rs +++ b/flo-server/src/engine/connection_handler/peer.rs @@ -1,5 +1,5 @@ -use protocol::{ProtocolMessage, EventStreamStatus}; +use protocol::{ProtocolMessage, PeerAnnounce, EventStreamStatus}; use engine::{ReceivedProtocolMessage, ConnectionId}; use super::connection_state::ConnectionState; use super::ConnectionHandlerResult; @@ -17,9 +17,16 @@ impl PeerConnectionState { pub fn initiate_outgoing_peer_connection(&mut self, state: &mut ConnectionState) { assert_eq!(PeerConnectionState::Init, *self); state.set_to_system_stream(); - let system_stream_state = state.get_current_stream_status(1); - let peer_announce = ProtocolMessage::PeerAnnounce(system_stream_state); - state.send_to_client(peer_announce).expect("failed to send peer announce when establishing outgoing connection"); + + let this_address = state.engine.get_this_instance_address() + .expect("Tried to initiate outgoing connection but this_instance_address is None"); + let announce = PeerAnnounce { + protocol_version: 1, + peer_address: this_address, + op_id: 1, + }; + let protocol_message = ProtocolMessage::PeerAnnounce(announce); + state.send_to_client(protocol_message).expect("failed to send peer announce when establishing outgoing connection"); *self = PeerConnectionState::AwaitingPeerResponse; } @@ -39,11 +46,9 @@ impl PeerConnectionState { Ok(()) } - fn stream_status_received(&mut self, state: &mut ConnectionState, status: EventStreamStatus) { + fn stream_status_received(&mut self, state: &mut ConnectionState, _status: EventStreamStatus) { self.set_state(state.connection_id, PeerConnectionState::Peer); - - - + unimplemented!() } fn set_state(&mut self, connection_id: ConnectionId, new_state: PeerConnectionState) { diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 18d6d99..894ebce 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -1,10 +1,9 @@ use std::net::SocketAddr; -use std::time::{Instant, Duration}; +use std::time::{Instant}; use event::EventCounter; -use engine::controller::outgoing_io::OutgoingConnectionCreator; -use engine::system_stream::FloInstanceId; +use protocol::FloInstanceId; #[derive(Debug, PartialEq, Clone)] diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index 51fa9a3..7f85d5a 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -1,3 +1,4 @@ +use std::sync::{Arc, RwLock}; use std::path::{PathBuf, Path}; use std::collections::HashMap; use std::fs::DirEntry; @@ -43,6 +44,8 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul // TODO: initialize system primary status to false once clustering works let system_primary_writer = AtomicBoolWriter::with_value(true); + let system_primary_server_addr = Arc::new(RwLock::new(None)); + let partition_result = init_system_partition(&storage_dir, system_primary_writer.reader(), &default_stream_options); @@ -57,13 +60,14 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul let flo_controller = FloController::new(system_partition, system_primary_writer, + system_primary_server_addr.clone(), user_streams, storage_dir, cluster_options, default_stream_options); let (system_partition_tx, system_partition_rx) = create_partition_channels(); - let engine_ref = create_engine_ref(&flo_controller, system_highest_counter, system_primary_reader, system_partition_tx); + let engine_ref = create_engine_ref(&flo_controller, system_highest_counter, system_primary_reader, system_primary_server_addr, system_partition_tx); run_controller_impl(flo_controller, system_partition_rx); engine_ref @@ -95,17 +99,20 @@ fn init_system_partition(storage_dir: &Path, system_primary_reader: AtomicBoolRe fn create_engine_ref(controller: &FloController, system_highest_counter: AtomicCounterReader, system_primary_reader: AtomicBoolReader, + system_primary_addr: Arc>>, system_sender: PartitionSender) -> EngineRef { let system_partition_ref = PartitionRef::new(system_stream_name(), 1, system_highest_counter, system_primary_reader, - system_sender); + system_sender, + system_primary_addr); let system_stream_ref = SystemStreamRef::new(system_partition_ref); let shared_stream_refs = controller.get_shared_streams(); - EngineRef::new(system_stream_ref, shared_stream_refs) + let this_addr = controller.get_this_instance_address(); + EngineRef::new(this_addr, system_stream_ref, shared_stream_refs) } diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index faf3209..b9233c3 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -1,11 +1,11 @@ mod cluster_state; -mod outgoing_io; mod system_stream; mod initialization; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, RwLock}; use std::collections::HashMap; +use std::net::SocketAddr; use engine::event_stream::{EventStreamRef, @@ -43,11 +43,19 @@ pub struct FloController { /// used to set the status of the system stream. There is only ever at most one instance in a cluster /// where this variable is true ...if things actually work correctly ;) system_primary_status_writer: AtomicBoolWriter, + + /// The address of the cluster's primary server, if one exists and it is known + system_primary_server_addr: Arc>>, + + /// cluster parameters that this instance was started with. We'll almost certainly want to replace this field later on + /// with something that can deal with more complexity + cluster_options: Option, } impl FloController { pub fn new(system_partition: PartitionImpl, system_primary_setter: AtomicBoolWriter, + system_primary_address: Arc>>, event_streams: HashMap, storage_dir: PathBuf, cluster_options: Option, @@ -64,6 +72,8 @@ impl FloController { storage_dir, default_stream_options, system_primary_status_writer: system_primary_setter, + system_primary_server_addr: system_primary_address, + cluster_options, } } @@ -78,6 +88,10 @@ impl FloController { fn get_shared_streams(&self) -> Arc>> { self.shared_event_stream_refs.clone() } + + fn get_this_instance_address(&self) -> Option { + self.cluster_options.as_ref().map(|opts| opts.this_instance_address) + } } diff --git a/flo-server/src/engine/controller/outgoing_io.rs b/flo-server/src/engine/controller/outgoing_io.rs deleted file mode 100644 index b0421f0..0000000 --- a/flo-server/src/engine/controller/outgoing_io.rs +++ /dev/null @@ -1,118 +0,0 @@ -use std::net::SocketAddr; -use std::io; - -use tokio_core::reactor::Handle; -use tokio_core::net::TcpStream; - -#[allow(deprecated)] -use tokio_core::io::Io; -use futures::{Future, Stream, Sink}; - -use event_loops::LoopHandles; -use engine::{EngineRef, create_client_channels, ReceivedProtocolMessage}; - -#[derive(Debug, Copy, Clone, PartialEq)] -pub struct CreateOutgoingConnection(SocketAddr); - -impl CreateOutgoingConnection { - pub fn new(addr: SocketAddr) -> CreateOutgoingConnection { - CreateOutgoingConnection(addr) - } - - fn addr(&self) -> &SocketAddr { - &self.0 - } -} - -pub trait OutgoingConnectionCreator { - fn establish_connection(&mut self, create: CreateOutgoingConnection); -} - -pub struct OutgoingConnectionCreatorImpl(::futures::sync::mpsc::UnboundedSender); - -impl OutgoingConnectionCreator for OutgoingConnectionCreatorImpl { - fn establish_connection(&mut self, create: CreateOutgoingConnection) { - self.0.unbounded_send(create).expect("Failed to send message to OutgoingConnectionCreator channel"); - } -} - - -pub fn start_outgoing_io(mut event_loops: LoopHandles, engine_ref: EngineRef) -> OutgoingConnectionCreatorImpl { - let (tx, rx) = ::futures::sync::mpsc::unbounded::(); - - let remote = event_loops.next_handle(); - let system_stream_ref = engine_ref.get_system_stream(); - - let future = rx.for_each( move |create_outgoing| { - - let engine = engine_ref.clone(); - event_loops.next_handle().spawn(move |handle| { - handle_outgoing_connection(handle.clone(), create_outgoing, engine) - }); - Ok(()) - }); - - remote.spawn(|handle| { - future - }); - - OutgoingConnectionCreatorImpl(tx) -} - - -fn handle_outgoing_connection(handle: Handle, create_outgoing: CreateOutgoingConnection, engine_ref: EngineRef) -> Box> { - use flo_io::{ProtocolMessageStream, ServerMessageStream}; - use engine::ConnectionHandler; - - // These copies are to work around the ownership rules, since we want to have slightly different error handling for - // connection failures depending on when it fails - let client_addr = *create_outgoing.addr(); - let client_addr_copy = client_addr.clone(); - - let mut system_stream = engine_ref.get_system_stream(); - let mut system_stream_copy = system_stream.clone(); - - let future = TcpStream::connect(create_outgoing.addr(), &handle).map_err( move |io_err| { - - error!("Failed to create outgoing connection to address: {:?}: {:?}", client_addr_copy, io_err); - system_stream.outgoing_connection_failed(0, client_addr_copy); - - }).and_then( move |tcp_stream| { - let connection_id = engine_ref.next_connection_id(); - debug!("Established connection to {:?} with connection_id: {}", client_addr, connection_id); - - if let Err(io_err) = tcp_stream.set_nodelay(true) { - error!("Error setting NODELAY for connection_id: {}. Nagle yet lives!: {:?}", connection_id, io_err); - } - - let (client_tx, client_rx) = create_client_channels(); - - #[allow(deprecated)] - let (read, write) = tcp_stream.split(); - - let server_to_client = ServerMessageStream::new(connection_id, client_rx, write); - let client_message_stream = ProtocolMessageStream::new(connection_id, read); - - let connection_handler = ConnectionHandler::new( - connection_id, - client_tx.clone(), - engine_ref, - handle.clone()); - - let client_to_server = connection_handler - .send_all(client_message_stream) - .map(|_| ()); - - client_to_server.select(server_to_client).then(move |res| { - if let Err((err, _)) = res { - warn!("Closing outgoing connection: {} due to err: {:?}", connection_id, err); - system_stream_copy.outgoing_connection_failed(connection_id, client_addr); - } - info!("Closed connection_id: {} to address: {}", connection_id, client_addr); - Ok(()) - }) - - }); - - Box::new(future) -} diff --git a/flo-server/src/engine/controller/peer_connection/mod.rs b/flo-server/src/engine/controller/peer_connection/mod.rs new file mode 100644 index 0000000..361598c --- /dev/null +++ b/flo-server/src/engine/controller/peer_connection/mod.rs @@ -0,0 +1,41 @@ +mod replication; +mod system; + +use std::fmt::Debug; +use std::net::SocketAddr; +use std::io; + +use engine::EngineRef; +use event_loops::LoopHandles; + +pub use self::replication::PeerReplicationConnection; +pub use self::system::PeerSystemConnection; + +pub type ConnectionSendResult = Result<(), T>; + + +/// Trait for creating outgoing connections (clever name, I know). +pub trait OutgoingConnectionCreator { + fn establish_system_connection(&mut self, address: SocketAddr) -> Box; + fn establish_replication_connection(&mut self, address: SocketAddr, event_stream: String) -> Box; +} + +pub struct OutgoingConnectionCreatorImpl { + event_loops: LoopHandles, + engine_ref: EngineRef, +} + +impl OutgoingConnectionCreator for OutgoingConnectionCreatorImpl { + fn establish_system_connection(&mut self, address: SocketAddr) -> Box { + unimplemented!() + } + + fn establish_replication_connection(&mut self, address: SocketAddr, event_stream: String) -> Box { + unimplemented!() + } +} + + + + + diff --git a/flo-server/src/engine/controller/peer_connection/replication.rs b/flo-server/src/engine/controller/peer_connection/replication.rs new file mode 100644 index 0000000..bb54272 --- /dev/null +++ b/flo-server/src/engine/controller/peer_connection/replication.rs @@ -0,0 +1,15 @@ +use std::fmt::Debug; + +use super::ConnectionSendResult; +use event::VersionVector; + + +/// trait representing an active peer connection to a particular event stream, with functions to control replication +pub trait PeerReplicationConnection: Debug + Send + 'static { + fn start_replication(&mut self, version_vector: VersionVector) -> ConnectionSendResult; + fn stop_replication(&mut self) -> ConnectionSendResult<()>; + fn close_connection(&mut self); +} + + + diff --git a/flo-server/src/engine/controller/peer_connection/system.rs b/flo-server/src/engine/controller/peer_connection/system.rs new file mode 100644 index 0000000..9b3e1f3 --- /dev/null +++ b/flo-server/src/engine/controller/peer_connection/system.rs @@ -0,0 +1,129 @@ +use std::fmt::Debug; +use std::io; +use std::net::SocketAddr; + +use tokio_core::reactor::Handle; +use tokio_core::net::TcpStream; +#[allow(deprecated)] +use tokio_core::io::Io; +use futures::{Future, Stream, Sink}; + +use event::VersionVector; +use event_loops::LoopHandles; +use engine::controller::SystemStreamRef; +use engine::{EngineRef, create_client_channels, ReceivedProtocolMessage, ClientSender, ClientReceiver}; +use engine::system_stream::{FloInstanceId, AppendEntriesCall, RequestVoteCall}; + +use super::ConnectionSendResult; + + +/// Trait representing an active peer connection, with functions to control it +pub trait PeerSystemConnection: Debug + Send + 'static { + fn send_append_entries(&mut self, append_entries: AppendEntriesCall) -> ConnectionSendResult; + fn send_request_vote(&mut self, request: RequestVoteCall) -> ConnectionSendResult; +} + + + +enum SystemConnectionMessage { + AppendEntries(AppendEntriesCall), + Vote(RequestVoteCall), +} + +fn create_outgoing_connection(loops: &mut LoopHandles, client_addr: SocketAddr, engine_ref: EngineRef) -> Box { + + // These copies are to work around the ownership rules, since we want to have slightly different error handling for + // connection failures depending on when it fails + let client_addr_copy = client_addr.clone(); + + let mut system_stream = engine_ref.get_system_stream(); + let mut system_stream_copy = system_stream.clone(); + + let (client_tx, client_rx) = create_client_channels(); + let client_tx_copy = client_tx.clone(); + + loops.next_handle().spawn( move |handle| { + + let owned_handle = handle.clone(); + let addr = client_addr_copy; + TcpStream::connect(&addr, handle).map_err( move |io_err| { + + error!("Failed to create outgoing connection to address: {:?}: {:?}", addr, io_err); + system_stream.outgoing_connection_failed(0, addr); + + }).and_then( move |tcp_stream| { + + outgoing_connection_future(owned_handle, + engine_ref, + client_addr, + tcp_stream, + client_tx_copy, + client_rx, + system_stream_copy) + }) + }); + + let outgoing = OutgoingPeerSystemConnection { + client_tx + }; + Box::new(outgoing) +} + +fn outgoing_connection_future(handle: Handle, + engine_ref: EngineRef, + client_addr: SocketAddr, + tcp_stream: TcpStream, + client_tx: ClientSender, + client_rx: ClientReceiver, + mut system_stream: SystemStreamRef) -> Box> { + + use flo_io::{ProtocolMessageStream, ServerMessageStream}; + use engine::ConnectionHandler; + + let connection_id = engine_ref.next_connection_id(); + debug!("Established connection to {:?} with connection_id: {}", client_addr, connection_id); + if let Err(io_err) = tcp_stream.set_nodelay(true) { + error!("Error setting NODELAY for connection_id: {}. Nagle yet lives!: {:?}", connection_id, io_err); + } + #[allow(deprecated)] + let (read, write) = tcp_stream.split(); + let server_to_client = ServerMessageStream::new(connection_id, client_rx, write); + let client_message_stream = ProtocolMessageStream::new(connection_id, read); + let mut connection_handler = ConnectionHandler::new( + connection_id, + client_tx, + engine_ref, + handle.clone()); + + // sends PeerAnnounce message to the other server and sets up the connection handler to expect the response + connection_handler.upgrade_to_outgoing_peer(); + + let client_to_server = connection_handler + .send_all(client_message_stream) + .map(|_| ()); + + let future = client_to_server.select(server_to_client).then(move |res| { + if let Err((err, _)) = res { + warn!("Closing outgoing connection: {} due to err: {:?}", connection_id, err); + system_stream.outgoing_connection_failed(connection_id, client_addr); + } + info!("Closed connection_id: {} to address: {}", connection_id, client_addr); + Ok(()) + }); + Box::new(future) +} + +#[derive(Debug)] +struct OutgoingPeerSystemConnection { + client_tx: ClientSender +} + +impl PeerSystemConnection for OutgoingPeerSystemConnection { + fn send_append_entries(&mut self, append_entries: AppendEntriesCall) -> ConnectionSendResult { + unimplemented!() + } + + fn send_request_vote(&mut self, request: RequestVoteCall) -> ConnectionSendResult { + unimplemented!() + } +} diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index e13c63e..ae873b2 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -4,6 +4,7 @@ mod event_reader; mod ops; pub mod controller; +use std::net::SocketAddr; use std::fmt::{self, Debug, Display}; use std::path::{Path, PathBuf}; use std::collections::VecDeque; @@ -179,17 +180,19 @@ pub struct PartitionRef { partition_num: ActorId, highest_event_counter: AtomicCounterReader, primary: AtomicBoolReader, + primary_server_address: Arc>>, sender: PartitionSender, } impl PartitionRef { - pub fn new(event_stream_name: String, partition_num: ActorId, highest_event_counter: AtomicCounterReader, primary: AtomicBoolReader, sender: PartitionSender) -> PartitionRef { + pub fn new(event_stream_name: String, partition_num: ActorId, highest_event_counter: AtomicCounterReader, primary: AtomicBoolReader, sender: PartitionSender, primary_server_address: Arc>>) -> PartitionRef { PartitionRef { - event_stream_name: event_stream_name, - partition_num: partition_num, - highest_event_counter: highest_event_counter, - primary: primary, - sender: sender, + event_stream_name, + partition_num, + highest_event_counter, + primary, + sender, + primary_server_address, } } pub fn partition_num(&self) -> ActorId { @@ -204,6 +207,11 @@ impl PartitionRef { self.primary.get_relaxed() } + pub fn get_primary_server_addr(&self) -> Option { + let value_ref = self.primary_server_address.read().unwrap(); + value_ref.clone() + } + pub fn event_stream_name(&self) -> &str { &self.event_stream_name } @@ -241,11 +249,14 @@ pub fn initialize_existing_partition(partition_num: ActorId, event_stream_options: &EventStreamOptions, highest_counter: HighestCounter) -> io::Result { + // TODO: for now we are starting every partition as primary. This will need to change once we have a raft implementation let status_writer = AtomicBoolWriter::with_value(true); + let primary_addr = Arc::new(RwLock::new(None)); + let partition_data_dir = get_partition_data_dir(event_stream_data_dir, partition_num); let partition_impl = PartitionImpl::init_existing(partition_num, partition_data_dir, event_stream_options, status_writer.reader(), highest_counter)?; - run_partition(partition_impl).map(|partition_ref| { + run_partition(partition_impl, primary_addr).map(|partition_ref| { PartitionRefMut { status_writer, partition_ref, @@ -260,9 +271,11 @@ pub fn initialize_new_partition(partition_num: ActorId, // TODO: for now we are starting every partition as primary. This will need to change once we have a raft implementation let status_writer = AtomicBoolWriter::with_value(true); + let primary_addr = Arc::new(RwLock::new(None)); + let partition_data_dir = get_partition_data_dir(event_stream_data_dir, partition_num); let partition_impl = PartitionImpl::init_new(partition_num, partition_data_dir, &event_stream_options, status_writer.reader(), highest_counter)?; - run_partition(partition_impl).map(|partition_ref| { + run_partition(partition_impl, primary_addr).map(|partition_ref| { PartitionRefMut { status_writer, partition_ref, @@ -271,7 +284,7 @@ pub fn initialize_new_partition(partition_num: ActorId, } -pub fn run_partition(partition_impl: PartitionImpl) -> io::Result { +pub fn run_partition(partition_impl: PartitionImpl, primary_server_addr: Arc>>) -> io::Result { let partition_num = partition_impl.partition_num(); let event_counter_reader = partition_impl.event_counter_reader(); let primary_status_reader = partition_impl.primary_status_reader(); @@ -303,7 +316,13 @@ pub fn run_partition(partition_impl: PartitionImpl) -> io::Result fsync_result); })?; - Ok(PartitionRef::new(event_stream_name, partition_num, event_counter_reader, primary_status_reader,tx)) + let partition = PartitionRef::new(event_stream_name, + partition_num, + event_counter_reader, + primary_status_reader, + tx, + primary_server_addr); + Ok(partition) } fn get_partition_thread_name(event_stream_name: &str, partition_num: ActorId) -> String { diff --git a/flo-server/src/engine/event_stream/partition/ops.rs b/flo-server/src/engine/event_stream/partition/ops.rs index 808f019..249f2c9 100644 --- a/flo-server/src/engine/event_stream/partition/ops.rs +++ b/flo-server/src/engine/event_stream/partition/ops.rs @@ -8,7 +8,6 @@ use futures::sync::oneshot; use engine::event_stream::partition::{EventFilter, PartitionReader}; use engine::ConnectionId; -use engine::system_stream::SystemOp; use protocol::ProduceEvent; use event::{FloEventId, EventCounter}; @@ -61,7 +60,12 @@ pub enum OpType { Consume(ConsumeOperation), StopConsumer, Tick, - System(SystemOp) + System(SystemOp), +} + +#[derive(Debug)] +pub enum SystemOp { + OutgoingConnectionFailed(SocketAddr), } diff --git a/flo-server/src/engine/mod.rs b/flo-server/src/engine/mod.rs index fbc93ef..8a26b5f 100644 --- a/flo-server/src/engine/mod.rs +++ b/flo-server/src/engine/mod.rs @@ -1,5 +1,4 @@ pub mod event_stream; -pub mod system_stream; mod controller; mod connection_handler; @@ -7,6 +6,7 @@ mod connection_handler; use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::sync::atomic::{AtomicUsize}; +use std::net::SocketAddr; use protocol::ProtocolMessage; use event::OwnedFloEvent; @@ -40,6 +40,8 @@ pub fn system_stream_name() -> String { #[derive(Clone, Debug)] pub struct EngineRef { + /// only known if this instance was started in clustering mode + this_instance_address: Option, current_connection_id: Arc, system_stream: SystemStreamRef, event_streams: Arc>> @@ -52,8 +54,9 @@ pub enum ConnectError { } impl EngineRef { - pub fn new(system_stream: SystemStreamRef, event_streams: Arc>>) -> EngineRef { + pub fn new(this_address: Option, system_stream: SystemStreamRef, event_streams: Arc>>) -> EngineRef { EngineRef { + this_instance_address: this_address, current_connection_id: Arc::new(AtomicUsize::new(0)), system_stream, event_streams @@ -91,6 +94,11 @@ impl EngineRef { pub fn get_system_stream(&self) -> SystemStreamRef { self.system_stream.clone() } + + /// Returns the address that this intance is reachable at. This will be `None` if the server was started started in non-clustered mode + pub fn get_this_instance_address(&self) -> Option { + self.this_instance_address + } } diff --git a/flo-server/src/engine/system_stream/mod.rs b/flo-server/src/engine/system_stream/mod.rs deleted file mode 100644 index ee9557d..0000000 --- a/flo-server/src/engine/system_stream/mod.rs +++ /dev/null @@ -1,112 +0,0 @@ -mod flo_instance_id; -mod replication_plan; - -use std::net::SocketAddr; - -use event::EventCounter; -use engine::ConnectionId; -use engine::event_stream::EventStreamOptions; - -pub use self::flo_instance_id::FloInstanceId; -pub use self::replication_plan::{SystemReplicationPlan, StreamReplicationPlan, PartitionReplicationPlan}; - -pub type Term = u64; - - -#[derive(Debug, PartialEq, Clone)] -pub struct FloServer { - pub id: FloInstanceId, - pub address: SocketAddr, -} - -#[derive(Debug, PartialEq, Clone)] -pub struct ServerStatus { - peer_id: FloInstanceId, - leader_id: FloInstanceId, - system_event_counter: EventCounter, - current_term: Term, - all_members: Vec, -} - -#[derive(Debug, Clone)] -pub enum SystemOp { - OutgoingConnectionFailed(SocketAddr), - PeerConnectionEstablished(ConnectionId, SocketAddr), - AppendResponse(AppendEntriesResponse), - VoteResponse(RequestVoteResponse), -} - - - -#[derive(Debug, PartialEq, Clone)] -pub struct AppendEntriesCall { - pub leader_id: FloInstanceId, - pub term: Term, - pub prev_entry_term: Term, - pub prev_entry_index: EventCounter, - pub leader_commit_index: EventCounter, - pub entries: Vec, -} - -impl AppendEntriesCall { - pub fn heartbeat(leader_id: FloInstanceId, - term: Term, - prev_entry_term: Term, - prev_entry_index: EventCounter, - leader_commit_index: EventCounter) -> AppendEntriesCall { - - AppendEntriesCall { - leader_id, - term, - prev_entry_term, - prev_entry_index, - leader_commit_index, - entries: Vec::new(), - } - } - - pub fn is_heartbeat(&self) -> bool { - self.entries.is_empty() - } -} - -#[derive(Debug, PartialEq, Clone)] -pub struct AppendEntriesResponse { - pub from_peer: FloInstanceId, - pub term: Term, - pub success: bool, -} - -#[derive(Debug, PartialEq, Clone)] -pub struct RequestVoteCall { - pub term: Term, - pub candidate_id: FloInstanceId, - pub last_log_index: EventCounter, - pub last_log_term: Term, -} - - -#[derive(Debug, PartialEq, Clone)] -pub struct RequestVoteResponse { - pub from_peer: FloInstanceId, - pub term: Term, - pub vote_granted: bool, -} - - -#[derive(Debug, Clone, PartialEq)] -pub enum SystemLogEntry { - /// Communicates which server should be the primary for each partition in each event stream - SetReplicationPlan(SystemReplicationPlan), - - /// Command to create a new event stream. Each follower should create the stream, but wait for - /// a replication plan before allowing any writes to it - CreateEventStream(EventStreamOptions), - - /// First phase of adding a new member to the cluster. New members are NOT eligible to become - /// primary until an `ActivateClusterMember` command with its id has been committed to the log. - AddClusterMember(FloServer), - - /// Once this entry has been committed, the given server is now eligible to be elected primary - ActivateClusterMember(FloInstanceId), -} diff --git a/flo-server/src/server/server_options.rs b/flo-server/src/server/server_options.rs index d08b5f3..74e2946 100644 --- a/flo-server/src/server/server_options.rs +++ b/flo-server/src/server/server_options.rs @@ -2,8 +2,6 @@ use std::path::PathBuf; use chrono::Duration; use std::net::SocketAddr; -use event::ActorId; - #[derive(Copy, Clone, PartialEq, Debug)] #[allow(dead_code)] From a35ac279483b9a16f9d1978836d54919112e78f4 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 3 Dec 2017 13:03:03 -0500 Subject: [PATCH 15/73] change connection handler to process either protocol or control messages --- flo-client-lib/src/async/mod.rs | 8 +++- flo-server/src/embedded/mod.rs | 29 ++++++++++++++- .../src/engine/connection_handler/input.rs | 37 +++++++++++++++++++ .../src/engine/connection_handler/mod.rs | 31 +++++++++++++--- flo-server/src/server/mod.rs | 3 +- 5 files changed, 98 insertions(+), 10 deletions(-) create mode 100644 flo-server/src/engine/connection_handler/input.rs diff --git a/flo-client-lib/src/async/mod.rs b/flo-client-lib/src/async/mod.rs index e0ebc54..03300a0 100644 --- a/flo-client-lib/src/async/mod.rs +++ b/flo-client-lib/src/async/mod.rs @@ -485,12 +485,14 @@ mod test { PartitionStatus { partition_num: 1, head: 7, - primary: true + primary: true, + primary_server_address: None, }, PartitionStatus { partition_num: 2, head: 5, primary: false, + primary_server_address: None, } ], })]; @@ -508,11 +510,13 @@ mod test { partition_num: 1, head: 7, writable: true, + primary_server_addr: None, }, PartitionState { partition_num: 2, head: 5, - writable: false + writable: false, + primary_server_addr: None, } ] }; diff --git a/flo-server/src/embedded/mod.rs b/flo-server/src/embedded/mod.rs index 667a36a..c68a85a 100644 --- a/flo-server/src/embedded/mod.rs +++ b/flo-server/src/embedded/mod.rs @@ -6,7 +6,7 @@ use std::fmt::Debug; use std::io; use tokio_core::reactor::{Handle, Remote}; -use futures::Stream; +use futures::{Stream, Sink, StartSend, AsyncSink, Async, Poll}; use protocol::ProtocolMessage; use flo_client_lib::async::{AsyncConnection, MessageReceiver, MessageSender, ClientProtocolMessage}; @@ -34,6 +34,9 @@ impl EmbeddedFloServer { client_sender.clone(), engine_ref, handle); + let embedded_connection = EmbeddedConnectionHandler { + inner: connection_handler + }; let receiver = client_receiver.map(|message| { message_to_owned(message) @@ -41,7 +44,7 @@ impl EmbeddedFloServer { io::Error::new(io::ErrorKind::UnexpectedEof, format!("Error reading from channel: {:?}", recv_err)) }); let recv = Box::new(receiver) as MessageReceiver; - let send = Box::new(connection_handler) as MessageSender; + let send = Box::new(embedded_connection) as MessageSender; AsyncConnection::new(name, send, recv, codec) } @@ -81,3 +84,25 @@ pub fn run_embedded_server(options: ControllerOptions, remote: Remote) -> io::Re }) } +struct EmbeddedConnectionHandler { + inner: ConnectionHandler +} + +impl Sink for EmbeddedConnectionHandler { + type SinkItem = ClientProtocolMessage; + type SinkError = io::Error; + + fn start_send(&mut self, item: Self::SinkItem) -> StartSend { + self.inner.start_send(item.into()).map(|async_sink| { + async_sink.map(|input| input.unwrap_protocol_message()) + }) + } + + fn poll_complete(&mut self) -> Poll<(), Self::SinkError> { + self.inner.poll_complete() + } + + fn close(&mut self) -> Poll<(), Self::SinkError> { + self.inner.close() + } +} diff --git a/flo-server/src/engine/connection_handler/input.rs b/flo-server/src/engine/connection_handler/input.rs new file mode 100644 index 0000000..2d8959b --- /dev/null +++ b/flo-server/src/engine/connection_handler/input.rs @@ -0,0 +1,37 @@ + +use engine::ReceivedProtocolMessage; + +#[derive(Debug)] +pub enum ConnectionHandlerInput { + IncomingMessage(ReceivedProtocolMessage), + Control(ConnectionControl), +} + +impl ConnectionHandlerInput { + pub fn unwrap_protocol_message(self) -> ReceivedProtocolMessage { + match self { + ConnectionHandlerInput::IncomingMessage(message) => message, + ConnectionHandlerInput::Control(ctrl) => { + panic!("Attempt to unwrap a protocol message from a Control input with control: {:?}", ctrl); + } + } + } +} + +#[derive(Debug)] +pub enum ConnectionControl { + InitiateOutgoingSystemConnection, +} + + +impl From for ConnectionHandlerInput { + fn from(message: ReceivedProtocolMessage) -> Self { + ConnectionHandlerInput::IncomingMessage(message) + } +} + +impl From for ConnectionHandlerInput { + fn from(control: ConnectionControl) -> Self { + ConnectionHandlerInput::Control(control) + } +} diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 8b4199e..c87d51b 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -2,6 +2,7 @@ pub mod connection_state; mod consumer; mod producer; mod peer; +mod input; use std::fmt::{self, Debug}; use std::io; @@ -17,6 +18,7 @@ use self::consumer::ConsumerConnectionState; use self::producer::ProducerConnectionState; use self::peer::PeerConnectionState; +pub use self::input::{ConnectionHandlerInput, ConnectionControl}; pub struct ConnectionHandler { common_state: ConnectionState, @@ -49,6 +51,17 @@ impl ConnectionHandler { !self.producer_state.requires_poll_complete() && !self.consumer_state.requires_poll_complete() } + pub fn handle_control(&mut self, control: ConnectionControl) -> ConnectionHandlerResult { + debug!("client: {:?} processing control: {:?}", self.common_state, control); + + match control { + ConnectionControl::InitiateOutgoingSystemConnection => { + self.upgrade_to_outgoing_peer() + } + } + Ok(()) + } + pub fn handle_incoming_message(&mut self, message: ReceivedProtocolMessage) -> ConnectionHandlerResult { trace!("client: {:?}, received message: {:?}", self.common_state, message); @@ -82,15 +95,23 @@ impl ConnectionHandler { impl Sink for ConnectionHandler { - type SinkItem = ReceivedProtocolMessage; + type SinkItem = ConnectionHandlerInput; type SinkError = io::Error; fn start_send(&mut self, item: Self::SinkItem) -> StartSend { - if !self.can_process(&item) { - return Ok(AsyncSink::NotReady(item)); - } + match item { + ConnectionHandlerInput::IncomingMessage(message) => { + if !self.can_process(&message) { + return Ok(AsyncSink::NotReady(message.into())); + } + + self.handle_incoming_message(message) + } + ConnectionHandlerInput::Control(control) => { + self.handle_control(control) + } - self.handle_incoming_message(item).map(|()| { + }.map(|()| { AsyncSink::Ready }).map_err(|err_string| { io::Error::new(io::ErrorKind::Other, err_string) diff --git a/flo-server/src/server/mod.rs b/flo-server/src/server/mod.rs index e6d5ab3..a9b07fa 100644 --- a/flo-server/src/server/mod.rs +++ b/flo-server/src/server/mod.rs @@ -88,7 +88,8 @@ pub fn run(mut options: ServerOptions) -> io::Result<()> { let server_to_client = ServerMessageStream::new(connection_id, client_rx, tcp_writer); - let client_message_stream = ProtocolMessageStream::new(connection_id, tcp_reader); + let client_message_stream = ProtocolMessageStream::new(connection_id, tcp_reader) + .map(|proto_message| proto_message.into()); let connection_handler = ConnectionHandler::new( connection_id, client_tx.clone(), From 7de19a3695fc39cf26ef6d9d7a8e3314fbb51f17 Mon Sep 17 00:00:00 2001 From: pfried Date: Wed, 6 Dec 2017 21:32:24 -0500 Subject: [PATCH 16/73] fix produce operation in flo-client cli --- flo-client-cli/src/client_cli/mod.rs | 4 ++++ flo-client-cli/src/client_cli/producer.rs | 4 ++-- flo-client-cli/src/main.rs | 8 +++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/flo-client-cli/src/client_cli/mod.rs b/flo-client-cli/src/client_cli/mod.rs index eadb356..b3cc980 100644 --- a/flo-client-cli/src/client_cli/mod.rs +++ b/flo-client-cli/src/client_cli/mod.rs @@ -65,6 +65,10 @@ impl Context { } } + pub fn debug(&self, message: M) { + self.println(message, Verbosity::Debug) + } + pub fn verbose(&self, message: M) { self.println(message, Verbosity::Verbose); } diff --git a/flo-client-cli/src/client_cli/producer.rs b/flo-client-cli/src/client_cli/producer.rs index 83f41e7..1c23e4e 100644 --- a/flo-client-cli/src/client_cli/producer.rs +++ b/flo-client-cli/src/client_cli/producer.rs @@ -21,11 +21,11 @@ impl FloCliCommand for Producer { fn run(ProduceOptions{host, port, partition, namespace, event_data, parent_id}: ProduceOptions, output: &Context) -> Result<(), Self::Error> { let server_address = format!("{}:{}", host, port); - output.verbose(format!("Attempting connection to: {:?}", &server_address)); + output.verbose(format!("Attempting connection to: {:?} to produce {} events", &server_address, event_data.len())); SyncConnection::connect_from_str(&server_address, "flo-client-cli", RawCodec, None).map_err(|handshake_err| { format!("Error establishing connection to flo server: {}", handshake_err) }).and_then(|mut connection| { - output.verbose(format!("connected to {}", &server_address)); + output.debug(format!("connected to {}", &server_address)); event_data.into_iter().fold(Ok(0), |events_produced, event_data| { events_produced.and_then(|count| { diff --git a/flo-client-cli/src/main.rs b/flo-client-cli/src/main.rs index 0da19c1..34209f5 100644 --- a/flo-client-cli/src/main.rs +++ b/flo-client-cli/src/main.rs @@ -187,13 +187,11 @@ fn get_parent_id(args: &ArgMatches, context: &Context) -> Option { fn get_event_data(args: &ArgMatches) -> Vec> { args.values_of(args::EVENT_DATA).map(|values| { - let mut vec = values.map(|str_val| { + + values.map(|str_val| { str_val.as_bytes().to_owned() - }).collect::>>(); + }).collect::>>() - // work around some weird bug where we keep getting an extra value that is an empty string - vec.pop(); - vec }).unwrap_or(Vec::new()) } From b41d76500cde8cf364048afa27ec93b1151035c6 Mon Sep 17 00:00:00 2001 From: pfried Date: Wed, 6 Dec 2017 23:15:54 -0500 Subject: [PATCH 17/73] factor out spawning of connection handler so it can be reused for creating outgoing connections change system stream to use it's own operation type --- .../src/engine/connection_handler/mod.rs | 15 +++-- .../src/engine/connection_handler/producer.rs | 4 +- .../engine/controller/controller_messages.rs | 57 ++++++++++++++++++ .../src/engine/controller/initialization.rs | 13 ++-- flo-server/src/engine/controller/mod.rs | 14 ++++- .../src/engine/controller/system_stream.rs | 5 +- .../src/engine/event_stream/partition/mod.rs | 41 +++++++++++-- flo-server/src/engine/mod.rs | 2 +- flo-server/src/flo_io/mod.rs | 56 ++++++++++++++++++ flo-server/src/server/mod.rs | 59 ++++--------------- 10 files changed, 193 insertions(+), 73 deletions(-) create mode 100644 flo-server/src/engine/controller/controller_messages.rs diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index c87d51b..3227e31 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -171,6 +171,7 @@ mod test { struct Fixture { #[allow(dead_code)] // TODO: add more connection handler tests partition_receivers: HashMap<(String, ActorId), PartitionReceiver>, + system_receiver: ::engine::controller::SystemPartitionReceiver, client_receiver: Option, engine: EngineRef, reactor: Core, @@ -185,26 +186,24 @@ mod test { let primary = AtomicBoolWriter::with_value(true); let primary_addr = Arc::new(RwLock::new(None)); - let (tx, rx) = create_partition_channels(); - let part_ref = PartitionRef::new(system_stream_name(), + let (tx, rx) = ::engine::controller::create_system_partition_channels(); + let part_ref = PartitionRef::system(system_stream_name(), 1, counter_writer.reader(), primary.reader(), - tx, + tx.clone(), primary_addr); - let system_stream = SystemStreamRef::new(part_ref); + let system_stream = SystemStreamRef::new(part_ref, tx); let streams = Arc::new(Mutex::new(HashMap::new())); let engine = EngineRef::new(None, system_stream, streams); let subject = ConnectionHandler::new(456, client_sender, engine.clone(), reactor.handle()); - let mut partition_receivers = HashMap::new(); - partition_receivers.insert((system_stream_name(), 1), rx); - let fixture = Fixture { - partition_receivers: partition_receivers, + partition_receivers: HashMap::new(), + system_receiver: rx, client_receiver: Some(client_rx), engine: engine, reactor: reactor diff --git a/flo-server/src/engine/connection_handler/producer.rs b/flo-server/src/engine/connection_handler/producer.rs index 5581f24..e7d6927 100644 --- a/flo-server/src/engine/connection_handler/producer.rs +++ b/flo-server/src/engine/connection_handler/producer.rs @@ -33,8 +33,8 @@ impl ProducerConnectionState { let receiver = { let partition = common_state.event_stream.get_partition(produce.partition).unwrap(); - partition.produce(connection_id, op_id, vec![produce]).map_err(|err| { - format!("Failed to send operation: {:?}", err.0) + partition.produce(connection_id, op_id, vec![produce]).map_err(|_| { + format!("Failed to send produce operation with op_id: {}", op_id) })? }; diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs new file mode 100644 index 0000000..2c9985f --- /dev/null +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -0,0 +1,57 @@ +use std::net::SocketAddr; +use std::time::Instant; + +use futures::sync::mpsc::UnboundedSender; + +use engine::event_stream::partition::{self, Operation}; +use engine::connection_handler::ConnectionControl; +use engine::ConnectionId; + +#[derive(Debug)] +pub struct ConnectionRef { + pub connection_id: ConnectionId, + pub remote_address: SocketAddr, + pub control_sender: UnboundedSender, +} + +#[derive(Debug)] +pub enum SystemOpType { + PartitionOp(partition::OpType), + IncomingConnectionEstablished(ConnectionRef), + ConnectionClosed(ConnectionId), + OutgoingConnectionFailed(SocketAddr), +} + +#[derive(Debug)] +pub struct SystemOperation { + pub connection_id: ConnectionId, + pub op_start_time: Instant, + pub op_type: SystemOpType, +} + +impl SystemOperation { + + pub fn outgoing_connection_failed(addr: SocketAddr) -> SystemOperation { + SystemOperation::new(0, SystemOpType::OutgoingConnectionFailed(addr)) + } + + fn new(connection_id: ConnectionId, op_type: SystemOpType) -> SystemOperation { + SystemOperation { + connection_id, + op_type, + op_start_time: Instant::now(), + } + } +} + + +impl From for SystemOperation { + fn from(op: Operation) -> SystemOperation { + let Operation { connection_id, client_message_recv_time, op_type } = op; + SystemOperation { + connection_id, + op_start_time: client_message_recv_time, + op_type: SystemOpType::PartitionOp(op_type), + } + } +} diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index 7f85d5a..f303dcc 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -8,6 +8,7 @@ use std::net::SocketAddr; use tokio_core::reactor::Remote; use engine::{EngineRef, system_stream_name, SYSTEM_STREAM_NAME}; +use engine::controller::{SystemPartitionSender, SystemPartitionReceiver, SystemOperation}; use engine::event_stream::{EventStreamRefMut, EventStreamOptions, init_existing_event_stream, @@ -66,7 +67,7 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul cluster_options, default_stream_options); - let (system_partition_tx, system_partition_rx) = create_partition_channels(); + let (system_partition_tx, system_partition_rx) = ::engine::controller::create_system_partition_channels(); let engine_ref = create_engine_ref(&flo_controller, system_highest_counter, system_primary_reader, system_primary_server_addr, system_partition_tx); run_controller_impl(flo_controller, system_partition_rx); @@ -100,16 +101,16 @@ fn create_engine_ref(controller: &FloController, system_highest_counter: AtomicCounterReader, system_primary_reader: AtomicBoolReader, system_primary_addr: Arc>>, - system_sender: PartitionSender) -> EngineRef { + system_partition_sender: SystemPartitionSender) -> EngineRef { - let system_partition_ref = PartitionRef::new(system_stream_name(), + let system_partition_ref = PartitionRef::system(system_stream_name(), 1, system_highest_counter, system_primary_reader, - system_sender, + system_partition_sender.clone(), system_primary_addr); + let system_stream_ref = SystemStreamRef::new(system_partition_ref, system_partition_sender); - let system_stream_ref = SystemStreamRef::new(system_partition_ref); let shared_stream_refs = controller.get_shared_streams(); let this_addr = controller.get_this_instance_address(); EngineRef::new(this_addr, system_stream_ref, shared_stream_refs) @@ -151,7 +152,7 @@ fn is_user_event_stream(dir_entry: &DirEntry) -> io::Result { } -fn run_controller_impl(mut controller: FloController, system_partition_rx: PartitionReceiver) { +fn run_controller_impl(mut controller: FloController, system_partition_rx: SystemPartitionReceiver) { ::std::thread::spawn(move || { debug!("Starting FloController processing"); while let Ok(operation) = system_partition_rx.recv() { diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index b9233c3..73e0302 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -1,6 +1,7 @@ mod cluster_state; mod system_stream; mod initialization; +mod controller_messages; use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; @@ -19,7 +20,14 @@ use atomics::AtomicBoolWriter; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; pub use self::system_stream::SystemStreamRef; +pub use self::controller_messages::{SystemOperation, SystemOpType, ConnectionRef}; +pub type SystemPartitionSender = ::std::sync::mpsc::Sender; +pub type SystemPartitionReceiver = ::std::sync::mpsc::Receiver; + +pub fn create_system_partition_channels() -> (SystemPartitionSender, SystemPartitionReceiver) { + ::std::sync::mpsc::channel() +} /// A specialized event stream that always has exactly one partition and manages the cluster state and consensus /// Of course there is no cluster state and thus no consensus at the moment, but we'll just leave this here... @@ -77,8 +85,10 @@ impl FloController { } } - fn process(&mut self, _operation: Operation) { - unimplemented!() + fn process(&mut self, operation: SystemOperation) { + debug!("Processing: {:?}", operation); + + //TODO: handle system operations } fn shutdown(&mut self) { diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index f49c684..54dffa6 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -2,19 +2,22 @@ use std::net::SocketAddr; use engine::event_stream::partition::{PartitionRef, Operation}; use engine::event_stream::EventStreamRef; +use engine::controller::SystemPartitionSender; use engine::ConnectionId; #[derive(Clone, Debug)] pub struct SystemStreamRef { + system_sender: SystemPartitionSender, inner: PartitionRef, } impl SystemStreamRef { - pub fn new(partition_ref: PartitionRef) -> SystemStreamRef { + pub fn new(partition_ref: PartitionRef, system_sender: SystemPartitionSender) -> SystemStreamRef { SystemStreamRef { + system_sender, inner: partition_ref, } } diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index ae873b2..c6c1914 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -15,6 +15,7 @@ use std::io; use atomics::{AtomicCounterReader, AtomicBoolReader, AtomicBoolWriter}; use engine::ConnectionId; use engine::event_stream::{EventStreamOptions, HighestCounter}; +use engine::controller::SystemPartitionSender; use protocol::{ProduceEvent}; use event::{EventCounter, ActorId}; use self::segment::SegmentReader; @@ -42,7 +43,7 @@ pub fn create_partition_channels() -> (PartitionSender, PartitionReceiver) { } #[derive(Debug)] -pub struct PartitionSendError(pub Operation); +pub struct PartitionSendError; pub type PartitionSendResult = Result<(), PartitionSendError>; @@ -174,6 +175,12 @@ impl PartitionRefMut { } } +#[derive(Debug, Clone)] +enum SenderType { + Normal(PartitionSender), + System(SystemPartitionSender) +} + #[derive(Clone, Debug)] pub struct PartitionRef { event_stream_name: String, @@ -181,7 +188,7 @@ pub struct PartitionRef { highest_event_counter: AtomicCounterReader, primary: AtomicBoolReader, primary_server_address: Arc>>, - sender: PartitionSender, + sender: SenderType, } impl PartitionRef { @@ -191,10 +198,22 @@ impl PartitionRef { partition_num, highest_event_counter, primary, - sender, + sender: SenderType::Normal(sender), primary_server_address, } } + + pub fn system(event_stream_name: String, partition_num: ActorId, highest_event_counter: AtomicCounterReader, primary: AtomicBoolReader, sender: SystemPartitionSender, primary_server_address: Arc>>) -> PartitionRef { + PartitionRef { + event_stream_name, + partition_num, + highest_event_counter, + primary, + sender: SenderType::System(sender), + primary_server_address, + } + } + pub fn partition_num(&self) -> ActorId { self.partition_num } @@ -236,9 +255,19 @@ impl PartitionRef { } pub fn send(&mut self, op: Operation) -> PartitionSendResult { - self.sender.send(op).map_err(|err| { - PartitionSendError(err.0) - }) + match self.sender { + SenderType::Normal(ref mut sender) => { + sender.send(op).map_err(|_| { + PartitionSendError + }) + } + SenderType::System(ref mut sender) => { + sender.send(op.into()).map_err(|_| { + PartitionSendError + }) + } + } + } } diff --git a/flo-server/src/engine/mod.rs b/flo-server/src/engine/mod.rs index 8a26b5f..d851d81 100644 --- a/flo-server/src/engine/mod.rs +++ b/flo-server/src/engine/mod.rs @@ -1,7 +1,7 @@ pub mod event_stream; +pub mod connection_handler; mod controller; -mod connection_handler; use std::collections::HashMap; use std::sync::{Arc, Mutex}; diff --git a/flo-server/src/flo_io/mod.rs b/flo-server/src/flo_io/mod.rs index 7586f09..cb42253 100644 --- a/flo-server/src/flo_io/mod.rs +++ b/flo-server/src/flo_io/mod.rs @@ -1,6 +1,62 @@ mod client_message_stream; mod server_message_stream; + +use std::net::SocketAddr; +use std::io; +#[allow(deprecated)] +use tokio_core::io::Io; +use tokio_core::reactor::Handle; +use tokio_core::net::TcpStream; +use futures::{Stream, Sink, Future}; + +use engine::{create_client_channels, ConnectionHandler}; +use engine::connection_handler::ConnectionHandlerInput; +use engine::EngineRef; + pub use self::client_message_stream::ProtocolMessageStream; pub use self::server_message_stream::ServerMessageStream; + + +pub fn spawn_connection_handler(client_handle: Handle, client_engine_ref: EngineRef, client_addr: SocketAddr, tcp_stream: TcpStream) -> Box> { + let connection_id = client_engine_ref.next_connection_id(); + info!("Opened connection_id: {} to address: {}", connection_id, client_addr); + let (client_tx, client_rx) = create_client_channels(); + + #[allow(deprecated)] + let (tcp_reader, tcp_writer) = tcp_stream.split(); + + let server_to_client = ServerMessageStream::new(connection_id, client_rx, tcp_writer); + + let client_message_stream = ProtocolMessageStream::new(connection_id, tcp_reader) + .map(|proto_message| proto_message.into()); + + let (control_tx, control_rx) = ::futures::sync::mpsc::unbounded::(); + + let joint_stream = control_rx + .map(|control| control.into()) + .map_err(|recv_err| { + io::Error::new(io::ErrorKind::Other, format!("Error receiving from control channel: {:?}", recv_err)) + }) + .select(client_message_stream); + + let connection_handler = ConnectionHandler::new( + connection_id, + client_tx.clone(), + client_engine_ref, + client_handle); + + let client_to_server = connection_handler + .send_all(joint_stream) + .map(|_| ()); + + let future = client_to_server.select(server_to_client).then(move |res| { + if let Err((err, _)) = res { + warn!("Closing connection: {} due to err: {:?}", connection_id, err); + } + info!("Closed connection_id: {} to address: {}", connection_id, client_addr); + Ok(()) + }); + Box::new(future) +} diff --git a/flo-server/src/server/mod.rs b/flo-server/src/server/mod.rs index a9b07fa..4b048ee 100644 --- a/flo-server/src/server/mod.rs +++ b/flo-server/src/server/mod.rs @@ -1,28 +1,22 @@ mod server_options; -use futures::{Stream, Sink, Future}; -use tokio_core::net::{TcpStream, TcpListener}; - -use event_loops; use std::net::{SocketAddr, Ipv4Addr, SocketAddrV4}; use std::io; +use tokio_core::net::{TcpStream, TcpListener}; +use futures::{Stream, Future}; + +use engine::{ControllerOptions, + ClusterOptions, + start_controller, + system_stream_name}; +use engine::event_stream::EventStreamOptions; +use flo_io::spawn_connection_handler; +use event_loops; pub use self::server_options::{ServerOptions, MemoryLimit, MemoryUnit}; - - pub fn run(mut options: ServerOptions) -> io::Result<()> { - #[allow(deprecated)] - use tokio_core::io::Io; - use engine::{ControllerOptions, - ClusterOptions, - start_controller, - system_stream_name, - create_client_channels, - ConnectionHandler}; - use engine::event_stream::EventStreamOptions; - use flo_io::{ProtocolMessageStream, ServerMessageStream}; const ONE_GB: usize = 1024 * 1024 * 1024; @@ -74,40 +68,10 @@ pub fn run(mut options: ServerOptions) -> io::Result<()> { () })?; let client_engine_ref = engine_ref.clone(); - let connection_id = client_engine_ref.next_connection_id(); let remote_handle = event_loop_handles.next_handle(); - let (client_tx, client_rx) = create_client_channels(); - - info!("Opened connection_id: {} to address: {}", connection_id, client_addr); - remote_handle.spawn(move |client_handle| { - - #[allow(deprecated)] - let (tcp_reader, tcp_writer) = tcp_stream.split(); - - let server_to_client = ServerMessageStream::new(connection_id, client_rx, tcp_writer); - - let client_message_stream = ProtocolMessageStream::new(connection_id, tcp_reader) - .map(|proto_message| proto_message.into()); - let connection_handler = ConnectionHandler::new( - connection_id, - client_tx.clone(), - client_engine_ref, - client_handle.clone()); - - let client_to_server = connection_handler - .send_all(client_message_stream) - .map(|_| ()); - - client_to_server.select(server_to_client).then(move |res| { - if let Err((err, _)) = res { - warn!("Closing connection: {} due to err: {:?}", connection_id, err); - } - info!("Closed connection_id: {} to address: {}", connection_id, client_addr); - Ok(()) - }) - + spawn_connection_handler(client_handle.clone(), client_engine_ref, client_addr, tcp_stream) }); Ok(()) @@ -118,3 +82,4 @@ pub fn run(mut options: ServerOptions) -> io::Result<()> { Ok(()) } + From 2ea608160605b53a1a683e4b34ea7d3ebca9df2f Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 9 Dec 2017 16:20:16 -0500 Subject: [PATCH 18/73] more shuffling things aroung in preparation for outgoing connections --- .../src/engine/connection_handler/mod.rs | 7 ++ .../engine/controller/controller_messages.rs | 8 +- flo-server/src/engine/controller/mod.rs | 1 + .../engine/controller/peer_connection/mod.rs | 35 ++++++ .../controller/peer_connection/system.rs | 113 ++---------------- .../src/engine/controller/system_stream.rs | 13 +- flo-server/src/engine/mod.rs | 3 +- flo-server/src/flo_io/mod.rs | 17 +-- flo-server/src/server/mod.rs | 46 ++++--- 9 files changed, 112 insertions(+), 131 deletions(-) diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 3227e31..6525d97 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -20,6 +20,13 @@ use self::peer::PeerConnectionState; pub use self::input::{ConnectionHandlerInput, ConnectionControl}; +pub type ConnectionControlSender = ::futures::sync::mpsc::UnboundedSender; +pub type ConnectionControlReceiver = ::futures::sync::mpsc::UnboundedReceiver; + +pub fn create_connection_control_channels() -> (ConnectionControlSender, ConnectionControlReceiver) { + ::futures::sync::mpsc::unbounded() +} + pub struct ConnectionHandler { common_state: ConnectionState, consumer_state: ConsumerConnectionState, diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 2c9985f..40829df 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -4,14 +4,14 @@ use std::time::Instant; use futures::sync::mpsc::UnboundedSender; use engine::event_stream::partition::{self, Operation}; -use engine::connection_handler::ConnectionControl; +use engine::connection_handler::{ConnectionControl, ConnectionControlSender}; use engine::ConnectionId; #[derive(Debug)] pub struct ConnectionRef { pub connection_id: ConnectionId, pub remote_address: SocketAddr, - pub control_sender: UnboundedSender, + pub control_sender: ConnectionControlSender, } #[derive(Debug)] @@ -31,6 +31,10 @@ pub struct SystemOperation { impl SystemOperation { + pub fn incoming_connection_established(connection: ConnectionRef) -> SystemOperation { + SystemOperation::new(connection.connection_id, SystemOpType::IncomingConnectionEstablished(connection)) + } + pub fn outgoing_connection_failed(addr: SocketAddr) -> SystemOperation { SystemOperation::new(0, SystemOpType::OutgoingConnectionFailed(addr)) } diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 73e0302..3bd3067 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -2,6 +2,7 @@ mod cluster_state; mod system_stream; mod initialization; mod controller_messages; +mod peer_connection; use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; diff --git a/flo-server/src/engine/controller/peer_connection/mod.rs b/flo-server/src/engine/controller/peer_connection/mod.rs index 361598c..794a3fc 100644 --- a/flo-server/src/engine/controller/peer_connection/mod.rs +++ b/flo-server/src/engine/controller/peer_connection/mod.rs @@ -4,9 +4,14 @@ mod system; use std::fmt::Debug; use std::net::SocketAddr; use std::io; +use tokio_core::reactor::Handle; +use tokio_core::net::TcpStream; +use futures::Future; use engine::EngineRef; +use engine::connection_handler::{create_connection_control_channels, ConnectionControlSender}; use event_loops::LoopHandles; +use flo_io::create_connection_handler; pub use self::replication::PeerReplicationConnection; pub use self::system::PeerSystemConnection; @@ -36,6 +41,36 @@ impl OutgoingConnectionCreator for OutgoingConnectionCreatorImpl { } +fn create_outgoing_connection(loops: &mut LoopHandles, client_addr: SocketAddr, engine_ref: EngineRef) -> ConnectionControlSender { + + let client_addr_copy = client_addr.clone(); + let mut system_stream = engine_ref.get_system_stream(); + + let (control_tx, control_rx) = create_connection_control_channels(); + + loops.next_handle().spawn( move |handle| { + + let owned_handle = handle.clone(); + let addr = client_addr_copy; + TcpStream::connect(&addr, handle).map_err( move |io_err| { + + error!("Failed to create outgoing connection to address: {:?}: {:?}", addr, io_err); + system_stream.outgoing_connection_failed(0, addr); + + }).and_then( move |tcp_stream| { + let connection_id = engine_ref.next_connection_id(); + + create_connection_handler(owned_handle, + engine_ref, + connection_id, + client_addr, + tcp_stream, + control_rx) + }) + }); + + control_tx +} diff --git a/flo-server/src/engine/controller/peer_connection/system.rs b/flo-server/src/engine/controller/peer_connection/system.rs index 9b3e1f3..cc65762 100644 --- a/flo-server/src/engine/controller/peer_connection/system.rs +++ b/flo-server/src/engine/controller/peer_connection/system.rs @@ -8,122 +8,31 @@ use tokio_core::net::TcpStream; use tokio_core::io::Io; use futures::{Future, Stream, Sink}; -use event::VersionVector; +use protocol::{FloInstanceId, Term}; +use event::EventCounter; use event_loops::LoopHandles; use engine::controller::SystemStreamRef; -use engine::{EngineRef, create_client_channels, ReceivedProtocolMessage, ClientSender, ClientReceiver}; -use engine::system_stream::{FloInstanceId, AppendEntriesCall, RequestVoteCall}; +use engine::connection_handler::ConnectionControlSender; use super::ConnectionSendResult; +pub struct CallAppendEntries { + pub leader_id: FloInstanceId, + pub term: Term, + pub prev_entry_term: Term, + pub prev_entry_index: EventCounter, + pub leader_commit_index: EventCounter, +} /// Trait representing an active peer connection, with functions to control it pub trait PeerSystemConnection: Debug + Send + 'static { - fn send_append_entries(&mut self, append_entries: AppendEntriesCall) -> ConnectionSendResult; - fn send_request_vote(&mut self, request: RequestVoteCall) -> ConnectionSendResult; -} - - - -enum SystemConnectionMessage { - AppendEntries(AppendEntriesCall), - Vote(RequestVoteCall), -} - -fn create_outgoing_connection(loops: &mut LoopHandles, client_addr: SocketAddr, engine_ref: EngineRef) -> Box { - - // These copies are to work around the ownership rules, since we want to have slightly different error handling for - // connection failures depending on when it fails - let client_addr_copy = client_addr.clone(); - - let mut system_stream = engine_ref.get_system_stream(); - let mut system_stream_copy = system_stream.clone(); - - let (client_tx, client_rx) = create_client_channels(); - let client_tx_copy = client_tx.clone(); - - loops.next_handle().spawn( move |handle| { - let owned_handle = handle.clone(); - let addr = client_addr_copy; - TcpStream::connect(&addr, handle).map_err( move |io_err| { - - error!("Failed to create outgoing connection to address: {:?}: {:?}", addr, io_err); - system_stream.outgoing_connection_failed(0, addr); - - }).and_then( move |tcp_stream| { - - outgoing_connection_future(owned_handle, - engine_ref, - client_addr, - tcp_stream, - client_tx_copy, - client_rx, - system_stream_copy) - }) - }); - - let outgoing = OutgoingPeerSystemConnection { - client_tx - }; - Box::new(outgoing) } -fn outgoing_connection_future(handle: Handle, - engine_ref: EngineRef, - client_addr: SocketAddr, - tcp_stream: TcpStream, - client_tx: ClientSender, - client_rx: ClientReceiver, - mut system_stream: SystemStreamRef) -> Box> { - - use flo_io::{ProtocolMessageStream, ServerMessageStream}; - use engine::ConnectionHandler; - - let connection_id = engine_ref.next_connection_id(); - debug!("Established connection to {:?} with connection_id: {}", client_addr, connection_id); - if let Err(io_err) = tcp_stream.set_nodelay(true) { - error!("Error setting NODELAY for connection_id: {}. Nagle yet lives!: {:?}", connection_id, io_err); - } - #[allow(deprecated)] - let (read, write) = tcp_stream.split(); - let server_to_client = ServerMessageStream::new(connection_id, client_rx, write); - let client_message_stream = ProtocolMessageStream::new(connection_id, read); - let mut connection_handler = ConnectionHandler::new( - connection_id, - client_tx, - engine_ref, - handle.clone()); - // sends PeerAnnounce message to the other server and sets up the connection handler to expect the response - connection_handler.upgrade_to_outgoing_peer(); - let client_to_server = connection_handler - .send_all(client_message_stream) - .map(|_| ()); - - let future = client_to_server.select(server_to_client).then(move |res| { - if let Err((err, _)) = res { - warn!("Closing outgoing connection: {} due to err: {:?}", connection_id, err); - system_stream.outgoing_connection_failed(connection_id, client_addr); - } - info!("Closed connection_id: {} to address: {}", connection_id, client_addr); - Ok(()) - }); - Box::new(future) -} #[derive(Debug)] struct OutgoingPeerSystemConnection { - client_tx: ClientSender -} - -impl PeerSystemConnection for OutgoingPeerSystemConnection { - fn send_append_entries(&mut self, append_entries: AppendEntriesCall) -> ConnectionSendResult { - unimplemented!() - } - - fn send_request_vote(&mut self, request: RequestVoteCall) -> ConnectionSendResult { - unimplemented!() - } + client_tx: ConnectionControlSender, } diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index 54dffa6..10a34d7 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -2,7 +2,7 @@ use std::net::SocketAddr; use engine::event_stream::partition::{PartitionRef, Operation}; use engine::event_stream::EventStreamRef; -use engine::controller::SystemPartitionSender; +use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef}; use engine::ConnectionId; @@ -28,11 +28,18 @@ impl SystemStreamRef { EventStreamRef::new(name, partition_ref) } + pub fn incomming_connection_accepted(&mut self, connection_ref: ConnectionRef) { + let op = SystemOperation::incoming_connection_established(connection_ref); + self.send(op); + } + pub fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, socket_addr: SocketAddr) { - let op = Operation::outgoing_connection_failed(connection_id, socket_addr); - self.inner.send(op).expect("System Stream has shutdown"); + unimplemented!() } + fn send(&mut self, op: SystemOperation) { + self.system_sender.send(op).expect("Failed to send to flo controller. System must have shut down"); + } } diff --git a/flo-server/src/engine/mod.rs b/flo-server/src/engine/mod.rs index d851d81..c2f6f6d 100644 --- a/flo-server/src/engine/mod.rs +++ b/flo-server/src/engine/mod.rs @@ -1,7 +1,6 @@ pub mod event_stream; pub mod connection_handler; - -mod controller; +pub mod controller; use std::collections::HashMap; use std::sync::{Arc, Mutex}; diff --git a/flo-server/src/flo_io/mod.rs b/flo-server/src/flo_io/mod.rs index cb42253..fd74772 100644 --- a/flo-server/src/flo_io/mod.rs +++ b/flo-server/src/flo_io/mod.rs @@ -11,16 +11,21 @@ use tokio_core::net::TcpStream; use futures::{Stream, Sink, Future}; use engine::{create_client_channels, ConnectionHandler}; -use engine::connection_handler::ConnectionHandlerInput; -use engine::EngineRef; +use engine::connection_handler::{ConnectionHandlerInput, ConnectionControlReceiver}; +use engine::{EngineRef, ConnectionId}; pub use self::client_message_stream::ProtocolMessageStream; pub use self::server_message_stream::ServerMessageStream; -pub fn spawn_connection_handler(client_handle: Handle, client_engine_ref: EngineRef, client_addr: SocketAddr, tcp_stream: TcpStream) -> Box> { - let connection_id = client_engine_ref.next_connection_id(); +pub fn create_connection_handler(client_handle: Handle, + client_engine_ref: EngineRef, + connection_id: ConnectionId, + client_addr: SocketAddr, + tcp_stream: TcpStream, + control_receiver: ConnectionControlReceiver) -> Box> { + info!("Opened connection_id: {} to address: {}", connection_id, client_addr); let (client_tx, client_rx) = create_client_channels(); @@ -32,9 +37,7 @@ pub fn spawn_connection_handler(client_handle: Handle, client_engine_ref: Engine let client_message_stream = ProtocolMessageStream::new(connection_id, tcp_reader) .map(|proto_message| proto_message.into()); - let (control_tx, control_rx) = ::futures::sync::mpsc::unbounded::(); - - let joint_stream = control_rx + let joint_stream = control_receiver .map(|control| control.into()) .map_err(|recv_err| { io::Error::new(io::ErrorKind::Other, format!("Error receiving from control channel: {:?}", recv_err)) diff --git a/flo-server/src/server/mod.rs b/flo-server/src/server/mod.rs index 4b048ee..0269c1b 100644 --- a/flo-server/src/server/mod.rs +++ b/flo-server/src/server/mod.rs @@ -3,22 +3,26 @@ mod server_options; use std::net::{SocketAddr, Ipv4Addr, SocketAddrV4}; use std::io; +use tokio_core::reactor::Remote; use tokio_core::net::{TcpStream, TcpListener}; use futures::{Stream, Future}; use engine::{ControllerOptions, + EngineRef, ClusterOptions, start_controller, system_stream_name}; +use engine::connection_handler::create_connection_control_channels; +use engine::controller::ConnectionRef; use engine::event_stream::EventStreamOptions; -use flo_io::spawn_connection_handler; +use flo_io::create_connection_handler; use event_loops; pub use self::server_options::{ServerOptions, MemoryLimit, MemoryUnit}; -pub fn run(mut options: ServerOptions) -> io::Result<()> { +const ONE_GB: usize = 1024 * 1024 * 1024; - const ONE_GB: usize = 1024 * 1024 * 1024; +pub fn run(mut options: ServerOptions) -> io::Result<()> { let (join_handle, mut event_loop_handles) = event_loops::spawn_event_loop_threads(options.max_io_threads).unwrap(); @@ -63,18 +67,8 @@ pub fn run(mut options: ServerOptions) -> io::Result<()> { incoming.map_err(|io_err| { error!("Error creating new connection: {:?}", io_err); }).for_each(move |(tcp_stream, client_addr): (TcpStream, SocketAddr)| { - tcp_stream.set_nodelay(true).map_err(|io_err| { - error!("Error setting NODELAY. Nagle yet lives!: {:?}", io_err); - () - })?; - let client_engine_ref = engine_ref.clone(); - let remote_handle = event_loop_handles.next_handle(); - - remote_handle.spawn(move |client_handle| { - spawn_connection_handler(client_handle.clone(), client_engine_ref, client_addr, tcp_stream) - }); - - Ok(()) + + handle_incoming_connection(&engine_ref, event_loop_handles.next_handle(), tcp_stream, client_addr) }) }); @@ -82,4 +76,26 @@ pub fn run(mut options: ServerOptions) -> io::Result<()> { Ok(()) } +fn handle_incoming_connection(engine_ref: &EngineRef, remote_handle: Remote, tcp_stream: TcpStream, client_addr: SocketAddr) -> Result<(), ()> { + tcp_stream.set_nodelay(true).map_err(|io_err| { + error!("Error setting NODELAY. Nagle yet lives!: {:?}", io_err); + () + })?; + + let mut client_engine_ref = engine_ref.clone(); + let connection_id = client_engine_ref.next_connection_id(); + let (control_tx, control_rx) = create_connection_control_channels(); + let connection_ref = ConnectionRef { + connection_id, + remote_address: client_addr, + control_sender: control_tx, + }; + client_engine_ref.system_stream().incomming_connection_accepted(connection_ref); + + remote_handle.spawn(move |client_handle| { + create_connection_handler(client_handle.clone(), client_engine_ref, connection_id, client_addr, tcp_stream, control_rx) + }); + + Ok(()) +} From 364e92dc2e91a8d51c17c898337b3853d8e37026 Mon Sep 17 00:00:00 2001 From: pfried Date: Wed, 13 Dec 2017 21:23:20 -0500 Subject: [PATCH 19/73] WIP on partition controller and managing peer connections, mostly just beating around the bush tbh --- flo-server/src/atomics/atomic_counter.rs | 8 +- .../consumer_stream/multi_partition_reader.rs | 2 +- .../src/engine/connection_handler/input.rs | 8 + .../{peer.rs => peer/mod.rs} | 1 + .../connection_handler/peer/peer_follower.rs | 25 +++ .../partition/controller/commit_manager.rs | 169 ++++++++++++++++++ .../event_stream/partition/controller/mod.rs | 28 ++- .../partition/event_reader/mod.rs | 25 ++- .../src/engine/event_stream/partition/mod.rs | 2 +- .../event_stream/partition/segment/mod.rs | 8 +- 10 files changed, 256 insertions(+), 20 deletions(-) rename flo-server/src/engine/connection_handler/{peer.rs => peer/mod.rs} (99%) create mode 100644 flo-server/src/engine/connection_handler/peer/peer_follower.rs create mode 100644 flo-server/src/engine/event_stream/partition/controller/commit_manager.rs diff --git a/flo-server/src/atomics/atomic_counter.rs b/flo-server/src/atomics/atomic_counter.rs index af73f50..3fa67a0 100644 --- a/flo-server/src/atomics/atomic_counter.rs +++ b/flo-server/src/atomics/atomic_counter.rs @@ -28,11 +28,15 @@ impl AtomicCounterWriter { old + amount } - /// sets the new value, only if it is greater than the current value - pub fn set_if_greater(&mut self, new_value: usize) { + /// sets the new value, only if it is greater than the current value. Returns true if the value was updated, and + /// otherwise false + pub fn set_if_greater(&mut self, new_value: usize) -> bool { let current = self.inner.load(Ordering::SeqCst); if new_value > current { self.inner.store(new_value, Ordering::SeqCst); + true + } else { + false } } diff --git a/flo-server/src/engine/connection_handler/consumer/consumer_stream/multi_partition_reader.rs b/flo-server/src/engine/connection_handler/consumer/consumer_stream/multi_partition_reader.rs index cc9982f..6f16f97 100644 --- a/flo-server/src/engine/connection_handler/consumer/consumer_stream/multi_partition_reader.rs +++ b/flo-server/src/engine/connection_handler/consumer/consumer_stream/multi_partition_reader.rs @@ -56,7 +56,7 @@ struct PartReaderInternal { impl PartReaderInternal { fn advance(&mut self) { if self.next_val.is_none() { - let next = self.reader.next_matching(); + let next = self.reader.read_next_uncommitted(); self.next_val = next; } } diff --git a/flo-server/src/engine/connection_handler/input.rs b/flo-server/src/engine/connection_handler/input.rs index 2d8959b..18819b2 100644 --- a/flo-server/src/engine/connection_handler/input.rs +++ b/flo-server/src/engine/connection_handler/input.rs @@ -1,4 +1,6 @@ +use protocol::{FloInstanceId, Term}; +use event::EventCounter; use engine::ReceivedProtocolMessage; #[derive(Debug)] @@ -35,3 +37,9 @@ impl From for ConnectionHandlerInput { ConnectionHandlerInput::Control(control) } } + +pub struct SendAppendEntries { + pub current_term: Term, + pub prev_entry_index: EventCounter, + pub prev_entry_term: Term, +} diff --git a/flo-server/src/engine/connection_handler/peer.rs b/flo-server/src/engine/connection_handler/peer/mod.rs similarity index 99% rename from flo-server/src/engine/connection_handler/peer.rs rename to flo-server/src/engine/connection_handler/peer/mod.rs index 11d9d80..99baaa7 100644 --- a/flo-server/src/engine/connection_handler/peer.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -1,3 +1,4 @@ +mod peer_follower; use protocol::{ProtocolMessage, PeerAnnounce, EventStreamStatus}; use engine::{ReceivedProtocolMessage, ConnectionId}; diff --git a/flo-server/src/engine/connection_handler/peer/peer_follower.rs b/flo-server/src/engine/connection_handler/peer/peer_follower.rs new file mode 100644 index 0000000..a30544b --- /dev/null +++ b/flo-server/src/engine/connection_handler/peer/peer_follower.rs @@ -0,0 +1,25 @@ + +use event::EventCounter; +use engine::event_stream::partition::PartitionReader; +use super::{ConnectionState, ConnectionHandlerResult}; + +pub struct PeerFollowerState { + last_acknowledged_id: EventCounter, + reader: PartitionReader, +} + + + +impl PeerFollowerState { + pub fn new(last_acknowledged_id: EventCounter, reader: PartitionReader) -> PeerFollowerState { + PeerFollowerState { + last_acknowledged_id, + reader + } + } + + + pub fn send_append_entries(&mut self, connection_state: &mut ConnectionState) -> ConnectionHandlerResult { + unimplemented!() + } +} diff --git a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs new file mode 100644 index 0000000..c249488 --- /dev/null +++ b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs @@ -0,0 +1,169 @@ + +use event::{EventCounter, ActorId}; +use protocol::FloInstanceId; +use atomics::{AtomicCounterReader, AtomicCounterWriter}; + + + +pub struct CommitManager { + /// The current commit index. Cannot ever go backwards + commit_index: AtomicCounterWriter, + /// The number of acknowledgements that must be received from peers in order to get a majority. + /// Note that this number does _not_ include the implicit acknowledgement from this instance. + /// Once this number of acks has been received from peers, the event will be considered committed + min_required_for_commit: ActorId, + /// The _other_ members of this cluster. Does not include this instance + peers: Vec<(FloInstanceId, EventCounter)>, +} + + +impl CommitManager { + + pub fn new(commit_index: AtomicCounterWriter) -> CommitManager { + CommitManager { + commit_index, + min_required_for_commit: 0, + peers: Vec::new(), + } + } + + pub fn acknowledgement_received(&mut self, from: FloInstanceId, acknowledged: EventCounter) -> Option { + let peer_count = self.peers.len(); + + for i in 0..peer_count { + let &mut (ref id, ref mut counter) = &mut self.peers[i]; + if *id == from { + *counter = acknowledged; + + } + } + + self.peers.sort_by_key(|elem| elem.1); + + let idx = self.min_required_for_commit; + let mut ack_counter = self.peers[idx as usize].1; + + if self.commit_index.set_if_greater(ack_counter as usize) { + Some(ack_counter) + } else { + None + } + } + + pub fn update_commit_index(&mut self, new_index: EventCounter) { + self.commit_index.set_if_greater(new_index as usize); + } + + pub fn add_member(&mut self, peer_id: FloInstanceId) { + self.peers.push((peer_id, 0)); + let new_ack_requirement = self.compute_min_required(); + self.min_required_for_commit = new_ack_requirement; + } + + pub fn get_commit_index_reader(&self) -> AtomicCounterReader { + self.commit_index.reader() + } + + fn compute_min_required(&self) -> ActorId { + let peer_count = self.peers.len() as ActorId; + + match peer_count { + 0 => 0, + 1 => 1, + 2 => 1, + other @ _ if other % 2 == 0 => { + other / 2 + } + other @ _ => { + (other / 2) + 1 + } + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + fn subject_with_peers(peers: &[FloInstanceId]) -> CommitManager { + let mut subject = CommitManager::new(AtomicCounterWriter::with_value(0)); + for id in peers { + subject.add_member(*id); + } + subject + } + + #[test] + fn commit_index_is_incremented_when_a_majority_of_all_members_have_acknowledged_an_event_greater_than_the_one_acknowledged() { + let peers = [ + FloInstanceId::generate_new(), + FloInstanceId::generate_new(), + FloInstanceId::generate_new(), + FloInstanceId::generate_new() + ]; + let mut subject = subject_with_peers(&peers[..]); + let commit_reader = subject.get_commit_index_reader(); + + assert!(subject.acknowledgement_received(peers[0], 25).is_none()); + let result = subject.acknowledgement_received(peers[1], 17); + assert_eq!(Some(17), result); + assert_eq!(17, commit_reader.load_relaxed() as u64); + + let result = subject.acknowledgement_received(peers[2], 19); + assert_eq!(Some(19), result); + assert_eq!(19, commit_reader.load_relaxed() as u64); + + // here's the tricky part. When peer[3] acknowledges event 30, the commit index should only jump to 25 + let result = subject.acknowledgement_received(peers[3], 30); + assert_eq!(Some(25), result); + assert_eq!(25, commit_reader.load_relaxed() as u64); + } + + #[test] + fn commit_index_is_incremented_when_a_majority_of_all_members_have_acknowledged_a_specific_event() { + let peers = [ + FloInstanceId::generate_new(), + FloInstanceId::generate_new(), + FloInstanceId::generate_new(), + FloInstanceId::generate_new() + ]; + let mut subject = subject_with_peers(&peers[..]); + + let event = 5; + assert!(subject.acknowledgement_received(peers[0], event).is_none()); + let result = subject.acknowledgement_received(peers[1], event); + assert_eq!(Some(event), result); + assert_eq!(event, subject.get_commit_index_reader().load_relaxed() as u64); + } + + #[test] + fn adding_members_sets_then_minimum_required_for_commit() { + let mut subject = CommitManager::new(AtomicCounterWriter::with_value(0)); + + assert_eq!(0, subject.min_required_for_commit); + + subject.add_member(FloInstanceId::generate_new()); + assert_eq!(1, subject.peers.len()); + assert_eq!(1, subject.min_required_for_commit); // 2 of 2 + + subject.add_member(FloInstanceId::generate_new()); + assert_eq!(2, subject.peers.len()); + assert_eq!(1, subject.min_required_for_commit); // 2 of 3 + + subject.add_member(FloInstanceId::generate_new()); + assert_eq!(3, subject.peers.len()); + assert_eq!(2, subject.min_required_for_commit); // 3 of 4 + + subject.add_member(FloInstanceId::generate_new()); + assert_eq!(4, subject.peers.len()); + assert_eq!(2, subject.min_required_for_commit); // 3 of 5 + + subject.add_member(FloInstanceId::generate_new()); + subject.add_member(FloInstanceId::generate_new()); + subject.add_member(FloInstanceId::generate_new()); + + assert_eq!(4, subject.min_required_for_commit); // 5 of 8 + } +} + + diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index e703acb..700d6e6 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -1,5 +1,6 @@ mod util; mod consumer_manager; +mod commit_manager; use std::io; use std::collections::VecDeque; @@ -29,7 +30,7 @@ pub struct PartitionImpl { segments: VecDeque, index: PartitionIndex, event_stream_highest_counter: HighestCounter, - partition_highest_counter: AtomicCounterWriter, + partition_highest_committed: AtomicCounterWriter, primary: AtomicBoolReader, /// new segments each have a reader added here. The readers are then accessed as needed by the EventReader @@ -61,6 +62,8 @@ impl PartitionImpl { initialized_segments.push_front(segment); reader_refs.add(reader); } + + //TODO: differentiate between highest committed and highest uncommitted when initializing existing partition let current_greatest_id = index.greatest_event_counter(); highest_counter.set_if_greater(current_greatest_id); let partition_id_counter = AtomicCounterWriter::with_value(current_greatest_id as usize); @@ -84,7 +87,7 @@ impl PartitionImpl { segments: initialized_segments, index: index, event_stream_highest_counter: highest_counter, - partition_highest_counter: partition_id_counter, + partition_highest_committed: partition_id_counter, primary: status_reader, reader_refs: reader_refs, consumer_manager: ConsumerManager::new(), @@ -108,7 +111,7 @@ impl PartitionImpl { segments: VecDeque::with_capacity(4), index: PartitionIndex::new(partition_num), event_stream_highest_counter: highest_counter, - partition_highest_counter: AtomicCounterWriter::zero(), + partition_highest_committed: AtomicCounterWriter::zero(), primary: status_reader, reader_refs: SharedReaderRefsMut::new(), consumer_manager: ConsumerManager::new(), @@ -120,7 +123,7 @@ impl PartitionImpl { } pub fn event_counter_reader(&self) -> AtomicCounterReader { - self.partition_highest_counter.reader() + self.partition_highest_committed.reader() } pub fn primary_status_reader(&self) -> AtomicBoolReader { @@ -211,8 +214,9 @@ impl PartitionImpl { self.append(&event)?; } debug!("partition: {} finished appending {} events ending with counter: {}", self.partition_num, event_count, event_counter); - // now increment our counter and notify consumers - self.partition_highest_counter.increment_and_get_relaxed(event_count); + + // fence to make sure that events are actually done being saved prior to the notify, since + // consumers may then immediately read that region of memory ::std::sync::atomic::fence(::std::sync::atomic::Ordering::SeqCst); self.consumer_manager.notify_uncommitted(); Ok(FloEventId::new(self.partition_num, event_counter)) @@ -321,7 +325,13 @@ impl PartitionImpl { } }; - PartitionReader::new(connection_id, self.partition_num, filter, current_segment, self.reader_refs.get_reader_refs()) + let commit_index_reader = self.partition_highest_committed.reader(); + PartitionReader::new(connection_id, + self.partition_num, + filter, + current_segment, + self.reader_refs.get_reader_refs(), + commit_index_reader) } @@ -425,9 +435,9 @@ mod test { partition.handle_produce(produce).unwrap(); let mut reader: PartitionReader = partition.create_reader(CONNECTION, EventFilter::All, 0); - let event = reader.next_matching().expect("read_next returned None").expect("read_next returned error"); + let event = reader.read_next_uncommitted().expect("read_next returned None").expect("read_next returned error"); assert_eq!(b"the quick", event.data()); - let event2 = reader.next_matching().expect("read_next returned None").expect("read_next returned error"); + let event2 = reader.read_next_uncommitted().expect("read_next returned None").expect("read_next returned error"); assert_eq!(b"brown fox", event2.data()); assert!(reader.next().is_none()); diff --git a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs index ca2fa98..163b849 100644 --- a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs +++ b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs @@ -7,6 +7,7 @@ use event::{FloEvent, ActorId}; use engine::ConnectionId; use engine::event_stream::partition::{SharedReaderRefs, SegmentNum}; use engine::event_stream::partition::segment::{SegmentReader, PersistentEvent}; +use atomics::AtomicCounterReader; pub use self::namespace::NamespaceGlob; @@ -38,6 +39,7 @@ pub struct PartitionReader { connection_id: ConnectionId, partition_num: ActorId, filter: EventFilter, + commit_index_reader: AtomicCounterReader, current_segment_reader: Option, segment_readers_ref: SharedReaderRefs, returned_error: bool, @@ -46,18 +48,25 @@ pub struct PartitionReader { impl PartitionReader { - pub fn new(connection_id: ConnectionId, partition_num: ActorId, filter: EventFilter, current_reader: Option, segment_refs: SharedReaderRefs) -> PartitionReader { + pub fn new(connection_id: ConnectionId, + partition_num: ActorId, + filter: EventFilter, + current_reader: Option, + segment_refs: SharedReaderRefs, + commit_index_reader: AtomicCounterReader) -> PartitionReader { + PartitionReader { - connection_id: connection_id, - partition_num: partition_num, - filter: filter, + connection_id, + partition_num, + filter, + commit_index_reader, current_segment_reader: current_reader, segment_readers_ref: segment_refs, returned_error: false, } } - pub fn next_matching(&mut self) -> Option> { + pub fn read_next_uncommitted(&mut self) -> Option> { let mut next = self.read_next(); while self.should_skip(&next) { next = self.read_next(); @@ -65,6 +74,10 @@ impl PartitionReader { next } + pub fn read_next_committed(&mut self) -> Option> { + unimplemented!() + } + fn should_skip(&self, result: &Option>) -> bool { if let Some(Ok(ref event)) = *result { !self.filter.matches(event) @@ -116,7 +129,7 @@ impl Iterator for PartitionReader { type Item = io::Result; fn next(&mut self) -> Option { - self.next_matching() + self.read_next_uncommitted() } } diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index c6c1914..861a181 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -1,8 +1,8 @@ +pub mod controller; mod segment; mod index; mod event_reader; mod ops; -pub mod controller; use std::net::SocketAddr; use std::fmt::{self, Debug, Display}; diff --git a/flo-server/src/engine/event_stream/partition/segment/mod.rs b/flo-server/src/engine/event_stream/partition/segment/mod.rs index 92e81eb..4bcf473 100644 --- a/flo-server/src/engine/event_stream/partition/segment/mod.rs +++ b/flo-server/src/engine/event_stream/partition/segment/mod.rs @@ -83,6 +83,7 @@ impl Segment { trace!("creating range iter starting at offset: {}", start); SegmentReader { segment_id: self.segment_num, + last_read_id: 0, reader: self.appender.reader(start) } } @@ -153,12 +154,17 @@ impl Segment { #[derive(Clone, Debug)] pub struct SegmentReader { pub segment_id: SegmentNum, + last_read_id: EventCounter, reader: MmapReader, } impl SegmentReader { pub fn read_next(&mut self) -> Option> { - self.reader.read_next() + let result = self.reader.read_next(); + if let Some(&Ok(ref event)) = result.as_ref() { + self.last_read_id = event.id().event_counter; + } + result } pub fn is_exhausted(&self) -> bool { From 03969e5ad7aa25cf75945d43605c50594ff92f40 Mon Sep 17 00:00:00 2001 From: pfried Date: Wed, 13 Dec 2017 21:35:17 -0500 Subject: [PATCH 20/73] remove peer replication connection module until we are actually ready for it --- .../src/engine/controller/peer_connection/mod.rs | 7 ------- .../controller/peer_connection/replication.rs | 15 --------------- 2 files changed, 22 deletions(-) delete mode 100644 flo-server/src/engine/controller/peer_connection/replication.rs diff --git a/flo-server/src/engine/controller/peer_connection/mod.rs b/flo-server/src/engine/controller/peer_connection/mod.rs index 794a3fc..6ca1e12 100644 --- a/flo-server/src/engine/controller/peer_connection/mod.rs +++ b/flo-server/src/engine/controller/peer_connection/mod.rs @@ -1,4 +1,3 @@ -mod replication; mod system; use std::fmt::Debug; @@ -13,7 +12,6 @@ use engine::connection_handler::{create_connection_control_channels, ConnectionC use event_loops::LoopHandles; use flo_io::create_connection_handler; -pub use self::replication::PeerReplicationConnection; pub use self::system::PeerSystemConnection; pub type ConnectionSendResult = Result<(), T>; @@ -22,7 +20,6 @@ pub type ConnectionSendResult = Result<(), T>; /// Trait for creating outgoing connections (clever name, I know). pub trait OutgoingConnectionCreator { fn establish_system_connection(&mut self, address: SocketAddr) -> Box; - fn establish_replication_connection(&mut self, address: SocketAddr, event_stream: String) -> Box; } pub struct OutgoingConnectionCreatorImpl { @@ -34,10 +31,6 @@ impl OutgoingConnectionCreator for OutgoingConnectionCreatorImpl { fn establish_system_connection(&mut self, address: SocketAddr) -> Box { unimplemented!() } - - fn establish_replication_connection(&mut self, address: SocketAddr, event_stream: String) -> Box { - unimplemented!() - } } diff --git a/flo-server/src/engine/controller/peer_connection/replication.rs b/flo-server/src/engine/controller/peer_connection/replication.rs deleted file mode 100644 index bb54272..0000000 --- a/flo-server/src/engine/controller/peer_connection/replication.rs +++ /dev/null @@ -1,15 +0,0 @@ -use std::fmt::Debug; - -use super::ConnectionSendResult; -use event::VersionVector; - - -/// trait representing an active peer connection to a particular event stream, with functions to control replication -pub trait PeerReplicationConnection: Debug + Send + 'static { - fn start_replication(&mut self, version_vector: VersionVector) -> ConnectionSendResult; - fn stop_replication(&mut self) -> ConnectionSendResult<()>; - fn close_connection(&mut self); -} - - - From 0e7ca67f57b4f8e4c9cda08d01de410dbbef53c3 Mon Sep 17 00:00:00 2001 From: pfried Date: Wed, 13 Dec 2017 23:06:51 -0500 Subject: [PATCH 21/73] add new fields to peer announce message so members can tell who the primary is --- flo-protocol/src/messages/flo_instance_id.rs | 18 +++++- flo-protocol/src/messages/mod.rs | 24 +++++++- flo-protocol/src/messages/peer_announce.rs | 59 +++++++++++++++----- 3 files changed, 84 insertions(+), 17 deletions(-) diff --git a/flo-protocol/src/messages/flo_instance_id.rs b/flo-protocol/src/messages/flo_instance_id.rs index 3eb846a..2844c94 100644 --- a/flo-protocol/src/messages/flo_instance_id.rs +++ b/flo-protocol/src/messages/flo_instance_id.rs @@ -8,8 +8,16 @@ pub struct FloInstanceId(u64); impl FloInstanceId { + pub fn null() -> FloInstanceId { + FloInstanceId(0) + } + pub fn generate_new() -> FloInstanceId { - FloInstanceId(::rand::random()) + let mut value = 0; + while value == 0 { + value = ::rand::random(); + } + FloInstanceId(value) } pub fn as_bytes(&self) -> [u8; 8] { @@ -43,3 +51,11 @@ impl FloSerialize for FloInstanceId { } named!{pub parse_flo_instance_id, map!(::nom::be_u64, |val| {FloInstanceId(val) } )} + +named!{pub parse_optional_flo_instance_id>, map!(::nom::be_u64, |val| { + if val > 0 { + Some(FloInstanceId(val)) + } else { + None + } +} )} diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index 9710532..5fd2cbc 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -43,7 +43,7 @@ use self::append_entries::{serialize_append_entries, parse_append_entries_call, use self::request_vote::{serialize_request_vote_call, parse_request_vote_call, serialize_request_vote_response, parse_request_vote_response}; pub use self::client_announce::ClientAnnounce; -pub use self::peer_announce::PeerAnnounce; +pub use self::peer_announce::{PeerAnnounce, ClusterMember}; pub use self::error::{ErrorMessage, ErrorKind}; pub use self::produce_event::ProduceEvent; pub use self::event_ack::EventAck; @@ -441,11 +441,26 @@ mod test { #[test] fn serde_peer_announce_with_ipv6_address() { - let addr = ::std::str::FromStr::from_str("[1:3:5::2]:4321").unwrap(); let announce = PeerAnnounce { protocol_version: 9, - peer_address: addr, op_id: 6543, + instance_id: FloInstanceId::generate_new(), + peer_address: addr("[1:3:5::2]:4321"), + system_primary_id: Some(FloInstanceId::generate_new()), + cluster_members: vec![ + ClusterMember { + id: FloInstanceId::generate_new(), + address: addr("1.2.3.4:5") + }, + ClusterMember { + id: FloInstanceId::generate_new(), + address: addr("127.0.0.1:2456") + }, + ClusterMember { + id: FloInstanceId::generate_new(), + address: addr("192.168.1.1:443") + }, + ] }; test_serialize_then_deserialize(&ProtocolMessage::PeerAnnounce(announce)); } @@ -455,8 +470,11 @@ mod test { let addr = ::std::str::FromStr::from_str("123.234.12.1:4321").unwrap(); let announce = PeerAnnounce { protocol_version: 9, + instance_id: FloInstanceId::generate_new(), peer_address: addr, op_id: 6543, + system_primary_id: None, + cluster_members: Vec::new(), }; test_serialize_then_deserialize(&ProtocolMessage::PeerAnnounce(announce)); } diff --git a/flo-protocol/src/messages/peer_announce.rs b/flo-protocol/src/messages/peer_announce.rs index 0adba62..f0281fa 100644 --- a/flo-protocol/src/messages/peer_announce.rs +++ b/flo-protocol/src/messages/peer_announce.rs @@ -1,9 +1,10 @@ use std::net::SocketAddr; -use nom::be_u32; -use serializer::Serializer; +use nom::{be_u32, be_u16}; +use serializer::{Serializer, FloSerialize}; use event::OwnedFloEvent; -use super::{parse_socket_addr, ProtocolMessage}; +use super::{parse_socket_addr, ProtocolMessage, FloInstanceId}; +use super::flo_instance_id::{parse_flo_instance_id, parse_optional_flo_instance_id}; pub const PEER_ANNOUNCE: u8 = 7; @@ -13,23 +14,48 @@ pub struct PeerAnnounce { pub protocol_version: u32, pub peer_address: SocketAddr, pub op_id: u32, + pub instance_id: FloInstanceId, + pub system_primary_id: Option, + pub cluster_members: Vec } +#[derive(Debug, PartialEq, Clone)] +pub struct ClusterMember { + pub id: FloInstanceId, + pub address: SocketAddr, +} + +impl FloSerialize for ClusterMember { + fn serialize<'a>(&'a self, serializer: Serializer<'a>) -> Serializer<'a> { + serializer.write(&self.id).write_socket_addr(self.address) + } +} + +named!{parse_cluster_member, do_parse!( + id: parse_flo_instance_id >> + address: parse_socket_addr >> + ( ClusterMember{id, address} ) +)} named!{pub parse_peer_announce>, - chain!( - _tag: tag!(&[PEER_ANNOUNCE]) ~ - protocol_version: be_u32 ~ - op_id: be_u32 ~ - peer_address: parse_socket_addr, - || { - ProtocolMessage::PeerAnnounce(PeerAnnounce { + do_parse!( + tag!(&[PEER_ANNOUNCE]) >> + protocol_version: be_u32 >> + op_id: be_u32 >> + instance_id: parse_flo_instance_id >> + peer_address: parse_socket_addr >> + system_primary_id: parse_optional_flo_instance_id >> + cluster_members: length_count!(be_u16, parse_cluster_member) >> + + ( ProtocolMessage::PeerAnnounce(PeerAnnounce { protocol_version, op_id, - peer_address - }) - } + instance_id, + peer_address, + system_primary_id, + cluster_members + }) ) ) } @@ -38,6 +64,13 @@ pub fn serialize_peer_announce(announce: &PeerAnnounce, buf: &mut [u8]) -> usize .write_u8(PEER_ANNOUNCE) .write_u32(announce.protocol_version) .write_u32(announce.op_id) + .write(&announce.instance_id) .write_socket_addr(announce.peer_address) + .write(&announce.system_primary_id.unwrap_or(FloInstanceId::null())) + .write_u16(announce.cluster_members.len() as u16) + .write_many(announce.cluster_members.iter(), |ser, member| { + ser.write(member) + }) .finish() + } From 06ab54d4698e2dc1dda18cbcc63e62b969d07ca0 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 17 Dec 2017 21:38:56 -0500 Subject: [PATCH 22/73] sketch in a real initialization flow for cluster state --- Cargo.lock | 76 ++++++++++++++++- flo-server/Cargo.toml | 5 +- .../src/engine/connection_handler/mod.rs | 12 ++- .../src/engine/connection_handler/peer/mod.rs | 33 ++++++-- .../engine/controller/cluster_state/mod.rs | 81 ++++++++++++++++--- .../controller/cluster_state/persistent.rs | 69 ++++++++++++++++ .../engine/controller/controller_messages.rs | 4 + .../src/engine/controller/initialization.rs | 69 ++++++++-------- flo-server/src/engine/controller/mod.rs | 29 +++---- .../engine/controller/peer_connection/mod.rs | 2 +- .../src/engine/controller/system_stream.rs | 22 ++++- flo-server/src/engine/mod.rs | 10 +-- flo-server/src/flo_io/mod.rs | 2 + flo-server/src/lib.rs | 3 + 14 files changed, 321 insertions(+), 96 deletions(-) create mode 100644 flo-server/src/engine/controller/cluster_state/persistent.rs diff --git a/Cargo.lock b/Cargo.lock index 0bbe0d3..f26a59d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -275,8 +275,9 @@ dependencies = [ "memmap 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -585,6 +586,11 @@ name = "quick-error" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "rand" version = "0.3.18" @@ -653,6 +659,11 @@ name = "serde" version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "serde" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "serde-value" version = "0.2.1" @@ -662,6 +673,25 @@ dependencies = [ "serde 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde_derive" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive_internals 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_derive_internals" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "serde_json" version = "0.9.10" @@ -673,6 +703,17 @@ dependencies = [ "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde_json" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "slab" version = "0.3.0" @@ -696,6 +737,24 @@ name = "strsim" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "syn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tempdir" version = "0.3.5" @@ -831,6 +890,11 @@ name = "unicode-width" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unreachable" version = "1.0.0" @@ -991,6 +1055,7 @@ dependencies = [ "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" "checksum png 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "48f397b84083c2753ba53c7b56ad023edb94512b2885ffe227c66ff7edb61868" "checksum quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eda5fe9b71976e62bc81b781206aaa076401769b2143379d3eb2118388babac4" +"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum rand 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)" = "6475140dfd8655aeb72e1fd4b7a1cc1c202be65d71669476e392fe62532b9edd" "checksum redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "8dde11f18c108289bef24469638a04dce49da56084f2d50618b226e47eb04509" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" @@ -1001,12 +1066,18 @@ dependencies = [ "checksum scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f417c22df063e9450888a7561788e9bd46d3bb3c1466435b4eccb903807f147d" "checksum serde 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)" = "1b0e0732aa8ec4267f61815a396a942ba3525062e3bd5520aa8419927cfc0a92" "checksum serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)" = "34b623917345a631dc9608d5194cc206b3fe6c3554cd1c75b937e55e285254af" +"checksum serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)" = "1c57ab4ec5fa85d08aaf8ed9245899d9bbdd66768945b21113b84d5f595cb6a1" "checksum serde-value 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d94076c6c6e05aaf18beaa024fb789f372be9a1dccbcf66e5748fdfe8cb2a00c" +"checksum serde_derive 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)" = "02c92ea07b6e49b959c1481804ebc9bfd92d3c459f1274c9a9546829e42a66ce" +"checksum serde_derive_internals 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75c6aac7b99801a16db5b40b7bf0d7e4ba16e76fbf231e32a4677f271cac0603" "checksum serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ad8bcf487be7d2e15d3d543f04312de991d631cfe1b43ea0ade69e6a8a5b16a1" +"checksum serde_json 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7cf5b0b5b4bd22eeecb7e01ac2e1225c7ef5e4272b79ee28a8392a8c8489c839" "checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" "checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d" "checksum stb_truetype 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fcf3270840fc9de208d63e836eb3fdebb85379e7532f42f1b2cbd505fb6fda08" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" +"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" "checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" @@ -1021,6 +1092,7 @@ dependencies = [ "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" "checksum unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "51ccda9ef9efa3f7ef5d91e8f9b83bbe6955f9bf86aec89d5cce2c874625920f" "checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" +"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f" "checksum url 0.2.38 (registry+https://github.com/rust-lang/crates.io-index)" = "cbaa8377a162d88e7d15db0cf110c8523453edcbc5bc66d2b6fffccffa34a068" diff --git a/flo-server/Cargo.toml b/flo-server/Cargo.toml index e1f5ed3..13496c9 100644 --- a/flo-server/Cargo.toml +++ b/flo-server/Cargo.toml @@ -19,8 +19,9 @@ glob = "0.2" chrono = "^0.2" memmap = "0.5.2" libc = "0.2.33" -serde = "0.9.15" -serde_json = "0.9.10" +serde = "1.0" +serde_json = "1.0" +serde_derive = "1.0" [dev-dependencies] env_logger = "*" diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 6525d97..5515b8a 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -172,7 +172,7 @@ mod test { use engine::event_stream::EventStreamRef; use engine::event_stream::partition::*; use engine::ClientReceiver; - use engine::controller::SystemStreamRef; + use engine::controller::{SystemStreamRef, SharedClusterState}; use atomics::{AtomicCounterWriter, AtomicBoolWriter}; struct Fixture { @@ -201,10 +201,16 @@ mod test { tx.clone(), primary_addr); - let system_stream = SystemStreamRef::new(part_ref, tx); + let cluster_state = SharedClusterState { + this_instance_id: FloInstanceId::generate_new(), + this_address: None, + system_primary: None, + peers: Vec::new(), + }; + let system_stream = SystemStreamRef::new(part_ref, tx, Arc::new(RwLock::new(cluster_state))); let streams = Arc::new(Mutex::new(HashMap::new())); - let engine = EngineRef::new(None, system_stream, streams); + let engine = EngineRef::new(system_stream, streams); let subject = ConnectionHandler::new(456, client_sender, engine.clone(), reactor.handle()); diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 99baaa7..13cc321 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -1,7 +1,8 @@ mod peer_follower; -use protocol::{ProtocolMessage, PeerAnnounce, EventStreamStatus}; +use protocol::{ProtocolMessage, PeerAnnounce, EventStreamStatus, ClusterMember}; use engine::{ReceivedProtocolMessage, ConnectionId}; +use engine::controller::SystemStreamRef; use super::connection_state::ConnectionState; use super::ConnectionHandlerResult; @@ -19,19 +20,35 @@ impl PeerConnectionState { assert_eq!(PeerConnectionState::Init, *self); state.set_to_system_stream(); - let this_address = state.engine.get_this_instance_address() - .expect("Tried to initiate outgoing connection but this_instance_address is None"); - let announce = PeerAnnounce { - protocol_version: 1, - peer_address: this_address, - op_id: 1, - }; + let announce = PeerConnectionState::create_peer_announce(&*state.engine.system_stream()); let protocol_message = ProtocolMessage::PeerAnnounce(announce); state.send_to_client(protocol_message).expect("failed to send peer announce when establishing outgoing connection"); *self = PeerConnectionState::AwaitingPeerResponse; } + fn create_peer_announce(system_stream: &SystemStreamRef) -> PeerAnnounce { + system_stream.with_cluster_state(|state| { + let instance_id = state.this_instance_id; + let address = state.this_address.expect("Attempted to send PeerAnnounce, but system is not in cluster mode"); + let primary = state.system_primary.as_ref().map(|peer| peer.id); + let members = state.peers.iter().map(|peer| { + ClusterMember { + id: peer.id, + address: peer.address, + } + }).collect::>(); + + PeerAnnounce { + protocol_version: 1, + instance_id, + peer_address: address, + op_id: 1, + system_primary_id: primary, + cluster_members: members, + } + }) + } pub fn message_received(&mut self, message: ReceivedProtocolMessage, state: &mut ConnectionState) -> ConnectionHandlerResult { diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 894ebce..45eb9fa 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -1,29 +1,84 @@ +mod persistent; +use std::io; +use std::sync::{Arc, RwLock}; use std::net::SocketAddr; use std::time::{Instant}; +use std::path::Path; use event::EventCounter; use protocol::FloInstanceId; +use atomics::{AtomicBoolWriter, AtomicBoolReader}; +use super::ClusterOptions; +pub use self::persistent::{FilePersistedState, PersistentClusterState}; -#[derive(Debug, PartialEq, Clone)] -struct PersistentClusterState { - current_term: u64, - voted_for: Option, -} +#[derive(Debug)] pub struct ClusterState { - persistent: PersistentClusterState, + primary_status_writer: AtomicBoolWriter, + cluster_options: Option, last_applied: EventCounter, + persistent: FilePersistedState, + system_partion_primary_address: SystemPrimaryAddressRef, + shared: Arc>, +} + +#[derive(Debug, PartialEq, Clone)] +pub struct SharedClusterState { + pub this_instance_id: FloInstanceId, + pub this_address: Option, + pub system_primary: Option, + pub peers: Vec, } -struct Peer { - id: FloInstanceId, - leader: bool, - peer_addr: SocketAddr, - last_ackgnowledged_entry: EventCounter, +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub struct Peer { + pub id: FloInstanceId, + pub address: SocketAddr, } -enum PeerState { - Connecting(Instant), + +impl ClusterState { + + pub fn system_primary_status_reader(&self) -> AtomicBoolReader { + self.primary_status_writer.reader() + } + + pub fn system_primary_address_reader(&self) -> SystemPrimaryAddressRef { + self.system_partion_primary_address.clone() + } + + pub fn reader(&self) -> ClusterStateReader { + self.shared.clone() + } } + +pub type SystemPrimaryAddressRef = Arc>>; + +pub type ClusterStateReader = Arc>; + + +pub fn init_cluster_state(storage_dir: &Path, options: Option) -> io::Result { + let path = storage_dir.join("cluster-state"); + let persistent_state = FilePersistedState::initialize(path)?; // early return if this fails + let this_address = options.as_ref().map(|opts| opts.this_instance_address); + let shared_state = persistent_state.initialize_shared_state(this_address); + + let start_as_primary = options.is_none(); + if start_as_primary { + debug!("Initializing in non-cluster mode, system stream will always be considered primary"); + } else { + debug!("Initializing in clustered mode, starting as secondary") + } + + Ok(ClusterState { + primary_status_writer: AtomicBoolWriter::with_value(start_as_primary), + cluster_options: options, + last_applied: 0, + persistent: persistent_state, + system_partion_primary_address: Arc::new(RwLock::new(None)), + shared: Arc::new(RwLock::new(shared_state)), + }) +} + diff --git a/flo-server/src/engine/controller/cluster_state/persistent.rs b/flo-server/src/engine/controller/cluster_state/persistent.rs new file mode 100644 index 0000000..33139e7 --- /dev/null +++ b/flo-server/src/engine/controller/cluster_state/persistent.rs @@ -0,0 +1,69 @@ +use std::net::SocketAddr; +use std::path::PathBuf; +use std::fs::{File, OpenOptions}; +use std::io::{self, Seek}; + +use protocol::{FloInstanceId, Term}; +use super::{Peer, ClusterOptions, SharedClusterState}; + +/// Holds all the cluster state that we want to survive a reboot. +/// We always persist the `FloInstanceId` because we prefer that to be stable across reboots. We do _not_ want to persist +/// the `SocketAddr` for the server, though, since that may well change after a restart, depending on environment. +#[derive(Debug, PartialEq, Clone)] +pub struct PersistentClusterState { + current_term: Term, + voted_for: Option, + this_instance_id: FloInstanceId, + cluster_members: Vec, +} + +impl PersistentClusterState { + /// called during system startup to initialize the shared cluster state that will be available to all the connection handlers + pub fn initialize_shared_state(&self, this_address: Option) -> SharedClusterState { + + SharedClusterState { + this_instance_id: self.this_instance_id, + this_address, + system_primary: None, // we are still starting up, so we have no idea who's primary + peers: self.cluster_members.clone(), + } + } +} + + +/// Placeholder for a wrapper struct that will take care of persisting the state as it changes +#[derive(Debug)] +pub struct FilePersistedState { + file: File, + path: PathBuf, + state: PersistentClusterState, +} + +impl FilePersistedState { + pub fn initialize(path: PathBuf) -> io::Result { + let file = OpenOptions::new().write(true).read(true).create(true).open(&path)?; // early return on failure + + let state = PersistentClusterState { + current_term: 0, + voted_for: None, + this_instance_id: FloInstanceId::generate_new(), + cluster_members: Vec::new(), + }; + debug!("Initialized {:?}", state); + + Ok(FilePersistedState { + file, + path, + state + }) + } +} + +impl ::std::ops::Deref for FilePersistedState { + type Target = PersistentClusterState; + + fn deref(&self) -> &Self::Target { + &self.state + } +} + diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 40829df..79bd3f5 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -35,6 +35,10 @@ impl SystemOperation { SystemOperation::new(connection.connection_id, SystemOpType::IncomingConnectionEstablished(connection)) } + pub fn connection_closed(connection_id: ConnectionId) -> SystemOperation { + SystemOperation::new(connection_id, SystemOpType::ConnectionClosed(connection_id)) + } + pub fn outgoing_connection_failed(addr: SocketAddr) -> SystemOperation { SystemOperation::new(0, SystemOpType::OutgoingConnectionFailed(addr)) } diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index f303dcc..028eaf6 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -9,6 +9,7 @@ use tokio_core::reactor::Remote; use engine::{EngineRef, system_stream_name, SYSTEM_STREAM_NAME}; use engine::controller::{SystemPartitionSender, SystemPartitionReceiver, SystemOperation}; +use engine::controller::cluster_state::{init_cluster_state, ClusterState}; use engine::event_stream::{EventStreamRefMut, EventStreamOptions, init_existing_event_stream, @@ -43,37 +44,34 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul debug!("Starting Flo Controller with: {:?}", options); let ControllerOptions{storage_dir, default_stream_options, cluster_options} = options; - // TODO: initialize system primary status to false once clustering works - let system_primary_writer = AtomicBoolWriter::with_value(true); - let system_primary_server_addr = Arc::new(RwLock::new(None)); - - let partition_result = init_system_partition(&storage_dir, - system_primary_writer.reader(), - &default_stream_options); - - partition_result.and_then(|system_partition| { - debug!("Initialized system partition"); - init_user_streams(&storage_dir, &default_stream_options, &remote).map(|user_streams| { - debug!("Initialized all {} user event streams", user_streams.len()); - - let system_primary_reader = system_primary_writer.reader(); - let system_highest_counter = system_partition.event_counter_reader(); - - let flo_controller = FloController::new(system_partition, - system_primary_writer, - system_primary_server_addr.clone(), - user_streams, - storage_dir, - cluster_options, - default_stream_options); - - let (system_partition_tx, system_partition_rx) = ::engine::controller::create_system_partition_channels(); - let engine_ref = create_engine_ref(&flo_controller, system_highest_counter, system_primary_reader, system_primary_server_addr, system_partition_tx); - - run_controller_impl(flo_controller, system_partition_rx); - engine_ref - }) - }) + let cluster_state = init_cluster_state(&storage_dir, cluster_options)?; // early return if this fails + + let system_partition = init_system_partition(&storage_dir, + cluster_state.system_primary_status_reader(), + &default_stream_options)?; // early return if this fails + debug!("Initialized system partition"); + + // early return if this fails + let user_streams = init_user_streams(&storage_dir, &default_stream_options, &remote)?; + debug!("Initialized all {} user event streams", user_streams.len()); + + let system_primary_reader = cluster_state.system_primary_status_reader(); + let system_primary_server_addr = cluster_state.system_primary_address_reader(); + let system_highest_counter = system_partition.event_counter_reader(); + + + let flo_controller = FloController::new(system_partition, + user_streams, + storage_dir, + cluster_state, + default_stream_options); + + let (system_partition_tx, system_partition_rx) = ::engine::controller::create_system_partition_channels(); + let engine_ref = create_engine_ref(&flo_controller, system_highest_counter, system_primary_reader, system_primary_server_addr, system_partition_tx); + + run_controller_impl(flo_controller, system_partition_rx); + + Ok(engine_ref) } fn init_system_partition(storage_dir: &Path, system_primary_reader: AtomicBoolReader, default_stream_options: &EventStreamOptions) -> io::Result { @@ -109,16 +107,19 @@ fn create_engine_ref(controller: &FloController, system_primary_reader, system_partition_sender.clone(), system_primary_addr); - let system_stream_ref = SystemStreamRef::new(system_partition_ref, system_partition_sender); + + let cluster_state_reader = controller.get_cluster_state_reader(); + let system_stream_ref = SystemStreamRef::new(system_partition_ref, system_partition_sender, cluster_state_reader); let shared_stream_refs = controller.get_shared_streams(); - let this_addr = controller.get_this_instance_address(); - EngineRef::new(this_addr, system_stream_ref, shared_stream_refs) + EngineRef::new(system_stream_ref, shared_stream_refs) } fn init_user_streams(storage_dir: &Path, options: &EventStreamOptions, remote: &Remote) -> io::Result> { let mut user_streams = HashMap::new(); + + // all sorts of early returns if there's filesystem failures for file_result in ::std::fs::read_dir(storage_dir)? { let entry = file_result?; if is_user_event_stream(&entry)? { diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 3bd3067..fa17bbb 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -1,4 +1,4 @@ -mod cluster_state; +pub mod cluster_state; mod system_stream; mod initialization; mod controller_messages; @@ -17,11 +17,13 @@ use engine::event_stream::{EventStreamRef, use engine::event_stream::partition::Operation; use engine::event_stream::partition::controller::PartitionImpl; use atomics::AtomicBoolWriter; +use self::cluster_state::ClusterState; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; pub use self::system_stream::SystemStreamRef; pub use self::controller_messages::{SystemOperation, SystemOpType, ConnectionRef}; +pub use self::cluster_state::{SharedClusterState, Peer, ClusterStateReader}; pub type SystemPartitionSender = ::std::sync::mpsc::Sender; pub type SystemPartitionReceiver = ::std::sync::mpsc::Receiver; @@ -49,25 +51,14 @@ pub struct FloController { /// the partition that persists system events. Used as the RAFT log system_partition: PartitionImpl, - /// used to set the status of the system stream. There is only ever at most one instance in a cluster - /// where this variable is true ...if things actually work correctly ;) - system_primary_status_writer: AtomicBoolWriter, - - /// The address of the cluster's primary server, if one exists and it is known - system_primary_server_addr: Arc>>, - - /// cluster parameters that this instance was started with. We'll almost certainly want to replace this field later on - /// with something that can deal with more complexity - cluster_options: Option, + cluster_state: ClusterState, } impl FloController { pub fn new(system_partition: PartitionImpl, - system_primary_setter: AtomicBoolWriter, - system_primary_address: Arc>>, event_streams: HashMap, storage_dir: PathBuf, - cluster_options: Option, + cluster_state: ClusterState, default_stream_options: EventStreamOptions) -> FloController { let stream_refs = event_streams.iter().map(|(k, v)| { @@ -80,14 +71,12 @@ impl FloController { system_partition, storage_dir, default_stream_options, - system_primary_status_writer: system_primary_setter, - system_primary_server_addr: system_primary_address, - cluster_options, + cluster_state, } } fn process(&mut self, operation: SystemOperation) { - debug!("Processing: {:?}", operation); + warn!("Ignoring SystemOperation: {:?}", operation); //TODO: handle system operations } @@ -100,8 +89,8 @@ impl FloController { self.shared_event_stream_refs.clone() } - fn get_this_instance_address(&self) -> Option { - self.cluster_options.as_ref().map(|opts| opts.this_instance_address) + fn get_cluster_state_reader(&self) -> ClusterStateReader { + self.cluster_state.reader() } } diff --git a/flo-server/src/engine/controller/peer_connection/mod.rs b/flo-server/src/engine/controller/peer_connection/mod.rs index 6ca1e12..6930644 100644 --- a/flo-server/src/engine/controller/peer_connection/mod.rs +++ b/flo-server/src/engine/controller/peer_connection/mod.rs @@ -48,7 +48,7 @@ fn create_outgoing_connection(loops: &mut LoopHandles, client_addr: SocketAddr, TcpStream::connect(&addr, handle).map_err( move |io_err| { error!("Failed to create outgoing connection to address: {:?}: {:?}", addr, io_err); - system_stream.outgoing_connection_failed(0, addr); + system_stream.outgoing_connection_failed(addr); }).and_then( move |tcp_stream| { let connection_id = engine_ref.next_connection_id(); diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index 10a34d7..06d24a7 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -3,25 +3,33 @@ use std::net::SocketAddr; use engine::event_stream::partition::{PartitionRef, Operation}; use engine::event_stream::EventStreamRef; use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef}; +use engine::controller::cluster_state::{SharedClusterState, ClusterStateReader}; use engine::ConnectionId; - +use atomics::AtomicBoolReader; #[derive(Clone, Debug)] pub struct SystemStreamRef { + cluster_state_reader: ClusterStateReader, system_sender: SystemPartitionSender, inner: PartitionRef, } impl SystemStreamRef { - pub fn new(partition_ref: PartitionRef, system_sender: SystemPartitionSender) -> SystemStreamRef { + pub fn new(partition_ref: PartitionRef, system_sender: SystemPartitionSender, cluster_state_reader: ClusterStateReader) -> SystemStreamRef { SystemStreamRef { + cluster_state_reader, system_sender, inner: partition_ref, } } + pub fn with_cluster_state(&self, fun: F) -> T where F: Fn(&SharedClusterState) -> T { + let state = self.cluster_state_reader.read().unwrap(); + fun(&*state) + } + pub fn to_event_stream(&self) -> EventStreamRef { let name = self.inner.event_stream_name().to_owned(); let partition_ref = vec![self.inner.clone()]; @@ -33,8 +41,14 @@ impl SystemStreamRef { self.send(op); } - pub fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, socket_addr: SocketAddr) { - unimplemented!() + pub fn connection_closed(&mut self, connection_id: ConnectionId) { + let op = SystemOperation::connection_closed(connection_id); + self.send(op); + } + + pub fn outgoing_connection_failed(&mut self, socket_addr: SocketAddr) { + let op = SystemOperation::outgoing_connection_failed(socket_addr); + self.send(op); } fn send(&mut self, op: SystemOperation) { diff --git a/flo-server/src/engine/mod.rs b/flo-server/src/engine/mod.rs index c2f6f6d..181bcbb 100644 --- a/flo-server/src/engine/mod.rs +++ b/flo-server/src/engine/mod.rs @@ -39,8 +39,6 @@ pub fn system_stream_name() -> String { #[derive(Clone, Debug)] pub struct EngineRef { - /// only known if this instance was started in clustering mode - this_instance_address: Option, current_connection_id: Arc, system_stream: SystemStreamRef, event_streams: Arc>> @@ -53,9 +51,8 @@ pub enum ConnectError { } impl EngineRef { - pub fn new(this_address: Option, system_stream: SystemStreamRef, event_streams: Arc>>) -> EngineRef { + pub fn new(system_stream: SystemStreamRef, event_streams: Arc>>) -> EngineRef { EngineRef { - this_instance_address: this_address, current_connection_id: Arc::new(AtomicUsize::new(0)), system_stream, event_streams @@ -93,11 +90,6 @@ impl EngineRef { pub fn get_system_stream(&self) -> SystemStreamRef { self.system_stream.clone() } - - /// Returns the address that this intance is reachable at. This will be `None` if the server was started started in non-clustered mode - pub fn get_this_instance_address(&self) -> Option { - self.this_instance_address - } } diff --git a/flo-server/src/flo_io/mod.rs b/flo-server/src/flo_io/mod.rs index fd74772..b5aa31d 100644 --- a/flo-server/src/flo_io/mod.rs +++ b/flo-server/src/flo_io/mod.rs @@ -44,6 +44,7 @@ pub fn create_connection_handler(client_handle: Handle, }) .select(client_message_stream); + let mut system_stream_ref = client_engine_ref.get_system_stream(); let connection_handler = ConnectionHandler::new( connection_id, client_tx.clone(), @@ -59,6 +60,7 @@ pub fn create_connection_handler(client_handle: Handle, warn!("Closing connection: {} due to err: {:?}", connection_id, err); } info!("Closed connection_id: {} to address: {}", connection_id, client_addr); + system_stream_ref.connection_closed(connection_id); Ok(()) }); Box::new(future) diff --git a/flo-server/src/lib.rs b/flo-server/src/lib.rs index c6f19b7..f6f25a8 100644 --- a/flo-server/src/lib.rs +++ b/flo-server/src/lib.rs @@ -16,6 +16,9 @@ extern crate clap; extern crate log4rs; extern crate num_cpus; extern crate byteorder; + +#[macro_use] +extern crate serde_derive; extern crate serde; extern crate serde_json; From 3d7be050714973bf2d3d4289596fe496da30843c Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 19 Dec 2017 22:02:22 -0500 Subject: [PATCH 23/73] factor out trait objects for some of the cluster-related state management, handle peerAnnounce handshake in connectionHandler --- .../src/engine/connection_handler/peer/mod.rs | 50 +++++---- .../engine/controller/cluster_state/mod.rs | 104 ++++++++++++++---- .../engine/controller/controller_messages.rs | 9 +- .../src/engine/controller/initialization.rs | 31 ++++-- flo-server/src/engine/controller/mod.rs | 10 +- .../engine/controller/peer_connection/mod.rs | 24 +++- .../controller/peer_connection/system.rs | 21 +++- .../src/engine/controller/system_stream.rs | 5 + .../partition/controller/commit_manager.rs | 2 +- flo-server/src/main.rs | 1 - flo-server/src/server/mod.rs | 1 + flo-server/tests/embedded_tests.rs | 1 - 12 files changed, 186 insertions(+), 73 deletions(-) diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 13cc321..7089737 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -22,11 +22,35 @@ impl PeerConnectionState { let announce = PeerConnectionState::create_peer_announce(&*state.engine.system_stream()); let protocol_message = ProtocolMessage::PeerAnnounce(announce); + // safe unwrap since this is called only when creating a brand new outgoing connection state.send_to_client(protocol_message).expect("failed to send peer announce when establishing outgoing connection"); *self = PeerConnectionState::AwaitingPeerResponse; } + pub fn peer_announce_received(&mut self, state: &mut ConnectionState) -> ConnectionHandlerResult { + let connection_id = state.connection_id; + let old_state = self.set_state(connection_id, PeerConnectionState::Peer); + + match old_state { + PeerConnectionState::Peer => { + // we've already gone through this, so something's wrong + let message = format!("received redundant PeerAnnounce for connection_id: {}, closing connection", connection_id); + return Err(message) + } + PeerConnectionState::Init => { + // This was an incoming connection, and this was the first peer message sent, so we need to respond in kind + let peer_announce = PeerConnectionState::create_peer_announce(state.get_system_stream()); + state.send_to_client(ProtocolMessage::PeerAnnounce(peer_announce))?; + } + PeerConnectionState::AwaitingPeerResponse => { } + } + + state.set_to_system_stream(); + state.get_system_stream().connection_upgraded_to_peer(connection_id); + Ok(()) + } + fn create_peer_announce(system_stream: &SystemStreamRef) -> PeerAnnounce { system_stream.with_cluster_state(|state| { let instance_id = state.this_instance_id; @@ -50,27 +74,13 @@ impl PeerConnectionState { }) } - pub fn message_received(&mut self, message: ReceivedProtocolMessage, state: &mut ConnectionState) -> ConnectionHandlerResult { - - match message { - ProtocolMessage::StreamStatus(status) => { - self.stream_status_received(state, status); - } - other @ _ => { - error!("Unhandled protocol message from peer: {:?}", other); - } - } - - Ok(()) + fn set_state(&mut self, connection_id: ConnectionId, new_state: PeerConnectionState) -> PeerConnectionState { + debug!("Transitioning connection_id: {} from {:?} to {:?}", connection_id, self, new_state); + ::std::mem::replace(self, new_state) } +} - fn stream_status_received(&mut self, state: &mut ConnectionState, _status: EventStreamStatus) { - self.set_state(state.connection_id, PeerConnectionState::Peer); - unimplemented!() - } +#[cfg(test)] +mod test { - fn set_state(&mut self, connection_id: ConnectionId, new_state: PeerConnectionState) { - debug!("Transitioning connection_id: {} from {:?} to {:?}", connection_id, self, new_state); - *self = new_state; - } } diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 45eb9fa..16047c1 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -7,23 +7,43 @@ use std::time::{Instant}; use std::path::Path; use event::EventCounter; +use engine::ConnectionId; use protocol::FloInstanceId; use atomics::{AtomicBoolWriter, AtomicBoolReader}; use super::ClusterOptions; +use super::peer_connection::PeerSystemConnection; pub use self::persistent::{FilePersistedState, PersistentClusterState}; +pub trait ConsensusProcessor: Send { + fn system_primary_address_reader(&self) -> Arc>>; + fn system_primary_status_reader(&self) -> AtomicBoolReader; + fn peer_connection_established(&mut self, peer_id: FloInstanceId, connection_id: ConnectionId); +} #[derive(Debug)] pub struct ClusterState { primary_status_writer: AtomicBoolWriter, - cluster_options: Option, + starting_peer_addresses: Vec, last_applied: EventCounter, persistent: FilePersistedState, system_partion_primary_address: SystemPrimaryAddressRef, shared: Arc>, } +impl ConsensusProcessor for ClusterState { + + fn peer_connection_established(&mut self, peer_id: FloInstanceId, connection_id: ConnectionId) { + unimplemented!() + } + fn system_primary_status_reader(&self) -> AtomicBoolReader { + self.primary_status_writer.reader() + } + fn system_primary_address_reader(&self) -> Arc>> { + self.system_partion_primary_address.clone() + } +} + #[derive(Debug, PartialEq, Clone)] pub struct SharedClusterState { pub this_instance_id: FloInstanceId, @@ -32,6 +52,17 @@ pub struct SharedClusterState { pub peers: Vec, } +impl SharedClusterState { + fn non_cluster() -> SharedClusterState { + SharedClusterState { + this_instance_id: FloInstanceId::generate_new(), + this_address: None, + system_primary: None, + peers: Vec::new(), + } + } +} + #[derive(Debug, PartialEq, Eq, Hash, Clone)] pub struct Peer { pub id: FloInstanceId, @@ -48,10 +79,6 @@ impl ClusterState { pub fn system_primary_address_reader(&self) -> SystemPrimaryAddressRef { self.system_partion_primary_address.clone() } - - pub fn reader(&self) -> ClusterStateReader { - self.shared.clone() - } } pub type SystemPrimaryAddressRef = Arc>>; @@ -59,26 +86,65 @@ pub type SystemPrimaryAddressRef = Arc>>; pub type ClusterStateReader = Arc>; -pub fn init_cluster_state(storage_dir: &Path, options: Option) -> io::Result { - let path = storage_dir.join("cluster-state"); - let persistent_state = FilePersistedState::initialize(path)?; // early return if this fails - let this_address = options.as_ref().map(|opts| opts.this_instance_address); - let shared_state = persistent_state.initialize_shared_state(this_address); - +pub fn init_consensus_processor(storage_dir: &Path, options: Option) -> io::Result<(Box, ClusterStateReader)> { let start_as_primary = options.is_none(); - if start_as_primary { - debug!("Initializing in non-cluster mode, system stream will always be considered primary"); + if let Some(options) = options { + debug!("Initializing in clustered mode, starting as secondary"); + init_cluster_state(storage_dir, options) } else { - debug!("Initializing in clustered mode, starting as secondary") + debug!("Initializing in non-cluster mode, system stream will always be considered primary"); + Ok(init_no_op_consensus_processor()) } +} + +fn init_cluster_state(storage_dir: &Path, options: ClusterOptions) -> io::Result<(Box, ClusterStateReader)> { + let path = storage_dir.join("cluster-state"); + let persistent_state = FilePersistedState::initialize(path)?; // early return if this fails + + let ClusterOptions{ this_instance_address, peer_addresses, event_loop_handles } = options; - Ok(ClusterState { - primary_status_writer: AtomicBoolWriter::with_value(start_as_primary), - cluster_options: options, + let shared_state = persistent_state.initialize_shared_state(Some(this_instance_address)); + let shared_state_ref = Arc::new(RwLock::new(shared_state)); + + let state = ClusterState { + primary_status_writer: AtomicBoolWriter::with_value(false), last_applied: 0, + starting_peer_addresses: peer_addresses, persistent: persistent_state, system_partion_primary_address: Arc::new(RwLock::new(None)), - shared: Arc::new(RwLock::new(shared_state)), - }) + shared: shared_state_ref.clone(), + }; + Ok((Box::new(state), shared_state_ref)) +} + +fn init_no_op_consensus_processor() -> (Box, ClusterStateReader) { + (Box::new(NoOpConsensusProcessor::new()), Arc::new(RwLock::new(SharedClusterState::non_cluster()))) } +#[derive(Debug)] +pub struct NoOpConsensusProcessor { + primary: AtomicBoolWriter, + primary_address: Arc>> +} + +impl NoOpConsensusProcessor { + fn new() -> NoOpConsensusProcessor { + NoOpConsensusProcessor { + primary: AtomicBoolWriter::with_value(true), + primary_address: Arc::new(RwLock::new(None)), + } + } +} + +impl ConsensusProcessor for NoOpConsensusProcessor { + + fn peer_connection_established(&mut self, peer_id: FloInstanceId, connection_id: ConnectionId) { + unimplemented!() + } + fn system_primary_status_reader(&self) -> AtomicBoolReader { + self.primary.reader() + } + fn system_primary_address_reader(&self) -> Arc>> { + self.primary_address.clone() + } +} diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 79bd3f5..6008a92 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -18,7 +18,8 @@ pub struct ConnectionRef { pub enum SystemOpType { PartitionOp(partition::OpType), IncomingConnectionEstablished(ConnectionRef), - ConnectionClosed(ConnectionId), + ConnectionUpgradeToPeer, + ConnectionClosed, OutgoingConnectionFailed(SocketAddr), } @@ -31,12 +32,16 @@ pub struct SystemOperation { impl SystemOperation { + pub fn connection_upgraded_to_peer(connection_id: ConnectionId) -> SystemOperation { + SystemOperation::new(connection_id, SystemOpType::ConnectionUpgradeToPeer) + } + pub fn incoming_connection_established(connection: ConnectionRef) -> SystemOperation { SystemOperation::new(connection.connection_id, SystemOpType::IncomingConnectionEstablished(connection)) } pub fn connection_closed(connection_id: ConnectionId) -> SystemOperation { - SystemOperation::new(connection_id, SystemOpType::ConnectionClosed(connection_id)) + SystemOperation::new(connection_id, SystemOpType::ConnectionClosed) } pub fn outgoing_connection_failed(addr: SocketAddr) -> SystemOperation { diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index 028eaf6..51eb05f 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -9,7 +9,8 @@ use tokio_core::reactor::Remote; use engine::{EngineRef, system_stream_name, SYSTEM_STREAM_NAME}; use engine::controller::{SystemPartitionSender, SystemPartitionReceiver, SystemOperation}; -use engine::controller::cluster_state::{init_cluster_state, ClusterState}; +use engine::controller::cluster_state::{init_consensus_processor, ClusterState, ClusterStateReader}; +use engine::controller::peer_connection::OutgoingConnectionCreatorImpl; use engine::event_stream::{EventStreamRefMut, EventStreamOptions, init_existing_event_stream, @@ -21,33 +22,34 @@ use engine::event_stream::partition::{PartitionSender, create_partition_channels}; use engine::event_stream::partition::controller::PartitionImpl; use atomics::{AtomicBoolReader, AtomicBoolWriter, AtomicCounterReader}; +use event_loops::LoopHandles; use super::{FloController, SystemStreamRef}; /// Options passed to the controller on startup that determine how this instance will start and behave. /// These options will come from the command line if this is a standalone server. -#[derive(Debug, PartialEq)] +#[derive(Debug)] pub struct ControllerOptions { pub storage_dir: PathBuf, pub default_stream_options: EventStreamOptions, pub cluster_options: Option, } -#[derive(Debug, PartialEq)] +#[derive(Debug)] pub struct ClusterOptions { pub this_instance_address: SocketAddr, pub peer_addresses: Vec, + pub event_loop_handles: LoopHandles, } pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Result { debug!("Starting Flo Controller with: {:?}", options); let ControllerOptions{storage_dir, default_stream_options, cluster_options} = options; - - let cluster_state = init_cluster_state(&storage_dir, cluster_options)?; // early return if this fails + let (consensus_processor, shared_cluster_state) = init_consensus_processor(&storage_dir, cluster_options)?; // early return if this fails let system_partition = init_system_partition(&storage_dir, - cluster_state.system_primary_status_reader(), + consensus_processor.system_primary_status_reader(), &default_stream_options)?; // early return if this fails debug!("Initialized system partition"); @@ -55,19 +57,24 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul let user_streams = init_user_streams(&storage_dir, &default_stream_options, &remote)?; debug!("Initialized all {} user event streams", user_streams.len()); - let system_primary_reader = cluster_state.system_primary_status_reader(); - let system_primary_server_addr = cluster_state.system_primary_address_reader(); + let system_primary_reader = consensus_processor.system_primary_status_reader(); + let system_primary_server_addr = consensus_processor.system_primary_address_reader(); let system_highest_counter = system_partition.event_counter_reader(); let flo_controller = FloController::new(system_partition, user_streams, storage_dir, - cluster_state, + consensus_processor, default_stream_options); let (system_partition_tx, system_partition_rx) = ::engine::controller::create_system_partition_channels(); - let engine_ref = create_engine_ref(&flo_controller, system_highest_counter, system_primary_reader, system_primary_server_addr, system_partition_tx); + let engine_ref = create_engine_ref(&flo_controller, + system_highest_counter, + system_primary_reader, + system_primary_server_addr, + system_partition_tx, + shared_cluster_state); run_controller_impl(flo_controller, system_partition_rx); @@ -99,7 +106,8 @@ fn create_engine_ref(controller: &FloController, system_highest_counter: AtomicCounterReader, system_primary_reader: AtomicBoolReader, system_primary_addr: Arc>>, - system_partition_sender: SystemPartitionSender) -> EngineRef { + system_partition_sender: SystemPartitionSender, + cluster_state_reader: ClusterStateReader) -> EngineRef { let system_partition_ref = PartitionRef::system(system_stream_name(), 1, @@ -108,7 +116,6 @@ fn create_engine_ref(controller: &FloController, system_partition_sender.clone(), system_primary_addr); - let cluster_state_reader = controller.get_cluster_state_reader(); let system_stream_ref = SystemStreamRef::new(system_partition_ref, system_partition_sender, cluster_state_reader); let shared_stream_refs = controller.get_shared_streams(); diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index fa17bbb..7e7f818 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -17,7 +17,7 @@ use engine::event_stream::{EventStreamRef, use engine::event_stream::partition::Operation; use engine::event_stream::partition::controller::PartitionImpl; use atomics::AtomicBoolWriter; -use self::cluster_state::ClusterState; +use self::cluster_state::{ClusterState, ConsensusProcessor}; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; @@ -51,14 +51,14 @@ pub struct FloController { /// the partition that persists system events. Used as the RAFT log system_partition: PartitionImpl, - cluster_state: ClusterState, + cluster_state: Box, } impl FloController { pub fn new(system_partition: PartitionImpl, event_streams: HashMap, storage_dir: PathBuf, - cluster_state: ClusterState, + cluster_state: Box, default_stream_options: EventStreamOptions) -> FloController { let stream_refs = event_streams.iter().map(|(k, v)| { @@ -88,10 +88,6 @@ impl FloController { fn get_shared_streams(&self) -> Arc>> { self.shared_event_stream_refs.clone() } - - fn get_cluster_state_reader(&self) -> ClusterStateReader { - self.cluster_state.reader() - } } diff --git a/flo-server/src/engine/controller/peer_connection/mod.rs b/flo-server/src/engine/controller/peer_connection/mod.rs index 6930644..af06229 100644 --- a/flo-server/src/engine/controller/peer_connection/mod.rs +++ b/flo-server/src/engine/controller/peer_connection/mod.rs @@ -7,10 +7,11 @@ use tokio_core::reactor::Handle; use tokio_core::net::TcpStream; use futures::Future; -use engine::EngineRef; +use engine::{EngineRef, ConnectionId}; use engine::connection_handler::{create_connection_control_channels, ConnectionControlSender}; use event_loops::LoopHandles; use flo_io::create_connection_handler; +use self::system::PeerConnectionImpl; pub use self::system::PeerSystemConnection; @@ -27,15 +28,27 @@ pub struct OutgoingConnectionCreatorImpl { engine_ref: EngineRef, } +impl OutgoingConnectionCreatorImpl { + pub fn new(loops: LoopHandles, engine: EngineRef) -> OutgoingConnectionCreatorImpl { + OutgoingConnectionCreatorImpl { + event_loops: loops, + engine_ref: engine, + } + } +} + impl OutgoingConnectionCreator for OutgoingConnectionCreatorImpl { fn establish_system_connection(&mut self, address: SocketAddr) -> Box { - unimplemented!() + let OutgoingConnectionCreatorImpl { ref mut event_loops, ref engine_ref } = *self; + + let (sender, connection_id) = create_outgoing_connection(event_loops, address, engine_ref.clone()); + Box::new(PeerConnectionImpl::new(connection_id, sender)) } } -fn create_outgoing_connection(loops: &mut LoopHandles, client_addr: SocketAddr, engine_ref: EngineRef) -> ConnectionControlSender { - +fn create_outgoing_connection(loops: &mut LoopHandles, client_addr: SocketAddr, engine_ref: EngineRef) -> (ConnectionControlSender, ConnectionId) { + let connection_id = engine_ref.next_connection_id(); let client_addr_copy = client_addr.clone(); let mut system_stream = engine_ref.get_system_stream(); @@ -51,7 +64,6 @@ fn create_outgoing_connection(loops: &mut LoopHandles, client_addr: SocketAddr, system_stream.outgoing_connection_failed(addr); }).and_then( move |tcp_stream| { - let connection_id = engine_ref.next_connection_id(); create_connection_handler(owned_handle, engine_ref, @@ -62,7 +74,7 @@ fn create_outgoing_connection(loops: &mut LoopHandles, client_addr: SocketAddr, }) }); - control_tx + (control_tx, connection_id) } diff --git a/flo-server/src/engine/controller/peer_connection/system.rs b/flo-server/src/engine/controller/peer_connection/system.rs index cc65762..1c177a6 100644 --- a/flo-server/src/engine/controller/peer_connection/system.rs +++ b/flo-server/src/engine/controller/peer_connection/system.rs @@ -11,6 +11,7 @@ use futures::{Future, Stream, Sink}; use protocol::{FloInstanceId, Term}; use event::EventCounter; use event_loops::LoopHandles; +use engine::ConnectionId; use engine::controller::SystemStreamRef; use engine::connection_handler::ConnectionControlSender; @@ -26,13 +27,25 @@ pub struct CallAppendEntries { /// Trait representing an active peer connection, with functions to control it pub trait PeerSystemConnection: Debug + Send + 'static { - + fn connection_id(&self) -> ConnectionId; } +#[derive(Debug)] +pub struct PeerConnectionImpl { + connection_id: ConnectionId, + sender: ConnectionControlSender +} +impl PeerConnectionImpl { + pub fn new(connection_id: ConnectionId, sender: ConnectionControlSender) -> PeerConnectionImpl { + PeerConnectionImpl { connection_id, sender } + } +} -#[derive(Debug)] -struct OutgoingPeerSystemConnection { - client_tx: ConnectionControlSender, +impl PeerSystemConnection for PeerConnectionImpl { + fn connection_id(&self) -> ConnectionId { + self.connection_id + } } + diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index 06d24a7..6f41d5c 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -51,6 +51,11 @@ impl SystemStreamRef { self.send(op); } + pub fn connection_upgraded_to_peer(&mut self, connection_id: ConnectionId) { + let op = SystemOperation::connection_upgraded_to_peer(connection_id); + self.send(op); + } + fn send(&mut self, op: SystemOperation) { self.system_sender.send(op).expect("Failed to send to flo controller. System must have shut down"); } diff --git a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs index c249488..2829745 100644 --- a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs +++ b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs @@ -41,7 +41,7 @@ impl CommitManager { self.peers.sort_by_key(|elem| elem.1); let idx = self.min_required_for_commit; - let mut ack_counter = self.peers[idx as usize].1; + let ack_counter = self.peers[idx as usize].1; if self.commit_index.set_if_greater(ack_counter as usize) { Some(ack_counter) diff --git a/flo-server/src/main.rs b/flo-server/src/main.rs index b8dbbcf..04e3d45 100644 --- a/flo-server/src/main.rs +++ b/flo-server/src/main.rs @@ -24,7 +24,6 @@ extern crate tempdir; pub use flo_server::*; use chrono::Duration; -use event::ActorId; use logging::{init_logging, LogLevelOption, LogFileOption}; use clap::{App, Arg, ArgMatches}; use std::str::FromStr; diff --git a/flo-server/src/server/mod.rs b/flo-server/src/server/mod.rs index 0269c1b..0857ed6 100644 --- a/flo-server/src/server/mod.rs +++ b/flo-server/src/server/mod.rs @@ -34,6 +34,7 @@ pub fn run(mut options: ServerOptions) -> io::Result<()> { ClusterOptions { this_instance_address: server_addr, peer_addresses: peers, + event_loop_handles: event_loop_handles.clone(), } }); diff --git a/flo-server/tests/embedded_tests.rs b/flo-server/tests/embedded_tests.rs index 011a07e..a4f2323 100644 --- a/flo-server/tests/embedded_tests.rs +++ b/flo-server/tests/embedded_tests.rs @@ -12,7 +12,6 @@ extern crate log; use std::fmt::Debug; use std::thread; use std::time::Duration; -use std::net::SocketAddr; use tokio_core::reactor::Core; use futures::{Stream, Future}; From 8383bb91c651fed05428a3bb46cf52e2aed4c3c3 Mon Sep 17 00:00:00 2001 From: pfried Date: Wed, 20 Dec 2017 00:39:23 -0500 Subject: [PATCH 24/73] Send a Tick operation to the system controller at a random interval between 150 and 300 milliseconds --- Cargo.lock | 33 ++++++++++ flo-server/Cargo.toml | 1 + .../engine/controller/controller_messages.rs | 5 ++ .../src/engine/controller/initialization.rs | 5 ++ flo-server/src/engine/controller/mod.rs | 1 + .../src/engine/controller/system_stream.rs | 11 ++++ .../src/engine/controller/tick_generator.rs | 62 +++++++++++++++++++ flo-server/src/lib.rs | 1 + 8 files changed, 119 insertions(+) create mode 100644 flo-server/src/engine/controller/tick_generator.rs diff --git a/Cargo.lock b/Cargo.lock index f26a59d..3667696 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,11 @@ name = "bitflags" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bitflags" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byteorder" version = "0.4.2" @@ -275,6 +280,7 @@ dependencies = [ "memmap 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -306,6 +312,15 @@ dependencies = [ "fuchsia-zircon-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fuchsia-zircon" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "fuchsia-zircon-sys" version = "0.2.0" @@ -314,6 +329,11 @@ dependencies = [ "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "futures" version = "0.1.17" @@ -600,6 +620,15 @@ dependencies = [ "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rand" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "redox_syscall" version = "0.1.31" @@ -996,6 +1025,7 @@ dependencies = [ "checksum atty 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "21e50800ec991574876040fff8ee46b136a53e985286fbe6a3bdfe6421b78860" "checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" "checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" +"checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf" "checksum byteorder 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "96c8b41881888cc08af32d47ac4edd52bc7fa27fef774be47a92443756451304" "checksum byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff81738b726f5d099632ceaffe7fb65b90212e8dce59d518729e7e8634032d3d" "checksum bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d828f97b58cc5de3e40c421d0cf2132d6b2da4ee0e11b8632fa838f0f9333ad6" @@ -1018,7 +1048,9 @@ dependencies = [ "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum fs2 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ab76cfd2aaa59b7bf6688ad9ba15bbae64bff97f04ea02144cfd3443e5c2866" "checksum fuchsia-zircon 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f6c0581a4e363262e52b87f59ee2afe3415361c6ec35e665924eb08afe8ff159" +"checksum fuchsia-zircon 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bd510087c325af53ba24f3be8f1c081b0982319adcb8b03cad764512923ccc19" "checksum fuchsia-zircon-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "43f3795b4bae048dc6123a6b972cadde2e676f9ded08aef6bb77f5f157684a82" +"checksum fuchsia-zircon-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "08b3a6f13ad6b96572b53ce7af74543132f1a7055ccceb6d073dd36c54481859" "checksum futures 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "118b49cac82e04121117cbd3121ede3147e885627d82c4546b87c702debb90c1" "checksum getopts 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)" = "65922871abd2f101a2eb0eaebadc66668e54a87ad9c3dd82520b5f86ede5eff9" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" @@ -1057,6 +1089,7 @@ dependencies = [ "checksum quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eda5fe9b71976e62bc81b781206aaa076401769b2143379d3eb2118388babac4" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum rand 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)" = "6475140dfd8655aeb72e1fd4b7a1cc1c202be65d71669476e392fe62532b9edd" +"checksum rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9d5f78082e6a6d042862611e9640cf20776185fee506cf6cf67e93c6225cee31" "checksum redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "8dde11f18c108289bef24469638a04dce49da56084f2d50618b226e47eb04509" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1731164734096285ec2a5ec7fea5248ae2f5485b3feeb0115af4fda2183b2d1b" diff --git a/flo-server/Cargo.toml b/flo-server/Cargo.toml index 13496c9..95ad88b 100644 --- a/flo-server/Cargo.toml +++ b/flo-server/Cargo.toml @@ -22,6 +22,7 @@ libc = "0.2.33" serde = "1.0" serde_json = "1.0" serde_derive = "1.0" +rand = "0.4" [dev-dependencies] env_logger = "*" diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 6008a92..1df5d56 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -16,6 +16,7 @@ pub struct ConnectionRef { #[derive(Debug)] pub enum SystemOpType { + Tick, PartitionOp(partition::OpType), IncomingConnectionEstablished(ConnectionRef), ConnectionUpgradeToPeer, @@ -48,6 +49,10 @@ impl SystemOperation { SystemOperation::new(0, SystemOpType::OutgoingConnectionFailed(addr)) } + pub fn tick() -> SystemOperation { + SystemOperation::new(0, SystemOpType::Tick) + } + fn new(connection_id: ConnectionId, op_type: SystemOpType) -> SystemOperation { SystemOperation { connection_id, diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index 51eb05f..5c7fdf3 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -75,9 +75,12 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul system_primary_server_addr, system_partition_tx, shared_cluster_state); + let system_stream_ref = engine_ref.get_system_stream(); run_controller_impl(flo_controller, system_partition_rx); + ::engine::controller::tick_generator::spawn_tick_generator(remote, system_stream_ref); + Ok(engine_ref) } @@ -169,3 +172,5 @@ fn run_controller_impl(mut controller: FloController, system_partition_rx: Syste controller.shutdown(); }); } + + diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 7e7f818..02ca9b7 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -3,6 +3,7 @@ mod system_stream; mod initialization; mod controller_messages; mod peer_connection; +mod tick_generator; use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index 6f41d5c..60bb13d 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -36,6 +36,16 @@ impl SystemStreamRef { EventStreamRef::new(name, partition_ref) } + pub fn tick(&mut self) -> Result<(), ()> { + let op = SystemOperation::tick(); + self.system_sender.send(op).map_err(|_| ()) + } + + pub fn tick_error(&mut self) { + // TODO: send a message to the system partition to let it know that there was an error so that it can resign as primary + unimplemented!() + } + pub fn incomming_connection_accepted(&mut self, connection_ref: ConnectionRef) { let op = SystemOperation::incoming_connection_established(connection_ref); self.send(op); @@ -57,6 +67,7 @@ impl SystemStreamRef { } fn send(&mut self, op: SystemOperation) { + // TODO: change this to propagate the error so that connectionHandlers can shut down gracefully self.system_sender.send(op).expect("Failed to send to flo controller. System must have shut down"); } } diff --git a/flo-server/src/engine/controller/tick_generator.rs b/flo-server/src/engine/controller/tick_generator.rs new file mode 100644 index 0000000..9946a8b --- /dev/null +++ b/flo-server/src/engine/controller/tick_generator.rs @@ -0,0 +1,62 @@ +use std::time::Duration; +use std::io; + +use tokio_core::reactor::{Interval, Remote}; +use futures::{Stream, Future}; +use rand::distributions::{Range, IndependentSample}; +use rand::thread_rng; + +use engine::controller::{SystemPartitionSender, SystemStreamRef}; + +#[derive(Debug)] +enum TickError { + Io(io::Error), + Send, +} + +impl From for TickError { + fn from(io_err: io::Error) -> Self { + TickError::Io(io_err) + } +} + + +pub fn spawn_tick_generator(remote: Remote, mut system_sender: SystemStreamRef) { + // clone the inputs so we can re-run in the case of a failure + let remote_copy = remote.clone(); + let mut system_stream_ref_clone = system_sender.clone(); + let tick_interval = get_controller_tick_interval_millis(); + info!("Using tick interval of {} milliseconds", tick_interval); + + remote.spawn(move |handle| { + // TODO: handle the failure to create the Interval + let interval = Interval::new(Duration::from_millis(tick_interval), handle) + .expect("failed to create tick interval"); + interval.map_err(|io_err| TickError::Io(io_err)) + .for_each(move |_| { + system_sender.tick().map_err(|_| TickError::Send) + }) + .then(move |result| { + match result { + Ok(()) => unreachable!(), + Err(TickError::Send) => { + // system controller has shut down, so just quit + debug!("Shut down system tick generator") + } + Err(TickError::Io(io_err)) => { + error!("Error generating tick operations for system stream: {}", io_err); + system_stream_ref_clone.tick_error(); + spawn_tick_generator(remote_copy, system_stream_ref_clone); + } + } + // always return Ok so that this future will be considered resolved. We'll just spawn a new one anyway + Ok(()) + }) + }); +} + +fn get_controller_tick_interval_millis() -> u64 { + + let range = Range::new(150u64, 300u64); + range.ind_sample(&mut thread_rng()) +} diff --git a/flo-server/src/lib.rs b/flo-server/src/lib.rs index f6f25a8..0078958 100644 --- a/flo-server/src/lib.rs +++ b/flo-server/src/lib.rs @@ -16,6 +16,7 @@ extern crate clap; extern crate log4rs; extern crate num_cpus; extern crate byteorder; +extern crate rand; #[macro_use] extern crate serde_derive; From 7e3b27390114806a5276d01b68427ff494de3db8 Mon Sep 17 00:00:00 2001 From: pfried Date: Wed, 20 Dec 2017 08:39:49 -0500 Subject: [PATCH 25/73] remove debug logging from travis builds --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 43ae6f6..e148615 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: rust env: global: - RUST_BACKTRACE=1 - - RUST_LOG=flo=debug rust: - stable - beta From 36732c42b92fa440fb6f3b17fae12eba18f6bcff Mon Sep 17 00:00:00 2001 From: pfried Date: Mon, 25 Dec 2017 21:03:33 -0500 Subject: [PATCH 26/73] work basic cluster state and management into initialization --- .../src/engine/connection_handler/mod.rs | 5 +- .../src/engine/connection_handler/peer/mod.rs | 9 +- .../engine/controller/cluster_state/mod.rs | 117 +++++++----------- .../engine/controller/controller_messages.rs | 9 +- .../src/engine/controller/initialization.rs | 74 ++++++++--- flo-server/src/engine/controller/mod.rs | 40 ++++-- .../engine/controller/peer_connection/mod.rs | 17 ++- .../controller/peer_connection/system.rs | 8 ++ .../src/engine/controller/system_stream.rs | 7 +- flo-server/src/server/mod.rs | 2 +- 10 files changed, 171 insertions(+), 117 deletions(-) diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 5515b8a..38c8eb6 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -72,7 +72,7 @@ impl ConnectionHandler { pub fn handle_incoming_message(&mut self, message: ReceivedProtocolMessage) -> ConnectionHandlerResult { trace!("client: {:?}, received message: {:?}", self.common_state, message); - let ConnectionHandler{ref mut common_state, ref mut consumer_state, ref mut producer_state, .. } = *self; + let ConnectionHandler{ref mut common_state, ref mut consumer_state, ref mut producer_state, ref mut peer_state } = *self; match message { ProtocolMessage::SetEventStream(SetEventStream{op_id, name}) => { @@ -81,6 +81,9 @@ impl ConnectionHandler { ProtocolMessage::Announce(announce) => { common_state.handle_announce_message(announce) }, + ProtocolMessage::PeerAnnounce(peer_announce) => { + peer_state.peer_announce_received(peer_announce, common_state) + } ProtocolMessage::ProduceEvent(produce) => { producer_state.handle_produce(produce, common_state) }, diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 7089737..b063cce 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -28,7 +28,7 @@ impl PeerConnectionState { *self = PeerConnectionState::AwaitingPeerResponse; } - pub fn peer_announce_received(&mut self, state: &mut ConnectionState) -> ConnectionHandlerResult { + pub fn peer_announce_received(&mut self, announce: PeerAnnounce, state: &mut ConnectionState) -> ConnectionHandlerResult { let connection_id = state.connection_id; let old_state = self.set_state(connection_id, PeerConnectionState::Peer); @@ -42,12 +42,15 @@ impl PeerConnectionState { // This was an incoming connection, and this was the first peer message sent, so we need to respond in kind let peer_announce = PeerConnectionState::create_peer_announce(state.get_system_stream()); state.send_to_client(ProtocolMessage::PeerAnnounce(peer_announce))?; + + } + PeerConnectionState::AwaitingPeerResponse => { + } - PeerConnectionState::AwaitingPeerResponse => { } } state.set_to_system_stream(); - state.get_system_stream().connection_upgraded_to_peer(connection_id); + state.get_system_stream().connection_upgraded_to_peer(connection_id, announce.instance_id); Ok(()) } diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 16047c1..5ea1e13 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -7,40 +7,37 @@ use std::time::{Instant}; use std::path::Path; use event::EventCounter; -use engine::ConnectionId; +use engine::{ConnectionId, EngineRef}; use protocol::FloInstanceId; use atomics::{AtomicBoolWriter, AtomicBoolReader}; use super::ClusterOptions; -use super::peer_connection::PeerSystemConnection; +use super::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator, OutgoingConnectionCreatorImpl}; pub use self::persistent::{FilePersistedState, PersistentClusterState}; pub trait ConsensusProcessor: Send { - fn system_primary_address_reader(&self) -> Arc>>; - fn system_primary_status_reader(&self) -> AtomicBoolReader; fn peer_connection_established(&mut self, peer_id: FloInstanceId, connection_id: ConnectionId); + fn outgoing_connection_failed(&mut self, address: SocketAddr); } #[derive(Debug)] -pub struct ClusterState { +pub struct ClusterManager { primary_status_writer: AtomicBoolWriter, starting_peer_addresses: Vec, last_applied: EventCounter, persistent: FilePersistedState, system_partion_primary_address: SystemPrimaryAddressRef, shared: Arc>, + outgoing_connection_creator: Box, } -impl ConsensusProcessor for ClusterState { +impl ConsensusProcessor for ClusterManager { - fn peer_connection_established(&mut self, peer_id: FloInstanceId, connection_id: ConnectionId) { + fn outgoing_connection_failed(&mut self, address: SocketAddr) { unimplemented!() } - fn system_primary_status_reader(&self) -> AtomicBoolReader { - self.primary_status_writer.reader() - } - fn system_primary_address_reader(&self) -> Arc>> { - self.system_partion_primary_address.clone() + fn peer_connection_established(&mut self, peer_id: FloInstanceId, connection_id: ConnectionId) { + unimplemented!() } } @@ -53,7 +50,7 @@ pub struct SharedClusterState { } impl SharedClusterState { - fn non_cluster() -> SharedClusterState { + pub fn non_cluster() -> SharedClusterState { SharedClusterState { this_instance_id: FloInstanceId::generate_new(), this_address: None, @@ -70,81 +67,63 @@ pub struct Peer { } -impl ClusterState { - - pub fn system_primary_status_reader(&self) -> AtomicBoolReader { - self.primary_status_writer.reader() - } - - pub fn system_primary_address_reader(&self) -> SystemPrimaryAddressRef { - self.system_partion_primary_address.clone() - } -} - pub type SystemPrimaryAddressRef = Arc>>; - pub type ClusterStateReader = Arc>; -pub fn init_consensus_processor(storage_dir: &Path, options: Option) -> io::Result<(Box, ClusterStateReader)> { - let start_as_primary = options.is_none(); - if let Some(options) = options { - debug!("Initializing in clustered mode, starting as secondary"); - init_cluster_state(storage_dir, options) - } else { - debug!("Initializing in non-cluster mode, system stream will always be considered primary"); - Ok(init_no_op_consensus_processor()) - } -} - -fn init_cluster_state(storage_dir: &Path, options: ClusterOptions) -> io::Result<(Box, ClusterStateReader)> { - let path = storage_dir.join("cluster-state"); - let persistent_state = FilePersistedState::initialize(path)?; // early return if this fails - - let ClusterOptions{ this_instance_address, peer_addresses, event_loop_handles } = options; - - let shared_state = persistent_state.initialize_shared_state(Some(this_instance_address)); - let shared_state_ref = Arc::new(RwLock::new(shared_state)); - - let state = ClusterState { - primary_status_writer: AtomicBoolWriter::with_value(false), +//pub fn init_consensus_processor(storage_dir: &Path, +// options: Option, +// engine_ref: EngineRef, +// mut system_primary: AtomicBoolWriter, +// primary_address: SystemPrimaryAddressRef) -> io::Result<(Box, ClusterStateReader)> { +// +// if let Some(options) = options { +// system_primary.set(false); +// debug!("Initializing in clustered mode, starting as secondary"); +// init_cluster_consensus_processor(storage_dir, options, engine_ref, system_primary, primary_address) +// } else { +// debug!("Initializing in non-cluster mode, system stream will always be considered primary"); +// system_primary.set(true); +// Ok(init_no_op_consensus_processor()) +// } +//} + +pub fn init_cluster_consensus_processor(persistent_state: FilePersistedState, + options: ClusterOptions, + engine_ref: EngineRef, + shared_state_ref: ClusterStateReader, + system_primary: AtomicBoolWriter, + primary_address: SystemPrimaryAddressRef) -> Box { + + let ClusterOptions{ peer_addresses, event_loop_handles, .. } = options; + + let outgoing_connection_creator = OutgoingConnectionCreatorImpl::new(event_loop_handles, engine_ref); + + let state = ClusterManager { + primary_status_writer: system_primary, last_applied: 0, starting_peer_addresses: peer_addresses, persistent: persistent_state, - system_partion_primary_address: Arc::new(RwLock::new(None)), - shared: shared_state_ref.clone(), + system_partion_primary_address: primary_address, + shared: shared_state_ref, + outgoing_connection_creator: Box::new(outgoing_connection_creator) }; - Ok((Box::new(state), shared_state_ref)) + Box::new(state) } fn init_no_op_consensus_processor() -> (Box, ClusterStateReader) { - (Box::new(NoOpConsensusProcessor::new()), Arc::new(RwLock::new(SharedClusterState::non_cluster()))) + (Box::new(NoOpConsensusProcessor), Arc::new(RwLock::new(SharedClusterState::non_cluster()))) } #[derive(Debug)] -pub struct NoOpConsensusProcessor { - primary: AtomicBoolWriter, - primary_address: Arc>> -} - -impl NoOpConsensusProcessor { - fn new() -> NoOpConsensusProcessor { - NoOpConsensusProcessor { - primary: AtomicBoolWriter::with_value(true), - primary_address: Arc::new(RwLock::new(None)), - } - } -} +pub struct NoOpConsensusProcessor; impl ConsensusProcessor for NoOpConsensusProcessor { fn peer_connection_established(&mut self, peer_id: FloInstanceId, connection_id: ConnectionId) { unimplemented!() } - fn system_primary_status_reader(&self) -> AtomicBoolReader { - self.primary.reader() - } - fn system_primary_address_reader(&self) -> Arc>> { - self.primary_address.clone() + fn outgoing_connection_failed(&mut self, address: SocketAddr) { + unimplemented!() } } diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 1df5d56..934a39d 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -3,11 +3,12 @@ use std::time::Instant; use futures::sync::mpsc::UnboundedSender; +use protocol::FloInstanceId; use engine::event_stream::partition::{self, Operation}; use engine::connection_handler::{ConnectionControl, ConnectionControlSender}; use engine::ConnectionId; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ConnectionRef { pub connection_id: ConnectionId, pub remote_address: SocketAddr, @@ -19,7 +20,7 @@ pub enum SystemOpType { Tick, PartitionOp(partition::OpType), IncomingConnectionEstablished(ConnectionRef), - ConnectionUpgradeToPeer, + ConnectionUpgradeToPeer(FloInstanceId), ConnectionClosed, OutgoingConnectionFailed(SocketAddr), } @@ -33,8 +34,8 @@ pub struct SystemOperation { impl SystemOperation { - pub fn connection_upgraded_to_peer(connection_id: ConnectionId) -> SystemOperation { - SystemOperation::new(connection_id, SystemOpType::ConnectionUpgradeToPeer) + pub fn connection_upgraded_to_peer(connection_id: ConnectionId, peer_id: FloInstanceId) -> SystemOperation { + SystemOperation::new(connection_id, SystemOpType::ConnectionUpgradeToPeer(peer_id)) } pub fn incoming_connection_established(connection: ConnectionRef) -> SystemOperation { diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index 5c7fdf3..8efc669 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -1,4 +1,4 @@ -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, RwLock, Mutex}; use std::path::{PathBuf, Path}; use std::collections::HashMap; use std::fs::DirEntry; @@ -9,9 +9,15 @@ use tokio_core::reactor::Remote; use engine::{EngineRef, system_stream_name, SYSTEM_STREAM_NAME}; use engine::controller::{SystemPartitionSender, SystemPartitionReceiver, SystemOperation}; -use engine::controller::cluster_state::{init_consensus_processor, ClusterState, ClusterStateReader}; -use engine::controller::peer_connection::OutgoingConnectionCreatorImpl; +use engine::controller::cluster_state::{init_cluster_consensus_processor, + ConsensusProcessor, + NoOpConsensusProcessor, + ClusterManager, + ClusterStateReader, + SystemPrimaryAddressRef, + FilePersistedState}; use engine::event_stream::{EventStreamRefMut, + EventStreamRef, EventStreamOptions, init_existing_event_stream, init_new_event_stream}; @@ -46,41 +52,70 @@ pub struct ClusterOptions { pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Result { debug!("Starting Flo Controller with: {:?}", options); let ControllerOptions{storage_dir, default_stream_options, cluster_options} = options; - let (consensus_processor, shared_cluster_state) = init_consensus_processor(&storage_dir, cluster_options)?; // early return if this fails + let use_cluster_mode = cluster_options.is_some(); + + let system_primary_status_writer = AtomicBoolWriter::with_value(false); + let system_primary_address: SystemPrimaryAddressRef = Arc::new(RwLock::new(None)); let system_partition = init_system_partition(&storage_dir, - consensus_processor.system_primary_status_reader(), + system_primary_status_writer.reader(), &default_stream_options)?; // early return if this fails debug!("Initialized system partition"); // early return if this fails let user_streams = init_user_streams(&storage_dir, &default_stream_options, &remote)?; debug!("Initialized all {} user event streams", user_streams.len()); + let shared_stream_refs = user_streams.iter().map(|(key, value)| { + (key.to_owned(), value.clone_ref()) + }).collect::>(); + let shared_stream_refs = Arc::new(Mutex::new(shared_stream_refs)); - let system_primary_reader = consensus_processor.system_primary_status_reader(); - let system_primary_server_addr = consensus_processor.system_primary_address_reader(); let system_highest_counter = system_partition.event_counter_reader(); + let (system_partition_tx, system_partition_rx) = ::engine::controller::create_system_partition_channels(); + + let (shared_state, file_cluster_state) = if use_cluster_mode { + let path = storage_dir.join("cluster-state"); + let file_state = FilePersistedState::initialize(path)?; + let this_address = cluster_options.as_ref().map(|opts| opts.this_instance_address); + let shared_cluster_state = file_state.initialize_shared_state(this_address); + (shared_cluster_state, Some(file_state)) + } else { + let shared_cluster_state = ::engine::controller::SharedClusterState::non_cluster(); + (shared_cluster_state, None) + }; + let cluster_state_ref = Arc::new(RwLock::new(shared_state)); + let engine_ref = create_engine_ref(shared_stream_refs.clone(), + system_highest_counter, + system_primary_status_writer.reader(), + system_primary_address.clone(), + system_partition_tx, + cluster_state_ref.clone()); + + let consensus_processor: Box = if use_cluster_mode { + let persistent_state = file_cluster_state.unwrap(); + let cluster_opts = cluster_options.unwrap(); + + init_cluster_consensus_processor(persistent_state, + cluster_opts, + engine_ref.clone(), + cluster_state_ref, + system_primary_status_writer, + system_primary_address) + } else { + Box::new(NoOpConsensusProcessor) + }; let flo_controller = FloController::new(system_partition, user_streams, + shared_stream_refs, storage_dir, consensus_processor, default_stream_options); - let (system_partition_tx, system_partition_rx) = ::engine::controller::create_system_partition_channels(); - let engine_ref = create_engine_ref(&flo_controller, - system_highest_counter, - system_primary_reader, - system_primary_server_addr, - system_partition_tx, - shared_cluster_state); - let system_stream_ref = engine_ref.get_system_stream(); - run_controller_impl(flo_controller, system_partition_rx); - + let system_stream_ref = engine_ref.get_system_stream(); ::engine::controller::tick_generator::spawn_tick_generator(remote, system_stream_ref); - Ok(engine_ref) } @@ -105,7 +140,7 @@ fn init_system_partition(storage_dir: &Path, system_primary_reader: AtomicBoolRe } } -fn create_engine_ref(controller: &FloController, +fn create_engine_ref(shared_stream_refs: Arc>>, system_highest_counter: AtomicCounterReader, system_primary_reader: AtomicBoolReader, system_primary_addr: Arc>>, @@ -121,7 +156,6 @@ fn create_engine_ref(controller: &FloController, let system_stream_ref = SystemStreamRef::new(system_partition_ref, system_partition_sender, cluster_state_reader); - let shared_stream_refs = controller.get_shared_streams(); EngineRef::new(system_stream_ref, shared_stream_refs) } diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 02ca9b7..c4e8f97 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -11,6 +11,7 @@ use std::collections::HashMap; use std::net::SocketAddr; +use engine::ConnectionId; use engine::event_stream::{EventStreamRef, EventStreamRefMut, EventStreamOptions}; @@ -18,7 +19,7 @@ use engine::event_stream::{EventStreamRef, use engine::event_stream::partition::Operation; use engine::event_stream::partition::controller::PartitionImpl; use atomics::AtomicBoolWriter; -use self::cluster_state::{ClusterState, ConsensusProcessor}; +use self::cluster_state::{ClusterManager, ConsensusProcessor}; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; @@ -53,42 +54,59 @@ pub struct FloController { system_partition: PartitionImpl, cluster_state: Box, + + all_connections: HashMap, } impl FloController { pub fn new(system_partition: PartitionImpl, event_streams: HashMap, + shared_stream_refs: Arc>>, storage_dir: PathBuf, cluster_state: Box, default_stream_options: EventStreamOptions) -> FloController { - let stream_refs = event_streams.iter().map(|(k, v)| { - (k.to_owned(), v.clone_ref()) - }).collect::>(); - FloController { - shared_event_stream_refs: Arc::new(Mutex::new(stream_refs)), + shared_event_stream_refs: shared_stream_refs, event_streams, system_partition, storage_dir, default_stream_options, cluster_state, + all_connections: HashMap::with_capacity(4), } } fn process(&mut self, operation: SystemOperation) { - warn!("Ignoring SystemOperation: {:?}", operation); - //TODO: handle system operations + let SystemOperation {connection_id, op_start_time, op_type } = operation; + trace!("Received system op: {:?}", op_type); + // TODO: time operation handling and record perf metrics + + match op_type { + SystemOpType::IncomingConnectionEstablished(connection_ref) => { + self.all_connections.insert(connection_id, connection_ref); + } + SystemOpType::OutgoingConnectionFailed(address) => { + self.cluster_state.outgoing_connection_failed(address); + } + SystemOpType::ConnectionUpgradeToPeer(peer_id) => { + if let Some(connection_ref) = self.all_connections.get(&connection_id).cloned() { + unimplemented!() + } else { + error!("Got ConnectionUpgradeToPeer for connection_id: {}, but no connection with that id exists", connection_id); + } + } + other @ _ => { + warn!("Ignoring SystemOperation: {:?}", other); + } + } } fn shutdown(&mut self) { info!("Shutting down FloController"); } - fn get_shared_streams(&self) -> Arc>> { - self.shared_event_stream_refs.clone() - } } diff --git a/flo-server/src/engine/controller/peer_connection/mod.rs b/flo-server/src/engine/controller/peer_connection/mod.rs index af06229..b890354 100644 --- a/flo-server/src/engine/controller/peer_connection/mod.rs +++ b/flo-server/src/engine/controller/peer_connection/mod.rs @@ -1,5 +1,6 @@ mod system; +use std::collections::HashMap; use std::fmt::Debug; use std::net::SocketAddr; use std::io; @@ -9,20 +10,22 @@ use futures::Future; use engine::{EngineRef, ConnectionId}; use engine::connection_handler::{create_connection_control_channels, ConnectionControlSender}; +use engine::controller::ConnectionRef; use event_loops::LoopHandles; use flo_io::create_connection_handler; use self::system::PeerConnectionImpl; -pub use self::system::PeerSystemConnection; +pub use self::system::{PeerSystemConnection, PendingSystemConnection}; pub type ConnectionSendResult = Result<(), T>; /// Trait for creating outgoing connections (clever name, I know). -pub trait OutgoingConnectionCreator { - fn establish_system_connection(&mut self, address: SocketAddr) -> Box; +pub trait OutgoingConnectionCreator: Debug + Send + 'static { + fn establish_system_connection(&mut self, address: SocketAddr) -> ConnectionRef; } +#[derive(Debug)] pub struct OutgoingConnectionCreatorImpl { event_loops: LoopHandles, engine_ref: EngineRef, @@ -38,11 +41,15 @@ impl OutgoingConnectionCreatorImpl { } impl OutgoingConnectionCreator for OutgoingConnectionCreatorImpl { - fn establish_system_connection(&mut self, address: SocketAddr) -> Box { + fn establish_system_connection(&mut self, address: SocketAddr) -> ConnectionRef { let OutgoingConnectionCreatorImpl { ref mut event_loops, ref engine_ref } = *self; let (sender, connection_id) = create_outgoing_connection(event_loops, address, engine_ref.clone()); - Box::new(PeerConnectionImpl::new(connection_id, sender)) + ConnectionRef { + connection_id, + remote_address: address, + control_sender: sender, + } } } diff --git a/flo-server/src/engine/controller/peer_connection/system.rs b/flo-server/src/engine/controller/peer_connection/system.rs index 1c177a6..ad9046d 100644 --- a/flo-server/src/engine/controller/peer_connection/system.rs +++ b/flo-server/src/engine/controller/peer_connection/system.rs @@ -25,12 +25,20 @@ pub struct CallAppendEntries { pub leader_commit_index: EventCounter, } +pub trait PendingSystemConnection: Debug + Send + 'static { + fn connection_id(&self) -> ConnectionId; + + fn complete(self: Box, id: FloInstanceId) -> Box; +} + /// Trait representing an active peer connection, with functions to control it pub trait PeerSystemConnection: Debug + Send + 'static { fn connection_id(&self) -> ConnectionId; } + + #[derive(Debug)] pub struct PeerConnectionImpl { connection_id: ConnectionId, diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index 60bb13d..ead496d 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -1,5 +1,6 @@ use std::net::SocketAddr; +use protocol::FloInstanceId; use engine::event_stream::partition::{PartitionRef, Operation}; use engine::event_stream::EventStreamRef; use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef}; @@ -46,7 +47,7 @@ impl SystemStreamRef { unimplemented!() } - pub fn incomming_connection_accepted(&mut self, connection_ref: ConnectionRef) { + pub fn incoming_connection_accepted(&mut self, connection_ref: ConnectionRef) { let op = SystemOperation::incoming_connection_established(connection_ref); self.send(op); } @@ -61,8 +62,8 @@ impl SystemStreamRef { self.send(op); } - pub fn connection_upgraded_to_peer(&mut self, connection_id: ConnectionId) { - let op = SystemOperation::connection_upgraded_to_peer(connection_id); + pub fn connection_upgraded_to_peer(&mut self, connection_id: ConnectionId, peer_id: FloInstanceId) { + let op = SystemOperation::connection_upgraded_to_peer(connection_id, peer_id); self.send(op); } diff --git a/flo-server/src/server/mod.rs b/flo-server/src/server/mod.rs index 0857ed6..a70e6f6 100644 --- a/flo-server/src/server/mod.rs +++ b/flo-server/src/server/mod.rs @@ -91,7 +91,7 @@ fn handle_incoming_connection(engine_ref: &EngineRef, remote_handle: Remote, tcp remote_address: client_addr, control_sender: control_tx, }; - client_engine_ref.system_stream().incomming_connection_accepted(connection_ref); + client_engine_ref.system_stream().incoming_connection_accepted(connection_ref); remote_handle.spawn(move |client_handle| { create_connection_handler(client_handle.clone(), client_engine_ref, connection_id, client_addr, tcp_stream, control_rx) From 2d6223c9da7b62376352fc92915f74b59266f546 Mon Sep 17 00:00:00 2001 From: pfried Date: Thu, 28 Dec 2017 21:26:53 -0500 Subject: [PATCH 27/73] connections established on startup with known cluster members --- .../src/engine/connection_handler/peer/mod.rs | 23 +- .../engine/controller/cluster_state/mod.rs | 286 +++++++++++++---- .../cluster_state/peer_connections.rs | 296 ++++++++++++++++++ .../controller/cluster_state/persistent.rs | 11 +- .../engine/controller/controller_messages.rs | 29 +- .../src/engine/controller/initialization.rs | 7 +- flo-server/src/engine/controller/mod.rs | 22 +- .../engine/controller/peer_connection/mod.rs | 54 +++- .../src/engine/controller/system_stream.rs | 10 +- .../src/engine/controller/tick_generator.rs | 11 +- flo-server/src/lib.rs | 3 + flo-server/src/main.rs | 14 + flo-server/src/server/mod.rs | 2 + flo-server/src/server/server_options.rs | 2 + flo-server/src/test_utils.rs | 6 + 15 files changed, 685 insertions(+), 91 deletions(-) create mode 100644 flo-server/src/engine/controller/cluster_state/peer_connections.rs create mode 100644 flo-server/src/test_utils.rs diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index b063cce..852b1c2 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -3,6 +3,7 @@ mod peer_follower; use protocol::{ProtocolMessage, PeerAnnounce, EventStreamStatus, ClusterMember}; use engine::{ReceivedProtocolMessage, ConnectionId}; use engine::controller::SystemStreamRef; +use engine::controller::Peer; use super::connection_state::ConnectionState; use super::ConnectionHandlerResult; @@ -48,9 +49,23 @@ impl PeerConnectionState { } } - state.set_to_system_stream(); - state.get_system_stream().connection_upgraded_to_peer(connection_id, announce.instance_id); + + let PeerAnnounce {instance_id, system_primary_id, cluster_members, ..} = announce; + let primary = system_primary_id.and_then(|primary_id| { + cluster_members.iter().find(|member| { + member.id == primary_id + }).map(|member| { + Peer { + id: member.id, + address: member.address, + } + }) + }); + let peers = cluster_members.into_iter().map(|member| { + member_to_peer(member) + }).collect(); + state.get_system_stream().connection_upgraded_to_peer(connection_id, announce.instance_id, primary, peers); Ok(()) } @@ -83,6 +98,10 @@ impl PeerConnectionState { } } +fn member_to_peer(ClusterMember{id, address}: ClusterMember) -> Peer { + Peer { id, address } +} + #[cfg(test)] mod test { diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 5ea1e13..d300575 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -1,43 +1,160 @@ mod persistent; +mod peer_connections; use std::io; use std::sync::{Arc, RwLock}; use std::net::SocketAddr; -use std::time::{Instant}; +use std::time::{Instant, Duration}; use std::path::Path; +use std::collections::{HashMap, HashSet}; use event::EventCounter; use engine::{ConnectionId, EngineRef}; use protocol::FloInstanceId; use atomics::{AtomicBoolWriter, AtomicBoolReader}; -use super::ClusterOptions; +use super::{ClusterOptions, ConnectionRef, Peer, PeerUpgrade}; use super::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator, OutgoingConnectionCreatorImpl}; +use self::peer_connections::PeerConnections; pub use self::persistent::{FilePersistedState, PersistentClusterState}; pub trait ConsensusProcessor: Send { - fn peer_connection_established(&mut self, peer_id: FloInstanceId, connection_id: ConnectionId); - fn outgoing_connection_failed(&mut self, address: SocketAddr); + fn tick(&mut self, now: Instant, all_connections: &mut HashMap); + fn is_primary(&self) -> bool; + fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap); + fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr); +} + +#[derive(Debug)] +pub struct NoOpConsensusProcessor; + +impl ConsensusProcessor for NoOpConsensusProcessor { + fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap) { + } + fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { + panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); + } + fn tick(&mut self, now: Instant, all_connections: &mut HashMap) { + } + fn is_primary(&self) -> bool { + true + } } #[derive(Debug)] pub struct ClusterManager { + state: State, + initialization_peers: HashSet, + election_timeout: Duration, + last_heartbeat: Instant, primary_status_writer: AtomicBoolWriter, - starting_peer_addresses: Vec, last_applied: EventCounter, persistent: FilePersistedState, - system_partion_primary_address: SystemPrimaryAddressRef, + system_partition_primary_address: SystemPrimaryAddressRef, shared: Arc>, - outgoing_connection_creator: Box, + connection_manager: PeerConnections, +} + +#[derive(Debug, PartialEq, Copy, Clone)] +enum State { + EstablishConnections, + DeterminePrimary, + Follower, + Voted, + Primary, +} + +impl ClusterManager { + fn new(election_timeout_millis: u64, + starting_peer_addresses: Vec, + persistent: FilePersistedState, + shared: Arc>, + primary_status_writer: AtomicBoolWriter, + system_partition_primary_address: SystemPrimaryAddressRef, + outgoing_connection_creator: Box) -> ClusterManager { + + let mut initialization_peers = starting_peer_addresses.iter().cloned().collect::>(); + for peer in persistent.cluster_members.iter() { + initialization_peers.insert(peer.address); + } + + let peer_connections = PeerConnections::new(starting_peer_addresses, outgoing_connection_creator, &persistent.cluster_members); + + ClusterManager { + state: State::EstablishConnections, + election_timeout: Duration::from_millis(election_timeout_millis), + last_heartbeat: Instant::now(), + connection_manager: peer_connections, + initialization_peers, + last_applied: 0, + primary_status_writer, + persistent, + system_partition_primary_address, + shared, + } + } + + fn transition_state(&mut self, new_state: State) { + debug!("Transitioning from state: {:?} to {:?}", self.state, new_state); + self.state = new_state; + } + + fn determine_primary_from_peer_upgrade(&mut self, upgrade: PeerUpgrade) { + let PeerUpgrade { peer_id, system_primary, cluster_members } = upgrade; + if let Some(primary) = system_primary { + info!("Determined primary of {:?} at {}, as told by instance: {:?}", primary.id, primary.address, peer_id); + self.transition_state(State::Follower); + { + let mut lock = self.system_partition_primary_address.write().unwrap(); + *lock = Some(primary.address); + } + { + let mut shared = self.shared.write().unwrap(); + shared.system_primary = Some(primary); + } + } + } + + fn connection_resolved(&mut self, address: SocketAddr) { + if self.state == State::DeterminePrimary && self.initialization_peers.remove(&address) { + if self.initialization_peers.is_empty() { + warn!("exhausted all peers without determining a primary member"); + self.transition_state(State::Follower); + } + } + } } impl ConsensusProcessor for ClusterManager { + fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { + self.connection_manager.outgoing_connection_failed(connection_id, address); + self.connection_resolved(address); + } + + fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap) { + let connection = all_connections.get(&connection_id) + .expect("Expected connection to be in all_connections on peer_connection_established"); + + self.connection_manager.peer_connection_established(upgrade.peer_id, connection); - fn outgoing_connection_failed(&mut self, address: SocketAddr) { - unimplemented!() + if let State::DeterminePrimary = self.state { + self.determine_primary_from_peer_upgrade(upgrade); + self.connection_resolved(connection.remote_address); + } + } + + fn tick(&mut self, now: Instant, all_connections: &mut HashMap) { + match self.state { + State::EstablishConnections => { + self.connection_manager.establish_connections(now, all_connections); + self.transition_state(State::DeterminePrimary); + } + _ => { } + } } - fn peer_connection_established(&mut self, peer_id: FloInstanceId, connection_id: ConnectionId) { - unimplemented!() + + fn is_primary(&self) -> bool { + self.state == State::Primary } } @@ -60,34 +177,10 @@ impl SharedClusterState { } } -#[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub struct Peer { - pub id: FloInstanceId, - pub address: SocketAddr, -} - pub type SystemPrimaryAddressRef = Arc>>; pub type ClusterStateReader = Arc>; - -//pub fn init_consensus_processor(storage_dir: &Path, -// options: Option, -// engine_ref: EngineRef, -// mut system_primary: AtomicBoolWriter, -// primary_address: SystemPrimaryAddressRef) -> io::Result<(Box, ClusterStateReader)> { -// -// if let Some(options) = options { -// system_primary.set(false); -// debug!("Initializing in clustered mode, starting as secondary"); -// init_cluster_consensus_processor(storage_dir, options, engine_ref, system_primary, primary_address) -// } else { -// debug!("Initializing in non-cluster mode, system stream will always be considered primary"); -// system_primary.set(true); -// Ok(init_no_op_consensus_processor()) -// } -//} - pub fn init_cluster_consensus_processor(persistent_state: FilePersistedState, options: ClusterOptions, engine_ref: EngineRef, @@ -95,35 +188,118 @@ pub fn init_cluster_consensus_processor(persistent_state: FilePersistedState, system_primary: AtomicBoolWriter, primary_address: SystemPrimaryAddressRef) -> Box { - let ClusterOptions{ peer_addresses, event_loop_handles, .. } = options; + let ClusterOptions{ peer_addresses, event_loop_handles, election_timeout_millis, .. } = options; let outgoing_connection_creator = OutgoingConnectionCreatorImpl::new(event_loop_handles, engine_ref); - let state = ClusterManager { - primary_status_writer: system_primary, - last_applied: 0, - starting_peer_addresses: peer_addresses, - persistent: persistent_state, - system_partion_primary_address: primary_address, - shared: shared_state_ref, - outgoing_connection_creator: Box::new(outgoing_connection_creator) - }; + let state = ClusterManager::new(election_timeout_millis, + peer_addresses, + persistent_state, + shared_state_ref, + system_primary, + primary_address, + Box::new(outgoing_connection_creator)); Box::new(state) } -fn init_no_op_consensus_processor() -> (Box, ClusterStateReader) { - (Box::new(NoOpConsensusProcessor), Arc::new(RwLock::new(SharedClusterState::non_cluster()))) -} +#[cfg(test)] +mod test { + use super::*; + use std::path::{Path, PathBuf}; + use std::collections::HashSet; + use tempdir::TempDir; + use engine::connection_handler::{ConnectionControlSender, ConnectionControlReceiver, ConnectionControl}; + use engine::controller::peer_connection::{OutgoingConnectionCreator, MockOutgoingConnectionCreator}; + use test_utils::addr; -#[derive(Debug)] -pub struct NoOpConsensusProcessor; + fn t(start: Instant, seconds: u64) -> Instant { + start + Duration::from_secs(seconds) + } -impl ConsensusProcessor for NoOpConsensusProcessor { + #[test] + fn cluster_manager_moves_to_follower_state_once_peer_announce_is_received_with_known_primary() { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let mut mock_creator = MockOutgoingConnectionCreator::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:3000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:3000") + }; + let (peer_1_connection, _) = mock_creator.stub(peer_1.address); + let _ = mock_creator.stub(peer_2.address); + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), mock_creator.boxed()); + + assert_eq!(State::EstablishConnections, subject.state); + let mut connections = HashMap::new(); + subject.tick(t(start, 1), &mut connections); + assert_eq!(State::DeterminePrimary, subject.state); + + let upgrade = PeerUpgrade { + peer_id: peer_1.id, + system_primary: Some(peer_2.clone()), + cluster_members: vec![peer_2.clone()], + }; + subject.peer_connection_established(upgrade, peer_1_connection.connection_id, &connections); + + assert_eq!(State::Follower, subject.state); + let actual_primary: Option = { + subject.system_partition_primary_address.read().unwrap().as_ref().cloned() + }; - fn peer_connection_established(&mut self, peer_id: FloInstanceId, connection_id: ConnectionId) { - unimplemented!() } - fn outgoing_connection_failed(&mut self, address: SocketAddr) { - unimplemented!() + + #[test] + fn cluster_manager_moves_to_follower_state_once_all_unknown_peer_connections_have_failed() { + let start = Instant::now(); + + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let mut mock_creator = MockOutgoingConnectionCreator::new(); + let peer_1_addr = addr("111.222.0.1:3000"); + let peer_2_addr = addr("111.222.0.2:3000"); + let _ = mock_creator.stub(peer_1_addr); + let _ = mock_creator.stub(peer_2_addr); + let mut subject = create_cluster_manager(vec![peer_1_addr, peer_2_addr], temp_dir.path(), mock_creator.boxed()); + + assert_eq!(State::EstablishConnections, subject.state); + let mut connections = HashMap::new(); + subject.tick(t(start, 1), &mut connections); + assert_eq!(State::DeterminePrimary, subject.state); + + assert_eq!(2, connections.len()); + let actual = connections.values().map(|cr| cr.remote_address).collect::>(); + + assert!(actual.contains(&peer_1_addr)); + assert!(actual.contains(&peer_2_addr)); + + // make sure it doesn't happen again + subject.tick(t(start, 5), &mut connections); + assert_eq!(2, connections.len()); + + subject.outgoing_connection_failed(1, peer_1_addr); + subject.outgoing_connection_failed(2, peer_2_addr); + assert_eq!(State::Follower, subject.state); + } + + fn this_instance_addr() -> SocketAddr { + addr("123.1.2.3:4567") + } + + fn create_cluster_manager(starting_peers: Vec, temp_dir: &Path, conn_creator: Box) -> ClusterManager { + let temp_file = temp_dir.join("cluster_state"); + let file_state = FilePersistedState::initialize(temp_file).expect("failed to init persistent state"); + + let shared = file_state.initialize_shared_state(Some(this_instance_addr())); + + ClusterManager::new(150, + starting_peers, + file_state, + Arc::new(RwLock::new(shared)), + AtomicBoolWriter::with_value(false), + Arc::new(RwLock::new(None)), + conn_creator) } } diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs new file mode 100644 index 0000000..b6f894d --- /dev/null +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -0,0 +1,296 @@ +use std::collections::HashMap; +use std::net::SocketAddr; +use std::time::{Instant, Duration}; + +use protocol::FloInstanceId; +use engine::ConnectionId; +use engine::controller::{ConnectionRef, Peer}; +use engine::controller::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator}; +use engine::connection_handler::ConnectionControl; + +#[derive(Debug)] +pub struct PeerConnections { + disconnected_peers: HashMap, + known_peers: HashMap, + active_connections: HashMap, + outgoing_connection_creator: Box, +} + +impl PeerConnections { + pub fn new(starting_peer_addresses: Vec, outgoing_connection_creator: Box, peers: &[Peer]) -> PeerConnections { + let known_peers = peers.iter().map(|peer| { + let connection = Connection::new(peer.address); + (peer.id, connection) + }).collect::>(); + + let mut starting_peers = starting_peer_addresses.into_iter().map(|addr| { + (addr, ConnectionAttempt::new()) + }).collect::>(); + + for peer in peers.iter() { + let address = peer.address; + if !starting_peers.contains_key(&address) { + starting_peers.insert(address, ConnectionAttempt::new()); + } + } + + PeerConnections { + disconnected_peers: starting_peers, + active_connections: HashMap::new(), + known_peers, + outgoing_connection_creator, + } + } + + pub fn establish_connections(&mut self, now: Instant, all_connections: &mut HashMap) { + let PeerConnections {ref mut disconnected_peers, ref mut outgoing_connection_creator, ..} = *self; + for (address, attempt) in disconnected_peers.iter_mut() { + if attempt.should_try_connect(now) { + let connection_ref = outgoing_connection_creator.establish_system_connection(*address); + all_connections.insert(connection_ref.connection_id, connection_ref.clone()); + attempt.attempt_time = now; + attempt.attempt_count += 1; + attempt.connection = Some(connection_ref); + } + } + } + + pub fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { + let PeerConnections {ref mut disconnected_peers, ref mut known_peers, ref mut outgoing_connection_creator, ..} = *self; + if let Some(attempt) = disconnected_peers.get_mut(&address) { + // remove the connection + if attempt.connection.take().is_some() { + // set the attempt time to the time of the failure, since it could take quite some time to get a connection error + attempt.attempt_time = Instant::now(); + } else { + warn!("Outgoing connection_id: {} to addr: {} failed, but no connection attempt was in progress", connection_id, address); + } + } else { + warn!("Outgoing connection_id: {} to addr: {} failed, but no outgoing connection could be found", connection_id, address); + } + } + + pub fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef) { + let disconnected = self.disconnected_peers.remove(&success_connection.remote_address); + let success_connection_id = success_connection.connection_id; + + if let Some(ConnectionAttempt {connection, attempt_count, ..}) = disconnected { + if let Some(outgoing_connection_ref) = connection { + // if there's an existing attempt to create a connection to this peer, verify that this is indeed + // the same connection. if this success is from a different connection than the one we were trying to establish, + // then we'll close the in progress attempt by dropping the connectionRef + if outgoing_connection_ref.connection_id != success_connection_id { + info!("Established a new connection for peer: {:?} with connection_id: {}, so existing connection: {} will be closed", + peer_id, + success_connection_id, + outgoing_connection_ref.connection_id) + } + } + } + + let mut connection = self.known_peers.entry(peer_id).or_insert_with(|| { + Connection::new(success_connection.remote_address) + }); + connection.state = PeerState::Connected(success_connection.clone()); + + self.active_connections.insert(success_connection_id, peer_id); + } + + pub fn connection_closed(&mut self, connection_id: ConnectionId) { + let address: Option = self.active_connections.remove(&connection_id).and_then(|peer_id| { + self.known_peers.get_mut(&peer_id).map(|connection| { + connection.state = PeerState::ConnectionFailed; + connection.peer_address + }) + }); + + if let Some(peer_address) = address { + // todo: as it is, we only deal with one connection per peer at a time, so we know there won't already be an entry in disconnected_peers. will need to change that once we deal with multiple connections per peer + self.disconnected_peers.insert(peer_address, ConnectionAttempt::new()); + } else { + // just means that this id was not for a peer connection + debug!("Got connection closed for connection_id: {}, but there was no active peer connection", connection_id); + } + } + + pub fn connection_peer_id(&self, connection_id: ConnectionId) -> Option { + self.active_connections.get(&connection_id).cloned() + } + +} + +#[derive(Debug)] +struct ConnectionAttempt { + attempt_count: u32, + attempt_time: Instant, + connection: Option, +} + +impl ConnectionAttempt { + fn new() -> ConnectionAttempt { + ConnectionAttempt { + attempt_count: 0, + attempt_time: Instant::now(), + connection: None, + } + } + fn should_try_connect(&self, now: Instant) -> bool { + if self.connection.is_some() { + return false; + } + + let time_to_wait = Duration::from_secs(self.attempt_count.min(30) as u64); + now >= self.attempt_time && (now - self.attempt_time) >= time_to_wait + } + + fn failed(&mut self, time: Instant) { + self.attempt_time = time; + self.attempt_count += 1; + self.connection = None; + } +} + +#[derive(Debug)] +struct Connection { + peer_address: SocketAddr, + state: PeerState, +} + +impl Connection { + fn new(peer_address: SocketAddr) -> Connection { + Connection { + peer_address, + state: PeerState::Init + } + } +} + +#[derive(Debug)] +enum PeerState { + Init, + ConnectionFailed, + OutgoingConnectAttempt, + Connected(ConnectionRef), +} + + +#[cfg(test)] +mod test { + use super::*; + use std::time::Duration; + use engine::controller::peer_connection::MockOutgoingConnectionCreator; + use test_utils::addr; + + #[test] + fn outgoing_connect_success_adds_known_peer_and_connection_closed_sets_it_to_disconnected() { + let peer_address = addr("123.45.67.8:3000"); + let peer_id = FloInstanceId::generate_new(); + + let mut creator = MockOutgoingConnectionCreator::new(); + let (peer_connection, rx) = creator.stub(peer_address); + + let mut subject = PeerConnections::new(vec![peer_address], creator.boxed(), &[]); + + let mut connections = HashMap::new(); + let time = Instant::now(); + subject.establish_connections(time, &mut connections); + + subject.peer_connection_established(peer_id, &peer_connection); + + assert_eq!(Some(peer_id), subject.connection_peer_id(peer_connection.connection_id)); + + subject.connection_closed(peer_connection.connection_id); + + assert!(subject.disconnected_peers.contains_key(&peer_address)); + assert_eq!(None, subject.connection_peer_id(peer_connection.connection_id)); + } + + #[test] + fn known_peers_are_added_to_disconnected_peers_when_struct_is_initialized() { + let peer_address = addr("123.45.67.8:3000"); + let peer = Peer { + id: FloInstanceId::generate_new(), + address: peer_address, + }; + let mut creator = MockOutgoingConnectionCreator::new(); + creator.stub(peer_address); + let mut subject = PeerConnections::new(Vec::new(), creator.boxed(), &[peer.clone()]); + + assert!(subject.disconnected_peers.contains_key(&peer_address)); + } + + #[test] + fn outgoing_connect_failed_sets_status_of_starting_peer() { + let peer_address = addr("123.45.67.8:3000"); + let new_peers = vec![peer_address]; + let mut creator = MockOutgoingConnectionCreator::new(); + creator.stub(peer_address); + let mut subject = PeerConnections::new(new_peers, creator.boxed(), &[]); + + let mut connections = HashMap::new(); + let time = Instant::now(); + subject.establish_connections(time, &mut connections); + + assert_eq!(1, connections.len()); + { + let attempt = subject.disconnected_peers.get(&peer_address).unwrap(); + assert_eq!(time, attempt.attempt_time); + assert_eq!(1, attempt.attempt_count); + assert!(attempt.connection.is_some()); + } + + subject.outgoing_connection_failed(1, peer_address); + let attempt = subject.disconnected_peers.get(&peer_address).unwrap(); + assert!(attempt.attempt_time > time); + assert_eq!(1, attempt.attempt_count); + assert!(attempt.connection.is_none()); + } + + #[test] + fn connection_attempt_should_try_returns_true_when_last_attempt_was_long_enough_in_the_past() { + let attempt = ConnectionAttempt { + attempt_time: Instant::now() - Duration::from_secs(30), + attempt_count: 999, + connection: None, + }; + assert!(attempt.should_try_connect(Instant::now())); + } + + #[test] + fn connection_attempt_should_try_returns_false_when_last_attempt_was_to_recent() { + let start = Instant::now(); + let attempt = ConnectionAttempt { + attempt_time: start - Duration::from_millis(750), + attempt_count: 1, + connection: None, + }; + assert!(!attempt.should_try_connect(start)); + } + + #[test] + fn connection_attempt_should_try_returns_true_when_attempts_is_0() { + let instant = Instant::now(); + let attempt = ConnectionAttempt { + attempt_time: instant, + attempt_count: 0, + connection: None, + }; + assert!(attempt.should_try_connect(instant)); + } + + #[test] + fn connection_attempt_should_try_returns_false_when_an_attempt_is_in_progress() { + let (tx, _rx) = ::engine::connection_handler::create_connection_control_channels(); + let conn = ConnectionRef { + connection_id: 0, + remote_address: addr("127.0.0.1:3456"), + control_sender: tx, + }; + let attempt = ConnectionAttempt { + attempt_count: 1, + attempt_time: Instant::now() - Duration::from_millis(5000), + connection: Some(conn), + }; + assert!(!attempt.should_try_connect(Instant::now())); + } +} diff --git a/flo-server/src/engine/controller/cluster_state/persistent.rs b/flo-server/src/engine/controller/cluster_state/persistent.rs index 33139e7..601412f 100644 --- a/flo-server/src/engine/controller/cluster_state/persistent.rs +++ b/flo-server/src/engine/controller/cluster_state/persistent.rs @@ -4,17 +4,18 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Seek}; use protocol::{FloInstanceId, Term}; -use super::{Peer, ClusterOptions, SharedClusterState}; +use engine::controller::controller_messages::Peer; +use super::{ClusterOptions, SharedClusterState}; /// Holds all the cluster state that we want to survive a reboot. /// We always persist the `FloInstanceId` because we prefer that to be stable across reboots. We do _not_ want to persist /// the `SocketAddr` for the server, though, since that may well change after a restart, depending on environment. #[derive(Debug, PartialEq, Clone)] pub struct PersistentClusterState { - current_term: Term, - voted_for: Option, - this_instance_id: FloInstanceId, - cluster_members: Vec, + pub current_term: Term, + pub voted_for: Option, + pub this_instance_id: FloInstanceId, + pub cluster_members: Vec, } impl PersistentClusterState { diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 934a39d..0c1d491 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -15,12 +15,26 @@ pub struct ConnectionRef { pub control_sender: ConnectionControlSender, } +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub struct Peer { + pub id: FloInstanceId, + pub address: SocketAddr, +} + +#[derive(Debug, Clone)] +pub struct PeerUpgrade { + pub peer_id: FloInstanceId, + pub system_primary: Option, + pub cluster_members: Vec, +} + + #[derive(Debug)] pub enum SystemOpType { Tick, PartitionOp(partition::OpType), IncomingConnectionEstablished(ConnectionRef), - ConnectionUpgradeToPeer(FloInstanceId), + ConnectionUpgradeToPeer(PeerUpgrade), ConnectionClosed, OutgoingConnectionFailed(SocketAddr), } @@ -34,8 +48,13 @@ pub struct SystemOperation { impl SystemOperation { - pub fn connection_upgraded_to_peer(connection_id: ConnectionId, peer_id: FloInstanceId) -> SystemOperation { - SystemOperation::new(connection_id, SystemOpType::ConnectionUpgradeToPeer(peer_id)) + pub fn connection_upgraded_to_peer(connection_id: ConnectionId, peer_id: FloInstanceId, system_primary: Option, cluster_members: Vec) -> SystemOperation { + let upgrade = PeerUpgrade { + peer_id, + system_primary, + cluster_members + }; + SystemOperation::new(connection_id, SystemOpType::ConnectionUpgradeToPeer(upgrade)) } pub fn incoming_connection_established(connection: ConnectionRef) -> SystemOperation { @@ -46,8 +65,8 @@ impl SystemOperation { SystemOperation::new(connection_id, SystemOpType::ConnectionClosed) } - pub fn outgoing_connection_failed(addr: SocketAddr) -> SystemOperation { - SystemOperation::new(0, SystemOpType::OutgoingConnectionFailed(addr)) + pub fn outgoing_connection_failed(connection_id: ConnectionId, addr: SocketAddr) -> SystemOperation { + SystemOperation::new(connection_id, SystemOpType::OutgoingConnectionFailed(addr)) } pub fn tick() -> SystemOperation { diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index 8efc669..be9596a 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -44,6 +44,8 @@ pub struct ControllerOptions { #[derive(Debug)] pub struct ClusterOptions { + pub election_timeout_millis: u64, + pub heartbeat_interval_millis: u64, pub this_instance_address: SocketAddr, pub peer_addresses: Vec, pub event_loop_handles: LoopHandles, @@ -96,6 +98,9 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul let persistent_state = file_cluster_state.unwrap(); let cluster_opts = cluster_options.unwrap(); + let system_stream_ref = engine_ref.get_system_stream(); + ::engine::controller::tick_generator::spawn_tick_generator(cluster_opts.heartbeat_interval_millis, remote, system_stream_ref); + init_cluster_consensus_processor(persistent_state, cluster_opts, engine_ref.clone(), @@ -114,8 +119,6 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul default_stream_options); run_controller_impl(flo_controller, system_partition_rx); - let system_stream_ref = engine_ref.get_system_stream(); - ::engine::controller::tick_generator::spawn_tick_generator(remote, system_stream_ref); Ok(engine_ref) } diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index c4e8f97..fe9b67b 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -1,9 +1,9 @@ pub mod cluster_state; +pub mod tick_generator; mod system_stream; mod initialization; mod controller_messages; mod peer_connection; -mod tick_generator; use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; @@ -24,8 +24,8 @@ use self::cluster_state::{ClusterManager, ConsensusProcessor}; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; pub use self::system_stream::SystemStreamRef; -pub use self::controller_messages::{SystemOperation, SystemOpType, ConnectionRef}; -pub use self::cluster_state::{SharedClusterState, Peer, ClusterStateReader}; +pub use self::controller_messages::{SystemOperation, SystemOpType, ConnectionRef, Peer, PeerUpgrade}; +pub use self::cluster_state::{SharedClusterState, ClusterStateReader}; pub type SystemPartitionSender = ::std::sync::mpsc::Sender; pub type SystemPartitionReceiver = ::std::sync::mpsc::Receiver; @@ -88,14 +88,16 @@ impl FloController { self.all_connections.insert(connection_id, connection_ref); } SystemOpType::OutgoingConnectionFailed(address) => { - self.cluster_state.outgoing_connection_failed(address); + self.all_connections.remove(&connection_id); + self.cluster_state.outgoing_connection_failed(connection_id, address); } - SystemOpType::ConnectionUpgradeToPeer(peer_id) => { - if let Some(connection_ref) = self.all_connections.get(&connection_id).cloned() { - unimplemented!() - } else { - error!("Got ConnectionUpgradeToPeer for connection_id: {}, but no connection with that id exists", connection_id); - } + SystemOpType::ConnectionUpgradeToPeer(upgrade) => { + let FloController{ref mut cluster_state, ref mut all_connections, ..} = *self; + cluster_state.peer_connection_established(upgrade, connection_id, all_connections); + } + SystemOpType::Tick => { + let FloController{ref mut cluster_state, ref mut all_connections, ..} = *self; + cluster_state.tick(op_start_time, all_connections); } other @ _ => { warn!("Ignoring SystemOperation: {:?}", other); diff --git a/flo-server/src/engine/controller/peer_connection/mod.rs b/flo-server/src/engine/controller/peer_connection/mod.rs index b890354..fa25c5a 100644 --- a/flo-server/src/engine/controller/peer_connection/mod.rs +++ b/flo-server/src/engine/controller/peer_connection/mod.rs @@ -68,7 +68,7 @@ fn create_outgoing_connection(loops: &mut LoopHandles, client_addr: SocketAddr, TcpStream::connect(&addr, handle).map_err( move |io_err| { error!("Failed to create outgoing connection to address: {:?}: {:?}", addr, io_err); - system_stream.outgoing_connection_failed(addr); + system_stream.outgoing_connection_failed(connection_id, addr); }).and_then( move |tcp_stream| { @@ -84,5 +84,57 @@ fn create_outgoing_connection(loops: &mut LoopHandles, client_addr: SocketAddr, (control_tx, connection_id) } +#[cfg(test)] +pub use self::test::MockOutgoingConnectionCreator; +#[cfg(test)] +mod test { + use super::*; + use engine::connection_handler::ConnectionControlReceiver; + + #[derive(Debug)] + pub struct MockOutgoingConnectionCreator { + current_connection_id: ConnectionId, + to_return: Vec<(SocketAddr, ConnectionRef)>, + calls: Vec, + } + + impl MockOutgoingConnectionCreator { + pub fn new() -> MockOutgoingConnectionCreator { + MockOutgoingConnectionCreator { + current_connection_id: 0, + to_return: Vec::new(), + calls: Vec::new() + } + } + + pub fn stub(&mut self, expected_address: SocketAddr) -> (ConnectionRef, ConnectionControlReceiver) { + let (tx, rx) = ::engine::connection_handler::create_connection_control_channels(); + + self.current_connection_id += 1; + let connection_ref = ConnectionRef { + connection_id: self.current_connection_id, + remote_address: expected_address, + control_sender: tx, + }; + self.to_return.push((expected_address, connection_ref.clone())); + (connection_ref, rx) + } + + pub fn boxed(self) -> Box { + Box::new(self) + } + } + + impl OutgoingConnectionCreator for MockOutgoingConnectionCreator { + fn establish_system_connection(&mut self, address: SocketAddr) -> ConnectionRef { + let stub_idx = self.to_return.iter().position(|stub| stub.0 == address); + if stub_idx.is_none() { + panic!("Missing stub for address: {:?}", address); + } + self.calls.push(address); + self.to_return.remove(stub_idx.unwrap()).1 + } + } +} diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index ead496d..aeef14e 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -3,7 +3,7 @@ use std::net::SocketAddr; use protocol::FloInstanceId; use engine::event_stream::partition::{PartitionRef, Operation}; use engine::event_stream::EventStreamRef; -use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef}; +use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer}; use engine::controller::cluster_state::{SharedClusterState, ClusterStateReader}; use engine::ConnectionId; use atomics::AtomicBoolReader; @@ -57,13 +57,13 @@ impl SystemStreamRef { self.send(op); } - pub fn outgoing_connection_failed(&mut self, socket_addr: SocketAddr) { - let op = SystemOperation::outgoing_connection_failed(socket_addr); + pub fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, socket_addr: SocketAddr) { + let op = SystemOperation::outgoing_connection_failed(connection_id, socket_addr); self.send(op); } - pub fn connection_upgraded_to_peer(&mut self, connection_id: ConnectionId, peer_id: FloInstanceId) { - let op = SystemOperation::connection_upgraded_to_peer(connection_id, peer_id); + pub fn connection_upgraded_to_peer(&mut self, connection_id: ConnectionId, peer_id: FloInstanceId, system_primary: Option, cluster_members: Vec) { + let op = SystemOperation::connection_upgraded_to_peer(connection_id, peer_id, system_primary, cluster_members); self.send(op); } diff --git a/flo-server/src/engine/controller/tick_generator.rs b/flo-server/src/engine/controller/tick_generator.rs index 9946a8b..f0bf020 100644 --- a/flo-server/src/engine/controller/tick_generator.rs +++ b/flo-server/src/engine/controller/tick_generator.rs @@ -21,16 +21,15 @@ impl From for TickError { } -pub fn spawn_tick_generator(remote: Remote, mut system_sender: SystemStreamRef) { +pub fn spawn_tick_generator(tick_interval_millis: u64, remote: Remote, mut system_sender: SystemStreamRef) { // clone the inputs so we can re-run in the case of a failure let remote_copy = remote.clone(); let mut system_stream_ref_clone = system_sender.clone(); - let tick_interval = get_controller_tick_interval_millis(); - info!("Using tick interval of {} milliseconds", tick_interval); + info!("Using tick interval of {} milliseconds", tick_interval_millis); remote.spawn(move |handle| { // TODO: handle the failure to create the Interval - let interval = Interval::new(Duration::from_millis(tick_interval), handle) + let interval = Interval::new(Duration::from_millis(tick_interval_millis), handle) .expect("failed to create tick interval"); interval.map_err(|io_err| TickError::Io(io_err)) .for_each(move |_| { @@ -46,7 +45,7 @@ pub fn spawn_tick_generator(remote: Remote, mut system_sender: SystemStreamRef) Err(TickError::Io(io_err)) => { error!("Error generating tick operations for system stream: {}", io_err); system_stream_ref_clone.tick_error(); - spawn_tick_generator(remote_copy, system_stream_ref_clone); + spawn_tick_generator(tick_interval_millis, remote_copy, system_stream_ref_clone); } } // always return Ok so that this future will be considered resolved. We'll just spawn a new one anyway @@ -55,7 +54,7 @@ pub fn spawn_tick_generator(remote: Remote, mut system_sender: SystemStreamRef) }); } -fn get_controller_tick_interval_millis() -> u64 { +pub fn get_election_timeout_millis() -> u64 { let range = Range::new(150u64, 300u64); range.ind_sample(&mut thread_rng()) diff --git a/flo-server/src/lib.rs b/flo-server/src/lib.rs index 0078958..802acc2 100644 --- a/flo-server/src/lib.rs +++ b/flo-server/src/lib.rs @@ -37,3 +37,6 @@ pub mod engine; pub mod event_loops; pub mod channels; pub mod atomics; + +#[cfg(test)] +pub mod test_utils; diff --git a/flo-server/src/main.rs b/flo-server/src/main.rs index 04e3d45..80d905e 100644 --- a/flo-server/src/main.rs +++ b/flo-server/src/main.rs @@ -88,6 +88,14 @@ fn app_args() -> App<'static, 'static> { .short("A") .takes_value(true) .help("The socket address that this server is reachable at. Will be removed once cluster support doesn't suck so bad")) + .arg(Arg::with_name("election-timeout") + .long("election-timeout") + .takes_value(true) + .help("Trigger an election after this number of milliseconds. Defaults to a random number between 150-300")) + .arg(Arg::with_name("heartbeat-interval") + .long("heartbeat-interval") + .takes_value(true) + .help("Number of milliseconds to go in between sending heartbeats. Defaults to 1/3 of the election timeout")) .arg(Arg::with_name("max-io-threads") .long("max-io-threads") .takes_value(true) @@ -132,6 +140,10 @@ fn main() { let default_eviction_period = ::std::cmp::min(retention_duration.num_hours() / 6, MAX_SEGMENT_PERIOD_HOURS); let eviction_period_hours = parse_arg_or_exit(&args, "eviction-period", default_eviction_period); + let default_election_timeout = ::engine::controller::tick_generator::get_election_timeout_millis(); + let election_timeout = parse_arg_or_exit(&args, "election-timeout", default_election_timeout); + let heartbeat_interval = parse_arg_or_exit(&args, "heartbeat-interval", election_timeout / 3); + let server_options = ServerOptions { event_retention_duration: retention_duration, event_eviction_period: Duration::hours(eviction_period_hours), @@ -140,6 +152,8 @@ fn main() { max_cache_memory: max_cache_memory, this_instance_address: this_address, cluster_addresses: cluster_addresses, + election_timeout_millis: election_timeout, + heartbeat_interval_millis: heartbeat_interval, max_io_threads: max_io_threads, }; diff --git a/flo-server/src/server/mod.rs b/flo-server/src/server/mod.rs index a70e6f6..b9f5ab5 100644 --- a/flo-server/src/server/mod.rs +++ b/flo-server/src/server/mod.rs @@ -32,6 +32,8 @@ pub fn run(mut options: ServerOptions) -> io::Result<()> { let cluster_options = this_address.map(|server_addr| { let peers = peer_addresses.unwrap(); ClusterOptions { + election_timeout_millis: options.election_timeout_millis, + heartbeat_interval_millis: options.heartbeat_interval_millis, this_instance_address: server_addr, peer_addresses: peers, event_loop_handles: event_loop_handles.clone(), diff --git a/flo-server/src/server/server_options.rs b/flo-server/src/server/server_options.rs index 74e2946..bb68dee 100644 --- a/flo-server/src/server/server_options.rs +++ b/flo-server/src/server/server_options.rs @@ -45,6 +45,8 @@ pub struct ServerOptions { pub max_cache_memory: MemoryLimit, pub this_instance_address: Option, pub cluster_addresses: Option>, + pub election_timeout_millis: u64, + pub heartbeat_interval_millis: u64, pub max_io_threads: Option, } diff --git a/flo-server/src/test_utils.rs b/flo-server/src/test_utils.rs new file mode 100644 index 0000000..47fffba --- /dev/null +++ b/flo-server/src/test_utils.rs @@ -0,0 +1,6 @@ + +use std::net::SocketAddr; + +pub fn addr(string: &str) -> SocketAddr { + ::std::str::FromStr::from_str(string).unwrap() +} From 2f2541350e87faa87b5cbfd843a6b4af065bac60 Mon Sep 17 00:00:00 2001 From: pfried Date: Thu, 28 Dec 2017 22:31:30 -0500 Subject: [PATCH 28/73] peer connections are completing the handshake --- .../src/engine/connection_handler/input.rs | 2 +- .../src/engine/connection_handler/peer/mod.rs | 2 ++ .../engine/controller/cluster_state/mod.rs | 11 ++++++++--- .../cluster_state/peer_connections.rs | 19 +++++++++++++++---- .../engine/controller/controller_messages.rs | 9 +++++++++ flo-server/src/engine/controller/mod.rs | 5 ++++- 6 files changed, 39 insertions(+), 9 deletions(-) diff --git a/flo-server/src/engine/connection_handler/input.rs b/flo-server/src/engine/connection_handler/input.rs index 18819b2..967e75d 100644 --- a/flo-server/src/engine/connection_handler/input.rs +++ b/flo-server/src/engine/connection_handler/input.rs @@ -20,7 +20,7 @@ impl ConnectionHandlerInput { } } -#[derive(Debug)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ConnectionControl { InitiateOutgoingSystemConnection, } diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 852b1c2..3940b50 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -33,6 +33,8 @@ impl PeerConnectionState { let connection_id = state.connection_id; let old_state = self.set_state(connection_id, PeerConnectionState::Peer); + debug!("PeerAnnounce received on connection_id: {}, {:?}, prev_connection_state: {:?}", connection_id, announce, old_state); + match old_state { PeerConnectionState::Peer => { // we've already gone through this, so something's wrong diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index d300575..a777f20 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -100,7 +100,7 @@ impl ClusterManager { } fn determine_primary_from_peer_upgrade(&mut self, upgrade: PeerUpgrade) { - let PeerUpgrade { peer_id, system_primary, cluster_members } = upgrade; + let PeerUpgrade { peer_id, system_primary, .. } = upgrade; if let Some(primary) = system_primary { info!("Determined primary of {:?} at {}, as told by instance: {:?}", primary.id, primary.address, peer_id); self.transition_state(State::Follower); @@ -112,6 +112,8 @@ impl ClusterManager { let mut shared = self.shared.write().unwrap(); shared.system_primary = Some(primary); } + } else { + debug!("peer announce from {:?} has unknown primary", peer_id); } } @@ -132,6 +134,7 @@ impl ConsensusProcessor for ClusterManager { } fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap) { + debug!("peer_connection_established: connection_id: {}, {:?}", connection_id, upgrade); let connection = all_connections.get(&connection_id) .expect("Expected connection to be in all_connections on peer_connection_established"); @@ -139,14 +142,16 @@ impl ConsensusProcessor for ClusterManager { if let State::DeterminePrimary = self.state { self.determine_primary_from_peer_upgrade(upgrade); - self.connection_resolved(connection.remote_address); } + self.connection_resolved(connection.remote_address); } fn tick(&mut self, now: Instant, all_connections: &mut HashMap) { + self.connection_manager.establish_connections(now, all_connections); + match self.state { State::EstablishConnections => { - self.connection_manager.establish_connections(now, all_connections); + // we'll only be in this initial status once, on startup self.transition_state(State::DeterminePrimary); } _ => { } diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs index b6f894d..5da83b9 100644 --- a/flo-server/src/engine/controller/cluster_state/peer_connections.rs +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -46,7 +46,9 @@ impl PeerConnections { let PeerConnections {ref mut disconnected_peers, ref mut outgoing_connection_creator, ..} = *self; for (address, attempt) in disconnected_peers.iter_mut() { if attempt.should_try_connect(now) { + debug!("Making outgoing connection attempt #{} to address: {}", attempt.attempt_count + 1, address); let connection_ref = outgoing_connection_creator.establish_system_connection(*address); + send(&connection_ref, ConnectionControl::InitiateOutgoingSystemConnection); all_connections.insert(connection_ref.connection_id, connection_ref.clone()); attempt.attempt_time = now; attempt.attempt_count += 1; @@ -74,6 +76,9 @@ impl PeerConnections { let disconnected = self.disconnected_peers.remove(&success_connection.remote_address); let success_connection_id = success_connection.connection_id; + info!("Successfully established connection_id: {} to peer_id: {} at address: {}", + success_connection.connection_id, peer_id, success_connection.remote_address); + if let Some(ConnectionAttempt {connection, attempt_count, ..}) = disconnected { if let Some(outgoing_connection_ref) = connection { // if there's an existing attempt to create a connection to this peer, verify that this is indeed @@ -81,14 +86,12 @@ impl PeerConnections { // then we'll close the in progress attempt by dropping the connectionRef if outgoing_connection_ref.connection_id != success_connection_id { info!("Established a new connection for peer: {:?} with connection_id: {}, so existing connection: {} will be closed", - peer_id, - success_connection_id, - outgoing_connection_ref.connection_id) + peer_id, success_connection_id, outgoing_connection_ref.connection_id) } } } - let mut connection = self.known_peers.entry(peer_id).or_insert_with(|| { + let connection = self.known_peers.entry(peer_id).or_insert_with(|| { Connection::new(success_connection.remote_address) }); connection.state = PeerState::Connected(success_connection.clone()); @@ -107,6 +110,7 @@ impl PeerConnections { if let Some(peer_address) = address { // todo: as it is, we only deal with one connection per peer at a time, so we know there won't already be an entry in disconnected_peers. will need to change that once we deal with multiple connections per peer self.disconnected_peers.insert(peer_address, ConnectionAttempt::new()); + info!("ConnectionClosed for connection_id: {} from peer_address: {}", connection_id, peer_address); } else { // just means that this id was not for a peer connection debug!("Got connection closed for connection_id: {}, but there was no active peer connection", connection_id); @@ -119,6 +123,13 @@ impl PeerConnections { } +fn send(connection: &ConnectionRef, control: ConnectionControl) { + let result = connection.control_sender.send(control); + if result.is_err() { + warn!("Error sending control to connection_id: {}, {:?}", connection.connection_id, control); + } +} + #[derive(Debug)] struct ConnectionAttempt { attempt_count: u32, diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 0c1d491..7917338 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -39,6 +39,15 @@ pub enum SystemOpType { OutgoingConnectionFailed(SocketAddr), } +impl SystemOpType { + pub fn is_tick(&self) -> bool { + match self { + &SystemOpType::Tick => true, + _ => false + } + } +} + #[derive(Debug)] pub struct SystemOperation { pub connection_id: ConnectionId, diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index fe9b67b..a9fe4b7 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -80,7 +80,10 @@ impl FloController { fn process(&mut self, operation: SystemOperation) { let SystemOperation {connection_id, op_start_time, op_type } = operation; - trace!("Received system op: {:?}", op_type); + + if !op_type.is_tick() { + trace!("Received system op: {:?}", op_type); + } // TODO: time operation handling and record perf metrics match op_type { From 405cb85ce47ec9cc44c475c93544e289ea75b32e Mon Sep 17 00:00:00 2001 From: pfried Date: Thu, 28 Dec 2017 23:20:35 -0500 Subject: [PATCH 29/73] change connectionHandler to shutdown on Drop instead or Sink::close --- flo-server/src/engine/connection_handler/consumer/mod.rs | 1 + flo-server/src/engine/connection_handler/mod.rs | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/flo-server/src/engine/connection_handler/consumer/mod.rs b/flo-server/src/engine/connection_handler/consumer/mod.rs index c7ede0f..635fdfc 100644 --- a/flo-server/src/engine/connection_handler/consumer/mod.rs +++ b/flo-server/src/engine/connection_handler/consumer/mod.rs @@ -41,6 +41,7 @@ impl ConsumerConnectionState { } pub fn shutdown(&mut self, connection: &mut ConnectionState) { + debug!("Shutting down connection_id: {}", connection.connection_id); if let Some(ref mut consumer) = self.consumer_ref.take() { // tell the active consumer to stop sending events consumer.status_setter.set(ConsumerStatus::Stop); diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 38c8eb6..b556056 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -141,12 +141,14 @@ impl Sink for ConnectionHandler { } fn close(&mut self) -> Poll<(), Self::SinkError> { - let _ = try_ready!(self.poll_complete()); + Ok(Async::Ready(())) + } +} +impl Drop for ConnectionHandler { + fn drop(&mut self) { let ConnectionHandler {ref mut common_state, ref mut consumer_state, ..} = *self; consumer_state.shutdown(common_state); - - Ok(Async::Ready(())) } } From de6e49d92bd034a72919908f344b949b78f5c1ce Mon Sep 17 00:00:00 2001 From: pfried Date: Fri, 29 Dec 2017 18:39:58 -0500 Subject: [PATCH 30/73] handle requestVote in connectionHandler --- flo-protocol/src/messages/error.rs | 5 + .../src/engine/connection_handler/input.rs | 7 +- .../src/engine/connection_handler/mod.rs | 152 ++++++++++++++++-- .../src/engine/connection_handler/peer/mod.rs | 121 ++++++++++---- .../engine/controller/cluster_state/mod.rs | 67 +++++++- .../cluster_state/peer_connections.rs | 6 +- .../controller/cluster_state/persistent.rs | 138 ++++++++++++++-- .../engine/controller/controller_messages.rs | 37 ++++- flo-server/src/engine/controller/mod.rs | 2 +- .../src/engine/controller/system_stream.rs | 7 +- .../src/engine/event_stream/partition/ops.rs | 31 ++-- 11 files changed, 493 insertions(+), 80 deletions(-) diff --git a/flo-protocol/src/messages/error.rs b/flo-protocol/src/messages/error.rs index 521e5f6..ded2a80 100644 --- a/flo-protocol/src/messages/error.rs +++ b/flo-protocol/src/messages/error.rs @@ -9,6 +9,7 @@ pub const ERROR_HEADER: u8 = 10; pub const ERROR_INVALID_NAMESPACE: u8 = 15; pub const ERROR_INVALID_CONSUMER_STATE: u8 = 16; +pub const ERROR_INVALID_PEER_STATE: u8 = 30; pub const ERROR_INVALID_VERSION_VECTOR: u8 = 17; pub const ERROR_STORAGE_ENGINE_IO: u8 = 18; pub const ERROR_NO_STREAM: u8 = 19; @@ -20,6 +21,8 @@ pub enum ErrorKind { InvalidNamespaceGlob, /// Indicates that the client connection was in an invalid state when it attempted some consumer operation InvalidConsumerState, + /// Indicates that the connection was in an invalid state when it attempted to process some peer operation + InvalidPeerState, /// Indicates that the provided version vector was invalid (contained more than one entry for at least one actor id) InvalidVersionVector, /// Unable to read or write to events file @@ -47,6 +50,7 @@ impl ErrorKind { match byte { ERROR_INVALID_NAMESPACE => Ok(ErrorKind::InvalidNamespaceGlob), ERROR_INVALID_CONSUMER_STATE => Ok(ErrorKind::InvalidConsumerState), + ERROR_INVALID_PEER_STATE => Ok(ErrorKind::InvalidPeerState), ERROR_INVALID_VERSION_VECTOR => Ok(ErrorKind::InvalidVersionVector), ERROR_STORAGE_ENGINE_IO => Ok(ErrorKind::StorageEngineError), ERROR_NO_STREAM => Ok(ErrorKind::NoSuchStream), @@ -59,6 +63,7 @@ impl ErrorKind { match self { &ErrorKind::InvalidNamespaceGlob => ERROR_INVALID_NAMESPACE, &ErrorKind::InvalidConsumerState => ERROR_INVALID_CONSUMER_STATE, + &ErrorKind::InvalidPeerState => ERROR_INVALID_PEER_STATE, &ErrorKind::InvalidVersionVector => ERROR_INVALID_VERSION_VECTOR, &ErrorKind::StorageEngineError => ERROR_STORAGE_ENGINE_IO, &ErrorKind::NoSuchStream => ERROR_NO_STREAM, diff --git a/flo-server/src/engine/connection_handler/input.rs b/flo-server/src/engine/connection_handler/input.rs index 967e75d..c2318a4 100644 --- a/flo-server/src/engine/connection_handler/input.rs +++ b/flo-server/src/engine/connection_handler/input.rs @@ -2,6 +2,7 @@ use protocol::{FloInstanceId, Term}; use event::EventCounter; use engine::ReceivedProtocolMessage; +use engine::controller::{CallRequestVote, VoteResponse}; #[derive(Debug)] pub enum ConnectionHandlerInput { @@ -20,9 +21,13 @@ impl ConnectionHandlerInput { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + + +#[derive(Debug, Clone, PartialEq)] pub enum ConnectionControl { InitiateOutgoingSystemConnection, + SendRequestVote(CallRequestVote), + SendVoteResponse(VoteResponse), } diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index b556056..9315544 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -43,17 +43,10 @@ impl ConnectionHandler { common_state: ConnectionState::new(connection, client_sender, engine, handle), consumer_state: ConsumerConnectionState::new(), producer_state: ProducerConnectionState::new(), - peer_state: PeerConnectionState::Init, + peer_state: PeerConnectionState::new(), } } - pub fn upgrade_to_outgoing_peer(&mut self) { - debug!("upgrading connection_id: {} to outgoing peer connection", self.common_state.connection_id); - let ConnectionHandler{ref mut common_state, ref mut peer_state, .. } = *self; - - peer_state.initiate_outgoing_peer_connection(common_state); - } - pub fn can_process(&self, _message: &ReceivedProtocolMessage) -> bool { !self.producer_state.requires_poll_complete() && !self.consumer_state.requires_poll_complete() } @@ -61,12 +54,21 @@ impl ConnectionHandler { pub fn handle_control(&mut self, control: ConnectionControl) -> ConnectionHandlerResult { debug!("client: {:?} processing control: {:?}", self.common_state, control); + let ConnectionHandler{ref mut common_state, ref mut peer_state, .. } = *self; + match control { ConnectionControl::InitiateOutgoingSystemConnection => { - self.upgrade_to_outgoing_peer() + peer_state.initiate_outgoing_peer_connection(common_state); + Ok(()) } + ConnectionControl::SendRequestVote(request) => { + peer_state.send_request_vote(request, common_state) + } + ConnectionControl::SendVoteResponse(response) => { + peer_state.send_vote_response(response, common_state) + } + _ => unimplemented!() } - Ok(()) } pub fn handle_incoming_message(&mut self, message: ReceivedProtocolMessage) -> ConnectionHandlerResult { @@ -96,6 +98,9 @@ impl ConnectionHandler { ProtocolMessage::StopConsuming(op_id) => { consumer_state.stop_consuming(op_id, common_state) } + ProtocolMessage::RequestVote(request_vote) => { + peer_state.request_vote_received(request_vote, common_state) + } _ => unimplemented!() } } @@ -169,6 +174,7 @@ impl Debug for ConnectionHandler { mod test { use std::collections::HashMap; use std::sync::{Arc, Mutex, RwLock}; + use std::net::SocketAddr; use tokio_core::reactor::Core; use super::*; @@ -177,8 +183,9 @@ mod test { use engine::event_stream::EventStreamRef; use engine::event_stream::partition::*; use engine::ClientReceiver; - use engine::controller::{SystemStreamRef, SharedClusterState}; + use engine::controller::{SystemStreamRef, SharedClusterState, SystemOpType, SystemOperation, CallRequestVote, VoteResponse}; use atomics::{AtomicCounterWriter, AtomicBoolWriter}; + use test_utils::addr; struct Fixture { #[allow(dead_code)] // TODO: add more connection handler tests @@ -187,9 +194,37 @@ mod test { client_receiver: Option, engine: EngineRef, reactor: Core, + instance_id: FloInstanceId, + instance_addr: SocketAddr, } impl Fixture { + + fn create_outgoing_peer_connection() -> (ConnectionHandler, Fixture) { + let (mut subject, mut fixture) = Fixture::create(); + subject.handle_control(ConnectionControl::InitiateOutgoingSystemConnection).unwrap(); + let announce = PeerAnnounce { + protocol_version: 1, + peer_address: fixture.instance_addr, + op_id: 1, + instance_id: fixture.instance_id, + system_primary_id: None, + cluster_members: Vec::new(), + }; + fixture.assert_sent_to_client(ProtocolMessage::PeerAnnounce(announce.clone())); + + let peer_id = FloInstanceId::generate_new(); + let peer_addr = addr("127.0.0.1:4000"); + let response = PeerAnnounce { + peer_address: peer_addr, + instance_id: peer_id, + .. announce + }; + // get the response + subject.handle_incoming_message(ProtocolMessage::PeerAnnounce(response)).unwrap(); + (subject, fixture) + } + fn create() -> (ConnectionHandler, Fixture) { let reactor = Core::new().unwrap(); @@ -205,10 +240,12 @@ mod test { primary.reader(), tx.clone(), primary_addr); + let instance_id = FloInstanceId::generate_new(); + let instance_addr = addr("127.0.0.1:3000"); let cluster_state = SharedClusterState { - this_instance_id: FloInstanceId::generate_new(), - this_address: None, + this_instance_id: instance_id, + this_address: Some(instance_addr), system_primary: None, peers: Vec::new(), }; @@ -223,8 +260,10 @@ mod test { partition_receivers: HashMap::new(), system_receiver: rx, client_receiver: Some(client_rx), - engine: engine, - reactor: reactor + engine, + reactor, + instance_id, + instance_addr }; (subject, fixture) } @@ -269,6 +308,20 @@ mod test { result.expect(&format!("partition: {} failed to receive message", partition_id)) } + fn assert_sent_to_system_stream(&mut self, expected: SystemOpType) { + let mut unequal_messages = Vec::new(); + + while let Ok(next) = self.system_receiver.try_recv() { + let received = next.op_type; + if received == expected { + return; + } else { + unequal_messages.push(received); + } + } + panic!("Expected system op: {:?}, but received: {:?}", expected, unequal_messages); + } + fn assert_sent_to_client(&mut self, expected: ProtocolMessage) { use tokio_core::reactor::Timeout; use futures::future::Either; @@ -292,6 +345,75 @@ mod test { } + #[test] + fn receiving_request_vote_results_in_error_when_connection_is_not_in_peer_state() { + let (mut subject, mut fixture) = Fixture::create(); + + let incoming = RequestVoteCall { + op_id: 99, + term: 5, + candidate_id: FloInstanceId::generate_new(), + last_log_index: 44, + last_log_term: 4, + }; + let error = subject.handle_incoming_message(ProtocolMessage::RequestVote(incoming)).unwrap_err(); + assert_eq!("Refusing to process RequestVote when connection is in Init state", &error); + + let expected = ErrorMessage { + op_id: 99, + kind: ErrorKind::InvalidPeerState, + description: error, + }; + fixture.assert_sent_to_client(ProtocolMessage::Error(expected)); + } + + #[test] + fn receiving_request_vote_forwards_request_to_system_stream_for_peer_connection_and_response_is_sent_back() { + let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); + let peer_id = FloInstanceId::generate_new(); + let incoming = RequestVoteCall { + op_id: 99, + term: 5, + candidate_id: peer_id, + last_log_index: 44, + last_log_term: 4, + }; + subject.handle_incoming_message(ProtocolMessage::RequestVote(incoming)).unwrap(); + + let response = VoteResponse { + term: 7, + granted: false, + }; + subject.handle_control(ConnectionControl::SendVoteResponse(response)).unwrap(); + let expected = RequestVoteResponse { + op_id: 99, + term: 7, + vote_granted: false, + }; + fixture.assert_sent_to_client(ProtocolMessage::VoteResponse(expected)); + } + + #[test] + fn send_request_vote_control_sends_request_vote_to_client() { + let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); + let request_vote = CallRequestVote { + term: 4, + candidate_id: fixture.instance_id, + last_log_index: 33, + last_log_term: 3, + }; + subject.handle_control(ConnectionControl::SendRequestVote(request_vote)).unwrap(); + + let expected = RequestVoteCall { + op_id: 2, + term: 4, + candidate_id: fixture.instance_id, + last_log_index: 33, + last_log_term: 3, + }; + fixture.assert_sent_to_client(ProtocolMessage::RequestVote(expected)); + } + #[test] fn set_event_stream_sets_event_stream_when_the_named_stream_exists() { let (mut subject, mut fixture) = Fixture::create(); diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 3940b50..3033be8 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -1,55 +1,117 @@ mod peer_follower; -use protocol::{ProtocolMessage, PeerAnnounce, EventStreamStatus, ClusterMember}; +use std::collections::VecDeque; +use protocol::{self, ProtocolMessage, PeerAnnounce, EventStreamStatus, ClusterMember, ErrorMessage, ErrorKind}; use engine::{ReceivedProtocolMessage, ConnectionId}; -use engine::controller::SystemStreamRef; -use engine::controller::Peer; +use engine::controller::{self, SystemStreamRef, Peer}; use super::connection_state::ConnectionState; use super::ConnectionHandlerResult; #[derive(Debug, Clone, Copy, PartialEq)] -pub enum PeerConnectionState { +enum State { Init, AwaitingPeerResponse, Peer, } +#[derive(Debug, Clone, PartialEq)] +pub struct PeerConnectionState { + state: State, + current_op_id: u32, + pending_operation_queue: VecDeque, +} + impl PeerConnectionState { + pub fn new() -> PeerConnectionState { + PeerConnectionState { + state: State::Init, + current_op_id: 0, + pending_operation_queue: VecDeque::new(), + } + } + + pub fn send_vote_response(&mut self, response: controller::VoteResponse, state: &mut ConnectionState) -> ConnectionHandlerResult { + let pending = self.pending_operation_queue.pop_back(); + if let Some(op_id) = pending { + let protocol_message = protocol::RequestVoteResponse { + op_id, + term: response.term, + vote_granted: response.granted, + }; + state.send_to_client(ProtocolMessage::VoteResponse(protocol_message)) + } else { + let err_message = format!("Refusing to send: {:?} as there is no pending operation", response); + error!("{}", err_message); + Err(err_message) + } + } + + pub fn request_vote_received(&mut self, request: protocol::RequestVoteCall, state: &mut ConnectionState) -> ConnectionHandlerResult { + if self.state == State::Peer { + let protocol::RequestVoteCall { op_id, term, candidate_id, last_log_index, last_log_term } = request; + self.pending_operation_queue.push_front(op_id); + let controller_message = controller::CallRequestVote { term, candidate_id, last_log_term, last_log_index }; + let connection_id = state.connection_id; + state.get_system_stream().request_vote_received(connection_id, controller_message); + Ok(()) + } else { + let err_message = format!("Refusing to process RequestVote when connection is in {:?} state", self.state); + let response = ErrorMessage { + op_id: request.op_id, + kind: ErrorKind::InvalidPeerState, + description: err_message.clone(), + }; + let _ = state.send_to_client(ProtocolMessage::Error(response)); + // return error so that connection will be closed + Err(err_message) + } + } + + pub fn send_request_vote(&mut self, request: controller::CallRequestVote, state: &mut ConnectionState) -> ConnectionHandlerResult { + let controller::CallRequestVote { term, candidate_id, last_log_index, last_log_term } = request; + let protocol_message = protocol::RequestVoteCall { + op_id: self.next_op_id(), + term, + candidate_id, + last_log_index, + last_log_term, + }; + state.send_to_client(ProtocolMessage::RequestVote(protocol_message)) + } + pub fn initiate_outgoing_peer_connection(&mut self, state: &mut ConnectionState) { - assert_eq!(PeerConnectionState::Init, *self); + info!("Upgrading connection_id: {} to outgoing peer-to-peer connection", state.connection_id); + assert_eq!(State::Init, self.state); state.set_to_system_stream(); - let announce = PeerConnectionState::create_peer_announce(&*state.engine.system_stream()); + let announce = self.create_peer_announce(&*state.engine.system_stream()); let protocol_message = ProtocolMessage::PeerAnnounce(announce); // safe unwrap since this is called only when creating a brand new outgoing connection state.send_to_client(protocol_message).expect("failed to send peer announce when establishing outgoing connection"); - *self = PeerConnectionState::AwaitingPeerResponse; + self.set_state(state.connection_id, State::AwaitingPeerResponse); } pub fn peer_announce_received(&mut self, announce: PeerAnnounce, state: &mut ConnectionState) -> ConnectionHandlerResult { let connection_id = state.connection_id; - let old_state = self.set_state(connection_id, PeerConnectionState::Peer); + let old_state = self.set_state(connection_id, State::Peer); debug!("PeerAnnounce received on connection_id: {}, {:?}, prev_connection_state: {:?}", connection_id, announce, old_state); - match old_state { - PeerConnectionState::Peer => { + State::Peer => { // we've already gone through this, so something's wrong let message = format!("received redundant PeerAnnounce for connection_id: {}, closing connection", connection_id); return Err(message) } - PeerConnectionState::Init => { + State::Init => { // This was an incoming connection, and this was the first peer message sent, so we need to respond in kind - let peer_announce = PeerConnectionState::create_peer_announce(state.get_system_stream()); + let peer_announce = self.create_peer_announce(state.get_system_stream()); state.send_to_client(ProtocolMessage::PeerAnnounce(peer_announce))?; } - PeerConnectionState::AwaitingPeerResponse => { - - } + State::AwaitingPeerResponse => { } } state.set_to_system_stream(); @@ -71,12 +133,13 @@ impl PeerConnectionState { Ok(()) } - fn create_peer_announce(system_stream: &SystemStreamRef) -> PeerAnnounce { + fn create_peer_announce(&mut self, system_stream: &SystemStreamRef) -> PeerAnnounce { + let op_id = self.next_op_id(); system_stream.with_cluster_state(|state| { let instance_id = state.this_instance_id; let address = state.this_address.expect("Attempted to send PeerAnnounce, but system is not in cluster mode"); - let primary = state.system_primary.as_ref().map(|peer| peer.id); - let members = state.peers.iter().map(|peer| { + let system_primary_id = state.system_primary.as_ref().map(|peer| peer.id); + let cluster_members = state.peers.iter().map(|peer| { ClusterMember { id: peer.id, address: peer.address, @@ -84,27 +147,27 @@ impl PeerConnectionState { }).collect::>(); PeerAnnounce { - protocol_version: 1, + op_id, instance_id, + system_primary_id, + cluster_members, + protocol_version: 1, peer_address: address, - op_id: 1, - system_primary_id: primary, - cluster_members: members, } }) } - fn set_state(&mut self, connection_id: ConnectionId, new_state: PeerConnectionState) -> PeerConnectionState { + fn next_op_id(&mut self) -> u32 { + self.current_op_id += 1; + self.current_op_id + } + + fn set_state(&mut self, connection_id: ConnectionId, new_state: State) -> State { debug!("Transitioning connection_id: {} from {:?} to {:?}", connection_id, self, new_state); - ::std::mem::replace(self, new_state) + ::std::mem::replace(&mut self.state, new_state) } } fn member_to_peer(ClusterMember{id, address}: ClusterMember) -> Peer { Peer { id, address } } - -#[cfg(test)] -mod test { - -} diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index a777f20..80944ec 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -1,4 +1,4 @@ -mod persistent; +pub mod persistent; mod peer_connections; use std::io; @@ -125,6 +125,25 @@ impl ClusterManager { } } } + + fn election_timed_out(&self, now: Instant) -> bool { + now - self.last_heartbeat > self.election_timeout + } + + fn start_new_election(&mut self) { + info!("Starting new election"); + let result = self.persistent.modify(|state| { + state.current_term += 1; + let my_id = state.this_instance_id; + state.voted_for = Some(my_id); + }); + if let Err(io_err) = result { + error!("Aborting new election due to error persisting state: {:?}", io_err); + return; + } + unimplemented!() + } + } impl ConsensusProcessor for ClusterManager { @@ -154,6 +173,11 @@ impl ConsensusProcessor for ClusterManager { // we'll only be in this initial status once, on startup self.transition_state(State::DeterminePrimary); } + State::Follower => { + if self.election_timed_out(now) { + self.start_new_election(); + } + } _ => { } } } @@ -221,6 +245,47 @@ mod test { start + Duration::from_secs(seconds) } + #[test] + fn cluster_manager_starts_new_election_after_timeout_elapses() { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let mut mock_creator = MockOutgoingConnectionCreator::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:3000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:3000") + }; + let (peer_1_connection, _) = mock_creator.stub(peer_1.address); + let (peer_2_connection, _) = mock_creator.stub(peer_2.address); + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), mock_creator.boxed()); + assert_eq!(0, subject.persistent.current_term); + + let mut connections = HashMap::new(); + subject.tick(t(start, 1), &mut connections); + + let upgrade_1 = PeerUpgrade { + peer_id: peer_1.id, + system_primary: Some(peer_2.clone()), + cluster_members: vec![peer_2.clone()], + }; + subject.peer_connection_established(upgrade_1, peer_1_connection.connection_id, &connections); + let upgrade_2 = PeerUpgrade { + peer_id: peer_2.id, + system_primary: Some(peer_2.clone()), + cluster_members: vec![peer_2.clone()], + }; + subject.peer_connection_established(upgrade_2, peer_2_connection.connection_id, &connections); + + subject.tick(t(start, 2), &mut connections); + + let this_id = subject.persistent.this_instance_id; + assert_eq!(Some(this_id), subject.persistent.voted_for); + assert_eq!(1, subject.persistent.current_term); + } + #[test] fn cluster_manager_moves_to_follower_state_once_peer_announce_is_received_with_known_primary() { let start = Instant::now(); diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs index 5da83b9..80262e4 100644 --- a/flo-server/src/engine/controller/cluster_state/peer_connections.rs +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -125,8 +125,8 @@ impl PeerConnections { fn send(connection: &ConnectionRef, control: ConnectionControl) { let result = connection.control_sender.send(control); - if result.is_err() { - warn!("Error sending control to connection_id: {}, {:?}", connection.connection_id, control); + if let Err(send_err) = result { + warn!("Error sending control to connection_id: {}, {:?}", connection.connection_id, send_err); } } @@ -225,7 +225,7 @@ mod test { }; let mut creator = MockOutgoingConnectionCreator::new(); creator.stub(peer_address); - let mut subject = PeerConnections::new(Vec::new(), creator.boxed(), &[peer.clone()]); + let subject = PeerConnections::new(Vec::new(), creator.boxed(), &[peer.clone()]); assert!(subject.disconnected_peers.contains_key(&peer_address)); } diff --git a/flo-server/src/engine/controller/cluster_state/persistent.rs b/flo-server/src/engine/controller/cluster_state/persistent.rs index 601412f..885abd1 100644 --- a/flo-server/src/engine/controller/cluster_state/persistent.rs +++ b/flo-server/src/engine/controller/cluster_state/persistent.rs @@ -1,7 +1,7 @@ use std::net::SocketAddr; use std::path::PathBuf; use std::fs::{File, OpenOptions}; -use std::io::{self, Seek}; +use std::io::{self, Seek, SeekFrom}; use protocol::{FloInstanceId, Term}; use engine::controller::controller_messages::Peer; @@ -10,10 +10,12 @@ use super::{ClusterOptions, SharedClusterState}; /// Holds all the cluster state that we want to survive a reboot. /// We always persist the `FloInstanceId` because we prefer that to be stable across reboots. We do _not_ want to persist /// the `SocketAddr` for the server, though, since that may well change after a restart, depending on environment. -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub struct PersistentClusterState { pub current_term: Term, + #[serde(with = "OptInstanceIdRemote")] pub voted_for: Option, + #[serde(with = "InstanceIdRemote")] pub this_instance_id: FloInstanceId, pub cluster_members: Vec, } @@ -31,6 +33,40 @@ impl PersistentClusterState { } } +#[derive(Serialize, Deserialize, PartialEq, Debug, Clone, Copy)] +#[serde(remote = "FloInstanceId")] +pub struct InstanceIdRemote{ + #[serde(getter = "FloInstanceId::as_bytes")] + bytes: [u8; 8] +} + +#[derive(Serialize, Deserialize, PartialEq, Debug, Clone, Copy)] +#[serde(remote = "Option")] +pub struct OptInstanceIdRemote{ + #[serde(getter = "id_as_bytes")] + bytes: [u8; 8] +} + +fn id_as_bytes(id: &Option) -> [u8; 8] { + id.map(|i| i.as_bytes()).unwrap_or([0; 8]) +} + +impl Into> for OptInstanceIdRemote { + fn into(self) -> Option { + let id = FloInstanceId::from_bytes(&self.bytes[..]); + if id == FloInstanceId::null() { + None + } else { + Some(id) + } + } +} + +impl Into for InstanceIdRemote { + fn into(self) -> FloInstanceId { + FloInstanceId::from_bytes(&self.bytes[..]) + } +} /// Placeholder for a wrapper struct that will take care of persisting the state as it changes #[derive(Debug)] @@ -42,15 +78,25 @@ pub struct FilePersistedState { impl FilePersistedState { pub fn initialize(path: PathBuf) -> io::Result { - let file = OpenOptions::new().write(true).read(true).create(true).open(&path)?; // early return on failure - let state = PersistentClusterState { - current_term: 0, - voted_for: None, - this_instance_id: FloInstanceId::generate_new(), - cluster_members: Vec::new(), + let (file, state) = if path.exists() { + let mut file = OpenOptions::new().write(true).read(true).open(&path)?; // early return on failure + let state = ::serde_json::from_reader(&mut file).map_err(|des_err| { + error!("Failed to read persistent cluster state: {:?}", des_err); + io::Error::new(io::ErrorKind::InvalidData, format!("Deserialization error: {}", des_err)) + })?; // early return if this fails + (file, state) + } else { + let file = OpenOptions::new().write(true).read(true).create(true).open(&path)?; // early return on failure + let state = PersistentClusterState { + current_term: 0, + voted_for: None, + this_instance_id: FloInstanceId::generate_new(), + cluster_members: Vec::new(), + }; + info!("Initialized brand new state: {:?}", state); + (file, state) }; - debug!("Initialized {:?}", state); Ok(FilePersistedState { file, @@ -58,6 +104,34 @@ impl FilePersistedState { state }) } + + pub fn modify(&mut self, fun: F) -> Result<(), io::Error> where F: Fn(&mut PersistentClusterState) { + fun(&mut self.state); + self.flush() + } + + pub fn flush(&mut self) -> io::Result<()> { + use serde_json::to_writer_pretty; + let FilePersistedState {ref mut file, ref state, ..} = *self; + // all the early returns + file.seek(SeekFrom::Start(0))?; + to_writer_pretty(&mut *file, state)?; + + // truncates the file in case the modified data is shorter than the old data, so there won't be invalid garbage at the end + let position = file.seek(SeekFrom::Current(0))?; + file.set_len(position)?; + + /* + It's debatable whether this is really necessary. _technically_, it's possible that an instance could cast a vote, have + a power failure that causes unsynced data to be lost, then power on and vot again in the same term. This is the reason + that Raft recommends persisting this information on disk in the first place, to prevent an instance from voting multiple + times in one term. Given an election timeout of ~300 milliseconds, though, it seems pretty unlikely for a server to + power cycle and vote twice, though. If we only need the persistent data to survive a restart of the flo process, then + the sync probably isn't necessary. Anyway, we'll play it safe for now until we determine that it's actually causing a + performance problem. + */ + file.sync_data() + } } impl ::std::ops::Deref for FilePersistedState { @@ -68,3 +142,49 @@ impl ::std::ops::Deref for FilePersistedState { } } +#[cfg(test)] +mod test { + use super::*; + use tempdir::TempDir; + use test_utils::addr; + + #[test] + fn modifying_state_persists_changes() { + use std::ops::Deref; + + let temp = TempDir::new("persistent_state_test").unwrap(); + let path = temp.path().join("test_cluster_state"); + + let mut subject = FilePersistedState::initialize(path.clone()).unwrap(); + subject.modify(|state| { + state.current_term = 9; + state.cluster_members = vec![ + Peer { + id: FloInstanceId::generate_new(), + address: addr("127.0.0.1:3456") + }, + Peer { + id: FloInstanceId::generate_new(), + address: addr("[2001:873::1]:3000") + }, + Peer { + id: FloInstanceId::generate_new(), + address: addr("127.0.0.1:456") + } + ]; + }).unwrap(); + + let subject2 = FilePersistedState::initialize(path.clone()).unwrap(); + assert_eq!(subject.deref(), subject2.deref()); + + // remove some data here. This will cause the second init to fail if we don't truncate the file + subject.modify(|state| { + state.cluster_members.pop(); + state.cluster_members.pop(); + }).unwrap(); + + let subject2 = FilePersistedState::initialize(path.clone()).unwrap(); + assert_eq!(subject.deref(), subject2.deref()); + } + +} diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 7917338..34b836c 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -3,10 +3,26 @@ use std::time::Instant; use futures::sync::mpsc::UnboundedSender; -use protocol::FloInstanceId; +use event::EventCounter; +use protocol::{FloInstanceId, Term}; use engine::event_stream::partition::{self, Operation}; use engine::connection_handler::{ConnectionControl, ConnectionControlSender}; use engine::ConnectionId; +use engine::controller::cluster_state::persistent::InstanceIdRemote; + +#[derive(Debug, Clone, PartialEq)] +pub struct CallRequestVote { + pub term: Term, + pub candidate_id: FloInstanceId, + pub last_log_index: EventCounter, + pub last_log_term: Term, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct VoteResponse { + pub term: Term, + pub granted: bool, +} #[derive(Debug, Clone)] pub struct ConnectionRef { @@ -15,13 +31,20 @@ pub struct ConnectionRef { pub control_sender: ConnectionControlSender, } -#[derive(Debug, PartialEq, Eq, Hash, Clone)] +impl PartialEq for ConnectionRef { + fn eq(&self, other: &ConnectionRef) -> bool { + self.connection_id == other.connection_id && self.remote_address == other.remote_address + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)] pub struct Peer { + #[serde(with = "InstanceIdRemote")] pub id: FloInstanceId, pub address: SocketAddr, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct PeerUpgrade { pub peer_id: FloInstanceId, pub system_primary: Option, @@ -29,7 +52,7 @@ pub struct PeerUpgrade { } -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum SystemOpType { Tick, PartitionOp(partition::OpType), @@ -37,6 +60,8 @@ pub enum SystemOpType { ConnectionUpgradeToPeer(PeerUpgrade), ConnectionClosed, OutgoingConnectionFailed(SocketAddr), + RequestVote(CallRequestVote), + VoteResponseReceived(VoteResponse), } impl SystemOpType { @@ -57,6 +82,10 @@ pub struct SystemOperation { impl SystemOperation { + pub fn request_vote_received(connection_id: ConnectionId, request: CallRequestVote) -> SystemOperation { + SystemOperation::new(connection_id, SystemOpType::RequestVote(request)) + } + pub fn connection_upgraded_to_peer(connection_id: ConnectionId, peer_id: FloInstanceId, system_primary: Option, cluster_members: Vec) -> SystemOperation { let upgrade = PeerUpgrade { peer_id, diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index a9fe4b7..faaffac 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -24,7 +24,7 @@ use self::cluster_state::{ClusterManager, ConsensusProcessor}; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; pub use self::system_stream::SystemStreamRef; -pub use self::controller_messages::{SystemOperation, SystemOpType, ConnectionRef, Peer, PeerUpgrade}; +pub use self::controller_messages::{SystemOperation, SystemOpType, ConnectionRef, Peer, PeerUpgrade, CallRequestVote, VoteResponse}; pub use self::cluster_state::{SharedClusterState, ClusterStateReader}; pub type SystemPartitionSender = ::std::sync::mpsc::Sender; diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index aeef14e..2f7317a 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -3,7 +3,7 @@ use std::net::SocketAddr; use protocol::FloInstanceId; use engine::event_stream::partition::{PartitionRef, Operation}; use engine::event_stream::EventStreamRef; -use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer}; +use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer, CallRequestVote}; use engine::controller::cluster_state::{SharedClusterState, ClusterStateReader}; use engine::ConnectionId; use atomics::AtomicBoolReader; @@ -67,6 +67,11 @@ impl SystemStreamRef { self.send(op); } + pub fn request_vote_received(&mut self, connection_id: ConnectionId, request: CallRequestVote) { + let op = SystemOperation::request_vote_received(connection_id, request); + self.send(op); + } + fn send(&mut self, op: SystemOperation) { // TODO: change this to propagate the error so that connectionHandlers can shut down gracefully self.system_sender.send(op).expect("Failed to send to flo controller. System must have shut down"); diff --git a/flo-server/src/engine/event_stream/partition/ops.rs b/flo-server/src/engine/event_stream/partition/ops.rs index 249f2c9..713bf8a 100644 --- a/flo-server/src/engine/event_stream/partition/ops.rs +++ b/flo-server/src/engine/event_stream/partition/ops.rs @@ -21,6 +21,12 @@ pub struct ProduceOperation { pub events: Vec, } +impl PartialEq for ProduceOperation { + fn eq(&self, other: &ProduceOperation) -> bool { + self.op_id == other.op_id && self.events == other.events + } +} + impl Debug for ProduceOperation { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -48,27 +54,28 @@ pub struct ConsumeOperation { pub notifier: Box, } +impl PartialEq for ConsumeOperation { + fn eq(&self, other: &ConsumeOperation) -> bool { + self.filter == other.filter && + self.start_exclusive == other.start_exclusive && + self.notifier.connection_id() == other.notifier.connection_id() + } +} + impl Debug for ConsumeOperation { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "ConsumeOperation {{ filter: {:?}, start_exclusive: {} }}", self.filter, self.start_exclusive) } } -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum OpType { Produce(ProduceOperation), Consume(ConsumeOperation), StopConsumer, Tick, - System(SystemOp), } -#[derive(Debug)] -pub enum SystemOp { - OutgoingConnectionFailed(SocketAddr), -} - - #[derive(Debug)] pub struct Operation { pub connection_id: ConnectionId, @@ -86,10 +93,6 @@ impl Operation { } } - fn system(connection_id: ConnectionId, system_op: SystemOp) -> Operation { - Operation::new(connection_id, OpType::System(system_op)) - } - pub fn consume(connection_id: ConnectionId, notifier: Box, filter: EventFilter, start_exclusive: EventCounter) -> (Operation, ConsumeResponseReceiver) { let (tx, rx) = oneshot::channel(); let consume = ConsumeOperation { @@ -120,9 +123,5 @@ impl Operation { pub fn tick() -> Operation { Operation::new(0, OpType::Tick) } - - pub fn outgoing_connection_failed(connection_id: ConnectionId, socket_addr: SocketAddr) -> Operation { - Operation::system(connection_id, SystemOp::OutgoingConnectionFailed(socket_addr)) - } } From 5397954ab583c100a4078c40660fecd701dad205 Mon Sep 17 00:00:00 2001 From: pfried Date: Fri, 29 Dec 2017 21:13:13 -0500 Subject: [PATCH 31/73] connectionHandler now also deals with VoteResponse messages --- .../src/engine/connection_handler/mod.rs | 52 ++++++++++++++++++- .../src/engine/connection_handler/peer/mod.rs | 35 +++++++++++-- .../engine/controller/controller_messages.rs | 4 ++ .../src/engine/controller/system_stream.rs | 7 ++- 4 files changed, 91 insertions(+), 7 deletions(-) diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 9315544..23951b8 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -101,6 +101,9 @@ impl ConnectionHandler { ProtocolMessage::RequestVote(request_vote) => { peer_state.request_vote_received(request_vote, common_state) } + ProtocolMessage::VoteResponse(response) => { + peer_state.vote_response_received(response, common_state) + } _ => unimplemented!() } } @@ -187,6 +190,8 @@ mod test { use atomics::{AtomicCounterWriter, AtomicBoolWriter}; use test_utils::addr; + const CONNECTION_ID: ConnectionId = 456; + struct Fixture { #[allow(dead_code)] // TODO: add more connection handler tests partition_receivers: HashMap<(String, ActorId), PartitionReceiver>, @@ -254,7 +259,7 @@ mod test { let streams = Arc::new(Mutex::new(HashMap::new())); let engine = EngineRef::new(system_stream, streams); - let subject = ConnectionHandler::new(456, client_sender, engine.clone(), reactor.handle()); + let subject = ConnectionHandler::new(CONNECTION_ID, client_sender, engine.clone(), reactor.handle()); let fixture = Fixture { partition_receivers: HashMap::new(), @@ -345,6 +350,51 @@ mod test { } + #[test] + fn error_is_returned_when_vote_response_is_unexpected() { + let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); + + let error = subject.handle_incoming_message(ProtocolMessage::VoteResponse(RequestVoteResponse { + op_id: 7, + term: 6, + vote_granted: true + })).unwrap_err(); + assert_eq!("connection_id: 456 received unexpected message: RequestVoteResponse { op_id: 7, term: 6, vote_granted: true }, expected op_id: None", &error); + } + + #[test] + fn vote_response_is_sent_to_system_stream_when_one_is_expected() { + let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); + let peer_id = FloInstanceId::generate_new(); + let request_vote = CallRequestVote { + term: 5, + candidate_id: peer_id, + last_log_index: 44, + last_log_term: 4, + }; + subject.handle_control(ConnectionControl::SendRequestVote(request_vote)).unwrap(); + fixture.assert_sent_to_client(ProtocolMessage::RequestVote(RequestVoteCall { + op_id: 2, + term: 5, + candidate_id: peer_id, + last_log_index: 44, + last_log_term: 4, + })); + + subject.handle_incoming_message(ProtocolMessage::VoteResponse(RequestVoteResponse { + op_id: 2, + term: 7, + vote_granted: false, + })).unwrap(); + + let expected = VoteResponse { + term: 7, + granted: false, + }; + fixture.assert_sent_to_system_stream(SystemOpType::VoteResponseReceived(expected)); + + } + #[test] fn receiving_request_vote_results_in_error_when_connection_is_not_in_peer_state() { let (mut subject, mut fixture) = Fixture::create(); diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 3033be8..7ca54b9 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -19,7 +19,8 @@ enum State { pub struct PeerConnectionState { state: State, current_op_id: u32, - pending_operation_queue: VecDeque, + controller_operation_queue: VecDeque, + peer_operation_queue: VecDeque, } @@ -28,12 +29,34 @@ impl PeerConnectionState { PeerConnectionState { state: State::Init, current_op_id: 0, - pending_operation_queue: VecDeque::new(), + controller_operation_queue: VecDeque::new(), + peer_operation_queue: VecDeque::new(), + } + } + + pub fn vote_response_received(&mut self, response: protocol::RequestVoteResponse, state: &mut ConnectionState) -> ConnectionHandlerResult { + let connection_id = state.connection_id; + let expected_op_id = self.peer_operation_queue.pop_back(); + + + if Some(response.op_id) == expected_op_id { + let protocol::RequestVoteResponse { op_id, term, vote_granted } = response; + let controller_message = controller::VoteResponse { + term, + granted: vote_granted + }; + state.get_system_stream().vote_response_received(connection_id, controller_message); + Ok(()) + } else { + let err_message = format!("connection_id: {} received unexpected message: {:?}, expected op_id: {:?}", + connection_id, response, expected_op_id); + error!("{}", err_message); + Err(err_message) } } pub fn send_vote_response(&mut self, response: controller::VoteResponse, state: &mut ConnectionState) -> ConnectionHandlerResult { - let pending = self.pending_operation_queue.pop_back(); + let pending = self.controller_operation_queue.pop_back(); if let Some(op_id) = pending { let protocol_message = protocol::RequestVoteResponse { op_id, @@ -51,7 +74,7 @@ impl PeerConnectionState { pub fn request_vote_received(&mut self, request: protocol::RequestVoteCall, state: &mut ConnectionState) -> ConnectionHandlerResult { if self.state == State::Peer { let protocol::RequestVoteCall { op_id, term, candidate_id, last_log_index, last_log_term } = request; - self.pending_operation_queue.push_front(op_id); + self.controller_operation_queue.push_front(op_id); let controller_message = controller::CallRequestVote { term, candidate_id, last_log_term, last_log_index }; let connection_id = state.connection_id; state.get_system_stream().request_vote_received(connection_id, controller_message); @@ -71,8 +94,10 @@ impl PeerConnectionState { pub fn send_request_vote(&mut self, request: controller::CallRequestVote, state: &mut ConnectionState) -> ConnectionHandlerResult { let controller::CallRequestVote { term, candidate_id, last_log_index, last_log_term } = request; + let op_id = self.next_op_id(); + self.peer_operation_queue.push_front(op_id); let protocol_message = protocol::RequestVoteCall { - op_id: self.next_op_id(), + op_id, term, candidate_id, last_log_index, diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 34b836c..1f9ca7e 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -82,6 +82,10 @@ pub struct SystemOperation { impl SystemOperation { + pub fn vote_response_received(connection_id: ConnectionId, response: VoteResponse) -> SystemOperation { + SystemOperation::new(connection_id, SystemOpType::VoteResponseReceived(response)) + } + pub fn request_vote_received(connection_id: ConnectionId, request: CallRequestVote) -> SystemOperation { SystemOperation::new(connection_id, SystemOpType::RequestVote(request)) } diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index 2f7317a..f247876 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -3,7 +3,7 @@ use std::net::SocketAddr; use protocol::FloInstanceId; use engine::event_stream::partition::{PartitionRef, Operation}; use engine::event_stream::EventStreamRef; -use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer, CallRequestVote}; +use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer, CallRequestVote, VoteResponse}; use engine::controller::cluster_state::{SharedClusterState, ClusterStateReader}; use engine::ConnectionId; use atomics::AtomicBoolReader; @@ -72,6 +72,11 @@ impl SystemStreamRef { self.send(op); } + pub fn vote_response_received(&mut self, connection_id: ConnectionId, response: VoteResponse) { + let op = SystemOperation::vote_response_received(connection_id, response); + self.send(op); + } + fn send(&mut self, op: SystemOperation) { // TODO: change this to propagate the error so that connectionHandlers can shut down gracefully self.system_sender.send(op).expect("Failed to send to flo controller. System must have shut down"); From 21c1461422e2b65019121c487b51113903dfb779 Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 30 Dec 2017 00:13:38 -0500 Subject: [PATCH 32/73] controller triggers an election after the timeout, but of course there is no resolution to the election --- .../engine/controller/cluster_state/mod.rs | 15 +++- .../cluster_state/peer_connections.rs | 74 ++++++++++++++++++- flo-server/src/test_utils.rs | 31 ++++++++ 3 files changed, 114 insertions(+), 6 deletions(-) diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 80944ec..bf1d252 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -10,7 +10,9 @@ use std::collections::{HashMap, HashSet}; use event::EventCounter; use engine::{ConnectionId, EngineRef}; -use protocol::FloInstanceId; +use engine::connection_handler::ConnectionControl; +use engine::controller::CallRequestVote; +use protocol::{FloInstanceId, Term}; use atomics::{AtomicBoolWriter, AtomicBoolReader}; use super::{ClusterOptions, ConnectionRef, Peer, PeerUpgrade}; use super::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator, OutgoingConnectionCreatorImpl}; @@ -49,6 +51,7 @@ pub struct ClusterManager { last_heartbeat: Instant, primary_status_writer: AtomicBoolWriter, last_applied: EventCounter, + last_applied_term: Term, persistent: FilePersistedState, system_partition_primary_address: SystemPrimaryAddressRef, shared: Arc>, @@ -87,6 +90,7 @@ impl ClusterManager { connection_manager: peer_connections, initialization_peers, last_applied: 0, + last_applied_term: 0, primary_status_writer, persistent, system_partition_primary_address, @@ -141,9 +145,16 @@ impl ClusterManager { error!("Aborting new election due to error persisting state: {:?}", io_err); return; } - unimplemented!() + let connection_control = ConnectionControl::SendRequestVote(CallRequestVote { + term: self.persistent.current_term, + candidate_id: self.persistent.this_instance_id, + last_log_index: self.last_applied, + last_log_term: self.last_applied_term, + }); + self.connection_manager.broadcast(connection_control); } + } impl ConsensusProcessor for ClusterManager { diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs index 80262e4..9a0a037 100644 --- a/flo-server/src/engine/controller/cluster_state/peer_connections.rs +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -2,9 +2,10 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::time::{Instant, Duration}; -use protocol::FloInstanceId; +use event::EventCounter; +use protocol::{FloInstanceId, Term}; use engine::ConnectionId; -use engine::controller::{ConnectionRef, Peer}; +use engine::controller::{ConnectionRef, Peer, CallRequestVote}; use engine::controller::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator}; use engine::connection_handler::ConnectionControl; @@ -121,10 +122,25 @@ impl PeerConnections { self.active_connections.get(&connection_id).cloned() } + pub fn broadcast(&mut self, control: ConnectionControl) { + debug!("Broadcasting {:?}", control); + for (peer, connection) in self.known_peers.iter() { + match connection.state { + PeerState::Connected(ref connection_ref) => { + send(connection_ref, control.clone()); + } + ref other @ _ => { + trace!("Skipping connection to {:?} because it is in state: {:?}", peer, other); + } + } + } + } + } fn send(connection: &ConnectionRef, control: ConnectionControl) { - let result = connection.control_sender.send(control); + trace!("Sending to connection_id: {}, control: {:?}", connection.connection_id, control); + let result = connection.control_sender.unbounded_send(control); if let Err(send_err) = result { warn!("Error sending control to connection_id: {}, {:?}", connection.connection_id, send_err); } @@ -190,7 +206,57 @@ mod test { use super::*; use std::time::Duration; use engine::controller::peer_connection::MockOutgoingConnectionCreator; - use test_utils::addr; + use engine::connection_handler::ConnectionControlReceiver; + use test_utils::{addr, expect_future_resolved}; + + fn assert_control_sent(rx: ConnectionControlReceiver, expected: &ConnectionControl) -> ConnectionControlReceiver { + use futures::Stream; + let (message, stream) = expect_future_resolved(rx.into_future()).expect("failed to receive control message"); + let message = message.expect("did not receive any control message"); + assert_eq!(&message, expected); + stream + } + + fn subject_with_connected_peers(peers: &[Peer], creator: MockOutgoingConnectionCreator) -> (PeerConnections, HashMap) { + let mut subject = PeerConnections::new(Vec::new(), creator.boxed(), peers); + let mut connections = HashMap::new(); + subject.establish_connections(Instant::now(), &mut connections); + for peer in peers { + let connection = connections.values().find(|conn| conn.remote_address == peer.address).unwrap(); + subject.peer_connection_established(peer.id, connection); + } + (subject, connections) + } + + #[test] + fn broadcast_sends_control_to_all_connected_peers() { + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("123.4.5.6:3000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("123.4.5.6:4000") + }; + let mut creator = MockOutgoingConnectionCreator::new(); + let (peer_1_conn, rx_1) = creator.stub(peer_1.address); + let (peer_2_conn, rx_2) = creator.stub(peer_2.address); + let (mut subject, mut connections) = subject_with_connected_peers(&[peer_1, peer_2], creator); + let rx_1 = assert_control_sent(rx_1, &ConnectionControl::InitiateOutgoingSystemConnection); + let rx_2 = assert_control_sent(rx_2, &ConnectionControl::InitiateOutgoingSystemConnection); + + let candidate = FloInstanceId::generate_new(); + let expected = ConnectionControl::SendRequestVote(CallRequestVote { + term: 7, + candidate_id: candidate, + last_log_index: 99, + last_log_term: 6, + }); + subject.broadcast(expected.clone()); + + assert_control_sent(rx_1, &expected); + assert_control_sent(rx_2, &expected); + } #[test] fn outgoing_connect_success_adds_known_peer_and_connection_closed_sets_it_to_disconnected() { diff --git a/flo-server/src/test_utils.rs b/flo-server/src/test_utils.rs index 47fffba..5e230ed 100644 --- a/flo-server/src/test_utils.rs +++ b/flo-server/src/test_utils.rs @@ -1,6 +1,37 @@ use std::net::SocketAddr; +use std::time::{Duration, Instant}; +use futures::executor::{spawn, Unpark}; +use futures::{Future, Async}; pub fn addr(string: &str) -> SocketAddr { ::std::str::FromStr::from_str(string).unwrap() } + +/// used for testing futures when we don't expect the future to need unparked +struct NoOpUnpark; +impl Unpark for NoOpUnpark { + fn unpark(&self) { + unimplemented!() + } +} + +pub fn expect_future_resolved(future: F) -> Result where F: Future { + let now = Instant::now(); + let mut s = spawn(future); + let unpark = ::std::sync::Arc::new(NoOpUnpark); + loop { + let result = s.poll_future(unpark.clone()); + match result { + Ok(Async::Ready(t)) => { + return Ok(t); + } + Err(e) => { + return Err(e); + } + Ok(Async::NotReady) => { + panic!("Future was not ready"); + } + } + } +} From f0d89f3d31f3cd5f5eb171c7a497bbcc06bdeae3 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 31 Dec 2017 00:29:47 -0500 Subject: [PATCH 33/73] factor out a PeerConnectionManager trait to make testing easier, implement responses to RequestVote in cluster manager --- .../src/engine/connection_handler/mod.rs | 6 +- .../engine/controller/cluster_state/mod.rs | 341 +++++++++++++++--- .../cluster_state/peer_connections.rs | 139 ++++++- .../controller/cluster_state/persistent.rs | 12 +- .../engine/controller/controller_messages.rs | 16 + 5 files changed, 429 insertions(+), 85 deletions(-) diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 23951b8..2e1e024 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -175,7 +175,7 @@ impl Debug for ConnectionHandler { #[cfg(test)] mod test { - use std::collections::HashMap; + use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex, RwLock}; use std::net::SocketAddr; use tokio_core::reactor::Core; @@ -252,7 +252,7 @@ mod test { this_instance_id: instance_id, this_address: Some(instance_addr), system_primary: None, - peers: Vec::new(), + peers: HashSet::new(), }; let system_stream = SystemStreamRef::new(part_ref, tx, Arc::new(RwLock::new(cluster_state))); @@ -352,7 +352,7 @@ mod test { #[test] fn error_is_returned_when_vote_response_is_unexpected() { - let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); + let (mut subject, fixture) = Fixture::create_outgoing_peer_connection(); let error = subject.handle_incoming_message(ProtocolMessage::VoteResponse(RequestVoteResponse { op_id: 7, diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index bf1d252..defa6e1 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -11,20 +11,29 @@ use std::collections::{HashMap, HashSet}; use event::EventCounter; use engine::{ConnectionId, EngineRef}; use engine::connection_handler::ConnectionControl; -use engine::controller::CallRequestVote; +use engine::controller::{CallRequestVote, VoteResponse}; use protocol::{FloInstanceId, Term}; use atomics::{AtomicBoolWriter, AtomicBoolReader}; use super::{ClusterOptions, ConnectionRef, Peer, PeerUpgrade}; use super::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator, OutgoingConnectionCreatorImpl}; -use self::peer_connections::PeerConnections; +use self::peer_connections::{PeerConnectionManager, PeerConnections}; pub use self::persistent::{FilePersistedState, PersistentClusterState}; +/// This is a placeholder for a somewhat better error handling when updating the persistent cluster state fails. +/// This situation is extremely problematic, since it may be possible to cast two different votes in the same term, if for +/// instance, changes to the `voted_for` field cannot be persisted. For now, we're just going to have the controller panic +/// whenever there's an IO error when persisting cluster state changes. Thankfully, this error should be fairly rare, since +/// the file that's used to persist state changes is opened during initialization. +static STATE_UPDATE_FAILED: &'static str = "failed to persist changes to cluster state! Panicking, since data consistency cannot be guaranteed under these circumstances"; + + pub trait ConsensusProcessor: Send { fn tick(&mut self, now: Instant, all_connections: &mut HashMap); fn is_primary(&self) -> bool; fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap); fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr); + fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote); } #[derive(Debug)] @@ -35,6 +44,9 @@ impl ConsensusProcessor for NoOpConsensusProcessor { } fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); + } + fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote) { + } fn tick(&mut self, now: Instant, all_connections: &mut HashMap) { } @@ -55,7 +67,7 @@ pub struct ClusterManager { persistent: FilePersistedState, system_partition_primary_address: SystemPrimaryAddressRef, shared: Arc>, - connection_manager: PeerConnections, + connection_manager: Box, } #[derive(Debug, PartialEq, Copy, Clone)] @@ -74,20 +86,18 @@ impl ClusterManager { shared: Arc>, primary_status_writer: AtomicBoolWriter, system_partition_primary_address: SystemPrimaryAddressRef, - outgoing_connection_creator: Box) -> ClusterManager { + peer_connection_manager: Box) -> ClusterManager { let mut initialization_peers = starting_peer_addresses.iter().cloned().collect::>(); for peer in persistent.cluster_members.iter() { initialization_peers.insert(peer.address); } - let peer_connections = PeerConnections::new(starting_peer_addresses, outgoing_connection_creator, &persistent.cluster_members); - ClusterManager { state: State::EstablishConnections, election_timeout: Duration::from_millis(election_timeout_millis), last_heartbeat: Instant::now(), - connection_manager: peer_connections, + connection_manager: peer_connection_manager, initialization_peers, last_applied: 0, last_applied_term: 0, @@ -140,20 +150,24 @@ impl ClusterManager { state.current_term += 1; let my_id = state.this_instance_id; state.voted_for = Some(my_id); - }); - if let Err(io_err) = result { - error!("Aborting new election due to error persisting state: {:?}", io_err); - return; - } + }).expect(STATE_UPDATE_FAILED); + let connection_control = ConnectionControl::SendRequestVote(CallRequestVote { term: self.persistent.current_term, candidate_id: self.persistent.this_instance_id, last_log_index: self.last_applied, last_log_term: self.last_applied_term, }); - self.connection_manager.broadcast(connection_control); + self.connection_manager.broadcast_to_peers(connection_control); } + fn can_grant_vote(&self, request: &CallRequestVote) -> bool { + (self.persistent.voted_for.is_none() || self.persistent.voted_for == Some(request.candidate_id)) && + self.persistent.current_term <= request.term && + self.last_applied <= request.last_log_index && + self.last_applied_term <= request.last_log_term && + self.persistent.cluster_members.iter().any(|peer| peer.id == request.candidate_id) + } } @@ -163,6 +177,29 @@ impl ConsensusProcessor for ClusterManager { self.connection_resolved(address); } + fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote) { + let candidate_id = request.candidate_id; + + let response = if self.can_grant_vote(&request) { + self.persistent.modify(|state| { + state.voted_for = Some(candidate_id); + state.current_term = request.term; + }).expect(STATE_UPDATE_FAILED); + + VoteResponse { + term: self.persistent.current_term, + granted: true + } + } else { + VoteResponse { + term: self.persistent.current_term, + granted: false, + } + }; + + self.connection_manager.send_to_peer(candidate_id, ConnectionControl::SendVoteResponse(response)); + } + fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap) { debug!("peer_connection_established: connection_id: {}, {:?}", connection_id, upgrade); let connection = all_connections.get(&connection_id) @@ -203,7 +240,7 @@ pub struct SharedClusterState { pub this_instance_id: FloInstanceId, pub this_address: Option, pub system_primary: Option, - pub peers: Vec, + pub peers: HashSet, } impl SharedClusterState { @@ -212,7 +249,7 @@ impl SharedClusterState { this_instance_id: FloInstanceId::generate_new(), this_address: None, system_primary: None, - peers: Vec::new(), + peers: HashSet::new(), } } } @@ -231,6 +268,7 @@ pub fn init_cluster_consensus_processor(persistent_state: FilePersistedState, let ClusterOptions{ peer_addresses, event_loop_handles, election_timeout_millis, .. } = options; let outgoing_connection_creator = OutgoingConnectionCreatorImpl::new(event_loop_handles, engine_ref); + let peer_connection_manager = PeerConnections::new(peer_addresses.clone(), Box::new(outgoing_connection_creator), &persistent_state.cluster_members); let state = ClusterManager::new(election_timeout_millis, peer_addresses, @@ -238,7 +276,7 @@ pub fn init_cluster_consensus_processor(persistent_state: FilePersistedState, shared_state_ref, system_primary, primary_address, - Box::new(outgoing_connection_creator)); + Box::new(peer_connection_manager)); Box::new(state) } @@ -250,17 +288,24 @@ mod test { use tempdir::TempDir; use engine::connection_handler::{ConnectionControlSender, ConnectionControlReceiver, ConnectionControl}; use engine::controller::peer_connection::{OutgoingConnectionCreator, MockOutgoingConnectionCreator}; - use test_utils::addr; + use test_utils::{addr, expect_future_resolved}; + use engine::controller::cluster_state::peer_connections::mock::{MockPeerConnectionManager, Invocation}; + use engine::controller::controller_messages::mock::mock_connection_ref; fn t(start: Instant, seconds: u64) -> Instant { start + Duration::from_secs(seconds) } - #[test] - fn cluster_manager_starts_new_election_after_timeout_elapses() { + struct VoteExpectation { + term: Term, + persistent_voted_for: Option, + granted: bool, + } + + fn vote_test(setup_fun: F) where F: Fn(&mut ClusterManager, &CallRequestVote, Peer, Peer) -> VoteExpectation { let start = Instant::now(); let temp_dir = TempDir::new("cluster_state_test").unwrap(); - let mut mock_creator = MockOutgoingConnectionCreator::new(); + let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { id: FloInstanceId::generate_new(), address: addr("111.222.0.1:3000") @@ -269,39 +314,213 @@ mod test { id: FloInstanceId::generate_new(), address: addr("111.222.0.2:3000") }; - let (peer_1_connection, _) = mock_creator.stub(peer_1.address); - let (peer_2_connection, _) = mock_creator.stub(peer_2.address); - let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), mock_creator.boxed()); - assert_eq!(0, subject.persistent.current_term); + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + subject.state = State::Follower; + + let request_vote = CallRequestVote { + term: 8, + candidate_id: peer_1.id, + last_log_index: 9, + last_log_term: 7, + }; + let expectation = setup_fun(&mut subject, &request_vote, peer_1.clone(), peer_2.clone()); - let mut connections = HashMap::new(); - subject.tick(t(start, 1), &mut connections); + subject.request_vote_received(678, request_vote); + + assert_eq!(expectation.persistent_voted_for, subject.persistent.voted_for); + assert_eq!(expectation.term, subject.persistent.current_term); - let upgrade_1 = PeerUpgrade { + connection_manager.verify_in_order(&Invocation::SendToPeer { peer_id: peer_1.id, - system_primary: Some(peer_2.clone()), - cluster_members: vec![peer_2.clone()], + connection_control: ConnectionControl::SendVoteResponse(VoteResponse { + term: expectation.term, + granted: expectation.granted, + }) + }); + + } + + #[test] + fn vote_is_granted_when_candidate_term_and_log_are_more_up_to_date() { + vote_test(|subject, request, peer_1, peer_2| { + subject.last_applied_term = request.last_log_term - 1; + subject.last_applied = request.last_log_index - 2; + subject.persistent.modify(|state| { + state.current_term = request.term - 1; + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + }).unwrap(); + + VoteExpectation { + term: request.term, + persistent_voted_for: Some(peer_1.id), + granted: true + } + }); + } + + #[test] + fn vote_is_denied_when_candidate_term_is_less_than_current_term() { + vote_test(|subject, request, peer_1, peer_2| { + subject.last_applied_term = request.last_log_term; + subject.last_applied = request.last_log_index; + subject.persistent.modify(|state| { + state.current_term = request.term + 1; // my current term is greater + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + }).unwrap(); + + VoteExpectation { + term: request.term + 1, + persistent_voted_for: None, + granted: false + } + }); + } + + #[test] + fn vote_is_denied_when_candidate_log_term_is_out_of_date() { + vote_test(|subject, request, peer_1, peer_2| { + subject.last_applied_term = request.last_log_term + 1; + // unless we've really screwed something up, the last_log_index should never be the same if the last_log_term is different. + // We're doing it this way in the test just to document the behavior in this case + subject.last_applied = request.last_log_index; + subject.persistent.modify(|state| { + state.current_term = request.term; + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + }).unwrap(); + + VoteExpectation { + term: request.term, + persistent_voted_for: None, + granted: false + } + }); + } + + #[test] + fn vote_is_denied_when_candidate_is_not_a_known_peer() { + vote_test(|subject, request, peer_1, peer_2| { + subject.last_applied_term = request.last_log_term; + subject.last_applied = request.last_log_index; + subject.persistent.modify(|state| { + state.current_term = request.term - 1; + // peer_1 is not a known member + state.cluster_members.insert(peer_2.clone()); + }).unwrap(); + + VoteExpectation { + term: request.term - 1, + persistent_voted_for: None, + granted: false + } + }); + } + + #[test] + fn vote_is_denied_when_candidate_log_is_out_of_date() { + vote_test(|subject, request, peer_1, peer_2| { + subject.last_applied_term = request.last_log_term; + subject.last_applied = request.last_log_index + 1; + subject.persistent.modify(|state| { + state.voted_for = None; + state.current_term = request.term; + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + }).unwrap(); + + VoteExpectation { + term: request.term, + persistent_voted_for: None, + granted: false, + } + }); + } + + #[test] + fn vote_is_denied_when_a_vote_was_already_cast_for_another_member_this_term() { + vote_test(|subject, request, peer_1, peer_2| { + subject.last_applied_term = request.last_log_term; + subject.last_applied = request.last_log_index; + subject.persistent.modify(|state| { + state.voted_for = Some(peer_2.id); // already voted for peer 2 + state.current_term = request.term; + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + }).unwrap(); + + VoteExpectation { + term: request.term, + persistent_voted_for: Some(peer_2.id), + granted: false, + } + }); + } + + #[test] + fn vote_is_granted_when_no_other_vote_was_granted_and_candidate_log_exactly_matches() { + vote_test(|subject, request, peer_1, peer_2| { + subject.last_applied_term = request.last_log_term; + subject.last_applied = request.last_log_index; + subject.persistent.modify(|state| { + state.current_term = request.term - 1; + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + }).unwrap(); + + VoteExpectation { + term: request.term, + persistent_voted_for: Some(peer_1.id), + granted: true + } + }); + } + + #[test] + fn cluster_manager_starts_new_election_after_timeout_elapses() { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let connection_manager = MockPeerConnectionManager::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:3000") }; - subject.peer_connection_established(upgrade_1, peer_1_connection.connection_id, &connections); - let upgrade_2 = PeerUpgrade { - peer_id: peer_2.id, - system_primary: Some(peer_2.clone()), - cluster_members: vec![peer_2.clone()], + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:3000") }; - subject.peer_connection_established(upgrade_2, peer_2_connection.connection_id, &connections); + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + subject.state = State::Follower; + subject.last_applied_term = 7; + subject.last_applied = 9; + subject.persistent.modify(|state| { + state.current_term = 7; + }).unwrap(); - subject.tick(t(start, 2), &mut connections); + let mut connections = HashMap::new(); + subject.tick(t(start, 1), &mut connections); + connection_manager.verify_in_order(&Invocation::EstablishConnections); let this_id = subject.persistent.this_instance_id; assert_eq!(Some(this_id), subject.persistent.voted_for); - assert_eq!(1, subject.persistent.current_term); + assert_eq!(8, subject.persistent.current_term); + + connection_manager.verify_in_order(&Invocation::BroadcastToPeers { + connection_control: ConnectionControl::SendRequestVote(CallRequestVote { + term: 8, + candidate_id: this_id, + last_log_index: 9, + last_log_term: 7, + }) + }); } #[test] fn cluster_manager_moves_to_follower_state_once_peer_announce_is_received_with_known_primary() { let start = Instant::now(); let temp_dir = TempDir::new("cluster_state_test").unwrap(); - let mut mock_creator = MockOutgoingConnectionCreator::new(); + let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { id: FloInstanceId::generate_new(), address: addr("111.222.0.1:3000") @@ -310,27 +529,34 @@ mod test { id: FloInstanceId::generate_new(), address: addr("111.222.0.2:3000") }; - let (peer_1_connection, _) = mock_creator.stub(peer_1.address); - let _ = mock_creator.stub(peer_2.address); - let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), mock_creator.boxed()); + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); assert_eq!(State::EstablishConnections, subject.state); let mut connections = HashMap::new(); subject.tick(t(start, 1), &mut connections); assert_eq!(State::DeterminePrimary, subject.state); + connection_manager.verify_in_order(&Invocation::EstablishConnections); + let conn_id = 5; let upgrade = PeerUpgrade { peer_id: peer_1.id, system_primary: Some(peer_2.clone()), cluster_members: vec![peer_2.clone()], }; - subject.peer_connection_established(upgrade, peer_1_connection.connection_id, &connections); + let (connection, _) = mock_connection_ref(conn_id, peer_1.address); + connections.insert(conn_id, connection.clone()); + + subject.peer_connection_established(upgrade, conn_id, &connections); + connection_manager.verify_in_order(&Invocation::PeerConnectionEstablished { + peer_id: peer_1.id, + success_connection: connection, + }); assert_eq!(State::Follower, subject.state); let actual_primary: Option = { subject.system_partition_primary_address.read().unwrap().as_ref().cloned() }; - + assert_eq!(Some(peer_2.address), actual_primary); } #[test] @@ -338,30 +564,27 @@ mod test { let start = Instant::now(); let temp_dir = TempDir::new("cluster_state_test").unwrap(); - let mut mock_creator = MockOutgoingConnectionCreator::new(); + let connection_manager = MockPeerConnectionManager::new(); let peer_1_addr = addr("111.222.0.1:3000"); let peer_2_addr = addr("111.222.0.2:3000"); - let _ = mock_creator.stub(peer_1_addr); - let _ = mock_creator.stub(peer_2_addr); - let mut subject = create_cluster_manager(vec![peer_1_addr, peer_2_addr], temp_dir.path(), mock_creator.boxed()); + let mut subject = create_cluster_manager(vec![peer_1_addr, peer_2_addr], temp_dir.path(), connection_manager.boxed_ref()); assert_eq!(State::EstablishConnections, subject.state); let mut connections = HashMap::new(); subject.tick(t(start, 1), &mut connections); + connection_manager.verify_in_order(&Invocation::EstablishConnections); assert_eq!(State::DeterminePrimary, subject.state); - assert_eq!(2, connections.len()); - let actual = connections.values().map(|cr| cr.remote_address).collect::>(); - - assert!(actual.contains(&peer_1_addr)); - assert!(actual.contains(&peer_2_addr)); - - // make sure it doesn't happen again - subject.tick(t(start, 5), &mut connections); - assert_eq!(2, connections.len()); - subject.outgoing_connection_failed(1, peer_1_addr); + connection_manager.verify_in_order(&Invocation::OutgoingConnectionFailed { + connection_id: 1, + addr: peer_1_addr, + }); subject.outgoing_connection_failed(2, peer_2_addr); + connection_manager.verify_in_order(&Invocation::OutgoingConnectionFailed { + connection_id: 2, + addr: peer_2_addr, + }); assert_eq!(State::Follower, subject.state); } @@ -369,7 +592,7 @@ mod test { addr("123.1.2.3:4567") } - fn create_cluster_manager(starting_peers: Vec, temp_dir: &Path, conn_creator: Box) -> ClusterManager { + fn create_cluster_manager(starting_peers: Vec, temp_dir: &Path, conn_manager: Box) -> ClusterManager { let temp_file = temp_dir.join("cluster_state"); let file_state = FilePersistedState::initialize(temp_file).expect("failed to init persistent state"); @@ -381,6 +604,6 @@ mod test { Arc::new(RwLock::new(shared)), AtomicBoolWriter::with_value(false), Arc::new(RwLock::new(None)), - conn_creator) + conn_manager) } } diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs index 9a0a037..1a0d924 100644 --- a/flo-server/src/engine/controller/cluster_state/peer_connections.rs +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -1,6 +1,7 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; use std::time::{Instant, Duration}; +use std::fmt::Debug; use event::EventCounter; use protocol::{FloInstanceId, Term}; @@ -9,6 +10,15 @@ use engine::controller::{ConnectionRef, Peer, CallRequestVote}; use engine::controller::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator}; use engine::connection_handler::ConnectionControl; +pub trait PeerConnectionManager: Send + Debug + 'static { + fn establish_connections(&mut self, now: Instant, all_connections: &mut HashMap); + fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, addr: SocketAddr); + fn connection_closed(&mut self, connection_id: ConnectionId); + fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef); + fn broadcast_to_peers(&mut self, connection_control: ConnectionControl); + fn send_to_peer(&mut self, peer_id: FloInstanceId, connection_control: ConnectionControl); +} + #[derive(Debug)] pub struct PeerConnections { disconnected_peers: HashMap, @@ -18,7 +28,7 @@ pub struct PeerConnections { } impl PeerConnections { - pub fn new(starting_peer_addresses: Vec, outgoing_connection_creator: Box, peers: &[Peer]) -> PeerConnections { + pub fn new(starting_peer_addresses: Vec, outgoing_connection_creator: Box, peers: &HashSet) -> PeerConnections { let known_peers = peers.iter().map(|peer| { let connection = Connection::new(peer.address); (peer.id, connection) @@ -43,7 +53,15 @@ impl PeerConnections { } } - pub fn establish_connections(&mut self, now: Instant, all_connections: &mut HashMap) { + + fn connection_peer_id(&self, connection_id: ConnectionId) -> Option { + self.active_connections.get(&connection_id).cloned() + } +} + +impl PeerConnectionManager for PeerConnections { + + fn establish_connections(&mut self, now: Instant, all_connections: &mut HashMap) { let PeerConnections {ref mut disconnected_peers, ref mut outgoing_connection_creator, ..} = *self; for (address, attempt) in disconnected_peers.iter_mut() { if attempt.should_try_connect(now) { @@ -58,7 +76,7 @@ impl PeerConnections { } } - pub fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { + fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { let PeerConnections {ref mut disconnected_peers, ref mut known_peers, ref mut outgoing_connection_creator, ..} = *self; if let Some(attempt) = disconnected_peers.get_mut(&address) { // remove the connection @@ -73,7 +91,7 @@ impl PeerConnections { } } - pub fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef) { + fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef) { let disconnected = self.disconnected_peers.remove(&success_connection.remote_address); let success_connection_id = success_connection.connection_id; @@ -100,7 +118,7 @@ impl PeerConnections { self.active_connections.insert(success_connection_id, peer_id); } - pub fn connection_closed(&mut self, connection_id: ConnectionId) { + fn connection_closed(&mut self, connection_id: ConnectionId) { let address: Option = self.active_connections.remove(&connection_id).and_then(|peer_id| { self.known_peers.get_mut(&peer_id).map(|connection| { connection.state = PeerState::ConnectionFailed; @@ -118,11 +136,7 @@ impl PeerConnections { } } - pub fn connection_peer_id(&self, connection_id: ConnectionId) -> Option { - self.active_connections.get(&connection_id).cloned() - } - - pub fn broadcast(&mut self, control: ConnectionControl) { + fn broadcast_to_peers(&mut self, control: ConnectionControl) { debug!("Broadcasting {:?}", control); for (peer, connection) in self.known_peers.iter() { match connection.state { @@ -136,6 +150,9 @@ impl PeerConnections { } } + fn send_to_peer(&mut self, peer_id: FloInstanceId, control: ConnectionControl) { + unimplemented!() + } } fn send(connection: &ConnectionRef, control: ConnectionControl) { @@ -218,7 +235,7 @@ mod test { } fn subject_with_connected_peers(peers: &[Peer], creator: MockOutgoingConnectionCreator) -> (PeerConnections, HashMap) { - let mut subject = PeerConnections::new(Vec::new(), creator.boxed(), peers); + let mut subject = PeerConnections::new(Vec::new(), creator.boxed(), &peers.iter().cloned().collect()); let mut connections = HashMap::new(); subject.establish_connections(Instant::now(), &mut connections); for peer in peers { @@ -241,7 +258,7 @@ mod test { let mut creator = MockOutgoingConnectionCreator::new(); let (peer_1_conn, rx_1) = creator.stub(peer_1.address); let (peer_2_conn, rx_2) = creator.stub(peer_2.address); - let (mut subject, mut connections) = subject_with_connected_peers(&[peer_1, peer_2], creator); + let (mut subject, connections) = subject_with_connected_peers(&[peer_1, peer_2], creator); let rx_1 = assert_control_sent(rx_1, &ConnectionControl::InitiateOutgoingSystemConnection); let rx_2 = assert_control_sent(rx_2, &ConnectionControl::InitiateOutgoingSystemConnection); @@ -252,7 +269,7 @@ mod test { last_log_index: 99, last_log_term: 6, }); - subject.broadcast(expected.clone()); + subject.broadcast_to_peers(expected.clone()); assert_control_sent(rx_1, &expected); assert_control_sent(rx_2, &expected); @@ -266,7 +283,7 @@ mod test { let mut creator = MockOutgoingConnectionCreator::new(); let (peer_connection, rx) = creator.stub(peer_address); - let mut subject = PeerConnections::new(vec![peer_address], creator.boxed(), &[]); + let mut subject = PeerConnections::new(vec![peer_address], creator.boxed(), &HashSet::new()); let mut connections = HashMap::new(); let time = Instant::now(); @@ -291,7 +308,9 @@ mod test { }; let mut creator = MockOutgoingConnectionCreator::new(); creator.stub(peer_address); - let subject = PeerConnections::new(Vec::new(), creator.boxed(), &[peer.clone()]); + let mut all_peers = HashSet::new(); + all_peers.insert(peer.clone()); + let subject = PeerConnections::new(Vec::new(), creator.boxed(), &all_peers); assert!(subject.disconnected_peers.contains_key(&peer_address)); } @@ -302,7 +321,7 @@ mod test { let new_peers = vec![peer_address]; let mut creator = MockOutgoingConnectionCreator::new(); creator.stub(peer_address); - let mut subject = PeerConnections::new(new_peers, creator.boxed(), &[]); + let mut subject = PeerConnections::new(new_peers, creator.boxed(), &HashSet::new()); let mut connections = HashMap::new(); let time = Instant::now(); @@ -370,4 +389,90 @@ mod test { }; assert!(!attempt.should_try_connect(Instant::now())); } + +} + +#[cfg(test)] +pub mod mock { + use super::*; + use std::sync::{Arc, Mutex}; + use std::collections::VecDeque; + + #[derive(Debug, Clone)] + pub struct MockPeerConnectionManager { + actual_invocations: Arc>>, + } + + impl MockPeerConnectionManager { + pub fn new() -> MockPeerConnectionManager { + MockPeerConnectionManager { + actual_invocations: Arc::new(Mutex::new(VecDeque::new())) + } + } + + pub fn verify_in_order(&self, expected: &Invocation) { + // lock will be poisoned if this panics + let mut lock = self.actual_invocations.lock().unwrap(); + + let missing_invocation_message = format!("Expected invocation: {:?}, but no calls were made on this mock", expected); + let next_invocation = lock.pop_front().expect(&missing_invocation_message); + if expected != &next_invocation { + panic!("Expected: {:?}, but actual was: {:?}. Other invocations were: {:?}", expected, next_invocation, ::std::ops::Deref::deref(&lock)); + } + } + + pub fn boxed_ref(&self) -> Box { + Box::new(self.clone()) + } + + fn push_invocation(&self, invocation: Invocation) { + let mut lock = self.actual_invocations.lock().unwrap(); + lock.push_back(invocation); + } + } + + impl PeerConnectionManager for MockPeerConnectionManager { + fn establish_connections(&mut self, _now: Instant, _all_connections: &mut HashMap) { + self.push_invocation(Invocation::EstablishConnections); + } + fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, addr: SocketAddr) { + self.push_invocation(Invocation::OutgoingConnectionFailed {connection_id, addr}); + } + fn connection_closed(&mut self, connection_id: ConnectionId) { + self.push_invocation(Invocation::ConnectionClosed {connection_id}); + } + fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef) { + self.push_invocation(Invocation::PeerConnectionEstablished {peer_id, success_connection: success_connection.clone()}); + } + fn broadcast_to_peers(&mut self, connection_control: ConnectionControl) { + self.push_invocation(Invocation::BroadcastToPeers {connection_control}); + } + fn send_to_peer(&mut self, peer_id: FloInstanceId, connection_control: ConnectionControl) { + self.push_invocation(Invocation::SendToPeer {peer_id, connection_control}); + } + } + + + + #[derive(Debug, PartialEq, Clone)] + pub enum Invocation { + EstablishConnections, + OutgoingConnectionFailed{ + connection_id: ConnectionId, + addr: SocketAddr, + }, + ConnectionClosed{ connection_id: ConnectionId }, + PeerConnectionEstablished{peer_id: FloInstanceId, success_connection: ConnectionRef}, + BroadcastToPeers{connection_control: ConnectionControl}, + SendToPeer{peer_id: FloInstanceId, connection_control :ConnectionControl}, + } + + /* + fn establish_connections(&mut self, now: Instant, all_connections: &mut HashMap); + fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, addr: SocketAddr); + fn connection_closed(&mut self, connection_id: ConnectionId); + fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef); + fn broadcast_to_peers(&mut self, connection_control: ConnectionControl); + fn send_to_peer(&mut self, peer_id: FloInstanceId, connection_control: ConnectionControl); + */ } diff --git a/flo-server/src/engine/controller/cluster_state/persistent.rs b/flo-server/src/engine/controller/cluster_state/persistent.rs index 885abd1..ada33f4 100644 --- a/flo-server/src/engine/controller/cluster_state/persistent.rs +++ b/flo-server/src/engine/controller/cluster_state/persistent.rs @@ -2,6 +2,7 @@ use std::net::SocketAddr; use std::path::PathBuf; use std::fs::{File, OpenOptions}; use std::io::{self, Seek, SeekFrom}; +use std::collections::HashSet; use protocol::{FloInstanceId, Term}; use engine::controller::controller_messages::Peer; @@ -17,7 +18,7 @@ pub struct PersistentClusterState { pub voted_for: Option, #[serde(with = "InstanceIdRemote")] pub this_instance_id: FloInstanceId, - pub cluster_members: Vec, + pub cluster_members: HashSet, } impl PersistentClusterState { @@ -92,7 +93,7 @@ impl FilePersistedState { current_term: 0, voted_for: None, this_instance_id: FloInstanceId::generate_new(), - cluster_members: Vec::new(), + cluster_members: HashSet::new(), }; info!("Initialized brand new state: {:?}", state); (file, state) @@ -158,7 +159,7 @@ mod test { let mut subject = FilePersistedState::initialize(path.clone()).unwrap(); subject.modify(|state| { state.current_term = 9; - state.cluster_members = vec![ + state.cluster_members = [ Peer { id: FloInstanceId::generate_new(), address: addr("127.0.0.1:3456") @@ -171,7 +172,7 @@ mod test { id: FloInstanceId::generate_new(), address: addr("127.0.0.1:456") } - ]; + ].iter().cloned().collect(); }).unwrap(); let subject2 = FilePersistedState::initialize(path.clone()).unwrap(); @@ -179,8 +180,7 @@ mod test { // remove some data here. This will cause the second init to fail if we don't truncate the file subject.modify(|state| { - state.cluster_members.pop(); - state.cluster_members.pop(); + state.cluster_members.clear(); }).unwrap(); let subject2 = FilePersistedState::initialize(path.clone()).unwrap(); diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 1f9ca7e..6fd1ea0 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -135,3 +135,19 @@ impl From for SystemOperation { } } } + +#[cfg(test)] +pub mod mock { + use super::*; + use engine::connection_handler::{create_connection_control_channels, ConnectionControlReceiver}; + + pub fn mock_connection_ref(connection_id: ConnectionId, addr: SocketAddr) -> (ConnectionRef, ConnectionControlReceiver) { + let (tx, rx) = create_connection_control_channels(); + let conn = ConnectionRef { + connection_id, + remote_address: addr, + control_sender: tx, + }; + (conn, rx) + } +} From da6c05eb68a00b29ca4492732c2cfafb5220dd04 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 31 Dec 2017 10:21:34 -0500 Subject: [PATCH 34/73] add doc comments to PartitionImpl fields so I remember wtf I was thinking --- .../event_stream/partition/controller/mod.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index 700d6e6..94cd63a 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -22,15 +22,29 @@ use self::consumer_manager::ConsumerManager; const FIRST_SEGMENT_NUM: SegmentNum = SegmentNum(1); pub struct PartitionImpl { + /// The name of the event stream that this partition is a member of. This is just here to make debugging _way_ easier. event_stream_name: String, + /// The partition number within this event stream partition_num: ActorId, + /// The directory used to store everything for this partition partition_dir: PathBuf, + /// The maximum size in bytes for any segment. This value may be exceeded when the size of a single event is larger than + /// the `max_segment_size`. In this case, you'll end up with a segment that includes just that one event max_segment_size: usize, + /// the maximum duration of any segment. Helps control the size of segments when there's relatively low frequency of events + /// added and a short TTL for events max_segment_duration: Duration, + /// The segments that make up this partition segments: VecDeque, + /// A simple index that maps `EventCounter`s to a tuple of segment number and file offset index: PartitionIndex, + /// Shared EventCounter for all partitions in the event stream. Serves as a Lamport clock to help reason about relative + /// order of events across multiple partitions. Used to generate new `EventCounter`s when events are appended event_stream_highest_counter: HighestCounter, + /// Tracks the highest committed event in this partition. This value is shared with the `ConnectionHandler`s partition_highest_committed: AtomicCounterWriter, + /// Whether this instance is the primary for this partition. This value is set by `FloController`, since it requires + /// consensus to modify which instance is primary for a partition. primary: AtomicBoolReader, /// new segments each have a reader added here. The readers are then accessed as needed by the EventReader From 89e2a5626fddd2b6308fecddd6b6379c576a9f42 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 31 Dec 2017 11:13:54 -0500 Subject: [PATCH 35/73] minor test refactor, factor out function for determining the minimum required votes for majority --- .../engine/controller/cluster_state/mod.rs | 125 +++++++++--------- .../partition/controller/commit_manager.rs | 14 +- flo-server/src/engine/mod.rs | 65 ++++++++- 3 files changed, 128 insertions(+), 76 deletions(-) diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index defa6e1..2a69503 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -292,68 +292,20 @@ mod test { use engine::controller::cluster_state::peer_connections::mock::{MockPeerConnectionManager, Invocation}; use engine::controller::controller_messages::mock::mock_connection_ref; - fn t(start: Instant, seconds: u64) -> Instant { - start + Duration::from_secs(seconds) - } - - struct VoteExpectation { - term: Term, - persistent_voted_for: Option, - granted: bool, - } - - fn vote_test(setup_fun: F) where F: Fn(&mut ClusterManager, &CallRequestVote, Peer, Peer) -> VoteExpectation { - let start = Instant::now(); - let temp_dir = TempDir::new("cluster_state_test").unwrap(); - let connection_manager = MockPeerConnectionManager::new(); - let peer_1 = Peer { - id: FloInstanceId::generate_new(), - address: addr("111.222.0.1:3000") - }; - let peer_2 = Peer { - id: FloInstanceId::generate_new(), - address: addr("111.222.0.2:3000") - }; - let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); - subject.state = State::Follower; - - let request_vote = CallRequestVote { - term: 8, - candidate_id: peer_1.id, - last_log_index: 9, - last_log_term: 7, - }; - let expectation = setup_fun(&mut subject, &request_vote, peer_1.clone(), peer_2.clone()); - - subject.request_vote_received(678, request_vote); - - assert_eq!(expectation.persistent_voted_for, subject.persistent.voted_for); - assert_eq!(expectation.term, subject.persistent.current_term); - - connection_manager.verify_in_order(&Invocation::SendToPeer { - peer_id: peer_1.id, - connection_control: ConnectionControl::SendVoteResponse(VoteResponse { - term: expectation.term, - granted: expectation.granted, - }) - }); - - } - #[test] fn vote_is_granted_when_candidate_term_and_log_are_more_up_to_date() { - vote_test(|subject, request, peer_1, peer_2| { + vote_test(|subject, request, candidate, peer_2| { subject.last_applied_term = request.last_log_term - 1; subject.last_applied = request.last_log_index - 2; subject.persistent.modify(|state| { state.current_term = request.term - 1; - state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(candidate.clone()); state.cluster_members.insert(peer_2.clone()); }).unwrap(); VoteExpectation { term: request.term, - persistent_voted_for: Some(peer_1.id), + persistent_voted_for: Some(candidate.id), granted: true } }); @@ -361,12 +313,12 @@ mod test { #[test] fn vote_is_denied_when_candidate_term_is_less_than_current_term() { - vote_test(|subject, request, peer_1, peer_2| { + vote_test(|subject, request, candidate, peer_2| { subject.last_applied_term = request.last_log_term; subject.last_applied = request.last_log_index; subject.persistent.modify(|state| { state.current_term = request.term + 1; // my current term is greater - state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(candidate.clone()); state.cluster_members.insert(peer_2.clone()); }).unwrap(); @@ -380,14 +332,14 @@ mod test { #[test] fn vote_is_denied_when_candidate_log_term_is_out_of_date() { - vote_test(|subject, request, peer_1, peer_2| { + vote_test(|subject, request, candidate, peer_2| { subject.last_applied_term = request.last_log_term + 1; // unless we've really screwed something up, the last_log_index should never be the same if the last_log_term is different. // We're doing it this way in the test just to document the behavior in this case subject.last_applied = request.last_log_index; subject.persistent.modify(|state| { state.current_term = request.term; - state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(candidate.clone()); state.cluster_members.insert(peer_2.clone()); }).unwrap(); @@ -401,7 +353,7 @@ mod test { #[test] fn vote_is_denied_when_candidate_is_not_a_known_peer() { - vote_test(|subject, request, peer_1, peer_2| { + vote_test(|subject, request, candidate, peer_2| { subject.last_applied_term = request.last_log_term; subject.last_applied = request.last_log_index; subject.persistent.modify(|state| { @@ -420,13 +372,13 @@ mod test { #[test] fn vote_is_denied_when_candidate_log_is_out_of_date() { - vote_test(|subject, request, peer_1, peer_2| { + vote_test(|subject, request, candidate, peer_2| { subject.last_applied_term = request.last_log_term; subject.last_applied = request.last_log_index + 1; subject.persistent.modify(|state| { state.voted_for = None; state.current_term = request.term; - state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(candidate.clone()); state.cluster_members.insert(peer_2.clone()); }).unwrap(); @@ -440,13 +392,13 @@ mod test { #[test] fn vote_is_denied_when_a_vote_was_already_cast_for_another_member_this_term() { - vote_test(|subject, request, peer_1, peer_2| { + vote_test(|subject, request, candidate, peer_2| { subject.last_applied_term = request.last_log_term; subject.last_applied = request.last_log_index; subject.persistent.modify(|state| { state.voted_for = Some(peer_2.id); // already voted for peer 2 state.current_term = request.term; - state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(candidate.clone()); state.cluster_members.insert(peer_2.clone()); }).unwrap(); @@ -460,18 +412,18 @@ mod test { #[test] fn vote_is_granted_when_no_other_vote_was_granted_and_candidate_log_exactly_matches() { - vote_test(|subject, request, peer_1, peer_2| { + vote_test(|subject, request, candidate, peer_2| { subject.last_applied_term = request.last_log_term; subject.last_applied = request.last_log_index; subject.persistent.modify(|state| { state.current_term = request.term - 1; - state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(candidate.clone()); state.cluster_members.insert(peer_2.clone()); }).unwrap(); VoteExpectation { term: request.term, - persistent_voted_for: Some(peer_1.id), + persistent_voted_for: Some(candidate.id), granted: true } }); @@ -606,4 +558,51 @@ mod test { Arc::new(RwLock::new(None)), conn_manager) } + + fn t(start: Instant, seconds: u64) -> Instant { + start + Duration::from_secs(seconds) + } + + struct VoteExpectation { + term: Term, + persistent_voted_for: Option, + granted: bool, + } + + fn vote_test(setup_fun: F) where F: Fn(&mut ClusterManager, &CallRequestVote, Peer, Peer) -> VoteExpectation { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let connection_manager = MockPeerConnectionManager::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:3000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:3000") + }; + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + subject.state = State::Follower; + + let request_vote = CallRequestVote { + term: 8, + candidate_id: peer_1.id, + last_log_index: 9, + last_log_term: 7, + }; + let expectation = setup_fun(&mut subject, &request_vote, peer_1.clone(), peer_2.clone()); + + subject.request_vote_received(678, request_vote); + + assert_eq!(expectation.persistent_voted_for, subject.persistent.voted_for); + assert_eq!(expectation.term, subject.persistent.current_term); + + connection_manager.verify_in_order(&Invocation::SendToPeer { + peer_id: peer_1.id, + connection_control: ConnectionControl::SendVoteResponse(VoteResponse { + term: expectation.term, + granted: expectation.granted, + }) + }); + } } diff --git a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs index 2829745..48698b0 100644 --- a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs +++ b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs @@ -41,7 +41,7 @@ impl CommitManager { self.peers.sort_by_key(|elem| elem.1); let idx = self.min_required_for_commit; - let ack_counter = self.peers[idx as usize].1; + let mut ack_counter = self.peers[idx as usize].1; if self.commit_index.set_if_greater(ack_counter as usize) { Some(ack_counter) @@ -67,17 +67,7 @@ impl CommitManager { fn compute_min_required(&self) -> ActorId { let peer_count = self.peers.len() as ActorId; - match peer_count { - 0 => 0, - 1 => 1, - 2 => 1, - other @ _ if other % 2 == 0 => { - other / 2 - } - other @ _ => { - (other / 2) + 1 - } - } + ::engine::minimum_required_votes_for_majority(peer_count) } } diff --git a/flo-server/src/engine/mod.rs b/flo-server/src/engine/mod.rs index 181bcbb..d538ce9 100644 --- a/flo-server/src/engine/mod.rs +++ b/flo-server/src/engine/mod.rs @@ -8,7 +8,7 @@ use std::sync::atomic::{AtomicUsize}; use std::net::SocketAddr; use protocol::ProtocolMessage; -use event::OwnedFloEvent; +use event::{OwnedFloEvent, ActorId}; use self::event_stream::EventStreamRef; pub use self::controller::{ControllerOptions, ClusterOptions, SystemStreamRef, start_controller}; @@ -37,6 +37,23 @@ pub fn system_stream_name() -> String { SYSTEM_STREAM_NAME.to_owned() } +/// Returns the minimum number of votes required to achieve a majority. Takes as input the number of _other_ peers in the +/// cluster, not including this instance. Returns the number of votes required from _other_ peers, again not including the +/// implicit vote from this instance +fn minimum_required_votes_for_majority(number_of_other_peers: ActorId) -> ActorId { + match number_of_other_peers { + 0 => 0, + 1 => 1, + 2 => 1, + other @ _ if other % 2 == 0 => { + other / 2 + } + other @ _ => { + (other / 2) + 1 + } + } +} + #[derive(Clone, Debug)] pub struct EngineRef { current_connection_id: Arc, @@ -93,5 +110,51 @@ impl EngineRef { } +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn minimum_required_votes_for_majority_returns_0_when_there_are_no_other_peers() { + // no-cluster mode, so no other votes are required + let result = minimum_required_votes_for_majority(0); + assert_eq!(0, result); + } + + #[test] + fn minimum_required_votes_for_majority_returns_1_when_there_is_1_other_peer() { + // total cluster size is only 2, so they must both agree + let result = minimum_required_votes_for_majority(1); + assert_eq!(1, result); + } + + #[test] + fn minimum_required_votes_for_majority_returns_1_when_there_are_2_other_peers() { + // total cluster size is 3, so one vote from another peer makes for 2/3 + let result = minimum_required_votes_for_majority(2); + assert_eq!(1, result); + } + + #[test] + fn minimum_required_votes_for_majority_returns_2_when_there_are_3_other_peers() { + // total cluster size is 4, so 2 votes makes for 3/4 + let result = minimum_required_votes_for_majority(3); + assert_eq!(2, result); + } + + #[test] + fn minimum_required_votes_for_majority_returns_2_when_there_are_4_other_peers() { + // total cluster size is 5, so 2 votes makes for 3/5 + let result = minimum_required_votes_for_majority(4); + assert_eq!(2, result); + } + + #[test] + fn minimum_required_votes_for_majority_returns_4_when_there_are_7_other_peers() { + // total cluster size is 8, so 4 votes makes for 5/8 + let result = minimum_required_votes_for_majority(7); + assert_eq!(4, result); + } +} From bbbd66dfcc0f90d52e0b24b8dd810d89cb41bf60 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 31 Dec 2017 19:24:24 -0500 Subject: [PATCH 36/73] start handling vote responses in flo controller --- .../engine/controller/cluster_state/mod.rs | 306 +++++++++++++++++- .../cluster_state/peer_connections.rs | 41 ++- 2 files changed, 314 insertions(+), 33 deletions(-) diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 2a69503..6d3ecd0 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -8,7 +8,7 @@ use std::time::{Instant, Duration}; use std::path::Path; use std::collections::{HashMap, HashSet}; -use event::EventCounter; +use event::{EventCounter, ActorId}; use engine::{ConnectionId, EngineRef}; use engine::connection_handler::ConnectionControl; use engine::controller::{CallRequestVote, VoteResponse}; @@ -34,6 +34,7 @@ pub trait ConsensusProcessor: Send { fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap); fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr); fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote); + fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse); } #[derive(Debug)] @@ -53,18 +54,23 @@ impl ConsensusProcessor for NoOpConsensusProcessor { fn is_primary(&self) -> bool { true } + fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse) { + + } } #[derive(Debug)] pub struct ClusterManager { state: State, initialization_peers: HashSet, + this_instance_address: SocketAddr, election_timeout: Duration, last_heartbeat: Instant, primary_status_writer: AtomicBoolWriter, last_applied: EventCounter, last_applied_term: Term, persistent: FilePersistedState, + votes_received: HashSet, system_partition_primary_address: SystemPrimaryAddressRef, shared: Arc>, connection_manager: Box, @@ -82,6 +88,7 @@ enum State { impl ClusterManager { fn new(election_timeout_millis: u64, starting_peer_addresses: Vec, + this_instance_address: SocketAddr, persistent: FilePersistedState, shared: Arc>, primary_status_writer: AtomicBoolWriter, @@ -98,6 +105,8 @@ impl ClusterManager { election_timeout: Duration::from_millis(election_timeout_millis), last_heartbeat: Instant::now(), connection_manager: peer_connection_manager, + votes_received: HashSet::new(), + this_instance_address, initialization_peers, last_applied: 0, last_applied_term: 0, @@ -145,7 +154,8 @@ impl ClusterManager { } fn start_new_election(&mut self) { - info!("Starting new election"); + info!("Starting new election with term: {}", self.persistent.current_term); + self.votes_received.clear(); let result = self.persistent.modify(|state| { state.current_term += 1; let my_id = state.this_instance_id; @@ -159,6 +169,7 @@ impl ClusterManager { last_log_term: self.last_applied_term, }); self.connection_manager.broadcast_to_peers(connection_control); + self.transition_state(State::Voted) } fn can_grant_vote(&self, request: &CallRequestVote) -> bool { @@ -169,6 +180,35 @@ impl ClusterManager { self.persistent.cluster_members.iter().any(|peer| peer.id == request.candidate_id) } + fn count_vote_response(&mut self, peer_id: FloInstanceId) -> bool { + if self.votes_received.insert(peer_id) { + trace!("Counting vote response from peer_id: {:?}", peer_id); + } else { + trace!("Not counting duplicate vote from peer_id: {:?}", peer_id); + } + let peer_count = self.persistent.cluster_members.len() as ActorId; + let vote_count = self.votes_received.len() as ActorId; + let required_count = ::engine::minimum_required_votes_for_majority(peer_count); + let election_won = vote_count >= required_count; + debug!("After counting vote from peer_id: {:?}, this instance has {} of {} required votes from a total of {} cluster members; election_won={}", + peer_id, vote_count, required_count, peer_count, election_won); + election_won + } + + fn transition_to_primary(&mut self) { + self.transition_state(State::Primary); + let this_peer = Peer { + id: self.persistent.this_instance_id, + address: self.this_instance_address, + }; + self.set_new_primary(Some(this_peer)); + self.primary_status_writer.set(true); + } + + fn set_new_primary(&mut self, primary: Option) { + let mut lock = self.shared.write().unwrap(); + lock.system_primary = primary; + } } impl ConsensusProcessor for ClusterManager { @@ -186,20 +226,41 @@ impl ConsensusProcessor for ClusterManager { state.current_term = request.term; }).expect(STATE_UPDATE_FAILED); - VoteResponse { - term: self.persistent.current_term, - granted: true - } + VoteResponse { term: self.persistent.current_term, granted: true } } else { - VoteResponse { - term: self.persistent.current_term, - granted: false, - } + VoteResponse { term: self.persistent.current_term, granted: false, } }; self.connection_manager.send_to_peer(candidate_id, ConnectionControl::SendVoteResponse(response)); } + fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse) { + let peer_id = self.connection_manager.get_peer_id(from); + if peer_id.is_none() { + error!("Ignoring Vote Response from connection_id: {}: {:?}", from, response); + return; + } + let peer_id = peer_id.unwrap(); + + if response.granted { + if self.count_vote_response(peer_id) { + self.transition_to_primary(); + } + } else { + let response_term = response.term; + debug!("VoteResponse from {:?} was not granted, response term: {}, current_term: {}", peer_id, response_term, self.persistent.current_term); + if response_term > self.persistent.current_term { + info!("Received response term of {}, which is greater than current term of: {}. Updating current term and clearing out {} existing votes", + response_term, self.persistent.current_term, self.votes_received.len()); + self.persistent.modify(|state| { + state.current_term = response_term; + }).expect(STATE_UPDATE_FAILED); + self.votes_received.clear(); + self.transition_state(State::Follower); + } + } + } + fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap) { debug!("peer_connection_established: connection_id: {}, {:?}", connection_id, upgrade); let connection = all_connections.get(&connection_id) @@ -207,7 +268,7 @@ impl ConsensusProcessor for ClusterManager { self.connection_manager.peer_connection_established(upgrade.peer_id, connection); - if let State::DeterminePrimary = self.state { + if State::DeterminePrimary == self.state { self.determine_primary_from_peer_upgrade(upgrade); } self.connection_resolved(connection.remote_address); @@ -252,6 +313,13 @@ impl SharedClusterState { peers: HashSet::new(), } } + + pub fn this_instance_is_primary(&self) -> bool { + let this_id = self.this_instance_id; + self.system_primary.as_ref().map(|primary| { + primary.id == this_id + }).unwrap_or(false) + } } @@ -265,13 +333,14 @@ pub fn init_cluster_consensus_processor(persistent_state: FilePersistedState, system_primary: AtomicBoolWriter, primary_address: SystemPrimaryAddressRef) -> Box { - let ClusterOptions{ peer_addresses, event_loop_handles, election_timeout_millis, .. } = options; + let ClusterOptions{ peer_addresses, event_loop_handles, election_timeout_millis, this_instance_address, .. } = options; let outgoing_connection_creator = OutgoingConnectionCreatorImpl::new(event_loop_handles, engine_ref); let peer_connection_manager = PeerConnections::new(peer_addresses.clone(), Box::new(outgoing_connection_creator), &persistent_state.cluster_members); let state = ClusterManager::new(election_timeout_millis, peer_addresses, + this_instance_address, persistent_state, shared_state_ref, system_primary, @@ -292,6 +361,167 @@ mod test { use engine::controller::cluster_state::peer_connections::mock::{MockPeerConnectionManager, Invocation}; use engine::controller::controller_messages::mock::mock_connection_ref; + // TODO: test for ensuring vote reponse is ignored when it is from an unknown peer + + #[test] + fn election_is_aborted_when_vote_response_contains_term_greater_than_current() { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let connection_manager = MockPeerConnectionManager::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:1000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:2000") + }; + let peer_3 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:3000") + }; + let peer_4 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:4000") + }; + let peer_1_connection = 1; + let peer_2_connection = 2; + let peer_3_connection = 3; + let peer_4_connection = 4; + connection_manager.stub_peer_connection(peer_1_connection, peer_1.id); + connection_manager.stub_peer_connection(peer_2_connection, peer_2.id); + connection_manager.stub_peer_connection(peer_3_connection, peer_3.id); + connection_manager.stub_peer_connection(peer_4_connection, peer_4.id); + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + + subject.persistent.modify(|state| { + let this_id = state.this_instance_id; + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + state.cluster_members.insert(peer_3.clone()); + state.cluster_members.insert(peer_4.clone()); + state.current_term = 5; + }).unwrap(); + subject.state = State::Voted; + subject.votes_received.insert(peer_1.id); // simulate one vote having been received already + + subject.vote_response_received(t_millis(start, 5), peer_2_connection, VoteResponse { + term: 7, + granted: false, + }); + assert!(subject.votes_received.is_empty()); + assert_eq!(7, subject.persistent.current_term); + assert_eq!(State::Follower, subject.state); + } + + #[test] + fn vote_response_is_not_counted_when_granted_is_false() { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let connection_manager = MockPeerConnectionManager::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:1000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:2000") + }; + let peer_1_connection = 1; + let peer_2_connection = 2; + connection_manager.stub_peer_connection(peer_1_connection, peer_1.id); + connection_manager.stub_peer_connection(peer_2_connection, peer_2.id); + + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + + subject.persistent.modify(|state| { + let this_id = state.this_instance_id; + state.voted_for = Some(this_id); + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + state.current_term = 5; + }).unwrap(); + subject.state = State::Voted; + subject.last_heartbeat = start; + + assert!(subject.votes_received.is_empty()); + subject.vote_response_received(t_millis(start, 4), peer_1_connection, VoteResponse{ + term: 7, + granted: false + }); + assert!(subject.votes_received.is_empty()); + assert_eq!(7, subject.persistent.current_term); + } + + #[test] + fn primary_status_is_set_after_a_majority_of_votes_are_granted_before_the_election_timeout() { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let connection_manager = MockPeerConnectionManager::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:1000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:2000") + }; + let peer_3 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:3000") + }; + let peer_4 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:4000") + }; + let peer_1_connection = 1; + let peer_2_connection = 2; + let peer_3_connection = 3; + let peer_4_connection = 4; + connection_manager.stub_peer_connection(peer_1_connection, peer_1.id); + connection_manager.stub_peer_connection(peer_2_connection, peer_2.id); + connection_manager.stub_peer_connection(peer_3_connection, peer_3.id); + connection_manager.stub_peer_connection(peer_4_connection, peer_4.id); + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + + subject.persistent.modify(|state| { + let this_id = state.this_instance_id; + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + state.cluster_members.insert(peer_3.clone()); + state.cluster_members.insert(peer_4.clone()); + state.current_term = 5; + }).unwrap(); + subject.state = State::Follower; + subject.last_applied_term = 5; + subject.last_applied = 9; + + let mut connections = HashMap::new(); + let election_start = t_sec(start, 1); + subject.tick(election_start, &mut connections); + connection_manager.verify_in_order(&Invocation::EstablishConnections); + connection_manager.verify_in_order(&Invocation::BroadcastToPeers { + connection_control: ConnectionControl::SendRequestVote(CallRequestVote { + term: 6, + candidate_id: subject.persistent.this_instance_id, + last_log_index: subject.last_applied, + last_log_term: subject.last_applied_term, + }), + }); + + subject.vote_response_received(t_millis(election_start, 3), peer_1_connection, VoteResponse { + term: 6, granted: true + }); + assert_eq!(State::Voted, subject.state); + subject.vote_response_received(t_millis(election_start, 3), peer_2_connection, VoteResponse { + term: 6, granted: true + }); + assert_eq!(State::Primary, subject.state); + assert!(subject.primary_status_writer.reader().get_relaxed()); + let shared = subject.shared.read().unwrap(); + assert!(shared.this_instance_is_primary()); + } + #[test] fn vote_is_granted_when_candidate_term_and_log_are_more_up_to_date() { vote_test(|subject, request, candidate, peer_2| { @@ -451,7 +681,7 @@ mod test { }).unwrap(); let mut connections = HashMap::new(); - subject.tick(t(start, 1), &mut connections); + subject.tick(t_sec(start, 1), &mut connections); connection_manager.verify_in_order(&Invocation::EstablishConnections); let this_id = subject.persistent.this_instance_id; @@ -485,7 +715,7 @@ mod test { assert_eq!(State::EstablishConnections, subject.state); let mut connections = HashMap::new(); - subject.tick(t(start, 1), &mut connections); + subject.tick(t_sec(start, 1), &mut connections); assert_eq!(State::DeterminePrimary, subject.state); connection_manager.verify_in_order(&Invocation::EstablishConnections); @@ -523,7 +753,7 @@ mod test { assert_eq!(State::EstablishConnections, subject.state); let mut connections = HashMap::new(); - subject.tick(t(start, 1), &mut connections); + subject.tick(t_sec(start, 1), &mut connections); connection_manager.verify_in_order(&Invocation::EstablishConnections); assert_eq!(State::DeterminePrimary, subject.state); @@ -540,6 +770,45 @@ mod test { assert_eq!(State::Follower, subject.state); } + #[test] + fn shared_state_this_instance_is_primary_returns_false_when_system_primary_does_not_match() { + let this_id = FloInstanceId::generate_new(); + let this_addr = addr("127.0.0.1:3000"); + let subject = SharedClusterState { + this_instance_id: this_id, + this_address: Some(this_addr), + system_primary: Some(Peer {id: FloInstanceId::generate_new(), address: this_addr}), + peers: HashSet::new(), + }; + assert!(!subject.this_instance_is_primary()); + } + + #[test] + fn shared_state_this_instance_is_primary_returns_false_when_system_primary_is_none() { + let this_id = FloInstanceId::generate_new(); + let this_addr = addr("127.0.0.1:3000"); + let subject = SharedClusterState { + this_instance_id: this_id, + this_address: Some(this_addr), + system_primary: None, + peers: HashSet::new(), + }; + assert!(!subject.this_instance_is_primary()); + } + + #[test] + fn shared_state_this_instance_is_primary_returns_true_when_this_instance_id_matches_primary() { + let this_id = FloInstanceId::generate_new(); + let this_addr = addr("127.0.0.1:3000"); + let subject = SharedClusterState { + this_instance_id: this_id, + this_address: Some(this_addr), + system_primary: Some(Peer {id: this_id, address: this_addr}), + peers: HashSet::new(), + }; + assert!(subject.this_instance_is_primary()); + } + fn this_instance_addr() -> SocketAddr { addr("123.1.2.3:4567") } @@ -552,6 +821,7 @@ mod test { ClusterManager::new(150, starting_peers, + this_instance_addr(), file_state, Arc::new(RwLock::new(shared)), AtomicBoolWriter::with_value(false), @@ -559,10 +829,14 @@ mod test { conn_manager) } - fn t(start: Instant, seconds: u64) -> Instant { + fn t_sec(start: Instant, seconds: u64) -> Instant { start + Duration::from_secs(seconds) } + fn t_millis(start: Instant, millis: u64) -> Instant { + start + Duration::from_millis(millis) + } + struct VoteExpectation { term: Term, persistent_voted_for: Option, diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs index 1a0d924..b36053b 100644 --- a/flo-server/src/engine/controller/cluster_state/peer_connections.rs +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -17,6 +17,7 @@ pub trait PeerConnectionManager: Send + Debug + 'static { fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef); fn broadcast_to_peers(&mut self, connection_control: ConnectionControl); fn send_to_peer(&mut self, peer_id: FloInstanceId, connection_control: ConnectionControl); + fn get_peer_id(&mut self, connection_id: ConnectionId) -> Option; } #[derive(Debug)] @@ -52,11 +53,6 @@ impl PeerConnections { outgoing_connection_creator, } } - - - fn connection_peer_id(&self, connection_id: ConnectionId) -> Option { - self.active_connections.get(&connection_id).cloned() - } } impl PeerConnectionManager for PeerConnections { @@ -153,6 +149,10 @@ impl PeerConnectionManager for PeerConnections { fn send_to_peer(&mut self, peer_id: FloInstanceId, control: ConnectionControl) { unimplemented!() } + + fn get_peer_id(&mut self, connection_id: ConnectionId) -> Option { + self.active_connections.get(&connection_id).cloned() + } } fn send(connection: &ConnectionRef, control: ConnectionControl) { @@ -291,12 +291,12 @@ mod test { subject.peer_connection_established(peer_id, &peer_connection); - assert_eq!(Some(peer_id), subject.connection_peer_id(peer_connection.connection_id)); + assert_eq!(Some(peer_id), subject.get_peer_id(peer_connection.connection_id)); subject.connection_closed(peer_connection.connection_id); assert!(subject.disconnected_peers.contains_key(&peer_address)); - assert_eq!(None, subject.connection_peer_id(peer_connection.connection_id)); + assert_eq!(None, subject.get_peer_id(peer_connection.connection_id)); } #[test] @@ -401,15 +401,22 @@ pub mod mock { #[derive(Debug, Clone)] pub struct MockPeerConnectionManager { actual_invocations: Arc>>, + peer_stubs: Arc>>, } impl MockPeerConnectionManager { pub fn new() -> MockPeerConnectionManager { MockPeerConnectionManager { - actual_invocations: Arc::new(Mutex::new(VecDeque::new())) + actual_invocations: Arc::new(Mutex::new(VecDeque::new())), + peer_stubs: Arc::new(Mutex::new(HashMap::new())), } } + pub fn stub_peer_connection(&self, connection_id: ConnectionId, peer_id: FloInstanceId) { + let mut lock = self.peer_stubs.lock().unwrap(); + lock.insert(connection_id, peer_id); + } + pub fn verify_in_order(&self, expected: &Invocation) { // lock will be poisoned if this panics let mut lock = self.actual_invocations.lock().unwrap(); @@ -437,12 +444,17 @@ pub mod mock { } fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, addr: SocketAddr) { self.push_invocation(Invocation::OutgoingConnectionFailed {connection_id, addr}); + let mut lock = self.peer_stubs.lock().unwrap(); + lock.remove(&connection_id); } fn connection_closed(&mut self, connection_id: ConnectionId) { self.push_invocation(Invocation::ConnectionClosed {connection_id}); + let mut lock = self.peer_stubs.lock().unwrap(); + lock.remove(&connection_id); } fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef) { self.push_invocation(Invocation::PeerConnectionEstablished {peer_id, success_connection: success_connection.clone()}); + self.stub_peer_connection(success_connection.connection_id, peer_id); } fn broadcast_to_peers(&mut self, connection_control: ConnectionControl) { self.push_invocation(Invocation::BroadcastToPeers {connection_control}); @@ -450,6 +462,10 @@ pub mod mock { fn send_to_peer(&mut self, peer_id: FloInstanceId, connection_control: ConnectionControl) { self.push_invocation(Invocation::SendToPeer {peer_id, connection_control}); } + fn get_peer_id(&mut self, connection_id: ConnectionId) -> Option { + let lock = self.peer_stubs.lock().unwrap(); + lock.get(&connection_id).cloned() + } } @@ -466,13 +482,4 @@ pub mod mock { BroadcastToPeers{connection_control: ConnectionControl}, SendToPeer{peer_id: FloInstanceId, connection_control :ConnectionControl}, } - - /* - fn establish_connections(&mut self, now: Instant, all_connections: &mut HashMap); - fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, addr: SocketAddr); - fn connection_closed(&mut self, connection_id: ConnectionId); - fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef); - fn broadcast_to_peers(&mut self, connection_control: ConnectionControl); - fn send_to_peer(&mut self, peer_id: FloInstanceId, connection_control: ConnectionControl); - */ } From 3a8331eff54b565e459fd0529fe5e4fd2d076877 Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 2 Jan 2018 10:06:47 -0500 Subject: [PATCH 37/73] ignore vote responses from unknown peers, restart election on timeout --- .../engine/controller/cluster_state/mod.rs | 100 +++++++++++++++++- 1 file changed, 96 insertions(+), 4 deletions(-) diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 6d3ecd0..a20812d 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -154,7 +154,7 @@ impl ClusterManager { } fn start_new_election(&mut self) { - info!("Starting new election with term: {}", self.persistent.current_term); + info!("Starting new election with term: {}", self.persistent.current_term + 1); self.votes_received.clear(); let result = self.persistent.modify(|state| { state.current_term += 1; @@ -282,12 +282,17 @@ impl ConsensusProcessor for ClusterManager { // we'll only be in this initial status once, on startup self.transition_state(State::DeterminePrimary); } - State::Follower => { + State::DeterminePrimary => { + trace!("Waiting to DeterminePrimary"); + } + State::Primary => { + trace!("This instance is primary"); + } + other @ _ => { if self.election_timed_out(now) { self.start_new_election(); } } - _ => { } } } @@ -361,7 +366,94 @@ mod test { use engine::controller::cluster_state::peer_connections::mock::{MockPeerConnectionManager, Invocation}; use engine::controller::controller_messages::mock::mock_connection_ref; - // TODO: test for ensuring vote reponse is ignored when it is from an unknown peer + #[test] + fn new_election_is_started_on_tick_when_current_election_goes_beyond_timeout() { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let connection_manager = MockPeerConnectionManager::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:1000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:2000") + }; + let peer_1_connection = 1; + let peer_2_connection = 2; + connection_manager.stub_peer_connection(peer_1_connection, peer_1.id); + connection_manager.stub_peer_connection(peer_2_connection, peer_2.id); + + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + + subject.persistent.modify(|state| { + let this_id = state.this_instance_id; + state.voted_for = Some(this_id); + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + state.current_term = 5; + }).unwrap(); + subject.last_applied = 99; + subject.last_applied_term = 4; + subject.state = State::Voted; + subject.last_heartbeat = start; + + let mut connections = HashMap::new(); + subject.tick(t_sec(start, 1), &mut connections); + + connection_manager.verify_in_order(&Invocation::EstablishConnections); + connection_manager.verify_in_order(&Invocation::BroadcastToPeers { + connection_control: ConnectionControl::SendRequestVote(CallRequestVote { + term: 6, + candidate_id: subject.persistent.this_instance_id, + last_log_index: 99, + last_log_term: 4, + }) + }); + + assert_eq!(6, subject.persistent.current_term); + assert_eq!(Some(subject.persistent.this_instance_id), subject.persistent.voted_for); + assert_eq!(State::Voted, subject.state); + } + + #[test] + fn vote_response_is_ignored_when_it_is_from_an_unknown_peer() { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let connection_manager = MockPeerConnectionManager::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:1000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:2000") + }; + let peer_1_connection = 1; + let peer_2_connection = 2; + let unknown_connection = 3; + connection_manager.stub_peer_connection(peer_1_connection, peer_1.id); + connection_manager.stub_peer_connection(peer_2_connection, peer_2.id); + + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + + subject.persistent.modify(|state| { + let this_id = state.this_instance_id; + state.voted_for = Some(this_id); + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + state.current_term = 5; + }).unwrap(); + subject.state = State::Voted; + subject.last_heartbeat = start; + + assert!(subject.votes_received.is_empty()); + subject.vote_response_received(t_millis(start, 4), unknown_connection, VoteResponse{ + term: 5, + granted: true, + }); + assert!(subject.votes_received.is_empty()); + } #[test] fn election_is_aborted_when_vote_response_contains_term_greater_than_current() { From 3619ff14ae97f6f5b863877e07212129b9628eba Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 2 Jan 2018 12:27:27 -0500 Subject: [PATCH 38/73] connection handler sends out append entries --- .../src/engine/connection_handler/input.rs | 8 ++- .../src/engine/connection_handler/mod.rs | 8 ++- .../src/engine/connection_handler/peer/mod.rs | 57 +++++++++++++++++-- .../src/engine/controller/initialization.rs | 5 +- flo-server/src/engine/controller/mod.rs | 2 + .../src/engine/controller/system_reader.rs | 47 +++++++++++++++ .../src/engine/controller/system_stream.rs | 16 ++++-- .../partition/controller/commit_manager.rs | 2 +- .../event_stream/partition/controller/mod.rs | 8 ++- .../partition/event_reader/mod.rs | 15 +++++ .../src/engine/event_stream/partition/mod.rs | 12 ++++ 11 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 flo-server/src/engine/controller/system_reader.rs diff --git a/flo-server/src/engine/connection_handler/input.rs b/flo-server/src/engine/connection_handler/input.rs index c2318a4..1e71635 100644 --- a/flo-server/src/engine/connection_handler/input.rs +++ b/flo-server/src/engine/connection_handler/input.rs @@ -3,6 +3,7 @@ use protocol::{FloInstanceId, Term}; use event::EventCounter; use engine::ReceivedProtocolMessage; use engine::controller::{CallRequestVote, VoteResponse}; +use engine::event_stream::partition::SegmentNum; #[derive(Debug)] pub enum ConnectionHandlerInput { @@ -28,6 +29,7 @@ pub enum ConnectionControl { InitiateOutgoingSystemConnection, SendRequestVote(CallRequestVote), SendVoteResponse(VoteResponse), + SendAppendEntried(CallAppendEntries) } @@ -43,8 +45,12 @@ impl From for ConnectionHandlerInput { } } -pub struct SendAppendEntries { +#[derive(Debug, Clone, PartialEq)] +pub struct CallAppendEntries { pub current_term: Term, pub prev_entry_index: EventCounter, pub prev_entry_term: Term, + pub reader_start_offset: usize, + pub reader_start_segment: SegmentNum, + pub reader_start_event: EventCounter, } diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 2e1e024..5a601f9 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -18,7 +18,7 @@ use self::consumer::ConsumerConnectionState; use self::producer::ProducerConnectionState; use self::peer::PeerConnectionState; -pub use self::input::{ConnectionHandlerInput, ConnectionControl}; +pub use self::input::{ConnectionHandlerInput, ConnectionControl, CallAppendEntries}; pub type ConnectionControlSender = ::futures::sync::mpsc::UnboundedSender; pub type ConnectionControlReceiver = ::futures::sync::mpsc::UnboundedReceiver; @@ -67,6 +67,9 @@ impl ConnectionHandler { ConnectionControl::SendVoteResponse(response) => { peer_state.send_vote_response(response, common_state) } + ConnectionControl::SendAppendEntried(append) => { + peer_state.send_append_entries(append, common_state) + } _ => unimplemented!() } } @@ -254,7 +257,8 @@ mod test { system_primary: None, peers: HashSet::new(), }; - let system_stream = SystemStreamRef::new(part_ref, tx, Arc::new(RwLock::new(cluster_state))); + let readers = ::engine::event_stream::partition::SharedReaderRefs::empty(); + let system_stream = SystemStreamRef::new(part_ref, tx, Arc::new(RwLock::new(cluster_state)), readers); let streams = Arc::new(Mutex::new(HashMap::new())); let engine = EngineRef::new(system_stream, streams); diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 7ca54b9..8d103fe 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -1,11 +1,11 @@ mod peer_follower; use std::collections::VecDeque; -use protocol::{self, ProtocolMessage, PeerAnnounce, EventStreamStatus, ClusterMember, ErrorMessage, ErrorKind}; +use protocol::{self, ProtocolMessage, PeerAnnounce, EventStreamStatus, ClusterMember, ErrorMessage, ErrorKind, FloInstanceId}; use engine::{ReceivedProtocolMessage, ConnectionId}; -use engine::controller::{self, SystemStreamRef, Peer}; +use engine::controller::{self, SystemStreamRef, Peer, SystemStreamReader}; use super::connection_state::ConnectionState; -use super::ConnectionHandlerResult; +use super::{ConnectionHandlerResult, CallAppendEntries}; #[derive(Debug, Clone, Copy, PartialEq)] @@ -15,12 +15,14 @@ enum State { Peer, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug)] pub struct PeerConnectionState { state: State, current_op_id: u32, controller_operation_queue: VecDeque, peer_operation_queue: VecDeque, + system_partition_reader: Option, + this_instance_id: Option, } @@ -31,9 +33,56 @@ impl PeerConnectionState { current_op_id: 0, controller_operation_queue: VecDeque::new(), peer_operation_queue: VecDeque::new(), + system_partition_reader: None, + this_instance_id: None, } } + pub fn send_append_entries(&mut self, append: CallAppendEntries, state: &mut ConnectionState) -> ConnectionHandlerResult { + let connection_id = state.connection_id; + let op_id = self.next_op_id(); + let this_instance_id = self.get_this_instance_id(state); + let CallAppendEntries {current_term, prev_entry_index, prev_entry_term, + reader_start_offset, reader_start_segment, reader_start_event} = append; + + if self.system_partition_reader.is_none() { + let reader = state.get_system_stream().create_system_stream_reader(connection_id); + self.system_partition_reader = Some(reader); + } + + let reader = self.system_partition_reader.as_mut().unwrap(); + reader.set_to(reader_start_segment, reader_start_offset).map_err(|io_err| { + format!("Failed to set position of system stream reader for connection_id: {} - {:?}", connection_id ,io_err) + })?; // just give up + + let num_events = reader.fill_buffer().map_err(|io_err| { + format!("Failed to rea events for AppendEntries for connection_id: {}, error: {:?}", connection_id, io_err) + })?; // give up on error + + let append = protocol::AppendEntriesCall { + op_id, + leader_id: this_instance_id, + term: current_term, + prev_entry_term, + prev_entry_index, + leader_commit_index: 0, //TODO: figure out what our current commit_index is + entry_count: num_events as u32, + }; + state.send_to_client(ProtocolMessage::SystemAppendCall(append))?; + for event in reader.drain() { + state.send_to_client(ProtocolMessage::ReceiveEvent(event))?; + } + Ok(()) + } + + fn get_this_instance_id(&mut self, state: &mut ConnectionState) -> FloInstanceId { + if self.this_instance_id.is_none() { + let id = state.get_system_stream().with_cluster_state(|cluster_state| cluster_state.this_instance_id); + self.this_instance_id = Some(id); + } + self.this_instance_id.unwrap() + } + pub fn vote_response_received(&mut self, response: protocol::RequestVoteResponse, state: &mut ConnectionState) -> ConnectionHandlerResult { let connection_id = state.connection_id; let expected_op_id = self.peer_operation_queue.pop_back(); diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index be9596a..7dfff91 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -25,6 +25,7 @@ use engine::event_stream::{EventStreamRefMut, use engine::event_stream::partition::{PartitionSender, PartitionReceiver, PartitionRef, + SharedReaderRefs, create_partition_channels}; use engine::event_stream::partition::controller::PartitionImpl; use atomics::{AtomicBoolReader, AtomicBoolWriter, AtomicCounterReader}; @@ -88,6 +89,7 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul let cluster_state_ref = Arc::new(RwLock::new(shared_state)); let engine_ref = create_engine_ref(shared_stream_refs.clone(), + system_partition.get_shared_reader_refs(), system_highest_counter, system_primary_status_writer.reader(), system_primary_address.clone(), @@ -144,6 +146,7 @@ fn init_system_partition(storage_dir: &Path, system_primary_reader: AtomicBoolRe } fn create_engine_ref(shared_stream_refs: Arc>>, + system_reader_refs: SharedReaderRefs, system_highest_counter: AtomicCounterReader, system_primary_reader: AtomicBoolReader, system_primary_addr: Arc>>, @@ -157,7 +160,7 @@ fn create_engine_ref(shared_stream_refs: Arc, +} + +impl SystemStreamReader { + pub fn new(connection_id: ConnectionId, shared_refs: SharedReaderRefs, commit_index_reader: AtomicCounterReader) -> SystemStreamReader { + use engine::event_stream::partition::EventFilter; + let part = PartitionReader::new(connection_id, 0, EventFilter::All, None, shared_refs, commit_index_reader); + SystemStreamReader { + inner: part, + event_buffer: Vec::with_capacity(SYSTEM_READER_BATCH_SIZE), + } + } + + /// sets the reader to the given segment and offset if it's not already there + pub fn set_to(&mut self, segment: SegmentNum, offset: usize) -> io::Result<()> { + self.inner.set_to(segment, offset) + } + + pub fn fill_buffer(&mut self) -> io::Result { + while self.event_buffer.len() < SYSTEM_READER_BATCH_SIZE { + let next = self.inner.next(); + if let Some(next_result) = next { + let event = next_result?; // return if read failed + self.event_buffer.push(event); + } else { + break; + } + } + Ok(self.event_buffer.len()) + } + + pub fn drain(&mut self) -> Drain { + self.event_buffer.drain(..) + } +} diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index f247876..84692f9 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -1,12 +1,12 @@ use std::net::SocketAddr; use protocol::FloInstanceId; -use engine::event_stream::partition::{PartitionRef, Operation}; +use engine::event_stream::partition::{PartitionRef, Operation, SharedReaderRefs, PartitionReader, SegmentNum}; use engine::event_stream::EventStreamRef; -use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer, CallRequestVote, VoteResponse}; +use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer, CallRequestVote, VoteResponse, SystemStreamReader}; use engine::controller::cluster_state::{SharedClusterState, ClusterStateReader}; use engine::ConnectionId; -use atomics::AtomicBoolReader; +use atomics::{AtomicBoolReader, AtomicCounterReader}; #[derive(Clone, Debug)] @@ -14,18 +14,25 @@ pub struct SystemStreamRef { cluster_state_reader: ClusterStateReader, system_sender: SystemPartitionSender, inner: PartitionRef, + system_segment_readers: SharedReaderRefs, } impl SystemStreamRef { - pub fn new(partition_ref: PartitionRef, system_sender: SystemPartitionSender, cluster_state_reader: ClusterStateReader) -> SystemStreamRef { + pub fn new(partition_ref: PartitionRef, system_sender: SystemPartitionSender, cluster_state_reader: ClusterStateReader, readers: SharedReaderRefs) -> SystemStreamRef { SystemStreamRef { cluster_state_reader, system_sender, inner: partition_ref, + system_segment_readers: readers } } + pub fn create_system_stream_reader(&self, connection_id: ConnectionId) -> SystemStreamReader { + debug!("Creating new SystemStreamReader for connection_id: {}", connection_id); + SystemStreamReader::new(connection_id, self.system_segment_readers.clone(), self.inner.get_highest_counter_reader()) + } + pub fn with_cluster_state(&self, fun: F) -> T where F: Fn(&SharedClusterState) -> T { let state = self.cluster_state_reader.read().unwrap(); fun(&*state) @@ -88,3 +95,4 @@ impl SystemStreamRef { + diff --git a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs index 48698b0..34ff8fc 100644 --- a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs +++ b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs @@ -41,7 +41,7 @@ impl CommitManager { self.peers.sort_by_key(|elem| elem.1); let idx = self.min_required_for_commit; - let mut ack_counter = self.peers[idx as usize].1; + let ack_counter = self.peers[idx as usize].1; if self.commit_index.set_if_greater(ack_counter as usize) { Some(ack_counter) diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index 94cd63a..a06d06e 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -11,7 +11,7 @@ use chrono::{Duration}; use atomics::{AtomicCounterWriter, AtomicCounterReader, AtomicBoolReader}; use protocol::ProduceEvent; use event::{ActorId, FloEventId, EventCounter, FloEvent, Timestamp, time}; -use super::{SharedReaderRefsMut, Operation, OpType, ProduceOperation, ConsumeOperation, PartitionReader, EventFilter, SegmentNum}; +use super::{SharedReaderRefs, SharedReaderRefsMut, Operation, OpType, ProduceOperation, ConsumeOperation, PartitionReader, EventFilter, SegmentNum}; use super::segment::Segment; use super::index::{PartitionIndex, IndexEntry}; use engine::event_stream::{EventStreamOptions, HighestCounter}; @@ -302,6 +302,10 @@ impl PartitionImpl { Ok(()) } + pub fn get_shared_reader_refs(&self) -> SharedReaderRefs { + self.reader_refs.get_reader_refs() + } + fn current_segment_num(&self) -> SegmentNum { self.segments.front().map(|s| s.segment_num).unwrap_or(SegmentNum(0)) } @@ -322,7 +326,7 @@ impl PartitionImpl { fn create_reader(&mut self, connection_id: ConnectionId, filter: EventFilter, start_exclusive: EventCounter) -> PartitionReader { let current_segment_num = self.current_segment_num(); let index_entry: Option = self.index.get_next_entry(start_exclusive); - let readers = self.reader_refs.get_reader_refs(); + let readers = self.get_shared_reader_refs(); let current_segment = match index_entry { Some(entry) => { diff --git a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs index 163b849..d144cc3 100644 --- a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs +++ b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs @@ -78,6 +78,21 @@ impl PartitionReader { unimplemented!() } + pub fn set_to(&mut self, segment_num: SegmentNum, offset: usize) -> io::Result<()> { + if !segment_num.is_set() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "Cannot set PartitionReader segment to 0")); + } + if self.current_reader_segment_id() != segment_num.0 { + let segment = self.segment_readers_ref.get_segment(segment_num) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, format!("No segment exists for {:?}", segment_num)) + })?; // return early if segment does not exist + self.current_segment_reader = Some(segment); + } + self.current_segment_reader.as_mut().unwrap().set_offset(offset); + Ok(()) + } + fn should_skip(&self, result: &Option>) -> bool { if let Some(Ok(ref event)) = *result { !self.filter.matches(event) diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index 861a181..e548fc4 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -119,6 +119,7 @@ impl SharedReaderRefsMut { } } +#[derive(Clone)] pub struct SharedReaderRefs { inner: Arc>> } @@ -134,6 +135,13 @@ impl Debug for SharedReaderRefs { } impl SharedReaderRefs { + #[cfg(test)] + pub fn empty() -> SharedReaderRefs { + SharedReaderRefs { + inner: Arc::new(RwLock::new(VecDeque::new())) + } + } + pub fn get_next_segment(&self, previous: SegmentNum) -> Option { let locked = self.inner.read().unwrap(); locked.front().map(|r| r.segment_id).and_then(|front_segment| { @@ -222,6 +230,10 @@ impl PartitionRef { self.highest_event_counter.load_relaxed() as EventCounter } + pub fn get_highest_counter_reader(&self) -> AtomicCounterReader { + self.highest_event_counter.clone() + } + pub fn is_primary(&self) -> bool { self.primary.get_relaxed() } From cf6b8acbe62d3bcd20a347065afb4bd215e5477a Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 2 Jan 2018 18:28:06 -0500 Subject: [PATCH 39/73] test sending heartbeat from connection handler, add commit index to CallAppendEntries --- .../src/engine/connection_handler/input.rs | 3 +- .../src/engine/connection_handler/mod.rs | 27 +++++++++++++++++- .../src/engine/connection_handler/peer/mod.rs | 28 +++++++++++-------- .../src/engine/event_stream/partition/mod.rs | 7 +++++ 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/flo-server/src/engine/connection_handler/input.rs b/flo-server/src/engine/connection_handler/input.rs index 1e71635..d477e21 100644 --- a/flo-server/src/engine/connection_handler/input.rs +++ b/flo-server/src/engine/connection_handler/input.rs @@ -29,7 +29,7 @@ pub enum ConnectionControl { InitiateOutgoingSystemConnection, SendRequestVote(CallRequestVote), SendVoteResponse(VoteResponse), - SendAppendEntried(CallAppendEntries) + SendAppendEntries(CallAppendEntries) } @@ -53,4 +53,5 @@ pub struct CallAppendEntries { pub reader_start_offset: usize, pub reader_start_segment: SegmentNum, pub reader_start_event: EventCounter, + pub commit_index: EventCounter, } diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 5a601f9..a921f79 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -67,7 +67,7 @@ impl ConnectionHandler { ConnectionControl::SendVoteResponse(response) => { peer_state.send_vote_response(response, common_state) } - ConnectionControl::SendAppendEntried(append) => { + ConnectionControl::SendAppendEntries(append) => { peer_state.send_append_entries(append, common_state) } _ => unimplemented!() @@ -353,6 +353,31 @@ mod test { } } + #[test] + fn append_entries_is_sent_without_any_events() { + let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); + + subject.handle_control(ConnectionControl::SendAppendEntries(CallAppendEntries { + current_term: 4, + prev_entry_index: 0, + prev_entry_term: 0, + reader_start_offset: 0, + reader_start_segment: SegmentNum::new_unset(), + reader_start_event: 0, + commit_index: 987, // just to show that this is just a dumb value and not interpreted by the connection handler + })).unwrap(); + + let expected_id = fixture.instance_id; + fixture.assert_sent_to_client(ProtocolMessage::SystemAppendCall(AppendEntriesCall { + op_id: 2, + leader_id: expected_id, + term: 4, + prev_entry_term: 0, + prev_entry_index: 0, + leader_commit_index: 987, + entry_count: 0, + })); + } #[test] fn error_is_returned_when_vote_response_is_unexpected() { diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 8d103fe..8302ccc 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -43,21 +43,25 @@ impl PeerConnectionState { let op_id = self.next_op_id(); let this_instance_id = self.get_this_instance_id(state); let CallAppendEntries {current_term, prev_entry_index, prev_entry_term, - reader_start_offset, reader_start_segment, reader_start_event} = append; + reader_start_offset, reader_start_segment, reader_start_event, commit_index} = append; if self.system_partition_reader.is_none() { let reader = state.get_system_stream().create_system_stream_reader(connection_id); self.system_partition_reader = Some(reader); } - let reader = self.system_partition_reader.as_mut().unwrap(); - reader.set_to(reader_start_segment, reader_start_offset).map_err(|io_err| { - format!("Failed to set position of system stream reader for connection_id: {} - {:?}", connection_id ,io_err) - })?; // just give up - let num_events = reader.fill_buffer().map_err(|io_err| { - format!("Failed to rea events for AppendEntries for connection_id: {}, error: {:?}", connection_id, io_err) - })?; // give up on error + let num_events = if reader_start_segment.is_set() { + reader.set_to(reader_start_segment, reader_start_offset).map_err(|io_err| { + format!("Failed to set position of system stream reader for connection_id: {} - {:?}", connection_id ,io_err) + })?; // just give up + + reader.fill_buffer().map_err(|io_err| { + format!("Failed to rea events for AppendEntries for connection_id: {}, error: {:?}", connection_id, io_err) + })? // give up on error + } else { + 0 + }; let append = protocol::AppendEntriesCall { op_id, @@ -65,12 +69,14 @@ impl PeerConnectionState { term: current_term, prev_entry_term, prev_entry_index, - leader_commit_index: 0, //TODO: figure out what our current commit_index is + leader_commit_index: commit_index, entry_count: num_events as u32, }; state.send_to_client(ProtocolMessage::SystemAppendCall(append))?; - for event in reader.drain() { - state.send_to_client(ProtocolMessage::ReceiveEvent(event))?; + if num_events > 0 { + for event in reader.drain() { + state.send_to_client(ProtocolMessage::ReceiveEvent(event))?; + } } Ok(()) } diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index e548fc4..7113142 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -62,6 +62,13 @@ fn get_events_file(partition_dir: &Path, segment_num: SegmentNum) -> PathBuf { pub struct SegmentNum(u64); impl SegmentNum { + + /// Used to create SegmentNum(0) for testing purposes. Normally, creating such instances would only be possible from within the partition module + #[cfg(test)] + pub fn new_unset() -> SegmentNum { + SegmentNum(0) + } + /// returns true if this segment is non-zero pub fn is_set(&self) -> bool { self.0 > 0 From 2d0328db23441da3b123d910511fa0920d8a5124 Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 2 Jan 2018 23:22:12 -0500 Subject: [PATCH 40/73] handle receiving append entries including events --- .../src/engine/connection_handler/mod.rs | 166 +++++++++++++++--- .../src/engine/connection_handler/peer/mod.rs | 74 +++++++- .../engine/controller/controller_messages.rs | 16 +- flo-server/src/engine/controller/mod.rs | 2 +- .../src/engine/controller/system_stream.rs | 7 +- 5 files changed, 232 insertions(+), 33 deletions(-) diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index a921f79..97b253c 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -107,6 +107,12 @@ impl ConnectionHandler { ProtocolMessage::VoteResponse(response) => { peer_state.vote_response_received(response, common_state) } + ProtocolMessage::SystemAppendCall(append_entries) => { + peer_state.append_entries_received(append_entries, common_state) + } + ProtocolMessage::ReceiveEvent(event) => { + peer_state.event_received(event, common_state) + } _ => unimplemented!() } } @@ -184,12 +190,12 @@ mod test { use tokio_core::reactor::Core; use super::*; - use event::ActorId; + use event::{ActorId, OwnedFloEvent, FloEventId, time}; use engine::{SYSTEM_STREAM_NAME, system_stream_name}; use engine::event_stream::EventStreamRef; use engine::event_stream::partition::*; use engine::ClientReceiver; - use engine::controller::{SystemStreamRef, SharedClusterState, SystemOpType, SystemOperation, CallRequestVote, VoteResponse}; + use engine::controller::*; use atomics::{AtomicCounterWriter, AtomicBoolWriter}; use test_utils::addr; @@ -230,6 +236,11 @@ mod test { }; // get the response subject.handle_incoming_message(ProtocolMessage::PeerAnnounce(response)).unwrap(); + fixture.assert_sent_to_system_stream(SystemOpType::ConnectionUpgradeToPeer(PeerUpgrade { + peer_id, + system_primary: None, + cluster_members: Vec::new(), + })); (subject, fixture) } @@ -331,6 +342,12 @@ mod test { panic!("Expected system op: {:?}, but received: {:?}", expected, unequal_messages); } + fn assert_nothing_sent_to_system_stream(&self) { + if let Ok(message) = self.system_receiver.try_recv() { + panic!("Expected no messages to be received by the system controller, but received: {:?}", message); + } + } + fn assert_sent_to_client(&mut self, expected: ProtocolMessage) { use tokio_core::reactor::Timeout; use futures::future::Either; @@ -345,39 +362,130 @@ mod test { self.client_receiver = Some(receiver); assert_eq!(Some(expected), message) }, - Ok(Either::B(_)) => panic!("Timed out on recv with Ok"), + Ok(Either::B(_)) => panic!("No message was received before timeout expired"), Err(Either::A(_)) => panic!("Recv err attempting to recv next message, expected: {:?}", expected), Err(Either::B(_)) => panic!("Timout Err") } - } + } + + #[test] + fn receive_event_returns_error_when_no_event_was_expected() { + let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); + + let event = OwnedFloEvent { + id: FloEventId::new(0, 457), + timestamp: time::now(), + parent_id: None, + namespace: "/system/foo".to_owned(), + data: vec![1, 2, 3, 4, 5], + }; + let result = subject.handle_incoming_message(ProtocolMessage::ReceiveEvent(event.clone())); + assert!(result.is_err()); + + fixture.assert_sent_to_client(ProtocolMessage::Error(ErrorMessage { + op_id: 0, + kind: ErrorKind::InvalidPeerState, + description: "No event was expected".to_owned(), + })); + } + + #[test] + fn receiving_append_entries_with_multiple_entries_sends_append_entries_to_system_controller_once_all_events_are_received() { + let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); + + let sender_id = FloInstanceId::generate_new(); + let append = ProtocolMessage::SystemAppendCall(AppendEntriesCall { + op_id: 2, + leader_id: sender_id, + term: 4, + prev_entry_term: 4, + prev_entry_index: 456, + leader_commit_index: 458, + entry_count: 2, + }); + + subject.handle_incoming_message(append).unwrap(); + fixture.assert_nothing_sent_to_system_stream(); + let event1 = OwnedFloEvent { + id: FloEventId::new(0, 457), + timestamp: time::now(), + parent_id: None, + namespace: "/system/foo".to_owned(), + data: vec![1, 2, 3, 4, 5], + }; + subject.handle_incoming_message(ProtocolMessage::ReceiveEvent(event1.clone())).unwrap(); + fixture.assert_nothing_sent_to_system_stream(); + let event2 = OwnedFloEvent { + id: FloEventId::new(0, 457), + timestamp: time::now(), + parent_id: None, + namespace: "/system/foo".to_owned(), + data: vec![1, 2, 3, 4, 5], + }; + subject.handle_incoming_message(ProtocolMessage::ReceiveEvent(event2.clone())).unwrap(); + + let expected = SystemOpType::AppendEntriesReceived(ReceiveAppendEntries { + term: 4, + prev_entry_index: 456, + prev_entry_term: 4, + commit_index: 458, + events: vec![event1, event2], + }); + fixture.assert_sent_to_system_stream(expected); + } + + #[test] + fn receiving_append_entries_with_0_entries_sends_append_entries_to_system_controller() { + let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); + + let sender_id = FloInstanceId::generate_new(); + let append = ProtocolMessage::SystemAppendCall(AppendEntriesCall { + op_id: 2, + leader_id: sender_id, + term: 4, + prev_entry_term: 4, + prev_entry_index: 456, + leader_commit_index: 987, + entry_count: 0, + }); + + subject.handle_incoming_message(append).unwrap(); + fixture.assert_sent_to_system_stream(SystemOpType::AppendEntriesReceived(ReceiveAppendEntries { + term: 4, + prev_entry_index: 456, + prev_entry_term: 4, + commit_index: 987, + events: Vec::new(), + })); + } + + #[test] + fn append_entries_is_sent_without_any_events() { + let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); + + subject.handle_control(ConnectionControl::SendAppendEntries(CallAppendEntries { + current_term: 4, + prev_entry_index: 0, + prev_entry_term: 0, + reader_start_offset: 0, + reader_start_segment: SegmentNum::new_unset(), + reader_start_event: 0, + commit_index: 987, // just to show that this is just a dumb value and not interpreted by the connection handler + })).unwrap(); + + let expected_id = fixture.instance_id; + fixture.assert_sent_to_client(ProtocolMessage::SystemAppendCall(AppendEntriesCall { + op_id: 2, + leader_id: expected_id, + term: 4, + prev_entry_term: 0, + prev_entry_index: 0, + leader_commit_index: 987, + entry_count: 0, + })); } - #[test] - fn append_entries_is_sent_without_any_events() { - let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); - - subject.handle_control(ConnectionControl::SendAppendEntries(CallAppendEntries { - current_term: 4, - prev_entry_index: 0, - prev_entry_term: 0, - reader_start_offset: 0, - reader_start_segment: SegmentNum::new_unset(), - reader_start_event: 0, - commit_index: 987, // just to show that this is just a dumb value and not interpreted by the connection handler - })).unwrap(); - - let expected_id = fixture.instance_id; - fixture.assert_sent_to_client(ProtocolMessage::SystemAppendCall(AppendEntriesCall { - op_id: 2, - leader_id: expected_id, - term: 4, - prev_entry_term: 0, - prev_entry_index: 0, - leader_commit_index: 987, - entry_count: 0, - })); - } #[test] fn error_is_returned_when_vote_response_is_unexpected() { diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 8302ccc..4212add 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -1,6 +1,8 @@ mod peer_follower; use std::collections::VecDeque; + +use event::OwnedFloEvent; use protocol::{self, ProtocolMessage, PeerAnnounce, EventStreamStatus, ClusterMember, ErrorMessage, ErrorKind, FloInstanceId}; use engine::{ReceivedProtocolMessage, ConnectionId}; use engine::controller::{self, SystemStreamRef, Peer, SystemStreamReader}; @@ -23,6 +25,7 @@ pub struct PeerConnectionState { peer_operation_queue: VecDeque, system_partition_reader: Option, this_instance_id: Option, + in_progress_append: Option, } @@ -35,9 +38,78 @@ impl PeerConnectionState { peer_operation_queue: VecDeque::new(), system_partition_reader: None, this_instance_id: None, + in_progress_append: None, } } + pub fn append_entries_received(&mut self, append: protocol::AppendEntriesCall, connection: &mut ConnectionState) -> ConnectionHandlerResult { + self.ensure_peer_state(Some(append.op_id), connection)?; + + self.controller_operation_queue.push_back(append.op_id); + let event_count = append.entry_count; + let controller_message = controller::ReceiveAppendEntries { + term: append.term, + prev_entry_index: append.prev_entry_index, + prev_entry_term: append.prev_entry_term, + commit_index: append.leader_commit_index, + events: Vec::with_capacity(event_count as usize), + }; + + if event_count == 0 { + let connection_id = connection.connection_id; + connection.get_system_stream().append_entries_received(connection_id, controller_message); + } else { + self.in_progress_append = Some(controller_message); + } + Ok(()) + } + + pub fn event_received(&mut self, event: OwnedFloEvent, state: &mut ConnectionState) -> ConnectionHandlerResult { + let connection_id = state.connection_id; + self.ensure_peer_state(None, state)?; + if self.in_progress_append.is_none() { + let error_message = ErrorMessage { + op_id: 0, + kind: ErrorKind::InvalidPeerState, + description: "No event was expected".to_owned(), + }; + let _ = state.send_to_client(ProtocolMessage::Error(error_message)); + Err(format!("Received event for connection_id: {}, when none was expected", connection_id)) + } else { + let receive_complete = { + let in_progress = self.in_progress_append.as_mut().unwrap(); + in_progress.events.push(event); + in_progress.events.capacity() == in_progress.events.len() + }; + if receive_complete { + let append = self.in_progress_append.take().unwrap(); + state.get_system_stream().append_entries_received(connection_id, append); + } + Ok(()) + } + } + + fn ensure_peer_state(&self, op_id: Option, state: &mut ConnectionState) -> ConnectionHandlerResult { + if self.state != State::Peer { + let err = ErrorMessage { + op_id: op_id.unwrap_or(0), + kind: ErrorKind::InvalidPeerState, + description: "No PeerAnnounce message has been received so peer operations are invalid".to_owned(), + }; + // ignore whatever send error might be raised here, since we're returning an error anyway, which will close the connection + let _ = state.send_to_client(ProtocolMessage::Error(err)); + Err(format!("Expected connection to be in {:?} state, but was {:?}", State::Peer, self.state)) + } else { + Ok(()) + } + } + + fn get_remaining_expected_events(&self) -> usize { + self.in_progress_append.as_ref().map(|append| { + append.events.capacity() - append.events.len() + }).unwrap_or(0) + } + pub fn send_append_entries(&mut self, append: CallAppendEntries, state: &mut ConnectionState) -> ConnectionHandlerResult { let connection_id = state.connection_id; let op_id = self.next_op_id(); @@ -91,9 +163,9 @@ impl PeerConnectionState { pub fn vote_response_received(&mut self, response: protocol::RequestVoteResponse, state: &mut ConnectionState) -> ConnectionHandlerResult { let connection_id = state.connection_id; + self.ensure_peer_state(Some(response.op_id), state)?; let expected_op_id = self.peer_operation_queue.pop_back(); - if Some(response.op_id) == expected_op_id { let protocol::RequestVoteResponse { op_id, term, vote_granted } = response; let controller_message = controller::VoteResponse { diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 6fd1ea0..1868ab2 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -3,7 +3,7 @@ use std::time::Instant; use futures::sync::mpsc::UnboundedSender; -use event::EventCounter; +use event::{EventCounter, OwnedFloEvent}; use protocol::{FloInstanceId, Term}; use engine::event_stream::partition::{self, Operation}; use engine::connection_handler::{ConnectionControl, ConnectionControlSender}; @@ -37,6 +37,15 @@ impl PartialEq for ConnectionRef { } } +#[derive(Debug, Clone, PartialEq)] +pub struct ReceiveAppendEntries { + pub term: Term, + pub prev_entry_index: EventCounter, + pub prev_entry_term: Term, + pub commit_index: EventCounter, + pub events: Vec, +} + #[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)] pub struct Peer { #[serde(with = "InstanceIdRemote")] @@ -62,6 +71,7 @@ pub enum SystemOpType { OutgoingConnectionFailed(SocketAddr), RequestVote(CallRequestVote), VoteResponseReceived(VoteResponse), + AppendEntriesReceived(ReceiveAppendEntries) } impl SystemOpType { @@ -82,6 +92,10 @@ pub struct SystemOperation { impl SystemOperation { + pub fn append_entries_received(connection_id: ConnectionId, append: ReceiveAppendEntries) -> SystemOperation { + SystemOperation::new(connection_id, SystemOpType::AppendEntriesReceived(append)) + } + pub fn vote_response_received(connection_id: ConnectionId, response: VoteResponse) -> SystemOperation { SystemOperation::new(connection_id, SystemOpType::VoteResponseReceived(response)) } diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index db884b6..90acf41 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -26,7 +26,7 @@ use self::cluster_state::{ClusterManager, ConsensusProcessor}; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; pub use self::system_stream::SystemStreamRef; pub use self::system_reader::SystemStreamReader; -pub use self::controller_messages::{SystemOperation, SystemOpType, ConnectionRef, Peer, PeerUpgrade, CallRequestVote, VoteResponse}; +pub use self::controller_messages::{SystemOperation, SystemOpType, ConnectionRef, Peer, PeerUpgrade, CallRequestVote, VoteResponse, ReceiveAppendEntries}; pub use self::cluster_state::{SharedClusterState, ClusterStateReader}; pub type SystemPartitionSender = ::std::sync::mpsc::Sender; diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index 84692f9..912fba1 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -3,7 +3,7 @@ use std::net::SocketAddr; use protocol::FloInstanceId; use engine::event_stream::partition::{PartitionRef, Operation, SharedReaderRefs, PartitionReader, SegmentNum}; use engine::event_stream::EventStreamRef; -use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer, CallRequestVote, VoteResponse, SystemStreamReader}; +use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer, CallRequestVote, VoteResponse, SystemStreamReader, ReceiveAppendEntries}; use engine::controller::cluster_state::{SharedClusterState, ClusterStateReader}; use engine::ConnectionId; use atomics::{AtomicBoolReader, AtomicCounterReader}; @@ -84,6 +84,11 @@ impl SystemStreamRef { self.send(op); } + pub fn append_entries_received(&mut self, connection_id: ConnectionId, append: ReceiveAppendEntries) { + let op = SystemOperation::append_entries_received(connection_id, append); + self.send(op); + } + fn send(&mut self, op: SystemOperation) { // TODO: change this to propagate the error so that connectionHandlers can shut down gracefully self.system_sender.send(op).expect("Failed to send to flo controller. System must have shut down"); From 81675a44ca65556062db2969f06583e4cfda2b77 Mon Sep 17 00:00:00 2001 From: pfried Date: Wed, 3 Jan 2018 21:15:43 -0500 Subject: [PATCH 41/73] handle sending and receiving appendEntries responses in connection handler --- flo-protocol/src/messages/append_entries.rs | 5 +-- flo-protocol/src/messages/mod.rs | 1 - .../src/engine/connection_handler/input.rs | 5 +-- .../src/engine/connection_handler/mod.rs | 35 +++++++++++++++++-- .../src/engine/connection_handler/peer/mod.rs | 29 +++++++++++++++ .../engine/controller/controller_messages.rs | 14 +++++++- flo-server/src/engine/controller/mod.rs | 2 +- .../src/engine/controller/system_stream.rs | 7 +++- 8 files changed, 86 insertions(+), 12 deletions(-) diff --git a/flo-protocol/src/messages/append_entries.rs b/flo-protocol/src/messages/append_entries.rs index e180574..376a3c9 100644 --- a/flo-protocol/src/messages/append_entries.rs +++ b/flo-protocol/src/messages/append_entries.rs @@ -56,7 +56,6 @@ impl AppendEntriesCall { #[derive(Debug, PartialEq, Clone)] pub struct AppendEntriesResponse { pub op_id: u32, - pub from_peer: FloInstanceId, pub term: Term, pub success: bool, } @@ -65,7 +64,6 @@ pub fn serialize_append_response(response: &AppendEntriesResponse, buf: &mut [u8 Serializer::new(buf) .write_u8(APPEND_ENTRIES_RESPONSE_HEADER) .write_u32(response.op_id) - .write(&response.from_peer) .write_u64(response.term) .write_bool(response.success) .finish() @@ -74,12 +72,11 @@ pub fn serialize_append_response(response: &AppendEntriesResponse, buf: &mut [u8 named!{pub parse_append_entries_response>, do_parse!( tag!(&[APPEND_ENTRIES_RESPONSE_HEADER]) >> op_id: be_u32 >> - from_peer: parse_flo_instance_id >> term: be_u64 >> success: map!(be_u8, |val| { val == 1 }) >> ( ProtocolMessage::SystemAppendResponse(AppendEntriesResponse { - op_id, from_peer, term, success, + op_id, term, success, }) ) )} diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index 5fd2cbc..8c4bf7a 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -406,7 +406,6 @@ mod test { fn serde_append_entries_response() { let response = AppendEntriesResponse { op_id: 45, - from_peer: FloInstanceId::generate_new(), term: 345, success: false, }; diff --git a/flo-server/src/engine/connection_handler/input.rs b/flo-server/src/engine/connection_handler/input.rs index d477e21..5ca14bc 100644 --- a/flo-server/src/engine/connection_handler/input.rs +++ b/flo-server/src/engine/connection_handler/input.rs @@ -2,7 +2,7 @@ use protocol::{FloInstanceId, Term}; use event::EventCounter; use engine::ReceivedProtocolMessage; -use engine::controller::{CallRequestVote, VoteResponse}; +use engine::controller::{CallRequestVote, VoteResponse, AppendEntriesResponse}; use engine::event_stream::partition::SegmentNum; #[derive(Debug)] @@ -29,7 +29,8 @@ pub enum ConnectionControl { InitiateOutgoingSystemConnection, SendRequestVote(CallRequestVote), SendVoteResponse(VoteResponse), - SendAppendEntries(CallAppendEntries) + SendAppendEntries(CallAppendEntries), + SendAppendEntriesResponse(AppendEntriesResponse), } diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 97b253c..8638aba 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -70,6 +70,9 @@ impl ConnectionHandler { ConnectionControl::SendAppendEntries(append) => { peer_state.send_append_entries(append, common_state) } + ConnectionControl::SendAppendEntriesResponse(response) => { + peer_state.send_append_entries_response(response, common_state) + } _ => unimplemented!() } } @@ -113,6 +116,9 @@ impl ConnectionHandler { ProtocolMessage::ReceiveEvent(event) => { peer_state.event_received(event, common_state) } + ProtocolMessage::SystemAppendResponse(response) => { + peer_state.append_entries_response_received(response, common_state) + } _ => unimplemented!() } } @@ -190,11 +196,13 @@ mod test { use tokio_core::reactor::Core; use super::*; + use protocol; use event::{ActorId, OwnedFloEvent, FloEventId, time}; use engine::{SYSTEM_STREAM_NAME, system_stream_name}; use engine::event_stream::EventStreamRef; use engine::event_stream::partition::*; use engine::ClientReceiver; + use engine::controller; use engine::controller::*; use atomics::{AtomicCounterWriter, AtomicBoolWriter}; use test_utils::addr; @@ -436,7 +444,7 @@ mod test { } #[test] - fn receiving_append_entries_with_0_entries_sends_append_entries_to_system_controller() { + fn receiving_append_entries_with_0_entries_sends_append_entries_to_system_controller_and_response_is_sent_out() { let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); let sender_id = FloInstanceId::generate_new(); @@ -459,10 +467,22 @@ mod test { commit_index: 987, events: Vec::new(), })); + + // now send the response to the client + subject.handle_control(ConnectionControl::SendAppendEntriesResponse(controller::AppendEntriesResponse { + term: 11, + success: false, + })).unwrap(); + + fixture.assert_sent_to_client(ProtocolMessage::SystemAppendResponse(protocol::AppendEntriesResponse { + op_id: 2, + term: 11, + success: false, + })); } #[test] - fn append_entries_is_sent_without_any_events() { + fn append_entries_is_sent_without_any_events_and_response_is_received() { let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); subject.handle_control(ConnectionControl::SendAppendEntries(CallAppendEntries { @@ -485,6 +505,17 @@ mod test { leader_commit_index: 987, entry_count: 0, })); + + // now receive the response + subject.handle_incoming_message(ProtocolMessage::SystemAppendResponse(protocol::AppendEntriesResponse { + op_id: 2, + term: 7, + success: false, + })).unwrap(); + fixture.assert_sent_to_system_stream(SystemOpType::AppendEntriesResponseReceived(controller::AppendEntriesResponse { + term: 7, + success: false, + })); } #[test] diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 4212add..12f17e9 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -42,6 +42,34 @@ impl PeerConnectionState { } } + pub fn append_entries_response_received(&mut self, response: protocol::AppendEntriesResponse, state: &mut ConnectionState) -> ConnectionHandlerResult { + let op_id = self.peer_operation_queue.pop_front().ok_or_else(|| { + "Received AppendEntriesResponse but no response was expected".to_owned() + })?; + if op_id != response.op_id { + return Err(format!("Expected AppendEntriesResponse with op_id: {}, but received: {:?}", op_id, response)); + } + let controller_message = controller::AppendEntriesResponse { + term: response.term, + success: response.success, + }; + let connection_id = state.connection_id; + state.get_system_stream().append_entries_response_received(connection_id, controller_message); + Ok(()) + } + + pub fn send_append_entries_response(&mut self, response: controller::AppendEntriesResponse, state: &mut ConnectionState) -> ConnectionHandlerResult { + let op_id = self.controller_operation_queue.pop_front().ok_or_else(|| { + "send_append_entries_response was called but there was no pending operation".to_owned() + })?; + let protocol_message = protocol::AppendEntriesResponse { + op_id, + term: response.term, + success: response.success, + }; + state.send_to_client(ProtocolMessage::SystemAppendResponse(protocol_message)) + } + pub fn append_entries_received(&mut self, append: protocol::AppendEntriesCall, connection: &mut ConnectionState) -> ConnectionHandlerResult { self.ensure_peer_state(Some(append.op_id), connection)?; @@ -113,6 +141,7 @@ impl PeerConnectionState { pub fn send_append_entries(&mut self, append: CallAppendEntries, state: &mut ConnectionState) -> ConnectionHandlerResult { let connection_id = state.connection_id; let op_id = self.next_op_id(); + self.peer_operation_queue.push_back(op_id); let this_instance_id = self.get_this_instance_id(state); let CallAppendEntries {current_term, prev_entry_index, prev_entry_term, reader_start_offset, reader_start_segment, reader_start_event, commit_index} = append; diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 1868ab2..826673b 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -46,6 +46,12 @@ pub struct ReceiveAppendEntries { pub events: Vec, } +#[derive(Debug, Clone, PartialEq)] +pub struct AppendEntriesResponse { + pub term: Term, + pub success: bool, +} + #[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)] pub struct Peer { #[serde(with = "InstanceIdRemote")] @@ -69,9 +75,11 @@ pub enum SystemOpType { ConnectionUpgradeToPeer(PeerUpgrade), ConnectionClosed, OutgoingConnectionFailed(SocketAddr), + RequestVote(CallRequestVote), VoteResponseReceived(VoteResponse), - AppendEntriesReceived(ReceiveAppendEntries) + AppendEntriesReceived(ReceiveAppendEntries), + AppendEntriesResponseReceived(AppendEntriesResponse), } impl SystemOpType { @@ -92,6 +100,10 @@ pub struct SystemOperation { impl SystemOperation { + pub fn append_entries_response_received(connection_id: ConnectionId, response: AppendEntriesResponse) -> SystemOperation { + SystemOperation::new(connection_id, SystemOpType::AppendEntriesResponseReceived(response)) + } + pub fn append_entries_received(connection_id: ConnectionId, append: ReceiveAppendEntries) -> SystemOperation { SystemOperation::new(connection_id, SystemOpType::AppendEntriesReceived(append)) } diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 90acf41..75d40a0 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -26,7 +26,7 @@ use self::cluster_state::{ClusterManager, ConsensusProcessor}; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; pub use self::system_stream::SystemStreamRef; pub use self::system_reader::SystemStreamReader; -pub use self::controller_messages::{SystemOperation, SystemOpType, ConnectionRef, Peer, PeerUpgrade, CallRequestVote, VoteResponse, ReceiveAppendEntries}; +pub use self::controller_messages::*; pub use self::cluster_state::{SharedClusterState, ClusterStateReader}; pub type SystemPartitionSender = ::std::sync::mpsc::Sender; diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index 912fba1..dda8af0 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -3,7 +3,7 @@ use std::net::SocketAddr; use protocol::FloInstanceId; use engine::event_stream::partition::{PartitionRef, Operation, SharedReaderRefs, PartitionReader, SegmentNum}; use engine::event_stream::EventStreamRef; -use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer, CallRequestVote, VoteResponse, SystemStreamReader, ReceiveAppendEntries}; +use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer, CallRequestVote, VoteResponse, SystemStreamReader, ReceiveAppendEntries, AppendEntriesResponse}; use engine::controller::cluster_state::{SharedClusterState, ClusterStateReader}; use engine::ConnectionId; use atomics::{AtomicBoolReader, AtomicCounterReader}; @@ -89,6 +89,11 @@ impl SystemStreamRef { self.send(op); } + pub fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse) { + let op = SystemOperation::append_entries_response_received(connection_id, response); + self.send(op); + } + fn send(&mut self, op: SystemOperation) { // TODO: change this to propagate the error so that connectionHandlers can shut down gracefully self.system_sender.send(op).expect("Failed to send to flo controller. System must have shut down"); From 1b3950a288bf4b59c474f038ca4d3baf4589fff1 Mon Sep 17 00:00:00 2001 From: pfried Date: Wed, 3 Jan 2018 23:00:24 -0500 Subject: [PATCH 42/73] rename FloEvent::to_owned to to_owned_event to differentiate it from std::borrow::ToOwned --- flo-event/src/lib.rs | 6 +- flo-server/src/embedded/mod.rs | 2 +- .../engine/controller/cluster_state/mod.rs | 96 ++++++++++++++----- .../src/engine/controller/system_reader.rs | 6 +- .../event_stream/partition/segment/mmap.rs | 10 +- .../partition/segment/persistent_event.rs | 4 +- 6 files changed, 90 insertions(+), 34 deletions(-) diff --git a/flo-event/src/lib.rs b/flo-event/src/lib.rs index 00e2996..7816583 100644 --- a/flo-event/src/lib.rs +++ b/flo-event/src/lib.rs @@ -203,7 +203,7 @@ pub trait FloEvent: Debug { /// Returns the arbitrary binary data associated with this event. fn data(&self) -> &[u8]; /// Converts this event into an `OwnedFloEvent`, cloning it in the process. - fn to_owned(&self) -> OwnedFloEvent { + fn to_owned_event(&self) -> OwnedFloEvent { let id = *self.id(); let data = self.data().to_owned(); OwnedFloEvent { @@ -233,7 +233,7 @@ impl FloEvent for T where T: AsRef + Debug { self.as_ref().data() } - fn to_owned(&self) -> OwnedFloEvent { + fn to_owned_event(&self) -> OwnedFloEvent { self.as_ref().clone() } @@ -286,7 +286,7 @@ impl FloEvent for OwnedFloEvent { &self.data } - fn to_owned(&self) -> OwnedFloEvent { + fn to_owned_event(&self) -> OwnedFloEvent { self.clone() } diff --git a/flo-server/src/embedded/mod.rs b/flo-server/src/embedded/mod.rs index c68a85a..d9a4a4c 100644 --- a/flo-server/src/embedded/mod.rs +++ b/flo-server/src/embedded/mod.rs @@ -55,7 +55,7 @@ impl EmbeddedFloServer { // probably figure out a way to avoid exposing the generic types via the public api. Seems like a 'later' problem fn message_to_owned(server_msg: SendProtocolMessage) -> ClientProtocolMessage { match server_msg { - ProtocolMessage::ReceiveEvent(event) => ProtocolMessage::ReceiveEvent(event.to_owned()), + ProtocolMessage::ReceiveEvent(event) => ProtocolMessage::ReceiveEvent(event.to_owned_event()), ProtocolMessage::StopConsuming(op) => ProtocolMessage::StopConsuming(op), ProtocolMessage::AwaitingEvents => ProtocolMessage::AwaitingEvents, ProtocolMessage::Error(op) => ProtocolMessage::Error(op), diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index a20812d..6492b4e 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -14,7 +14,7 @@ use engine::connection_handler::ConnectionControl; use engine::controller::{CallRequestVote, VoteResponse}; use protocol::{FloInstanceId, Term}; use atomics::{AtomicBoolWriter, AtomicBoolReader}; -use super::{ClusterOptions, ConnectionRef, Peer, PeerUpgrade}; +use super::{ClusterOptions, ConnectionRef, Peer, PeerUpgrade, AppendEntriesResponse, ReceiveAppendEntries}; use super::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator, OutgoingConnectionCreatorImpl}; use self::peer_connections::{PeerConnectionManager, PeerConnections}; @@ -33,30 +33,12 @@ pub trait ConsensusProcessor: Send { fn is_primary(&self) -> bool; fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap); fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr); + fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote); fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse); -} - -#[derive(Debug)] -pub struct NoOpConsensusProcessor; - -impl ConsensusProcessor for NoOpConsensusProcessor { - fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap) { - } - fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { - panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); - } - fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote) { - - } - fn tick(&mut self, now: Instant, all_connections: &mut HashMap) { - } - fn is_primary(&self) -> bool { - true - } - fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse) { - } + fn append_entries_received(&mut self, append: ReceiveAppendEntries); + fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse); } #[derive(Debug)] @@ -299,6 +281,14 @@ impl ConsensusProcessor for ClusterManager { fn is_primary(&self) -> bool { self.state == State::Primary } + + fn append_entries_received(&mut self, append: ReceiveAppendEntries) { + unimplemented!() + } + + fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse) { + unimplemented!() + } } #[derive(Debug, PartialEq, Clone)] @@ -366,6 +356,37 @@ mod test { use engine::controller::cluster_state::peer_connections::mock::{MockPeerConnectionManager, Invocation}; use engine::controller::controller_messages::mock::mock_connection_ref; + #[test] + fn append_entries_is_sent_on_tick_when_state_is_primary() { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let connection_manager = MockPeerConnectionManager::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:1000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:2000") + }; + let peer_1_connection = 1; + let peer_2_connection = 2; + connection_manager.stub_peer_connection(peer_1_connection, peer_1.id); + connection_manager.stub_peer_connection(peer_2_connection, peer_2.id); + + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + + subject.persistent.modify(|state| { + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + state.current_term = 5; + }).unwrap(); + subject.last_applied = 99; + subject.last_applied_term = 4; + subject.state = State::Voted; + subject.last_heartbeat = start; + } + #[test] fn new_election_is_started_on_tick_when_current_election_goes_beyond_timeout() { let start = Instant::now(); @@ -972,3 +993,34 @@ mod test { }); } } + +/// Used when the server is running in standalone mode +#[derive(Debug)] +pub struct NoOpConsensusProcessor; + +// TODO: reasonable and consistent behavior for calls into NoOpConsensusProcessor that should never actually happen +impl ConsensusProcessor for NoOpConsensusProcessor { + fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap) { + } + fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { + panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); + } + fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote) { + + } + fn tick(&mut self, now: Instant, all_connections: &mut HashMap) { + } + fn is_primary(&self) -> bool { + true + } + fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse) { + + } + fn append_entries_received(&mut self, append: ReceiveAppendEntries) { + unimplemented!() + } + + fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse) { + unimplemented!() + } +} diff --git a/flo-server/src/engine/controller/system_reader.rs b/flo-server/src/engine/controller/system_reader.rs index 9bc4add..1672ff3 100644 --- a/flo-server/src/engine/controller/system_reader.rs +++ b/flo-server/src/engine/controller/system_reader.rs @@ -5,7 +5,11 @@ use engine::ConnectionId; use engine::event_stream::partition::{PartitionReader, SegmentNum, SharedReaderRefs, EventFilter, PersistentEvent}; use atomics::AtomicCounterReader; -const SYSTEM_READER_BATCH_SIZE: usize = 8; +/// The max number of events that will be sent with a single AppendEntries call. +/// This is currently a bit hacky. When the controller requests an AppendEntries to be sent, it does not say how many events +/// should be included. The controller just assumes that all available events will be sent, to a maximum of this value. +/// A better plan may be to have the controller simply tell the ConnectionHandler how many events to send. +pub const SYSTEM_READER_BATCH_SIZE: usize = 8; #[derive(Debug)] pub struct SystemStreamReader { diff --git a/flo-server/src/engine/event_stream/partition/segment/mmap.rs b/flo-server/src/engine/event_stream/partition/segment/mmap.rs index 7e0955c..b01e2a0 100644 --- a/flo-server/src/engine/event_stream/partition/segment/mmap.rs +++ b/flo-server/src/engine/event_stream/partition/segment/mmap.rs @@ -261,7 +261,7 @@ mod test { subject.append(&input).unwrap(); let result = reader.read_next().expect("reader returned none").expect("failed to read event"); - assert_eq!(input, result.to_owned()); + assert_eq!(input, result.to_owned_event()); } #[test] @@ -306,7 +306,7 @@ mod test { subject.append(&input).unwrap(); let result = reader.read_next().expect("reader returned none").expect("failed to read event"); - assert_eq!(input, result.to_owned()); + assert_eq!(input, result.to_owned_event()); let len = PersistentEvent::get_repr_length(&input); assert_eq!(len, PersistentEvent::get_repr_length(&result)); @@ -339,7 +339,7 @@ mod test { let off_3 = subject.append(&input3).unwrap().expect("write returned none"); let read_all = subject.reader(0) - .map(|result| result.expect("failed to read event").to_owned()) + .map(|result| result.expect("failed to read event").to_owned_event()) .collect::>(); let expected = vec![input1.clone(), input2.clone(), input3.clone()]; assert_eq!(expected, read_all); @@ -347,10 +347,10 @@ mod test { let read_2 = subject.reader(off_2).next().unwrap().expect("failed to read event 2"); let read2_offset = read_2.file_offset(); assert_eq!(off_2, read2_offset); - assert_eq!(input2, read_2.to_owned()); + assert_eq!(input2, read_2.to_owned_event()); let read_3 = subject.reader(off_3).next().unwrap().expect("failed to read event 3"); - assert_eq!(input3, read_3.to_owned()); + assert_eq!(input3, read_3.to_owned_event()); } fn assert_read_err(expected_description: &str, modify_buffer_fun: F) { diff --git a/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs b/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs index e2e9f4e..7c5fa87 100644 --- a/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs +++ b/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs @@ -7,7 +7,7 @@ use engine::event_stream::partition::segment::mmap::{MmapRef}; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PersistentEvent { id: FloEventId, file_offset: usize, @@ -163,7 +163,7 @@ impl FloEvent for PersistentEvent { self.as_buf(48 + ns_len, data_len) } - fn to_owned(&self) -> OwnedFloEvent { + fn to_owned_event(&self) -> OwnedFloEvent { let id = *self.id(); let parent_id = self.parent_id(); let timestamp = self.timestamp(); From a0dda4933c191aaea0549ef1572b8934f88bb132 Mon Sep 17 00:00:00 2001 From: pfried Date: Thu, 4 Jan 2018 23:03:24 -0500 Subject: [PATCH 43/73] add basic skeleton of SystemEvent --- Cargo.lock | 23 +++++ flo-server/Cargo.toml | 2 + flo-server/src/engine/controller/mod.rs | 1 + .../src/engine/controller/system_event.rs | 94 +++++++++++++++++++ flo-server/src/lib.rs | 2 + 5 files changed, 122 insertions(+) create mode 100644 flo-server/src/engine/controller/system_event.rs diff --git a/Cargo.lock b/Cargo.lock index 3667696..56e533f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -281,6 +281,8 @@ dependencies = [ "nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp-serde 0.13.7 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -659,6 +661,25 @@ name = "regex-syntax" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "rmp" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rmp-serde" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rustc-serialize" version = "0.3.24" @@ -1094,6 +1115,8 @@ dependencies = [ "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1731164734096285ec2a5ec7fea5248ae2f5485b3feeb0115af4fda2183b2d1b" "checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db" +"checksum rmp 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a3d45d7afc9b132b34a2479648863aa95c5c88e98b32285326a6ebadc80ec5c9" +"checksum rmp-serde 0.13.7 (registry+https://github.com/rust-lang/crates.io-index)" = "011e1d58446e9fa3af7cdc1fb91295b10621d3ac4cb3a85cc86385ee9ca50cd3" "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum rusttype 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "07b8848db3b5b5ba97020c6a756c0fdf2dbf2ad7c0d06aa4344a3f2f49c3fe17" "checksum scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f417c22df063e9450888a7561788e9bd46d3bb3c1466435b4eccb903807f147d" diff --git a/flo-server/Cargo.toml b/flo-server/Cargo.toml index 95ad88b..c279863 100644 --- a/flo-server/Cargo.toml +++ b/flo-server/Cargo.toml @@ -23,6 +23,8 @@ serde = "1.0" serde_json = "1.0" serde_derive = "1.0" rand = "0.4" +rmp = "^0.8" +rmp-serde = "^0.13" [dev-dependencies] env_logger = "*" diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 75d40a0..b8470df 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -1,5 +1,6 @@ pub mod cluster_state; pub mod tick_generator; +pub mod system_event; mod system_stream; mod initialization; mod controller_messages; diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs new file mode 100644 index 0000000..58626c2 --- /dev/null +++ b/flo-server/src/engine/controller/system_event.rs @@ -0,0 +1,94 @@ +use rmp_serde::decode::Error; + +use protocol::Term; +use event::{FloEvent, FloEventId, OwnedFloEvent, EventCounter, ActorId, Timestamp, time}; + +#[derive(Debug, PartialEq)] +pub struct SystemEvent { + wrapped: E, +} + +impl SystemEvent { + + pub fn from_event(event: E) -> Result, Error> { + { + let _ = ::rmp_serde::decode::from_slice::(event.data())?; + } + Ok(SystemEvent{ + wrapped: event + }) + } + + pub fn system_data(&self) -> Result { + ::rmp_serde::from_slice(self.data()) + } +} + + +impl SystemEvent { + pub fn new(id: FloEventId, parent: Option, namespace: String, time: Timestamp, data: &SystemEventData) -> SystemEvent { + // TODO: I feel like this is probably a safe unwrap, but might be good to double check + let data = ::rmp_serde::to_vec(data).unwrap(); + let event = OwnedFloEvent::new(id, parent, time, namespace, data); + SystemEvent { + wrapped: event + } + } +} + +impl FloEvent for SystemEvent { + + fn id(&self) -> &FloEventId { + self.wrapped.id() + } + fn timestamp(&self) -> Timestamp { + self.wrapped.timestamp() + } + fn parent_id(&self) -> Option { + self.wrapped.parent_id() + } + fn namespace(&self) -> &str { + self.wrapped.namespace() + } + fn data_len(&self) -> u32 { + self.wrapped.data_len() + } + fn data(&self) -> &[u8] { + self.wrapped.data() + } +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct SystemEventData { + pub term: Term, +} + + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn from_event_returns_error_when_event_data_cannot_be_deserialized() { + let data = vec![ 0xff, 0xff, 0xff, 0xff, 0x00, 0x00 ]; + let event = OwnedFloEvent::new(FloEventId::new(1, 2), None, time::now(), String::new(), data); + let result = SystemEvent::from_event(event); + assert!(result.is_err()); + } + + #[test] + fn system_event_data_is_serialized_and_deserialized_inside_system_event() { + let data = SystemEventData { + term: 33, + }; + let id = FloEventId::new(3, 4); + let parent = Some(FloEventId::new(2, 3)); + let time = time::from_millis_since_epoch(1234567); + let namespace = "/system/foo".to_owned(); + let event = SystemEvent::new(id, parent, namespace, time, &data); + let as_owned = event.to_owned_event(); + let result = SystemEvent::from_event(as_owned).unwrap(); + assert_eq!(event, result); + } +} + diff --git a/flo-server/src/lib.rs b/flo-server/src/lib.rs index 802acc2..2e535a2 100644 --- a/flo-server/src/lib.rs +++ b/flo-server/src/lib.rs @@ -17,6 +17,8 @@ extern crate log4rs; extern crate num_cpus; extern crate byteorder; extern crate rand; +extern crate rmp; +extern crate rmp_serde; #[macro_use] extern crate serde_derive; From a7972cd83c0f1106f62d7496ebc097fde7596fdf Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 7 Jan 2018 14:38:56 -0500 Subject: [PATCH 44/73] change how AppendEntries is sent so that peer offset is maintained by the connection handler --- .../src/engine/connection_handler/input.rs | 7 +- .../src/engine/connection_handler/mod.rs | 12 +-- .../src/engine/connection_handler/peer/mod.rs | 69 +++++---------- .../peer/system_read_wrapper.rs | 85 +++++++++++++++++++ .../engine/controller/controller_messages.rs | 2 +- flo-server/src/engine/controller/mod.rs | 3 +- .../src/engine/controller/system_event.rs | 24 +++++- .../src/engine/controller/system_reader.rs | 24 +++--- 8 files changed, 152 insertions(+), 74 deletions(-) create mode 100644 flo-server/src/engine/connection_handler/peer/system_read_wrapper.rs diff --git a/flo-server/src/engine/connection_handler/input.rs b/flo-server/src/engine/connection_handler/input.rs index 5ca14bc..9cdaa6c 100644 --- a/flo-server/src/engine/connection_handler/input.rs +++ b/flo-server/src/engine/connection_handler/input.rs @@ -49,10 +49,15 @@ impl From for ConnectionHandlerInput { #[derive(Debug, Clone, PartialEq)] pub struct CallAppendEntries { pub current_term: Term, + pub commit_index: EventCounter, + pub reader_start_position: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AppendEntriesStart { pub prev_entry_index: EventCounter, pub prev_entry_term: Term, pub reader_start_offset: usize, pub reader_start_segment: SegmentNum, pub reader_start_event: EventCounter, - pub commit_index: EventCounter, } diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 8638aba..4f0cd3a 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -18,7 +18,7 @@ use self::consumer::ConsumerConnectionState; use self::producer::ProducerConnectionState; use self::peer::PeerConnectionState; -pub use self::input::{ConnectionHandlerInput, ConnectionControl, CallAppendEntries}; +pub use self::input::{ConnectionHandlerInput, ConnectionControl, CallAppendEntries, AppendEntriesStart}; pub type ConnectionControlSender = ::futures::sync::mpsc::UnboundedSender; pub type ConnectionControlReceiver = ::futures::sync::mpsc::UnboundedReceiver; @@ -471,7 +471,7 @@ mod test { // now send the response to the client subject.handle_control(ConnectionControl::SendAppendEntriesResponse(controller::AppendEntriesResponse { term: 11, - success: false, + success: None, })).unwrap(); fixture.assert_sent_to_client(ProtocolMessage::SystemAppendResponse(protocol::AppendEntriesResponse { @@ -487,12 +487,8 @@ mod test { subject.handle_control(ConnectionControl::SendAppendEntries(CallAppendEntries { current_term: 4, - prev_entry_index: 0, - prev_entry_term: 0, - reader_start_offset: 0, - reader_start_segment: SegmentNum::new_unset(), - reader_start_event: 0, commit_index: 987, // just to show that this is just a dumb value and not interpreted by the connection handler + reader_start_position: None, })).unwrap(); let expected_id = fixture.instance_id; @@ -514,7 +510,7 @@ mod test { })).unwrap(); fixture.assert_sent_to_system_stream(SystemOpType::AppendEntriesResponseReceived(controller::AppendEntriesResponse { term: 7, - success: false, + success: None, })); } diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 12f17e9..6eb99a2 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -1,13 +1,16 @@ mod peer_follower; +mod system_read_wrapper; use std::collections::VecDeque; -use event::OwnedFloEvent; -use protocol::{self, ProtocolMessage, PeerAnnounce, EventStreamStatus, ClusterMember, ErrorMessage, ErrorKind, FloInstanceId}; +use event::{EventCounter, OwnedFloEvent, FloEvent}; +use protocol::{self, ProtocolMessage, PeerAnnounce, EventStreamStatus, ClusterMember, ErrorMessage, ErrorKind, FloInstanceId, Term}; use engine::{ReceivedProtocolMessage, ConnectionId}; -use engine::controller::{self, SystemStreamRef, Peer, SystemStreamReader}; +use engine::controller::{self, SystemStreamRef, Peer, SystemEvent, SystemStreamReader, SYSTEM_READER_BATCH_SIZE}; +use engine::event_stream::partition::PersistentEvent; use super::connection_state::ConnectionState; -use super::{ConnectionHandlerResult, CallAppendEntries}; +use super::{ConnectionHandlerResult, CallAppendEntries, AppendEntriesStart}; +use self::system_read_wrapper::SystemReaderWrapper; #[derive(Debug, Clone, Copy, PartialEq)] @@ -23,7 +26,7 @@ pub struct PeerConnectionState { current_op_id: u32, controller_operation_queue: VecDeque, peer_operation_queue: VecDeque, - system_partition_reader: Option, + system_partition_reader: Option, this_instance_id: Option, in_progress_append: Option, } @@ -49,9 +52,16 @@ impl PeerConnectionState { if op_id != response.op_id { return Err(format!("Expected AppendEntriesResponse with op_id: {}, but received: {:?}", op_id, response)); } + let success = if response.success { + self.system_partition_reader.as_mut().map(|reader| { + reader.append_acknowledged() + }) + } else { + None + }; let controller_message = controller::AppendEntriesResponse { term: response.term, - success: response.success, + success, }; let connection_id = state.connection_id; state.get_system_stream().append_entries_response_received(connection_id, controller_message); @@ -65,7 +75,7 @@ impl PeerConnectionState { let protocol_message = protocol::AppendEntriesResponse { op_id, term: response.term, - success: response.success, + success: response.success.is_some(), }; state.send_to_client(ProtocolMessage::SystemAppendResponse(protocol_message)) } @@ -117,10 +127,10 @@ impl PeerConnectionState { } } - fn ensure_peer_state(&self, op_id: Option, state: &mut ConnectionState) -> ConnectionHandlerResult { + fn ensure_peer_state(&self, incoming_message_op_id: Option, state: &mut ConnectionState) -> ConnectionHandlerResult { if self.state != State::Peer { let err = ErrorMessage { - op_id: op_id.unwrap_or(0), + op_id: incoming_message_op_id.unwrap_or(0), kind: ErrorKind::InvalidPeerState, description: "No PeerAnnounce message has been received so peer operations are invalid".to_owned(), }; @@ -132,54 +142,17 @@ impl PeerConnectionState { } } - fn get_remaining_expected_events(&self) -> usize { - self.in_progress_append.as_ref().map(|append| { - append.events.capacity() - append.events.len() - }).unwrap_or(0) - } - pub fn send_append_entries(&mut self, append: CallAppendEntries, state: &mut ConnectionState) -> ConnectionHandlerResult { - let connection_id = state.connection_id; let op_id = self.next_op_id(); self.peer_operation_queue.push_back(op_id); let this_instance_id = self.get_this_instance_id(state); - let CallAppendEntries {current_term, prev_entry_index, prev_entry_term, - reader_start_offset, reader_start_segment, reader_start_event, commit_index} = append; if self.system_partition_reader.is_none() { - let reader = state.get_system_stream().create_system_stream_reader(connection_id); - self.system_partition_reader = Some(reader); + self.system_partition_reader = Some(SystemReaderWrapper::new(state)); } let reader = self.system_partition_reader.as_mut().unwrap(); - let num_events = if reader_start_segment.is_set() { - reader.set_to(reader_start_segment, reader_start_offset).map_err(|io_err| { - format!("Failed to set position of system stream reader for connection_id: {} - {:?}", connection_id ,io_err) - })?; // just give up - - reader.fill_buffer().map_err(|io_err| { - format!("Failed to rea events for AppendEntries for connection_id: {}, error: {:?}", connection_id, io_err) - })? // give up on error - } else { - 0 - }; - - let append = protocol::AppendEntriesCall { - op_id, - leader_id: this_instance_id, - term: current_term, - prev_entry_term, - prev_entry_index, - leader_commit_index: commit_index, - entry_count: num_events as u32, - }; - state.send_to_client(ProtocolMessage::SystemAppendCall(append))?; - if num_events > 0 { - for event in reader.drain() { - state.send_to_client(ProtocolMessage::ReceiveEvent(event))?; - } - } - Ok(()) + reader.send_append_entries(op_id, this_instance_id, append, state) } fn get_this_instance_id(&mut self, state: &mut ConnectionState) -> FloInstanceId { diff --git a/flo-server/src/engine/connection_handler/peer/system_read_wrapper.rs b/flo-server/src/engine/connection_handler/peer/system_read_wrapper.rs new file mode 100644 index 0000000..bbaa190 --- /dev/null +++ b/flo-server/src/engine/connection_handler/peer/system_read_wrapper.rs @@ -0,0 +1,85 @@ +use event::{EventCounter, FloEvent}; +use protocol::{self, ProtocolMessage, Term, FloInstanceId}; +use engine::controller::{SystemEvent, SystemStreamReader, SYSTEM_READER_BATCH_SIZE}; +use engine::event_stream::partition::PersistentEvent; +use engine::connection_handler::{CallAppendEntries, ConnectionHandlerResult}; +use engine::connection_handler::connection_state::ConnectionState; + + +#[derive(Debug)] +pub struct SystemReaderWrapper { + prev_event_index: EventCounter, + prev_event_term: Term, + last_sent_index: EventCounter, + last_sent_term: Term, + reader: SystemStreamReader, + event_buffer: Vec>, +} + +impl SystemReaderWrapper { + pub fn new(common: &mut ConnectionState) -> SystemReaderWrapper { + let connection_id = common.connection_id; + let reader = common.get_system_stream().create_system_stream_reader(connection_id); + SystemReaderWrapper { + prev_event_index: 0, + prev_event_term: 0, + last_sent_index: 0, + last_sent_term: 0, + event_buffer: Vec::with_capacity(SYSTEM_READER_BATCH_SIZE), + reader + } + } + + pub fn append_acknowledged(&mut self) -> EventCounter { + self.prev_event_term = self.last_sent_term; + self.prev_event_index = self.last_sent_index; + self.last_sent_index + } + + pub fn send_append_entries(&mut self, op_id: u32, this_instance_id: FloInstanceId, call: CallAppendEntries, common: &mut ConnectionState) -> ConnectionHandlerResult { + + let connection_id = common.connection_id; + let CallAppendEntries {current_term, commit_index, reader_start_position} = call; + + if let Some(start) = reader_start_position { + info!("Setting start position for outgoing AppendEntries for connection_id: {} to {:?}", connection_id, start); + self.prev_event_term = start.prev_entry_term; + self.prev_event_index = start.prev_entry_index; + self.reader.set_to(start.reader_start_segment, start.reader_start_offset).map_err(|io_err| { + format!("Failed to set reader position for connection_id: {} to: {:?}, io_err: {:?}", connection_id, start, io_err) + })?; // early return on failure + } + + let prev_term = self.prev_event_term; + let prev_index = self.prev_event_index; + + let event_count = { + let SystemReaderWrapper {ref mut event_buffer, ref mut reader, ..} = *self; + event_buffer.clear(); + reader.fill_buffer(event_buffer).map_err(|io_err| { + format!("Error reading system events for connection_id: {}, io_err: {:?}", connection_id, io_err) + })? // early return on failure + }; + + let append = protocol::AppendEntriesCall { + op_id, + leader_id: this_instance_id, + term: current_term, + prev_entry_term: prev_term, + prev_entry_index: prev_index, + leader_commit_index: commit_index, + entry_count: event_count as u32, + }; + + if let Some(event) = self.event_buffer.last() { + self.last_sent_term = event.term(); + self.last_sent_index = event.id().event_counter; + } + + common.send_to_client(ProtocolMessage::SystemAppendCall(append))?; + for event in self.event_buffer.drain(..) { + common.send_to_client(ProtocolMessage::ReceiveEvent(event.into()))?; + } + Ok(()) + } +} diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 826673b..f8d2f78 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -49,7 +49,7 @@ pub struct ReceiveAppendEntries { #[derive(Debug, Clone, PartialEq)] pub struct AppendEntriesResponse { pub term: Term, - pub success: bool, + pub success: Option, } #[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)] diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index b8470df..4e650ad 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -26,7 +26,8 @@ use self::cluster_state::{ClusterManager, ConsensusProcessor}; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; pub use self::system_stream::SystemStreamRef; -pub use self::system_reader::SystemStreamReader; +pub use self::system_event::{SystemEvent, SystemEventData}; +pub use self::system_reader::{SystemStreamReader, SYSTEM_READER_BATCH_SIZE}; pub use self::controller_messages::*; pub use self::cluster_state::{SharedClusterState, ClusterStateReader}; diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index 58626c2..5b343c8 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -2,35 +2,53 @@ use rmp_serde::decode::Error; use protocol::Term; use event::{FloEvent, FloEventId, OwnedFloEvent, EventCounter, ActorId, Timestamp, time}; +use engine::event_stream::partition::PersistentEvent; #[derive(Debug, PartialEq)] pub struct SystemEvent { + term: Term, wrapped: E, } impl SystemEvent { pub fn from_event(event: E) -> Result, Error> { - { - let _ = ::rmp_serde::decode::from_slice::(event.data())?; - } + let term = { + + let data = ::rmp_serde::decode::from_slice::(event.data())?; + data.term + + }; Ok(SystemEvent{ + term, wrapped: event }) } + pub fn term(&self) -> Term { + self.term + } + pub fn system_data(&self) -> Result { ::rmp_serde::from_slice(self.data()) } } +impl Into for SystemEvent { + fn into(self) -> PersistentEvent { + self.wrapped + } +} + impl SystemEvent { pub fn new(id: FloEventId, parent: Option, namespace: String, time: Timestamp, data: &SystemEventData) -> SystemEvent { + let term = data.term; // TODO: I feel like this is probably a safe unwrap, but might be good to double check let data = ::rmp_serde::to_vec(data).unwrap(); let event = OwnedFloEvent::new(id, parent, time, namespace, data); SystemEvent { + term, wrapped: event } } diff --git a/flo-server/src/engine/controller/system_reader.rs b/flo-server/src/engine/controller/system_reader.rs index 1672ff3..72daa4b 100644 --- a/flo-server/src/engine/controller/system_reader.rs +++ b/flo-server/src/engine/controller/system_reader.rs @@ -1,8 +1,9 @@ use std::io; -use std::vec::Drain; +use event::FloEvent; use engine::ConnectionId; use engine::event_stream::partition::{PartitionReader, SegmentNum, SharedReaderRefs, EventFilter, PersistentEvent}; +use engine::controller::system_event::SystemEvent; use atomics::AtomicCounterReader; /// The max number of events that will be sent with a single AppendEntries call. @@ -14,16 +15,13 @@ pub const SYSTEM_READER_BATCH_SIZE: usize = 8; #[derive(Debug)] pub struct SystemStreamReader { inner: PartitionReader, - event_buffer: Vec, } impl SystemStreamReader { pub fn new(connection_id: ConnectionId, shared_refs: SharedReaderRefs, commit_index_reader: AtomicCounterReader) -> SystemStreamReader { - use engine::event_stream::partition::EventFilter; let part = PartitionReader::new(connection_id, 0, EventFilter::All, None, shared_refs, commit_index_reader); SystemStreamReader { inner: part, - event_buffer: Vec::with_capacity(SYSTEM_READER_BATCH_SIZE), } } @@ -32,20 +30,22 @@ impl SystemStreamReader { self.inner.set_to(segment, offset) } - pub fn fill_buffer(&mut self) -> io::Result { - while self.event_buffer.len() < SYSTEM_READER_BATCH_SIZE { + pub fn fill_buffer(&mut self, event_buffer: &mut Vec>) -> io::Result { + event_buffer.clear(); + while event_buffer.len() < SYSTEM_READER_BATCH_SIZE { let next = self.inner.next(); if let Some(next_result) = next { let event = next_result?; // return if read failed - self.event_buffer.push(event); + let event_id = *event.id(); + let system_event = SystemEvent::from_event(event).map_err(|des_err| { + error!("Error deserializing system event: {}, err: {:?}", event_id, des_err); + io::Error::new(io::ErrorKind::InvalidData, des_err) + })?; + event_buffer.push(system_event); } else { break; } } - Ok(self.event_buffer.len()) - } - - pub fn drain(&mut self) -> Drain { - self.event_buffer.drain(..) + Ok(event_buffer.len()) } } From 1aa8d21ea80ed4be1c20729188db9a0482467e1a Mon Sep 17 00:00:00 2001 From: pfried Date: Wed, 10 Jan 2018 09:12:07 -0500 Subject: [PATCH 45/73] WIP on sending AppendEntries --- flo-server/src/atomics/atomic_counter.rs | 6 + .../src/engine/connection_handler/input.rs | 1 - .../engine/controller/cluster_state/mod.rs | 70 ++++--- .../cluster_state/peer_connections.rs | 154 +++++++++++++-- .../src/engine/controller/controller_state.rs | 177 ++++++++++++++++++ flo-server/src/engine/controller/mod.rs | 59 +++--- .../event_stream/partition/controller/mod.rs | 21 ++- .../partition/event_reader/mod.rs | 6 + .../src/engine/event_stream/partition/mod.rs | 1 + .../event_stream/partition/segment/mmap.rs | 4 + .../event_stream/partition/segment/mod.rs | 8 + 11 files changed, 424 insertions(+), 83 deletions(-) create mode 100644 flo-server/src/engine/controller/controller_state.rs diff --git a/flo-server/src/atomics/atomic_counter.rs b/flo-server/src/atomics/atomic_counter.rs index 3fa67a0..176d563 100644 --- a/flo-server/src/atomics/atomic_counter.rs +++ b/flo-server/src/atomics/atomic_counter.rs @@ -44,6 +44,12 @@ impl AtomicCounterWriter { self.inner.fetch_add(amount, ordering) } + /// returns the current value of the counter using relaxed ordering since it's guaranteed that there is only one Writer + #[allow(unused_mut)] + pub fn get(&self) -> usize { + self.inner.load(Ordering::Relaxed) + } + /// creates a reader for the value. The reader is only able to read the value, and can never mutate it pub fn reader(&self) -> AtomicCounterReader { AtomicCounterReader { diff --git a/flo-server/src/engine/connection_handler/input.rs b/flo-server/src/engine/connection_handler/input.rs index 9cdaa6c..e0c8405 100644 --- a/flo-server/src/engine/connection_handler/input.rs +++ b/flo-server/src/engine/connection_handler/input.rs @@ -59,5 +59,4 @@ pub struct AppendEntriesStart { pub prev_entry_term: Term, pub reader_start_offset: usize, pub reader_start_segment: SegmentNum, - pub reader_start_event: EventCounter, } diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 6492b4e..8edb916 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -14,7 +14,7 @@ use engine::connection_handler::ConnectionControl; use engine::controller::{CallRequestVote, VoteResponse}; use protocol::{FloInstanceId, Term}; use atomics::{AtomicBoolWriter, AtomicBoolReader}; -use super::{ClusterOptions, ConnectionRef, Peer, PeerUpgrade, AppendEntriesResponse, ReceiveAppendEntries}; +use super::{ClusterOptions, ConnectionRef, Peer, PeerUpgrade, AppendEntriesResponse, ReceiveAppendEntries, ControllerState}; use super::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator, OutgoingConnectionCreatorImpl}; use self::peer_connections::{PeerConnectionManager, PeerConnections}; @@ -29,16 +29,16 @@ static STATE_UPDATE_FAILED: &'static str = "failed to persist changes to cluster pub trait ConsensusProcessor: Send { - fn tick(&mut self, now: Instant, all_connections: &mut HashMap); + fn tick(&mut self, now: Instant, controller_state: &mut ControllerState); fn is_primary(&self) -> bool; - fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap); + fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, controller_state: &ControllerState); fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr); fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote); fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse); - fn append_entries_received(&mut self, append: ReceiveAppendEntries); - fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse); + fn append_entries_received(&mut self, append: ReceiveAppendEntries, controller_state: &mut ControllerState); + fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState); } #[derive(Debug)] @@ -191,6 +191,11 @@ impl ClusterManager { let mut lock = self.shared.write().unwrap(); lock.system_primary = primary; } + + fn send_append_entries(&mut self, controller_state: &mut ControllerState) { + let term = self.persistent.current_term; + self.connection_manager.broadcast_append_entries(term, controller_state); + } } impl ConsensusProcessor for ClusterManager { @@ -243,9 +248,9 @@ impl ConsensusProcessor for ClusterManager { } } - fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap) { + fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, controller_state: &ControllerState) { debug!("peer_connection_established: connection_id: {}, {:?}", connection_id, upgrade); - let connection = all_connections.get(&connection_id) + let connection = controller_state.get_connection(connection_id) .expect("Expected connection to be in all_connections on peer_connection_established"); self.connection_manager.peer_connection_established(upgrade.peer_id, connection); @@ -256,8 +261,8 @@ impl ConsensusProcessor for ClusterManager { self.connection_resolved(connection.remote_address); } - fn tick(&mut self, now: Instant, all_connections: &mut HashMap) { - self.connection_manager.establish_connections(now, all_connections); + fn tick(&mut self, now: Instant, controller_state: &mut ControllerState) { + self.connection_manager.establish_connections(now, controller_state); match self.state { State::EstablishConnections => { @@ -269,6 +274,7 @@ impl ConsensusProcessor for ClusterManager { } State::Primary => { trace!("This instance is primary"); + self.send_append_entries(controller_state); } other @ _ => { if self.election_timed_out(now) { @@ -282,11 +288,11 @@ impl ConsensusProcessor for ClusterManager { self.state == State::Primary } - fn append_entries_received(&mut self, append: ReceiveAppendEntries) { + fn append_entries_received(&mut self, append: ReceiveAppendEntries, controller_state: &mut ControllerState) { unimplemented!() } - fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse) { + fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState) { unimplemented!() } } @@ -350,11 +356,12 @@ mod test { use std::path::{Path, PathBuf}; use std::collections::HashSet; use tempdir::TempDir; - use engine::connection_handler::{ConnectionControlSender, ConnectionControlReceiver, ConnectionControl}; + use engine::connection_handler::{ConnectionControlSender, ConnectionControlReceiver, ConnectionControl, CallAppendEntries}; use engine::controller::peer_connection::{OutgoingConnectionCreator, MockOutgoingConnectionCreator}; use test_utils::{addr, expect_future_resolved}; use engine::controller::cluster_state::peer_connections::mock::{MockPeerConnectionManager, Invocation}; use engine::controller::controller_messages::mock::mock_connection_ref; + use engine::controller::mock::MockControllerState; #[test] fn append_entries_is_sent_on_tick_when_state_is_primary() { @@ -383,8 +390,13 @@ mod test { }).unwrap(); subject.last_applied = 99; subject.last_applied_term = 4; - subject.state = State::Voted; + subject.state = State::Primary; subject.last_heartbeat = start; + + let mut controller_state = MockControllerState::with_commit_index(101); + subject.send_append_entries(&mut controller_state); + + connection_manager.verify_in_order(&Invocation::BroadcastAppendEntries { term: 5 }) } #[test] @@ -419,8 +431,8 @@ mod test { subject.state = State::Voted; subject.last_heartbeat = start; - let mut connections = HashMap::new(); - subject.tick(t_sec(start, 1), &mut connections); + let mut controller_state = MockControllerState::new(); + subject.tick(t_sec(start, 1), &mut controller_state); connection_manager.verify_in_order(&Invocation::EstablishConnections); connection_manager.verify_in_order(&Invocation::BroadcastToPeers { @@ -609,9 +621,9 @@ mod test { subject.last_applied_term = 5; subject.last_applied = 9; - let mut connections = HashMap::new(); + let mut controller_state = MockControllerState::new(); let election_start = t_sec(start, 1); - subject.tick(election_start, &mut connections); + subject.tick(election_start, &mut controller_state); connection_manager.verify_in_order(&Invocation::EstablishConnections); connection_manager.verify_in_order(&Invocation::BroadcastToPeers { connection_control: ConnectionControl::SendRequestVote(CallRequestVote { @@ -793,8 +805,8 @@ mod test { state.current_term = 7; }).unwrap(); - let mut connections = HashMap::new(); - subject.tick(t_sec(start, 1), &mut connections); + let mut controller_state = MockControllerState::new(); + subject.tick(t_sec(start, 1), &mut controller_state); connection_manager.verify_in_order(&Invocation::EstablishConnections); let this_id = subject.persistent.this_instance_id; @@ -827,8 +839,8 @@ mod test { let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); assert_eq!(State::EstablishConnections, subject.state); - let mut connections = HashMap::new(); - subject.tick(t_sec(start, 1), &mut connections); + let mut controller_state = MockControllerState::new(); + subject.tick(t_sec(start, 1), &mut controller_state); assert_eq!(State::DeterminePrimary, subject.state); connection_manager.verify_in_order(&Invocation::EstablishConnections); @@ -839,9 +851,9 @@ mod test { cluster_members: vec![peer_2.clone()], }; let (connection, _) = mock_connection_ref(conn_id, peer_1.address); - connections.insert(conn_id, connection.clone()); + controller_state.add_connection(connection.clone()); - subject.peer_connection_established(upgrade, conn_id, &connections); + subject.peer_connection_established(upgrade, conn_id, &controller_state); connection_manager.verify_in_order(&Invocation::PeerConnectionEstablished { peer_id: peer_1.id, success_connection: connection, @@ -865,8 +877,8 @@ mod test { let mut subject = create_cluster_manager(vec![peer_1_addr, peer_2_addr], temp_dir.path(), connection_manager.boxed_ref()); assert_eq!(State::EstablishConnections, subject.state); - let mut connections = HashMap::new(); - subject.tick(t_sec(start, 1), &mut connections); + let mut controller_state = MockControllerState::new(); + subject.tick(t_sec(start, 1), &mut controller_state); connection_manager.verify_in_order(&Invocation::EstablishConnections); assert_eq!(State::DeterminePrimary, subject.state); @@ -1000,7 +1012,7 @@ pub struct NoOpConsensusProcessor; // TODO: reasonable and consistent behavior for calls into NoOpConsensusProcessor that should never actually happen impl ConsensusProcessor for NoOpConsensusProcessor { - fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, all_connections: &HashMap) { + fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, controller_state: &ControllerState) { } fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); @@ -1008,7 +1020,7 @@ impl ConsensusProcessor for NoOpConsensusProcessor { fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote) { } - fn tick(&mut self, now: Instant, all_connections: &mut HashMap) { + fn tick(&mut self, now: Instant, controller_state: &mut ControllerState) { } fn is_primary(&self) -> bool { true @@ -1016,11 +1028,11 @@ impl ConsensusProcessor for NoOpConsensusProcessor { fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse) { } - fn append_entries_received(&mut self, append: ReceiveAppendEntries) { + fn append_entries_received(&mut self, append: ReceiveAppendEntries, controller_state: &mut ControllerState) { unimplemented!() } - fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse) { + fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState) { unimplemented!() } } diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs index b36053b..2d7f34d 100644 --- a/flo-server/src/engine/controller/cluster_state/peer_connections.rs +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -2,19 +2,22 @@ use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; use std::time::{Instant, Duration}; use std::fmt::Debug; +use std::io; use event::EventCounter; use protocol::{FloInstanceId, Term}; use engine::ConnectionId; -use engine::controller::{ConnectionRef, Peer, CallRequestVote}; +use engine::controller::{ConnectionRef, Peer, CallRequestVote, ControllerState}; use engine::controller::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator}; -use engine::connection_handler::ConnectionControl; +use engine::connection_handler::{ConnectionControl, CallAppendEntries, AppendEntriesStart}; pub trait PeerConnectionManager: Send + Debug + 'static { - fn establish_connections(&mut self, now: Instant, all_connections: &mut HashMap); + fn establish_connections(&mut self, now: Instant, controller_state: &mut ControllerState); fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, addr: SocketAddr); fn connection_closed(&mut self, connection_id: ConnectionId); fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef); + + fn broadcast_append_entries(&mut self, term: Term, controller_state: &mut ControllerState); fn broadcast_to_peers(&mut self, connection_control: ConnectionControl); fn send_to_peer(&mut self, peer_id: FloInstanceId, connection_control: ConnectionControl); fn get_peer_id(&mut self, connection_id: ConnectionId) -> Option; @@ -53,18 +56,39 @@ impl PeerConnections { outgoing_connection_creator, } } + +} + +fn start_after_commit_index(controller_state: &mut ControllerState, peer: &mut Connection, commit_index: EventCounter) -> io::Result { + // safe unwrap of the option, since we're asking for the last committed event + let result = controller_state.get_system_event(commit_index).unwrap(); + result.map(|event| { + let next_entry = controller_state.get_next_entry(commit_index); + let (reader_start_segment, reader_start_offset) = next_entry.map(|entry| { + (entry.segment, entry.file_offset) + }).unwrap_or_else(|| { + controller_state.get_current_file_offset() + }); + + AppendEntriesStart { + prev_entry_index: commit_index, + prev_entry_term: event.term(), + reader_start_segment, + reader_start_offset, + } + }) } impl PeerConnectionManager for PeerConnections { - fn establish_connections(&mut self, now: Instant, all_connections: &mut HashMap) { + fn establish_connections(&mut self, now: Instant, controller_state: &mut ControllerState) { let PeerConnections {ref mut disconnected_peers, ref mut outgoing_connection_creator, ..} = *self; for (address, attempt) in disconnected_peers.iter_mut() { if attempt.should_try_connect(now) { debug!("Making outgoing connection attempt #{} to address: {}", attempt.attempt_count + 1, address); let connection_ref = outgoing_connection_creator.establish_system_connection(*address); send(&connection_ref, ConnectionControl::InitiateOutgoingSystemConnection); - all_connections.insert(connection_ref.connection_id, connection_ref.clone()); + controller_state.add_connection(connection_ref.clone()); attempt.attempt_time = now; attempt.attempt_count += 1; attempt.connection = Some(connection_ref); @@ -146,6 +170,36 @@ impl PeerConnectionManager for PeerConnections { } } + fn broadcast_append_entries(&mut self, term: Term, controller_state: &mut ControllerState) { + + let commit_index = controller_state.get_system_commit_index(); + for (peer_id, peer) in self.known_peers.iter_mut() { + if peer.is_connected() { + let start = if peer.last_acknowledged_index.is_none() { + match start_after_commit_index(controller_state, peer, commit_index) { + Ok(start_after) => Some(start_after), + Err(io_err) => { + error!("Failed to create AppendEntriesStart, refusing to send any more AppendEntries this time: {:?}", io_err); + return; + } + } + } else { + None + }; + + let append = CallAppendEntries { + current_term: term, + reader_start_position: start, + commit_index, + }; + + peer.send_append_entries(append); + } else { + debug!("Not sending AppendEntries to peer_id: {} because it is disconnected", peer_id); + } + } + } + fn send_to_peer(&mut self, peer_id: FloInstanceId, control: ConnectionControl) { unimplemented!() } @@ -196,6 +250,7 @@ impl ConnectionAttempt { #[derive(Debug)] struct Connection { + last_acknowledged_index: Option, peer_address: SocketAddr, state: PeerState, } @@ -203,10 +258,33 @@ struct Connection { impl Connection { fn new(peer_address: SocketAddr) -> Connection { Connection { + last_acknowledged_index: None, peer_address, state: PeerState::Init } } + + fn is_connected(&self) -> bool { + match self.state { + PeerState::Connected(_) => true, + _ => false + } + } + + fn send_append_entries(&mut self, append: CallAppendEntries) { + let send_err = match &mut self.state { + &mut PeerState::Connected(ref mut conn) => { + let result = conn.control_sender.send(ConnectionControl::SendAppendEntries(append)); + result.is_err() + } + other @ _ => { + panic!("Attempted to send AppendEntries to disconnected peer in state: {:?}", other); + } + }; + if send_err { + warn!("Failed to send AppendEntries control message to connectionHandler for peer at: {}", self.peer_address); + } + } } #[derive(Debug)] @@ -223,6 +301,8 @@ mod test { use super::*; use std::time::Duration; use engine::controller::peer_connection::MockOutgoingConnectionCreator; + use engine::controller::mock::MockControllerState; + use engine::controller::ControllerState; use engine::connection_handler::ConnectionControlReceiver; use test_utils::{addr, expect_future_resolved}; @@ -234,15 +314,41 @@ mod test { stream } - fn subject_with_connected_peers(peers: &[Peer], creator: MockOutgoingConnectionCreator) -> (PeerConnections, HashMap) { + fn subject_with_connected_peers(peers: &[Peer], creator: MockOutgoingConnectionCreator) -> (PeerConnections, MockControllerState) { let mut subject = PeerConnections::new(Vec::new(), creator.boxed(), &peers.iter().cloned().collect()); - let mut connections = HashMap::new(); - subject.establish_connections(Instant::now(), &mut connections); + let mut controller_state = MockControllerState::new(); + subject.establish_connections(Instant::now(), &mut controller_state); for peer in peers { - let connection = connections.values().find(|conn| conn.remote_address == peer.address).unwrap(); + let connection = controller_state.all_connections.values() + .find(|conn| conn.remote_address == peer.address) + .unwrap(); subject.peer_connection_established(peer.id, connection); } - (subject, connections) + (subject, controller_state) + } + + #[test] + fn broadcast_append_entries_sends_append_entries_control_to_all_connected_peers() { + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("123.4.5.6:3000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("123.4.5.6:4000") + }; + let mut creator = MockOutgoingConnectionCreator::new(); + let (peer_1_conn, rx_1) = creator.stub(peer_1.address); + let (peer_2_conn, rx_2) = creator.stub(peer_2.address); + let (mut subject, mut controller_state) = subject_with_connected_peers(&[peer_1, peer_2], creator); + controller_state.set_commit_index(7); + + // TODO: stub out return values for controller_state + + subject.broadcast_append_entries(7, &mut controller_state); + + + // TODO: verify that append entries was sent } #[test] @@ -285,9 +391,9 @@ mod test { let mut subject = PeerConnections::new(vec![peer_address], creator.boxed(), &HashSet::new()); - let mut connections = HashMap::new(); + let mut controller_state = MockControllerState::new(); let time = Instant::now(); - subject.establish_connections(time, &mut connections); + subject.establish_connections(time, &mut controller_state); subject.peer_connection_established(peer_id, &peer_connection); @@ -323,11 +429,11 @@ mod test { creator.stub(peer_address); let mut subject = PeerConnections::new(new_peers, creator.boxed(), &HashSet::new()); - let mut connections = HashMap::new(); + let mut controller_state = MockControllerState::new(); let time = Instant::now(); - subject.establish_connections(time, &mut connections); + subject.establish_connections(time, &mut controller_state); - assert_eq!(1, connections.len()); + assert_eq!(1, controller_state.all_connections.len()); { let attempt = subject.disconnected_peers.get(&peer_address).unwrap(); assert_eq!(time, attempt.attempt_time); @@ -428,6 +534,18 @@ pub mod mock { } } + pub fn verify_any_order(&self, expected: &Invocation) { + // lock will be poisoned if this panics + let mut lock = self.actual_invocations.lock().unwrap(); + + let index = lock.iter().position(|actual| actual == expected); + if let Some(idx) = index { + lock.remove(idx); + } else { + panic!("Expected: {:?} in any order, other invocations on this mock: {:?}", expected, lock.as_slices()); + } + } + pub fn boxed_ref(&self) -> Box { Box::new(self.clone()) } @@ -439,7 +557,7 @@ pub mod mock { } impl PeerConnectionManager for MockPeerConnectionManager { - fn establish_connections(&mut self, _now: Instant, _all_connections: &mut HashMap) { + fn establish_connections(&mut self, _now: Instant, _controller_state: &mut ControllerState) { self.push_invocation(Invocation::EstablishConnections); } fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, addr: SocketAddr) { @@ -466,6 +584,9 @@ pub mod mock { let lock = self.peer_stubs.lock().unwrap(); lock.get(&connection_id).cloned() } + fn broadcast_append_entries(&mut self, term: Term, controller_state: &mut ControllerState) { + self.push_invocation(Invocation::BroadcastAppendEntries {term}) + } } @@ -481,5 +602,6 @@ pub mod mock { PeerConnectionEstablished{peer_id: FloInstanceId, success_connection: ConnectionRef}, BroadcastToPeers{connection_control: ConnectionControl}, SendToPeer{peer_id: FloInstanceId, connection_control :ConnectionControl}, + BroadcastAppendEntries { term: Term }, } } diff --git a/flo-server/src/engine/controller/controller_state.rs b/flo-server/src/engine/controller/controller_state.rs new file mode 100644 index 0000000..0fcd326 --- /dev/null +++ b/flo-server/src/engine/controller/controller_state.rs @@ -0,0 +1,177 @@ +use std::sync::{Arc, Mutex}; +use std::path::PathBuf; +use std::collections::HashMap; +use std::io; + +use event::EventCounter; +use engine::ConnectionId; +use engine::controller::{ConnectionRef, SystemEvent}; +use engine::event_stream::{EventStreamRef, EventStreamRefMut, EventStreamOptions}; +use engine::event_stream::partition::{SegmentNum, IndexEntry, PersistentEvent, PartitionReader}; +use engine::event_stream::partition::controller::PartitionImpl; + + +pub trait ControllerState { + fn add_connection(&mut self, connection: ConnectionRef); + fn remove_connection(&mut self, connection_id: ConnectionId); + fn get_connection(&self, connection_id: ConnectionId) -> Option<&ConnectionRef>; + + fn set_commit_index(&mut self, new_index: EventCounter); + fn get_system_commit_index(&self) -> EventCounter; + fn get_system_event(&mut self, event_counter: EventCounter) -> Option>>; + fn get_next_entry(&self, event_counter: EventCounter) -> Option; + fn get_current_file_offset(&self) -> (SegmentNum, usize); +} + +pub struct ControllerStateImpl { + /// Shared references to all event streams in the system + pub shared_event_stream_refs: Arc>>, + + /// Unique mutable references to every event stream in the system + pub event_streams: HashMap, + + /// used as defaults when creating new event streams + pub default_stream_options: EventStreamOptions, + + /// directory in which all event stream data is stored + pub storage_dir: PathBuf, + + /// the partition that persists system events. Used as the RAFT log + pub system_partition: PartitionImpl, + + /// a reader for the system partition, since we'll occasionally need to look up events + pub system_partition_reader: PartitionReader, + + /// Stores all of the active connections with this instance + pub all_connections: HashMap, +} + +impl ControllerStateImpl { + pub fn new(mut system_partition: PartitionImpl, + event_streams: HashMap, + shared_event_stream_refs: Arc>>, + storage_dir: PathBuf, + default_stream_options: EventStreamOptions) -> ControllerStateImpl { + + let system_partition_reader = system_partition.create_reader(0, ::engine::event_stream::partition::EventFilter::All, 0); + + ControllerStateImpl { + shared_event_stream_refs, + event_streams, + default_stream_options, + storage_dir, + system_partition, + system_partition_reader, + all_connections: HashMap::new(), + } + } + + fn read_event(&mut self, entry: IndexEntry) -> io::Result> { + self.system_partition_reader.set_to(entry.segment, entry.file_offset)?; + let result = self.system_partition_reader.read_next_uncommitted().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, format!("Expected to find an event for {:?} but no event was read", entry)) + })?; + result.and_then(|event| { + SystemEvent::from_event(event).map_err(|des_err| { + error!("Failed to deserialize system event data for {:?}: {:?}", entry, des_err); + io::Error::new(io::ErrorKind::InvalidData, format!("Failed to deserialize system event data: {:?}", des_err)) + }) + }) + } +} + +impl ControllerState for ControllerStateImpl { + + fn add_connection(&mut self, connection: ConnectionRef) { + self.all_connections.insert(connection.connection_id, connection); + } + + fn remove_connection(&mut self, connection_id: ConnectionId) { + self.all_connections.remove(&connection_id); + } + + fn get_connection(&self, connection_id: ConnectionId) -> Option<&ConnectionRef> { + self.all_connections.get(&connection_id) + } + + fn get_system_commit_index(&self) -> EventCounter { + self.system_partition.get_commit_index() + } + + fn set_commit_index(&mut self, new_index: EventCounter) { + self.system_partition.set_commit_index(new_index); + } + + fn get_system_event(&mut self, event_counter: EventCounter) -> Option>> { + self.get_next_entry(event_counter.saturating_sub(1)).and_then(|index_entry| { + if index_entry.counter == event_counter { + Some(self.read_event(index_entry)) + } else { + None + } + }) + } + + fn get_next_entry(&self, previous: EventCounter) -> Option { + self.system_partition.get_next_index_entry(previous) + } + + fn get_current_file_offset(&self) -> (SegmentNum, usize) { + self.system_partition.get_head_position() + } +} + + + +#[cfg(test)] +pub mod mock { + use super::*; + + pub struct MockControllerState { + pub commit_index: EventCounter, + pub all_connections: HashMap, + } + + impl MockControllerState { + pub fn new() -> MockControllerState { + MockControllerState::with_commit_index(0) + } + + pub fn with_commit_index(index: EventCounter) -> MockControllerState { + MockControllerState { + commit_index: index, + all_connections: HashMap::new(), + } + } + } + + impl ControllerState for MockControllerState { + fn add_connection(&mut self, connection: ConnectionRef) { + self.all_connections.insert(connection.connection_id, connection); + } + fn remove_connection(&mut self, connection_id: ConnectionId) { + self.all_connections.remove(&connection_id); + } + fn get_connection(&self, connection_id: ConnectionId) -> Option<&ConnectionRef> { + self.all_connections.get(&connection_id) + } + fn get_system_commit_index(&self) -> EventCounter { + unimplemented!() + } + + fn get_system_event(&mut self, event_counter: EventCounter) -> Option>> { + unimplemented!() + } + fn get_next_entry(&self, event_counter: EventCounter) -> Option { + unimplemented!() + } + + fn get_current_file_offset(&self) -> (SegmentNum, usize) { + unimplemented!() + } + fn set_commit_index(&mut self, new_index: EventCounter) { + unimplemented!() + } + } + +} diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 4e650ad..df63e85 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -6,22 +6,21 @@ mod initialization; mod controller_messages; mod peer_connection; mod system_reader; +mod controller_state; use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; use std::collections::HashMap; -use std::net::SocketAddr; - +use std::io; +use event::EventCounter; use engine::ConnectionId; use engine::event_stream::{EventStreamRef, EventStreamRefMut, EventStreamOptions}; - -use engine::event_stream::partition::Operation; +use engine::event_stream::partition::{PersistentEvent, IndexEntry, SegmentNum}; use engine::event_stream::partition::controller::PartitionImpl; -use atomics::AtomicBoolWriter; -use self::cluster_state::{ClusterManager, ConsensusProcessor}; +use self::cluster_state::ConsensusProcessor; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; @@ -30,6 +29,10 @@ pub use self::system_event::{SystemEvent, SystemEventData}; pub use self::system_reader::{SystemStreamReader, SYSTEM_READER_BATCH_SIZE}; pub use self::controller_messages::*; pub use self::cluster_state::{SharedClusterState, ClusterStateReader}; +pub use self::controller_state::{ControllerState, ControllerStateImpl}; + +#[cfg(test)] +pub use self::controller_state::mock; pub type SystemPartitionSender = ::std::sync::mpsc::Sender; pub type SystemPartitionReceiver = ::std::sync::mpsc::Receiver; @@ -38,28 +41,14 @@ pub fn create_system_partition_channels() -> (SystemPartitionSender, SystemParti ::std::sync::mpsc::channel() } + + /// A specialized event stream that always has exactly one partition and manages the cluster state and consensus /// Of course there is no cluster state and thus no consensus at the moment, but we'll just leave this here... #[allow(dead_code)] pub struct FloController { - /// Shared references to all event streams in the system - shared_event_stream_refs: Arc>>, - - /// Unique mutable references to every event stream in the system - event_streams: HashMap, - - /// used as defaults when creating new event streams - default_stream_options: EventStreamOptions, - - /// directory in which all event stream data is stored - storage_dir: PathBuf, - - /// the partition that persists system events. Used as the RAFT log - system_partition: PartitionImpl, - + controller_state: ControllerStateImpl, cluster_state: Box, - - all_connections: HashMap, } impl FloController { @@ -70,14 +59,12 @@ impl FloController { cluster_state: Box, default_stream_options: EventStreamOptions) -> FloController { + let controller_state = ControllerStateImpl::new(system_partition, event_streams, shared_stream_refs, + storage_dir, default_stream_options); + FloController { - shared_event_stream_refs: shared_stream_refs, - event_streams, - system_partition, - storage_dir, - default_stream_options, cluster_state, - all_connections: HashMap::with_capacity(4), + controller_state, } } @@ -92,19 +79,19 @@ impl FloController { match op_type { SystemOpType::IncomingConnectionEstablished(connection_ref) => { - self.all_connections.insert(connection_id, connection_ref); + self.controller_state.all_connections.insert(connection_id, connection_ref); } SystemOpType::OutgoingConnectionFailed(address) => { - self.all_connections.remove(&connection_id); + self.controller_state.all_connections.remove(&connection_id); self.cluster_state.outgoing_connection_failed(connection_id, address); } SystemOpType::ConnectionUpgradeToPeer(upgrade) => { - let FloController{ref mut cluster_state, ref mut all_connections, ..} = *self; - cluster_state.peer_connection_established(upgrade, connection_id, all_connections); + let FloController{ref mut cluster_state, ref mut controller_state, ..} = *self; + cluster_state.peer_connection_established(upgrade, connection_id, controller_state); } SystemOpType::Tick => { - let FloController{ref mut cluster_state, ref mut all_connections, ..} = *self; - cluster_state.tick(op_start_time, all_connections); + let FloController{ref mut cluster_state, ref mut controller_state, ..} = *self; + cluster_state.tick(op_start_time, controller_state); } other @ _ => { warn!("Ignoring SystemOperation: {:?}", other); @@ -114,9 +101,11 @@ impl FloController { fn shutdown(&mut self) { info!("Shutting down FloController"); + //TODO: either do something on shutdown or delete this function } } + diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index a06d06e..b84c0aa 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -140,6 +140,14 @@ impl PartitionImpl { self.partition_highest_committed.reader() } + pub fn set_commit_index(&mut self, new_index: EventCounter) { + self.partition_highest_committed.set_if_greater(new_index as usize); + } + + pub fn get_commit_index(&self) -> EventCounter { + self.partition_highest_committed.get() as EventCounter + } + pub fn primary_status_reader(&self) -> AtomicBoolReader { self.primary.clone() } @@ -306,6 +314,12 @@ impl PartitionImpl { self.reader_refs.get_reader_refs() } + pub fn get_head_position(&self) -> (SegmentNum, usize) { + self.segments.front().map(|s| { + (s.segment_num, s.head_position()) + }).unwrap_or((SegmentNum(0), 0)) + } + fn current_segment_num(&self) -> SegmentNum { self.segments.front().map(|s| s.segment_num).unwrap_or(SegmentNum(0)) } @@ -323,9 +337,9 @@ impl PartitionImpl { Ok(()) } - fn create_reader(&mut self, connection_id: ConnectionId, filter: EventFilter, start_exclusive: EventCounter) -> PartitionReader { + pub fn create_reader(&mut self, connection_id: ConnectionId, filter: EventFilter, start_exclusive: EventCounter) -> PartitionReader { let current_segment_num = self.current_segment_num(); - let index_entry: Option = self.index.get_next_entry(start_exclusive); + let index_entry: Option = self.get_next_index_entry(start_exclusive); let readers = self.get_shared_reader_refs(); let current_segment = match index_entry { @@ -352,6 +366,9 @@ impl PartitionImpl { commit_index_reader) } + pub fn get_next_index_entry(&self, previous: EventCounter) -> Option { + self.index.get_next_entry(previous) + } } diff --git a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs index d144cc3..5243af1 100644 --- a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs +++ b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs @@ -93,6 +93,12 @@ impl PartitionReader { Ok(()) } + pub fn get_current_file_offset(&self) -> (SegmentNum, usize) { + self.current_segment_reader.as_ref().map(|segment| { + (segment.segment_id, segment.current_offset()) + }).unwrap_or((SegmentNum(0), 0)) + } + fn should_skip(&self, result: &Option>) -> bool { if let Some(Ok(ref event)) = *result { !self.filter.matches(event) diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index 7113142..10fec9a 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -34,6 +34,7 @@ pub use self::ops::{OpType, }; pub use self::event_reader::{PartitionReader, EventFilter}; pub use self::segment::PersistentEvent; +pub use self::index::IndexEntry; pub type PartitionSender = ::std::sync::mpsc::Sender; pub type PartitionReceiver = ::std::sync::mpsc::Receiver; diff --git a/flo-server/src/engine/event_stream/partition/segment/mmap.rs b/flo-server/src/engine/event_stream/partition/segment/mmap.rs index b01e2a0..c285c5e 100644 --- a/flo-server/src/engine/event_stream/partition/segment/mmap.rs +++ b/flo-server/src/engine/event_stream/partition/segment/mmap.rs @@ -223,6 +223,10 @@ impl MmapReader { self.current_offset >= head } + pub fn get_current_offset(&self) -> usize { + self.current_offset + } + fn get_current_head(&self) -> usize { self.inner.head.load(Ordering::Relaxed) } diff --git a/flo-server/src/engine/event_stream/partition/segment/mod.rs b/flo-server/src/engine/event_stream/partition/segment/mod.rs index 4bcf473..8ca9b9b 100644 --- a/flo-server/src/engine/event_stream/partition/segment/mod.rs +++ b/flo-server/src/engine/event_stream/partition/segment/mod.rs @@ -49,6 +49,10 @@ pub struct Segment { impl Segment { + pub fn head_position(&self) -> usize { + self.appender.get_file_position() + } + pub fn is_expired(&self, now: Timestamp) -> bool { now < self.segment_end_time } @@ -178,6 +182,10 @@ impl SegmentReader { pub fn set_offset(&mut self, new_offset: usize) { self.reader.set_offset(new_offset) } + + pub fn current_offset(&self) -> usize { + self.reader.get_current_offset() + } } impl Iterator for SegmentReader { From 1e04eac07079d9652af797c73e3660e786785daf Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 20 Jan 2018 10:36:13 -0500 Subject: [PATCH 46/73] controller receives appendEntries and sends response, still not actually appending any entries, though --- Cargo.lock | 168 ++++++++++-- flo-server/Cargo.toml | 4 +- flo-server/src/atomics/atomic_boolean.rs | 5 + .../engine/controller/cluster_state/mod.rs | 257 ++++++++++++++++-- .../cluster_state/peer_connections.rs | 126 +++------ .../controller/cluster_state/primary_state.rs | 136 +++++++++ .../src/engine/controller/controller_state.rs | 111 +++++--- .../src/engine/event_stream/partition/mod.rs | 5 + flo-server/src/logging.rs | 38 +-- flo-server/src/test_utils.rs | 1 - 10 files changed, 653 insertions(+), 198 deletions(-) create mode 100644 flo-server/src/engine/controller/cluster_state/primary_state.rs diff --git a/Cargo.lock b/Cargo.lock index 56e533f..74c314c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,6 +85,11 @@ dependencies = [ "iovec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "cc" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "cfg-if" version = "0.1.2" @@ -99,6 +104,15 @@ dependencies = [ "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "chrono" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "chunked_transfer" version = "0.3.1" @@ -128,7 +142,7 @@ dependencies = [ [[package]] name = "crossbeam" -version = "0.2.10" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -211,6 +225,15 @@ dependencies = [ "regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "flate2" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "flo-bench-cli" version = "0.2.0" @@ -275,8 +298,8 @@ dependencies = [ "futures 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "log4rs 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log4rs 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "memmap 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -372,7 +395,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "humantime" -version = "0.1.3" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -431,27 +454,51 @@ name = "libc" version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "linked-hash-map" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "log" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "log" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log-mdc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "log4rs" -version = "0.4.8" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "humantime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "flate2 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde-value 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log-mdc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "serde-value 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_yaml 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -478,6 +525,15 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "miniz-sys" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "mio" version = "0.6.11" @@ -581,10 +637,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ordered-float" -version = "0.1.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -699,11 +756,6 @@ name = "scoped-tls" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "serde" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "serde" version = "0.9.15" @@ -716,11 +768,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde-value" -version = "0.2.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ordered-float 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)", + "ordered-float 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -764,6 +816,17 @@ dependencies = [ "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde_yaml" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "linked-hash-map 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "yaml-rust 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "slab" version = "0.3.0" @@ -945,6 +1008,14 @@ name = "unicode-xid" version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unreachable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "unreachable" version = "1.0.0" @@ -1021,11 +1092,30 @@ name = "winapi" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "winapi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "winapi-build" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "ws2_32-sys" version = "0.2.1" @@ -1035,6 +1125,14 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "yaml-rust" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "linked-hash-map 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [metadata] "checksum adler32 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6cbd0b9af8587c72beadc9f72d35b9fbb070982c9e6203e46e93f10df25f8f45" "checksum aho-corasick 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "500909c4f87a9e52355b26626d890833e9e1d53ac566db76c36faa984b889699" @@ -1050,12 +1148,14 @@ dependencies = [ "checksum byteorder 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "96c8b41881888cc08af32d47ac4edd52bc7fa27fef774be47a92443756451304" "checksum byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff81738b726f5d099632ceaffe7fb65b90212e8dce59d518729e7e8634032d3d" "checksum bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d828f97b58cc5de3e40c421d0cf2132d6b2da4ee0e11b8632fa838f0f9333ad6" +"checksum cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "deaf9ec656256bb25b404c51ef50097207b9cbb29c933d31f92cae5a8a0ffee0" "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" "checksum chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)" = "9213f7cd7c27e95c2b57c49f0e69b1ea65b27138da84a170133fd21b07659c00" +"checksum chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c20ebe0b2b08b0aeddba49c609fe7957ba2e33449882cb186a180bc60682fa9" "checksum chunked_transfer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "498d20a7aaf62625b9bf26e637cf7736417cde1d0c99f1d04d1170229a85cf87" "checksum clap 2.27.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1b8c532887f1a292d17de05ae858a8fe50a301e196f9ef0ddb7ccd0d1d00f180" "checksum clocksource 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3a858a8b189b5ab63739bce11781a2dfe68678c362246a0438dcad5002f4e66a" -"checksum crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0c5ea215664ca264da8a9d9c3be80d2eaf30923c259d03e870388eb927508f97" +"checksum crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "24ce9782d4d5c53674646a6a4c1863a21a8fc0cb649b3c94dfc16e45071dea19" "checksum deflate 0.7.17 (registry+https://github.com/rust-lang/crates.io-index)" = "4dddda59aaab719767ab11d3efd9a714e95b610c4445d4435765021e9d52dfb1" "checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab" "checksum encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" @@ -1066,6 +1166,7 @@ dependencies = [ "checksum encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" "checksum encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" "checksum env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b" +"checksum flate2 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9fac2277e84e5e858483756647a9d0aa8d9a2b7cba517fd84325a0aaa69a0909" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum fs2 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ab76cfd2aaa59b7bf6688ad9ba15bbae64bff97f04ea02144cfd3443e5c2866" "checksum fuchsia-zircon 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f6c0581a4e363262e52b87f59ee2afe3415361c6ec35e665924eb08afe8ff159" @@ -1078,7 +1179,7 @@ dependencies = [ "checksum heatmap 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4c9551a9016b91c9b81fbc093e5ad0dd11c80ff4082fd2266170a210c2890051" "checksum histogram 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1bdcec4094c1ca961b685384ea7af76af5718230b3f34657d1a71fd2dcf4cc9d" "checksum hsl 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "575fb7f1167f3b88ed825e90eb14918ac460461fdeaa3965c6a50951dee1c970" -"checksum humantime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6629498cf74d09ee3c5ce8358a1b7bcca486c5b60c179c8ff532f2121573df4f" +"checksum humantime 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0e9298fffb2a54569e1fcb818e9c2ff77caa2fad68d64b6e409b9f777bdb1960" "checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d" "checksum inflate 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d1238524675af3938a7c74980899535854b88ba07907bb1c944abe5b8fc437e5" "checksum iovec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b6e8b9c2247fcf6c6a1151f1156932be5606c9fd6f55a2d7f9fc1cb29386b2f7" @@ -1087,11 +1188,15 @@ dependencies = [ "checksum lazy_static 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "236eb37a62591d4a41a89b7763d7de3e06ca02d5ab2815446a8bae5d2f8c2d57" "checksum lazycell 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3b585b7a6811fb03aa10e74b278a0f00f8dd9b45dc681f148bb29fa5cb61859b" "checksum libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "5ba3df4dcb460b9dfbd070d41c94c19209620c191b0340b929ce748a2bcd42d2" +"checksum linked-hash-map 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2d2aab0478615bb586559b0114d94dd8eca4fdbb73b443adcb0d00b61692b4bf" "checksum log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "880f77541efa6e5cc74e76910c9884d9859683118839d6a1dc3b11e63512565b" -"checksum log4rs 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6a7074be77422e232a2f02470bdab3331187110f54f7e9c05d84741671e0583a" +"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" +"checksum log-mdc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a94d21414c1f4a51209ad204c1776a3d0765002c76c6abcb602a6f09f1e881c7" +"checksum log4rs 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a1f16090a553200fba94e104310b3e53e71f500fd9db7dc2143055aa3cc7ae63" "checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376" "checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" "checksum memmap 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "46f3c7359028b31999287dae4e5047ddfe90a23b7dca2282ce759b491080c99b" +"checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4" "checksum mio 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0e8411968194c7b139e9105bc4ae7db0bae232af087147e72f0616ebf5fdb9cb" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum mpmc 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "cb947c698d784291c6b1d97269b0615beb966178537d4502ce90970507e1cf3b" @@ -1104,7 +1209,7 @@ dependencies = [ "checksum num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "99843c856d68d8b4313b03a17e33c4bb42ae8f6610ea81b28abe076ac721b9b0" "checksum num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "514f0d73e64be53ff320680ca671b64fe3fb91da01e1ae2ddc99eb51d453b20d" "checksum odds 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)" = "c3df9b730298cea3a1c3faa90b7e2f9df3a9c400d0936d6015e6165734eefcba" -"checksum ordered-float 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4d961410be0435ccb80048a6516d95a4b91becde403a957d162f3fba4943b7e3" +"checksum ordered-float 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "58d25b6c0e47b20d05226d288ff434940296e7e2f8b877975da32f862152241f" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" "checksum png 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "48f397b84083c2753ba53c7b56ad023edb94512b2885ffe227c66ff7edb61868" "checksum quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eda5fe9b71976e62bc81b781206aaa076401769b2143379d3eb2118388babac4" @@ -1120,14 +1225,14 @@ dependencies = [ "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum rusttype 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "07b8848db3b5b5ba97020c6a756c0fdf2dbf2ad7c0d06aa4344a3f2f49c3fe17" "checksum scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f417c22df063e9450888a7561788e9bd46d3bb3c1466435b4eccb903807f147d" -"checksum serde 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)" = "1b0e0732aa8ec4267f61815a396a942ba3525062e3bd5520aa8419927cfc0a92" "checksum serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)" = "34b623917345a631dc9608d5194cc206b3fe6c3554cd1c75b937e55e285254af" "checksum serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)" = "1c57ab4ec5fa85d08aaf8ed9245899d9bbdd66768945b21113b84d5f595cb6a1" -"checksum serde-value 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d94076c6c6e05aaf18beaa024fb789f372be9a1dccbcf66e5748fdfe8cb2a00c" +"checksum serde-value 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "71187cf90819445c78d64f749d16499ba210d7724f16a95754dd03e0f207356d" "checksum serde_derive 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)" = "02c92ea07b6e49b959c1481804ebc9bfd92d3c459f1274c9a9546829e42a66ce" "checksum serde_derive_internals 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75c6aac7b99801a16db5b40b7bf0d7e4ba16e76fbf231e32a4677f271cac0603" "checksum serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ad8bcf487be7d2e15d3d543f04312de991d631cfe1b43ea0ade69e6a8a5b16a1" "checksum serde_json 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7cf5b0b5b4bd22eeecb7e01ac2e1225c7ef5e4272b79ee28a8392a8c8489c839" +"checksum serde_yaml 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e0f868d400d9d13d00988da49f7f02aeac6ef00f11901a8c535bd59d777b9e19" "checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" "checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d" "checksum stb_truetype 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fcf3270840fc9de208d63e836eb3fdebb85379e7532f42f1b2cbd505fb6fda08" @@ -1149,6 +1254,7 @@ dependencies = [ "checksum unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "51ccda9ef9efa3f7ef5d91e8f9b83bbe6955f9bf86aec89d5cce2c874625920f" "checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" "checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" +"checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f" "checksum url 0.2.38 (registry+https://github.com/rust-lang/crates.io-index)" = "cbaa8377a162d88e7d15db0cf110c8523453edcbc5bc66d2b6fffccffa34a068" @@ -1159,5 +1265,9 @@ dependencies = [ "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum waterfall 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfd2a19feb20d152820c6d01acfda726c305fa7ea67f685359d24f4d6040729" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +"checksum winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b09fb3b6f248ea4cd42c9a65113a847d612e17505d6ebd1f7357ad68a8bf8693" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" +"checksum winapi-i686-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ec6667f60c23eca65c561e63a13d81b44234c2e38a6b6c959025ee907ec614cc" +"checksum winapi-x86_64-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98f12c52b2630cd05d2c3ffd8e008f7f48252c042b4871c72aed9dc733b96668" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +"checksum yaml-rust 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57ab38ee1a4a266ed033496cf9af1828d8d6e6c1cfa5f643a2809effcae4d628" diff --git a/flo-server/Cargo.toml b/flo-server/Cargo.toml index c279863..add43c6 100644 --- a/flo-server/Cargo.toml +++ b/flo-server/Cargo.toml @@ -8,8 +8,8 @@ flo-event = { path = "../flo-event" } flo-protocol = { path = "../flo-protocol" } flo-client-lib = { path = "../flo-client-lib" } num_cpus = "^1.2" -log = "0.3" -log4rs = "0.4" +log = "0.4" +log4rs = "0.8" clap = "2.5" nom = "2.0" byteorder = "1" diff --git a/flo-server/src/atomics/atomic_boolean.rs b/flo-server/src/atomics/atomic_boolean.rs index a0f5a60..ad4b86c 100644 --- a/flo-server/src/atomics/atomic_boolean.rs +++ b/flo-server/src/atomics/atomic_boolean.rs @@ -18,6 +18,11 @@ impl AtomicBoolWriter { self.inner.swap(value, Ordering::Relaxed) } + #[allow(unused_mut)] + pub fn get(&mut self) -> bool { + self.inner.load(Ordering::Relaxed) + } + pub fn reader(&self) -> AtomicBoolReader { AtomicBoolReader { inner: self.inner.clone() diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 8edb916..878123f 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -1,5 +1,6 @@ pub mod persistent; mod peer_connections; +mod primary_state; use std::io; use std::sync::{Arc, RwLock}; @@ -8,7 +9,7 @@ use std::time::{Instant, Duration}; use std::path::Path; use std::collections::{HashMap, HashSet}; -use event::{EventCounter, ActorId}; +use event::{EventCounter, ActorId, OwnedFloEvent}; use engine::{ConnectionId, EngineRef}; use engine::connection_handler::ConnectionControl; use engine::controller::{CallRequestVote, VoteResponse}; @@ -17,6 +18,7 @@ use atomics::{AtomicBoolWriter, AtomicBoolReader}; use super::{ClusterOptions, ConnectionRef, Peer, PeerUpgrade, AppendEntriesResponse, ReceiveAppendEntries, ControllerState}; use super::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator, OutgoingConnectionCreatorImpl}; use self::peer_connections::{PeerConnectionManager, PeerConnections}; +use self::primary_state::PrimaryState; pub use self::persistent::{FilePersistedState, PersistentClusterState}; @@ -35,15 +37,16 @@ pub trait ConsensusProcessor: Send { fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr); fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote); - fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse); + fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse, controller: &mut ControllerState); - fn append_entries_received(&mut self, append: ReceiveAppendEntries, controller_state: &mut ControllerState); + fn append_entries_received(&mut self, connection_id: ConnectionId, append: ReceiveAppendEntries, controller_state: &mut ControllerState); fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState); } #[derive(Debug)] pub struct ClusterManager { state: State, + primary_state: Option, initialization_peers: HashSet, this_instance_address: SocketAddr, election_timeout: Duration, @@ -55,6 +58,7 @@ pub struct ClusterManager { votes_received: HashSet, system_partition_primary_address: SystemPrimaryAddressRef, shared: Arc>, + current_primary: Option, connection_manager: Box, } @@ -84,10 +88,12 @@ impl ClusterManager { ClusterManager { state: State::EstablishConnections, + primary_state: None, election_timeout: Duration::from_millis(election_timeout_millis), last_heartbeat: Instant::now(), connection_manager: peer_connection_manager, votes_received: HashSet::new(), + current_primary: None, this_instance_address, initialization_peers, last_applied: 0, @@ -104,6 +110,10 @@ impl ClusterManager { self.state = new_state; } + fn update_last_heartbeat_to_now(&mut self) { + self.last_heartbeat = Instant::now(); + } + fn determine_primary_from_peer_upgrade(&mut self, upgrade: PeerUpgrade) { let PeerUpgrade { peer_id, system_primary, .. } = upgrade; if let Some(primary) = system_primary { @@ -177,7 +187,8 @@ impl ClusterManager { election_won } - fn transition_to_primary(&mut self) { + fn transition_to_primary(&mut self, controller: &mut ControllerState) { + info!("Transitioning to system Primary"); self.transition_state(State::Primary); let this_peer = Peer { id: self.persistent.this_instance_id, @@ -185,16 +196,41 @@ impl ClusterManager { }; self.set_new_primary(Some(this_peer)); self.primary_status_writer.set(true); + let primary_state = PrimaryState::new(self.persistent.current_term); + self.primary_state = Some(primary_state); + self.send_append_entries(controller); } fn set_new_primary(&mut self, primary: Option) { + let primary_id = primary.as_ref().map(|p| p.id); + self.current_primary = primary_id; let mut lock = self.shared.write().unwrap(); lock.system_primary = primary; } + fn set_follower_status(&mut self, primary: Option) { + info!("Transitioning to follower state with new primary: {:?}", primary); + self.primary_status_writer.set(false); + self.primary_state = None; + self.set_new_primary(primary); + self.state = State::Follower; + } + fn send_append_entries(&mut self, controller_state: &mut ControllerState) { - let term = self.persistent.current_term; - self.connection_manager.broadcast_append_entries(term, controller_state); + let ClusterManager { ref mut primary_state, ref mut connection_manager, ref persistent, ..} = *self; + + primary_state.as_mut().map(|state| { + let all_peers = persistent.cluster_members.iter().map(|peer| peer.id); + state.send_append_entries(controller_state, connection_manager.as_mut(), all_peers); + }); + } + + fn append_system_events(&mut self, events: &[OwnedFloEvent]) -> io::Result { + // TODO: actually persist the system events from ApendEntries + if !events.is_empty() { + error!("Appending of system events was never implemented!!!, ignoring: {:?}", events); + } + Ok(0) } } @@ -221,7 +257,7 @@ impl ConsensusProcessor for ClusterManager { self.connection_manager.send_to_peer(candidate_id, ConnectionControl::SendVoteResponse(response)); } - fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse) { + fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse, controller: &mut ControllerState) { let peer_id = self.connection_manager.get_peer_id(from); if peer_id.is_none() { error!("Ignoring Vote Response from connection_id: {}: {:?}", from, response); @@ -231,7 +267,7 @@ impl ConsensusProcessor for ClusterManager { if response.granted { if self.count_vote_response(peer_id) { - self.transition_to_primary(); + self.transition_to_primary(controller); } } else { let response_term = response.term; @@ -288,8 +324,55 @@ impl ConsensusProcessor for ClusterManager { self.state == State::Primary } - fn append_entries_received(&mut self, append: ReceiveAppendEntries, controller_state: &mut ControllerState) { - unimplemented!() + fn append_entries_received(&mut self, connection_id: ConnectionId, append: ReceiveAppendEntries, controller_state: &mut ControllerState) { + let from = self.connection_manager.get_peer_id(connection_id); + if from.is_none() { + error!("Received AppendEntries from connection_id: {}, which is not a peer connection, received: {:?}", connection_id, append); + return; + } + let peer_id = from.unwrap(); + let my_current_term = self.persistent.current_term; + + // Check if this message indicates a change of system primary + if self.current_primary.map(|current| current != peer_id).unwrap_or(true) { + // this message indicates a new primary, so we need to transition + // safe unwrap here since we are just receiving a message from this connection + let peer_address = controller_state.get_connection(connection_id).map(|conn| conn.remote_address).unwrap(); + self.set_follower_status(Some(Peer { + id: peer_id, + address: peer_address + })); + } + + self.update_last_heartbeat_to_now(); + + // TODO: check to see if we have an event at the previous index with a matching term + let response = match controller_state.get_next_event(append.prev_entry_index.saturating_sub(1)) { + Some(Ok((actual_prev_index, actual_prev_term))) => { + if actual_prev_index == append.prev_entry_index && actual_prev_term == append.prev_entry_term { + // TODO: Append the new entries + trace!("AppendEntries looks successful, will append: {} entries and return success", append.events.len()); + self.append_system_events(append.events.as_slice()).map(|last_counter| { + Some(last_counter) + }).unwrap_or_else(|io_err| { + error!("Error appending system events: {:?}, returning false", io_err); + None + }) + } else { + info!("AppendEntries: {:?} does not match the prev index/term stored for this instance: index: {}, term: {}", append, actual_prev_index, actual_prev_term); + None + } + } + other @ _ => { + error!("failed to read previous system event at: {} with err: {:?}", append.prev_entry_index, other); + None + } + }; + + self.connection_manager.send_to_peer(peer_id, ConnectionControl::SendAppendEntriesResponse(AppendEntriesResponse { + term: my_current_term, + success: response, + })); } fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState) { @@ -356,12 +439,92 @@ mod test { use std::path::{Path, PathBuf}; use std::collections::HashSet; use tempdir::TempDir; - use engine::connection_handler::{ConnectionControlSender, ConnectionControlReceiver, ConnectionControl, CallAppendEntries}; - use engine::controller::peer_connection::{OutgoingConnectionCreator, MockOutgoingConnectionCreator}; + use engine::connection_handler::{ConnectionControlSender, ConnectionControlReceiver, ConnectionControl, CallAppendEntries, AppendEntriesStart}; use test_utils::{addr, expect_future_resolved}; + use engine::event_stream::partition::SegmentNum; + use engine::controller::peer_connection::{OutgoingConnectionCreator, MockOutgoingConnectionCreator}; use engine::controller::cluster_state::peer_connections::mock::{MockPeerConnectionManager, Invocation}; use engine::controller::controller_messages::mock::mock_connection_ref; - use engine::controller::mock::MockControllerState; + use engine::controller::mock::{MockControllerState, MockSystemEvent}; + + #[test] + fn reverts_to_follower_when_append_entries_is_received_from_another_peer() { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let connection_manager = MockPeerConnectionManager::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:1000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:2000") + }; + let peer_1_connection = 1; + let peer_2_connection = 2; + connection_manager.stub_peer_connection(peer_1_connection, peer_1.id); + connection_manager.stub_peer_connection(peer_2_connection, peer_2.id); + + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + + subject.persistent.modify(|state| { + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + state.current_term = 5; + }).unwrap(); + subject.last_applied = 99; + subject.last_applied_term = 4; + subject.state = State::Primary; + subject.primary_state = Some(PrimaryState::new(5)); + subject.last_heartbeat = start; + + let mock_system_events = vec![ + MockSystemEvent { + id: 1, + term: 1, + segment: SegmentNum::new(1), + file_offset: 0, + }, + MockSystemEvent { + id: 2, + term: 3, + segment: SegmentNum::new(1), + file_offset: 55, + }, + MockSystemEvent { + id: 3, + term: 4, + segment: SegmentNum::new(2), + file_offset: 77, + }, + ]; + + let (peer_1_tx, peer_1_rx) = ::engine::connection_handler::create_connection_control_channels(); + let mut controller_state = MockControllerState::new() + .with_commit_index(2) + .with_mocked_events(mock_system_events.as_slice()) + .with_connection(ConnectionRef { + connection_id: peer_1_connection, + remote_address: peer_1.address, + control_sender: peer_1_tx, + }); + + let append = ReceiveAppendEntries { + term: 6, + prev_entry_index: 3, + prev_entry_term: 4, + commit_index: 3, + events: Vec::new(), + }; + subject.append_entries_received(peer_1_connection, append, &mut controller_state); + + assert_eq!(State::Follower, subject.state); + assert!(subject.primary_state.is_none()); + let actual = subject.shared.read().unwrap(); + assert_eq!(Some(peer_1.clone()), actual.system_primary); + assert_eq!(Some(peer_1.id), subject.current_primary); + assert!(!subject.primary_status_writer.get()) + } #[test] fn append_entries_is_sent_on_tick_when_state_is_primary() { @@ -391,12 +554,52 @@ mod test { subject.last_applied = 99; subject.last_applied_term = 4; subject.state = State::Primary; + subject.primary_state = Some(PrimaryState::new(5)); subject.last_heartbeat = start; - let mut controller_state = MockControllerState::with_commit_index(101); + let mock_system_events = vec![ + MockSystemEvent { + id: 1, + term: 1, + segment: SegmentNum::new(1), + file_offset: 0, + }, + MockSystemEvent { + id: 2, + term: 3, + segment: SegmentNum::new(1), + file_offset: 55, + }, + MockSystemEvent { + id: 3, + term: 4, + segment: SegmentNum::new(2), + file_offset: 77, + }, + ]; + + let mut controller_state = MockControllerState::new().with_commit_index(2).with_mocked_events(mock_system_events.as_slice()); subject.send_append_entries(&mut controller_state); - connection_manager.verify_in_order(&Invocation::BroadcastAppendEntries { term: 5 }) + let expected = CallAppendEntries { + current_term: 5, + commit_index: 2, + reader_start_position: Some(AppendEntriesStart { + prev_entry_index: 2, + prev_entry_term: 3, + reader_start_offset: 77, + reader_start_segment: SegmentNum::new(2), + }), + }; + + connection_manager.verify_any_order(&Invocation::SendToPeer { + peer_id: peer_1.id, + connection_control: ConnectionControl::SendAppendEntries(expected.clone()), + }); + connection_manager.verify_any_order(&Invocation::SendToPeer { + peer_id: peer_2.id, + connection_control: ConnectionControl::SendAppendEntries(expected.clone()), + }); } #[test] @@ -481,10 +684,12 @@ mod test { subject.last_heartbeat = start; assert!(subject.votes_received.is_empty()); + + let mut mock_controller = MockControllerState::new(); subject.vote_response_received(t_millis(start, 4), unknown_connection, VoteResponse{ term: 5, granted: true, - }); + }, &mut mock_controller); assert!(subject.votes_received.is_empty()); } @@ -530,10 +735,11 @@ mod test { subject.state = State::Voted; subject.votes_received.insert(peer_1.id); // simulate one vote having been received already + let mut controller = MockControllerState::new(); subject.vote_response_received(t_millis(start, 5), peer_2_connection, VoteResponse { term: 7, granted: false, - }); + }, &mut controller); assert!(subject.votes_received.is_empty()); assert_eq!(7, subject.persistent.current_term); assert_eq!(State::Follower, subject.state); @@ -570,10 +776,11 @@ mod test { subject.last_heartbeat = start; assert!(subject.votes_received.is_empty()); + let mut mock_controller = MockControllerState::new(); subject.vote_response_received(t_millis(start, 4), peer_1_connection, VoteResponse{ term: 7, granted: false - }); + }, &mut mock_controller); assert!(subject.votes_received.is_empty()); assert_eq!(7, subject.persistent.current_term); } @@ -636,15 +843,17 @@ mod test { subject.vote_response_received(t_millis(election_start, 3), peer_1_connection, VoteResponse { term: 6, granted: true - }); + }, &mut controller_state); assert_eq!(State::Voted, subject.state); subject.vote_response_received(t_millis(election_start, 3), peer_2_connection, VoteResponse { term: 6, granted: true - }); + }, &mut controller_state); assert_eq!(State::Primary, subject.state); - assert!(subject.primary_status_writer.reader().get_relaxed()); + assert!(subject.primary_status_writer.get()); let shared = subject.shared.read().unwrap(); assert!(shared.this_instance_is_primary()); + let subject_id = subject.persistent.this_instance_id; + assert_eq!(Some(subject_id), subject.current_primary); } #[test] @@ -1025,10 +1234,10 @@ impl ConsensusProcessor for NoOpConsensusProcessor { fn is_primary(&self) -> bool { true } - fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse) { - + fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse, controller: &mut ControllerState) { + unimplemented!() } - fn append_entries_received(&mut self, append: ReceiveAppendEntries, controller_state: &mut ControllerState) { + fn append_entries_received(&mut self, connection_id: ConnectionId, append: ReceiveAppendEntries, controller_state: &mut ControllerState) { unimplemented!() } diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs index 2d7f34d..cd92bce 100644 --- a/flo-server/src/engine/controller/cluster_state/peer_connections.rs +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -17,7 +17,6 @@ pub trait PeerConnectionManager: Send + Debug + 'static { fn connection_closed(&mut self, connection_id: ConnectionId); fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef); - fn broadcast_append_entries(&mut self, term: Term, controller_state: &mut ControllerState); fn broadcast_to_peers(&mut self, connection_control: ConnectionControl); fn send_to_peer(&mut self, peer_id: FloInstanceId, connection_control: ConnectionControl); fn get_peer_id(&mut self, connection_id: ConnectionId) -> Option; @@ -59,25 +58,6 @@ impl PeerConnections { } -fn start_after_commit_index(controller_state: &mut ControllerState, peer: &mut Connection, commit_index: EventCounter) -> io::Result { - // safe unwrap of the option, since we're asking for the last committed event - let result = controller_state.get_system_event(commit_index).unwrap(); - result.map(|event| { - let next_entry = controller_state.get_next_entry(commit_index); - let (reader_start_segment, reader_start_offset) = next_entry.map(|entry| { - (entry.segment, entry.file_offset) - }).unwrap_or_else(|| { - controller_state.get_current_file_offset() - }); - - AppendEntriesStart { - prev_entry_index: commit_index, - prev_entry_term: event.term(), - reader_start_segment, - reader_start_offset, - } - }) -} impl PeerConnectionManager for PeerConnections { @@ -170,38 +150,12 @@ impl PeerConnectionManager for PeerConnections { } } - fn broadcast_append_entries(&mut self, term: Term, controller_state: &mut ControllerState) { - - let commit_index = controller_state.get_system_commit_index(); - for (peer_id, peer) in self.known_peers.iter_mut() { - if peer.is_connected() { - let start = if peer.last_acknowledged_index.is_none() { - match start_after_commit_index(controller_state, peer, commit_index) { - Ok(start_after) => Some(start_after), - Err(io_err) => { - error!("Failed to create AppendEntriesStart, refusing to send any more AppendEntries this time: {:?}", io_err); - return; - } - } - } else { - None - }; - - let append = CallAppendEntries { - current_term: term, - reader_start_position: start, - commit_index, - }; - - peer.send_append_entries(append); - } else { - debug!("Not sending AppendEntries to peer_id: {} because it is disconnected", peer_id); - } - } - } - fn send_to_peer(&mut self, peer_id: FloInstanceId, control: ConnectionControl) { - unimplemented!() + if let Some(connection) = self.known_peers.get_mut(&peer_id) { + connection.send_if_connected(control); + } else { + debug!("Cannot send control to unknown peer_id: {:?}, dropping message: {:?}", peer_id, control); + } } fn get_peer_id(&mut self, connection_id: ConnectionId) -> Option { @@ -210,10 +164,10 @@ impl PeerConnectionManager for PeerConnections { } fn send(connection: &ConnectionRef, control: ConnectionControl) { - trace!("Sending to connection_id: {}, control: {:?}", connection.connection_id, control); + debug!("Sending to connection_id: {}, control: {:?}", connection.connection_id, control); let result = connection.control_sender.unbounded_send(control); if let Err(send_err) = result { - warn!("Error sending control to connection_id: {}, {:?}", connection.connection_id, send_err); + info!("Error sending control to connection_id: {}, {:?}", connection.connection_id, send_err); } } @@ -250,7 +204,6 @@ impl ConnectionAttempt { #[derive(Debug)] struct Connection { - last_acknowledged_index: Option, peer_address: SocketAddr, state: PeerState, } @@ -258,31 +211,26 @@ struct Connection { impl Connection { fn new(peer_address: SocketAddr) -> Connection { Connection { - last_acknowledged_index: None, peer_address, state: PeerState::Init } } - fn is_connected(&self) -> bool { + fn send_if_connected(&mut self, control: ConnectionControl) { match self.state { - PeerState::Connected(_) => true, - _ => false + PeerState::Connected(ref mut connection_ref) => { + send(connection_ref, control); + return; + } + _ => {} } + debug!("Not sending control to peer: {:?} because it is not connected, dropping message: {:?}", self, control); } - fn send_append_entries(&mut self, append: CallAppendEntries) { - let send_err = match &mut self.state { - &mut PeerState::Connected(ref mut conn) => { - let result = conn.control_sender.send(ConnectionControl::SendAppendEntries(append)); - result.is_err() - } - other @ _ => { - panic!("Attempted to send AppendEntries to disconnected peer in state: {:?}", other); - } - }; - if send_err { - warn!("Failed to send AppendEntries control message to connectionHandler for peer at: {}", self.peer_address); + fn is_connected(&self) -> bool { + match self.state { + PeerState::Connected(_) => true, + _ => false } } } @@ -328,27 +276,25 @@ mod test { } #[test] - fn broadcast_append_entries_sends_append_entries_control_to_all_connected_peers() { - let peer_1 = Peer { - id: FloInstanceId::generate_new(), + fn send_to_peer_sends_connection_control_to_a_connected_peer() { + let peer_id = FloInstanceId::generate_new(); + let peer = Peer { + id: peer_id, address: addr("123.4.5.6:3000") }; - let peer_2 = Peer { - id: FloInstanceId::generate_new(), - address: addr("123.4.5.6:4000") - }; let mut creator = MockOutgoingConnectionCreator::new(); - let (peer_1_conn, rx_1) = creator.stub(peer_1.address); - let (peer_2_conn, rx_2) = creator.stub(peer_2.address); - let (mut subject, mut controller_state) = subject_with_connected_peers(&[peer_1, peer_2], creator); - controller_state.set_commit_index(7); + let (peer_conn, rx) = creator.stub(peer.address); - // TODO: stub out return values for controller_state + let (mut subject, connections) = subject_with_connected_peers(&[peer], creator); + let rx = assert_control_sent(rx, &ConnectionControl::InitiateOutgoingSystemConnection); - subject.broadcast_append_entries(7, &mut controller_state); - - - // TODO: verify that append entries was sent + let expected = ConnectionControl::SendAppendEntries(CallAppendEntries { + current_term: 3, + commit_index: 5, + reader_start_position: None, + }); + subject.send_to_peer(peer_id, expected.clone()); + assert_control_sent(rx, &expected); } #[test] @@ -530,7 +476,7 @@ pub mod mock { let missing_invocation_message = format!("Expected invocation: {:?}, but no calls were made on this mock", expected); let next_invocation = lock.pop_front().expect(&missing_invocation_message); if expected != &next_invocation { - panic!("Expected: {:?}, but actual was: {:?}. Other invocations were: {:?}", expected, next_invocation, ::std::ops::Deref::deref(&lock)); + panic!("Expected: {:#?}, but actual was: {:#?}. Other invocations were: {:#?}", expected, next_invocation, ::std::ops::Deref::deref(&lock)); } } @@ -542,7 +488,7 @@ pub mod mock { if let Some(idx) = index { lock.remove(idx); } else { - panic!("Expected: {:?} in any order, other invocations on this mock: {:?}", expected, lock.as_slices()); + panic!("Expected: {:#?} in any order, other invocations on this mock: {:#?}", expected, lock.as_slices()); } } @@ -584,9 +530,6 @@ pub mod mock { let lock = self.peer_stubs.lock().unwrap(); lock.get(&connection_id).cloned() } - fn broadcast_append_entries(&mut self, term: Term, controller_state: &mut ControllerState) { - self.push_invocation(Invocation::BroadcastAppendEntries {term}) - } } @@ -602,6 +545,5 @@ pub mod mock { PeerConnectionEstablished{peer_id: FloInstanceId, success_connection: ConnectionRef}, BroadcastToPeers{connection_control: ConnectionControl}, SendToPeer{peer_id: FloInstanceId, connection_control :ConnectionControl}, - BroadcastAppendEntries { term: Term }, } } diff --git a/flo-server/src/engine/controller/cluster_state/primary_state.rs b/flo-server/src/engine/controller/cluster_state/primary_state.rs new file mode 100644 index 0000000..b5628cf --- /dev/null +++ b/flo-server/src/engine/controller/cluster_state/primary_state.rs @@ -0,0 +1,136 @@ +use std::collections::HashMap; +use std::io; + +use event::EventCounter; +use protocol::{Term, FloInstanceId}; +use engine::controller::cluster_state::peer_connections::PeerConnectionManager; +use engine::controller::ControllerState; +use engine::connection_handler::{ConnectionControl, AppendEntriesStart, CallAppendEntries}; + +#[derive(Debug)] +pub struct PrimaryState { + term: Term, + peer_positions: HashMap, +} + +#[derive(Debug, Copy, Clone, PartialEq)] +enum Position { + SetStart(EventCounter), + TrackingFrom(EventCounter), +} + + +impl PrimaryState { + + pub fn new(term: Term) -> PrimaryState { + PrimaryState { + term, + peer_positions: HashMap::new() + } + } + + pub fn send_append_entries>(&mut self, controller: &mut ControllerState, + connection_manager: &mut PeerConnectionManager, + all_peers: I) { + + let (commit_index, _commit_term) = match controller.get_last_committed() { + Ok(result) => result, + Err(io_err) => { + error!("Unable to read last committed: {:?}", io_err); + return; + } + }; + for peer in all_peers { + let current_position = self.peer_positions.entry(peer).or_insert(Position::SetStart(commit_index)); + + let (new_position, start) = match *current_position { + Position::SetStart(new_start) => { + let append_start = match PrimaryState::get_start(new_start, controller) { + Ok(s) => s, + Err(io_err) => { + error!("Failed to get start position due to io error: {:?}, No more appendEntries will be sent", io_err); + return; + } + }; + (new_start, Some(append_start)) + } + Position::TrackingFrom(current) => { + (current, None) + } + }; + + let append = CallAppendEntries { + commit_index, + current_term: self.term, + reader_start_position: start, + }; + connection_manager.send_to_peer(peer, ConnectionControl::SendAppendEntries(append)); + + *current_position = Position::TrackingFrom(new_position); + } + } + + fn get_start(start_after_index: EventCounter, controller: &mut ControllerState) -> io::Result { + let (current_segment, current_offset) = controller.get_current_file_offset(); + + if start_after_index == 0 { + Ok(AppendEntriesStart { + prev_entry_index: 0, + prev_entry_term: 0, + reader_start_offset: current_offset, + reader_start_segment: current_segment, + }) + } else { + match controller.get_next_event(start_after_index.saturating_sub(1)) { + Some(result) => { + let (prev_index, prev_term) = result?; // return error if there is one + let (start_segment, start_offset) = controller.get_next_entry(start_after_index).map(|entry| { + (entry.segment, entry.file_offset) + }).unwrap_or((current_segment, current_offset)); + + Ok(AppendEntriesStart { + prev_entry_index: prev_index, + prev_entry_term: prev_term, + reader_start_offset: start_offset, + reader_start_segment: start_segment, + }) + } + None => { + // If we end up here, we've really fucked something up. If we have a non-zero number of events, then we + // should always have an event for the current position of each peer. + Err(io::Error::new(io::ErrorKind::InvalidInput, format!("Expected an event for index: {}, but no event was found", start_after_index))) + } + } + + } + } + + pub fn append_entries_response(&mut self, peer_id: FloInstanceId, ack_through: Option, + controller: &mut ControllerState, connection_manager: &mut PeerConnectionManager) { + if let Some(new_counter) = ack_through { + // TODO: peer has acknowledged events through this index, tell the partitionImpl about the ack and see if it allows us to commit any events + if log_enabled!(::log::Level::Debug) { + let current = self.peer_positions.get(&peer_id); + debug!("Received acknowledgement from peer_id: {:?}, new_position: {}, old_position: {:?}", peer_id, new_counter, current); + } + self.peer_positions.insert(peer_id, Position::TrackingFrom(new_counter)); + } else { + // If the peer did not acknowledge the last append, then we take their last known position and move it back. + // We move it back by 8, because that's the current batch size, but the actual amount is only a matter of efficiency. + // This amount does not affect correctness. + let current_position = self.get_peer_position(&peer_id); + let new_position = current_position.saturating_sub(8); + debug!("peer_id: {} did NOT acknowledge last append entries, changing position from {} to {}", peer_id, current_position, new_position); + self.peer_positions.insert(peer_id, Position::SetStart(new_position)); + } + } + + fn get_peer_position(&self, peer: &FloInstanceId) -> EventCounter { + self.peer_positions.get(peer).map(|position| { + match *position { + Position::SetStart(start) => start, + Position::TrackingFrom(last) => last + } + }).unwrap_or(0) + } +} diff --git a/flo-server/src/engine/controller/controller_state.rs b/flo-server/src/engine/controller/controller_state.rs index 0fcd326..22e2b07 100644 --- a/flo-server/src/engine/controller/controller_state.rs +++ b/flo-server/src/engine/controller/controller_state.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use std::collections::HashMap; use std::io; +use protocol::Term; use event::EventCounter; use engine::ConnectionId; use engine::controller::{ConnectionRef, SystemEvent}; @@ -16,9 +17,10 @@ pub trait ControllerState { fn remove_connection(&mut self, connection_id: ConnectionId); fn get_connection(&self, connection_id: ConnectionId) -> Option<&ConnectionRef>; + fn get_last_committed(&mut self) -> io::Result<(EventCounter, Term)>; + fn get_next_event(&mut self, start_after: EventCounter) -> Option>; + fn set_commit_index(&mut self, new_index: EventCounter); - fn get_system_commit_index(&self) -> EventCounter; - fn get_system_event(&mut self, event_counter: EventCounter) -> Option>>; fn get_next_entry(&self, event_counter: EventCounter) -> Option; fn get_current_file_offset(&self) -> (SegmentNum, usize); } @@ -94,24 +96,10 @@ impl ControllerState for ControllerStateImpl { self.all_connections.get(&connection_id) } - fn get_system_commit_index(&self) -> EventCounter { - self.system_partition.get_commit_index() - } - fn set_commit_index(&mut self, new_index: EventCounter) { self.system_partition.set_commit_index(new_index); } - fn get_system_event(&mut self, event_counter: EventCounter) -> Option>> { - self.get_next_entry(event_counter.saturating_sub(1)).and_then(|index_entry| { - if index_entry.counter == event_counter { - Some(self.read_event(index_entry)) - } else { - None - } - }) - } - fn get_next_entry(&self, previous: EventCounter) -> Option { self.system_partition.get_next_index_entry(previous) } @@ -119,6 +107,28 @@ impl ControllerState for ControllerStateImpl { fn get_current_file_offset(&self) -> (SegmentNum, usize) { self.system_partition.get_head_position() } + + fn get_last_committed(&mut self) -> io::Result<(EventCounter, Term)> { + let start_after = self.system_partition.get_commit_index(); + if start_after == 0 { + return Ok((0, 0)); + } + match self.get_next_event(start_after) { + Some(result) => result, + None => { + error!("No SystemEvent was found for the commit index: {}. System partition is in an invalid state!", start_after); + Err(io::Error::new(io::ErrorKind::InvalidData, format!("Expected to have a SystemEvent at index: {} since that is the commit index!", start_after))) + } + } + } + + fn get_next_event(&mut self, start_after: EventCounter) -> Option> { + self.get_next_entry(start_after.saturating_sub(1)).map(|index_entry| { + self.read_event(index_entry).map(|system_event| { + (start_after, system_event.term()) + }) + }) + } } @@ -126,23 +136,44 @@ impl ControllerState for ControllerStateImpl { #[cfg(test)] pub mod mock { use super::*; + use std::collections::BTreeMap; + #[derive(Debug)] pub struct MockControllerState { pub commit_index: EventCounter, pub all_connections: HashMap, + pub system_events: BTreeMap, } impl MockControllerState { pub fn new() -> MockControllerState { - MockControllerState::with_commit_index(0) - } - - pub fn with_commit_index(index: EventCounter) -> MockControllerState { MockControllerState { - commit_index: index, + commit_index: 0, all_connections: HashMap::new(), + system_events: BTreeMap::new(), } } + + pub fn with_commit_index(mut self, index: EventCounter) -> MockControllerState { + self.set_commit_index(index); + self + } + + pub fn with_connection(mut self, conn: ConnectionRef) -> Self { + self.add_connection(conn); + self + } + + pub fn with_mocked_events(mut self, events: &[MockSystemEvent]) -> Self { + for e in events { + self.add_mock_event(e.clone()); + } + self + } + + pub fn add_mock_event(&mut self, event: MockSystemEvent) { + self.system_events.insert(event.id, event); + } } impl ControllerState for MockControllerState { @@ -155,23 +186,41 @@ pub mod mock { fn get_connection(&self, connection_id: ConnectionId) -> Option<&ConnectionRef> { self.all_connections.get(&connection_id) } - fn get_system_commit_index(&self) -> EventCounter { - unimplemented!() - } - - fn get_system_event(&mut self, event_counter: EventCounter) -> Option>> { - unimplemented!() - } fn get_next_entry(&self, event_counter: EventCounter) -> Option { - unimplemented!() + self.system_events.range((event_counter + 1)..).next().map(|(id, sys)| { + IndexEntry::new(*id, sys.segment, sys.file_offset) + }) } fn get_current_file_offset(&self) -> (SegmentNum, usize) { - unimplemented!() + self.system_events.values().last().map(|sys| { + (sys.segment, sys.file_offset) + }).unwrap_or((SegmentNum::new_unset(), 0)) } fn set_commit_index(&mut self, new_index: EventCounter) { - unimplemented!() + self.commit_index = new_index; + } + fn get_last_committed(&mut self) -> io::Result<(EventCounter, Term)> { + let commit_idx = self.commit_index; + self.system_events.get(&commit_idx).map(|sys| { + (sys.id, sys.term) + }).ok_or_else(|| { + io::Error::new(io::ErrorKind::Other, format!("Test error because there is no stubbed event for the commit index: {}", commit_idx)) + }) } + fn get_next_event(&mut self, start_after: EventCounter) -> Option> { + self.system_events.range((start_after + 1)..).next().map(|(id, sys)| { + Ok((*id, sys.term)) + }) + } + } + + #[derive(Clone, Debug, PartialEq)] + pub struct MockSystemEvent { + pub id: EventCounter, + pub term: Term, + pub segment: SegmentNum, + pub file_offset: usize, } } diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index 10fec9a..1387d9f 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -70,6 +70,11 @@ impl SegmentNum { SegmentNum(0) } + #[cfg(test)] + pub fn new(num: u64) -> SegmentNum { + SegmentNum(num) + } + /// returns true if this segment is non-zero pub fn is_set(&self) -> bool { self.0 > 0 diff --git a/flo-server/src/logging.rs b/flo-server/src/logging.rs index 261d58f..3974754 100644 --- a/flo-server/src/logging.rs +++ b/flo-server/src/logging.rs @@ -4,7 +4,7 @@ use log4rs::append::file::FileAppender; use log4rs::append::Append; use log4rs::init_config; -use log::{LogLevelFilter, LogLevel}; +use log::{LevelFilter, Level}; use std::boxed::Box; use std::str::FromStr; @@ -21,7 +21,7 @@ pub enum LogFileOption { #[derive(Debug, PartialEq, Clone)] pub struct LogLevelOption { module: String, - log_level: LogLevel + log_level: Level } impl FromStr for LogLevelOption { @@ -33,7 +33,7 @@ impl FromStr for LogLevelOption { return Err(format!("Invalid Log Option: '{}', must be in the format = (with exactly one '=')", s)); } - LogLevel::from_str(parts[1]).map_err(|_| { + Level::from_str(parts[1]).map_err(|_| { format!("invalid log level: '{}'", parts[1]) }).map(|level| { LogLevelOption { @@ -59,14 +59,14 @@ pub fn init_logging(log_dest: LogFileOption, levels: Vec) { let override_default = levels.iter().any(|level_opt| DEFAULT_LOG_MODULE == &level_opt.module); if !override_default { - config = config.logger(Logger::builder().additive(true).build("flo".to_owned(), LogLevelFilter::Info)); + config = config.logger(Logger::builder().additive(true).build("flo".to_owned(), LevelFilter::Info)); } for level_opt in levels { - config = config.logger(Logger::builder().additive(true).build(level_opt.module, level_opt.log_level.to_log_level_filter())); + config = config.logger(Logger::builder().additive(true).build(level_opt.module, level_opt.log_level.to_level_filter())); } - let root = Root::builder().appender(LOG_APPENDER.to_string()).build(LogLevelFilter::Warn); + let root = Root::builder().appender(LOG_APPENDER.to_string()).build(LevelFilter::Warn); let config = config.build(root).unwrap(); init_config(config).unwrap(); } @@ -74,9 +74,9 @@ pub fn init_logging(log_dest: LogFileOption, levels: Vec) { #[cfg(test)] mod test { use super::*; - use log::LogLevel; + use log::Level; - fn test_valid_log_option(module: &str, level_str: &str, level: LogLevel) { + fn test_valid_log_option(module: &str, level_str: &str, level: Level) { let input = format!("{}={}", module, level_str); let result = LogLevelOption::from_str(&input).expect("failed to create log option"); @@ -101,16 +101,16 @@ mod test { #[test] fn log_option_is_created_from_str_with_valid_log_level() { - test_valid_log_option("my::module", "trace", LogLevel::Trace); - test_valid_log_option("my::module", "TRACE", LogLevel::Trace); - test_valid_log_option("my::module", "TraCe", LogLevel::Trace); - test_valid_log_option("my::module", "debug", LogLevel::Debug); - test_valid_log_option("my::module", "Debug", LogLevel::Debug); - test_valid_log_option("my::module", "info", LogLevel::Info); - test_valid_log_option("my::module", "iNFo", LogLevel::Info); - test_valid_log_option("my::module", "warn", LogLevel::Warn); - test_valid_log_option("my::module", "WARN", LogLevel::Warn); - test_valid_log_option("my::module", "error", LogLevel::Error); - test_valid_log_option("my::module", "erroR", LogLevel::Error); + test_valid_log_option("my::module", "trace", Level::Trace); + test_valid_log_option("my::module", "TRACE", Level::Trace); + test_valid_log_option("my::module", "TraCe", Level::Trace); + test_valid_log_option("my::module", "debug", Level::Debug); + test_valid_log_option("my::module", "Debug", Level::Debug); + test_valid_log_option("my::module", "info", Level::Info); + test_valid_log_option("my::module", "iNFo", Level::Info); + test_valid_log_option("my::module", "warn", Level::Warn); + test_valid_log_option("my::module", "WARN", Level::Warn); + test_valid_log_option("my::module", "error", Level::Error); + test_valid_log_option("my::module", "erroR", Level::Error); } } diff --git a/flo-server/src/test_utils.rs b/flo-server/src/test_utils.rs index 5e230ed..23db3d6 100644 --- a/flo-server/src/test_utils.rs +++ b/flo-server/src/test_utils.rs @@ -17,7 +17,6 @@ impl Unpark for NoOpUnpark { } pub fn expect_future_resolved(future: F) -> Result where F: Future { - let now = Instant::now(); let mut s = spawn(future); let unpark = ::std::sync::Arc::new(NoOpUnpark); loop { From 44cfd283d314760a376cd2cc637944fd423beaa6 Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 20 Jan 2018 20:38:31 -0500 Subject: [PATCH 47/73] primary is elected, seemingly correctly, but still no way to actually add entries to the system partition --- .../connection_handler/connection_state.rs | 13 +- .../src/engine/connection_handler/mod.rs | 27 ++- .../src/engine/connection_handler/peer/mod.rs | 52 +++-- .../engine/controller/cluster_state/mod.rs | 187 +++++++++++++++--- .../cluster_state/peer_connections.rs | 158 ++++++++++----- .../engine/controller/controller_messages.rs | 6 +- flo-server/src/engine/controller/mod.rs | 29 ++- .../src/engine/controller/system_reader.rs | 7 +- .../src/engine/controller/system_stream.rs | 4 +- .../partition/event_reader/mod.rs | 10 + flo-server/src/server/mod.rs | 1 + 11 files changed, 378 insertions(+), 116 deletions(-) diff --git a/flo-server/src/engine/connection_handler/connection_state.rs b/flo-server/src/engine/connection_handler/connection_state.rs index 4af9d60..9a933f3 100644 --- a/flo-server/src/engine/connection_handler/connection_state.rs +++ b/flo-server/src/engine/connection_handler/connection_state.rs @@ -1,4 +1,4 @@ - +use std::fmt::{self, Debug}; use tokio_core::reactor::Handle; use protocol::*; @@ -11,7 +11,6 @@ use super::ConnectionHandlerResult; const DEFAULT_CONSUME_BATCH_SIZE: u32 = 10_000; -#[derive(Debug)] pub struct ConnectionState { pub client_name: Option, pub connection_id: ConnectionId, @@ -22,6 +21,16 @@ pub struct ConnectionState { pub consume_batch_size: u32, } +impl Debug for ConnectionState { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("ConnectionState") + .field("connection_id", &self.connection_id) + .field("client_name", &self.client_name) + .field("event_stream", &self.event_stream.name()) + .field("consume_batch_size", &self.consume_batch_size) + .finish() + } +} impl ConnectionState { pub fn new(connection_id: ConnectionId, client_sender: ClientSender, engine: EngineRef, reactor: Handle) -> ConnectionState { diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 4f0cd3a..a7b5137 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -27,6 +27,7 @@ pub fn create_connection_control_channels() -> (ConnectionControlSender, Connect ::futures::sync::mpsc::unbounded() } +#[derive(Debug)] pub struct ConnectionHandler { common_state: ConnectionState, consumer_state: ConsumerConnectionState, @@ -147,6 +148,7 @@ impl Sink for ConnectionHandler { }.map(|()| { AsyncSink::Ready }).map_err(|err_string| { + warn!("Error in connection handler: '{}' - handler: {:#?}", err_string, self); io::Error::new(io::ErrorKind::Other, err_string) }) } @@ -176,18 +178,6 @@ impl Drop for ConnectionHandler { } -impl Debug for ConnectionHandler { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("ConnectionHandler") - .field("common_state", &self.common_state) - .field("consumer_state", &self.consumer_state) - .field("producer_state", &self.producer_state) - .finish() - } -} - - - #[cfg(test)] mod test { use std::collections::{HashMap, HashSet}; @@ -225,13 +215,14 @@ mod test { fn create_outgoing_peer_connection() -> (ConnectionHandler, Fixture) { let (mut subject, mut fixture) = Fixture::create(); subject.handle_control(ConnectionControl::InitiateOutgoingSystemConnection).unwrap(); + let announce = PeerAnnounce { protocol_version: 1, peer_address: fixture.instance_addr, op_id: 1, instance_id: fixture.instance_id, system_primary_id: None, - cluster_members: Vec::new(), + cluster_members: vec![ClusterMember{id: fixture.instance_id, address: fixture.instance_addr}], }; fixture.assert_sent_to_client(ProtocolMessage::PeerAnnounce(announce.clone())); @@ -240,14 +231,20 @@ mod test { let response = PeerAnnounce { peer_address: peer_addr, instance_id: peer_id, + cluster_members: vec![ClusterMember {id: peer_id, address: peer_addr}], .. announce }; // get the response subject.handle_incoming_message(ProtocolMessage::PeerAnnounce(response)).unwrap(); + + let expected_peer = Peer { + id: peer_id, + address: peer_addr, + }; fixture.assert_sent_to_system_stream(SystemOpType::ConnectionUpgradeToPeer(PeerUpgrade { - peer_id, + peer: expected_peer.clone(), system_primary: None, - cluster_members: Vec::new(), + cluster_members: vec![expected_peer.clone()], })); (subject, fixture) } diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 6eb99a2..8af7572 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -27,7 +27,7 @@ pub struct PeerConnectionState { controller_operation_queue: VecDeque, peer_operation_queue: VecDeque, system_partition_reader: Option, - this_instance_id: Option, + this_peer_info: Option, in_progress_append: Option, } @@ -40,7 +40,7 @@ impl PeerConnectionState { controller_operation_queue: VecDeque::new(), peer_operation_queue: VecDeque::new(), system_partition_reader: None, - this_instance_id: None, + this_peer_info: None, in_progress_append: None, } } @@ -145,22 +145,27 @@ impl PeerConnectionState { pub fn send_append_entries(&mut self, append: CallAppendEntries, state: &mut ConnectionState) -> ConnectionHandlerResult { let op_id = self.next_op_id(); self.peer_operation_queue.push_back(op_id); - let this_instance_id = self.get_this_instance_id(state); + let this_peer = self.get_this_peer_info(state); if self.system_partition_reader.is_none() { self.system_partition_reader = Some(SystemReaderWrapper::new(state)); } let reader = self.system_partition_reader.as_mut().unwrap(); - reader.send_append_entries(op_id, this_instance_id, append, state) + reader.send_append_entries(op_id, this_peer.id, append, state) } - fn get_this_instance_id(&mut self, state: &mut ConnectionState) -> FloInstanceId { - if self.this_instance_id.is_none() { - let id = state.get_system_stream().with_cluster_state(|cluster_state| cluster_state.this_instance_id); - self.this_instance_id = Some(id); + fn get_this_peer_info(&mut self, state: &mut ConnectionState) -> Peer { + if self.this_peer_info.is_none() { + let peer = state.get_system_stream().with_cluster_state(|cluster_state| { + Peer { + id: cluster_state.this_instance_id, + address: cluster_state.this_address.clone().unwrap() + } + }); + self.this_peer_info = Some(peer); } - self.this_instance_id.unwrap() + self.this_peer_info.as_ref().cloned().unwrap() } pub fn vote_response_received(&mut self, response: protocol::RequestVoteResponse, state: &mut ConnectionState) -> ConnectionHandlerResult { @@ -270,20 +275,25 @@ impl PeerConnectionState { state.set_to_system_stream(); let PeerAnnounce {instance_id, system_primary_id, cluster_members, ..} = announce; + let maybe_peer = cluster_members.iter().find(|member| { + member.id == instance_id + }).map(|member| member_to_peer(member.clone()) ); + + if maybe_peer.is_none() { + return Err(format!("Received invalid peer announce message. Missing a cluster member for the instance identified as peer: {:?}, in: {:?}", instance_id, cluster_members)); + } + + let peer = maybe_peer.unwrap(); let primary = system_primary_id.and_then(|primary_id| { cluster_members.iter().find(|member| { member.id == primary_id - }).map(|member| { - Peer { - id: member.id, - address: member.address, - } - }) + }).map(|member| member_to_peer(member.clone())) }); + let peers = cluster_members.into_iter().map(|member| { member_to_peer(member) }).collect(); - state.get_system_stream().connection_upgraded_to_peer(connection_id, announce.instance_id, primary, peers); + state.get_system_stream().connection_upgraded_to_peer(connection_id, peer, primary, peers); Ok(()) } @@ -293,13 +303,21 @@ impl PeerConnectionState { let instance_id = state.this_instance_id; let address = state.this_address.expect("Attempted to send PeerAnnounce, but system is not in cluster mode"); let system_primary_id = state.system_primary.as_ref().map(|peer| peer.id); - let cluster_members = state.peers.iter().map(|peer| { + let mut cluster_members = state.peers.iter().map(|peer| { ClusterMember { id: peer.id, address: peer.address, } }).collect::>(); + // The members enumerated in the shared state do not include an entry for this instance, only the _other_ instances + // in the cluster. The PeerAnnounce protocol message requires an entry for _every_ member, including this one. + // So, we always add an entry for ourselves when creating the message + cluster_members.push(ClusterMember { + id: instance_id, + address + }); + PeerAnnounce { op_id, instance_id, diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 878123f..fe48573 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -35,6 +35,7 @@ pub trait ConsensusProcessor: Send { fn is_primary(&self) -> bool; fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, controller_state: &ControllerState); fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr); + fn connection_closed(&mut self, connection_id: ConnectionId); fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote); fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse, controller: &mut ControllerState); @@ -115,9 +116,9 @@ impl ClusterManager { } fn determine_primary_from_peer_upgrade(&mut self, upgrade: PeerUpgrade) { - let PeerUpgrade { peer_id, system_primary, .. } = upgrade; + let PeerUpgrade { peer, system_primary, .. } = upgrade; if let Some(primary) = system_primary { - info!("Determined primary of {:?} at {}, as told by instance: {:?}", primary.id, primary.address, peer_id); + info!("Determined primary of {:?} at {}, as told by instance: {:?}", primary.id, primary.address, peer); self.transition_state(State::Follower); { let mut lock = self.system_partition_primary_address.write().unwrap(); @@ -128,7 +129,7 @@ impl ClusterManager { shared.system_primary = Some(primary); } } else { - debug!("peer announce from {:?} has unknown primary", peer_id); + debug!("peer announce from {:?} has unknown primary", peer); } } @@ -142,7 +143,10 @@ impl ClusterManager { } fn election_timed_out(&self, now: Instant) -> bool { - now - self.last_heartbeat > self.election_timeout + // since `now` comes from the original Tick operation, it's possible that the `last_heartbeat` was since updated + // and ends up being after `now`. If we don't check for this case, it can cause the controller to panic + now > self.last_heartbeat && + (now - self.last_heartbeat) > self.election_timeout } fn start_new_election(&mut self) { @@ -161,12 +165,26 @@ impl ClusterManager { last_log_term: self.last_applied_term, }); self.connection_manager.broadcast_to_peers(connection_control); - self.transition_state(State::Voted) + self.transition_state(State::Voted); + // update the timestamp to make sure that we allow the proper amount of time for this election + self.update_last_heartbeat_to_now(); } fn can_grant_vote(&self, request: &CallRequestVote) -> bool { - (self.persistent.voted_for.is_none() || self.persistent.voted_for == Some(request.candidate_id)) && - self.persistent.current_term <= request.term && + let request_term = request.term; + let my_term = self.persistent.current_term; + + if request_term < my_term { + return false; + } else if request_term == my_term { + if self.persistent.voted_for == Some(request.candidate_id) { + return true; + } else if self.persistent.voted_for.is_some() { + return false; + } + } + + self.persistent.current_term <= request_term && self.last_applied <= request.last_log_index && self.last_applied_term <= request.last_log_term && self.persistent.cluster_members.iter().any(|peer| peer.id == request.candidate_id) @@ -232,6 +250,16 @@ impl ClusterManager { } Ok(0) } + + fn create_append_result(&mut self, events: &[OwnedFloEvent]) -> Option { + trace!("AppendEntries looks successful, will append: {} entries", events.len()); + self.append_system_events(events).map(|last_counter| { + Some(last_counter) + }).unwrap_or_else(|io_err| { + error!("Error appending system events: {:?}, returning false", io_err); + None + }) + } } impl ConsensusProcessor for ClusterManager { @@ -240,6 +268,10 @@ impl ConsensusProcessor for ClusterManager { self.connection_resolved(address); } + fn connection_closed(&mut self, connection_id: ConnectionId) { + self.connection_manager.connection_closed(connection_id); + } + fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote) { let candidate_id = request.candidate_id; @@ -254,6 +286,7 @@ impl ConsensusProcessor for ClusterManager { VoteResponse { term: self.persistent.current_term, granted: false, } }; + debug!("Got request vote from connection_id: {}: {:?} - sending response: {:?}", from, request, response); self.connection_manager.send_to_peer(candidate_id, ConnectionControl::SendVoteResponse(response)); } @@ -273,8 +306,8 @@ impl ConsensusProcessor for ClusterManager { let response_term = response.term; debug!("VoteResponse from {:?} was not granted, response term: {}, current_term: {}", peer_id, response_term, self.persistent.current_term); if response_term > self.persistent.current_term { - info!("Received response term of {}, which is greater than current term of: {}. Updating current term and clearing out {} existing votes", - response_term, self.persistent.current_term, self.votes_received.len()); + info!("Received response term of {} from peer: {:?}, which is greater than current term of: {}. Updating current term and clearing out {} existing votes", + response_term, peer_id, self.persistent.current_term, self.votes_received.len()); self.persistent.modify(|state| { state.current_term = response_term; }).expect(STATE_UPDATE_FAILED); @@ -286,10 +319,24 @@ impl ConsensusProcessor for ClusterManager { fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, controller_state: &ControllerState) { debug!("peer_connection_established: connection_id: {}, {:?}", connection_id, upgrade); - let connection = controller_state.get_connection(connection_id) - .expect("Expected connection to be in all_connections on peer_connection_established"); + let connection = controller_state.get_connection(connection_id); + if connection.is_none() { + debug!("Ignoring peer_connection_established: {:?} for connection_id: {}, because that connection has already been closed", + upgrade, connection_id); + return; + } + let connection = connection.unwrap(); - self.connection_manager.peer_connection_established(upgrade.peer_id, connection); + if !self.persistent.cluster_members.contains(&upgrade.peer) { + self.persistent.modify(|state| { + state.cluster_members.insert(upgrade.peer.clone()); + }).expect(STATE_UPDATE_FAILED); + + let mut lock = self.shared.write().unwrap(); + lock.peers.insert(upgrade.peer.clone()); + } + + self.connection_manager.peer_connection_established(upgrade.peer.clone(), connection); if State::DeterminePrimary == self.state { self.determine_primary_from_peer_upgrade(upgrade); @@ -346,23 +393,25 @@ impl ConsensusProcessor for ClusterManager { self.update_last_heartbeat_to_now(); - // TODO: check to see if we have an event at the previous index with a matching term let response = match controller_state.get_next_event(append.prev_entry_index.saturating_sub(1)) { Some(Ok((actual_prev_index, actual_prev_term))) => { if actual_prev_index == append.prev_entry_index && actual_prev_term == append.prev_entry_term { - // TODO: Append the new entries - trace!("AppendEntries looks successful, will append: {} entries and return success", append.events.len()); - self.append_system_events(append.events.as_slice()).map(|last_counter| { - Some(last_counter) - }).unwrap_or_else(|io_err| { - error!("Error appending system events: {:?}, returning false", io_err); - None - }) + self.create_append_result(append.events.as_slice()) } else { info!("AppendEntries: {:?} does not match the prev index/term stored for this instance: index: {}, term: {}", append, actual_prev_index, actual_prev_term); None } } + None => { + if append.prev_entry_index == 0 && append.prev_entry_term == 0 { + // special case for when we're at the very beginning and have literally no events in the log + self.create_append_result(append.events.as_slice()) + } else { + debug!("System partition with head at: {} is behind AppendEntries with index: {}, term: {}, returning negative result", + self.last_applied, append.prev_entry_index, append.prev_entry_term); + None + } + } other @ _ => { error!("failed to read previous system event at: {} with err: {:?}", append.prev_entry_index, other); None @@ -376,7 +425,30 @@ impl ConsensusProcessor for ClusterManager { } fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState) { - unimplemented!() + let from = self.connection_manager.get_peer_id(connection_id); + if from.is_none() { + error!("Received AppendEntriesResponse from connection_id: {}, which is not a peer connection. Received: {:?}", connection_id, response); + return; + } + let peer_id = from.unwrap(); + let response_term = response.term; + + if response_term > self.persistent.current_term { + // Apparently, we've fallen behind! This instance is no longer primary, so we need to step down if we haven't already + warn!("Peer: {:?} responded with term: {}, which is higher than the current term: {}. Stepping down as primary", + peer_id, response_term, self.persistent.current_term); + if self.is_primary() { + self.set_follower_status(None); + } + self.persistent.modify(|state| { + state.current_term = response_term; + }).expect(STATE_UPDATE_FAILED) + } + + if response.success.is_some() && self.is_primary() { + let peer_counter = response.success.unwrap(); + // TODO: confirm entries with partition impl + } } } @@ -447,6 +519,50 @@ mod test { use engine::controller::controller_messages::mock::mock_connection_ref; use engine::controller::mock::{MockControllerState, MockSystemEvent}; + #[test] + fn reverts_to_follower_when_append_entries_response_indicates_term_greater_than_current_term() { + let start = Instant::now(); + let temp_dir = TempDir::new("cluster_state_test").unwrap(); + let connection_manager = MockPeerConnectionManager::new(); + let peer_1 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.1:1000") + }; + let peer_2 = Peer { + id: FloInstanceId::generate_new(), + address: addr("111.222.0.2:2000") + }; + let peer_1_connection = 1; + let peer_2_connection = 2; + connection_manager.stub_peer_connection(peer_1_connection, peer_1.id); + connection_manager.stub_peer_connection(peer_2_connection, peer_2.id); + + let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + + subject.persistent.modify(|state| { + state.cluster_members.insert(peer_1.clone()); + state.cluster_members.insert(peer_2.clone()); + state.current_term = 5; + }).unwrap(); + subject.last_applied = 99; + subject.last_applied_term = 4; + subject.state = State::Primary; + subject.primary_state = Some(PrimaryState::new(5)); + subject.last_heartbeat = start; + + let response = AppendEntriesResponse { + term: 6, + success: None, + }; + let mut controller = MockControllerState::new(); + subject.append_entries_response_received(peer_1_connection, response, &mut controller); + + assert_eq!(State::Follower, subject.state); + assert!(!subject.is_primary()); + assert!(subject.primary_state.is_none()); + assert_eq!(6, subject.persistent.current_term); + } + #[test] fn reverts_to_follower_when_append_entries_is_received_from_another_peer() { let start = Instant::now(); @@ -993,6 +1109,26 @@ mod test { }); } + #[test] + fn vote_is_granted_when_voted_for_is_already_populated_but_candidate_term_is_greater() { + vote_test(|subject, request, candidate, peer_2| { + subject.last_applied_term = request.last_log_term; + subject.last_applied = request.last_log_index; + subject.persistent.modify(|state| { + state.current_term = request.term - 1; + state.voted_for = Some(state.this_instance_id); + state.cluster_members.insert(candidate.clone()); + state.cluster_members.insert(peer_2.clone()); + }).unwrap(); + + VoteExpectation { + term: request.term, + persistent_voted_for: Some(candidate.id), + granted: true + } + }); + } + #[test] fn cluster_manager_starts_new_election_after_timeout_elapses() { let start = Instant::now(); @@ -1055,7 +1191,7 @@ mod test { let conn_id = 5; let upgrade = PeerUpgrade { - peer_id: peer_1.id, + peer: peer_1.clone(), system_primary: Some(peer_2.clone()), cluster_members: vec![peer_2.clone()], }; @@ -1064,7 +1200,7 @@ mod test { subject.peer_connection_established(upgrade, conn_id, &controller_state); connection_manager.verify_in_order(&Invocation::PeerConnectionEstablished { - peer_id: peer_1.id, + peer: peer_1.clone(), success_connection: connection, }); @@ -1244,4 +1380,7 @@ impl ConsensusProcessor for NoOpConsensusProcessor { fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState) { unimplemented!() } + fn connection_closed(&mut self, connection_id: ConnectionId) { + unimplemented!() + } } diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs index cd92bce..76b8340 100644 --- a/flo-server/src/engine/controller/cluster_state/peer_connections.rs +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -15,7 +15,7 @@ pub trait PeerConnectionManager: Send + Debug + 'static { fn establish_connections(&mut self, now: Instant, controller_state: &mut ControllerState); fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, addr: SocketAddr); fn connection_closed(&mut self, connection_id: ConnectionId); - fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef); + fn peer_connection_established(&mut self, peer: Peer, success_connection: &ConnectionRef); fn broadcast_to_peers(&mut self, connection_control: ConnectionControl); fn send_to_peer(&mut self, peer_id: FloInstanceId, connection_control: ConnectionControl); @@ -56,6 +56,33 @@ impl PeerConnections { } } + fn close_connection(&mut self, connection_id: ConnectionId, attempt_reconnect: bool) { + let address: Option = self.active_connections.remove(&connection_id).and_then(|peer_id| { + self.known_peers.get_mut(&peer_id).and_then(|connection| { + // only set the connection state back to ConnectionFailed if the connection here is the same as the one we are closing + if connection.get_connection_id() == Some(connection_id) { + connection.state = PeerState::ConnectionFailed; + Some(connection.peer_address) + } else { + // The connection in `known_peers` has a different connection id. This means that we won't want to + // add the address to `disconected_peers`, even if `attempt_reconnect` is true, because it would result in + // multiple connections open for the same peer + None + } + }) + }); + + if attempt_reconnect { + if let Some(peer_address) = address { + // todo: as it is, we only deal with one connection per peer at a time, so we know there won't already be an entry in disconnected_peers. will need to change that once we deal with multiple connections per peer + self.disconnected_peers.insert(peer_address, ConnectionAttempt::new()); + info!("ConnectionClosed for connection_id: {} from peer_address: {}", connection_id, peer_address); + } else { + // just means that this id was not for a peer connection + debug!("Got connection closed for connection_id: {}, but there was no active peer connection", connection_id); + } + } + } } @@ -67,7 +94,7 @@ impl PeerConnectionManager for PeerConnections { if attempt.should_try_connect(now) { debug!("Making outgoing connection attempt #{} to address: {}", attempt.attempt_count + 1, address); let connection_ref = outgoing_connection_creator.establish_system_connection(*address); - send(&connection_ref, ConnectionControl::InitiateOutgoingSystemConnection); + let _ = send(&connection_ref, ConnectionControl::InitiateOutgoingSystemConnection); // ignore result since it can't really fail controller_state.add_connection(connection_ref.clone()); attempt.attempt_time = now; attempt.attempt_count += 1; @@ -91,12 +118,12 @@ impl PeerConnectionManager for PeerConnections { } } - fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef) { - let disconnected = self.disconnected_peers.remove(&success_connection.remote_address); + fn peer_connection_established(&mut self, peer: Peer, success_connection: &ConnectionRef) { + let disconnected = self.disconnected_peers.remove(&peer.address); let success_connection_id = success_connection.connection_id; - info!("Successfully established connection_id: {} to peer_id: {} at address: {}", - success_connection.connection_id, peer_id, success_connection.remote_address); + info!("Successfully established connection_id: {} to peer: {:?} at address: {}", + success_connection.connection_id, peer, success_connection.remote_address); if let Some(ConnectionAttempt {connection, attempt_count, ..}) = disconnected { if let Some(outgoing_connection_ref) = connection { @@ -105,56 +132,77 @@ impl PeerConnectionManager for PeerConnections { // then we'll close the in progress attempt by dropping the connectionRef if outgoing_connection_ref.connection_id != success_connection_id { info!("Established a new connection for peer: {:?} with connection_id: {}, so existing connection: {} will be closed", - peer_id, success_connection_id, outgoing_connection_ref.connection_id) + peer, success_connection_id, outgoing_connection_ref.connection_id) } } } - let connection = self.known_peers.entry(peer_id).or_insert_with(|| { - Connection::new(success_connection.remote_address) - }); - connection.state = PeerState::Connected(success_connection.clone()); + let prev_connection = { + let connection = self.known_peers.entry(peer.id).or_insert_with(|| { + // TODO: We should instead just refuse connections from unknown peers for now + Connection::new(peer.address) + }); + connection.connection_established(success_connection.clone()) + }; + if let Some(to_close) = prev_connection { + debug!("A second connection was established for peer: {:?} with connection_id: {}, connection_id: {} will be closed", peer, success_connection.connection_id, to_close); + self.close_connection(to_close, false); + } - self.active_connections.insert(success_connection_id, peer_id); + self.active_connections.insert(success_connection_id, peer.id); } fn connection_closed(&mut self, connection_id: ConnectionId) { - let address: Option = self.active_connections.remove(&connection_id).and_then(|peer_id| { - self.known_peers.get_mut(&peer_id).map(|connection| { - connection.state = PeerState::ConnectionFailed; - connection.peer_address - }) - }); - - if let Some(peer_address) = address { - // todo: as it is, we only deal with one connection per peer at a time, so we know there won't already be an entry in disconnected_peers. will need to change that once we deal with multiple connections per peer - self.disconnected_peers.insert(peer_address, ConnectionAttempt::new()); - info!("ConnectionClosed for connection_id: {} from peer_address: {}", connection_id, peer_address); - } else { - // just means that this id was not for a peer connection - debug!("Got connection closed for connection_id: {}, but there was no active peer connection", connection_id); - } + self.close_connection(connection_id, true); } fn broadcast_to_peers(&mut self, control: ConnectionControl) { debug!("Broadcasting {:?}", control); + let mut errors = Vec::new(); for (peer, connection) in self.known_peers.iter() { match connection.state { PeerState::Connected(ref connection_ref) => { - send(connection_ref, control.clone()); + let result = send(connection_ref, control.clone()); + if result.is_err() { + info!("Error broadcasting connection control to handler for connection_id: {}, peer: {:?}, closing connection", connection_ref.connection_id, peer); + errors.push(connection_ref.connection_id); + } } ref other @ _ => { trace!("Skipping connection to {:?} because it is in state: {:?}", peer, other); } } } + + for bad_connection in errors { + self.connection_closed(bad_connection); + } } fn send_to_peer(&mut self, peer_id: FloInstanceId, control: ConnectionControl) { - if let Some(connection) = self.known_peers.get_mut(&peer_id) { - connection.send_if_connected(control); + let disconnect = if let Some(connection) = self.known_peers.get_mut(&peer_id) { + match &mut connection.state { + &mut PeerState::Connected(ref mut connection_ref) => { + let result = connection_ref.control_sender.send(control); + if result.is_err() { + info!("Failed to send ConnectionControl to peer connection handler for {:?}, connection_id: {}, closing connection", peer_id, connection_ref.connection_id); + Some(connection_ref.connection_id) + } else { + None + } + } + other @ _ => { + debug!("Cannot send control to peer_id: {:?} because it is in state: {:?}, dropping message: {:?}", peer_id, other, control); + None + } + } } else { debug!("Cannot send control to unknown peer_id: {:?}, dropping message: {:?}", peer_id, control); + None + }; + + if let Some(id) = disconnect { + self.connection_closed(id); } } @@ -163,12 +211,11 @@ impl PeerConnectionManager for PeerConnections { } } -fn send(connection: &ConnectionRef, control: ConnectionControl) { +fn send(connection: &ConnectionRef, control: ConnectionControl) -> Result<(), ()> { debug!("Sending to connection_id: {}, control: {:?}", connection.connection_id, control); - let result = connection.control_sender.unbounded_send(control); - if let Err(send_err) = result { + connection.control_sender.unbounded_send(control).map_err(|send_err| { info!("Error sending control to connection_id: {}, {:?}", connection.connection_id, send_err); - } + }) } #[derive(Debug)] @@ -216,15 +263,23 @@ impl Connection { } } - fn send_if_connected(&mut self, control: ConnectionControl) { + fn send_if_connected(&mut self, control: ConnectionControl) -> Result<(), ()> { match self.state { PeerState::Connected(ref mut connection_ref) => { - send(connection_ref, control); - return; + send(connection_ref, control) + } + _ => { + debug!("Not sending control to peer: {:?} because it is not connected, dropping message: {:?}", self, control); + Ok(()) } - _ => {} } - debug!("Not sending control to peer: {:?} because it is not connected, dropping message: {:?}", self, control); + } + + fn get_connection_id(&self) -> Option { + match self.state { + PeerState::Connected(ref conn) => Some(conn.connection_id), + _ => None + } } fn is_connected(&self) -> bool { @@ -233,6 +288,15 @@ impl Connection { _ => false } } + + fn connection_established(&mut self, connection_ref: ConnectionRef) -> Option { + let prev_conn = match self.state { + PeerState::Connected(ref existing) => Some(existing.connection_id), + _ => None + }; + self.state = PeerState::Connected(connection_ref); + prev_conn + } } #[derive(Debug)] @@ -270,7 +334,7 @@ mod test { let connection = controller_state.all_connections.values() .find(|conn| conn.remote_address == peer.address) .unwrap(); - subject.peer_connection_established(peer.id, connection); + subject.peer_connection_established(peer.clone(), connection); } (subject, controller_state) } @@ -341,7 +405,11 @@ mod test { let time = Instant::now(); subject.establish_connections(time, &mut controller_state); - subject.peer_connection_established(peer_id, &peer_connection); + let peer = Peer { + id: peer_id, + address: peer_address, + }; + subject.peer_connection_established(peer, &peer_connection); assert_eq!(Some(peer_id), subject.get_peer_id(peer_connection.connection_id)); @@ -516,9 +584,9 @@ pub mod mock { let mut lock = self.peer_stubs.lock().unwrap(); lock.remove(&connection_id); } - fn peer_connection_established(&mut self, peer_id: FloInstanceId, success_connection: &ConnectionRef) { - self.push_invocation(Invocation::PeerConnectionEstablished {peer_id, success_connection: success_connection.clone()}); - self.stub_peer_connection(success_connection.connection_id, peer_id); + fn peer_connection_established(&mut self, peer: Peer, success_connection: &ConnectionRef) { + self.stub_peer_connection(success_connection.connection_id, peer.id); + self.push_invocation(Invocation::PeerConnectionEstablished {peer, success_connection: success_connection.clone()}); } fn broadcast_to_peers(&mut self, connection_control: ConnectionControl) { self.push_invocation(Invocation::BroadcastToPeers {connection_control}); @@ -542,7 +610,7 @@ pub mod mock { addr: SocketAddr, }, ConnectionClosed{ connection_id: ConnectionId }, - PeerConnectionEstablished{peer_id: FloInstanceId, success_connection: ConnectionRef}, + PeerConnectionEstablished{peer: Peer, success_connection: ConnectionRef}, BroadcastToPeers{connection_control: ConnectionControl}, SendToPeer{peer_id: FloInstanceId, connection_control :ConnectionControl}, } diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index f8d2f78..85f07c2 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -61,7 +61,7 @@ pub struct Peer { #[derive(Debug, Clone, PartialEq)] pub struct PeerUpgrade { - pub peer_id: FloInstanceId, + pub peer: Peer, pub system_primary: Option, pub cluster_members: Vec, } @@ -116,9 +116,9 @@ impl SystemOperation { SystemOperation::new(connection_id, SystemOpType::RequestVote(request)) } - pub fn connection_upgraded_to_peer(connection_id: ConnectionId, peer_id: FloInstanceId, system_primary: Option, cluster_members: Vec) -> SystemOperation { + pub fn connection_upgraded_to_peer(connection_id: ConnectionId, peer: Peer, system_primary: Option, cluster_members: Vec) -> SystemOperation { let upgrade = PeerUpgrade { - peer_id, + peer, system_primary, cluster_members }; diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index df63e85..38e5d07 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -77,24 +77,39 @@ impl FloController { } // TODO: time operation handling and record perf metrics + let FloController{ref mut cluster_state, ref mut controller_state, ..} = *self; match op_type { SystemOpType::IncomingConnectionEstablished(connection_ref) => { - self.controller_state.all_connections.insert(connection_id, connection_ref); + controller_state.all_connections.insert(connection_id, connection_ref); + } + SystemOpType::ConnectionClosed => { + // TODO: do we need to inform ConsensusProcessor about the connection in case it is a peer connection? + controller_state.all_connections.remove(&connection_id); } SystemOpType::OutgoingConnectionFailed(address) => { - self.controller_state.all_connections.remove(&connection_id); - self.cluster_state.outgoing_connection_failed(connection_id, address); + cluster_state.outgoing_connection_failed(connection_id, address); + controller_state.all_connections.remove(&connection_id); } SystemOpType::ConnectionUpgradeToPeer(upgrade) => { - let FloController{ref mut cluster_state, ref mut controller_state, ..} = *self; cluster_state.peer_connection_established(upgrade, connection_id, controller_state); } SystemOpType::Tick => { - let FloController{ref mut cluster_state, ref mut controller_state, ..} = *self; cluster_state.tick(op_start_time, controller_state); } - other @ _ => { - warn!("Ignoring SystemOperation: {:?}", other); + SystemOpType::RequestVote(request) => { + cluster_state.request_vote_received(connection_id, request); + } + SystemOpType::VoteResponseReceived(response) => { + cluster_state.vote_response_received(op_start_time, connection_id, response, controller_state); + } + SystemOpType::AppendEntriesReceived(append) => { + cluster_state.append_entries_received(connection_id, append, controller_state); + } + SystemOpType::AppendEntriesResponseReceived(response) => { + cluster_state.append_entries_response_received(connection_id, response, controller_state); + } + SystemOpType::PartitionOp(partition_op) => { + warn!("Ignoring PartitionOp: {:?}", partition_op); } } } diff --git a/flo-server/src/engine/controller/system_reader.rs b/flo-server/src/engine/controller/system_reader.rs index 72daa4b..0ff32f7 100644 --- a/flo-server/src/engine/controller/system_reader.rs +++ b/flo-server/src/engine/controller/system_reader.rs @@ -27,7 +27,12 @@ impl SystemStreamReader { /// sets the reader to the given segment and offset if it's not already there pub fn set_to(&mut self, segment: SegmentNum, offset: usize) -> io::Result<()> { - self.inner.set_to(segment, offset) + if segment.is_set() { + self.inner.set_to(segment, offset) + } else { + self.inner.set_to_beginning(); + Ok(()) + } } pub fn fill_buffer(&mut self, event_buffer: &mut Vec>) -> io::Result { diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index dda8af0..5371b1f 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -69,8 +69,8 @@ impl SystemStreamRef { self.send(op); } - pub fn connection_upgraded_to_peer(&mut self, connection_id: ConnectionId, peer_id: FloInstanceId, system_primary: Option, cluster_members: Vec) { - let op = SystemOperation::connection_upgraded_to_peer(connection_id, peer_id, system_primary, cluster_members); + pub fn connection_upgraded_to_peer(&mut self, connection_id: ConnectionId, peer: Peer, system_primary: Option, cluster_members: Vec) { + let op = SystemOperation::connection_upgraded_to_peer(connection_id, peer, system_primary, cluster_members); self.send(op); } diff --git a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs index 5243af1..32cfefe 100644 --- a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs +++ b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs @@ -78,6 +78,11 @@ impl PartitionReader { unimplemented!() } + pub fn set_to_beginning(&mut self) { + self.current_segment_reader = None; + self.clear_error(); + } + pub fn set_to(&mut self, segment_num: SegmentNum, offset: usize) -> io::Result<()> { if !segment_num.is_set() { return Err(io::Error::new(io::ErrorKind::InvalidInput, "Cannot set PartitionReader segment to 0")); @@ -90,6 +95,7 @@ impl PartitionReader { self.current_segment_reader = Some(segment); } self.current_segment_reader.as_mut().unwrap().set_offset(offset); + self.clear_error(); Ok(()) } @@ -99,6 +105,10 @@ impl PartitionReader { }).unwrap_or((SegmentNum(0), 0)) } + fn clear_error(&mut self) { + self.returned_error = false; + } + fn should_skip(&self, result: &Option>) -> bool { if let Some(Ok(ref event)) = *result { !self.filter.matches(event) diff --git a/flo-server/src/server/mod.rs b/flo-server/src/server/mod.rs index b9f5ab5..54baaba 100644 --- a/flo-server/src/server/mod.rs +++ b/flo-server/src/server/mod.rs @@ -39,6 +39,7 @@ pub fn run(mut options: ServerOptions) -> io::Result<()> { event_loop_handles: event_loop_handles.clone(), } }); + info!("Using {:?}", cluster_options); let controller_options = ControllerOptions { storage_dir: options.data_dir.clone(), From b2610eccc3b9ac4cfd49b8cf03995e29a8e5c334 Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 20 Jan 2018 20:45:00 -0500 Subject: [PATCH 48/73] tweak travis config to try to speed up builds --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e148615..78675d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ +sudo: false language: rust +cache: cargo env: global: - RUST_BACKTRACE=1 @@ -17,4 +19,4 @@ matrix: os: osx - rust: nightly os: osx -script: cargo test --all --no-fail-fast +script: cargo test --all From 6c73000849e6df86150a2960109066257f74f30d Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 20 Jan 2018 23:11:57 -0500 Subject: [PATCH 49/73] basic wiring in of commit manager into partitionImpl --- .../src/engine/controller/initialization.rs | 4 +- .../src/engine/controller/system_event.rs | 3 +- .../partition/controller/commit_manager.rs | 4 + .../event_stream/partition/controller/mod.rs | 122 ++++++++++++++++-- .../partition/event_reader/mod.rs | 30 ++++- .../src/engine/event_stream/partition/mod.rs | 4 +- 6 files changed, 145 insertions(+), 22 deletions(-) diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index 7dfff91..576f7d0 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -73,7 +73,7 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul }).collect::>(); let shared_stream_refs = Arc::new(Mutex::new(shared_stream_refs)); - let system_highest_counter = system_partition.event_counter_reader(); + let system_commit_index_reader = system_partition.commit_index_reader(); let (system_partition_tx, system_partition_rx) = ::engine::controller::create_system_partition_channels(); let (shared_state, file_cluster_state) = if use_cluster_mode { @@ -90,7 +90,7 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul let engine_ref = create_engine_ref(shared_stream_refs.clone(), system_partition.get_shared_reader_refs(), - system_highest_counter, + system_commit_index_reader, system_primary_status_writer.reader(), system_primary_address.clone(), system_partition_tx, diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index 5b343c8..8ec9a0b 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -2,6 +2,7 @@ use rmp_serde::decode::Error; use protocol::Term; use event::{FloEvent, FloEventId, OwnedFloEvent, EventCounter, ActorId, Timestamp, time}; +use engine::event_stream::EventStreamOptions; use engine::event_stream::partition::PersistentEvent; #[derive(Debug, PartialEq)] @@ -76,7 +77,7 @@ impl FloEvent for SystemEvent { } } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub struct SystemEventData { pub term: Term, } diff --git a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs index 34ff8fc..f41bb4e 100644 --- a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs +++ b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs @@ -54,6 +54,10 @@ impl CommitManager { self.commit_index.set_if_greater(new_index as usize); } + pub fn get_commit_index(&self) -> EventCounter { + self.commit_index.get() as EventCounter + } + pub fn add_member(&mut self, peer_id: FloInstanceId) { self.peers.push((peer_id, 0)); let new_ack_requirement = self.compute_min_required(); diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index b84c0aa..541b2b1 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -9,7 +9,7 @@ use std::path::PathBuf; use chrono::{Duration}; use atomics::{AtomicCounterWriter, AtomicCounterReader, AtomicBoolReader}; -use protocol::ProduceEvent; +use protocol::{ProduceEvent, FloInstanceId}; use event::{ActorId, FloEventId, EventCounter, FloEvent, Timestamp, time}; use super::{SharedReaderRefs, SharedReaderRefsMut, Operation, OpType, ProduceOperation, ConsumeOperation, PartitionReader, EventFilter, SegmentNum}; use super::segment::Segment; @@ -18,6 +18,7 @@ use engine::event_stream::{EventStreamOptions, HighestCounter}; use engine::ConnectionId; use self::util::get_segment_files; use self::consumer_manager::ConsumerManager; +use self::commit_manager::CommitManager; const FIRST_SEGMENT_NUM: SegmentNum = SegmentNum(1); @@ -41,8 +42,6 @@ pub struct PartitionImpl { /// Shared EventCounter for all partitions in the event stream. Serves as a Lamport clock to help reason about relative /// order of events across multiple partitions. Used to generate new `EventCounter`s when events are appended event_stream_highest_counter: HighestCounter, - /// Tracks the highest committed event in this partition. This value is shared with the `ConnectionHandler`s - partition_highest_committed: AtomicCounterWriter, /// Whether this instance is the primary for this partition. This value is set by `FloController`, since it requires /// consensus to modify which instance is primary for a partition. primary: AtomicBoolReader, @@ -52,6 +51,9 @@ pub struct PartitionImpl { /// consumers each have a notifier added here consumer_manager: ConsumerManager, + + /// determines when events are committed + commit_manager: CommitManager, } impl PartitionImpl { @@ -80,7 +82,7 @@ impl PartitionImpl { //TODO: differentiate between highest committed and highest uncommitted when initializing existing partition let current_greatest_id = index.greatest_event_counter(); highest_counter.set_if_greater(current_greatest_id); - let partition_id_counter = AtomicCounterWriter::with_value(current_greatest_id as usize); + let partition_commit_counter = AtomicCounterWriter::with_value(current_greatest_id as usize); // TODO: factor out a more legit method of timing and logging perf stats let init_time = start_time.elapsed(); @@ -101,7 +103,7 @@ impl PartitionImpl { segments: initialized_segments, index: index, event_stream_highest_counter: highest_counter, - partition_highest_committed: partition_id_counter, + commit_manager: CommitManager::new(partition_commit_counter), primary: status_reader, reader_refs: reader_refs, consumer_manager: ConsumerManager::new(), @@ -125,27 +127,35 @@ impl PartitionImpl { segments: VecDeque::with_capacity(4), index: PartitionIndex::new(partition_num), event_stream_highest_counter: highest_counter, - partition_highest_committed: AtomicCounterWriter::zero(), + commit_manager: CommitManager::new(AtomicCounterWriter::with_value(0)), primary: status_reader, reader_refs: SharedReaderRefsMut::new(), consumer_manager: ConsumerManager::new(), }) } + pub fn add_replication_node(&mut self, peer: FloInstanceId) { + self.commit_manager.add_member(peer); + } + + pub fn events_acknowledged(&mut self, peer_id: FloInstanceId, counter: EventCounter) { + self.commit_manager.acknowledgement_received(peer_id, counter); + } + pub fn event_stream_name(&self) -> &str { &self.event_stream_name } - pub fn event_counter_reader(&self) -> AtomicCounterReader { - self.partition_highest_committed.reader() + pub fn commit_index_reader(&self) -> AtomicCounterReader { + self.commit_manager.get_commit_index_reader() } pub fn set_commit_index(&mut self, new_index: EventCounter) { - self.partition_highest_committed.set_if_greater(new_index as usize); + self.commit_manager.update_commit_index(new_index) } pub fn get_commit_index(&self) -> EventCounter { - self.partition_highest_committed.get() as EventCounter + self.commit_manager.get_commit_index() } pub fn primary_status_reader(&self) -> AtomicBoolReader { @@ -357,7 +367,7 @@ impl PartitionImpl { } }; - let commit_index_reader = self.partition_highest_committed.reader(); + let commit_index_reader = self.commit_index_reader(); PartitionReader::new(connection_id, self.partition_num, filter, @@ -423,7 +433,95 @@ mod test { const CONNECTION: ConnectionId = 55; #[test] - fn partition_impl_integration_test() { + fn events_are_committed_when_acknowledged_by_a_majority() { + let status = AtomicBoolWriter::with_value(true); + let options = EventStreamOptions { + name: "superduper".to_owned(), + num_partitions: 1, + event_retention: Duration::seconds(20), + max_segment_duration: Duration::seconds(5), + segment_max_size_bytes: 256, + }; + let tempdir = TempDir::new("partition_persist_events_and_read_them_back").unwrap(); + + // Init a new partition and append a bunch of events in two groups + let mut partition = PartitionImpl::init_new(PARTITION_NUM, + tempdir.path().to_owned(), + &options, + status.reader(), + HighestCounter::zero()).unwrap(); + + let peer_1 = FloInstanceId::generate_new(); + let peer_2 = FloInstanceId::generate_new(); + let peer_3 = FloInstanceId::generate_new(); + let peer_4 = FloInstanceId::generate_new(); + partition.add_replication_node(peer_1); + partition.add_replication_node(peer_2); + partition.add_replication_node(peer_3); + partition.add_replication_node(peer_4); + + let events = vec![ + ProduceEvent { + op_id: 3, + partition: PARTITION_NUM, + namespace: "/foo/bar".to_owned(), + parent_id: None, + data: "the quick".to_owned().into_bytes(), + }, + ProduceEvent { + op_id: 3, + partition: PARTITION_NUM, + namespace: "/foo/bar".to_owned(), + parent_id: None, + data: "brown fox".to_owned().into_bytes(), + }, + ProduceEvent { + op_id: 3, + partition: PARTITION_NUM, + namespace: "/foo/bar".to_owned(), + parent_id: None, + data: "jumped over".to_owned().into_bytes(), + }, + ProduceEvent { + op_id: 3, + partition: PARTITION_NUM, + namespace: "/foo/bar".to_owned(), + parent_id: None, + data: "the lazy dog".to_owned().into_bytes(), + }, + ]; + partition.append_all(events); + assert_eq!(0, partition.get_commit_index()); + + let mut reader = partition.create_reader(3, EventFilter::All, 0); + assert!(reader.read_next_committed().is_none()); + + let mut event_count = 0; + for _ in 0..4 { + reader.read_next_uncommitted().expect("read uncommitted returned None").expect("failed to read uncommitted"); + } + + partition.events_acknowledged(peer_1, 2); + assert_eq!(0, partition.get_commit_index()); + + partition.events_acknowledged(peer_3, 3); + assert_eq!(2, partition.get_commit_index()); // 3 of 5 acknowledge at least event 2 + + partition.events_acknowledged(peer_2, 4); + assert_eq!(3, partition.get_commit_index()); + + reader.set_to_beginning(); + + for _ in 0..3 { + reader.read_next_committed().expect("read committed returned None").expect("failed to read committed"); + } + + let last_event = reader.read_next_committed(); + assert!(last_event.is_none()); + } + + #[test] + fn events_are_persisted_and_can_be_read_after_reinitializing_in_the_same_directory() { let _ = ::env_logger::init(); let status = AtomicBoolWriter::with_value(true); diff --git a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs index 32cfefe..c54ac38 100644 --- a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs +++ b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs @@ -2,7 +2,7 @@ mod namespace; use std::io; -use event::{FloEvent, ActorId}; +use event::{FloEvent, ActorId, EventCounter}; use engine::ConnectionId; use engine::event_stream::partition::{SharedReaderRefs, SegmentNum}; @@ -43,6 +43,7 @@ pub struct PartitionReader { current_segment_reader: Option, segment_readers_ref: SharedReaderRefs, returned_error: bool, + next_buffer: Option, } @@ -63,6 +64,7 @@ impl PartitionReader { current_segment_reader: current_reader, segment_readers_ref: segment_refs, returned_error: false, + next_buffer: None, } } @@ -75,12 +77,26 @@ impl PartitionReader { } pub fn read_next_committed(&mut self) -> Option> { - unimplemented!() + let result = self.read_next_uncommitted(); + let commit_index = self.commit_index_reader.load_relaxed() as EventCounter; + let buffer = match &result { + &Some(Ok(ref event)) if event.id().event_counter > commit_index => true, + _ => false, + }; + + if buffer { + // hold onto this event until later + let event = result.unwrap().unwrap(); + self.next_buffer = Some(event); + None + } else { + result + } } pub fn set_to_beginning(&mut self) { self.current_segment_reader = None; - self.clear_error(); + self.reset(); } pub fn set_to(&mut self, segment_num: SegmentNum, offset: usize) -> io::Result<()> { @@ -95,7 +111,7 @@ impl PartitionReader { self.current_segment_reader = Some(segment); } self.current_segment_reader.as_mut().unwrap().set_offset(offset); - self.clear_error(); + self.reset(); Ok(()) } @@ -105,8 +121,9 @@ impl PartitionReader { }).unwrap_or((SegmentNum(0), 0)) } - fn clear_error(&mut self) { + fn reset(&mut self) { self.returned_error = false; + self.next_buffer = None; } fn should_skip(&self, result: &Option>) -> bool { @@ -130,6 +147,9 @@ impl PartitionReader { fn read_next(&mut self) -> Option> { if self.returned_error { return None; + } else if self.next_buffer.is_some() { + let next = self.next_buffer.take().unwrap(); + return Some(Ok(next)); } if self.current_reader_is_exhausted() { diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index 1387d9f..e2bf87a 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -340,7 +340,7 @@ pub fn initialize_new_partition(partition_num: ActorId, pub fn run_partition(partition_impl: PartitionImpl, primary_server_addr: Arc>>) -> io::Result { let partition_num = partition_impl.partition_num(); - let event_counter_reader = partition_impl.event_counter_reader(); + let commit_index_reader = partition_impl.commit_index_reader(); let primary_status_reader = partition_impl.primary_status_reader(); let event_stream_name = partition_impl.event_stream_name().to_owned(); let (tx, rx) = create_partition_channels(); @@ -372,7 +372,7 @@ pub fn run_partition(partition_impl: PartitionImpl, primary_server_addr: Arc Date: Sun, 21 Jan 2018 09:43:44 -0500 Subject: [PATCH 50/73] remove unused method from controller state, tell consensus processor about closed connections --- flo-server/src/engine/controller/controller_state.rs | 10 +--------- flo-server/src/engine/controller/mod.rs | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/flo-server/src/engine/controller/controller_state.rs b/flo-server/src/engine/controller/controller_state.rs index 22e2b07..9a1dca9 100644 --- a/flo-server/src/engine/controller/controller_state.rs +++ b/flo-server/src/engine/controller/controller_state.rs @@ -20,7 +20,6 @@ pub trait ControllerState { fn get_last_committed(&mut self) -> io::Result<(EventCounter, Term)>; fn get_next_event(&mut self, start_after: EventCounter) -> Option>; - fn set_commit_index(&mut self, new_index: EventCounter); fn get_next_entry(&self, event_counter: EventCounter) -> Option; fn get_current_file_offset(&self) -> (SegmentNum, usize); } @@ -96,10 +95,6 @@ impl ControllerState for ControllerStateImpl { self.all_connections.get(&connection_id) } - fn set_commit_index(&mut self, new_index: EventCounter) { - self.system_partition.set_commit_index(new_index); - } - fn get_next_entry(&self, previous: EventCounter) -> Option { self.system_partition.get_next_index_entry(previous) } @@ -155,7 +150,7 @@ pub mod mock { } pub fn with_commit_index(mut self, index: EventCounter) -> MockControllerState { - self.set_commit_index(index); + self.commit_index = index; self } @@ -197,9 +192,6 @@ pub mod mock { (sys.segment, sys.file_offset) }).unwrap_or((SegmentNum::new_unset(), 0)) } - fn set_commit_index(&mut self, new_index: EventCounter) { - self.commit_index = new_index; - } fn get_last_committed(&mut self) -> io::Result<(EventCounter, Term)> { let commit_idx = self.commit_index; self.system_events.get(&commit_idx).map(|sys| { diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 38e5d07..65b9b3a 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -83,7 +83,7 @@ impl FloController { controller_state.all_connections.insert(connection_id, connection_ref); } SystemOpType::ConnectionClosed => { - // TODO: do we need to inform ConsensusProcessor about the connection in case it is a peer connection? + cluster_state.connection_closed(connection_id); controller_state.all_connections.remove(&connection_id); } SystemOpType::OutgoingConnectionFailed(address) => { From 1041e565e0df229c44ff1bee47d08bcffa371c38 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 21 Jan 2018 12:07:58 -0500 Subject: [PATCH 51/73] start adding ability for partition to notify producers once events are committed, only used for system partition right now --- .../src/engine/connection_handler/mod.rs | 1 - .../engine/controller/cluster_state/mod.rs | 41 +++++++---- .../src/engine/controller/controller_state.rs | 11 ++- flo-server/src/engine/controller/mod.rs | 73 +++++++++++++++++-- .../src/engine/controller/system_event.rs | 10 ++- .../partition/controller/commit_manager.rs | 4 + .../partition/controller/consumer_manager.rs | 46 ++++++++---- .../event_stream/partition/controller/mod.rs | 51 +++++++++---- .../partition/controller/pending_produce.rs | 53 ++++++++++++++ 9 files changed, 236 insertions(+), 54 deletions(-) create mode 100644 flo-server/src/engine/event_stream/partition/controller/pending_produce.rs diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index a7b5137..1d9ab38 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -74,7 +74,6 @@ impl ConnectionHandler { ConnectionControl::SendAppendEntriesResponse(response) => { peer_state.send_append_entries_response(response, common_state) } - _ => unimplemented!() } } diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index fe48573..82aac60 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -33,6 +33,7 @@ static STATE_UPDATE_FAILED: &'static str = "failed to persist changes to cluster pub trait ConsensusProcessor: Send { fn tick(&mut self, now: Instant, controller_state: &mut ControllerState); fn is_primary(&self) -> bool; + fn get_current_term(&self) -> Term; fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, controller_state: &ControllerState); fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr); fn connection_closed(&mut self, connection_id: ConnectionId); @@ -42,6 +43,8 @@ pub trait ConsensusProcessor: Send { fn append_entries_received(&mut self, connection_id: ConnectionId, append: ReceiveAppendEntries, controller_state: &mut ControllerState); fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState); + + fn send_append_entries(&mut self, controller_state: &mut ControllerState); } #[derive(Debug)] @@ -234,15 +237,6 @@ impl ClusterManager { self.state = State::Follower; } - fn send_append_entries(&mut self, controller_state: &mut ControllerState) { - let ClusterManager { ref mut primary_state, ref mut connection_manager, ref persistent, ..} = *self; - - primary_state.as_mut().map(|state| { - let all_peers = persistent.cluster_members.iter().map(|peer| peer.id); - state.send_append_entries(controller_state, connection_manager.as_mut(), all_peers); - }); - } - fn append_system_events(&mut self, events: &[OwnedFloEvent]) -> io::Result { // TODO: actually persist the system events from ApendEntries if !events.is_empty() { @@ -263,6 +257,16 @@ impl ClusterManager { } impl ConsensusProcessor for ClusterManager { + + fn send_append_entries(&mut self, controller_state: &mut ControllerState) { + let ClusterManager { ref mut primary_state, ref mut connection_manager, ref persistent, ..} = *self; + + primary_state.as_mut().map(|state| { + let all_peers = persistent.cluster_members.iter().map(|peer| peer.id); + state.send_append_entries(controller_state, connection_manager.as_mut(), all_peers); + }); + } + fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { self.connection_manager.outgoing_connection_failed(connection_id, address); self.connection_resolved(address); @@ -450,6 +454,9 @@ impl ConsensusProcessor for ClusterManager { // TODO: confirm entries with partition impl } } + fn get_current_term(&self) -> Term { + self.persistent.current_term + } } #[derive(Debug, PartialEq, Clone)] @@ -1363,7 +1370,7 @@ impl ConsensusProcessor for NoOpConsensusProcessor { panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); } fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote) { - + panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); } fn tick(&mut self, now: Instant, controller_state: &mut ControllerState) { } @@ -1371,16 +1378,22 @@ impl ConsensusProcessor for NoOpConsensusProcessor { true } fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse, controller: &mut ControllerState) { - unimplemented!() + panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); } fn append_entries_received(&mut self, connection_id: ConnectionId, append: ReceiveAppendEntries, controller_state: &mut ControllerState) { - unimplemented!() + panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); } fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState) { - unimplemented!() + panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); } fn connection_closed(&mut self, connection_id: ConnectionId) { - unimplemented!() + + } + fn send_append_entries(&mut self, _controller_state: &mut ControllerState) { + // do nothing + } + fn get_current_term(&self) -> Term { + 0 } } diff --git a/flo-server/src/engine/controller/controller_state.rs b/flo-server/src/engine/controller/controller_state.rs index 9a1dca9..bf68a91 100644 --- a/flo-server/src/engine/controller/controller_state.rs +++ b/flo-server/src/engine/controller/controller_state.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use std::collections::HashMap; use std::io; -use protocol::Term; +use protocol::{Term, FloInstanceId}; use event::EventCounter; use engine::ConnectionId; use engine::controller::{ConnectionRef, SystemEvent}; @@ -22,6 +22,8 @@ pub trait ControllerState { fn get_next_entry(&self, event_counter: EventCounter) -> Option; fn get_current_file_offset(&self) -> (SegmentNum, usize); + + fn add_system_replication_node(&mut self, peer: FloInstanceId); } pub struct ControllerStateImpl { @@ -124,6 +126,9 @@ impl ControllerState for ControllerStateImpl { }) }) } + fn add_system_replication_node(&mut self, peer: FloInstanceId) { + self.system_partition.add_replication_node(peer); + } } @@ -205,6 +210,10 @@ pub mod mock { Ok((*id, sys.term)) }) } + + fn add_system_replication_node(&mut self, peer: FloInstanceId) { + unimplemented!() + } } #[derive(Clone, Debug, PartialEq)] diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 65b9b3a..abc4dee 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -12,13 +12,15 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; use std::collections::HashMap; use std::io; +use std::time::Instant; +use protocol::{ProduceEvent, Term}; use event::EventCounter; use engine::ConnectionId; use engine::event_stream::{EventStreamRef, EventStreamRefMut, EventStreamOptions}; -use engine::event_stream::partition::{PersistentEvent, IndexEntry, SegmentNum}; +use engine::event_stream::partition::{self, PersistentEvent, IndexEntry, SegmentNum}; use engine::event_stream::partition::controller::PartitionImpl; use self::cluster_state::ConsensusProcessor; @@ -42,9 +44,9 @@ pub fn create_system_partition_channels() -> (SystemPartitionSender, SystemParti } - -/// A specialized event stream that always has exactly one partition and manages the cluster state and consensus -/// Of course there is no cluster state and thus no consensus at the moment, but we'll just leave this here... +/// A specialized event stream that always has exactly one partition and manages the cluster state and consensus processor +/// The `ConsensusProcessor` manages most operations on the `ControllerState`. If this instance is running in standalone mode +/// then all operations will basically just immediately succeed #[allow(dead_code)] pub struct FloController { controller_state: ControllerStateImpl, @@ -109,7 +111,7 @@ impl FloController { cluster_state.append_entries_response_received(connection_id, response, controller_state); } SystemOpType::PartitionOp(partition_op) => { - warn!("Ignoring PartitionOp: {:?}", partition_op); + handle_partition_op(connection_id, op_start_time, partition_op, cluster_state.as_mut(), controller_state); } } } @@ -122,5 +124,66 @@ impl FloController { } +fn handle_partition_op(connection_id: ConnectionId, op_start_time: Instant, op: partition::OpType, + cluster_state: &mut ConsensusProcessor, controller_state: &mut ControllerStateImpl) { + use engine::event_stream::partition::OpType::*; + match op { + Produce(produce_op) => { + produce_system_events(produce_op, cluster_state, controller_state); + } + Consume(consume_op) => { + // `handle_consume` always just returns `Ok(())` at the moment anyway. Should probably just change it to unit return type + let _ = controller_state.system_partition.handle_consume(connection_id, consume_op); + } + StopConsumer => { + controller_state.system_partition.stop_consumer(connection_id); + } + Tick => { + // only used by the partition to expire old events, which we do not do for the system partition. + // anyway, nothing is creating these operations anyway + } + } +} + +fn produce_system_events(mut produce_op: partition::ProduceOperation, cluster_state: &mut ConsensusProcessor, controller_state: &mut ControllerStateImpl) { + if cluster_state.is_primary() { + let term = cluster_state.get_current_term(); + let convert_result = convert_to_system_events(&mut produce_op.events, term); + + if let Err(err) = convert_result { + // We're done here + produce_op.client.complete(Err(err)); + } else { + // hand off the modified operation to the partition, which will complete it + let result = controller_state.system_partition.handle_produce(produce_op); + if let Err(partition_err) = result { + error!("Partition error creating new system events: {:?}", partition_err); + } else { + // success! send out AppendEntries + cluster_state.send_append_entries(controller_state) + } + } + } else { + let err = io::Error::new(io::ErrorKind::Other, "Not primary"); + produce_op.client.complete(Err(err)); + } +} + + +fn convert_to_system_events(events: &mut Vec, term: Term) -> io::Result<()> { + for event in events.iter_mut() { + // TODO: Once system events have any sort of body, this function will need to change a bit + // for now, we'll just make sure that the body is empty + // in the future, we'll attempt to deserialize the body as a SystemEventType + if event.data.is_empty() { + let new_body = SystemEventData { term }; + let serialized = new_body.serialize(); + event.data = serialized; + } else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid system event body")); + } + } + Ok(()) +} diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index 8ec9a0b..af3d08c 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -46,8 +46,8 @@ impl SystemEvent { pub fn new(id: FloEventId, parent: Option, namespace: String, time: Timestamp, data: &SystemEventData) -> SystemEvent { let term = data.term; // TODO: I feel like this is probably a safe unwrap, but might be good to double check - let data = ::rmp_serde::to_vec(data).unwrap(); - let event = OwnedFloEvent::new(id, parent, time, namespace, data); + let serialized = data.serialize(); + let event = OwnedFloEvent::new(id, parent, time, namespace, serialized); SystemEvent { term, wrapped: event @@ -82,6 +82,12 @@ pub struct SystemEventData { pub term: Term, } +impl SystemEventData { + pub fn serialize(&self) -> Vec { + ::rmp_serde::to_vec(self).unwrap() + } +} + #[cfg(test)] mod test { diff --git a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs index f41bb4e..a443b61 100644 --- a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs +++ b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs @@ -50,6 +50,10 @@ impl CommitManager { } } + pub fn is_standalone(&self) -> bool { + self.min_required_for_commit == 0 + } + pub fn update_commit_index(&mut self, new_index: EventCounter) { self.commit_index.set_if_greater(new_index as usize); } diff --git a/flo-server/src/engine/event_stream/partition/controller/consumer_manager.rs b/flo-server/src/engine/event_stream/partition/controller/consumer_manager.rs index 4b0d827..1806c82 100644 --- a/flo-server/src/engine/event_stream/partition/controller/consumer_manager.rs +++ b/flo-server/src/engine/event_stream/partition/controller/consumer_manager.rs @@ -3,39 +3,53 @@ use engine::ConnectionId; use engine::event_stream::partition::ConsumerNotifier; pub struct ConsumerManager { - uncommitted_consumers: Vec>, + uncommitted: Vec>, + committed: Vec>, } impl ConsumerManager { pub fn new() -> ConsumerManager { ConsumerManager { - uncommitted_consumers: Vec::with_capacity(4) + uncommitted: Vec::with_capacity(4), + committed: Vec::with_capacity(4) } } + pub fn add_committed(&mut self, consumer: Box) { + self.committed.push(consumer); + } + pub fn add_uncommitted(&mut self, consumer: Box) { - self.uncommitted_consumers.push(consumer); + self.uncommitted.push(consumer); } pub fn remove(&mut self, consumer: ConnectionId) { - self.uncommitted_consumers.retain(|notifier| { + self.uncommitted.retain(|notifier| { notifier.connection_id() != consumer }) } + pub fn notify_committed(&mut self) { + do_notify("committed", &mut self.committed); + } + pub fn notify_uncommitted(&mut self) { - let mut count = 0; - self.uncommitted_consumers.retain(|consumer| { - let active = consumer.is_active(); - if active { - consumer.notify(); - count += 1; - } else { - debug!("Removing consumer for connection_id: {} because it is inactive", consumer.connection_id()); - } - active - }); - debug!("Notified {} uncommitted consumers", count); + do_notify("uncommitted", &mut self.uncommitted); } } + +fn do_notify(consumer_type: &str, consumers: &mut Vec>) { + let mut count = 0; + consumers.retain(|consumer| { + let active = consumer.is_active(); + if active { + consumer.notify(); + count += 1; + } else { + debug!("Removing consumer for connection_id: {} because it is inactive", consumer.connection_id()); + } + active + }); + debug!("Notified {} {} consumers", count, consumer_type); +} diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index 541b2b1..2a81676 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -1,6 +1,7 @@ mod util; mod consumer_manager; mod commit_manager; +mod pending_produce; use std::io; use std::collections::VecDeque; @@ -19,6 +20,7 @@ use engine::ConnectionId; use self::util::get_segment_files; use self::consumer_manager::ConsumerManager; use self::commit_manager::CommitManager; +use self::pending_produce::PendingProduceOperations; const FIRST_SEGMENT_NUM: SegmentNum = SegmentNum(1); @@ -54,6 +56,9 @@ pub struct PartitionImpl { /// determines when events are committed commit_manager: CommitManager, + + /// handles notifying producers when their events have been committed + pending_produce_operations: PendingProduceOperations, } impl PartitionImpl { @@ -107,6 +112,7 @@ impl PartitionImpl { primary: status_reader, reader_refs: reader_refs, consumer_manager: ConsumerManager::new(), + pending_produce_operations: PendingProduceOperations::new(partition_num), }) } @@ -131,6 +137,7 @@ impl PartitionImpl { primary: status_reader, reader_refs: SharedReaderRefsMut::new(), consumer_manager: ConsumerManager::new(), + pending_produce_operations: PendingProduceOperations::new(partition_num), }) } @@ -139,7 +146,12 @@ impl PartitionImpl { } pub fn events_acknowledged(&mut self, peer_id: FloInstanceId, counter: EventCounter) { - self.commit_manager.acknowledgement_received(peer_id, counter); + let new_index = self.commit_manager.acknowledgement_received(peer_id, counter); + if let Some(committed_event) = new_index { + // This acknowledgement has caused a new event to be commmitted. Notify the producers of the successful operations + // and also notify any consumers of committed events + + } } pub fn event_stream_name(&self) -> &str { @@ -166,6 +178,10 @@ impl PartitionImpl { self.partition_num } + pub fn stop_consumer(&mut self, connection_id: ConnectionId) { + self.consumer_manager.remove(connection_id); + } + pub fn process(&mut self, operation: Operation) -> io::Result<()> { trace!("Partition: {}, got operation: {:?}", self.partition_num, operation); @@ -180,17 +196,13 @@ impl PartitionImpl { self.handle_consume(connection_id, consume_op) } OpType::StopConsumer => { - self.consumer_manager.remove(connection_id); + self.stop_consumer(connection_id); Ok(()) } OpType::Tick => { self.expire_old_events(); Ok(()) } - other @ _ => { - warn!("received unexpected OpType: {:?}", other); - Ok(()) - } } } @@ -216,15 +228,25 @@ impl PartitionImpl { }); } - fn handle_produce(&mut self, produce: ProduceOperation) -> io::Result<()> { + pub fn handle_produce(&mut self, produce: ProduceOperation) -> io::Result<()> { let ProduceOperation {client, op_id, events} = produce; let result = self.append_all(events); - if let Err(e) = result.as_ref() { - error!("Failed to handle produce operation for op_id: {}, err: {:?}", op_id, e); + + match result { + Ok(id) => { + if self.commit_manager.is_standalone() { + // No biggie if the receiving end has hung up already. The operation will still be considered complete and successful + let _ = client.complete(Ok(id)); + } else { + self.pending_produce_operations.add(op_id, id.event_counter, client); + } + } + Err(io_err) => { + error!("Failed to handle produce operation for op_id: {}, err: {:?}", op_id, io_err); + let _ = client.complete(Err(io_err)); + } } - // No biggie if the receiving end has hung up already. The operation will still be considered complete and successful - // TODO: Consider logging this if the receiving end has hung up already? - let _ = client.send(result); + // TODO: separate out error that gets returned to connection handler so that we can also return an io::Error from this function if one occurs Ok(()) } @@ -334,7 +356,7 @@ impl PartitionImpl { self.segments.front().map(|s| s.segment_num).unwrap_or(SegmentNum(0)) } - fn handle_consume(&mut self, connection_id: ConnectionId, consume: ConsumeOperation) -> io::Result<()> { + pub fn handle_consume(&mut self, connection_id: ConnectionId, consume: ConsumeOperation) -> io::Result<()> { let ConsumeOperation {client_sender, filter, start_exclusive, notifier} = consume; let reader = self.create_reader(connection_id, filter, start_exclusive); @@ -490,13 +512,12 @@ mod test { data: "the lazy dog".to_owned().into_bytes(), }, ]; - partition.append_all(events); + partition.append_all(events).expect("failed to append events"); assert_eq!(0, partition.get_commit_index()); let mut reader = partition.create_reader(3, EventFilter::All, 0); assert!(reader.read_next_committed().is_none()); - let mut event_count = 0; for _ in 0..4 { reader.read_next_uncommitted().expect("read uncommitted returned None").expect("failed to read uncommitted"); } diff --git a/flo-server/src/engine/event_stream/partition/controller/pending_produce.rs b/flo-server/src/engine/event_stream/partition/controller/pending_produce.rs new file mode 100644 index 0000000..a22637e --- /dev/null +++ b/flo-server/src/engine/event_stream/partition/controller/pending_produce.rs @@ -0,0 +1,53 @@ +use std::io; +use std::collections::VecDeque; + +use futures::sync::oneshot::Sender; + +use event::{FloEventId, ActorId, EventCounter}; + +struct Pending { + sender: Sender>, + op_id: u32, + event: EventCounter, +} + +pub struct PendingProduceOperations { + partition: ActorId, + pending: VecDeque, +} + + +impl PendingProduceOperations { + + pub fn new(partition: ActorId) -> PendingProduceOperations { + PendingProduceOperations { + partition, + pending: VecDeque::new(), // Don't allocate any space, since we may just be in standalone mode + } + } + + pub fn add(&mut self, op_id: u32, event: EventCounter, sender: Sender>) { + self.pending.push_back(Pending { + sender, op_id, event + }); + } + + pub fn commit_success(&mut self, commit_index: EventCounter) { + let count = self.pending.iter().take_while(|op| { + op.event <= commit_index + }).count(); + debug!("New commit index of {} will complete {} pending operations", commit_index, count); + + let partition = self.partition; + for _ in 0..count { + let op = self.pending.pop_front().unwrap(); + debug!("Notifying producer of committed op_id: {}, event: {}", op.op_id, op.event); + let _ = op.sender.complete(Ok(FloEventId::new(partition, op.event))); + } + } + + // just a guess at the api we'll probably want here +// pub fn commit_fail(&mut self, fail_through: EventCounter) { +// +// } +} From 5f0c8c6affdb389f8a87fdcea100b1c8d32d42f9 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 21 Jan 2018 13:52:27 -0500 Subject: [PATCH 52/73] prevent system stream from being re-initialized as a user stream, add ability for client and cli to use a specific event stream --- flo-client-cli/src/client_cli/consumer.rs | 15 +++- flo-client-cli/src/client_cli/producer.rs | 19 ++++- flo-client-cli/src/main.rs | 23 +++++-- flo-client-lib/src/async/mod.rs | 10 ++- flo-client-lib/src/async/ops/mod.rs | 2 + .../src/async/ops/set_event_stream.rs | 69 +++++++++++++++++++ flo-client-lib/src/sync/mod.rs | 18 +++++ .../engine/controller/cluster_state/mod.rs | 2 +- .../src/engine/controller/initialization.rs | 3 +- .../src/engine/event_stream/partition/mod.rs | 2 +- flo-server/src/engine/mod.rs | 6 ++ 11 files changed, 154 insertions(+), 15 deletions(-) create mode 100644 flo-client-lib/src/async/ops/set_event_stream.rs diff --git a/flo-client-cli/src/client_cli/consumer.rs b/flo-client-cli/src/client_cli/consumer.rs index d656658..c7de392 100644 --- a/flo-client-cli/src/client_cli/consumer.rs +++ b/flo-client-cli/src/client_cli/consumer.rs @@ -6,6 +6,7 @@ use flo_client_lib::{Event, FloEventId, VersionVector}; use std::fmt::{self, Display}; pub struct CliConsumerOptions { + pub event_stream: Option, pub host: String, pub port: u16, pub namespace: String, @@ -23,13 +24,21 @@ impl FloCliCommand for CliConsumer { type Error = ConsumerError; fn run(input: Self::Input, output: &CliContext) -> Result<(), Self::Error> { - let CliConsumerOptions { host, port, namespace, limit, await, start_position, batch_size} = input; - + let CliConsumerOptions { host, port, event_stream, namespace, limit, await, start_position, batch_size} = input; let address = format!("{}:{}", host, port); output.verbose(format!("Connecting to: {}", &address)); - let connection = SyncConnection::connect_from_str(&address, "flo-client-cli", LossyStringCodec, batch_size)?; + let mut connection = SyncConnection::connect_from_str(&address, "flo-client-cli", LossyStringCodec, batch_size)?; + + if let Some(stream) = event_stream { + connection.set_event_stream(stream)?; + } + { + let status = connection.current_stream().unwrap(); + output.debug(format!("stream status: {:?}", status)); + output.normal(format!("Using event stream: '{}'", &status.name)); + } let mut version_vector = VersionVector::new(); if let Some(id) = start_position { diff --git a/flo-client-cli/src/client_cli/producer.rs b/flo-client-cli/src/client_cli/producer.rs index 1c23e4e..bcf42be 100644 --- a/flo-client-cli/src/client_cli/producer.rs +++ b/flo-client-cli/src/client_cli/producer.rs @@ -6,6 +6,7 @@ use super::{Context, FloCliCommand}; pub struct ProduceOptions { pub host: String, pub port: u16, + pub event_stream: Option, pub namespace: String, pub partition: ActorId, pub event_data: Vec>, @@ -19,13 +20,29 @@ impl FloCliCommand for Producer { type Input = ProduceOptions; type Error = String; - fn run(ProduceOptions{host, port, partition, namespace, event_data, parent_id}: ProduceOptions, output: &Context) -> Result<(), Self::Error> { + fn run(ProduceOptions{host, port, event_stream, partition, namespace, event_data, parent_id}: ProduceOptions, output: &Context) -> Result<(), Self::Error> { let server_address = format!("{}:{}", host, port); output.verbose(format!("Attempting connection to: {:?} to produce {} events", &server_address, event_data.len())); SyncConnection::connect_from_str(&server_address, "flo-client-cli", RawCodec, None).map_err(|handshake_err| { format!("Error establishing connection to flo server: {}", handshake_err) }).and_then(|mut connection| { output.debug(format!("connected to {}", &server_address)); + if let Some(stream) = event_stream { + connection.set_event_stream(stream).map_err(|err| { + format!("Error setting event stream: {:?}", err) + }).map(|_| { + connection + }) + } else { + Ok(connection) + } + + }).and_then(|mut connection| { + { + let status = connection.current_stream().unwrap(); + output.debug(format!("stream status: {:?}", status)); + output.normal(format!("Using event stream: '{}'", &status.name)); + } event_data.into_iter().fold(Ok(0), |events_produced, event_data| { events_produced.and_then(|count| { diff --git a/flo-client-cli/src/main.rs b/flo-client-cli/src/main.rs index 34209f5..8a53f91 100644 --- a/flo-client-cli/src/main.rs +++ b/flo-client-cli/src/main.rs @@ -23,6 +23,7 @@ mod args { pub const VERBOSE: &'static str = "verbose"; pub const HOST: &'static str = "server-host"; pub const PORT: &'static str = "server-port"; + pub const EVENT_STREAM: &'static str = "event-stream"; pub const NAMESPACE: &'static str = "namespace"; //produce options @@ -58,6 +59,11 @@ fn create_app_args() -> App<'static, 'static> { .long("port") .help("The port number of the flo server") .default_value("3000")) + .arg(Arg::with_name(args::EVENT_STREAM) + .short("e") + .long("event-stream") + .takes_value(true) + .help("The name of the event stream to interact with. Uses the default stream defined by the server if unspecified")) .subcommand(SubCommand::with_name(args::PRODUCE) .about("Used to produce events onto the stream") .arg(Arg::with_name(args::NAMESPACE) @@ -118,6 +124,7 @@ fn main() { let context = create_context(&args); let host = args.value_of(args::HOST).or_abort_process(&context).to_owned(); let port = args.value_of(args::PORT).or_abort_process(&context).parse::().or_abort_with_message("invalid port argument", &context); + let event_stream = args.value_of(args::EVENT_STREAM).map(|stream| stream.to_owned()); match args.subcommand() { (args::PRODUCE, Some(produce_args)) => { @@ -128,6 +135,7 @@ fn main() { let produce_options = ProduceOptions { host, port, + event_stream, namespace, partition, event_data, @@ -143,13 +151,14 @@ fn main() { let batch_size = parse_opt_or_exit::(args::CONSUME_BATCH, &consume_args, &context); let consume_opts = CliConsumerOptions { - host: host, - port: port, - namespace: namespace, - start_position: start_position, - limit: limit, - await: await, - batch_size: batch_size, + host, + port, + event_stream, + namespace, + start_position, + limit, + await, + batch_size, }; ::client_cli::run::(consume_opts, context); diff --git a/flo-client-lib/src/async/mod.rs b/flo-client-lib/src/async/mod.rs index 03300a0..a3b44bc 100644 --- a/flo-client-lib/src/async/mod.rs +++ b/flo-client-lib/src/async/mod.rs @@ -20,7 +20,7 @@ use event::{FloEventId, ActorId, VersionVector, OwnedFloEvent}; use codec::EventCodec; use self::recv::MessageRecvStream; use self::send::MessageSendSink; -use self::ops::{ProduceOne, ProduceAll, EventToProduce, Consume, Handshake}; +use self::ops::{ProduceOne, ProduceAll, EventToProduce, Consume, Handshake, SetEventStream}; pub use self::tcp_connect::{tcp_connect, tcp_connect_with, AsyncTcpClientConnect}; @@ -91,6 +91,14 @@ impl AsyncConnection { self.inner.current_stream.as_ref() } + /// Sets the event stream to use with this connection. If unset, then the default stream configured in the server will + /// be used. If the returned future completed successfully, then the connection will use the given event stream for all + /// operations from this point forward, and `current_steam` will return a `CurrentStreamState` corresponding to the given + /// stream. + pub fn set_event_stream>(self, new_stream: S) -> SetEventStream { + SetEventStream::new(self, new_stream.into()) + } + /// Produce a single event on the stream and await acknowledgement that it was persisted. Returns a future that resolves /// to a tuple of the `FloEventId` of the produced event and this `AsyncConnection`. pub fn produce(self, event: EventToProduce) -> ProduceOne { diff --git a/flo-client-lib/src/async/ops/mod.rs b/flo-client-lib/src/async/ops/mod.rs index d53ddf4..05c3d6d 100644 --- a/flo-client-lib/src/async/ops/mod.rs +++ b/flo-client-lib/src/async/ops/mod.rs @@ -4,6 +4,7 @@ mod await_response; mod consume; mod request_response; mod handshake; +mod set_event_stream; pub use self::send_message::{SendMessage, SendError}; pub use self::await_response::{AwaitResponse, AwaitResponseError}; @@ -11,3 +12,4 @@ pub use self::produce::{ProduceOne, ProduceErr, EventToProduce, ProduceAll, Prod pub use self::consume::{Consume, ConsumeError}; pub use self::request_response::{RequestResponse, RequestResponseError}; pub use self::handshake::{Handshake, HandshakeError}; +pub use self::set_event_stream::{SetEventStream, SetEventStreamError}; diff --git a/flo-client-lib/src/async/ops/set_event_stream.rs b/flo-client-lib/src/async/ops/set_event_stream.rs new file mode 100644 index 0000000..da2ce95 --- /dev/null +++ b/flo-client-lib/src/async/ops/set_event_stream.rs @@ -0,0 +1,69 @@ +use std::fmt::Debug; +use futures::{Future, Async, Poll}; + +use protocol::{self, ProtocolMessage}; +use async::{AsyncConnection, ErrorType, ClientProtocolMessage}; +use async::ops::{RequestResponse, RequestResponseError}; + +pub struct SetEventStream { + inner: RequestResponse, +} + +impl SetEventStream { + pub fn new(mut connection: AsyncConnection, stream_name: String) -> SetEventStream { + let op_id = connection.next_op_id(); + let to_send = ProtocolMessage::SetEventStream(protocol::SetEventStream { + op_id, + name: stream_name, + }); + SetEventStream { + inner: RequestResponse::new(connection, to_send) + } + } +} + +#[derive(Debug)] +pub struct SetEventStreamError { + pub error: ErrorType, + pub connection: AsyncConnection, +} + +impl From> for SetEventStreamError { + fn from(rr_err: RequestResponseError) -> Self { + let RequestResponseError {connection, error} = rr_err; + SetEventStreamError { + connection, + error: error.into() + } + } +} + +impl Future for SetEventStream { + type Item = AsyncConnection; + type Error = SetEventStreamError; + + fn poll(&mut self) -> Poll { + let (message, mut connection) = try_ready!(self.inner.poll()); + match message { + ProtocolMessage::StreamStatus(status) => { + let our_status = status.into(); + connection.inner.current_stream = Some(our_status); + Ok(Async::Ready(connection)) + } + ProtocolMessage::Error(err_message) => { + let err = SetEventStreamError { + connection, + error: err_message.into(), + }; + Err(err) + } + other @ _ => { + let err = SetEventStreamError { + connection, + error: ErrorType::unexpected_message("EventStreamStatus", other) + }; + Err(err) + } + } + } +} diff --git a/flo-client-lib/src/sync/mod.rs b/flo-client-lib/src/sync/mod.rs index b9ac534..20c5359 100644 --- a/flo-client-lib/src/sync/mod.rs +++ b/flo-client-lib/src/sync/mod.rs @@ -157,6 +157,24 @@ impl SyncConnection { self.async_connection.as_ref().and_then(|conn| conn.current_stream()) } + /// Sets the event stream to use for all operations from this point forward on this connection. + pub fn set_event_stream>(&mut self, event_stream: S) -> Result<(), ErrorType> { + use async::ops::SetEventStreamError; + + let conn = self.async_connection.take().unwrap(); + let result = run_future(conn.set_event_stream(event_stream)); + match result { + Ok(conn) => { + self.async_connection = Some(conn); + Ok(()) + } + Err(SetEventStreamError {connection, error}) => { + self.async_connection = Some(connection); + Err(error) + } + } + } + } /// An iterator of events from an event stream. Each element in the iterator is a `Result, ErrorType>`. If an error is diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 82aac60..2febe3f 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -234,7 +234,7 @@ impl ClusterManager { self.primary_status_writer.set(false); self.primary_state = None; self.set_new_primary(primary); - self.state = State::Follower; + self.transition_state(State::Follower); } fn append_system_events(&mut self, events: &[OwnedFloEvent]) -> io::Result { diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index 576f7d0..f65818f 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -183,7 +183,8 @@ fn init_user_streams(storage_dir: &Path, options: &EventStreamOptions, remote: & } } - if !user_streams.contains_key(&options.name) { + if !user_streams.contains_key(&options.name) && SYSTEM_STREAM_NAME != options.name.as_str() { + info!("Initializing new default event stream: '{}'", options.name); let new_stream_dir = storage_dir.join(&options.name); let new_stream = init_new_event_stream( new_stream_dir, diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index e2bf87a..90e130f 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -349,7 +349,7 @@ pub fn run_partition(partition_impl: PartitionImpl, primary_server_addr: Arc Result { + if stream_name == SYSTEM_STREAM_NAME { + // user wants to interact with the system stream, so we'll convert it to a "normal" stream ref + return Ok(self.system_stream.to_event_stream()); + } + + // go for a user stream let streams = self.event_streams.lock().unwrap(); if let Some(stream) = streams.get(stream_name).map(|s| s.clone()) { Ok(stream) From 3bdf125875dd543b4c3017744fc75ba48bd77bb8 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 21 Jan 2018 14:26:45 -0500 Subject: [PATCH 53/73] change default stream name to 'user' since the system stream is now locked down --- flo-server/src/server/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flo-server/src/server/mod.rs b/flo-server/src/server/mod.rs index 54baaba..35f356a 100644 --- a/flo-server/src/server/mod.rs +++ b/flo-server/src/server/mod.rs @@ -44,7 +44,7 @@ pub fn run(mut options: ServerOptions) -> io::Result<()> { let controller_options = ControllerOptions { storage_dir: options.data_dir.clone(), default_stream_options: EventStreamOptions{ - name: system_stream_name(), + name: "user".to_owned(), num_partitions: 1, event_retention: options.event_retention_duration, max_segment_duration: options.event_eviction_period, From 80c2e75a330da0c8afa9ba21e31d31dd55f2cc7c Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 11 Mar 2018 20:42:23 -0400 Subject: [PATCH 54/73] backup commit, wip on replication operations --- flo-server/src/engine/controller/mod.rs | 3 ++ .../event_stream/partition/controller/mod.rs | 53 ++++++++++++++++++- .../src/engine/event_stream/partition/mod.rs | 5 +- .../src/engine/event_stream/partition/ops.rs | 45 +++++++++++++++- 4 files changed, 101 insertions(+), 5 deletions(-) diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index abc4dee..58ed415 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -138,6 +138,9 @@ fn handle_partition_op(connection_id: ConnectionId, op_start_time: Instant, op: StopConsumer => { controller_state.system_partition.stop_consumer(connection_id); } + Replicate(_) => { + error!("Received normal Replicate operation from connection_id: {} in system partition", connection_id); + } Tick => { // only used by the partition to expire old events, which we do not do for the system partition. // anyway, nothing is creating these operations anyway diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index 2a81676..414dcba 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -12,7 +12,8 @@ use chrono::{Duration}; use atomics::{AtomicCounterWriter, AtomicCounterReader, AtomicBoolReader}; use protocol::{ProduceEvent, FloInstanceId}; use event::{ActorId, FloEventId, EventCounter, FloEvent, Timestamp, time}; -use super::{SharedReaderRefs, SharedReaderRefsMut, Operation, OpType, ProduceOperation, ConsumeOperation, PartitionReader, EventFilter, SegmentNum}; +use super::{SharedReaderRefs, SharedReaderRefsMut, Operation, OpType, ProduceOperation, ConsumeOperation, PartitionReader, + EventFilter, SegmentNum, ReplicateOperation, ReplicationResult}; use super::segment::Segment; use super::index::{PartitionIndex, IndexEntry}; use engine::event_stream::{EventStreamOptions, HighestCounter}; @@ -199,6 +200,10 @@ impl PartitionImpl { self.stop_consumer(connection_id); Ok(()) } + OpType::Replicate(rep) => { + self.handle_replicate(rep) + Ok(()) + } OpType::Tick => { self.expire_old_events(); Ok(()) @@ -216,6 +221,50 @@ impl PartitionImpl { } } + // returns unit, since we have to send a result back to the connection handler, regardless of outcome + fn handle_replicate(&mut self, ReplicateOperation{op_id, client_sender, events, prev_event_counter, prev_event_term}: ReplicateOperation) { + // TODO: think about handling empty `events` vec. maybe best to handle that in the connection handler? + // TODO: ensure that we are in Follower status and that the events came from the leader + let result = self.replicate_events(op_id, events); + let response = result.unwrap_or_else(|io_err| { + let head = self.index.greatest_event_counter(); + error!("Error handling replicate operation with op_id: {}, io error: '{}', sending fail response with head: {}", op_id, io_err, head); + ReplicationResult { + op_id, + success: false, + highest_event_counter: head, + } + }); + client_sender.complete(response); + } + + /// persists events in this partition. Panics if `events` is empty + fn replicate_events(&mut self, op_id: u32, events: Vec) -> io::Result { + let commit_index = self.commit_manager.get_commit_index(); + let current_head = self.index.greatest_event_counter(); + + // ignore any events with id < commit_index + let first_repl_event_index = events.iter().enumerate().find(|e| e.id().event_counter > commit_index); + if first_repl_event_index.is_none() { + // All the events in this message have already been committed, so just return our current commit index + return Ok(ReplicationResult { + op_id: op_id, + success: true, // success = true because the log is consistent. We just happen to have all these entries already + highest_event_counter: commit_index, + }); + } + let mut first_event_to_append = first_repl_event_index.unwrap(); + + // now we may still skip events that are uncommitted if the event id and crc matches what we already have in the log + + // if any uncommitted events have a different crc, then mark that event and all the events that come after it as deleted + // at this point, we should be in a state where the last non-deleted event in the log is the same as the previous event info in the message + + // append entries starting at `first_event_to_append` + + unimplemented!() + } + fn drop_segments_through_index(&mut self, segment_index: usize) { info!("Dropping first {} segment(s)", segment_index + 1); let PartitionImpl { ref mut segments, ref mut index, ref mut reader_refs, .. } = *self; @@ -276,7 +325,7 @@ impl PartitionImpl { Ok(FloEventId::new(self.partition_num, event_counter)) } - fn append(&mut self, event: &EventToProduce) -> io::Result<()> { + fn append(&mut self, event: &E) -> io::Result<()> { use super::SegmentNum; use super::segment::AppendResult; diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index 90e130f..d52dc2d 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -31,7 +31,10 @@ pub use self::ops::{OpType, ConsumeResponseReceiver, ConsumeResponder, ConsumerNotifier, -}; + ReplicateOperation, + ReplicateResultReceiver, + ReplicateResultSender, + ReplicationResult}; pub use self::event_reader::{PartitionReader, EventFilter}; pub use self::segment::PersistentEvent; pub use self::index::IndexEntry; diff --git a/flo-server/src/engine/event_stream/partition/ops.rs b/flo-server/src/engine/event_stream/partition/ops.rs index 713bf8a..f31efe9 100644 --- a/flo-server/src/engine/event_stream/partition/ops.rs +++ b/flo-server/src/engine/event_stream/partition/ops.rs @@ -8,8 +8,8 @@ use futures::sync::oneshot; use engine::event_stream::partition::{EventFilter, PartitionReader}; use engine::ConnectionId; -use protocol::ProduceEvent; -use event::{FloEventId, EventCounter}; +use protocol::{ProduceEvent, Term}; +use event::{OwnedFloEvent, FloEventId, EventCounter}; pub type ProduceResult = Result; pub type ProduceResponder = oneshot::Sender; @@ -68,10 +68,35 @@ impl Debug for ConsumeOperation { } } +#[derive(Debug, PartialEq, Clone)] +pub struct ReplicationResult { + pub op_id: u32, + pub success: bool, + pub highest_event_counter: EventCounter +} +pub type ReplicateResultSender = oneshot::Sender; +pub type ReplicateResultReceiver = oneshot::Receiver; + +#[derive(Debug)] +pub struct ReplicateOperation { + pub client_sender: ReplicateResultSender, + pub op_id: u32, + pub prev_event_counter: EventCounter, + pub prev_event_term: Term, + pub events: Vec, +} + +impl PartialEq for ReplicateOperation { + fn eq(&self, other: &ReplicateOperation) -> bool { + self.events == other.events + } +} + #[derive(Debug, PartialEq)] pub enum OpType { Produce(ProduceOperation), Consume(ConsumeOperation), + Replicate(ReplicateOperation), StopConsumer, Tick, } @@ -120,6 +145,22 @@ impl Operation { (op, rx) } + pub fn replicate(connection_id: ConnectionId, op_id: u32, prev_event_counter: EventCounter, prev_event_term: Term, events: Vec) -> (Operation, ReplicateResultReceiver) { + let (tx, rx) = oneshot::channel(); + let rep = ReplicateOperation { + client_sender: tx, + prev_event_counter, + prev_event_term, + events, + }; + let op = Operation { + connection_id, + client_message_recv_time: Instant::now(), + op_type: OpType::Replicate(rep), + }; + (op, rx) + } + pub fn tick() -> Operation { Operation::new(0, OpType::Tick) } From 0dc9b020b1256c1b2dc158ed1339ad0bbbdd7aa4 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 29 Apr 2018 16:04:19 -0400 Subject: [PATCH 55/73] add crc function to flo-event --- Cargo.lock | 806 +++++++++++++-------- flo-client-cli/Cargo.lock | 219 ------ flo-event/Cargo.lock | 100 --- flo-event/Cargo.toml | 4 +- flo-event/src/id.rs | 133 ++++ flo-event/src/lib.rs | 213 ++---- flo-event/src/time.rs | 16 +- flo-protocol/Cargo.toml | 2 +- flo-protocol/src/messages/produce_event.rs | 13 +- flo-server/Cargo.toml | 2 +- 10 files changed, 736 insertions(+), 772 deletions(-) delete mode 100644 flo-client-cli/Cargo.lock delete mode 100644 flo-event/Cargo.lock create mode 100644 flo-event/src/id.rs diff --git a/Cargo.lock b/Cargo.lock index 74c314c..0433a94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5,10 +5,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "aho-corasick" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -18,8 +18,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ansi_term" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "antidote" @@ -28,11 +31,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "arrayvec" -version = "0.3.24" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "odds 0.2.26 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "arrayvec" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "odds 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -42,13 +53,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "atty" -version = "0.2.3" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -58,12 +68,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "bitflags" -version = "0.9.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "bitflags" -version = "1.0.1" +name = "build_const" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -73,21 +83,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "byteorder" -version = "1.1.0" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "bytes" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "cc" -version = "1.0.4" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -100,17 +110,18 @@ name = "chrono" version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", + "num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "chrono" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -120,13 +131,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "clap" -version = "2.27.1" +version = "2.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -134,10 +145,18 @@ dependencies = [ [[package]] name = "clocksource" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -145,13 +164,51 @@ name = "crossbeam" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "crossbeam-deque" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-epoch 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "deflate" -version = "0.7.17" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "adler32 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -218,11 +275,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "env_logger" -version = "0.4.3" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -230,7 +290,7 @@ name = "flate2" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", "miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -238,7 +298,7 @@ dependencies = [ name = "flo-bench-cli" version = "0.2.0" dependencies = [ - "clap 2.27.1 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", "flo-client-lib 0.2.0", "tic 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -247,7 +307,7 @@ dependencies = [ name = "flo-client-cli" version = "0.2.0" dependencies = [ - "clap 2.27.1 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", "flo-client-lib 0.2.0", ] @@ -255,63 +315,65 @@ dependencies = [ name = "flo-client-lib" version = "0.2.0" dependencies = [ - "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", "flo-event 0.2.0", "flo-protocol 0.2.0", - "futures 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "flo-event" version = "0.2.0" dependencies = [ - "chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "flo-protocol" version = "0.2.0" dependencies = [ - "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "flo-event 0.2.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "flo-server" version = "0.2.0" dependencies = [ - "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.27.1 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", "flo-client-lib 0.2.0", "flo-event 0.2.0", "flo-protocol 0.2.0", - "futures 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "log4rs 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "memmap 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rmp 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)", "rmp-serde 0.13.7 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -321,52 +383,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "fs2" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "fuchsia-zircon" -version = "0.2.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "fuchsia-zircon-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "fuchsia-zircon" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-zircon-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.2.0" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "fuchsia-zircon-sys" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "futures" -version = "0.1.17" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "getopts" -version = "0.2.15" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -380,7 +425,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "histogram 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -395,7 +440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "humantime" -version = "1.0.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -418,10 +463,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "iovec" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -430,6 +475,11 @@ name = "itoa" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "itoa" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -441,28 +491,31 @@ dependencies = [ [[package]] name = "lazy_static" -version = "0.2.10" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "lazycell" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libc" -version = "0.2.33" +version = "0.2.40" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "linked-hash-map" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "log" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "log" @@ -470,7 +523,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -484,21 +537,21 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "humantime 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "log-mdc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", - "serde-value 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", + "serde-value 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)", "serde_yaml 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -508,10 +561,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "memchr" -version = "1.0.2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -519,36 +572,41 @@ name = "memmap" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "fs2 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "memoffset" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "miniz-sys" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "mio" -version = "0.6.11" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "fuchsia-zircon 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-zircon-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazycell 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", - "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -558,7 +616,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -570,14 +628,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "net2" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -592,47 +648,55 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "num" -version = "0.1.40" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", - "num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", + "num-iter 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-integer" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-iter" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-traits" -version = "0.1.40" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-traits" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "num_cpus" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "odds" -version = "0.2.25" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -640,7 +704,7 @@ name = "ordered-float" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", "unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -655,9 +719,17 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "deflate 0.7.17 (registry+https://github.com/rust-lang/crates.io-index)", + "deflate 0.7.18 (registry+https://github.com/rust-lang/crates.io-index)", "inflate 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", + "num-iter 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro2" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -667,30 +739,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "quote" -version = "0.3.15" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "rand" -version = "0.3.18" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "fuchsia-zircon 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rand" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "fuchsia-zircon 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "redox_syscall" -version = "0.1.31" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -698,33 +775,44 @@ name = "redox_termios" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "regex" -version = "0.2.2" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "regex-syntax" -version = "0.4.1" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "remove_dir_all" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "rmp" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -732,9 +820,9 @@ name = "rmp-serde" version = "0.13.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "rmp 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -747,13 +835,18 @@ name = "rusttype" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "arrayvec 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "arrayvec 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)", "stb_truetype 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "scoped-tls" -version = "0.1.0" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "scopeguard" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -763,35 +856,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde" -version = "1.0.24" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde-value" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ordered-float 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_derive" -version = "1.0.24" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive_internals 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive_internals 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.13.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_derive_internals" -version = "0.18.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", - "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.13.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -801,19 +895,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_json" -version = "1.0.8" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -821,17 +914,12 @@ name = "serde_yaml" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "linked-hash-map 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", "yaml-rust 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "slab" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "slab" version = "0.4.0" @@ -847,33 +935,34 @@ dependencies = [ [[package]] name = "strsim" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "syn" -version = "0.11.11" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "synom" -version = "0.11.3" +name = "tempdir" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "tempdir" -version = "0.3.5" +name = "termcolor" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rand 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -881,8 +970,8 @@ name = "termion" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -896,10 +985,10 @@ dependencies = [ [[package]] name = "thread_local" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -909,67 +998,153 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "allan 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", - "clocksource 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "clocksource 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "getopts 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)", + "getopts 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", "heatmap 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "histogram 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "mpmc 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", - "tiny_http 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny_http 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", "waterfall 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "time" -version = "0.1.38" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tiny_http" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ascii 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)", "chunked_transfer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "url 0.2.38 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tokio" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-tcp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-threadpool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-udp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tokio-core" -version = "0.1.10" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "scoped-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-executor" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-io" -version = "0.1.4" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-timer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-udp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -985,6 +1160,11 @@ dependencies = [ "unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ucd-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicode-bidi" version = "0.3.4" @@ -1005,7 +1185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "unicode-xid" -version = "0.0.4" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1044,7 +1224,7 @@ dependencies = [ [[package]] name = "url" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1062,7 +1242,7 @@ name = "uuid" version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rand 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1094,11 +1274,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "winapi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "winapi-i686-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-x86_64-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1108,14 +1288,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "winapi-i686-pc-windows-gnu" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "winapi-x86_64-pc-windows-gnu" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "wincolor" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ws2_32-sys" version = "0.2.1" @@ -1130,33 +1318,39 @@ name = "yaml-rust" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "linked-hash-map 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [metadata] "checksum adler32 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6cbd0b9af8587c72beadc9f72d35b9fbb070982c9e6203e46e93f10df25f8f45" -"checksum aho-corasick 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "500909c4f87a9e52355b26626d890833e9e1d53ac566db76c36faa984b889699" +"checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" "checksum allan 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "62ed9db31078b3c9e56ce77857fa21f6bdb062988c24a5c989c3f44fa1317b47" -"checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" +"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5" -"checksum arrayvec 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "e003cbf6e0e1c43a0fc8df2ea8ea24174514d35cbcf60c35ca6112e0139f65e2" +"checksum arrayvec 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)" = "06f59fe10306bb78facd90d28c2038ad23ffaaefa85bac43c8a434cde383334f" +"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" "checksum ascii 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae7d751998c189c1d4468cf0a39bb2eae052a9c58d50ebb3b9591ee3813ad50" -"checksum atty 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "21e50800ec991574876040fff8ee46b136a53e985286fbe6a3bdfe6421b78860" +"checksum atty 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6609a866dd1a1b2d0ee1362195bf3e4f6438abb2d80120b83b1e1f4fb6476dd0" "checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" -"checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" -"checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf" +"checksum bitflags 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1b2bf7093258c32e0825b635948de528a5949799dcd61bef39534c8aab95870c" +"checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" "checksum byteorder 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "96c8b41881888cc08af32d47ac4edd52bc7fa27fef774be47a92443756451304" -"checksum byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff81738b726f5d099632ceaffe7fb65b90212e8dce59d518729e7e8634032d3d" -"checksum bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d828f97b58cc5de3e40c421d0cf2132d6b2da4ee0e11b8632fa838f0f9333ad6" -"checksum cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "deaf9ec656256bb25b404c51ef50097207b9cbb29c933d31f92cae5a8a0ffee0" +"checksum byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "73b5bdfe7ee3ad0b99c9801d58807a9dbc9e09196365b0203853b99889ab3c87" +"checksum bytes 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "2f1d50c876fb7545f5f289cd8b2aee3f359d073ae819eed5d6373638e2c61e59" +"checksum cc 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "38fb45eeb2c9216a6700cf675b418d6c26ee15b55a3700970112da9fedfb8694" "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" "checksum chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)" = "9213f7cd7c27e95c2b57c49f0e69b1ea65b27138da84a170133fd21b07659c00" -"checksum chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c20ebe0b2b08b0aeddba49c609fe7957ba2e33449882cb186a180bc60682fa9" +"checksum chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1cce36c92cb605414e9b824f866f5babe0a0368e39ea07393b9b63cf3844c0e6" "checksum chunked_transfer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "498d20a7aaf62625b9bf26e637cf7736417cde1d0c99f1d04d1170229a85cf87" -"checksum clap 2.27.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1b8c532887f1a292d17de05ae858a8fe50a301e196f9ef0ddb7ccd0d1d00f180" -"checksum clocksource 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3a858a8b189b5ab63739bce11781a2dfe68678c362246a0438dcad5002f4e66a" +"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" +"checksum clocksource 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ffcfe846ac61510fe67bf890f503a883f4b0e141aee37d6904a4e53401110a11" +"checksum crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" "checksum crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "24ce9782d4d5c53674646a6a4c1863a21a8fc0cb649b3c94dfc16e45071dea19" -"checksum deflate 0.7.17 (registry+https://github.com/rust-lang/crates.io-index)" = "4dddda59aaab719767ab11d3efd9a714e95b610c4445d4435765021e9d52dfb1" +"checksum crossbeam-deque 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c1bdc73742c36f7f35ebcda81dbb33a7e0d33757d03a06d9ddca762712ec5ea2" +"checksum crossbeam-epoch 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9b4e2817eb773f770dcb294127c011e22771899c21d18fce7dd739c0b9832e81" +"checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" +"checksum crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d636a8b3bcc1b409d7ffd3facef8f21dcb4009626adbd0c5e6c4305c07253c7b" +"checksum deflate 0.7.18 (registry+https://github.com/rust-lang/crates.io-index)" = "32c8120d981901a9970a3a1c97cf8b630e0fa8c3ca31e75b6fd6fd5f9f427b31" "checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab" "checksum encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" "checksum encoding-index-japanese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91" @@ -1165,109 +1359,121 @@ dependencies = [ "checksum encoding-index-singlebyte 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" "checksum encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" "checksum encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" -"checksum env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b" +"checksum env_logger 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)" = "00c45cec4cde3daac5f036c74098b4956151525cdf360cff5ee0092c98823e54" "checksum flate2 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9fac2277e84e5e858483756647a9d0aa8d9a2b7cba517fd84325a0aaa69a0909" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" -"checksum fs2 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ab76cfd2aaa59b7bf6688ad9ba15bbae64bff97f04ea02144cfd3443e5c2866" -"checksum fuchsia-zircon 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f6c0581a4e363262e52b87f59ee2afe3415361c6ec35e665924eb08afe8ff159" -"checksum fuchsia-zircon 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bd510087c325af53ba24f3be8f1c081b0982319adcb8b03cad764512923ccc19" -"checksum fuchsia-zircon-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "43f3795b4bae048dc6123a6b972cadde2e676f9ded08aef6bb77f5f157684a82" -"checksum fuchsia-zircon-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "08b3a6f13ad6b96572b53ce7af74543132f1a7055ccceb6d073dd36c54481859" -"checksum futures 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "118b49cac82e04121117cbd3121ede3147e885627d82c4546b87c702debb90c1" -"checksum getopts 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)" = "65922871abd2f101a2eb0eaebadc66668e54a87ad9c3dd82520b5f86ede5eff9" +"checksum fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "1a70b146671de62ec8c8ed572219ca5d594d9b06c0b364d5e67b722fc559b48c" +"checksum getopts 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "b900c08c1939860ce8b54dc6a89e26e00c04c380fd0e09796799bd7f12861e05" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" "checksum heatmap 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4c9551a9016b91c9b81fbc093e5ad0dd11c80ff4082fd2266170a210c2890051" "checksum histogram 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1bdcec4094c1ca961b685384ea7af76af5718230b3f34657d1a71fd2dcf4cc9d" "checksum hsl 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "575fb7f1167f3b88ed825e90eb14918ac460461fdeaa3965c6a50951dee1c970" -"checksum humantime 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0e9298fffb2a54569e1fcb818e9c2ff77caa2fad68d64b6e409b9f777bdb1960" +"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" "checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d" "checksum inflate 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d1238524675af3938a7c74980899535854b88ba07907bb1c944abe5b8fc437e5" -"checksum iovec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b6e8b9c2247fcf6c6a1151f1156932be5606c9fd6f55a2d7f9fc1cb29386b2f7" +"checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" "checksum itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c" +"checksum itoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c069bbec61e1ca5a596166e55dfe4773ff745c3d16b700013bcaff9a6df2c682" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -"checksum lazy_static 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "236eb37a62591d4a41a89b7763d7de3e06ca02d5ab2815446a8bae5d2f8c2d57" -"checksum lazycell 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3b585b7a6811fb03aa10e74b278a0f00f8dd9b45dc681f148bb29fa5cb61859b" -"checksum libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "5ba3df4dcb460b9dfbd070d41c94c19209620c191b0340b929ce748a2bcd42d2" -"checksum linked-hash-map 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2d2aab0478615bb586559b0114d94dd8eca4fdbb73b443adcb0d00b61692b4bf" -"checksum log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "880f77541efa6e5cc74e76910c9884d9859683118839d6a1dc3b11e63512565b" +"checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d" +"checksum lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a6f08839bc70ef4a3fe1d566d5350f519c5912ea86be0df1740a7d247c7fc0ef" +"checksum libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "6fd41f331ac7c5b8ac259b8bf82c75c0fb2e469bbf37d2becbba9a6a2221965b" +"checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e" +"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" "checksum log-mdc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a94d21414c1f4a51209ad204c1776a3d0765002c76c6abcb602a6f09f1e881c7" "checksum log4rs 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a1f16090a553200fba94e104310b3e53e71f500fd9db7dc2143055aa3cc7ae63" "checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376" -"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" +"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" "checksum memmap 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "46f3c7359028b31999287dae4e5047ddfe90a23b7dca2282ce759b491080c99b" +"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" "checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4" -"checksum mio 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0e8411968194c7b139e9105bc4ae7db0bae232af087147e72f0616ebf5fdb9cb" +"checksum mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)" = "6d771e3ef92d58a8da8df7d6976bfca9371ed1de6619d9d5a5ce5b1f29b85bfe" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum mpmc 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "cb947c698d784291c6b1d97269b0615beb966178537d4502ce90970507e1cf3b" -"checksum net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)" = "3a80f842784ef6c9a958b68b7516bc7e35883c614004dd94959a4dca1b716c09" +"checksum net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)" = "9044faf1413a1057267be51b5afba8eb1090bd2231c693664aa1db716fe1eae0" "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" "checksum nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff" -"checksum num 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "a311b77ebdc5dd4cf6449d81e4135d9f0e3b153839ac90e648a8ef538f923525" -"checksum num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "d1452e8b06e448a07f0e6ebb0bb1d92b8890eea63288c0b627331d53514d0fba" -"checksum num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)" = "7485fcc84f85b4ecd0ea527b14189281cf27d60e583ae65ebc9c088b13dffe01" -"checksum num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "99843c856d68d8b4313b03a17e33c4bb42ae8f6610ea81b28abe076ac721b9b0" -"checksum num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "514f0d73e64be53ff320680ca671b64fe3fb91da01e1ae2ddc99eb51d453b20d" -"checksum odds 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)" = "c3df9b730298cea3a1c3faa90b7e2f9df3a9c400d0936d6015e6165734eefcba" +"checksum num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" +"checksum num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f8d26da319fb45674985c78f1d1caf99aa4941f785d384a2ae36d0740bc3e2fe" +"checksum num-iter 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "4b226df12c5a59b63569dd57fafb926d91b385dfce33d8074a412411b689d593" +"checksum num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +"checksum num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dee092fcdf725aee04dd7da1d21debff559237d49ef1cb3e69bcb8ece44c7364" +"checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30" +"checksum odds 0.2.26 (registry+https://github.com/rust-lang/crates.io-index)" = "4eae0151b9dacf24fcc170d9995e511669a082856a91f958a2fe380bfab3fb22" "checksum ordered-float 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "58d25b6c0e47b20d05226d288ff434940296e7e2f8b877975da32f862152241f" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" "checksum png 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "48f397b84083c2753ba53c7b56ad023edb94512b2885ffe227c66ff7edb61868" +"checksum proc-macro2 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "b16749538926f394755373f0dfec0852d79b3bd512a5906ceaeb72ee64a4eaa0" "checksum quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eda5fe9b71976e62bc81b781206aaa076401769b2143379d3eb2118388babac4" -"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" -"checksum rand 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)" = "6475140dfd8655aeb72e1fd4b7a1cc1c202be65d71669476e392fe62532b9edd" -"checksum rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9d5f78082e6a6d042862611e9640cf20776185fee506cf6cf67e93c6225cee31" -"checksum redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "8dde11f18c108289bef24469638a04dce49da56084f2d50618b226e47eb04509" +"checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" +"checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" +"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" +"checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" -"checksum regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1731164734096285ec2a5ec7fea5248ae2f5485b3feeb0115af4fda2183b2d1b" -"checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db" +"checksum regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "aec3f58d903a7d2a9dc2bf0e41a746f4530e0cab6b615494e058f67a3ef947fb" +"checksum regex-syntax 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "bd90079345f4a4c3409214734ae220fd773c6f2e8a543d07370c6c1c369cfbfb" +"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum rmp 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a3d45d7afc9b132b34a2479648863aa95c5c88e98b32285326a6ebadc80ec5c9" "checksum rmp-serde 0.13.7 (registry+https://github.com/rust-lang/crates.io-index)" = "011e1d58446e9fa3af7cdc1fb91295b10621d3ac4cb3a85cc86385ee9ca50cd3" "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum rusttype 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "07b8848db3b5b5ba97020c6a756c0fdf2dbf2ad7c0d06aa4344a3f2f49c3fe17" -"checksum scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f417c22df063e9450888a7561788e9bd46d3bb3c1466435b4eccb903807f147d" +"checksum scoped-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8674d439c964889e2476f474a3bf198cc9e199e77499960893bac5de7e9218a4" +"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)" = "34b623917345a631dc9608d5194cc206b3fe6c3554cd1c75b937e55e285254af" -"checksum serde 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)" = "1c57ab4ec5fa85d08aaf8ed9245899d9bbdd66768945b21113b84d5f595cb6a1" -"checksum serde-value 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "71187cf90819445c78d64f749d16499ba210d7724f16a95754dd03e0f207356d" -"checksum serde_derive 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)" = "02c92ea07b6e49b959c1481804ebc9bfd92d3c459f1274c9a9546829e42a66ce" -"checksum serde_derive_internals 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75c6aac7b99801a16db5b40b7bf0d7e4ba16e76fbf231e32a4677f271cac0603" +"checksum serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)" = "0c855d888276f20d140223bd06515e5bf1647fd6d02593cb5792466d9a8ec2d0" +"checksum serde-value 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "52903ade2290cbd61a0937a66a268f26cebf246e3ddd7964a8babb297111fb0d" +"checksum serde_derive 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)" = "aa113e5fc4b008a626ba2bbd41330b56c9987d667f79f7b243e5a2d03d91ed1c" +"checksum serde_derive_internals 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9d30c4596450fd7bbda79ef15559683f9a79ac0193ea819db90000d7e1cae794" "checksum serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ad8bcf487be7d2e15d3d543f04312de991d631cfe1b43ea0ade69e6a8a5b16a1" -"checksum serde_json 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7cf5b0b5b4bd22eeecb7e01ac2e1225c7ef5e4272b79ee28a8392a8c8489c839" +"checksum serde_json 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)" = "8c6c4e049dc657a99e394bd85c22acbf97356feeec6dbf44150f2dcf79fb3118" "checksum serde_yaml 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e0f868d400d9d13d00988da49f7f02aeac6ef00f11901a8c535bd59d777b9e19" -"checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" "checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d" "checksum stb_truetype 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fcf3270840fc9de208d63e836eb3fdebb85379e7532f42f1b2cbd505fb6fda08" -"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" -"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" -"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" -"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" +"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" +"checksum syn 0.13.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1f9bcba3e27a55cf7a1e46acc1568ccf3d92be8e517b8593ff83cb8183cf5cd6" +"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" -"checksum thread_local 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1697c4b57aeeb7a536b647165a2825faddffb1d3bad386d507709bd51a90bb14" +"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" "checksum tic 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b430518516916da193a0e291754fa906101428802c04ac6fdf92ff81d8b01a7e" -"checksum time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)" = "d5d788d3aa77bc0ef3e9621256885555368b47bd495c13dd2e7413c89f845520" -"checksum tiny_http 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)" = "016f040cfc9b5be610de3619eaaa57017fa0b0b678187327bde75fc146e2a41f" -"checksum tokio-core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "c843a027f7c1df5f81e7734a0df3f67bf329411781ebf36393ce67beef6071e3" -"checksum tokio-io 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "514aae203178929dbf03318ad7c683126672d4d96eccb77b29603d33c9e25743" +"checksum time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "a15375f1df02096fb3317256ce2cee6a1f42fc84ea5ad5fc8c421cfe40c73098" +"checksum tiny_http 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)" = "2f4d55c9a213880d1f0c89ded183f209c6e45b912ca6c7df6f93c163773572e1" +"checksum tokio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "be15ef40f675c9fe66e354d74c73f3ed012ca1aa14d65846a33ee48f1ae8d922" +"checksum tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "aeeffbbb94209023feaef3c196a41cbcdafa06b4a6f893f68779bb5e53796f71" +"checksum tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8cac2a7883ff3567e9d66bb09100d09b33d90311feca0206c7ca034bc0c55113" +"checksum tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "6af9eb326f64b2d6b68438e1953341e00ab3cf54de7e35d92bfc73af8555313a" +"checksum tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3cedc8e5af5131dc3423ffa4f877cce78ad25259a9a62de0613735a13ebc64b" +"checksum tokio-tcp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ec9b094851aadd2caf83ba3ad8e8c4ce65a42104f7b94d9e6550023f0407853f" +"checksum tokio-threadpool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3d05cdd6a78005e535d2b27c21521bdf91fbb321027a62d8e178929d18966d" +"checksum tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "29a89e4ad0c8f1e4c9860e605c38c69bfdad3cccd4ea446e58ff588c1c07a397" +"checksum tokio-udp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "137bda266504893ac4774e0ec4c2108f7ccdbcb7ac8dced6305fe9e4e0b5041a" "checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" "checksum typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "653be63c80a3296da5551e1bfd2cca35227e13cdd08c6668903ae2f4f77aa1f6" +"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" "checksum unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "51ccda9ef9efa3f7ef5d91e8f9b83bbe6955f9bf86aec89d5cce2c874625920f" "checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" -"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f" "checksum url 0.2.38 (registry+https://github.com/rust-lang/crates.io-index)" = "cbaa8377a162d88e7d15db0cf110c8523453edcbc5bc66d2b6fffccffa34a068" -"checksum url 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fa35e768d4daf1d85733418a49fb42e10d7f633e394fccab4ab7aba897053fe2" +"checksum url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f808aadd8cfec6ef90e4a14eb46f24511824d1ac596b9682703c87056c8678b7" "checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" "checksum uuid 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "78c590b5bd79ed10aad8fb75f078a59d8db445af6c743e55c4a53227fc01c13f" "checksum vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "887b5b631c2ad01628bbbaa7dd4c869f80d3186688f8d0b6f58774fbe324988c" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum waterfall 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfd2a19feb20d152820c6d01acfda726c305fa7ea67f685359d24f4d6040729" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" -"checksum winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b09fb3b6f248ea4cd42c9a65113a847d612e17505d6ebd1f7357ad68a8bf8693" +"checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" -"checksum winapi-i686-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ec6667f60c23eca65c561e63a13d81b44234c2e38a6b6c959025ee907ec614cc" -"checksum winapi-x86_64-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98f12c52b2630cd05d2c3ffd8e008f7f48252c042b4871c72aed9dc733b96668" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" "checksum yaml-rust 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57ab38ee1a4a266ed033496cf9af1828d8d6e6c1cfa5f643a2809effcae4d628" diff --git a/flo-client-cli/Cargo.lock b/flo-client-cli/Cargo.lock deleted file mode 100644 index f9b268d..0000000 --- a/flo-client-cli/Cargo.lock +++ /dev/null @@ -1,219 +0,0 @@ -[root] -name = "flo-client-cli" -version = "0.1.0" -dependencies = [ - "clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)", - "flo-client-lib 0.1.0", -] - -[[package]] -name = "ansi_term" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "atty" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bitflags" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "byteorder" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "chrono" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "clap" -version = "2.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "term_size 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-segmentation 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "vec_map 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "flo-client-lib" -version = "0.1.0" -dependencies = [ - "flo-event 0.1.0", - "flo-protocol 0.1.0", - "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "flo-event" -version = "0.1.0" -dependencies = [ - "chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "flo-protocol" -version = "0.1.0" -dependencies = [ - "byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "flo-event 0.1.0", - "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "nom 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "libc" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "log" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "nom" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "num" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-integer 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)", - "num-iter 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num-integer" -version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num-iter" -version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-integer 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num-traits" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "redox_syscall" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "strsim" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "term_size" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "time" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "unicode-segmentation" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicode-width" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "vec_map" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[metadata] -"checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" -"checksum atty 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d912da0db7fa85514874458ca3651fe2cddace8d0b0505571dbdcd41ab490159" -"checksum bitflags 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e1ab483fc81a8143faa7203c4a3c02888ebd1a782e37e41fa34753ba9a162" -"checksum byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855" -"checksum chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)" = "9213f7cd7c27e95c2b57c49f0e69b1ea65b27138da84a170133fd21b07659c00" -"checksum clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)" = "74a80f603221c9cd9aa27a28f52af452850051598537bb6b359c38a7d61e5cda" -"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135" -"checksum log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "5141eca02775a762cc6cd564d8d2c50f67c0ea3a372cbf1c51592b3e029e10ad" -"checksum nom 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e5d4598834859fedb9a0a69d5b862a970e77982a92f544d547257a4d49469067" -"checksum num 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "98b15ba84e910ea7a1973bccd3df7b31ae282bf9d8bd2897779950c9b8303d40" -"checksum num-integer 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "21e4df1098d1d797d27ef0c69c178c3fab64941559b290fcae198e0825c9c8b5" -"checksum num-iter 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "f7d1891bd7b936f12349b7d1403761c8a0b85a18b148e9da4429d5d102c1a41e" -"checksum num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "e1cbfa3781f3fe73dc05321bed52a06d2d491eaa764c52335cf4399f046ece99" -"checksum redox_syscall 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "8dd35cc9a8bdec562c757e3d43c1526b5c6d2653e23e2315065bc25556550753" -"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" -"checksum term_size 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "07b6c1ac5b3fffd75073276bca1ceed01f67a28537097a2a9539e116e50fb21a" -"checksum time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "211b63c112206356ef1ff9b19355f43740fc3f85960c598a93d3a3d3ba7beade" -"checksum unicode-segmentation 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18127285758f0e2c6cf325bb3f3d138a12fee27de4f23e146cd6a179f26c2cf3" -"checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" -"checksum vec_map 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8cdc8b93bd0198ed872357fb2e667f7125646b1762f16d60b2c96350d361897" -"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" -"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" diff --git a/flo-event/Cargo.lock b/flo-event/Cargo.lock deleted file mode 100644 index 00b453e..0000000 --- a/flo-event/Cargo.lock +++ /dev/null @@ -1,100 +0,0 @@ -[root] -name = "flo-event" -version = "0.1.0" -dependencies = [ - "chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "chrono" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "libc" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "num" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-integer 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)", - "num-iter 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num-integer" -version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num-iter" -version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-integer 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num-traits" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "redox_syscall" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "time" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[metadata] -"checksum chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)" = "9213f7cd7c27e95c2b57c49f0e69b1ea65b27138da84a170133fd21b07659c00" -"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135" -"checksum num 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "98b15ba84e910ea7a1973bccd3df7b31ae282bf9d8bd2897779950c9b8303d40" -"checksum num-integer 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "21e4df1098d1d797d27ef0c69c178c3fab64941559b290fcae198e0825c9c8b5" -"checksum num-iter 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "f7d1891bd7b936f12349b7d1403761c8a0b85a18b148e9da4429d5d102c1a41e" -"checksum num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "e1cbfa3781f3fe73dc05321bed52a06d2d491eaa764c52335cf4399f046ece99" -"checksum redox_syscall 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "8dd35cc9a8bdec562c757e3d43c1526b5c6d2653e23e2315065bc25556550753" -"checksum time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "211b63c112206356ef1ff9b19355f43740fc3f85960c598a93d3a3d3ba7beade" -"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" -"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" diff --git a/flo-event/Cargo.toml b/flo-event/Cargo.toml index ac2cb50..269bff4 100644 --- a/flo-event/Cargo.toml +++ b/flo-event/Cargo.toml @@ -4,4 +4,6 @@ version = "0.2.0" authors = ["pfried "] [dependencies] -chrono = "^0.2" +chrono = "^0.4.0" +crc = "^1.0" +byteorder = "1" diff --git a/flo-event/src/id.rs b/flo-event/src/id.rs new file mode 100644 index 0000000..3b60cc7 --- /dev/null +++ b/flo-event/src/id.rs @@ -0,0 +1,133 @@ +use std::cmp::{Ord, PartialOrd, Ordering}; +use std::fmt::{self, Display}; +use std::str::FromStr; + + +/// An actor is a flo server instance that participates in the cluster. Each actor must have it's own id that is unique +/// among all the running flo server instances +pub type ActorId = u16; + +/// This is just a dumb counter that is increased sequentially for each event produced. Note that each actor keeps it's own +/// EventCounter, but will fast-forward the counter to always be one greater than the highest known event counter at any +/// given point in time. +pub type EventCounter = u64; + +/// This is the primary key for an event. It is immutable and unique within the entire event stream. +/// FloEventIds are strictly ordered first based on the `EventCounter` and then on the `ActorId`. +/// This ordering is exactly the same as the ordering of events in the stream. +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +pub struct FloEventId { + pub actor: ActorId, + pub event_counter: EventCounter, +} + +impl Display for FloEventId { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "{}.{}", self.event_counter, self.actor) + } +} + +impl FromStr for FloEventId { + type Err = &'static str; + + fn from_str(input: &str) -> Result { + let err_message = "FloEventId must be an event counter and actor id separated by a single '.'"; + + if let Some(dot_index) = input.find('.') { + println!("dot index: {}", dot_index); + let (counter, actor) = input.split_at(dot_index); + counter.parse::().and_then(|c| { + (&actor[1..]).parse::().map(|a| { + FloEventId::new(a, c) + }) + }).map_err(|_| err_message) + } else { + Err(err_message) + } + } +} + +#[cfg(test)] +mod event_id_test { + use super::*; + + #[test] + fn flo_event_id_is_displayed_as_string() { + let id = FloEventId::new(7, 12345); + let result = format!("{}", id); + let expected = "12345.7"; + assert_eq!(expected, &result); + } + + #[test] + fn from_str_returns_err_when_actor_id_is_missing() { + assert!(FloEventId::from_str("4.").is_err()) + } + + #[test] + fn from_str_returns_err_when_event_counter_is_missing() { + assert!(FloEventId::from_str(".4").is_err()) + } + + #[test] + fn from_str_returns_err_when_number_does_not_contain_dot() { + assert!(FloEventId::from_str("7654").is_err()) + } + + #[test] + fn flo_event_id_is_parsed_from_a_dot_separated_string() { + let input = "8.2"; + let result = FloEventId::from_str(input).unwrap(); + assert_eq!(FloEventId::new(2, 8), result); + } +} + +pub const ZERO_EVENT_ID: FloEventId = FloEventId{event_counter: 0, actor: 0}; +pub const MAX_EVENT_ID: FloEventId = FloEventId{event_counter: ::std::u64::MAX, actor: ::std::u16::MAX}; + +impl FloEventId { + + /// A zero id represents the lack of an id. All valid ids start with an event counter of 1. + #[inline] + pub fn zero() -> FloEventId { + ZERO_EVENT_ID + } + + /// Returns the maximum possible event id. All other event ids will be less than or equal to this id + #[inline] + pub fn max() -> FloEventId { + MAX_EVENT_ID + } + + /// Constructs a new FloEventId with the given actor and counter values + pub fn new(actor: ActorId, event_counter: EventCounter) -> FloEventId { + FloEventId { + event_counter: event_counter, + actor: actor, + } + } + + pub fn is_zero(&self) -> bool { + *self == ZERO_EVENT_ID + } +} + +impl Ord for FloEventId { + fn cmp(&self, other: &Self) -> Ordering { + if self.event_counter == other.event_counter { + self.actor.cmp(&other.actor) + } else { + self.event_counter.cmp(&other.event_counter) + } + } +} + +impl PartialOrd for FloEventId { + fn partial_cmp(&self, other: &FloEventId) -> Option { + if self.event_counter == other.event_counter { + self.actor.partial_cmp(&other.actor) + } else { + self.event_counter.partial_cmp(&other.event_counter) + } + } +} diff --git a/flo-event/src/lib.rs b/flo-event/src/lib.rs index 7816583..a85993f 100644 --- a/flo-event/src/lib.rs +++ b/flo-event/src/lib.rs @@ -36,150 +36,23 @@ //! from `/**/*`. //! extern crate chrono; +extern crate crc; +extern crate byteorder; pub mod time; +mod id; mod version_vec; pub use version_vec::VersionVector; +pub use id::*; -use std::cmp::{Ord, PartialOrd, Ordering}; -use std::fmt::{self, Display, Debug}; -use std::str::FromStr; - -use chrono::{DateTime, UTC}; +use std::fmt::Debug; +use chrono::{DateTime, Utc}; /// All event timestamps are non-monotonic UTC timestamps with millisecond precision. Although the chrono crate can represent /// nanosecond precision, this resolution is not preserved by flo's wire protocol. -pub type Timestamp = DateTime; - -/// An actor is a flo server instance that participates in the cluster. Each actor must have it's own id that is unique -/// among all the running flo server instances -pub type ActorId = u16; - -/// This is just a dumb counter that is increased sequentially for each event produced. Note that each actor keeps it's own -/// EventCounter, but will fast-forward the counter to always be one greater than the highest known event counter at any -/// given point in time. -pub type EventCounter = u64; - -/// This is the primary key for an event. It is immutable and unique within the entire event stream. -/// FloEventIds are strictly ordered first based on the `EventCounter` and then on the `ActorId`. -/// This ordering is exactly the same as the ordering of events in the stream. -#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] -pub struct FloEventId { - pub actor: ActorId, - pub event_counter: EventCounter, -} - -impl Display for FloEventId { - fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { - write!(f, "{}.{}", self.event_counter, self.actor) - } -} - -impl FromStr for FloEventId { - type Err = &'static str; - - fn from_str(input: &str) -> Result { - let err_message = "FloEventId must be an event counter and actor id separated by a single '.'"; - - if let Some(dot_index) = input.find('.') { - println!("dot index: {}", dot_index); - let (counter, actor) = input.split_at(dot_index); - counter.parse::().and_then(|c| { - (&actor[1..]).parse::().map(|a| { - FloEventId::new(a, c) - }) - }).map_err(|_| err_message) - } else { - Err(err_message) - } - } -} - -#[cfg(test)] -mod event_id_test { - use super::*; - - #[test] - fn flo_event_id_is_displayed_as_string() { - let id = FloEventId::new(7, 12345); - let result = format!("{}", id); - let expected = "12345.7"; - assert_eq!(expected, &result); - } - - #[test] - fn from_str_returns_err_when_actor_id_is_missing() { - assert!(FloEventId::from_str("4.").is_err()) - } - - #[test] - fn from_str_returns_err_when_event_counter_is_missing() { - assert!(FloEventId::from_str(".4").is_err()) - } - - #[test] - fn from_str_returns_err_when_number_does_not_contain_dot() { - assert!(FloEventId::from_str("7654").is_err()) - } - - #[test] - fn flo_event_id_is_parsed_from_a_dot_separated_string() { - let input = "8.2"; - let result = FloEventId::from_str(input).unwrap(); - assert_eq!(FloEventId::new(2, 8), result); - } -} - -pub const ZERO_EVENT_ID: FloEventId = FloEventId{event_counter: 0, actor: 0}; -pub const MAX_EVENT_ID: FloEventId = FloEventId{event_counter: ::std::u64::MAX, actor: ::std::u16::MAX}; - -impl FloEventId { - - /// A zero id represents the lack of an id. All valid ids start with an event counter of 1. - #[inline] - pub fn zero() -> FloEventId { - ZERO_EVENT_ID - } - - /// Returns the maximum possible event id. All other event ids will be less than or equal to this id - #[inline] - pub fn max() -> FloEventId { - MAX_EVENT_ID - } - - /// Constructs a new FloEventId with the given actor and counter values - pub fn new(actor: ActorId, event_counter: EventCounter) -> FloEventId { - FloEventId { - event_counter: event_counter, - actor: actor, - } - } - - pub fn is_zero(&self) -> bool { - *self == ZERO_EVENT_ID - } -} - -impl Ord for FloEventId { - fn cmp(&self, other: &Self) -> Ordering { - if self.event_counter == other.event_counter { - self.actor.cmp(&other.actor) - } else { - self.event_counter.cmp(&other.event_counter) - } - } -} +pub type Timestamp = DateTime; -impl PartialOrd for FloEventId { - fn partial_cmp(&self, other: &FloEventId) -> Option { - if self.event_counter == other.event_counter { - self.actor.partial_cmp(&other.actor) - } else { - self.event_counter.partial_cmp(&other.event_counter) - } - } -} /// Defines an event from a client's perspective. An event Consists of some specific header information followed by /// an optional section of binary data. Flo imposes no restrictions or opinions on what can be contained in the `data` @@ -214,6 +87,62 @@ pub trait FloEvent: Debug { data: data, } } + +} + +pub trait EventData { + fn event_namespace(&self) -> &str; + fn event_parent_id(&self) -> Option; + fn event_data(&self) -> &[u8]; + + /// Returns the CRC for this event, which is a CRC32C of the following fields in order: + /// + /// - `namespace` + /// - `parent_id` + /// - `data` + /// + /// This CRC does _not_ use the timestamp or the `id` + fn compute_data_crc(&self) -> u32 { + use crc::crc32::{update, CASTAGNOLI_TABLE}; + use byteorder::{ByteOrder, BigEndian}; + + fn update_hash(value: u32, bytes: &[u8]) -> u32 { + update(value, &CASTAGNOLI_TABLE, bytes) + } + + // start with the crc of the parent id + let mut value: u32 = { + update_hash(0, self.event_namespace().as_bytes()) + }; + + // convert the parent id to a consistent binary representation + { + let mut parent_id_bytes = [0u8; 11]; + let parent_id = self.event_parent_id(); + if parent_id.is_some() { + let id = parent_id.as_ref().unwrap(); + parent_id_bytes[0] = 1; + BigEndian::write_u64(&mut parent_id_bytes[1..9], id.event_counter); + BigEndian::write_u16(&mut parent_id_bytes[9..], id.actor); + } + value = update_hash(value, &parent_id_bytes); + } + + // finally, the data + update_hash(value, self.event_data()) + } +} + +impl EventData for T where T: FloEvent { + fn event_namespace(&self) -> &str { + self.namespace() + } + fn event_parent_id(&self) -> Option { + self.parent_id() + } + fn event_data(&self) -> &[u8] { + self.data() + } } impl FloEvent for T where T: AsRef + Debug { @@ -221,6 +150,14 @@ impl FloEvent for T where T: AsRef + Debug { self.as_ref().id() } + fn timestamp(&self) -> Timestamp { + self.as_ref().timestamp() + } + + fn parent_id(&self) -> Option { + self.as_ref().parent_id() + } + fn namespace(&self) -> &str { self.as_ref().namespace() } @@ -236,14 +173,6 @@ impl FloEvent for T where T: AsRef + Debug { fn to_owned_event(&self) -> OwnedFloEvent { self.as_ref().clone() } - - fn parent_id(&self) -> Option { - self.as_ref().parent_id() - } - - fn timestamp(&self) -> Timestamp { - self.as_ref().timestamp() - } } /// This is the main `FloEvent` implementation that clients will deal with. All of the fields returned by the `FloEvent` diff --git a/flo-event/src/time.rs b/flo-event/src/time.rs index 10ad88d..ecc3c7f 100644 --- a/flo-event/src/time.rs +++ b/flo-event/src/time.rs @@ -1,11 +1,13 @@ -use chrono::{UTC, TimeZone}; +use std::borrow::Borrow; +use chrono::{TimeZone, Utc}; -use ::Timestamp; // type alias for DateTime defined in lib.rs +use Timestamp; // type alias for DateTime defined in lib.rs const NANOS_IN_MILLISECOND: u64 = 1_000_000u64; const MILLIS_IN_SECOND: u64 = 1_000; -pub fn millis_since_epoch(time: Timestamp) -> u64 { +pub fn millis_since_epoch>(ts: T) -> u64 { + let time = ts.borrow(); debug_assert!(time.timestamp() >= 0, "Timestamp must be after the unix epoch"); let secs = time.timestamp() as u64; let millis = time.timestamp_subsec_nanos() as u64 / NANOS_IN_MILLISECOND; @@ -15,17 +17,17 @@ pub fn millis_since_epoch(time: Timestamp) -> u64 { pub fn from_millis_since_epoch(millis_since_unix_epoch: u64) -> Timestamp { let seconds = millis_since_unix_epoch / MILLIS_IN_SECOND; let subsec_nanos = (millis_since_unix_epoch % MILLIS_IN_SECOND) * NANOS_IN_MILLISECOND; - UTC.timestamp(seconds as i64, subsec_nanos as u32) + Utc.timestamp(seconds as i64, subsec_nanos as u32) } pub fn now() -> Timestamp { - UTC::now() + Utc::now() } #[cfg(test)] mod test { use super::*; - use chrono::UTC; + use chrono::Utc; #[test] fn timestamp_is_converted_from_u64_and_back() { @@ -38,7 +40,7 @@ mod test { #[test] #[should_panic] fn millis_since_epoch_panics_if_timestamp_is_prior_to_unix_epoch() { - let early_timestamp = UTC.ymd(1932, 1, 10).and_hms(9, 30, 05); + let early_timestamp = Utc.ymd(1932, 1, 10).and_hms(9, 30, 05); let _ = millis_since_epoch(early_timestamp); } } diff --git a/flo-protocol/Cargo.toml b/flo-protocol/Cargo.toml index 3fd224b..607c453 100644 --- a/flo-protocol/Cargo.toml +++ b/flo-protocol/Cargo.toml @@ -8,6 +8,6 @@ flo-event = { path = "../flo-event" } log = "0.3" nom = "2.2.1" byteorder = "1" -chrono = "^0.2" +chrono = "^0.4.0" rand = "0.3" diff --git a/flo-protocol/src/messages/produce_event.rs b/flo-protocol/src/messages/produce_event.rs index c5cd6a6..0d40759 100644 --- a/flo-protocol/src/messages/produce_event.rs +++ b/flo-protocol/src/messages/produce_event.rs @@ -1,6 +1,6 @@ use nom::{be_u32, be_u16}; -use event::{OwnedFloEvent, FloEventId, ActorId}; +use event::{EventData, OwnedFloEvent, FloEventId, ActorId}; use serializer::Serializer; use super::{ProtocolMessage, parse_str, parse_event_id}; @@ -30,6 +30,17 @@ pub struct ProduceEvent { pub data: Vec, } +impl EventData for ProduceEvent { + fn event_namespace(&self) -> &str { + self.namespace.as_str() + } + fn event_parent_id(&self) -> Option { + self.parent_id + } + fn event_data(&self) -> &[u8] { + self.data.as_slice() + } +} named!{pub parse_new_producer_event>, chain!( diff --git a/flo-server/Cargo.toml b/flo-server/Cargo.toml index add43c6..471b836 100644 --- a/flo-server/Cargo.toml +++ b/flo-server/Cargo.toml @@ -16,7 +16,7 @@ byteorder = "1" tokio-core = "0.1.10" futures = "0.1.16" glob = "0.2" -chrono = "^0.2" +chrono = "^0.4.0" memmap = "0.5.2" libc = "0.2.33" serde = "1.0" From 8aaa098c33f41eafbfff47c2266785ba0cb08427 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 29 Apr 2018 16:05:24 -0400 Subject: [PATCH 56/73] get flo-server compiling again --- .../src/engine/event_stream/partition/controller/mod.rs | 8 ++++---- flo-server/src/engine/event_stream/partition/mod.rs | 2 +- flo-server/src/engine/event_stream/partition/ops.rs | 1 + .../src/engine/event_stream/partition/segment/mod.rs | 2 -- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index 414dcba..ec3d591 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -156,7 +156,7 @@ impl PartitionImpl { } pub fn event_stream_name(&self) -> &str { - &self.event_stream_name + self.event_stream_name.as_str() } pub fn commit_index_reader(&self) -> AtomicCounterReader { @@ -201,7 +201,7 @@ impl PartitionImpl { Ok(()) } OpType::Replicate(rep) => { - self.handle_replicate(rep) + self.handle_replicate(rep); Ok(()) } OpType::Tick => { @@ -244,11 +244,11 @@ impl PartitionImpl { let current_head = self.index.greatest_event_counter(); // ignore any events with id < commit_index - let first_repl_event_index = events.iter().enumerate().find(|e| e.id().event_counter > commit_index); + let first_repl_event_index = events.iter().position(|e| e.id().event_counter > commit_index); if first_repl_event_index.is_none() { // All the events in this message have already been committed, so just return our current commit index return Ok(ReplicationResult { - op_id: op_id, + op_id, success: true, // success = true because the log is consistent. We just happen to have all these entries already highest_event_counter: commit_index, }); diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index d52dc2d..1c972fd 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -260,7 +260,7 @@ impl PartitionRef { } pub fn event_stream_name(&self) -> &str { - &self.event_stream_name + self.event_stream_name.as_str() } pub fn consume(&mut self, connection_id: ConnectionId, _op_id: u32, notifier: Box, filter: EventFilter, start: EventCounter) -> AsyncConsumeResult { diff --git a/flo-server/src/engine/event_stream/partition/ops.rs b/flo-server/src/engine/event_stream/partition/ops.rs index f31efe9..781f10b 100644 --- a/flo-server/src/engine/event_stream/partition/ops.rs +++ b/flo-server/src/engine/event_stream/partition/ops.rs @@ -149,6 +149,7 @@ impl Operation { let (tx, rx) = oneshot::channel(); let rep = ReplicateOperation { client_sender: tx, + op_id, prev_event_counter, prev_event_term, events, diff --git a/flo-server/src/engine/event_stream/partition/segment/mod.rs b/flo-server/src/engine/event_stream/partition/segment/mod.rs index 8ca9b9b..decc79c 100644 --- a/flo-server/src/engine/event_stream/partition/segment/mod.rs +++ b/flo-server/src/engine/event_stream/partition/segment/mod.rs @@ -212,8 +212,6 @@ mod test { #[test] fn write_one_event_to_segment_and_read_it_back() { - let _ = ::env_logger::init(); - let tmpdir = TempDir::new("write_events_to_segment").unwrap(); let event = event(1); let segment_num = SegmentNum(1); From 56a9babf0938f95b9ecc6a5f2f66aefef8f9c7c6 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 6 May 2018 17:44:12 -0400 Subject: [PATCH 57/73] add a CRC to events --- flo-client-lib/src/async/mod.rs | 70 +++++------ flo-client-lib/src/async/ops/produce.rs | 12 +- flo-client-lib/src/codec/mod.rs | 2 +- flo-event/src/crc_error.rs | 20 +++ flo-event/src/lib.rs | 101 ++++++++++++--- flo-protocol/src/messages/mod.rs | 33 ++--- flo-protocol/src/messages/produce_event.rs | 27 ++++ flo-protocol/src/messages/receive_event.rs | 4 + .../src/engine/connection_handler/mod.rs | 42 +++---- .../src/engine/controller/system_event.rs | 21 +++- .../event_stream/partition/controller/mod.rs | 118 ++++++++++-------- .../event_stream/partition/segment/mmap.rs | 8 +- .../partition/segment/persistent_event.rs | 113 ++++++++++------- flo-server/src/test_utils.rs | 10 ++ flo-server/tests/embedded_tests.rs | 6 +- flo-server/tests/sync_client_tests.rs | 4 +- 16 files changed, 391 insertions(+), 200 deletions(-) create mode 100644 flo-event/src/crc_error.rs diff --git a/flo-client-lib/src/async/mod.rs b/flo-client-lib/src/async/mod.rs index a3b44bc..cd2a9a8 100644 --- a/flo-client-lib/src/async/mod.rs +++ b/flo-client-lib/src/async/mod.rs @@ -387,27 +387,27 @@ mod test { #[test] fn produce_all_produces_multiple_events_in_sequence() { let expected_sent = vec![ - ProtocolMessage::ProduceEvent(ProduceEvent { - op_id: 1, - partition: 1, - namespace: "/foo".to_owned(), - parent_id: None, - data: Vec::new(), - }), - ProtocolMessage::ProduceEvent(ProduceEvent { - op_id: 2, - partition: 2, - namespace: "/bar".to_owned(), - parent_id: None, - data: Vec::new(), - }), - ProtocolMessage::ProduceEvent(ProduceEvent { - op_id: 3, - partition: 3, - namespace: "/baz".to_owned(), - parent_id: None, - data: Vec::new(), - }) + ProtocolMessage::ProduceEvent(ProduceEvent::with_crc( + 1, + 1, + "/foo".to_owned(), + None, + Vec::new(), + )), + ProtocolMessage::ProduceEvent(ProduceEvent::with_crc( + 2, + 2, + "/bar".to_owned(), + None, + Vec::new(), + )), + ProtocolMessage::ProduceEvent(ProduceEvent::with_crc( + 3, + 3, + "/baz".to_owned(), + None, + Vec::new(), + )) ]; let to_recv = vec![ ProtocolMessage::AckEvent(EventAck{ @@ -577,22 +577,22 @@ mod test { let to_receive = vec![ ProtocolMessage::CursorCreated(CursorInfo{ op_id: consume_op_id, batch_size: 1 }), - ProtocolMessage::ReceiveEvent(OwnedFloEvent { - id: FloEventId::new(3, 4), - timestamp: time::from_millis_since_epoch(8), - parent_id: None, - namespace: "/foo/bar".to_owned(), - data: "first event data".as_bytes().to_owned(), - }), + ProtocolMessage::ReceiveEvent(OwnedFloEvent::new( + FloEventId::new(3, 4), + None, + time::from_millis_since_epoch(8), + "/foo/bar".to_owned(), + "first event data".as_bytes().to_owned(), + )), ProtocolMessage::EndOfBatch, ProtocolMessage::AwaitingEvents, - ProtocolMessage::ReceiveEvent(OwnedFloEvent { - id: FloEventId::new(3, 5), - timestamp: time::from_millis_since_epoch(9), - parent_id: Some(FloEventId::new(3, 4)), - namespace: "/foo/bar".to_owned(), - data: "second event data".as_bytes().to_owned(), - }), + ProtocolMessage::ReceiveEvent(OwnedFloEvent::new( + FloEventId::new(3, 5), + Some(FloEventId::new(3, 4)), + time::from_millis_since_epoch(9), + "/foo/bar".to_owned(), + "second event data".as_bytes().to_owned(), + )), ]; let receiver = MockReceiveStream::will_produce(to_receive); let (sender, mut send_verify) = MockSendStream::new(); diff --git a/flo-client-lib/src/async/ops/produce.rs b/flo-client-lib/src/async/ops/produce.rs index 632d832..e68f6ce 100644 --- a/flo-client-lib/src/async/ops/produce.rs +++ b/flo-client-lib/src/async/ops/produce.rs @@ -32,13 +32,11 @@ impl ProduceOne { let op_id = connection.next_op_id(); let inner: Inner = match connection.inner.codec.convert_produced(&namespace, data) { Ok(converted) => { - let proto_msg = ProduceEvent{ - op_id, - partition, - namespace, - parent_id, - data: converted, - }; + let proto_msg = ProduceEvent::with_crc(op_id, + partition, + namespace, + parent_id, + converted); Inner::RequestResp(RequestResponse::new(connection, ProtocolMessage::ProduceEvent(proto_msg))) } Err(codec_err) => { diff --git a/flo-client-lib/src/codec/mod.rs b/flo-client-lib/src/codec/mod.rs index 5f4d864..67d2155 100644 --- a/flo-client-lib/src/codec/mod.rs +++ b/flo-client-lib/src/codec/mod.rs @@ -18,7 +18,7 @@ pub trait EventCodec { fn convert_produced(&self, namespace: &str, data: Self::EventData) -> Result, Box>; fn convert_from_message(&self, input: OwnedFloEvent) -> Result, Box> { - let OwnedFloEvent{id, parent_id, namespace, timestamp, data} = input; + let OwnedFloEvent{id, parent_id, namespace, timestamp, data, ..} = input; let converted = { self.convert_received(&namespace, data) }; diff --git a/flo-event/src/crc_error.rs b/flo-event/src/crc_error.rs new file mode 100644 index 0000000..e8dff4f --- /dev/null +++ b/flo-event/src/crc_error.rs @@ -0,0 +1,20 @@ +use std::error::Error; +use std::fmt::{self, Display}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct InvalidCrcError{ + pub computed: u32, + pub actual: u32 +} + +impl Display for InvalidCrcError { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "Invalid CRC32C Checksum, expected: {}, actual: {}", self.computed, self.actual) + } +} + +impl Error for InvalidCrcError { + fn description(&self) -> &str { + "Invalid CRC32C Checksum" + } +} diff --git a/flo-event/src/lib.rs b/flo-event/src/lib.rs index a85993f..ca61a03 100644 --- a/flo-event/src/lib.rs +++ b/flo-event/src/lib.rs @@ -42,7 +42,9 @@ extern crate byteorder; pub mod time; mod id; mod version_vec; +mod crc_error; +pub use self::crc_error::InvalidCrcError; pub use version_vec::VersionVector; pub use id::*; @@ -57,7 +59,7 @@ pub type Timestamp = DateTime; /// Defines an event from a client's perspective. An event Consists of some specific header information followed by /// an optional section of binary data. Flo imposes no restrictions or opinions on what can be contained in the `data` /// section. -pub trait FloEvent: Debug { +pub trait FloEvent: Debug + EventData { /// The primary key of the event. This is immutable and never changes. fn id(&self) -> &FloEventId; /// The UTC timestamp (generated by the server) when this event was persisted. @@ -79,13 +81,10 @@ pub trait FloEvent: Debug { fn to_owned_event(&self) -> OwnedFloEvent { let id = *self.id(); let data = self.data().to_owned(); - OwnedFloEvent { - id: id, - timestamp: self.timestamp(), - parent_id: self.parent_id(), - namespace: self.namespace().to_owned(), - data: data, - } + let parent_id = self.parent_id(); + let timestamp = self.timestamp(); + let namespace = self.namespace().to_owned(); + OwnedFloEvent::new(id, parent_id, timestamp, namespace, data) } } @@ -95,6 +94,8 @@ pub trait EventData { fn event_parent_id(&self) -> Option; fn event_data(&self) -> &[u8]; + fn get_precomputed_crc(&self) -> Option; + /// Returns the CRC for this event, which is a CRC32C of the following fields in order: /// /// - `namespace` @@ -131,21 +132,50 @@ pub trait EventData { // finally, the data update_hash(value, self.event_data()) } + + fn validate_crc(&self) -> Result<(), InvalidCrcError> { + self.get_precomputed_crc().map(|crc| { + let computed = self.compute_data_crc(); + if crc == computed { + Ok(()) + } else { + Err(InvalidCrcError{ + computed, + actual: crc, + }) + } + }).unwrap_or(Ok(())) + } + + fn get_or_compute_crc(&self) -> u32 { + self.get_precomputed_crc().unwrap_or_else(|| self.compute_data_crc()) + } +} + +pub struct BorrowedEventData<'a> { + pub namespace: &'a str, + pub parent_id: Option, + pub data: &'a [u8], } -impl EventData for T where T: FloEvent { +impl <'a> EventData for BorrowedEventData<'a> { fn event_namespace(&self) -> &str { - self.namespace() + self.namespace } + fn event_parent_id(&self) -> Option { - self.parent_id() + self.parent_id } + fn event_data(&self) -> &[u8] { - self.data() + self.data + } + fn get_precomputed_crc(&self) -> Option { + None } } -impl FloEvent for T where T: AsRef + Debug { +impl FloEvent for T where T: AsRef + Debug + EventData { fn id(&self) -> &FloEventId { self.as_ref().id() } @@ -184,18 +214,53 @@ pub struct OwnedFloEvent { pub parent_id: Option, pub namespace: String, pub data: Vec, + pub crc: u32, } impl OwnedFloEvent { pub fn new(id: FloEventId, parent_id: Option, timestamp: Timestamp, namespace: String, data: Vec) -> OwnedFloEvent { + let crc = { + BorrowedEventData { + parent_id, + namespace: namespace.as_str(), + data: data.as_slice() + }.compute_data_crc() + }; OwnedFloEvent { - id: id, - timestamp: timestamp, - parent_id: parent_id, - namespace: namespace, - data: data, + id, + timestamp, + parent_id, + namespace, + data, + crc } } + + pub fn validate_crc(&self) -> Result<(), u32> { + let computed = self.compute_data_crc(); + if self.crc == computed { + Ok(()) + } else { + Err(computed) + } + } +} + +impl EventData for OwnedFloEvent { + fn event_namespace(&self) -> &str { + self.namespace() + } + + fn event_parent_id(&self) -> Option { + self.parent_id() + } + + fn event_data(&self) -> &[u8] { + self.data() + } + fn get_precomputed_crc(&self) -> Option { + Some(self.crc) + } } impl FloEvent for OwnedFloEvent { diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index 8c4bf7a..07520fc 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -24,7 +24,7 @@ mod append_entries; mod request_vote; mod flo_instance_id; -use nom::{be_u64, be_u32, be_u16, be_u8}; +use nom::{be_u64, be_u32, be_u16, be_u8, IResult}; use event::{time, OwnedFloEvent, FloEvent, FloEventId, Timestamp}; use serializer::Serializer; use std::net::SocketAddr; @@ -552,13 +552,13 @@ mod test { #[test] fn serde_receive_event() { - let event = OwnedFloEvent { - id: FloEventId::new(4, 5), - timestamp: time::from_millis_since_epoch(99), - parent_id: Some(FloEventId::new(4, 3)), - namespace: "/foo/bar".to_owned(), - data: vec![9; 99], - }; + let event = OwnedFloEvent::new( + FloEventId::new(4, 5), + Some(FloEventId::new(4, 3)), + time::from_millis_since_epoch(99), + "/foo/bar".to_owned(), + vec![9; 99], + ); let message = ProtocolMessage::ReceiveEvent(event.clone()); let result = serde_with_body(&message, true); assert_eq!(message, result); @@ -609,13 +609,13 @@ mod test { #[test] fn parse_producer_event_parses_the_header_but_not_the_data() { - let input = ProduceEvent { - namespace: "/the/namespace".to_owned(), - parent_id: Some(FloEventId::new(123, 456)), - op_id: 9, - partition: 7, - data: vec![9; 5] - }; + let input = ProduceEvent::with_crc( + 9, + 7, + "/the/namespace".to_owned(), + Some(FloEventId::new(123, 456)), + vec![9; 5] + ); let mut message_input = ProtocolMessage::ProduceEvent(input.clone()); let message_result = ser_de(&mut message_input); @@ -656,6 +656,7 @@ mod test { fn this_works_how_i_think_it_does() { let input = vec![ 3, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 93, 77, 45, 214, 26, @@ -663,7 +664,7 @@ mod test { ]; let result = parse_any(&input); - let expected = IResult::Incomplete(Needed::Size(12164)); + let expected = IResult::Incomplete(Needed::Size(12168)); assert_eq!(expected, result); } } diff --git a/flo-protocol/src/messages/produce_event.rs b/flo-protocol/src/messages/produce_event.rs index 0d40759..a1fde31 100644 --- a/flo-protocol/src/messages/produce_event.rs +++ b/flo-protocol/src/messages/produce_event.rs @@ -28,6 +28,27 @@ pub struct ProduceEvent { /// The event payload. As far as the flo server is concerned, this is just an opaque byte array. Note that events with /// 0-length bodies are perfectly fine. pub data: Vec, + /// The crc32c of the event + pub crc: u32, +} + +impl ProduceEvent { + pub fn new(op_id: u32, partition: ActorId, namespace: String, parent_id: Option, data: Vec, crc: u32) -> ProduceEvent { + ProduceEvent { + op_id, partition, namespace, parent_id, data, crc + } + } + + pub fn with_crc(op_id: u32, partition: ActorId, namespace: String, parent_id: Option, data: Vec) -> ProduceEvent { + let mut evt = ProduceEvent { op_id, partition, namespace, parent_id, data, crc: 0 }; + evt.set_crc(); + evt + } + + pub fn set_crc(&mut self) { + let new_crc = self.compute_data_crc(); + self.crc = new_crc; + } } impl EventData for ProduceEvent { @@ -40,11 +61,15 @@ impl EventData for ProduceEvent { fn event_data(&self) -> &[u8] { self.data.as_slice() } + fn get_precomputed_crc(&self) -> Option { + Some(self.crc) + } } named!{pub parse_new_producer_event>, chain!( _tag: tag!(&[PRODUCE_EVENT]) ~ + crc: be_u32 ~ namespace: parse_str ~ parent_id: parse_event_id ~ op_id: be_u32 ~ @@ -56,6 +81,7 @@ named!{pub parse_new_producer_event>, parent_id: parent_id, op_id: op_id, partition: partition, + crc: crc, data: Vec::with_capacity(data_len as usize), }) } @@ -68,6 +94,7 @@ pub fn serialize_new_produce_header(header: &ProduceEvent, buf: &mut [u8]) -> us }).unwrap_or((0, 0)); Serializer::new(buf).write_u8(PRODUCE_EVENT) + .write_u32(header.crc) .write_string(&header.namespace) .write_u64(counter) .write_u16(actor) diff --git a/flo-protocol/src/messages/receive_event.rs b/flo-protocol/src/messages/receive_event.rs index 01ae4b9..44c27fc 100644 --- a/flo-protocol/src/messages/receive_event.rs +++ b/flo-protocol/src/messages/receive_event.rs @@ -9,6 +9,7 @@ pub const RECEIVE_EVENT: u8 = 3; named!{pub parse_receive_event_header>, chain!( _tag: tag!(&[RECEIVE_EVENT]) ~ + crc: be_u32 ~ id: parse_non_zero_event_id ~ parent_id: parse_event_id ~ timestamp: parse_timestamp ~ @@ -21,14 +22,17 @@ named!{pub parse_receive_event_header>, namespace: namespace, timestamp: timestamp, data: data.to_vec(), + crc: crc }) } ) } pub fn serialize_receive_event_header(event: &E, buf: &mut [u8]) -> usize { + let crc = event.get_or_compute_crc(); Serializer::new(buf) .write_u8(RECEIVE_EVENT) + .write_u32(crc) .write_u64(event.id().event_counter) .write_u16(event.id().actor) .write_u64(event.parent_id().map(|id| id.event_counter).unwrap_or(0)) diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 1d9ab38..969d094 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -377,13 +377,13 @@ mod test { fn receive_event_returns_error_when_no_event_was_expected() { let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); - let event = OwnedFloEvent { - id: FloEventId::new(0, 457), - timestamp: time::now(), - parent_id: None, - namespace: "/system/foo".to_owned(), - data: vec![1, 2, 3, 4, 5], - }; + let event = OwnedFloEvent::new( + FloEventId::new(0, 457), + None, + time::now(), + "/system/foo".to_owned(), + vec![1, 2, 3, 4, 5], + ); let result = subject.handle_incoming_message(ProtocolMessage::ReceiveEvent(event.clone())); assert!(result.is_err()); @@ -411,22 +411,22 @@ mod test { subject.handle_incoming_message(append).unwrap(); fixture.assert_nothing_sent_to_system_stream(); - let event1 = OwnedFloEvent { - id: FloEventId::new(0, 457), - timestamp: time::now(), - parent_id: None, - namespace: "/system/foo".to_owned(), - data: vec![1, 2, 3, 4, 5], - }; + let event1 = OwnedFloEvent::new( + FloEventId::new(0, 457), + None, + time::now(), + "/system/foo".to_owned(), + vec![1, 2, 3, 4, 5], + ); subject.handle_incoming_message(ProtocolMessage::ReceiveEvent(event1.clone())).unwrap(); fixture.assert_nothing_sent_to_system_stream(); - let event2 = OwnedFloEvent { - id: FloEventId::new(0, 457), - timestamp: time::now(), - parent_id: None, - namespace: "/system/foo".to_owned(), - data: vec![1, 2, 3, 4, 5], - }; + let event2 = OwnedFloEvent::new( + FloEventId::new(0, 457), + None, + time::now(), + "/system/foo".to_owned(), + vec![1, 2, 3, 4, 5], + ); subject.handle_incoming_message(ProtocolMessage::ReceiveEvent(event2.clone())).unwrap(); let expected = SystemOpType::AppendEntriesReceived(ReceiveAppendEntries { diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index af3d08c..98d4e5c 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -1,7 +1,8 @@ +use std::fmt::Debug; use rmp_serde::decode::Error; use protocol::Term; -use event::{FloEvent, FloEventId, OwnedFloEvent, EventCounter, ActorId, Timestamp, time}; +use event::{FloEvent, EventData, FloEventId, OwnedFloEvent, EventCounter, ActorId, Timestamp, time}; use engine::event_stream::EventStreamOptions; use engine::event_stream::partition::PersistentEvent; @@ -55,6 +56,24 @@ impl SystemEvent { } } +impl EventData for SystemEvent { + fn event_namespace(&self) -> &str { + self.wrapped.event_namespace() + } + + fn event_parent_id(&self) -> Option { + self.wrapped.event_parent_id() + } + + fn event_data(&self) -> &[u8] { + self.wrapped.event_data() + } + + fn get_precomputed_crc(&self) -> Option { + self.wrapped.get_precomputed_crc() + } +} + impl FloEvent for SystemEvent { fn id(&self) -> &FloEventId { diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index ec3d591..3405e23 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -11,7 +11,7 @@ use chrono::{Duration}; use atomics::{AtomicCounterWriter, AtomicCounterReader, AtomicBoolReader}; use protocol::{ProduceEvent, FloInstanceId}; -use event::{ActorId, FloEventId, EventCounter, FloEvent, Timestamp, time}; +use event::{ActorId, FloEventId, EventCounter, FloEvent, EventData, Timestamp, time}; use super::{SharedReaderRefs, SharedReaderRefsMut, Operation, OpType, ProduceOperation, ConsumeOperation, PartitionReader, EventFilter, SegmentNum, ReplicateOperation, ReplicationResult}; use super::segment::Segment; @@ -460,6 +460,24 @@ struct EventToProduce { produce: ProduceEvent, } +impl EventData for EventToProduce { + fn event_namespace(&self) -> &str { + self.produce.event_namespace() + } + + fn event_parent_id(&self) -> Option { + self.produce.event_parent_id() + } + + fn event_data(&self) -> &[u8] { + self.produce.event_data() + } + + fn get_precomputed_crc(&self) -> Option { + self.produce.get_precomputed_crc() + } +} + impl FloEvent for EventToProduce { fn id(&self) -> &FloEventId { &self.id @@ -532,34 +550,34 @@ mod test { partition.add_replication_node(peer_4); let events = vec![ - ProduceEvent { - op_id: 3, - partition: PARTITION_NUM, - namespace: "/foo/bar".to_owned(), - parent_id: None, - data: "the quick".to_owned().into_bytes(), - }, - ProduceEvent { - op_id: 3, - partition: PARTITION_NUM, - namespace: "/foo/bar".to_owned(), - parent_id: None, - data: "brown fox".to_owned().into_bytes(), - }, - ProduceEvent { - op_id: 3, - partition: PARTITION_NUM, - namespace: "/foo/bar".to_owned(), - parent_id: None, - data: "jumped over".to_owned().into_bytes(), - }, - ProduceEvent { - op_id: 3, - partition: PARTITION_NUM, - namespace: "/foo/bar".to_owned(), - parent_id: None, - data: "the lazy dog".to_owned().into_bytes(), - }, + ProduceEvent::with_crc( + 3, + PARTITION_NUM, + "/foo/bar".to_owned(), + None, + "the quick".to_owned().into_bytes(), + ), + ProduceEvent::with_crc( + 3, + PARTITION_NUM, + "/foo/bar".to_owned(), + None, + "brown fox".to_owned().into_bytes(), + ), + ProduceEvent::with_crc( + 3, + PARTITION_NUM, + "/foo/bar".to_owned(), + None, + "jumped over".to_owned().into_bytes(), + ), + ProduceEvent::with_crc( + 3, + PARTITION_NUM, + "/foo/bar".to_owned(), + None, + "the lazy dog".to_owned().into_bytes(), + ), ]; partition.append_all(events).expect("failed to append events"); assert_eq!(0, partition.get_commit_index()); @@ -618,20 +636,20 @@ mod test { client: client_tx, op_id: 3, events: vec![ - ProduceEvent { - op_id: 3, - partition: PARTITION_NUM, - namespace: "/foo/bar".to_owned(), - parent_id: None, - data: "the quick".to_owned().into_bytes(), - }, - ProduceEvent { - op_id: 3, - partition: PARTITION_NUM, - namespace: "/foo/bar".to_owned(), - parent_id: None, - data: "brown fox".to_owned().into_bytes(), - } + ProduceEvent::with_crc( + 3, + PARTITION_NUM, + "/foo/bar".to_owned(), + None, + "the quick".to_owned().into_bytes(), + ), + ProduceEvent::with_crc( + 3, + PARTITION_NUM, + "/foo/bar".to_owned(), + None, + "brown fox".to_owned().into_bytes(), + ) ], }; @@ -645,13 +663,13 @@ mod test { assert!(reader.next().is_none()); let moar_events = (0..100).map(|_| { - ProduceEvent { - op_id: 4, - partition: PARTITION_NUM, - namespace: "/boo/hoo".to_owned(), - parent_id: None, - data: "stew".to_owned().into_bytes() - } + ProduceEvent::with_crc( + 4, + PARTITION_NUM, + "/boo/hoo".to_owned(), + None, + "stew".to_owned().into_bytes() + ) }).collect::>(); let (client_tx, _client_rx) = oneshot::channel(); diff --git a/flo-server/src/engine/event_stream/partition/segment/mmap.rs b/flo-server/src/engine/event_stream/partition/segment/mmap.rs index c285c5e..70e003c 100644 --- a/flo-server/src/engine/event_stream/partition/segment/mmap.rs +++ b/flo-server/src/engine/event_stream/partition/segment/mmap.rs @@ -271,28 +271,28 @@ mod test { #[test] fn read_event_returns_error_when_namespace_length_is_too_large() { assert_read_err("namespace length too large", |buf| { - buf[41] = 56; + buf[45] = 56; }) } #[test] fn read_event_returns_error_when_namespace_length_is_too_small() { assert_read_err("mismatched lengths", |buf| { - buf[43] = 7; //make the namespace length 7 instead of 8 + buf[47] = 7; //make the namespace length 7 instead of 8 }) } #[test] fn read_event_returns_error_when_data_length_is_too_large() { assert_read_err("mismatched lengths", |buf| { - buf[55] = 6; //make the data length 6 instead of 5 + buf[59] = 6; //make the data length 6 instead of 5 }) } #[test] fn read_event_returns_error_when_data_length_is_too_small() { assert_read_err("mismatched lengths", |buf| { - buf[55] = 4; //make the data length 4 instead of 5 + buf[59] = 4; //make the data length 4 instead of 5 }) } diff --git a/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs b/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs index 7c5fa87..0604a22 100644 --- a/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs +++ b/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs @@ -2,7 +2,7 @@ use std::io; use byteorder::{ByteOrder, BigEndian}; -use event::{FloEvent, OwnedFloEvent, FloEventId, Timestamp, time}; +use event::{FloEvent, EventData, OwnedFloEvent, FloEventId, Timestamp, time}; use engine::event_stream::partition::segment::mmap::{MmapRef}; @@ -21,17 +21,18 @@ impl PersistentEvent { // Don't change this function without also changing `write_event` below! // // 4 for total_size + start = 0 - // 8 for header marker + start = 4 - // 10 for id + start = 12 - // 10 for parent_id + start = 22 - // 8 for timestamp + start = 32 - // 4 for namespace.len + start = 40 - // x for namespace + start = 44 - // 4 for data.len + start = 44 + x = ? - // y for data start = 48 + x = ? + // 8 for header marker + start = 8 + // 4 for crc + start = 12 + // 10 for id + start = 16 + // 10 for parent_id + start = 26 + // 8 for timestamp + start = 36 + // 4 for namespace.len + start = 44 + // x for namespace + start = 48 + // 4 for data.len + start = 48 + x = ? + // y for data start = 52 + x = ? // - // = 48 + x + y - 48u32 + event.namespace().len() as u32 + event.data_len() + // = 52 + x + y + 52u32 + event.namespace().len() as u32 + event.data_len() } pub fn total_repr_len(&self) -> usize { @@ -67,7 +68,7 @@ impl PersistentEvent { } fn validate(buffer: &[u8]) -> io::Result<(FloEventId, u32)> { - if buffer.len() < 48 { + if buffer.len() < 52 { return Err(io::Error::new(io::ErrorKind::InvalidData, "buffer is not large enough")); } @@ -78,23 +79,26 @@ impl PersistentEvent { return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid marker bytes")); } - let partition_buf = &buffer[12..14]; + let crc_buf = &buffer[12..16]; + let crc = BigEndian::read_u32(crc_buf); + + let partition_buf = &buffer[16..18]; let partition_num = BigEndian::read_u16(partition_buf); - let counter_buf = &buffer[14..22]; + let counter_buf = &buffer[18..26]; let counter = BigEndian::read_u64(counter_buf); // check the namespace and data lengths to ensure that they line up OK - let ns_len = BigEndian::read_u32(&buffer[40..44]); + let ns_len = BigEndian::read_u32(&buffer[44..48]); - if ns_len as usize + 48 > buffer.len() { + if ns_len as usize + 52 > buffer.len() { return Err(io::Error::new(io::ErrorKind::InvalidData, "namespace length too large")); } - let data_len_pos = 44usize + ns_len as usize; + let data_len_pos = 48usize + ns_len as usize; let data_len_buf = &buffer[data_len_pos..(data_len_pos + 4)]; let data_len = BigEndian::read_u32(data_len_buf); - if total_len != 48 + ns_len + data_len { + if total_len != 52 + ns_len + data_len { return Err(io::Error::new(io::ErrorKind::InvalidData, "mismatched lengths")); } @@ -106,7 +110,7 @@ impl PersistentEvent { } fn namespace_len(&self) -> u32 { - let buf = self.as_buf(40, 4); + let buf = self.as_buf(44, 4); BigEndian::read_u32(buf) } } @@ -121,46 +125,65 @@ impl PartialEq for PersistentEvent { } } +impl EventData for PersistentEvent { + fn event_namespace(&self) -> &str { + let ns_len = self.namespace_len() as usize; + let ns_buf = self.as_buf(48, ns_len); + unsafe { + ::std::str::from_utf8_unchecked(ns_buf) + } + } + + fn event_parent_id(&self) -> Option { + let buf = self.as_buf(26, 10); + let partition = BigEndian::read_u16(&buf[0..2]); + let counter = BigEndian::read_u64(&buf[2..]); + if counter > 0 { + Some(FloEventId::new(partition, counter)) + } else { + None + } + } + + fn event_data(&self) -> &[u8] { + self.data() + } + + fn get_precomputed_crc(&self) -> Option { + let buf = self.as_buf(8, 4); + Some(BigEndian::read_u32(buf)) + } +} + impl FloEvent for PersistentEvent { fn id(&self) -> &FloEventId { &self.id } fn timestamp(&self) -> Timestamp { - let buf = self.as_buf(32, 8); + let buf = self.as_buf(36, 8); let as_u64 = BigEndian::read_u64(buf); time::from_millis_since_epoch(as_u64) } fn parent_id(&self) -> Option { - let buf = self.as_buf(22, 10); - let partition = BigEndian::read_u16(&buf[0..2]); - let counter = BigEndian::read_u64(&buf[2..]); - if counter > 0 { - Some(FloEventId::new(partition, counter)) - } else { - None - } + self.event_parent_id() } fn namespace(&self) -> &str { - let ns_len = self.namespace_len() as usize; - let ns_buf = self.as_buf(44, ns_len); - unsafe { - ::std::str::from_utf8_unchecked(ns_buf) - } + self.event_namespace() } fn data_len(&self) -> u32 { let ns_len = self.namespace_len() as usize; - let data_len_buf = self.as_buf(44 + ns_len, 4); + let data_len_buf = self.as_buf(48 + ns_len, 4); BigEndian::read_u32(data_len_buf) } fn data(&self) -> &[u8] { let ns_len = self.namespace_len() as usize; let data_len = self.data_len() as usize; - self.as_buf(48 + ns_len, data_len) + self.as_buf(52 + ns_len, data_len) } fn to_owned_event(&self) -> OwnedFloEvent { @@ -182,20 +205,22 @@ fn write_event_unchecked(buffer: &mut [u8], event: &E, total_size: // Don't change this function without also changing `get_repr_len` above! // // 4 for total_size + start = 0 - // 8 for header marker + start = 4 - // 10 for id + start = 12 - // 10 for parent_id + start = 22 - // 8 for timestamp + start = 32 - // 4 for namespace.len + start = 40 - // x for namespace + start = 44 - // 4 for data.len + start = 44 + x = ? - // y for data start = 48 + x = ? + // 8 for header marker + start = 8 + // 4 for crc + start = 12 + // 10 for id + start = 16 + // 10 for parent_id + start = 26 + // 8 for timestamp + start = 36 + // 4 for namespace.len + start = 44 + // x for namespace + start = 48 + // 4 for data.len + start = 48 + x = ? + // y for data start = 52 + x = ? // - // = 48 + x + y + // = 52 + x + y Serializer::new(buffer) .write_u32(total_size) .write_bytes(b"FLO_EVT\n") + .write_u32(event.get_or_compute_crc()) .write_u16(event.id().actor) .write_u64(event.id().event_counter) .write_u16(event.parent_id().map(|e| e.actor).unwrap_or(0)) diff --git a/flo-server/src/test_utils.rs b/flo-server/src/test_utils.rs index 23db3d6..fa44c5a 100644 --- a/flo-server/src/test_utils.rs +++ b/flo-server/src/test_utils.rs @@ -1,9 +1,19 @@ use std::net::SocketAddr; use std::time::{Duration, Instant}; +use std::sync::{Once, ONCE_INIT}; + use futures::executor::{spawn, Unpark}; use futures::{Future, Async}; +static LOGGER_INIT: Once = ONCE_INIT; + +pub fn init_logging() { + LOGGER_INIT.call_once(|| { + let _ = ::env_logger::init(); + }); +} + pub fn addr(string: &str) -> SocketAddr { ::std::str::FromStr::from_str(string).unwrap() } diff --git a/flo-server/tests/embedded_tests.rs b/flo-server/tests/embedded_tests.rs index a4f2323..f9daf24 100644 --- a/flo-server/tests/embedded_tests.rs +++ b/flo-server/tests/embedded_tests.rs @@ -9,6 +9,8 @@ extern crate chrono; extern crate log; +mod test_utils; + use std::fmt::Debug; use std::thread; use std::time::Duration; @@ -16,6 +18,7 @@ use std::time::Duration; use tokio_core::reactor::Core; use futures::{Stream, Future}; +use test_utils::init_logger; use flo_server::embedded::{EmbeddedFloServer, ControllerOptions, EventStreamOptions, run_embedded_server}; use flo_client_lib::{VersionVector, FloEventId, Event, EventCounter, ActorId}; @@ -30,8 +33,9 @@ fn codec() -> Box> { } + fn integration_test(test_name: &'static str, stream_opts: EventStreamOptions, fun: F) where F: Fn(EmbeddedFloServer, Core) { - let _ = env_logger::init(); + init_logger(); println!("starting test: {}", test_name); let dir_name = test_name.replace("\\w", "-"); diff --git a/flo-server/tests/sync_client_tests.rs b/flo-server/tests/sync_client_tests.rs index 7522557..2767a0d 100644 --- a/flo-server/tests/sync_client_tests.rs +++ b/flo-server/tests/sync_client_tests.rs @@ -1,6 +1,6 @@ extern crate flo_client_lib; extern crate url; -extern crate env_logger; +extern crate flo_server; extern crate tempdir; use std::fmt::Debug; @@ -31,7 +31,7 @@ fn simple_event, D: Debug, E: Into>(namespace: N, data: E) -> fn test_with_server(test_name: &'static str, flo_server_args: Vec<&str>, test_fun: F) { use tempdir::TempDir; - let _ = ::env_logger::init(); + init_logger(); let (proc_type, port) = get_server_port(); let temp_dir = TempDir::new(test_name).unwrap(); From 1a49cc42963218ac773a19c8eee45747f90b7901 Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 12 May 2018 21:45:37 -0400 Subject: [PATCH 58/73] POC implementation of event replication is sort of working in partition controller --- .../event_stream/partition/controller/mod.rs | 152 +++++++++++++++++- .../partition/event_reader/mod.rs | 2 +- .../event_stream/partition/segment/mmap.rs | 2 +- .../partition/segment/persistent_event.rs | 93 +++++++++-- 4 files changed, 223 insertions(+), 26 deletions(-) diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index 3405e23..b181c84 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -147,11 +147,17 @@ impl PartitionImpl { } pub fn events_acknowledged(&mut self, peer_id: FloInstanceId, counter: EventCounter) { + if !self.is_current_primary() { + debug!("partition: {} ignoring events_acknowledged from peer: {} with counter: {} because this instance is no longer primary", + self.partition_num, peer_id, counter); + return; + } + let new_index = self.commit_manager.acknowledgement_received(peer_id, counter); if let Some(committed_event) = new_index { // This acknowledgement has caused a new event to be commmitted. Notify the producers of the successful operations // and also notify any consumers of committed events - + self.pending_produce_operations.commit_success(committed_event); } } @@ -183,6 +189,10 @@ impl PartitionImpl { self.consumer_manager.remove(connection_id); } + pub fn is_current_primary(&self) -> bool { + self.primary.get_relaxed() + } + pub fn process(&mut self, operation: Operation) -> io::Result<()> { trace!("Partition: {}, got operation: {:?}", self.partition_num, operation); @@ -239,32 +249,97 @@ impl PartitionImpl { } /// persists events in this partition. Panics if `events` is empty - fn replicate_events(&mut self, op_id: u32, events: Vec) -> io::Result { + pub fn replicate_events(&mut self, op_id: u32, events: Vec) -> io::Result { let commit_index = self.commit_manager.get_commit_index(); let current_head = self.index.greatest_event_counter(); - // ignore any events with id < commit_index + // ignore any events with id.event_counter < commit_index + // We don't bother checking CRCs for these, since they are already committed let first_repl_event_index = events.iter().position(|e| e.id().event_counter > commit_index); if first_repl_event_index.is_none() { // All the events in this message have already been committed, so just return our current commit index return Ok(ReplicationResult { op_id, success: true, // success = true because the log is consistent. We just happen to have all these entries already - highest_event_counter: commit_index, + highest_event_counter: self.index.greatest_event_counter(), }); } - let mut first_event_to_append = first_repl_event_index.unwrap(); + let index_of_first_uncommitted = first_repl_event_index.unwrap(); // now we may still skip events that are uncommitted if the event id and crc matches what we already have in the log + // so we'll check these new events against what we already have to see if they match. This will return a slice of events to append + // checking the events against the ones we already have could result in an error, so we may return early here + let (invalidate_start_counter, events_to_replicate) = self.check_events_to_replicate_against_existing(&events[index_of_first_uncommitted..])?; + + // if any uncommitted events had a different crc, then mark that event and all the events that come after it as deleted + for start_of_invalid_events in invalidate_start_counter { + self.invalidate_uncommitted_events(start_of_invalid_events); + } - // if any uncommitted events have a different crc, then mark that event and all the events that come after it as deleted // at this point, we should be in a state where the last non-deleted event in the log is the same as the previous event info in the message + // append entries starting at `index_of_first_to_replicate` + self.append_replicated_events(events_to_replicate)?; + Ok(ReplicationResult{ + op_id, + success: true, + highest_event_counter: self.index.greatest_event_counter(), + }) + } + + /// Checks a new sequence of events to replicate against the events that may already be persisted to see if we need to + /// either A, delete any existing uncommitted events because the new ones are different, or B, skip writing any of the + /// new events if we already have them persisted + fn check_events_to_replicate_against_existing<'a>(&mut self, to_replicate: &'a [impl FloEvent]) -> io::Result<(Option, &'a [impl FloEvent])> { + let my_start_id = to_replicate[0].id().event_counter; + let my_reader = self.create_reader(0, EventFilter::All, my_start_id); + let mut index_of_first_to_replicate = 0; + let mut invalidate_start_counter: Option = None; + + for persisted_event_result in my_reader { + if index_of_first_to_replicate >= to_replicate.len() { + break; // There will be no work to do here + } + + let persisted_event = persisted_event_result?; // return the io error if we failed to read the event we already have + let existing_event_id = persisted_event.id(); + + let new_event = &to_replicate[index_of_first_to_replicate]; // safe since we check the index bounds above - // append entries starting at `first_event_to_append` + if PartitionImpl::are_same_event(&persisted_event, new_event) { + // If the two events are the same, then we can skip persisting the new one + index_of_first_to_replicate += 1; + } else { + // if the two events are different, then it's because the uncommitted events that we have need to be deleted + // We'll start at this event and invalidate (mark deleted) it and every event that follows it + invalidate_start_counter = Some(persisted_event.id().event_counter); + break; + } + } + let events_to_replicate = &to_replicate[index_of_first_to_replicate..]; + Ok((invalidate_start_counter, events_to_replicate)) + } + + fn append_replicated_events(&mut self, events: &[E]) -> io::Result<()> { + if !events.is_empty() { + debug!("partition: {} replicating {} new events starting with event_counter: {}", self.partition_num, events.len(), events[0].id().event_counter) + } + for event in events { + self.append(event)?; + } + Ok(()) + } + + fn invalidate_uncommitted_events(&mut self, start_inclusive: EventCounter) { + info!("Invalidating existing uncommitted events for partition: {}, starting with event_counter: {}", + self.partition_num, start_inclusive); unimplemented!() } + fn are_same_event(existing: &E, new: &N) -> bool { + existing.id() == new.id() && existing.get_or_compute_crc() == new.get_or_compute_crc() + } + fn drop_segments_through_index(&mut self, segment_index: usize) { info!("Dropping first {} segment(s)", segment_index + 1); let PartitionImpl { ref mut segments, ref mut index, ref mut reader_refs, .. } = *self; @@ -325,6 +400,7 @@ impl PartitionImpl { Ok(FloEventId::new(self.partition_num, event_counter)) } + /// Writes the event to disk and updates the index so that we know where it is fn append(&mut self, event: &E) -> io::Result<()> { use super::SegmentNum; use super::segment::AppendResult; @@ -521,6 +597,68 @@ mod test { const PARTITION_NUM: ActorId = 1; const CONNECTION: ConnectionId = 55; + #[test] + fn events_are_replicated_from_a_peer() { + let status = AtomicBoolWriter::with_value(true); + let options = EventStreamOptions { + name: "superduper".to_owned(), + num_partitions: 1, + event_retention: Duration::seconds(20), + max_segment_duration: Duration::seconds(5), + segment_max_size_bytes: 256, + }; + let tempdir = TempDir::new("partition_persist_events_and_read_them_back").unwrap(); + + // Init a new partition and append a bunch of events in two groups + let mut partition = PartitionImpl::init_new(PARTITION_NUM, + tempdir.path().to_owned(), + &options, + status.reader(), + HighestCounter::zero()).unwrap(); + + let peer_1 = FloInstanceId::generate_new(); + let peer_2 = FloInstanceId::generate_new(); + let peer_3 = FloInstanceId::generate_new(); + let peer_4 = FloInstanceId::generate_new(); + partition.add_replication_node(peer_1); + partition.add_replication_node(peer_2); + partition.add_replication_node(peer_3); + partition.add_replication_node(peer_4); + + let events = vec![ + ProduceEvent::with_crc( + 3, + PARTITION_NUM, + "/foo/bar".to_owned(), + None, + "the quick".to_owned().into_bytes(), + ), + ProduceEvent::with_crc( + 3, + PARTITION_NUM, + "/foo/bar".to_owned(), + None, + "brown fox".to_owned().into_bytes(), + ), + ProduceEvent::with_crc( + 3, + PARTITION_NUM, + "/foo/bar".to_owned(), + None, + "jumped over".to_owned().into_bytes(), + ), + ProduceEvent::with_crc( + 3, + PARTITION_NUM, + "/foo/bar".to_owned(), + None, + "the lazy dog".to_owned().into_bytes(), + ), + ]; + partition.append_all(events).expect("failed to append events"); + + } + #[test] fn events_are_committed_when_acknowledged_by_a_majority() { let status = AtomicBoolWriter::with_value(true); diff --git a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs index c54ac38..8e45fd6 100644 --- a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs +++ b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs @@ -128,7 +128,7 @@ impl PartitionReader { fn should_skip(&self, result: &Option>) -> bool { if let Some(Ok(ref event)) = *result { - !self.filter.matches(event) + !event.is_deleted() && !self.filter.matches(event) } else { false } diff --git a/flo-server/src/engine/event_stream/partition/segment/mmap.rs b/flo-server/src/engine/event_stream/partition/segment/mmap.rs index 70e003c..8e17af2 100644 --- a/flo-server/src/engine/event_stream/partition/segment/mmap.rs +++ b/flo-server/src/engine/event_stream/partition/segment/mmap.rs @@ -35,7 +35,7 @@ impl MmapInner { } } - unsafe fn get_write_slice(&self, start_offset: usize) -> &mut [u8] { + pub unsafe fn get_write_slice(&self, start_offset: usize) -> &mut [u8] { let mmap = &mut *self.region.get(); &mut mmap.as_mut_slice()[start_offset..] } diff --git a/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs b/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs index 0604a22..34e1426 100644 --- a/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs +++ b/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs @@ -6,6 +6,46 @@ use event::{FloEvent, EventData, OwnedFloEvent, FloEventId, Timestamp, time}; use engine::event_stream::partition::segment::mmap::{MmapRef}; +mod offsets { + pub const TOTAL_LEN_START: usize = 0; + pub const TOTAL_LEN_LEN: usize = 4; + + pub const HEADER_MARKER_START: usize = TOTAL_LEN_START + TOTAL_LEN_LEN; + pub const HEADER_MARKER_LEN: usize = 8; + + pub const CRC_START: usize = HEADER_MARKER_START + HEADER_MARKER_LEN; + pub const CRC_LEN: usize = 4; + + pub const ID_START: usize = CRC_START + CRC_LEN; + + pub const ID_PARTITION_LEN: usize = 2; + pub const ID_COUNTER_LEN: usize = 8; + pub const ID_LEN: usize = ID_PARTITION_LEN + ID_COUNTER_LEN; + + pub const PARENT_ID_START: usize = ID_START + ID_LEN; + + pub const TIMESTAMP_START: usize = PARENT_ID_START + ID_LEN; + pub const TIMESTAMP_LEN: usize = 8; + + pub const NS_LEN_START: usize = TIMESTAMP_START + TIMESTAMP_LEN; + pub const NS_LEN_LEN: usize = 4; + + pub const NS_BYTES_START: usize = NS_LEN_START + NS_LEN_LEN; + + pub fn data_len_start(namespace_len: usize) -> usize { + NS_BYTES_START + namespace_len + } + pub const DATA_LEN_LEN: usize = 4; + + pub fn data_bytes_start(namespace_len: usize) -> usize { + data_len_start(namespace_len) + 4 + } + + pub const MIN_EVENT_LEN: usize = NS_BYTES_START + DATA_LEN_LEN; +} + +static HEADER_NORMAL: &[u8; 8] = b"FLO_EVT\n"; +static HEADER_DELETED: &[u8; 8] = b"FLO_DEL\n"; #[derive(Debug, Clone)] pub struct PersistentEvent { @@ -21,7 +61,7 @@ impl PersistentEvent { // Don't change this function without also changing `write_event` below! // // 4 for total_size + start = 0 - // 8 for header marker + start = 8 + // 8 for header marker + start = 4 // 4 for crc + start = 12 // 10 for id + start = 16 // 10 for parent_id + start = 26 @@ -32,13 +72,27 @@ impl PersistentEvent { // y for data start = 52 + x = ? // // = 52 + x + y - 52u32 + event.namespace().len() as u32 + event.data_len() + offsets::MIN_EVENT_LEN as u32 + event.namespace().len() as u32 + event.data_len() } pub fn total_repr_len(&self) -> usize { PersistentEvent::get_repr_length(self) as usize } + pub fn is_deleted(&self) -> bool { + let marker_buf = self.as_buf(offsets::HEADER_MARKER_START, offsets::HEADER_MARKER_LEN); + marker_buf != &HEADER_NORMAL[..] + } + + pub unsafe fn set_deleted(&mut self) { + use std::io::Write; + + let start_offset = self.file_offset + offsets::HEADER_MARKER_START; + let mut buffer = self.raw_data.get_write_slice(start_offset); + // this write can't really fail, since we already know that there's space in the slice + let _ = buffer.write_all(HEADER_DELETED); + } + pub unsafe fn write_unchecked(event: &E, buffer: &mut [u8]) { let len = PersistentEvent::get_repr_length(event); write_event_unchecked(buffer, event, len); @@ -67,44 +121,49 @@ impl PersistentEvent { }) } + //TODO: currently PersistentEvent does not validate the crc. Not sure if it's worth it or not fn validate(buffer: &[u8]) -> io::Result<(FloEventId, u32)> { - if buffer.len() < 52 { + use self::offsets::*; + + if buffer.len() < MIN_EVENT_LEN { return Err(io::Error::new(io::ErrorKind::InvalidData, "buffer is not large enough")); } - let total_len = BigEndian::read_u32(&buffer[..4]); + let total_len = BigEndian::read_u32(&buffer[..TOTAL_LEN_LEN]); - let header_bytes = &buffer[4..12]; - if header_bytes != b"FLO_EVT\n" { + let header_bytes = &buffer[HEADER_MARKER_START..(HEADER_MARKER_START + HEADER_MARKER_LEN)]; + if header_bytes != HEADER_NORMAL && header_bytes != HEADER_DELETED { return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid marker bytes")); } - let crc_buf = &buffer[12..16]; - let crc = BigEndian::read_u32(crc_buf); - - let partition_buf = &buffer[16..18]; + let partition_buf = &buffer[ID_START..(ID_START + ID_PARTITION_LEN)]; let partition_num = BigEndian::read_u16(partition_buf); - let counter_buf = &buffer[18..26]; + let counter_buf = &buffer[(ID_START + ID_PARTITION_LEN)..(ID_START + ID_LEN)]; let counter = BigEndian::read_u64(counter_buf); // check the namespace and data lengths to ensure that they line up OK - let ns_len = BigEndian::read_u32(&buffer[44..48]); + let ns_len = BigEndian::read_u32(&buffer[NS_LEN_START..(NS_LEN_START + NS_LEN_LEN)]); - if ns_len as usize + 52 > buffer.len() { + if ns_len as usize + MIN_EVENT_LEN > buffer.len() { return Err(io::Error::new(io::ErrorKind::InvalidData, "namespace length too large")); } - let data_len_pos = 48usize + ns_len as usize; - let data_len_buf = &buffer[data_len_pos..(data_len_pos + 4)]; + let data_len_pos = data_len_start(ns_len as usize); + let data_len_buf = &buffer[data_len_pos..(data_len_pos + DATA_LEN_LEN)]; let data_len = BigEndian::read_u32(data_len_buf); - if total_len != 52 + ns_len + data_len { + if total_len != MIN_EVENT_LEN as u32 + ns_len + data_len { return Err(io::Error::new(io::ErrorKind::InvalidData, "mismatched lengths")); } Ok((FloEventId::new(partition_num, counter), total_len)) } + fn read_value_unchecked(&self, start: usize, len: usize, fun: F) -> V where F: Fn(&[u8]) -> V { + let buffer_slice = self.as_buf(start, len); + fun(buffer_slice) + } + fn as_buf(&self, start: usize, len: usize) -> &[u8] { &self.raw_data.get_read_slice(self.file_offset + start)[..len] } @@ -219,7 +278,7 @@ fn write_event_unchecked(buffer: &mut [u8], event: &E, total_size: Serializer::new(buffer) .write_u32(total_size) - .write_bytes(b"FLO_EVT\n") + .write_bytes(HEADER_NORMAL) .write_u32(event.get_or_compute_crc()) .write_u16(event.id().actor) .write_u64(event.id().event_counter) From 5424d2606b284fac303100d0978deba517c66fb4 Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 12 May 2018 23:32:31 -0400 Subject: [PATCH 59/73] finish the firs basic replication test, make partitionReader explicit about iterating committed vs uncommitted events --- .../src/engine/controller/system_reader.rs | 2 +- .../event_stream/partition/controller/mod.rs | 59 ++++++++++++------- .../partition/event_reader/mod.rs | 39 +++++++++++- 3 files changed, 74 insertions(+), 26 deletions(-) diff --git a/flo-server/src/engine/controller/system_reader.rs b/flo-server/src/engine/controller/system_reader.rs index 0ff32f7..37f6dcc 100644 --- a/flo-server/src/engine/controller/system_reader.rs +++ b/flo-server/src/engine/controller/system_reader.rs @@ -38,7 +38,7 @@ impl SystemStreamReader { pub fn fill_buffer(&mut self, event_buffer: &mut Vec>) -> io::Result { event_buffer.clear(); while event_buffer.len() < SYSTEM_READER_BATCH_SIZE { - let next = self.inner.next(); + let next = self.inner.read_next_uncommitted(); if let Some(next_result) = next { let event = next_result?; // return if read failed let event_id = *event.id(); diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index b181c84..65913f4 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -295,7 +295,7 @@ impl PartitionImpl { let mut index_of_first_to_replicate = 0; let mut invalidate_start_counter: Option = None; - for persisted_event_result in my_reader { + for persisted_event_result in my_reader.into_iter_uncommitted() { if index_of_first_to_replicate >= to_replicate.len() { break; // There will be no work to do here } @@ -588,6 +588,7 @@ mod test { use futures::sync::oneshot; use super::*; + use event::{OwnedFloEvent, FloEventId, time::from_millis_since_epoch}; use protocol::ProduceEvent; use engine::event_stream::partition::{ProduceOperation, EventFilter, PartitionReader}; use engine::event_stream::{EventStreamOptions, HighestCounter}; @@ -597,8 +598,12 @@ mod test { const PARTITION_NUM: ActorId = 1; const CONNECTION: ConnectionId = 55; + fn event_id(counter: EventCounter) -> FloEventId { + FloEventId::new(PARTITION_NUM, counter) + } + #[test] - fn events_are_replicated_from_a_peer() { + fn events_are_replicated_from_a_peer_when_none_of_them_already_exist() { let status = AtomicBoolWriter::with_value(true); let options = EventStreamOptions { name: "superduper".to_owned(), @@ -626,37 +631,47 @@ mod test { partition.add_replication_node(peer_4); let events = vec![ - ProduceEvent::with_crc( - 3, - PARTITION_NUM, - "/foo/bar".to_owned(), + OwnedFloEvent::new( + event_id(1), None, + from_millis_since_epoch(1), + "/foo/bar".to_owned(), "the quick".to_owned().into_bytes(), ), - ProduceEvent::with_crc( - 3, - PARTITION_NUM, - "/foo/bar".to_owned(), + OwnedFloEvent::new( + event_id(2), None, + from_millis_since_epoch(3), + "/foo/bar".to_owned(), "brown fox".to_owned().into_bytes(), ), - ProduceEvent::with_crc( - 3, - PARTITION_NUM, - "/foo/bar".to_owned(), + OwnedFloEvent::new( + event_id(3), None, + from_millis_since_epoch(5), + "/foo/bar".to_owned(), "jumped over".to_owned().into_bytes(), ), - ProduceEvent::with_crc( - 3, - PARTITION_NUM, - "/foo/bar".to_owned(), + OwnedFloEvent::new( + event_id(4), None, + from_millis_since_epoch(7), + "/foo/bar".to_owned(), "the lazy dog".to_owned().into_bytes(), ), ]; - partition.append_all(events).expect("failed to append events"); + let op_id = 567; + let result = partition.replicate_events(op_id, events.clone()).expect("Failed to replicate events"); + assert!(result.success); + assert_eq!(op_id, result.op_id); + assert_eq!(4, result.highest_event_counter); + + let reader = partition.create_reader(0, EventFilter::All, 0); + let persisted_events = reader.into_iter_uncommitted().map(|e| { + e.expect("Failed to read event").to_owned_event() + }).collect::>(); + assert_eq!(events, persisted_events); } #[test] @@ -798,7 +813,7 @@ mod test { assert_eq!(b"the quick", event.data()); let event2 = reader.read_next_uncommitted().expect("read_next returned None").expect("read_next returned error"); assert_eq!(b"brown fox", event2.data()); - assert!(reader.next().is_none()); + assert!(reader.read_next_uncommitted().is_none()); let moar_events = (0..100).map(|_| { ProduceEvent::with_crc( @@ -819,7 +834,7 @@ mod test { }).expect("failed to persist large batch"); let mut total = 0; - for result in reader { + for result in reader.into_iter_uncommitted() { result.expect("failed to read event"); total += 1; } @@ -832,7 +847,7 @@ mod test { let mut partition = result.expect("Failed to init partitionImpl"); let reader = partition.create_reader(77, EventFilter::All, 0); - let count = reader.map(|read_result| { + let count = reader.into_iter_uncommitted().map(|read_result| { read_result.expect("failed to read event after re-init"); }).count(); assert_eq!(102, count); diff --git a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs index 8e45fd6..dc5861e 100644 --- a/flo-server/src/engine/event_stream/partition/event_reader/mod.rs +++ b/flo-server/src/engine/event_stream/partition/event_reader/mod.rs @@ -121,6 +121,14 @@ impl PartitionReader { }).unwrap_or((SegmentNum(0), 0)) } + pub fn into_iter_uncommitted(self) -> PartitionIterUncommitted { + PartitionIterUncommitted(self) + } + + pub fn into_iter_committed(self) -> PartitionIterCommitted { + PartitionIterCommitted(self) + } + fn reset(&mut self) { self.returned_error = false; self.next_buffer = None; @@ -176,11 +184,36 @@ impl PartitionReader { } } -impl Iterator for PartitionReader { +pub struct PartitionIterUncommitted(PartitionReader); + +impl Iterator for PartitionIterUncommitted { + type Item = io::Result; + + fn next(&mut self) -> Option<::Item> { + self.0.read_next_uncommitted() + } +} + +impl Into for PartitionIterUncommitted { + fn into(self) -> PartitionReader { + self.0 + } +} + + +pub struct PartitionIterCommitted(PartitionReader); + +impl Iterator for PartitionIterCommitted { type Item = io::Result; - fn next(&mut self) -> Option { - self.read_next_uncommitted() + fn next(&mut self) -> Option<::Item> { + self.0.read_next_committed() + } +} + +impl Into for PartitionIterCommitted { + fn into(self) -> PartitionReader { + self.0 } } From 814895659aa196bf7a98ce30bba3aa514b29f4db Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 13 May 2018 13:00:05 -0400 Subject: [PATCH 60/73] server handles committed and uncommitted consumers separately and protocol allows clients to distinguish which one they want --- Cargo.lock | 1 + flo-client-lib/src/async/mod.rs | 1 + flo-client-lib/src/async/ops/consume.rs | 3 +- flo-protocol/Cargo.toml | 1 + flo-protocol/src/lib.rs | 3 ++ flo-protocol/src/messages/consume_start.rs | 32 +++++++++++++--- flo-protocol/src/messages/mod.rs | 4 +- .../engine/connection_handler/consumer/mod.rs | 13 ++++--- .../partition/controller/commit_manager.rs | 8 +++- .../partition/controller/consumer_manager.rs | 2 + .../event_stream/partition/controller/mod.rs | 37 +++++++++++++++---- .../src/engine/event_stream/partition/mod.rs | 4 +- .../src/engine/event_stream/partition/ops.rs | 17 ++++++--- 13 files changed, 97 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0433a94..ab8a3ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -338,6 +338,7 @@ dependencies = [ name = "flo-protocol" version = "0.2.0" dependencies = [ + "bitflags 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "flo-event 0.2.0", diff --git a/flo-client-lib/src/async/mod.rs b/flo-client-lib/src/async/mod.rs index cd2a9a8..ba49bd1 100644 --- a/flo-client-lib/src/async/mod.rs +++ b/flo-client-lib/src/async/mod.rs @@ -613,6 +613,7 @@ mod test { let expected_sent = vec![ ProtocolMessage::NewStartConsuming(NewConsumerStart { op_id: consume_op_id, + options: Default::default(), version_vector: vec![FloEventId::new(1, 2), FloEventId::new(2, 8), FloEventId::new(3, 4)], max_events: 2, namespace: "/foo/*".to_owned(), diff --git a/flo-client-lib/src/async/ops/consume.rs b/flo-client-lib/src/async/ops/consume.rs index 9f2f735..28c8a10 100644 --- a/flo-client-lib/src/async/ops/consume.rs +++ b/flo-client-lib/src/async/ops/consume.rs @@ -5,7 +5,7 @@ use std::io; use futures::{Future, Async, Poll, Stream}; use event::{VersionVector, OwnedFloEvent}; -use protocol::{ProtocolMessage, NewConsumerStart, CONSUME_UNLIMITED}; +use protocol::{ProtocolMessage, NewConsumerStart, ConsumerFlags, CONSUME_UNLIMITED}; use async::{AsyncConnection, ErrorType, ClientProtocolMessage}; use async::ops::{SendMessage, SendError, AwaitResponse, AwaitResponseError, RequestResponse}; use ::Event; @@ -28,6 +28,7 @@ impl Consume { let op_id = connection.next_op_id(); let consumer_start = NewConsumerStart { op_id: op_id, + options: ConsumerFlags::default(), version_vector: version_vec.snapshot(), max_events: event_limit.unwrap_or(CONSUME_UNLIMITED), namespace: namespace.clone(), diff --git a/flo-protocol/Cargo.toml b/flo-protocol/Cargo.toml index 607c453..ef11b01 100644 --- a/flo-protocol/Cargo.toml +++ b/flo-protocol/Cargo.toml @@ -5,6 +5,7 @@ authors = ["pfried "] [dependencies] flo-event = { path = "../flo-event" } +bitflags = "1.0" log = "0.3" nom = "2.2.1" byteorder = "1" diff --git a/flo-protocol/src/lib.rs b/flo-protocol/src/lib.rs index 50a5480..56b7a0b 100644 --- a/flo-protocol/src/lib.rs +++ b/flo-protocol/src/lib.rs @@ -4,6 +4,9 @@ extern crate nom; #[macro_use] extern crate log; +#[macro_use] +extern crate bitflags; + extern crate flo_event as event; extern crate byteorder; extern crate chrono; diff --git a/flo-protocol/src/messages/consume_start.rs b/flo-protocol/src/messages/consume_start.rs index 2ee8d42..4d201ba 100644 --- a/flo-protocol/src/messages/consume_start.rs +++ b/flo-protocol/src/messages/consume_start.rs @@ -10,28 +10,49 @@ pub const NEW_START_CONSUMING: u8 = 17; pub const CONSUME_UNLIMITED: u64 = 0; +bitflags! { + pub struct ConsumerFlags: u32 { + const ConsumeUncommitted = 1; + } +} + +impl Default for ConsumerFlags { + fn default() -> Self { + ConsumerFlags::empty() + } +} + /// New message sent from client to server to begin reading events from the stream #[derive(Debug, PartialEq, Clone)] pub struct NewConsumerStart { pub op_id: u32, + pub options: ConsumerFlags, pub version_vector: Vec, pub max_events: u64, pub namespace: String, } +named!{parse_consumer_options, + map_res!(be_u32, |unvalidated_options| { + ConsumerFlags::from_bits(unvalidated_options).ok_or("invalid consumer options bit flags") + }) +} + named!{pub parse_new_start_consuming>, chain!( _tag: tag!(&[NEW_START_CONSUMING]) ~ op_id: be_u32 ~ - version_vec: parse_version_vec ~ + options: parse_consumer_options ~ + version_vector: parse_version_vec ~ max_events: be_u64 ~ namespace: parse_str, || { ProtocolMessage::NewStartConsuming(NewConsumerStart { - op_id: op_id, - version_vector: version_vec, - max_events: max_events, - namespace: namespace, + op_id, + options, + version_vector, + max_events, + namespace, }) } ) @@ -40,6 +61,7 @@ named!{pub parse_new_start_consuming>, pub fn serialize_consumer_start(start: &NewConsumerStart, buf: &mut [u8]) -> usize { let mut serializer = Serializer::new(buf).write_u8(NEW_START_CONSUMING) .write_u32(start.op_id) + .write_u32(start.options.bits) .write_u16(start.version_vector.len() as u16); for id in start.version_vector.iter() { diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index 07520fc..b1fbd98 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -47,7 +47,7 @@ pub use self::peer_announce::{PeerAnnounce, ClusterMember}; pub use self::error::{ErrorMessage, ErrorKind}; pub use self::produce_event::ProduceEvent; pub use self::event_ack::EventAck; -pub use self::consume_start::{NewConsumerStart, CONSUME_UNLIMITED}; +pub use self::consume_start::{NewConsumerStart, ConsumerFlags, CONSUME_UNLIMITED}; pub use self::cursor_info::CursorInfo; pub use self::event_stream_status::{EventStreamStatus, PartitionStatus}; pub use self::set_event_stream::SetEventStream; @@ -532,6 +532,7 @@ mod test { ]; test_serialize_then_deserialize(&ProtocolMessage::NewStartConsuming(NewConsumerStart{ op_id: 321, + options: ConsumerFlags::default(), version_vector: version_vec, max_events: 987, namespace: "/foo/bar/*".to_owned(), @@ -543,6 +544,7 @@ mod test { let vv = vec![FloEventId::new(1, 0)]; let msg = ProtocolMessage::NewStartConsuming(NewConsumerStart { op_id: 3, + options: ConsumerFlags::ConsumeUncommitted, version_vector: vv, max_events: 1, namespace: "/foo/*".to_owned(), diff --git a/flo-server/src/engine/connection_handler/consumer/mod.rs b/flo-server/src/engine/connection_handler/consumer/mod.rs index 635fdfc..09de1f5 100644 --- a/flo-server/src/engine/connection_handler/consumer/mod.rs +++ b/flo-server/src/engine/connection_handler/consumer/mod.rs @@ -77,12 +77,13 @@ impl ConsumerConnectionState { } pub fn handle_start_consuming(&mut self, start: NewConsumerStart, connection: &mut ConnectionState) -> ConnectionHandlerResult { - let NewConsumerStart {op_id, version_vector, namespace, max_events} = start; + let NewConsumerStart {op_id, version_vector, namespace, max_events, options} = start; let event_limit = if max_events == CONSUME_UNLIMITED { None } else { Some(max_events) }; + let consume_uncommitted = options.contains(ConsumerFlags::ConsumeUncommitted); match EventFilter::parse(&namespace) { Ok(filter) => { @@ -94,11 +95,11 @@ impl ConsumerConnectionState { let partition = id.actor; let notifier = pending_consume.create_notifier(connection_id); - let send_result = connection.event_stream.get_partition(partition).unwrap().consume(connection_id, - op_id, - notifier, - filter.clone(), - start); + //TODO: don't call unwrap so that an invalid partition id in the version vector doesn't cause the connection handler to panic + let send_result = connection.event_stream + .get_partition(partition) + .unwrap() + .consume(connection_id, op_id, notifier, filter.clone(), start, consume_uncommitted); let receiver = send_result.map_err(|err| { format!("Failed to send consume operation to partition: {} : {:?}", partition, err) diff --git a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs index a443b61..dfabd04 100644 --- a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs +++ b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs @@ -54,7 +54,13 @@ impl CommitManager { self.min_required_for_commit == 0 } - pub fn update_commit_index(&mut self, new_index: EventCounter) { + pub fn events_written(&mut self, highest_event_counter: EventCounter) { + if self.is_standalone() { + self.update_commit_index(highest_event_counter); + } + } + + fn update_commit_index(&mut self, new_index: EventCounter) { self.commit_index.set_if_greater(new_index as usize); } diff --git a/flo-server/src/engine/event_stream/partition/controller/consumer_manager.rs b/flo-server/src/engine/event_stream/partition/controller/consumer_manager.rs index 1806c82..228813f 100644 --- a/flo-server/src/engine/event_stream/partition/controller/consumer_manager.rs +++ b/flo-server/src/engine/event_stream/partition/controller/consumer_manager.rs @@ -31,6 +31,8 @@ impl ConsumerManager { } pub fn notify_committed(&mut self) { + // consumers of uncommitted events should also get notified so that they can send out an update about the commit index + self.notify_uncommitted(); do_notify("committed", &mut self.committed); } diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index 65913f4..0729ae9 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -158,6 +158,7 @@ impl PartitionImpl { // This acknowledgement has caused a new event to be commmitted. Notify the producers of the successful operations // and also notify any consumers of committed events self.pending_produce_operations.commit_success(committed_event); + self.consumer_manager.notify_committed(); } } @@ -169,10 +170,6 @@ impl PartitionImpl { self.commit_manager.get_commit_index_reader() } - pub fn set_commit_index(&mut self, new_index: EventCounter) { - self.commit_manager.update_commit_index(new_index) - } - pub fn get_commit_index(&self) -> EventCounter { self.commit_manager.get_commit_index() } @@ -278,7 +275,12 @@ impl PartitionImpl { // at this point, we should be in a state where the last non-deleted event in the log is the same as the previous event info in the message // append entries starting at `index_of_first_to_replicate` - self.append_replicated_events(events_to_replicate)?; + if !events_to_replicate.is_empty() { + let id_of_first_replicated = events_to_replicate[0].id().event_counter; + self.append_replicated_events(events_to_replicate)?; // return early if the actual write fails + + self.notify_consumers_events_appended(id_of_first_replicated); + } Ok(ReplicationResult{ op_id, success: true, @@ -381,6 +383,9 @@ impl PartitionImpl { let timestamp = time::now(); let mut event_counter = new_highest - event_count as u64; + + // keep track of the first event that gets added, since this is what we pass to `notify_consumers_events_appended` + let first_event_id = event_counter; for produce_event in events { event_counter += 1; let event = EventToProduce { @@ -393,13 +398,25 @@ impl PartitionImpl { } debug!("partition: {} finished appending {} events ending with counter: {}", self.partition_num, event_count, event_counter); + // may update the commit index if we're in standalone mode, we we want this to be before the fence + self.commit_manager.events_written(new_highest); + // fence to make sure that events are actually done being saved prior to the notify, since // consumers may then immediately read that region of memory ::std::sync::atomic::fence(::std::sync::atomic::Ordering::SeqCst); - self.consumer_manager.notify_uncommitted(); + + self.notify_consumers_events_appended(first_event_id); Ok(FloEventId::new(self.partition_num, event_counter)) } + fn notify_consumers_events_appended(&mut self, first_event_added: EventCounter) { + if self.get_commit_index() >= first_event_added { + self.consumer_manager.notify_committed() + } else { + self.consumer_manager.notify_uncommitted() + } + } + /// Writes the event to disk and updates the index so that we know where it is fn append(&mut self, event: &E) -> io::Result<()> { use super::SegmentNum; @@ -482,14 +499,18 @@ impl PartitionImpl { } pub fn handle_consume(&mut self, connection_id: ConnectionId, consume: ConsumeOperation) -> io::Result<()> { - let ConsumeOperation {client_sender, filter, start_exclusive, notifier} = consume; + let ConsumeOperation {client_sender, filter, start_exclusive, notifier, consume_uncommitted} = consume; let reader = self.create_reader(connection_id, filter, start_exclusive); // We don't really care if the receiving end has hung up already // but we don't want to actually add the notifier to the consumer manager in that case let result = client_sender.send(reader); if let Ok(_) = result { - self.consumer_manager.add_uncommitted(notifier); + if consume_uncommitted { + self.consumer_manager.add_uncommitted(notifier); + } else { + self.consumer_manager.add_committed(notifier); + } } Ok(()) } diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index 1c972fd..f4c5a4a 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -263,8 +263,8 @@ impl PartitionRef { self.event_stream_name.as_str() } - pub fn consume(&mut self, connection_id: ConnectionId, _op_id: u32, notifier: Box, filter: EventFilter, start: EventCounter) -> AsyncConsumeResult { - let (op, rx) = Operation::consume(connection_id, notifier, filter, start); + pub fn consume(&mut self, connection_id: ConnectionId, _op_id: u32, notifier: Box, filter: EventFilter, start: EventCounter, consume_uncommmitted: bool) -> AsyncConsumeResult { + let (op, rx) = Operation::consume(connection_id, notifier, filter, start, consume_uncommmitted); self.send(op).map(|()| rx) } diff --git a/flo-server/src/engine/event_stream/partition/ops.rs b/flo-server/src/engine/event_stream/partition/ops.rs index 781f10b..d79d2a3 100644 --- a/flo-server/src/engine/event_stream/partition/ops.rs +++ b/flo-server/src/engine/event_stream/partition/ops.rs @@ -51,6 +51,7 @@ pub struct ConsumeOperation { pub client_sender: oneshot::Sender, pub filter: EventFilter, pub start_exclusive: EventCounter, + pub consume_uncommitted: bool, pub notifier: Box, } @@ -58,13 +59,18 @@ impl PartialEq for ConsumeOperation { fn eq(&self, other: &ConsumeOperation) -> bool { self.filter == other.filter && self.start_exclusive == other.start_exclusive && + self.consume_uncommitted == other.consume_uncommitted && self.notifier.connection_id() == other.notifier.connection_id() } } impl Debug for ConsumeOperation { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "ConsumeOperation {{ filter: {:?}, start_exclusive: {} }}", self.filter, self.start_exclusive) + f.debug_struct("ConsumeOperation") + .field("filter", &self.filter) + .field("start_exclusive", &self.start_exclusive) + .field("consume_uncommitted", &self.consume_uncommitted) + .finish() } } @@ -118,13 +124,14 @@ impl Operation { } } - pub fn consume(connection_id: ConnectionId, notifier: Box, filter: EventFilter, start_exclusive: EventCounter) -> (Operation, ConsumeResponseReceiver) { + pub fn consume(connection_id: ConnectionId, notifier: Box, filter: EventFilter, start_exclusive: EventCounter, consume_uncommitted: bool) -> (Operation, ConsumeResponseReceiver) { let (tx, rx) = oneshot::channel(); let consume = ConsumeOperation { client_sender: tx, - filter: filter, - start_exclusive: start_exclusive, - notifier: notifier, + filter, + start_exclusive, + notifier, + consume_uncommitted, }; let op = Operation::new(connection_id, OpType::Consume(consume)); (op, rx) From aef29dc7b62959a1554793cdfcceca26390b42de Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 19 May 2018 12:13:42 -0400 Subject: [PATCH 61/73] add a commitIndexUpdated protocol message --- flo-protocol/src/messages/mod.rs | 27 ++++++++++++++++++- flo-server/src/embedded/mod.rs | 1 + flo-server/src/engine/controller/mod.rs | 17 ++++-------- .../src/engine/controller/system_event.rs | 2 -- .../event_stream/partition/controller/mod.rs | 14 +++++++--- 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index b1fbd98..2bc3045 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -24,7 +24,7 @@ mod append_entries; mod request_vote; mod flo_instance_id; -use nom::{be_u64, be_u32, be_u16, be_u8, IResult}; +use nom::{be_u64, be_u32, be_u16, be_u8}; use event::{time, OwnedFloEvent, FloEvent, FloEventId, Timestamp}; use serializer::Serializer; use std::net::SocketAddr; @@ -68,6 +68,7 @@ pub mod headers { pub const NEXT_BATCH: u8 = 13; pub const END_OF_BATCH: u8 = 14; pub const STOP_CONSUMING: u8 = 15; + pub const COMMIT_INDEX_UPDATED: u8 = 20; } use self::headers::*; @@ -83,6 +84,8 @@ pub enum ProtocolMessage { AwaitingEvents, /// Always the first message sent by the client to the server Announce(ClientAnnounce), + /// Sent to consumers of uncommitted events to let them know when events become committed + CommitIndexUpdated(FloEventId), /// send by the server to a client in response to a StartConsuming message to indicate the start of a series of events CursorCreated(CursorInfo), /// Sent by the server to notify a consumer that it has reached the end of a batch and that more events can be sent @@ -205,6 +208,14 @@ named!{parse_bool, map!(::nom::be_u8, |val| { val == 1 } )} named!{parse_awaiting_events>, map!(tag!(&[AWAITING_EVENTS]), |_| {ProtocolMessage::AwaitingEvents})} +named!{parse_commit_index_updated>, chain!( + _tag: tag!(&[COMMIT_INDEX_UPDATED]) ~ + id: parse_non_zero_event_id, + || { + ProtocolMessage::CommitIndexUpdated(id) + } +)} + named!{parse_next_batch>, map!(tag!(&[NEXT_BATCH]), |_| {ProtocolMessage::NextBatch})} named!{parse_end_of_batch>, map!(tag!(&[END_OF_BATCH]), |_| {ProtocolMessage::EndOfBatch})} @@ -225,6 +236,7 @@ named!{pub parse_any>, alt!( parse_awaiting_events | parse_new_producer_event | parse_next_batch | + parse_commit_index_updated | parse_end_of_batch | parse_stop_consuming | parse_cursor_created | @@ -259,6 +271,13 @@ impl ProtocolMessage { ProtocolMessage::ReceiveEvent(ref event) => { serialize_receive_event_header(event, buf) } + ProtocolMessage::CommitIndexUpdated(ref id) => { + Serializer::new(buf) + .write_u8(COMMIT_INDEX_UPDATED) + .write_u64(id.event_counter) + .write_u16(id.actor) + .finish() + } ProtocolMessage::CursorCreated(ref info) => { serialize_cursor_created(info, buf) } @@ -380,6 +399,12 @@ mod test { ::std::str::FromStr::from_str(addr_string).unwrap() } + #[test] + fn serde_commit_index_updated() { + let message = ProtocolMessage::CommitIndexUpdated(FloEventId::new(5, 6)); + test_serialize_then_deserialize(&message); + } + #[test] fn serde_request_vote_response() { let response = RequestVoteResponse { diff --git a/flo-server/src/embedded/mod.rs b/flo-server/src/embedded/mod.rs index d9a4a4c..98807c7 100644 --- a/flo-server/src/embedded/mod.rs +++ b/flo-server/src/embedded/mod.rs @@ -73,6 +73,7 @@ fn message_to_owned(server_msg: SendProtocolMessage) -> ClientProtocolMessage { ProtocolMessage::SystemAppendResponse(op) => ProtocolMessage::SystemAppendResponse(op), ProtocolMessage::RequestVote(op) => ProtocolMessage::RequestVote(op), ProtocolMessage::VoteResponse(op) => ProtocolMessage::VoteResponse(op), + ProtocolMessage::CommitIndexUpdated(id) => ProtocolMessage::CommitIndexUpdated(id), } } diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 58ed415..6e09901 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -151,9 +151,9 @@ fn handle_partition_op(connection_id: ConnectionId, op_start_time: Instant, op: fn produce_system_events(mut produce_op: partition::ProduceOperation, cluster_state: &mut ConsensusProcessor, controller_state: &mut ControllerStateImpl) { if cluster_state.is_primary() { let term = cluster_state.get_current_term(); - let convert_result = convert_to_system_events(&mut produce_op.events, term); + let validate_result = validate_system_event(&mut produce_op.events, term); - if let Err(err) = convert_result { + if let Err(err) = validate_result { // We're done here produce_op.client.complete(Err(err)); } else { @@ -173,17 +173,10 @@ fn produce_system_events(mut produce_op: partition::ProduceOperation, cluster_st } -fn convert_to_system_events(events: &mut Vec, term: Term) -> io::Result<()> { - for event in events.iter_mut() { - // TODO: Once system events have any sort of body, this function will need to change a bit - // for now, we'll just make sure that the body is empty - // in the future, we'll attempt to deserialize the body as a SystemEventType +fn validate_system_event(events: &Vec, term: Term) -> io::Result<()> { + for event in events.iter() { if event.data.is_empty() { - let new_body = SystemEventData { term }; - let serialized = new_body.serialize(); - event.data = serialized; - } else { - return Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid system event body")); + return Err(io::Error::new(io::ErrorKind::InvalidData, "Event has no body")); } } Ok(()) diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index 98d4e5c..cb39ddc 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -3,7 +3,6 @@ use rmp_serde::decode::Error; use protocol::Term; use event::{FloEvent, EventData, FloEventId, OwnedFloEvent, EventCounter, ActorId, Timestamp, time}; -use engine::event_stream::EventStreamOptions; use engine::event_stream::partition::PersistentEvent; #[derive(Debug, PartialEq)] @@ -46,7 +45,6 @@ impl Into for SystemEvent { impl SystemEvent { pub fn new(id: FloEventId, parent: Option, namespace: String, time: Timestamp, data: &SystemEventData) -> SystemEvent { let term = data.term; - // TODO: I feel like this is probably a safe unwrap, but might be good to double check let serialized = data.serialize(); let event = OwnedFloEvent::new(id, parent, time, namespace, serialized); SystemEvent { diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index 0729ae9..5eb39d4 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -270,7 +270,7 @@ impl PartitionImpl { // if any uncommitted events had a different crc, then mark that event and all the events that come after it as deleted for start_of_invalid_events in invalidate_start_counter { - self.invalidate_uncommitted_events(start_of_invalid_events); + self.invalidate_uncommitted_events(start_of_invalid_events)?; } // at this point, we should be in a state where the last non-deleted event in the log is the same as the previous event info in the message @@ -332,10 +332,18 @@ impl PartitionImpl { Ok(()) } - fn invalidate_uncommitted_events(&mut self, start_inclusive: EventCounter) { + fn invalidate_uncommitted_events(&mut self, start_inclusive: EventCounter) -> io::Result<()> { info!("Invalidating existing uncommitted events for partition: {}, starting with event_counter: {}", self.partition_num, start_inclusive); - unimplemented!() + let reader = self.create_reader(0, EventFilter::All, start_inclusive - 1); + + for event_result in reader.into_iter_uncommitted() { + let mut event = event_result?; + unsafe { + event.set_deleted(); + } + } + Ok(()) } fn are_same_event(existing: &E, new: &N) -> bool { From 5f68b1d4cb8c1daea5afba1c2f43c60b6fcef02f Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 20 May 2018 10:16:30 -0400 Subject: [PATCH 62/73] change SystemEvent representation to include deserialized data --- .../src/engine/controller/system_event.rs | 49 ++++++++++++------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index cb39ddc..ec2e90e 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -7,31 +7,26 @@ use engine::event_stream::partition::PersistentEvent; #[derive(Debug, PartialEq)] pub struct SystemEvent { - term: Term, wrapped: E, + deserialized_data: SystemEventData } impl SystemEvent { pub fn from_event(event: E) -> Result, Error> { - let term = { - - let data = ::rmp_serde::decode::from_slice::(event.data())?; - data.term - - }; + let data = ::rmp_serde::decode::from_slice::(event.data())?; Ok(SystemEvent{ - term, - wrapped: event + wrapped: event, + deserialized_data: data }) } pub fn term(&self) -> Term { - self.term + self.system_data().term } - pub fn system_data(&self) -> Result { - ::rmp_serde::from_slice(self.data()) + pub fn system_data(&self) -> &SystemEventData { + &self.deserialized_data } } @@ -43,13 +38,12 @@ impl Into for SystemEvent { impl SystemEvent { - pub fn new(id: FloEventId, parent: Option, namespace: String, time: Timestamp, data: &SystemEventData) -> SystemEvent { - let term = data.term; + pub fn new(id: FloEventId, parent: Option, namespace: String, time: Timestamp, data: SystemEventData) -> SystemEvent { let serialized = data.serialize(); let event = OwnedFloEvent::new(id, parent, time, namespace, serialized); SystemEvent { - term, - wrapped: event + wrapped: event, + deserialized_data: data, } } } @@ -94,12 +88,27 @@ impl FloEvent for SystemEvent { } } + +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct QualifiedPartitionId { + pub event_stream: String, + pub partition: ActorId, +} + +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub enum SystemEventKind { + ResignPrimary(QualifiedPartitionId), + AssignPrimary(QualifiedPartitionId), +} + + #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub struct SystemEventData { pub term: Term, + pub kind: SystemEventKind, } -impl SystemEventData { +impl SystemEventData { pub fn serialize(&self) -> Vec { ::rmp_serde::to_vec(self).unwrap() } @@ -122,12 +131,16 @@ mod test { fn system_event_data_is_serialized_and_deserialized_inside_system_event() { let data = SystemEventData { term: 33, + kind: SystemEventKind::ResignPrimary(QualifiedPartitionId { + event_stream: "whatever".to_owned(), + partition: 4 + }) }; let id = FloEventId::new(3, 4); let parent = Some(FloEventId::new(2, 3)); let time = time::from_millis_since_epoch(1234567); let namespace = "/system/foo".to_owned(); - let event = SystemEvent::new(id, parent, namespace, time, &data); + let event = SystemEvent::new(id, parent, namespace, time, data); let as_owned = event.to_owned_event(); let result = SystemEvent::from_event(as_owned).unwrap(); assert_eq!(event, result); From 824474d1cda72c343b9d5164663a074816fd5f87 Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 9 Jun 2018 12:27:43 -0400 Subject: [PATCH 63/73] back out system event additions, change FloInstanceId to be a typedef instead of a wrapper type to make serde simpler --- flo-protocol/src/messages/append_entries.rs | 2 +- flo-protocol/src/messages/flo_instance_id.rs | 64 +++++++---------- flo-protocol/src/messages/mod.rs | 18 ++--- flo-protocol/src/messages/peer_announce.rs | 8 +-- flo-protocol/src/messages/request_vote.rs | 2 +- .../src/engine/connection_handler/mod.rs | 14 ++-- .../engine/controller/cluster_state/mod.rs | 69 ++++++++++--------- .../cluster_state/peer_connections.rs | 13 ++-- .../controller/cluster_state/persistent.rs | 63 +++++------------ .../controller/cluster_state/primary_state.rs | 20 ------ .../engine/controller/controller_messages.rs | 2 - flo-server/src/engine/controller/mod.rs | 2 +- .../src/engine/controller/system_event.rs | 4 +- .../partition/controller/commit_manager.rs | 31 +++++---- .../event_stream/partition/controller/mod.rs | 18 ++--- 15 files changed, 138 insertions(+), 192 deletions(-) diff --git a/flo-protocol/src/messages/append_entries.rs b/flo-protocol/src/messages/append_entries.rs index 376a3c9..5740eab 100644 --- a/flo-protocol/src/messages/append_entries.rs +++ b/flo-protocol/src/messages/append_entries.rs @@ -84,7 +84,7 @@ pub fn serialize_append_entries(append: &AppendEntriesCall, buf: &mut [u8]) -> u Serializer::new(buf) .write_u8(APPEND_ENTRIES_CALL_HEADER) .write_u32(append.op_id) - .write(&append.leader_id) + .write_u64(append.leader_id) .write_u64(append.term) .write_u64(append.prev_entry_term) .write_u64(append.prev_entry_index) diff --git a/flo-protocol/src/messages/flo_instance_id.rs b/flo-protocol/src/messages/flo_instance_id.rs index 2844c94..f50b745 100644 --- a/flo-protocol/src/messages/flo_instance_id.rs +++ b/flo-protocol/src/messages/flo_instance_id.rs @@ -2,59 +2,45 @@ use std::fmt::{self, Display}; use serializer::{Serializer, FloSerialize}; -/// An opaque identifier used to uniquely identify flo instances. -#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] -pub struct FloInstanceId(u64); +pub type FloInstanceId = u64; -impl FloInstanceId { +pub const NULL_INSTANCE_ID: FloInstanceId = 0; - pub fn null() -> FloInstanceId { - FloInstanceId(0) - } - - pub fn generate_new() -> FloInstanceId { - let mut value = 0; - while value == 0 { - value = ::rand::random(); - } - FloInstanceId(value) - } - - pub fn as_bytes(&self) -> [u8; 8] { - use ::byteorder::{ByteOrder, BigEndian}; - - let mut bytes = [0; 8]; - BigEndian::write_u64(&mut bytes[..], self.0); - bytes +pub fn generate_new() -> FloInstanceId { + let mut value = 0; + while value == 0 { + value = ::rand::random(); } + value +} - pub fn from_bytes(bytes: &[u8]) -> FloInstanceId { - use ::byteorder::{ByteOrder, BigEndian}; +pub fn as_bytes(instance_id: FloInstanceId) -> [u8; 8] { + use ::byteorder::{ByteOrder, BigEndian}; - let b = &bytes[0..8]; - let num = BigEndian::read_u64(b); - FloInstanceId(num) - } + let mut bytes = [0; 8]; + BigEndian::write_u64(&mut bytes[..], instance_id); + bytes } -impl Display for FloInstanceId { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "FloInstanceId({})", self.0) - } +pub fn from_bytes(bytes: &[u8]) -> FloInstanceId { + use ::byteorder::{ByteOrder, BigEndian}; + + let b = &bytes[0..8]; + BigEndian::read_u64(b) } -impl FloSerialize for FloInstanceId { - fn serialize<'a>(&'a self, serializer: Serializer<'a>) -> Serializer<'a> { - serializer.write_u64(self.0) +named!{pub parse_flo_instance_id, map_res!(::nom::be_u64, |val| { + if val == 0 { + Err(()) + } else { + Ok(val) } -} - -named!{pub parse_flo_instance_id, map!(::nom::be_u64, |val| {FloInstanceId(val) } )} +})} named!{pub parse_optional_flo_instance_id>, map!(::nom::be_u64, |val| { if val > 0 { - Some(FloInstanceId(val)) + Some(val) } else { None } diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index 2bc3045..dcaadfc 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -10,6 +10,7 @@ //! //! All numbers use big endian byte order. //! All Strings are newline terminated. +pub mod flo_instance_id; mod client_announce; mod peer_announce; mod error; @@ -22,7 +23,6 @@ mod set_event_stream; mod receive_event; mod append_entries; mod request_vote; -mod flo_instance_id; use nom::{be_u64, be_u32, be_u16, be_u8}; use event::{time, OwnedFloEvent, FloEvent, FloEventId, Timestamp}; @@ -420,7 +420,7 @@ mod test { let request = RequestVoteCall { op_id: 3, term: 4, - candidate_id: FloInstanceId::generate_new(), + candidate_id: flo_instance_id::generate_new(), last_log_index: 567, last_log_term: 8910, }; @@ -441,7 +441,7 @@ mod test { fn serde_append_entries_call() { let append = AppendEntriesCall { op_id: 345, - leader_id: FloInstanceId::generate_new(), + leader_id: flo_instance_id::generate_new(), term: 987, prev_entry_term: 986, prev_entry_index: 134, @@ -468,20 +468,20 @@ mod test { let announce = PeerAnnounce { protocol_version: 9, op_id: 6543, - instance_id: FloInstanceId::generate_new(), + instance_id: flo_instance_id::generate_new(), peer_address: addr("[1:3:5::2]:4321"), - system_primary_id: Some(FloInstanceId::generate_new()), + system_primary_id: Some(flo_instance_id::generate_new()), cluster_members: vec![ ClusterMember { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("1.2.3.4:5") }, ClusterMember { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("127.0.0.1:2456") }, ClusterMember { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("192.168.1.1:443") }, ] @@ -494,7 +494,7 @@ mod test { let addr = ::std::str::FromStr::from_str("123.234.12.1:4321").unwrap(); let announce = PeerAnnounce { protocol_version: 9, - instance_id: FloInstanceId::generate_new(), + instance_id: flo_instance_id::generate_new(), peer_address: addr, op_id: 6543, system_primary_id: None, diff --git a/flo-protocol/src/messages/peer_announce.rs b/flo-protocol/src/messages/peer_announce.rs index f0281fa..1dc949f 100644 --- a/flo-protocol/src/messages/peer_announce.rs +++ b/flo-protocol/src/messages/peer_announce.rs @@ -4,7 +4,7 @@ use nom::{be_u32, be_u16}; use serializer::{Serializer, FloSerialize}; use event::OwnedFloEvent; use super::{parse_socket_addr, ProtocolMessage, FloInstanceId}; -use super::flo_instance_id::{parse_flo_instance_id, parse_optional_flo_instance_id}; +use super::flo_instance_id::{parse_flo_instance_id, parse_optional_flo_instance_id, NULL_INSTANCE_ID}; pub const PEER_ANNOUNCE: u8 = 7; @@ -27,7 +27,7 @@ pub struct ClusterMember { impl FloSerialize for ClusterMember { fn serialize<'a>(&'a self, serializer: Serializer<'a>) -> Serializer<'a> { - serializer.write(&self.id).write_socket_addr(self.address) + serializer.write_u64(self.id).write_socket_addr(self.address) } } @@ -64,9 +64,9 @@ pub fn serialize_peer_announce(announce: &PeerAnnounce, buf: &mut [u8]) -> usize .write_u8(PEER_ANNOUNCE) .write_u32(announce.protocol_version) .write_u32(announce.op_id) - .write(&announce.instance_id) + .write_u64(announce.instance_id) .write_socket_addr(announce.peer_address) - .write(&announce.system_primary_id.unwrap_or(FloInstanceId::null())) + .write_u64(announce.system_primary_id.unwrap_or(NULL_INSTANCE_ID)) .write_u16(announce.cluster_members.len() as u16) .write_many(announce.cluster_members.iter(), |ser, member| { ser.write(member) diff --git a/flo-protocol/src/messages/request_vote.rs b/flo-protocol/src/messages/request_vote.rs index 33de0e7..73fbed8 100644 --- a/flo-protocol/src/messages/request_vote.rs +++ b/flo-protocol/src/messages/request_vote.rs @@ -51,7 +51,7 @@ pub fn serialize_request_vote_call(req: &RequestVoteCall, buf: &mut [u8]) -> usi .write_u8(REQUEST_VOTE_CALL_HEADER) .write_u32(req.op_id) .write_u64(req.term) - .write(&req.candidate_id) + .write_u64(req.candidate_id) .write_u64(req.last_log_index) .write_u64(req.last_log_term) .finish() diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 969d094..701629d 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -225,7 +225,7 @@ mod test { }; fixture.assert_sent_to_client(ProtocolMessage::PeerAnnounce(announce.clone())); - let peer_id = FloInstanceId::generate_new(); + let peer_id = flo_instance_id::generate_new(); let peer_addr = addr("127.0.0.1:4000"); let response = PeerAnnounce { peer_address: peer_addr, @@ -263,7 +263,7 @@ mod test { primary.reader(), tx.clone(), primary_addr); - let instance_id = FloInstanceId::generate_new(); + let instance_id = flo_instance_id::generate_new(); let instance_addr = addr("127.0.0.1:3000"); let cluster_state = SharedClusterState { @@ -398,7 +398,7 @@ mod test { fn receiving_append_entries_with_multiple_entries_sends_append_entries_to_system_controller_once_all_events_are_received() { let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); - let sender_id = FloInstanceId::generate_new(); + let sender_id = flo_instance_id::generate_new(); let append = ProtocolMessage::SystemAppendCall(AppendEntriesCall { op_id: 2, leader_id: sender_id, @@ -443,7 +443,7 @@ mod test { fn receiving_append_entries_with_0_entries_sends_append_entries_to_system_controller_and_response_is_sent_out() { let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); - let sender_id = FloInstanceId::generate_new(); + let sender_id = flo_instance_id::generate_new(); let append = ProtocolMessage::SystemAppendCall(AppendEntriesCall { op_id: 2, leader_id: sender_id, @@ -525,7 +525,7 @@ mod test { #[test] fn vote_response_is_sent_to_system_stream_when_one_is_expected() { let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); - let peer_id = FloInstanceId::generate_new(); + let peer_id = flo_instance_id::generate_new(); let request_vote = CallRequestVote { term: 5, candidate_id: peer_id, @@ -562,7 +562,7 @@ mod test { let incoming = RequestVoteCall { op_id: 99, term: 5, - candidate_id: FloInstanceId::generate_new(), + candidate_id: flo_instance_id::generate_new(), last_log_index: 44, last_log_term: 4, }; @@ -580,7 +580,7 @@ mod test { #[test] fn receiving_request_vote_forwards_request_to_system_stream_for_peer_connection_and_response_is_sent_back() { let (mut subject, mut fixture) = Fixture::create_outgoing_peer_connection(); - let peer_id = FloInstanceId::generate_new(); + let peer_id = flo_instance_id::generate_new(); let incoming = RequestVoteCall { op_id: 99, term: 5, diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 2febe3f..861ed84 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -13,7 +13,7 @@ use event::{EventCounter, ActorId, OwnedFloEvent}; use engine::{ConnectionId, EngineRef}; use engine::connection_handler::ConnectionControl; use engine::controller::{CallRequestVote, VoteResponse}; -use protocol::{FloInstanceId, Term}; +use protocol::{FloInstanceId, flo_instance_id, Term}; use atomics::{AtomicBoolWriter, AtomicBoolReader}; use super::{ClusterOptions, ConnectionRef, Peer, PeerUpgrade, AppendEntriesResponse, ReceiveAppendEntries, ControllerState}; use super::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator, OutgoingConnectionCreatorImpl}; @@ -254,6 +254,10 @@ impl ClusterManager { None }) } + + fn apply_committed(&mut self, commit_index: EventCounter, controller_state: &mut ControllerState) -> io::Result<()> { + unimplemented!() + } } impl ConsensusProcessor for ClusterManager { @@ -454,6 +458,7 @@ impl ConsensusProcessor for ClusterManager { // TODO: confirm entries with partition impl } } + fn get_current_term(&self) -> Term { self.persistent.current_term } @@ -470,7 +475,7 @@ pub struct SharedClusterState { impl SharedClusterState { pub fn non_cluster() -> SharedClusterState { SharedClusterState { - this_instance_id: FloInstanceId::generate_new(), + this_instance_id: flo_instance_id::generate_new(), this_address: None, system_primary: None, peers: HashSet::new(), @@ -532,11 +537,11 @@ mod test { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.1:1000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:2000") }; let peer_1_connection = 1; @@ -576,11 +581,11 @@ mod test { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.1:1000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:2000") }; let peer_1_connection = 1; @@ -655,11 +660,11 @@ mod test { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.1:1000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:2000") }; let peer_1_connection = 1; @@ -731,11 +736,11 @@ mod test { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.1:1000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:2000") }; let peer_1_connection = 1; @@ -781,11 +786,11 @@ mod test { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.1:1000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:2000") }; let peer_1_connection = 1; @@ -822,19 +827,19 @@ mod test { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.1:1000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:2000") }; let peer_3 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:3000") }; let peer_4 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:4000") }; let peer_1_connection = 1; @@ -874,11 +879,11 @@ mod test { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.1:1000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:2000") }; let peer_1_connection = 1; @@ -914,19 +919,19 @@ mod test { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.1:1000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:2000") }; let peer_3 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:3000") }; let peer_4 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:4000") }; let peer_1_connection = 1; @@ -1142,11 +1147,11 @@ mod test { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.1:3000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:3000") }; let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); @@ -1181,11 +1186,11 @@ mod test { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.1:3000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:3000") }; let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); @@ -1249,12 +1254,12 @@ mod test { #[test] fn shared_state_this_instance_is_primary_returns_false_when_system_primary_does_not_match() { - let this_id = FloInstanceId::generate_new(); + let this_id = flo_instance_id::generate_new(); let this_addr = addr("127.0.0.1:3000"); let subject = SharedClusterState { this_instance_id: this_id, this_address: Some(this_addr), - system_primary: Some(Peer {id: FloInstanceId::generate_new(), address: this_addr}), + system_primary: Some(Peer {id: flo_instance_id::generate_new(), address: this_addr}), peers: HashSet::new(), }; assert!(!subject.this_instance_is_primary()); @@ -1262,7 +1267,7 @@ mod test { #[test] fn shared_state_this_instance_is_primary_returns_false_when_system_primary_is_none() { - let this_id = FloInstanceId::generate_new(); + let this_id = flo_instance_id::generate_new(); let this_addr = addr("127.0.0.1:3000"); let subject = SharedClusterState { this_instance_id: this_id, @@ -1275,7 +1280,7 @@ mod test { #[test] fn shared_state_this_instance_is_primary_returns_true_when_this_instance_id_matches_primary() { - let this_id = FloInstanceId::generate_new(); + let this_id = flo_instance_id::generate_new(); let this_addr = addr("127.0.0.1:3000"); let subject = SharedClusterState { this_instance_id: this_id, @@ -1325,11 +1330,11 @@ mod test { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.1:3000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("111.222.0.2:3000") }; let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs index 76b8340..fb23103 100644 --- a/flo-server/src/engine/controller/cluster_state/peer_connections.rs +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -317,6 +317,7 @@ mod test { use engine::controller::ControllerState; use engine::connection_handler::ConnectionControlReceiver; use test_utils::{addr, expect_future_resolved}; + use protocol::flo_instance_id; fn assert_control_sent(rx: ConnectionControlReceiver, expected: &ConnectionControl) -> ConnectionControlReceiver { use futures::Stream; @@ -341,7 +342,7 @@ mod test { #[test] fn send_to_peer_sends_connection_control_to_a_connected_peer() { - let peer_id = FloInstanceId::generate_new(); + let peer_id = flo_instance_id::generate_new(); let peer = Peer { id: peer_id, address: addr("123.4.5.6:3000") @@ -364,11 +365,11 @@ mod test { #[test] fn broadcast_sends_control_to_all_connected_peers() { let peer_1 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("123.4.5.6:3000") }; let peer_2 = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("123.4.5.6:4000") }; let mut creator = MockOutgoingConnectionCreator::new(); @@ -378,7 +379,7 @@ mod test { let rx_1 = assert_control_sent(rx_1, &ConnectionControl::InitiateOutgoingSystemConnection); let rx_2 = assert_control_sent(rx_2, &ConnectionControl::InitiateOutgoingSystemConnection); - let candidate = FloInstanceId::generate_new(); + let candidate = flo_instance_id::generate_new(); let expected = ConnectionControl::SendRequestVote(CallRequestVote { term: 7, candidate_id: candidate, @@ -394,7 +395,7 @@ mod test { #[test] fn outgoing_connect_success_adds_known_peer_and_connection_closed_sets_it_to_disconnected() { let peer_address = addr("123.45.67.8:3000"); - let peer_id = FloInstanceId::generate_new(); + let peer_id = flo_instance_id::generate_new(); let mut creator = MockOutgoingConnectionCreator::new(); let (peer_connection, rx) = creator.stub(peer_address); @@ -423,7 +424,7 @@ mod test { fn known_peers_are_added_to_disconnected_peers_when_struct_is_initialized() { let peer_address = addr("123.45.67.8:3000"); let peer = Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: peer_address, }; let mut creator = MockOutgoingConnectionCreator::new(); diff --git a/flo-server/src/engine/controller/cluster_state/persistent.rs b/flo-server/src/engine/controller/cluster_state/persistent.rs index ada33f4..2bcc27b 100644 --- a/flo-server/src/engine/controller/cluster_state/persistent.rs +++ b/flo-server/src/engine/controller/cluster_state/persistent.rs @@ -4,7 +4,9 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Seek, SeekFrom}; use std::collections::HashSet; -use protocol::{FloInstanceId, Term}; +use event::EventCounter; +use protocol::Term; +use protocol::flo_instance_id::{self, FloInstanceId}; use engine::controller::controller_messages::Peer; use super::{ClusterOptions, SharedClusterState}; @@ -14,9 +16,7 @@ use super::{ClusterOptions, SharedClusterState}; #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub struct PersistentClusterState { pub current_term: Term, - #[serde(with = "OptInstanceIdRemote")] pub voted_for: Option, - #[serde(with = "InstanceIdRemote")] pub this_instance_id: FloInstanceId, pub cluster_members: HashSet, } @@ -32,43 +32,23 @@ impl PersistentClusterState { peers: self.cluster_members.clone(), } } -} - -#[derive(Serialize, Deserialize, PartialEq, Debug, Clone, Copy)] -#[serde(remote = "FloInstanceId")] -pub struct InstanceIdRemote{ - #[serde(getter = "FloInstanceId::as_bytes")] - bytes: [u8; 8] -} - -#[derive(Serialize, Deserialize, PartialEq, Debug, Clone, Copy)] -#[serde(remote = "Option")] -pub struct OptInstanceIdRemote{ - #[serde(getter = "id_as_bytes")] - bytes: [u8; 8] -} - -fn id_as_bytes(id: &Option) -> [u8; 8] { - id.map(|i| i.as_bytes()).unwrap_or([0; 8]) -} -impl Into> for OptInstanceIdRemote { - fn into(self) -> Option { - let id = FloInstanceId::from_bytes(&self.bytes[..]); - if id == FloInstanceId::null() { - None - } else { - Some(id) + pub fn generate_new() -> PersistentClusterState { + /* + TODO: generate new clusterState with a null FloInstanceId and have the primary assign the real one + Technically, there's a _very_ small chance that two separate instances could generate the same instance id. + The way around this is to have the system primary node just generate and assign all instance ids. This would + change the startup procedure significantly to account for that new step before the node enters a normal state + */ + PersistentClusterState { + current_term: 0, + voted_for: None, + this_instance_id: flo_instance_id::generate_new(), + cluster_members: HashSet::new(), } } } -impl Into for InstanceIdRemote { - fn into(self) -> FloInstanceId { - FloInstanceId::from_bytes(&self.bytes[..]) - } -} - /// Placeholder for a wrapper struct that will take care of persisting the state as it changes #[derive(Debug)] pub struct FilePersistedState { @@ -89,12 +69,7 @@ impl FilePersistedState { (file, state) } else { let file = OpenOptions::new().write(true).read(true).create(true).open(&path)?; // early return on failure - let state = PersistentClusterState { - current_term: 0, - voted_for: None, - this_instance_id: FloInstanceId::generate_new(), - cluster_members: HashSet::new(), - }; + let state = PersistentClusterState::generate_new(); info!("Initialized brand new state: {:?}", state); (file, state) }; @@ -161,15 +136,15 @@ mod test { state.current_term = 9; state.cluster_members = [ Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("127.0.0.1:3456") }, Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("[2001:873::1]:3000") }, Peer { - id: FloInstanceId::generate_new(), + id: flo_instance_id::generate_new(), address: addr("127.0.0.1:456") } ].iter().cloned().collect(); diff --git a/flo-server/src/engine/controller/cluster_state/primary_state.rs b/flo-server/src/engine/controller/cluster_state/primary_state.rs index b5628cf..fe9e93b 100644 --- a/flo-server/src/engine/controller/cluster_state/primary_state.rs +++ b/flo-server/src/engine/controller/cluster_state/primary_state.rs @@ -105,26 +105,6 @@ impl PrimaryState { } } - pub fn append_entries_response(&mut self, peer_id: FloInstanceId, ack_through: Option, - controller: &mut ControllerState, connection_manager: &mut PeerConnectionManager) { - if let Some(new_counter) = ack_through { - // TODO: peer has acknowledged events through this index, tell the partitionImpl about the ack and see if it allows us to commit any events - if log_enabled!(::log::Level::Debug) { - let current = self.peer_positions.get(&peer_id); - debug!("Received acknowledgement from peer_id: {:?}, new_position: {}, old_position: {:?}", peer_id, new_counter, current); - } - self.peer_positions.insert(peer_id, Position::TrackingFrom(new_counter)); - } else { - // If the peer did not acknowledge the last append, then we take their last known position and move it back. - // We move it back by 8, because that's the current batch size, but the actual amount is only a matter of efficiency. - // This amount does not affect correctness. - let current_position = self.get_peer_position(&peer_id); - let new_position = current_position.saturating_sub(8); - debug!("peer_id: {} did NOT acknowledge last append entries, changing position from {} to {}", peer_id, current_position, new_position); - self.peer_positions.insert(peer_id, Position::SetStart(new_position)); - } - } - fn get_peer_position(&self, peer: &FloInstanceId) -> EventCounter { self.peer_positions.get(peer).map(|position| { match *position { diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 85f07c2..0b56db4 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -8,7 +8,6 @@ use protocol::{FloInstanceId, Term}; use engine::event_stream::partition::{self, Operation}; use engine::connection_handler::{ConnectionControl, ConnectionControlSender}; use engine::ConnectionId; -use engine::controller::cluster_state::persistent::InstanceIdRemote; #[derive(Debug, Clone, PartialEq)] pub struct CallRequestVote { @@ -54,7 +53,6 @@ pub struct AppendEntriesResponse { #[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)] pub struct Peer { - #[serde(with = "InstanceIdRemote")] pub id: FloInstanceId, pub address: SocketAddr, } diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 6e09901..93d99cc 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -27,7 +27,7 @@ use self::cluster_state::ConsensusProcessor; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; pub use self::system_stream::SystemStreamRef; -pub use self::system_event::{SystemEvent, SystemEventData}; +pub use self::system_event::{SystemEvent, SystemEventData, SystemEventKind, QualifiedPartitionId}; pub use self::system_reader::{SystemStreamReader, SYSTEM_READER_BATCH_SIZE}; pub use self::controller_messages::*; pub use self::cluster_state::{SharedClusterState, ClusterStateReader}; diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index ec2e90e..1740cd9 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -1,7 +1,7 @@ use std::fmt::Debug; use rmp_serde::decode::Error; -use protocol::Term; +use protocol::{Term, FloInstanceId}; use event::{FloEvent, EventData, FloEventId, OwnedFloEvent, EventCounter, ActorId, Timestamp, time}; use engine::event_stream::partition::PersistentEvent; @@ -98,7 +98,7 @@ pub struct QualifiedPartitionId { #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub enum SystemEventKind { ResignPrimary(QualifiedPartitionId), - AssignPrimary(QualifiedPartitionId), + AssignPrimary(QualifiedPartitionId, FloInstanceId), } diff --git a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs index dfabd04..1f2be90 100644 --- a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs +++ b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs @@ -88,6 +88,7 @@ impl CommitManager { #[cfg(test)] mod test { use super::*; + use protocol::flo_instance_id; fn subject_with_peers(peers: &[FloInstanceId]) -> CommitManager { let mut subject = CommitManager::new(AtomicCounterWriter::with_value(0)); @@ -100,10 +101,10 @@ mod test { #[test] fn commit_index_is_incremented_when_a_majority_of_all_members_have_acknowledged_an_event_greater_than_the_one_acknowledged() { let peers = [ - FloInstanceId::generate_new(), - FloInstanceId::generate_new(), - FloInstanceId::generate_new(), - FloInstanceId::generate_new() + flo_instance_id::generate_new(), + flo_instance_id::generate_new(), + flo_instance_id::generate_new(), + flo_instance_id::generate_new() ]; let mut subject = subject_with_peers(&peers[..]); let commit_reader = subject.get_commit_index_reader(); @@ -126,10 +127,10 @@ mod test { #[test] fn commit_index_is_incremented_when_a_majority_of_all_members_have_acknowledged_a_specific_event() { let peers = [ - FloInstanceId::generate_new(), - FloInstanceId::generate_new(), - FloInstanceId::generate_new(), - FloInstanceId::generate_new() + flo_instance_id::generate_new(), + flo_instance_id::generate_new(), + flo_instance_id::generate_new(), + flo_instance_id::generate_new() ]; let mut subject = subject_with_peers(&peers[..]); @@ -146,25 +147,25 @@ mod test { assert_eq!(0, subject.min_required_for_commit); - subject.add_member(FloInstanceId::generate_new()); + subject.add_member(flo_instance_id::generate_new()); assert_eq!(1, subject.peers.len()); assert_eq!(1, subject.min_required_for_commit); // 2 of 2 - subject.add_member(FloInstanceId::generate_new()); + subject.add_member(flo_instance_id::generate_new()); assert_eq!(2, subject.peers.len()); assert_eq!(1, subject.min_required_for_commit); // 2 of 3 - subject.add_member(FloInstanceId::generate_new()); + subject.add_member(flo_instance_id::generate_new()); assert_eq!(3, subject.peers.len()); assert_eq!(2, subject.min_required_for_commit); // 3 of 4 - subject.add_member(FloInstanceId::generate_new()); + subject.add_member(flo_instance_id::generate_new()); assert_eq!(4, subject.peers.len()); assert_eq!(2, subject.min_required_for_commit); // 3 of 5 - subject.add_member(FloInstanceId::generate_new()); - subject.add_member(FloInstanceId::generate_new()); - subject.add_member(FloInstanceId::generate_new()); + subject.add_member(flo_instance_id::generate_new()); + subject.add_member(flo_instance_id::generate_new()); + subject.add_member(flo_instance_id::generate_new()); assert_eq!(4, subject.min_required_for_commit); // 5 of 8 } diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index 5eb39d4..e30398c 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -618,7 +618,7 @@ mod test { use super::*; use event::{OwnedFloEvent, FloEventId, time::from_millis_since_epoch}; - use protocol::ProduceEvent; + use protocol::{ProduceEvent, flo_instance_id}; use engine::event_stream::partition::{ProduceOperation, EventFilter, PartitionReader}; use engine::event_stream::{EventStreamOptions, HighestCounter}; use engine::ConnectionId; @@ -650,10 +650,10 @@ mod test { status.reader(), HighestCounter::zero()).unwrap(); - let peer_1 = FloInstanceId::generate_new(); - let peer_2 = FloInstanceId::generate_new(); - let peer_3 = FloInstanceId::generate_new(); - let peer_4 = FloInstanceId::generate_new(); + let peer_1 = flo_instance_id::generate_new(); + let peer_2 = flo_instance_id::generate_new(); + let peer_3 = flo_instance_id::generate_new(); + let peer_4 = flo_instance_id::generate_new(); partition.add_replication_node(peer_1); partition.add_replication_node(peer_2); partition.add_replication_node(peer_3); @@ -722,10 +722,10 @@ mod test { status.reader(), HighestCounter::zero()).unwrap(); - let peer_1 = FloInstanceId::generate_new(); - let peer_2 = FloInstanceId::generate_new(); - let peer_3 = FloInstanceId::generate_new(); - let peer_4 = FloInstanceId::generate_new(); + let peer_1 = flo_instance_id::generate_new(); + let peer_2 = flo_instance_id::generate_new(); + let peer_3 = flo_instance_id::generate_new(); + let peer_4 = flo_instance_id::generate_new(); partition.add_replication_node(peer_1); partition.add_replication_node(peer_2); partition.add_replication_node(peer_3); From c21e0e5ba027209f2b420ea91b7ff116ebe4c7b5 Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 9 Jun 2018 14:26:22 -0400 Subject: [PATCH 64/73] cleanup compiler warnings --- flo-bench-cli/src/main.rs | 2 - .../src/async/ops/set_event_stream.rs | 9 +- flo-protocol/src/messages/consume_start.rs | 2 +- flo-protocol/src/messages/flo_instance_id.rs | 20 ---- flo-protocol/src/messages/mod.rs | 2 +- flo-server/src/embedded/mod.rs | 2 +- .../engine/connection_handler/consumer/mod.rs | 2 +- .../src/engine/connection_handler/input.rs | 2 +- .../src/engine/connection_handler/mod.rs | 3 +- .../src/engine/connection_handler/peer/mod.rs | 21 ++-- .../connection_handler/peer/peer_follower.rs | 25 ----- .../engine/controller/cluster_state/mod.rs | 104 ++++++++---------- .../cluster_state/peer_connections.rs | 73 ++++-------- .../controller/cluster_state/persistent.rs | 3 +- .../controller/cluster_state/primary_state.rs | 9 -- .../engine/controller/controller_messages.rs | 4 +- .../src/engine/controller/controller_state.rs | 2 +- .../src/engine/controller/initialization.rs | 9 +- flo-server/src/engine/controller/mod.rs | 13 +-- .../engine/controller/peer_connection/mod.rs | 11 -- .../controller/peer_connection/system.rs | 59 ---------- .../src/engine/controller/system_event.rs | 4 +- .../src/engine/controller/system_stream.rs | 4 +- .../src/engine/controller/tick_generator.rs | 2 +- .../event_stream/partition/controller/mod.rs | 11 +- .../partition/controller/pending_produce.rs | 2 +- .../src/engine/event_stream/partition/ops.rs | 1 - .../partition/segment/persistent_event.rs | 9 -- flo-server/src/engine/mod.rs | 1 - flo-server/src/flo_io/mod.rs | 24 ++-- flo-server/src/server/mod.rs | 6 +- flo-server/src/test_utils.rs | 17 ++- 32 files changed, 129 insertions(+), 329 deletions(-) delete mode 100644 flo-server/src/engine/connection_handler/peer/peer_follower.rs delete mode 100644 flo-server/src/engine/controller/peer_connection/system.rs diff --git a/flo-bench-cli/src/main.rs b/flo-bench-cli/src/main.rs index 1ac352a..84eed2a 100644 --- a/flo-bench-cli/src/main.rs +++ b/flo-bench-cli/src/main.rs @@ -39,8 +39,6 @@ impl FromStr for Metric { type Err = (); fn from_str(s: &str) -> Result { - use std::ascii::AsciiExt; - match s.to_ascii_lowercase().as_ref() { PRODUCE_COMMAND => Ok(Metric::Produce), _ => Err(()) diff --git a/flo-client-lib/src/async/ops/set_event_stream.rs b/flo-client-lib/src/async/ops/set_event_stream.rs index da2ce95..9729e5b 100644 --- a/flo-client-lib/src/async/ops/set_event_stream.rs +++ b/flo-client-lib/src/async/ops/set_event_stream.rs @@ -1,9 +1,8 @@ -use std::fmt::Debug; -use futures::{Future, Async, Poll}; - -use protocol::{self, ProtocolMessage}; -use async::{AsyncConnection, ErrorType, ClientProtocolMessage}; +use async::{AsyncConnection, ErrorType}; use async::ops::{RequestResponse, RequestResponseError}; +use futures::{Async, Future, Poll}; +use protocol::{self, ProtocolMessage}; +use std::fmt::Debug; pub struct SetEventStream { inner: RequestResponse, diff --git a/flo-protocol/src/messages/consume_start.rs b/flo-protocol/src/messages/consume_start.rs index 4d201ba..21c2a2e 100644 --- a/flo-protocol/src/messages/consume_start.rs +++ b/flo-protocol/src/messages/consume_start.rs @@ -12,7 +12,7 @@ pub const CONSUME_UNLIMITED: u64 = 0; bitflags! { pub struct ConsumerFlags: u32 { - const ConsumeUncommitted = 1; + const CONSUME_UNCOMMITTED = 1; } } diff --git a/flo-protocol/src/messages/flo_instance_id.rs b/flo-protocol/src/messages/flo_instance_id.rs index f50b745..7bcdb49 100644 --- a/flo-protocol/src/messages/flo_instance_id.rs +++ b/flo-protocol/src/messages/flo_instance_id.rs @@ -1,7 +1,3 @@ - -use std::fmt::{self, Display}; -use serializer::{Serializer, FloSerialize}; - pub type FloInstanceId = u64; pub const NULL_INSTANCE_ID: FloInstanceId = 0; @@ -14,22 +10,6 @@ pub fn generate_new() -> FloInstanceId { value } -pub fn as_bytes(instance_id: FloInstanceId) -> [u8; 8] { - use ::byteorder::{ByteOrder, BigEndian}; - - let mut bytes = [0; 8]; - BigEndian::write_u64(&mut bytes[..], instance_id); - bytes -} - -pub fn from_bytes(bytes: &[u8]) -> FloInstanceId { - use ::byteorder::{ByteOrder, BigEndian}; - - let b = &bytes[0..8]; - BigEndian::read_u64(b) -} - - named!{pub parse_flo_instance_id, map_res!(::nom::be_u64, |val| { if val == 0 { Err(()) diff --git a/flo-protocol/src/messages/mod.rs b/flo-protocol/src/messages/mod.rs index dcaadfc..0002e32 100644 --- a/flo-protocol/src/messages/mod.rs +++ b/flo-protocol/src/messages/mod.rs @@ -569,7 +569,7 @@ mod test { let vv = vec![FloEventId::new(1, 0)]; let msg = ProtocolMessage::NewStartConsuming(NewConsumerStart { op_id: 3, - options: ConsumerFlags::ConsumeUncommitted, + options: ConsumerFlags::CONSUME_UNCOMMITTED, version_vector: vv, max_events: 1, namespace: "/foo/*".to_owned(), diff --git a/flo-server/src/embedded/mod.rs b/flo-server/src/embedded/mod.rs index 98807c7..e456cc8 100644 --- a/flo-server/src/embedded/mod.rs +++ b/flo-server/src/embedded/mod.rs @@ -6,7 +6,7 @@ use std::fmt::Debug; use std::io; use tokio_core::reactor::{Handle, Remote}; -use futures::{Stream, Sink, StartSend, AsyncSink, Async, Poll}; +use futures::{Stream, Sink, StartSend, Poll}; use protocol::ProtocolMessage; use flo_client_lib::async::{AsyncConnection, MessageReceiver, MessageSender, ClientProtocolMessage}; diff --git a/flo-server/src/engine/connection_handler/consumer/mod.rs b/flo-server/src/engine/connection_handler/consumer/mod.rs index 09de1f5..4140769 100644 --- a/flo-server/src/engine/connection_handler/consumer/mod.rs +++ b/flo-server/src/engine/connection_handler/consumer/mod.rs @@ -83,7 +83,7 @@ impl ConsumerConnectionState { } else { Some(max_events) }; - let consume_uncommitted = options.contains(ConsumerFlags::ConsumeUncommitted); + let consume_uncommitted = options.contains(ConsumerFlags::CONSUME_UNCOMMITTED); match EventFilter::parse(&namespace) { Ok(filter) => { diff --git a/flo-server/src/engine/connection_handler/input.rs b/flo-server/src/engine/connection_handler/input.rs index e0c8405..289304f 100644 --- a/flo-server/src/engine/connection_handler/input.rs +++ b/flo-server/src/engine/connection_handler/input.rs @@ -1,5 +1,5 @@ -use protocol::{FloInstanceId, Term}; +use protocol::Term; use event::EventCounter; use engine::ReceivedProtocolMessage; use engine::controller::{CallRequestVote, VoteResponse, AppendEntriesResponse}; diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 701629d..6efdc45 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -4,7 +4,6 @@ mod producer; mod peer; mod input; -use std::fmt::{self, Debug}; use std::io; #[allow(unused_imports)] @@ -512,7 +511,7 @@ mod test { #[test] fn error_is_returned_when_vote_response_is_unexpected() { - let (mut subject, fixture) = Fixture::create_outgoing_peer_connection(); + let (mut subject, _fixture) = Fixture::create_outgoing_peer_connection(); let error = subject.handle_incoming_message(ProtocolMessage::VoteResponse(RequestVoteResponse { op_id: 7, diff --git a/flo-server/src/engine/connection_handler/peer/mod.rs b/flo-server/src/engine/connection_handler/peer/mod.rs index 8af7572..3fb8087 100644 --- a/flo-server/src/engine/connection_handler/peer/mod.rs +++ b/flo-server/src/engine/connection_handler/peer/mod.rs @@ -1,16 +1,13 @@ -mod peer_follower; -mod system_read_wrapper; - +use engine::ConnectionId; +use engine::controller::{self, Peer, SystemStreamRef}; +use event::OwnedFloEvent; +use protocol::{self, ClusterMember, ErrorKind, ErrorMessage, PeerAnnounce, ProtocolMessage}; +use self::system_read_wrapper::SystemReaderWrapper; use std::collections::VecDeque; - -use event::{EventCounter, OwnedFloEvent, FloEvent}; -use protocol::{self, ProtocolMessage, PeerAnnounce, EventStreamStatus, ClusterMember, ErrorMessage, ErrorKind, FloInstanceId, Term}; -use engine::{ReceivedProtocolMessage, ConnectionId}; -use engine::controller::{self, SystemStreamRef, Peer, SystemEvent, SystemStreamReader, SYSTEM_READER_BATCH_SIZE}; -use engine::event_stream::partition::PersistentEvent; +use super::{CallAppendEntries, ConnectionHandlerResult}; use super::connection_state::ConnectionState; -use super::{ConnectionHandlerResult, CallAppendEntries, AppendEntriesStart}; -use self::system_read_wrapper::SystemReaderWrapper; + +mod system_read_wrapper; #[derive(Debug, Clone, Copy, PartialEq)] @@ -174,7 +171,7 @@ impl PeerConnectionState { let expected_op_id = self.peer_operation_queue.pop_back(); if Some(response.op_id) == expected_op_id { - let protocol::RequestVoteResponse { op_id, term, vote_granted } = response; + let protocol::RequestVoteResponse {term, vote_granted, ..} = response; let controller_message = controller::VoteResponse { term, granted: vote_granted diff --git a/flo-server/src/engine/connection_handler/peer/peer_follower.rs b/flo-server/src/engine/connection_handler/peer/peer_follower.rs deleted file mode 100644 index a30544b..0000000 --- a/flo-server/src/engine/connection_handler/peer/peer_follower.rs +++ /dev/null @@ -1,25 +0,0 @@ - -use event::EventCounter; -use engine::event_stream::partition::PartitionReader; -use super::{ConnectionState, ConnectionHandlerResult}; - -pub struct PeerFollowerState { - last_acknowledged_id: EventCounter, - reader: PartitionReader, -} - - - -impl PeerFollowerState { - pub fn new(last_acknowledged_id: EventCounter, reader: PartitionReader) -> PeerFollowerState { - PeerFollowerState { - last_acknowledged_id, - reader - } - } - - - pub fn send_append_entries(&mut self, connection_state: &mut ConnectionState) -> ConnectionHandlerResult { - unimplemented!() - } -} diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 861ed84..a29fe29 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -1,26 +1,23 @@ -pub mod persistent; -mod peer_connections; -mod primary_state; - -use std::io; -use std::sync::{Arc, RwLock}; -use std::net::SocketAddr; -use std::time::{Instant, Duration}; -use std::path::Path; -use std::collections::{HashMap, HashSet}; - -use event::{EventCounter, ActorId, OwnedFloEvent}; +use atomics::AtomicBoolWriter; use engine::{ConnectionId, EngineRef}; use engine::connection_handler::ConnectionControl; use engine::controller::{CallRequestVote, VoteResponse}; -use protocol::{FloInstanceId, flo_instance_id, Term}; -use atomics::{AtomicBoolWriter, AtomicBoolReader}; -use super::{ClusterOptions, ConnectionRef, Peer, PeerUpgrade, AppendEntriesResponse, ReceiveAppendEntries, ControllerState}; -use super::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator, OutgoingConnectionCreatorImpl}; +use event::{ActorId, EventCounter, OwnedFloEvent}; +use protocol::{flo_instance_id, FloInstanceId, Term}; use self::peer_connections::{PeerConnectionManager, PeerConnections}; +pub use self::persistent::{FilePersistedState, PersistentClusterState}; use self::primary_state::PrimaryState; +use std::collections::HashSet; +use std::io; +use std::net::SocketAddr; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; +use super::{AppendEntriesResponse, ClusterOptions, ControllerState, Peer, PeerUpgrade, ReceiveAppendEntries}; +use super::peer_connection::OutgoingConnectionCreatorImpl; -pub use self::persistent::{FilePersistedState, PersistentClusterState}; +pub mod persistent; +mod peer_connections; +mod primary_state; /// This is a placeholder for a somewhat better error handling when updating the persistent cluster state fails. /// This situation is extremely problematic, since it may be possible to cast two different votes in the same term, if for @@ -155,7 +152,7 @@ impl ClusterManager { fn start_new_election(&mut self) { info!("Starting new election with term: {}", self.persistent.current_term + 1); self.votes_received.clear(); - let result = self.persistent.modify(|state| { + self.persistent.modify(|state| { state.current_term += 1; let my_id = state.this_instance_id; state.voted_for = Some(my_id); @@ -254,10 +251,6 @@ impl ClusterManager { None }) } - - fn apply_committed(&mut self, commit_index: EventCounter, controller_state: &mut ControllerState) -> io::Result<()> { - unimplemented!() - } } impl ConsensusProcessor for ClusterManager { @@ -298,7 +291,7 @@ impl ConsensusProcessor for ClusterManager { self.connection_manager.send_to_peer(candidate_id, ConnectionControl::SendVoteResponse(response)); } - fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse, controller: &mut ControllerState) { + fn vote_response_received(&mut self, _now: Instant, from: ConnectionId, response: VoteResponse, controller: &mut ControllerState) { let peer_id = self.connection_manager.get_peer_id(from); if peer_id.is_none() { error!("Ignoring Vote Response from connection_id: {}: {:?}", from, response); @@ -367,7 +360,7 @@ impl ConsensusProcessor for ClusterManager { trace!("This instance is primary"); self.send_append_entries(controller_state); } - other @ _ => { + _ => { if self.election_timed_out(now) { self.start_new_election(); } @@ -432,7 +425,7 @@ impl ConsensusProcessor for ClusterManager { })); } - fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState) { + fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, _controller_state: &mut ControllerState) { let from = self.connection_manager.get_peer_id(connection_id); if from.is_none() { error!("Received AppendEntriesResponse from connection_id: {}, which is not a peer connection. Received: {:?}", connection_id, response); @@ -454,7 +447,7 @@ impl ConsensusProcessor for ClusterManager { } if response.success.is_some() && self.is_primary() { - let peer_counter = response.success.unwrap(); + let _peer_counter = response.success.unwrap(); // TODO: confirm entries with partition impl } } @@ -519,17 +512,17 @@ pub fn init_cluster_consensus_processor(persistent_state: FilePersistedState, #[cfg(test)] mod test { - use super::*; - use std::path::{Path, PathBuf}; - use std::collections::HashSet; - use tempdir::TempDir; - use engine::connection_handler::{ConnectionControlSender, ConnectionControlReceiver, ConnectionControl, CallAppendEntries, AppendEntriesStart}; - use test_utils::{addr, expect_future_resolved}; - use engine::event_stream::partition::SegmentNum; - use engine::controller::peer_connection::{OutgoingConnectionCreator, MockOutgoingConnectionCreator}; - use engine::controller::cluster_state::peer_connections::mock::{MockPeerConnectionManager, Invocation}; + use engine::connection_handler::{AppendEntriesStart, CallAppendEntries, ConnectionControl}; + use engine::controller::cluster_state::peer_connections::mock::{Invocation, MockPeerConnectionManager}; use engine::controller::controller_messages::mock::mock_connection_ref; + use engine::controller::ConnectionRef; use engine::controller::mock::{MockControllerState, MockSystemEvent}; + use engine::event_stream::partition::SegmentNum; + use std::collections::HashSet; + use std::path::Path; + use super::*; + use tempdir::TempDir; + use test_utils::addr; #[test] fn reverts_to_follower_when_append_entries_response_indicates_term_greater_than_current_term() { @@ -627,7 +620,7 @@ mod test { }, ]; - let (peer_1_tx, peer_1_rx) = ::engine::connection_handler::create_connection_control_channels(); + let (peer_1_tx, _peer_1_rx) = ::engine::connection_handler::create_connection_control_channels(); let mut controller_state = MockControllerState::new() .with_commit_index(2) .with_mocked_events(mock_system_events.as_slice()) @@ -853,7 +846,6 @@ mod test { let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); subject.persistent.modify(|state| { - let this_id = state.this_instance_id; state.cluster_members.insert(peer_1.clone()); state.cluster_members.insert(peer_2.clone()); state.cluster_members.insert(peer_3.clone()); @@ -945,7 +937,6 @@ mod test { let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); subject.persistent.modify(|state| { - let this_id = state.this_instance_id; state.cluster_members.insert(peer_1.clone()); state.cluster_members.insert(peer_2.clone()); state.cluster_members.insert(peer_3.clone()); @@ -1045,7 +1036,7 @@ mod test { #[test] fn vote_is_denied_when_candidate_is_not_a_known_peer() { - vote_test(|subject, request, candidate, peer_2| { + vote_test(|subject, request, _candidate, peer_2| { subject.last_applied_term = request.last_log_term; subject.last_applied = request.last_log_index; subject.persistent.modify(|state| { @@ -1326,7 +1317,6 @@ mod test { } fn vote_test(setup_fun: F) where F: Fn(&mut ClusterManager, &CallRequestVote, Peer, Peer) -> VoteExpectation { - let start = Instant::now(); let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { @@ -1369,36 +1359,36 @@ pub struct NoOpConsensusProcessor; // TODO: reasonable and consistent behavior for calls into NoOpConsensusProcessor that should never actually happen impl ConsensusProcessor for NoOpConsensusProcessor { - fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, controller_state: &ControllerState) { - } - fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { - panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); - } - fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote) { - panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); - } - fn tick(&mut self, now: Instant, controller_state: &mut ControllerState) { + fn tick(&mut self, _now: Instant, _controller_state: &mut ControllerState) { } fn is_primary(&self) -> bool { true } - fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse, controller: &mut ControllerState) { - panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); + fn get_current_term(&self) -> Term { + 0 } - fn append_entries_received(&mut self, connection_id: ConnectionId, append: ReceiveAppendEntries, controller_state: &mut ControllerState) { + fn peer_connection_established(&mut self, _upgrade: PeerUpgrade, _connection_id: ConnectionId, _controller_state: &ControllerState) { + } + fn outgoing_connection_failed(&mut self, _connection_id: ConnectionId, _address: SocketAddr) { panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); } + fn connection_closed(&mut self, _connection_id: ConnectionId) { - fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState) { + } + fn request_vote_received(&mut self, _from: ConnectionId, _request: CallRequestVote) { panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); } - fn connection_closed(&mut self, connection_id: ConnectionId) { + fn vote_response_received(&mut self, _now: Instant, _from: ConnectionId, _response: VoteResponse, _controller: &mut ControllerState) { + panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); + } + fn append_entries_received(&mut self, _connection_id: ConnectionId, _append: ReceiveAppendEntries, _controller_state: &mut ControllerState) { + panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); + } + fn append_entries_response_received(&mut self, _connection_id: ConnectionId, _response: AppendEntriesResponse, _controller_state: &mut ControllerState) { + panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); } fn send_append_entries(&mut self, _controller_state: &mut ControllerState) { // do nothing } - fn get_current_term(&self) -> Term { - 0 - } } diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs index fb23103..a74cec0 100644 --- a/flo-server/src/engine/controller/cluster_state/peer_connections.rs +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -1,15 +1,12 @@ +use engine::connection_handler::ConnectionControl; +use engine::ConnectionId; +use engine::controller::{ConnectionRef, ControllerState, Peer}; +use engine::controller::peer_connection::OutgoingConnectionCreator; +use protocol::FloInstanceId; use std::collections::{HashMap, HashSet}; -use std::net::SocketAddr; -use std::time::{Instant, Duration}; use std::fmt::Debug; -use std::io; - -use event::EventCounter; -use protocol::{FloInstanceId, Term}; -use engine::ConnectionId; -use engine::controller::{ConnectionRef, Peer, CallRequestVote, ControllerState}; -use engine::controller::peer_connection::{PeerSystemConnection, OutgoingConnectionCreator}; -use engine::connection_handler::{ConnectionControl, CallAppendEntries, AppendEntriesStart}; +use std::net::SocketAddr; +use std::time::{Duration, Instant}; pub trait PeerConnectionManager: Send + Debug + 'static { fn establish_connections(&mut self, now: Instant, controller_state: &mut ControllerState); @@ -104,8 +101,7 @@ impl PeerConnectionManager for PeerConnections { } fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { - let PeerConnections {ref mut disconnected_peers, ref mut known_peers, ref mut outgoing_connection_creator, ..} = *self; - if let Some(attempt) = disconnected_peers.get_mut(&address) { + if let Some(attempt) = self.disconnected_peers.get_mut(&address) { // remove the connection if attempt.connection.take().is_some() { // set the attempt time to the time of the failure, since it could take quite some time to get a connection error @@ -125,7 +121,7 @@ impl PeerConnectionManager for PeerConnections { info!("Successfully established connection_id: {} to peer: {:?} at address: {}", success_connection.connection_id, peer, success_connection.remote_address); - if let Some(ConnectionAttempt {connection, attempt_count, ..}) = disconnected { + if let Some(ConnectionAttempt {connection, ..}) = disconnected { if let Some(outgoing_connection_ref) = connection { // if there's an existing attempt to create a connection to this peer, verify that this is indeed // the same connection. if this success is from a different connection than the one we were trying to establish, @@ -183,7 +179,7 @@ impl PeerConnectionManager for PeerConnections { let disconnect = if let Some(connection) = self.known_peers.get_mut(&peer_id) { match &mut connection.state { &mut PeerState::Connected(ref mut connection_ref) => { - let result = connection_ref.control_sender.send(control); + let result = connection_ref.control_sender.unbounded_send(control); if result.is_err() { info!("Failed to send ConnectionControl to peer connection handler for {:?}, connection_id: {}, closing connection", peer_id, connection_ref.connection_id); Some(connection_ref.connection_id) @@ -242,11 +238,6 @@ impl ConnectionAttempt { now >= self.attempt_time && (now - self.attempt_time) >= time_to_wait } - fn failed(&mut self, time: Instant) { - self.attempt_time = time; - self.attempt_count += 1; - self.connection = None; - } } #[derive(Debug)] @@ -263,18 +254,6 @@ impl Connection { } } - fn send_if_connected(&mut self, control: ConnectionControl) -> Result<(), ()> { - match self.state { - PeerState::Connected(ref mut connection_ref) => { - send(connection_ref, control) - } - _ => { - debug!("Not sending control to peer: {:?} because it is not connected, dropping message: {:?}", self, control); - Ok(()) - } - } - } - fn get_connection_id(&self) -> Option { match self.state { PeerState::Connected(ref conn) => Some(conn.connection_id), @@ -282,13 +261,6 @@ impl Connection { } } - fn is_connected(&self) -> bool { - match self.state { - PeerState::Connected(_) => true, - _ => false - } - } - fn connection_established(&mut self, connection_ref: ConnectionRef) -> Option { let prev_conn = match self.state { PeerState::Connected(ref existing) => Some(existing.connection_id), @@ -303,21 +275,20 @@ impl Connection { enum PeerState { Init, ConnectionFailed, - OutgoingConnectAttempt, Connected(ConnectionRef), } #[cfg(test)] mod test { - use super::*; - use std::time::Duration; - use engine::controller::peer_connection::MockOutgoingConnectionCreator; + use engine::connection_handler::{CallAppendEntries, ConnectionControlReceiver}; + use engine::controller::CallRequestVote; use engine::controller::mock::MockControllerState; - use engine::controller::ControllerState; - use engine::connection_handler::ConnectionControlReceiver; - use test_utils::{addr, expect_future_resolved}; + use engine::controller::peer_connection::MockOutgoingConnectionCreator; use protocol::flo_instance_id; + use std::time::Duration; + use super::*; + use test_utils::{addr, expect_future_resolved}; fn assert_control_sent(rx: ConnectionControlReceiver, expected: &ConnectionControl) -> ConnectionControlReceiver { use futures::Stream; @@ -348,9 +319,9 @@ mod test { address: addr("123.4.5.6:3000") }; let mut creator = MockOutgoingConnectionCreator::new(); - let (peer_conn, rx) = creator.stub(peer.address); + let (_peer_conn, rx) = creator.stub(peer.address); - let (mut subject, connections) = subject_with_connected_peers(&[peer], creator); + let (mut subject, _connections) = subject_with_connected_peers(&[peer], creator); let rx = assert_control_sent(rx, &ConnectionControl::InitiateOutgoingSystemConnection); let expected = ConnectionControl::SendAppendEntries(CallAppendEntries { @@ -373,9 +344,9 @@ mod test { address: addr("123.4.5.6:4000") }; let mut creator = MockOutgoingConnectionCreator::new(); - let (peer_1_conn, rx_1) = creator.stub(peer_1.address); - let (peer_2_conn, rx_2) = creator.stub(peer_2.address); - let (mut subject, connections) = subject_with_connected_peers(&[peer_1, peer_2], creator); + let (_peer_1_conn, rx_1) = creator.stub(peer_1.address); + let (_peer_2_conn, rx_2) = creator.stub(peer_2.address); + let (mut subject, _connections) = subject_with_connected_peers(&[peer_1, peer_2], creator); let rx_1 = assert_control_sent(rx_1, &ConnectionControl::InitiateOutgoingSystemConnection); let rx_2 = assert_control_sent(rx_2, &ConnectionControl::InitiateOutgoingSystemConnection); @@ -398,7 +369,7 @@ mod test { let peer_id = flo_instance_id::generate_new(); let mut creator = MockOutgoingConnectionCreator::new(); - let (peer_connection, rx) = creator.stub(peer_address); + let (peer_connection, _rx) = creator.stub(peer_address); let mut subject = PeerConnections::new(vec![peer_address], creator.boxed(), &HashSet::new()); diff --git a/flo-server/src/engine/controller/cluster_state/persistent.rs b/flo-server/src/engine/controller/cluster_state/persistent.rs index 2bcc27b..6d88cdd 100644 --- a/flo-server/src/engine/controller/cluster_state/persistent.rs +++ b/flo-server/src/engine/controller/cluster_state/persistent.rs @@ -4,11 +4,10 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Seek, SeekFrom}; use std::collections::HashSet; -use event::EventCounter; use protocol::Term; use protocol::flo_instance_id::{self, FloInstanceId}; use engine::controller::controller_messages::Peer; -use super::{ClusterOptions, SharedClusterState}; +use super::SharedClusterState; /// Holds all the cluster state that we want to survive a reboot. /// We always persist the `FloInstanceId` because we prefer that to be stable across reboots. We do _not_ want to persist diff --git a/flo-server/src/engine/controller/cluster_state/primary_state.rs b/flo-server/src/engine/controller/cluster_state/primary_state.rs index fe9e93b..cad8106 100644 --- a/flo-server/src/engine/controller/cluster_state/primary_state.rs +++ b/flo-server/src/engine/controller/cluster_state/primary_state.rs @@ -104,13 +104,4 @@ impl PrimaryState { } } - - fn get_peer_position(&self, peer: &FloInstanceId) -> EventCounter { - self.peer_positions.get(peer).map(|position| { - match *position { - Position::SetStart(start) => start, - Position::TrackingFrom(last) => last - } - }).unwrap_or(0) - } } diff --git a/flo-server/src/engine/controller/controller_messages.rs b/flo-server/src/engine/controller/controller_messages.rs index 0b56db4..377f3e1 100644 --- a/flo-server/src/engine/controller/controller_messages.rs +++ b/flo-server/src/engine/controller/controller_messages.rs @@ -1,12 +1,10 @@ use std::net::SocketAddr; use std::time::Instant; -use futures::sync::mpsc::UnboundedSender; - use event::{EventCounter, OwnedFloEvent}; use protocol::{FloInstanceId, Term}; use engine::event_stream::partition::{self, Operation}; -use engine::connection_handler::{ConnectionControl, ConnectionControlSender}; +use engine::connection_handler::ConnectionControlSender; use engine::ConnectionId; #[derive(Debug, Clone, PartialEq)] diff --git a/flo-server/src/engine/controller/controller_state.rs b/flo-server/src/engine/controller/controller_state.rs index bf68a91..c8a4791 100644 --- a/flo-server/src/engine/controller/controller_state.rs +++ b/flo-server/src/engine/controller/controller_state.rs @@ -211,7 +211,7 @@ pub mod mock { }) } - fn add_system_replication_node(&mut self, peer: FloInstanceId) { + fn add_system_replication_node(&mut self, _peer: FloInstanceId) { unimplemented!() } } diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index f65818f..4aef7b5 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -8,11 +8,10 @@ use std::net::SocketAddr; use tokio_core::reactor::Remote; use engine::{EngineRef, system_stream_name, SYSTEM_STREAM_NAME}; -use engine::controller::{SystemPartitionSender, SystemPartitionReceiver, SystemOperation}; +use engine::controller::{SystemPartitionSender, SystemPartitionReceiver}; use engine::controller::cluster_state::{init_cluster_consensus_processor, ConsensusProcessor, NoOpConsensusProcessor, - ClusterManager, ClusterStateReader, SystemPrimaryAddressRef, FilePersistedState}; @@ -22,11 +21,7 @@ use engine::event_stream::{EventStreamRefMut, init_existing_event_stream, init_new_event_stream}; -use engine::event_stream::partition::{PartitionSender, - PartitionReceiver, - PartitionRef, - SharedReaderRefs, - create_partition_channels}; +use engine::event_stream::partition::{PartitionRef, SharedReaderRefs}; use engine::event_stream::partition::controller::PartitionImpl; use atomics::{AtomicBoolReader, AtomicBoolWriter, AtomicCounterReader}; use event_loops::LoopHandles; diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 93d99cc..459694f 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -9,18 +9,17 @@ mod system_reader; mod controller_state; use std::path::PathBuf; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use std::collections::HashMap; use std::io; use std::time::Instant; use protocol::{ProduceEvent, Term}; -use event::EventCounter; use engine::ConnectionId; use engine::event_stream::{EventStreamRef, EventStreamRefMut, EventStreamOptions}; -use engine::event_stream::partition::{self, PersistentEvent, IndexEntry, SegmentNum}; +use engine::event_stream::partition; use engine::event_stream::partition::controller::PartitionImpl; use self::cluster_state::ConsensusProcessor; @@ -124,7 +123,7 @@ impl FloController { } -fn handle_partition_op(connection_id: ConnectionId, op_start_time: Instant, op: partition::OpType, +fn handle_partition_op(connection_id: ConnectionId, _op_start_time: Instant, op: partition::OpType, cluster_state: &mut ConsensusProcessor, controller_state: &mut ControllerStateImpl) { use engine::event_stream::partition::OpType::*; match op { @@ -155,7 +154,7 @@ fn produce_system_events(mut produce_op: partition::ProduceOperation, cluster_st if let Err(err) = validate_result { // We're done here - produce_op.client.complete(Err(err)); + let _ = produce_op.client.send(Err(err)); } else { // hand off the modified operation to the partition, which will complete it let result = controller_state.system_partition.handle_produce(produce_op); @@ -168,12 +167,12 @@ fn produce_system_events(mut produce_op: partition::ProduceOperation, cluster_st } } else { let err = io::Error::new(io::ErrorKind::Other, "Not primary"); - produce_op.client.complete(Err(err)); + let _ = produce_op.client.send(Err(err)); } } -fn validate_system_event(events: &Vec, term: Term) -> io::Result<()> { +fn validate_system_event(events: &Vec, _term: Term) -> io::Result<()> { for event in events.iter() { if event.data.is_empty() { return Err(io::Error::new(io::ErrorKind::InvalidData, "Event has no body")); diff --git a/flo-server/src/engine/controller/peer_connection/mod.rs b/flo-server/src/engine/controller/peer_connection/mod.rs index fa25c5a..d217143 100644 --- a/flo-server/src/engine/controller/peer_connection/mod.rs +++ b/flo-server/src/engine/controller/peer_connection/mod.rs @@ -1,10 +1,5 @@ -mod system; - -use std::collections::HashMap; use std::fmt::Debug; use std::net::SocketAddr; -use std::io; -use tokio_core::reactor::Handle; use tokio_core::net::TcpStream; use futures::Future; @@ -13,12 +8,6 @@ use engine::connection_handler::{create_connection_control_channels, ConnectionC use engine::controller::ConnectionRef; use event_loops::LoopHandles; use flo_io::create_connection_handler; -use self::system::PeerConnectionImpl; - -pub use self::system::{PeerSystemConnection, PendingSystemConnection}; - -pub type ConnectionSendResult = Result<(), T>; - /// Trait for creating outgoing connections (clever name, I know). pub trait OutgoingConnectionCreator: Debug + Send + 'static { diff --git a/flo-server/src/engine/controller/peer_connection/system.rs b/flo-server/src/engine/controller/peer_connection/system.rs deleted file mode 100644 index ad9046d..0000000 --- a/flo-server/src/engine/controller/peer_connection/system.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::fmt::Debug; -use std::io; -use std::net::SocketAddr; - -use tokio_core::reactor::Handle; -use tokio_core::net::TcpStream; -#[allow(deprecated)] -use tokio_core::io::Io; -use futures::{Future, Stream, Sink}; - -use protocol::{FloInstanceId, Term}; -use event::EventCounter; -use event_loops::LoopHandles; -use engine::ConnectionId; -use engine::controller::SystemStreamRef; -use engine::connection_handler::ConnectionControlSender; - -use super::ConnectionSendResult; - -pub struct CallAppendEntries { - pub leader_id: FloInstanceId, - pub term: Term, - pub prev_entry_term: Term, - pub prev_entry_index: EventCounter, - pub leader_commit_index: EventCounter, -} - -pub trait PendingSystemConnection: Debug + Send + 'static { - fn connection_id(&self) -> ConnectionId; - - fn complete(self: Box, id: FloInstanceId) -> Box; -} - -/// Trait representing an active peer connection, with functions to control it -pub trait PeerSystemConnection: Debug + Send + 'static { - fn connection_id(&self) -> ConnectionId; -} - - - - -#[derive(Debug)] -pub struct PeerConnectionImpl { - connection_id: ConnectionId, - sender: ConnectionControlSender -} - -impl PeerConnectionImpl { - pub fn new(connection_id: ConnectionId, sender: ConnectionControlSender) -> PeerConnectionImpl { - PeerConnectionImpl { connection_id, sender } - } -} - -impl PeerSystemConnection for PeerConnectionImpl { - fn connection_id(&self) -> ConnectionId { - self.connection_id - } -} - diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index 1740cd9..3a613e4 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -1,8 +1,7 @@ -use std::fmt::Debug; use rmp_serde::decode::Error; use protocol::{Term, FloInstanceId}; -use event::{FloEvent, EventData, FloEventId, OwnedFloEvent, EventCounter, ActorId, Timestamp, time}; +use event::{FloEvent, EventData, FloEventId, OwnedFloEvent, ActorId, Timestamp}; use engine::event_stream::partition::PersistentEvent; #[derive(Debug, PartialEq)] @@ -118,6 +117,7 @@ impl SystemEventData { #[cfg(test)] mod test { use super::*; + use event::time; #[test] fn from_event_returns_error_when_event_data_cannot_be_deserialized() { diff --git a/flo-server/src/engine/controller/system_stream.rs b/flo-server/src/engine/controller/system_stream.rs index 5371b1f..8cf1550 100644 --- a/flo-server/src/engine/controller/system_stream.rs +++ b/flo-server/src/engine/controller/system_stream.rs @@ -1,12 +1,10 @@ use std::net::SocketAddr; -use protocol::FloInstanceId; -use engine::event_stream::partition::{PartitionRef, Operation, SharedReaderRefs, PartitionReader, SegmentNum}; +use engine::event_stream::partition::{PartitionRef, SharedReaderRefs}; use engine::event_stream::EventStreamRef; use engine::controller::{SystemPartitionSender, SystemOperation, ConnectionRef, Peer, CallRequestVote, VoteResponse, SystemStreamReader, ReceiveAppendEntries, AppendEntriesResponse}; use engine::controller::cluster_state::{SharedClusterState, ClusterStateReader}; use engine::ConnectionId; -use atomics::{AtomicBoolReader, AtomicCounterReader}; #[derive(Clone, Debug)] diff --git a/flo-server/src/engine/controller/tick_generator.rs b/flo-server/src/engine/controller/tick_generator.rs index f0bf020..eca494b 100644 --- a/flo-server/src/engine/controller/tick_generator.rs +++ b/flo-server/src/engine/controller/tick_generator.rs @@ -6,7 +6,7 @@ use futures::{Stream, Future}; use rand::distributions::{Range, IndependentSample}; use rand::thread_rng; -use engine::controller::{SystemPartitionSender, SystemStreamRef}; +use engine::controller::SystemStreamRef; #[derive(Debug)] enum TickError { diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index e30398c..593166a 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -229,7 +229,8 @@ impl PartitionImpl { } // returns unit, since we have to send a result back to the connection handler, regardless of outcome - fn handle_replicate(&mut self, ReplicateOperation{op_id, client_sender, events, prev_event_counter, prev_event_term}: ReplicateOperation) { + fn handle_replicate(&mut self, ReplicateOperation{op_id, client_sender, events, ..}: ReplicateOperation) { + // TODO: ensure that prev_event_counter and prev_event_term are checked for system event replication // TODO: think about handling empty `events` vec. maybe best to handle that in the connection handler? // TODO: ensure that we are in Follower status and that the events came from the leader let result = self.replicate_events(op_id, events); @@ -242,13 +243,12 @@ impl PartitionImpl { highest_event_counter: head, } }); - client_sender.complete(response); + let _ = client_sender.send(response); } /// persists events in this partition. Panics if `events` is empty pub fn replicate_events(&mut self, op_id: u32, events: Vec) -> io::Result { let commit_index = self.commit_manager.get_commit_index(); - let current_head = self.index.greatest_event_counter(); // ignore any events with id.event_counter < commit_index // We don't bother checking CRCs for these, since they are already committed @@ -303,7 +303,6 @@ impl PartitionImpl { } let persisted_event = persisted_event_result?; // return the io error if we failed to read the event we already have - let existing_event_id = persisted_event.id(); let new_event = &to_replicate[index_of_first_to_replicate]; // safe since we check the index bounds above @@ -370,14 +369,14 @@ impl PartitionImpl { Ok(id) => { if self.commit_manager.is_standalone() { // No biggie if the receiving end has hung up already. The operation will still be considered complete and successful - let _ = client.complete(Ok(id)); + let _ = client.send(Ok(id)); } else { self.pending_produce_operations.add(op_id, id.event_counter, client); } } Err(io_err) => { error!("Failed to handle produce operation for op_id: {}, err: {:?}", op_id, io_err); - let _ = client.complete(Err(io_err)); + let _ = client.send(Err(io_err)); } } // TODO: separate out error that gets returned to connection handler so that we can also return an io::Error from this function if one occurs diff --git a/flo-server/src/engine/event_stream/partition/controller/pending_produce.rs b/flo-server/src/engine/event_stream/partition/controller/pending_produce.rs index a22637e..00c3edb 100644 --- a/flo-server/src/engine/event_stream/partition/controller/pending_produce.rs +++ b/flo-server/src/engine/event_stream/partition/controller/pending_produce.rs @@ -42,7 +42,7 @@ impl PendingProduceOperations { for _ in 0..count { let op = self.pending.pop_front().unwrap(); debug!("Notifying producer of committed op_id: {}, event: {}", op.op_id, op.event); - let _ = op.sender.complete(Ok(FloEventId::new(partition, op.event))); + let _ = op.sender.send(Ok(FloEventId::new(partition, op.event))); } } diff --git a/flo-server/src/engine/event_stream/partition/ops.rs b/flo-server/src/engine/event_stream/partition/ops.rs index d79d2a3..cdc6b55 100644 --- a/flo-server/src/engine/event_stream/partition/ops.rs +++ b/flo-server/src/engine/event_stream/partition/ops.rs @@ -2,7 +2,6 @@ use std::io; use std::fmt::{self, Debug}; use std::time::Instant; -use std::net::SocketAddr; use futures::sync::oneshot; diff --git a/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs b/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs index 34e1426..8693ffc 100644 --- a/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs +++ b/flo-server/src/engine/event_stream/partition/segment/persistent_event.rs @@ -37,10 +37,6 @@ mod offsets { } pub const DATA_LEN_LEN: usize = 4; - pub fn data_bytes_start(namespace_len: usize) -> usize { - data_len_start(namespace_len) + 4 - } - pub const MIN_EVENT_LEN: usize = NS_BYTES_START + DATA_LEN_LEN; } @@ -159,11 +155,6 @@ impl PersistentEvent { Ok((FloEventId::new(partition_num, counter), total_len)) } - fn read_value_unchecked(&self, start: usize, len: usize, fun: F) -> V where F: Fn(&[u8]) -> V { - let buffer_slice = self.as_buf(start, len); - fun(buffer_slice) - } - fn as_buf(&self, start: usize, len: usize) -> &[u8] { &self.raw_data.get_read_slice(self.file_offset + start)[..len] } diff --git a/flo-server/src/engine/mod.rs b/flo-server/src/engine/mod.rs index cad401b..c63d2d1 100644 --- a/flo-server/src/engine/mod.rs +++ b/flo-server/src/engine/mod.rs @@ -5,7 +5,6 @@ pub mod controller; use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::sync::atomic::{AtomicUsize}; -use std::net::SocketAddr; use protocol::ProtocolMessage; use event::{OwnedFloEvent, ActorId}; diff --git a/flo-server/src/flo_io/mod.rs b/flo-server/src/flo_io/mod.rs index b5aa31d..3d490a6 100644 --- a/flo-server/src/flo_io/mod.rs +++ b/flo-server/src/flo_io/mod.rs @@ -1,22 +1,18 @@ -mod client_message_stream; -mod server_message_stream; - - -use std::net::SocketAddr; +use engine::{ConnectionHandler, create_client_channels}; +use engine::{ConnectionId, EngineRef}; +use engine::connection_handler::ConnectionControlReceiver; +use futures::{Future, Sink, Stream}; +pub use self::client_message_stream::ProtocolMessageStream; +pub use self::server_message_stream::ServerMessageStream; use std::io; +use std::net::SocketAddr; #[allow(deprecated)] use tokio_core::io::Io; -use tokio_core::reactor::Handle; use tokio_core::net::TcpStream; -use futures::{Stream, Sink, Future}; - -use engine::{create_client_channels, ConnectionHandler}; -use engine::connection_handler::{ConnectionHandlerInput, ConnectionControlReceiver}; -use engine::{EngineRef, ConnectionId}; - -pub use self::client_message_stream::ProtocolMessageStream; -pub use self::server_message_stream::ServerMessageStream; +use tokio_core::reactor::Handle; +mod client_message_stream; +mod server_message_stream; pub fn create_connection_handler(client_handle: Handle, diff --git a/flo-server/src/server/mod.rs b/flo-server/src/server/mod.rs index 35f356a..5e1cbd0 100644 --- a/flo-server/src/server/mod.rs +++ b/flo-server/src/server/mod.rs @@ -1,17 +1,15 @@ mod server_options; - use std::net::{SocketAddr, Ipv4Addr, SocketAddrV4}; use std::io; use tokio_core::reactor::Remote; use tokio_core::net::{TcpStream, TcpListener}; -use futures::{Stream, Future}; +use futures::Stream; use engine::{ControllerOptions, EngineRef, ClusterOptions, - start_controller, - system_stream_name}; + start_controller}; use engine::connection_handler::create_connection_control_channels; use engine::controller::ConnectionRef; use engine::event_stream::EventStreamOptions; diff --git a/flo-server/src/test_utils.rs b/flo-server/src/test_utils.rs index fa44c5a..84bc44e 100644 --- a/flo-server/src/test_utils.rs +++ b/flo-server/src/test_utils.rs @@ -1,9 +1,8 @@ use std::net::SocketAddr; -use std::time::{Duration, Instant}; use std::sync::{Once, ONCE_INIT}; -use futures::executor::{spawn, Unpark}; +use futures::executor::{spawn, Notify}; use futures::{Future, Async}; static LOGGER_INIT: Once = ONCE_INIT; @@ -18,19 +17,19 @@ pub fn addr(string: &str) -> SocketAddr { ::std::str::FromStr::from_str(string).unwrap() } -/// used for testing futures when we don't expect the future to need unparked -struct NoOpUnpark; -impl Unpark for NoOpUnpark { - fn unpark(&self) { - unimplemented!() +/// used for testing futures when we don't expect the future to need Notified +struct NoOpNotify; +impl Notify for NoOpNotify { + fn notify(&self, _id: usize) { + panic!("Expected that this future should not be notified"); } } pub fn expect_future_resolved(future: F) -> Result where F: Future { let mut s = spawn(future); - let unpark = ::std::sync::Arc::new(NoOpUnpark); + let unpark = ::std::sync::Arc::new(NoOpNotify); loop { - let result = s.poll_future(unpark.clone()); + let result = s.poll_future_notify(&unpark, 0); match result { Ok(Async::Ready(t)) => { return Ok(t); From c09e15fcfc1a69a1878bd9bda4c35328c3757cdb Mon Sep 17 00:00:00 2001 From: pfried Date: Sat, 9 Jun 2018 18:56:40 -0400 Subject: [PATCH 65/73] prevent users from producing invalid system events --- flo-server/src/engine/controller/mod.rs | 39 ++++++++++--------- .../src/engine/controller/system_event.rs | 38 +++++++++++++----- 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 459694f..a4292fb 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -26,7 +26,7 @@ use self::cluster_state::ConsensusProcessor; pub use self::initialization::{start_controller, ControllerOptions, ClusterOptions}; pub use self::system_stream::SystemStreamRef; -pub use self::system_event::{SystemEvent, SystemEventData, SystemEventKind, QualifiedPartitionId}; +pub use self::system_event::{SystemEvent, SystemEventData, SystemEventKind}; pub use self::system_reader::{SystemStreamReader, SYSTEM_READER_BATCH_SIZE}; pub use self::controller_messages::*; pub use self::cluster_state::{SharedClusterState, ClusterStateReader}; @@ -147,22 +147,25 @@ fn handle_partition_op(connection_id: ConnectionId, _op_start_time: Instant, op: } } -fn produce_system_events(mut produce_op: partition::ProduceOperation, cluster_state: &mut ConsensusProcessor, controller_state: &mut ControllerStateImpl) { +fn produce_system_events(produce_op: partition::ProduceOperation, cluster_state: &mut ConsensusProcessor, controller_state: &mut ControllerStateImpl) { if cluster_state.is_primary() { let term = cluster_state.get_current_term(); - let validate_result = validate_system_event(&mut produce_op.events, term); - - if let Err(err) = validate_result { - // We're done here - let _ = produce_op.client.send(Err(err)); - } else { - // hand off the modified operation to the partition, which will complete it - let result = controller_state.system_partition.handle_produce(produce_op); - if let Err(partition_err) = result { - error!("Partition error creating new system events: {:?}", partition_err); - } else { - // success! send out AppendEntries - cluster_state.send_append_entries(controller_state) + + match validate_system_event(&produce_op.events, term) { + Ok(()) => { + // hand off the modified operation to the partition, which will complete it + + let result = controller_state.system_partition.handle_produce(produce_op); + if let Err(partition_err) = result { + error!("Partition error creating new system events: {:?}", partition_err); + } else { + // success! send out AppendEntries + cluster_state.send_append_entries(controller_state) + } + } + Err(err) => { + // We're done here + let _ = produce_op.client.send(Err(err)); } } } else { @@ -172,11 +175,9 @@ fn produce_system_events(mut produce_op: partition::ProduceOperation, cluster_st } -fn validate_system_event(events: &Vec, _term: Term) -> io::Result<()> { +fn validate_system_event(events: &Vec, term: Term) -> io::Result<()> { for event in events.iter() { - if event.data.is_empty() { - return Err(io::Error::new(io::ErrorKind::InvalidData, "Event has no body")); - } + self::system_event::validate_data(event.data.as_slice(), term)?; } Ok(()) } diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index 3a613e4..701b0f2 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -20,6 +20,7 @@ impl SystemEvent { }) } + pub fn term(&self) -> Term { self.system_data().term } @@ -29,6 +30,21 @@ impl SystemEvent { } } +pub fn validate_data(event_data: &[u8], term: Term) -> ::std::io::Result<()> { + use std::io; + ::rmp_serde::decode::from_slice::(event_data).map_err(|serde_err| { + io::Error::new(io::ErrorKind::InvalidData, format!("Invalid system event body: {}", serde_err)) + }).and_then(|system_data| { + if system_data.term == term { + Ok(()) + } else { + let msg = format!("Refusing to produce event with term: {} because current term of {} is greater", + system_data.term, term); + Err(io::Error::new(io::ErrorKind::InvalidData, msg)) + } + }) +} + impl Into for SystemEvent { fn into(self) -> PersistentEvent { self.wrapped @@ -89,18 +105,23 @@ impl FloEvent for SystemEvent { #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] -pub struct QualifiedPartitionId { - pub event_stream: String, - pub partition: ActorId, +pub struct ClusterMemberJoining { + new_member: FloInstanceId, } #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] -pub enum SystemEventKind { - ResignPrimary(QualifiedPartitionId), - AssignPrimary(QualifiedPartitionId, FloInstanceId), +pub struct ClusterMemberJoined { + new_member: FloInstanceId, + new_member_partition_num: ActorId, } +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub enum SystemEventKind { + NewClusterMemberJoining(ClusterMemberJoining), + NewClusterMemberJoined(ClusterMemberJoined), +} + #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub struct SystemEventData { pub term: Term, @@ -131,9 +152,8 @@ mod test { fn system_event_data_is_serialized_and_deserialized_inside_system_event() { let data = SystemEventData { term: 33, - kind: SystemEventKind::ResignPrimary(QualifiedPartitionId { - event_stream: "whatever".to_owned(), - partition: 4 + kind: SystemEventKind::NewClusterMemberJoining(ClusterMemberJoining { + new_member: 555, }) }; let id = FloEventId::new(3, 4); From dbe87d015aa0aa0e2e83c9a86e75dc8ce1e26dc5 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 10 Jun 2018 09:45:36 -0400 Subject: [PATCH 66/73] add script to start cluster with multiple members --- dev-start-cluster.sh | 145 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100755 dev-start-cluster.sh diff --git a/dev-start-cluster.sh b/dev-start-cluster.sh new file mode 100755 index 0000000..90445c7 --- /dev/null +++ b/dev-start-cluster.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +USAGE=$(cat < Sets the root directory, under which all data directories will be created +-p Starting number of a port range that instances will listen on. Will use ports sequentially from there. +-a Adds an additional argument to all flo instances started +-x Do not start any servers (will only print the flo commands that would have been run). Can be used to delete data and logs without starting servers +EOF +) +NUM_INSTANCES=3 +PORT_NUM_START=3000 +DATA_ROOT="${TMPDIR}" +DELETE_DATA_DIRS="no" +ADDITIONAL_ARGS=() +DRY_RUN="no" + +PIDS=() + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +FLO_EXE_PATH="${SCRIPT_DIR}/target/debug/flo" +IP_ADDR="127.0.0.1" + +function stderr() { + echo "$@" >&2 +} + +function start_instance() { + local instance_num="$1" + shift + local peers="$@" + peers=($(echo "${peers[@]//$instance_num}")) + stderr "instance_num: ${instance_num} filtered peers: ${peers[@]}" + + local peer_ports=() + for peer in "${peers[@]}"; do + peer_ports+=($(($PORT_NUM_START + $peer))) + done + + local this_port=$(($PORT_NUM_START + $instance_num)) + local this_addr="${IP_ADDR}:${this_port}" + local peer_addr_args="${peer_ports[@]/#/-P $IP_ADDR:}" + + stderr "this_port='${this_port}', this_addr='${this_addr}', peer_args='${peer_addr_args}'" + local command="${FLO_EXE_PATH} -d ${DATA_ROOT}/flo${instance_num} -A ${this_addr} -p ${this_port} ${peer_addr_args} ${ADDITIONAL_ARGS[@]} -o ${DATA_ROOT}/flo${instance_num}.log" + stderr "command: $command" + + if [[ "${DRY_RUN}" == "no" ]]; then + $command & + local pid="$!" + PIDS+=("${pid}") + stderr "Started flo process with pid: ${pid}" + fi +} + +function delete_data_dirs() { + for instance_num in $@; do + stderr "Deleting data and log for instance: ${instance_num}" + local dir="${DATA_ROOT}/flo${instance_num}" + local log="${dir}.log" + stderr "Deleting dir: '${dir}'" + stderr "Deleting log file: '${log}'" + if [[ -d "${dir}" ]]; then + rm -R "${dir}" + fi + if [[ -f "${log}" ]]; then + rm "${log}" + fi + done +} + +function tail_logs() { + local peers=$@ + local peer_logs=() + for peer in $peers; do + peer_logs+=( "${DATA_ROOT}/flo${peer}.log" ) + done + + tail -f "${peer_logs[@]}" +} + +function stop_all() { + for pid in ${PIDS[@]}; do + stderr "Stopping flo pid: ${pid}" + kill "$pid" || true + done +} + +function start_all() { + local all_peers=($(seq 0 1 "$((NUM_INSTANCES - 1))")) + stderr "All_peers: ${all_peers[@]}" + + # first stop any running instances + stop_all + + if [[ "${DELETE_DATA_DIRS}" == "yes" ]]; then + delete_data_dirs "${all_peers[@]}" + fi + + for instance_num in "${all_peers[@]}"; do + stderr "loop instance num: ${instance_num}" + start_instance "$instance_num" "${all_peers[@]}" + done + + if [[ "${DRY_RUN}" == "no" ]]; then + tail_logs "${all_peers[@]}" + fi +} + + +while getopts "n:p:d:a:rx" opt; do + case $opt in + n) + NUM_INSTANCES="$OPTARG" + stderr "Starting ${num_instances}" >&2 + ;; + p) + PORT_NUM_START="$OPTARG" + stderr "Starting port number is: ${OPTARG}" + ;; + d) + DATA_ROOT="$OPTARG" + stderr "Data root is: ${OPTARG}" + ;; + a) + ADDITIONAL_ARGS+=("$OPTARG") + stderr "Using additional argument: '${OPTARG}'" + ;; + r) + DELETE_DATA_DIRS="yes" + stderr "Will delete data directories" + ;; + x) + DRY_RUN="yes" + stderr "Will be a dry run" + ;; + \?) + stderr "Invalid option: -$OPTARG" >&2 + ;; + esac +done + +trap stop_all INT +start_all \ No newline at end of file From 51b3c58078e5ef90aa296f0d20561e958231ce1a Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 10 Jun 2018 10:51:21 -0400 Subject: [PATCH 67/73] minor improvements to dev-start-cluster.sh --- dev-start-cluster.sh | 58 +++++++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/dev-start-cluster.sh b/dev-start-cluster.sh index 90445c7..710dbf9 100755 --- a/dev-start-cluster.sh +++ b/dev-start-cluster.sh @@ -1,12 +1,16 @@ #!/bin/bash USAGE=$(cat < Sets the root directory, under which all data directories will be created -p Starting number of a port range that instances will listen on. Will use ports sequentially from there. -a Adds an additional argument to all flo instances started +-r Remove existing data directories and log files before starting the cluster -x Do not start any servers (will only print the flo commands that would have been run). Can be used to delete data and logs without starting servers +-b Build all targets before starting cluster EOF ) NUM_INSTANCES=3 @@ -15,6 +19,7 @@ DATA_ROOT="${TMPDIR}" DELETE_DATA_DIRS="no" ADDITIONAL_ARGS=() DRY_RUN="no" +BUILD="no" PIDS=() @@ -31,7 +36,6 @@ function start_instance() { shift local peers="$@" peers=($(echo "${peers[@]//$instance_num}")) - stderr "instance_num: ${instance_num} filtered peers: ${peers[@]}" local peer_ports=() for peer in "${peers[@]}"; do @@ -42,16 +46,13 @@ function start_instance() { local this_addr="${IP_ADDR}:${this_port}" local peer_addr_args="${peer_ports[@]/#/-P $IP_ADDR:}" - stderr "this_port='${this_port}', this_addr='${this_addr}', peer_args='${peer_addr_args}'" local command="${FLO_EXE_PATH} -d ${DATA_ROOT}/flo${instance_num} -A ${this_addr} -p ${this_port} ${peer_addr_args} ${ADDITIONAL_ARGS[@]} -o ${DATA_ROOT}/flo${instance_num}.log" stderr "command: $command" - if [[ "${DRY_RUN}" == "no" ]]; then - $command & - local pid="$!" - PIDS+=("${pid}") - stderr "Started flo process with pid: ${pid}" - fi + $command & + local pid="$!" + PIDS+=("${pid}") + stderr "Started flo process with pid: ${pid}" } function delete_data_dirs() { @@ -81,6 +82,7 @@ function tail_logs() { } function stop_all() { + echo "" for pid in ${PIDS[@]}; do stderr "Stopping flo pid: ${pid}" kill "$pid" || true @@ -98,19 +100,24 @@ function start_all() { delete_data_dirs "${all_peers[@]}" fi - for instance_num in "${all_peers[@]}"; do - stderr "loop instance num: ${instance_num}" - start_instance "$instance_num" "${all_peers[@]}" - done - if [[ "${DRY_RUN}" == "no" ]]; then + for instance_num in "${all_peers[@]}"; do + start_instance "$instance_num" "${all_peers[@]}" + done + + # Sleep for a short bit to make sure that the log files actually exist before we start tailing them + sleep 1 tail_logs "${all_peers[@]}" fi } -while getopts "n:p:d:a:rx" opt; do +while getopts "n:p:d:a:rxbh" opt; do case $opt in + b) + BUILD="yes" + stderr "Will build before running" + ;; n) NUM_INSTANCES="$OPTARG" stderr "Starting ${num_instances}" >&2 @@ -135,11 +142,28 @@ while getopts "n:p:d:a:rx" opt; do DRY_RUN="yes" stderr "Will be a dry run" ;; + h) + echo "${USAGE}" + exit 0 + ;; \?) - stderr "Invalid option: -$OPTARG" >&2 - ;; + stderr "Invalid option: -$OPTARG" >&2 + stderr "${USAGE}" + exit 1 + ;; esac done +function do_build() { + local start_dir="$(pwd)" + cd "${SCRIPT_DIR}" + cargo build --all + cd "${start_dir}" +} + +if [[ "${BUILD}" == "yes" ]]; then + do_build +fi + trap stop_all INT start_all \ No newline at end of file From dd0c68132430c55f84381e34f3b3f753c35ab42c Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 12 Jun 2018 19:44:00 -0400 Subject: [PATCH 68/73] - Make all partitions start as non-writable when in cluster mode (stil working on actually setting them to writable again) - fix blanket impl of FloEvent - cleanups and foundational work in preparation for actually using system events --- dev-start-cluster.sh | 5 + flo-event/src/lib.rs | 56 ++--- .../src/engine/connection_handler/mod.rs | 1 + .../engine/controller/cluster_state/mod.rs | 235 +++++++++--------- .../controller/cluster_state/persistent.rs | 4 + .../controller/cluster_state/primary_state.rs | 2 +- .../src/engine/controller/controller_state.rs | 224 ++++++++++++++--- .../src/engine/controller/initialization.rs | 16 +- flo-server/src/engine/controller/mod.rs | 6 +- .../src/engine/controller/system_event.rs | 21 +- .../engine/event_stream/highest_counter.rs | 12 +- flo-server/src/engine/event_stream/mod.rs | 59 +++-- .../partition/controller/commit_manager.rs | 2 +- .../event_stream/partition/controller/mod.rs | 22 +- .../src/engine/event_stream/partition/mod.rs | 26 +- flo-server/tests/embedded_tests.rs | 2 +- 16 files changed, 454 insertions(+), 239 deletions(-) diff --git a/dev-start-cluster.sh b/dev-start-cluster.sh index 710dbf9..9c8c606 100755 --- a/dev-start-cluster.sh +++ b/dev-start-cluster.sh @@ -158,7 +158,12 @@ function do_build() { local start_dir="$(pwd)" cd "${SCRIPT_DIR}" cargo build --all + local build_success="$?" cd "${start_dir}" + + if [[ "${build_success}" != "0" ]]; then + exit 1 + fi } if [[ "${BUILD}" == "yes" ]]; then diff --git a/flo-event/src/lib.rs b/flo-event/src/lib.rs index ca61a03..6c4bd8a 100644 --- a/flo-event/src/lib.rs +++ b/flo-event/src/lib.rs @@ -177,27 +177,27 @@ impl <'a> EventData for BorrowedEventData<'a> { impl FloEvent for T where T: AsRef + Debug + EventData { fn id(&self) -> &FloEventId { - self.as_ref().id() + &self.as_ref().id } fn timestamp(&self) -> Timestamp { - self.as_ref().timestamp() + self.as_ref().timestamp } fn parent_id(&self) -> Option { - self.as_ref().parent_id() + self.as_ref().parent_id } fn namespace(&self) -> &str { - self.as_ref().namespace() + self.as_ref().namespace.as_str() } fn data_len(&self) -> u32 { - self.as_ref().data_len() + self.as_ref().data.len() as u32 } fn data(&self) -> &[u8] { - self.as_ref().data() + self.as_ref().data.as_slice() } fn to_owned_event(&self) -> OwnedFloEvent { @@ -205,6 +205,12 @@ impl FloEvent for T where T: AsRef + Debug + EventData { } } +impl AsRef for OwnedFloEvent { + fn as_ref(&self) -> &OwnedFloEvent { + self + } +} + /// This is the main `FloEvent` implementation that clients will deal with. All of the fields returned by the `FloEvent` /// methods are simply owned fields in this struct. #[derive(Debug, PartialEq, Clone)] @@ -246,48 +252,20 @@ impl OwnedFloEvent { } } -impl EventData for OwnedFloEvent { +impl EventData for T where T: AsRef { fn event_namespace(&self) -> &str { - self.namespace() + self.as_ref().namespace.as_str() } fn event_parent_id(&self) -> Option { - self.parent_id() + self.as_ref().parent_id } fn event_data(&self) -> &[u8] { - self.data() + self.as_ref().data.as_slice() } fn get_precomputed_crc(&self) -> Option { - Some(self.crc) + Some(self.as_ref().crc) } } -impl FloEvent for OwnedFloEvent { - fn id(&self) -> &FloEventId { - &self.id - } - - fn namespace(&self) -> &str { - &self.namespace - } - - fn data_len(&self) -> u32 { - self.data.len() as u32 - } - - fn data(&self) -> &[u8] { - &self.data - } - - fn to_owned_event(&self) -> OwnedFloEvent { - self.clone() - } - - fn parent_id(&self) -> Option { - self.parent_id - } - fn timestamp(&self) -> Timestamp { - self.timestamp - } -} diff --git a/flo-server/src/engine/connection_handler/mod.rs b/flo-server/src/engine/connection_handler/mod.rs index 6efdc45..0a5e3d7 100644 --- a/flo-server/src/engine/connection_handler/mod.rs +++ b/flo-server/src/engine/connection_handler/mod.rs @@ -270,6 +270,7 @@ mod test { this_address: Some(instance_addr), system_primary: None, peers: HashSet::new(), + this_partition_num: Some(1), }; let readers = ::engine::event_stream::partition::SharedReaderRefs::empty(); let system_stream = SystemStreamRef::new(part_ref, tx, Arc::new(RwLock::new(cluster_state)), readers); diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index a29fe29..229a6de 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -1,3 +1,7 @@ +pub mod persistent; +mod peer_connections; +mod primary_state; + use atomics::AtomicBoolWriter; use engine::{ConnectionId, EngineRef}; use engine::connection_handler::ConnectionControl; @@ -15,9 +19,6 @@ use std::time::{Duration, Instant}; use super::{AppendEntriesResponse, ClusterOptions, ControllerState, Peer, PeerUpgrade, ReceiveAppendEntries}; use super::peer_connection::OutgoingConnectionCreatorImpl; -pub mod persistent; -mod peer_connections; -mod primary_state; /// This is a placeholder for a somewhat better error handling when updating the persistent cluster state fails. /// This situation is extremely problematic, since it may be possible to cast two different votes in the same term, if for @@ -106,6 +107,20 @@ impl ClusterManager { } } + fn update_commit_index(&mut self, new_commit_index: EventCounter, term: Term, controller_state: &mut ControllerState) { + if new_commit_index <= self.last_applied { + return; + } + let last_commit_index = controller_state.get_commit_index(); + info!("Applying committed system events new_commit_index: {}, term: {}, last_applied: {}, last_committed: {}", + new_commit_index, term, self.last_applied, last_commit_index); + controller_state.set_commit_index(new_commit_index); + + // while self.last_applied < new_commit_index { + // let event = controller_state.get_next_event(self.last_applied) + // } + } + fn transition_state(&mut self, new_state: State) { debug!("Transitioning from state: {:?} to {:?}", self.state, new_state); self.state = new_state; @@ -119,14 +134,9 @@ impl ClusterManager { let PeerUpgrade { peer, system_primary, .. } = upgrade; if let Some(primary) = system_primary { info!("Determined primary of {:?} at {}, as told by instance: {:?}", primary.id, primary.address, peer); - self.transition_state(State::Follower); - { - let mut lock = self.system_partition_primary_address.write().unwrap(); - *lock = Some(primary.address); - } - { - let mut shared = self.shared.write().unwrap(); - shared.system_primary = Some(primary); + + if primary.id != self.persistent.this_instance_id { + self.set_follower_status(Some(primary)) } } else { debug!("peer announce from {:?} has unknown primary", peer); @@ -223,10 +233,17 @@ impl ClusterManager { let primary_id = primary.as_ref().map(|p| p.id); self.current_primary = primary_id; let mut lock = self.shared.write().unwrap(); - lock.system_primary = primary; + lock.system_primary = primary.clone(); + + let mut lock = self.system_partition_primary_address.write().unwrap(); + *lock = primary.map(|p| p.address) } fn set_follower_status(&mut self, primary: Option) { + if primary.as_ref().map(|new_primary| new_primary.id) == self.current_primary { + // if nothing has changed, then we won't bother doing anything + return; + } info!("Transitioning to follower state with new primary: {:?}", primary); self.primary_status_writer.set(false); self.primary_state = None; @@ -234,17 +251,15 @@ impl ClusterManager { self.transition_state(State::Follower); } - fn append_system_events(&mut self, events: &[OwnedFloEvent]) -> io::Result { - // TODO: actually persist the system events from ApendEntries - if !events.is_empty() { - error!("Appending of system events was never implemented!!!, ignoring: {:?}", events); - } - Ok(0) + fn append_system_events(&mut self, events: &[OwnedFloEvent], controller_state: &mut ControllerState) -> io::Result { + controller_state.replicate_system_events(events).map(|repl_result| { + repl_result.highest_event_counter + }) } - fn create_append_result(&mut self, events: &[OwnedFloEvent]) -> Option { + fn create_append_result(&mut self, events: &[OwnedFloEvent], controller_state: &mut ControllerState) -> Option { trace!("AppendEntries looks successful, will append: {} entries", events.len()); - self.append_system_events(events).map(|last_counter| { + self.append_system_events(events, controller_state).map(|last_counter| { Some(last_counter) }).unwrap_or_else(|io_err| { error!("Error appending system events: {:?}, returning false", io_err); @@ -254,14 +269,62 @@ impl ClusterManager { } impl ConsensusProcessor for ClusterManager { + fn tick(&mut self, now: Instant, controller_state: &mut ControllerState) { + self.connection_manager.establish_connections(now, controller_state); - fn send_append_entries(&mut self, controller_state: &mut ControllerState) { - let ClusterManager { ref mut primary_state, ref mut connection_manager, ref persistent, ..} = *self; + match self.state { + State::EstablishConnections => { + // we'll only be in this initial status once, on startup + self.transition_state(State::DeterminePrimary); + } + State::DeterminePrimary => { + trace!("Waiting to DeterminePrimary"); + } + State::Primary => { + trace!("This instance is primary"); + self.send_append_entries(controller_state); + } + _ => { + if self.election_timed_out(now) { + self.start_new_election(); + } + } + } + } - primary_state.as_mut().map(|state| { - let all_peers = persistent.cluster_members.iter().map(|peer| peer.id); - state.send_append_entries(controller_state, connection_manager.as_mut(), all_peers); - }); + fn is_primary(&self) -> bool { + self.state == State::Primary + } + + fn get_current_term(&self) -> Term { + self.persistent.current_term + } + + fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, controller_state: &ControllerState) { + debug!("peer_connection_established: connection_id: {}, {:?}", connection_id, upgrade); + let connection = controller_state.get_connection(connection_id); + if connection.is_none() { + debug!("Ignoring peer_connection_established: {:?} for connection_id: {}, because that connection has already been closed", + upgrade, connection_id); + return; + } + let connection = connection.unwrap(); + + if !self.persistent.cluster_members.contains(&upgrade.peer) { + self.persistent.modify(|state| { + state.cluster_members.insert(upgrade.peer.clone()); + }).expect(STATE_UPDATE_FAILED); + + let mut lock = self.shared.write().unwrap(); + lock.peers.insert(upgrade.peer.clone()); + } + + self.connection_manager.peer_connection_established(upgrade.peer.clone(), connection); + + if State::DeterminePrimary == self.state { + self.determine_primary_from_peer_upgrade(upgrade); + } + self.connection_resolved(connection.remote_address); } fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr) { @@ -318,60 +381,6 @@ impl ConsensusProcessor for ClusterManager { } } - fn peer_connection_established(&mut self, upgrade: PeerUpgrade, connection_id: ConnectionId, controller_state: &ControllerState) { - debug!("peer_connection_established: connection_id: {}, {:?}", connection_id, upgrade); - let connection = controller_state.get_connection(connection_id); - if connection.is_none() { - debug!("Ignoring peer_connection_established: {:?} for connection_id: {}, because that connection has already been closed", - upgrade, connection_id); - return; - } - let connection = connection.unwrap(); - - if !self.persistent.cluster_members.contains(&upgrade.peer) { - self.persistent.modify(|state| { - state.cluster_members.insert(upgrade.peer.clone()); - }).expect(STATE_UPDATE_FAILED); - - let mut lock = self.shared.write().unwrap(); - lock.peers.insert(upgrade.peer.clone()); - } - - self.connection_manager.peer_connection_established(upgrade.peer.clone(), connection); - - if State::DeterminePrimary == self.state { - self.determine_primary_from_peer_upgrade(upgrade); - } - self.connection_resolved(connection.remote_address); - } - - fn tick(&mut self, now: Instant, controller_state: &mut ControllerState) { - self.connection_manager.establish_connections(now, controller_state); - - match self.state { - State::EstablishConnections => { - // we'll only be in this initial status once, on startup - self.transition_state(State::DeterminePrimary); - } - State::DeterminePrimary => { - trace!("Waiting to DeterminePrimary"); - } - State::Primary => { - trace!("This instance is primary"); - self.send_append_entries(controller_state); - } - _ => { - if self.election_timed_out(now) { - self.start_new_election(); - } - } - } - } - - fn is_primary(&self) -> bool { - self.state == State::Primary - } - fn append_entries_received(&mut self, connection_id: ConnectionId, append: ReceiveAppendEntries, controller_state: &mut ControllerState) { let from = self.connection_manager.get_peer_id(connection_id); if from.is_none() { @@ -394,10 +403,10 @@ impl ConsensusProcessor for ClusterManager { self.update_last_heartbeat_to_now(); - let response = match controller_state.get_next_event(append.prev_entry_index.saturating_sub(1)) { + let response = match controller_state.get_next_counter_and_term(append.prev_entry_index.saturating_sub(1)) { Some(Ok((actual_prev_index, actual_prev_term))) => { if actual_prev_index == append.prev_entry_index && actual_prev_term == append.prev_entry_term { - self.create_append_result(append.events.as_slice()) + self.create_append_result(append.events.as_slice(), controller_state) } else { info!("AppendEntries: {:?} does not match the prev index/term stored for this instance: index: {}, term: {}", append, actual_prev_index, actual_prev_term); None @@ -406,7 +415,7 @@ impl ConsensusProcessor for ClusterManager { None => { if append.prev_entry_index == 0 && append.prev_entry_term == 0 { // special case for when we're at the very beginning and have literally no events in the log - self.create_append_result(append.events.as_slice()) + self.create_append_result(append.events.as_slice(), controller_state) } else { debug!("System partition with head at: {} is behind AppendEntries with index: {}, term: {}, returning negative result", self.last_applied, append.prev_entry_index, append.prev_entry_term); @@ -419,6 +428,8 @@ impl ConsensusProcessor for ClusterManager { } }; + self.update_commit_index(append.commit_index, append.term, controller_state); + self.connection_manager.send_to_peer(peer_id, ConnectionControl::SendAppendEntriesResponse(AppendEntriesResponse { term: my_current_term, success: response, @@ -452,14 +463,20 @@ impl ConsensusProcessor for ClusterManager { } } - fn get_current_term(&self) -> Term { - self.persistent.current_term + fn send_append_entries(&mut self, controller_state: &mut ControllerState) { + let ClusterManager { ref mut primary_state, ref mut connection_manager, ref persistent, ..} = *self; + + primary_state.as_mut().map(|state| { + let all_peers = persistent.cluster_members.iter().map(|peer| peer.id); + state.send_append_entries(controller_state, connection_manager.as_mut(), all_peers); + }); } } #[derive(Debug, PartialEq, Clone)] pub struct SharedClusterState { pub this_instance_id: FloInstanceId, + pub this_partition_num: Option, pub this_address: Option, pub system_primary: Option, pub peers: HashSet, @@ -469,6 +486,7 @@ impl SharedClusterState { pub fn non_cluster() -> SharedClusterState { SharedClusterState { this_instance_id: flo_instance_id::generate_new(), + this_partition_num: Some(0), this_address: None, system_primary: None, peers: HashSet::new(), @@ -543,12 +561,14 @@ mod test { connection_manager.stub_peer_connection(peer_2_connection, peer_2.id); let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); + let subject_id = subject.persistent.this_instance_id; subject.persistent.modify(|state| { state.cluster_members.insert(peer_1.clone()); state.cluster_members.insert(peer_2.clone()); state.current_term = 5; }).unwrap(); + subject.current_primary = Some(subject_id); subject.last_applied = 99; subject.last_applied_term = 4; subject.state = State::Primary; @@ -600,24 +620,9 @@ mod test { subject.last_heartbeat = start; let mock_system_events = vec![ - MockSystemEvent { - id: 1, - term: 1, - segment: SegmentNum::new(1), - file_offset: 0, - }, - MockSystemEvent { - id: 2, - term: 3, - segment: SegmentNum::new(1), - file_offset: 55, - }, - MockSystemEvent { - id: 3, - term: 4, - segment: SegmentNum::new(2), - file_offset: 77, - }, + MockSystemEvent::with_any_data(1, 1, SegmentNum::new(1), 0), + MockSystemEvent::with_any_data(2, 3, SegmentNum::new(1), 55), + MockSystemEvent::with_any_data(3, 4, SegmentNum::new(2), 77), ]; let (peer_1_tx, _peer_1_rx) = ::engine::connection_handler::create_connection_control_channels(); @@ -679,24 +684,9 @@ mod test { subject.last_heartbeat = start; let mock_system_events = vec![ - MockSystemEvent { - id: 1, - term: 1, - segment: SegmentNum::new(1), - file_offset: 0, - }, - MockSystemEvent { - id: 2, - term: 3, - segment: SegmentNum::new(1), - file_offset: 55, - }, - MockSystemEvent { - id: 3, - term: 4, - segment: SegmentNum::new(2), - file_offset: 77, - }, + MockSystemEvent::with_any_data(1, 1, SegmentNum::new(1), 0), + MockSystemEvent::with_any_data(2, 3, SegmentNum::new(1), 55), + MockSystemEvent::with_any_data(3, 4, SegmentNum::new(2), 77), ]; let mut controller_state = MockControllerState::new().with_commit_index(2).with_mocked_events(mock_system_events.as_slice()); @@ -1252,6 +1242,7 @@ mod test { this_address: Some(this_addr), system_primary: Some(Peer {id: flo_instance_id::generate_new(), address: this_addr}), peers: HashSet::new(), + this_partition_num: None, }; assert!(!subject.this_instance_is_primary()); } @@ -1265,6 +1256,7 @@ mod test { this_address: Some(this_addr), system_primary: None, peers: HashSet::new(), + this_partition_num: None, }; assert!(!subject.this_instance_is_primary()); } @@ -1278,6 +1270,7 @@ mod test { this_address: Some(this_addr), system_primary: Some(Peer {id: this_id, address: this_addr}), peers: HashSet::new(), + this_partition_num: None, }; assert!(subject.this_instance_is_primary()); } diff --git a/flo-server/src/engine/controller/cluster_state/persistent.rs b/flo-server/src/engine/controller/cluster_state/persistent.rs index 6d88cdd..fa0d1bb 100644 --- a/flo-server/src/engine/controller/cluster_state/persistent.rs +++ b/flo-server/src/engine/controller/cluster_state/persistent.rs @@ -4,6 +4,7 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Seek, SeekFrom}; use std::collections::HashSet; +use event::ActorId; use protocol::Term; use protocol::flo_instance_id::{self, FloInstanceId}; use engine::controller::controller_messages::Peer; @@ -17,6 +18,7 @@ pub struct PersistentClusterState { pub current_term: Term, pub voted_for: Option, pub this_instance_id: FloInstanceId, + pub this_partition_num: Option, pub cluster_members: HashSet, } @@ -25,6 +27,7 @@ impl PersistentClusterState { pub fn initialize_shared_state(&self, this_address: Option) -> SharedClusterState { SharedClusterState { + this_partition_num: self.this_partition_num, this_instance_id: self.this_instance_id, this_address, system_primary: None, // we are still starting up, so we have no idea who's primary @@ -43,6 +46,7 @@ impl PersistentClusterState { current_term: 0, voted_for: None, this_instance_id: flo_instance_id::generate_new(), + this_partition_num: None, cluster_members: HashSet::new(), } } diff --git a/flo-server/src/engine/controller/cluster_state/primary_state.rs b/flo-server/src/engine/controller/cluster_state/primary_state.rs index cad8106..913008b 100644 --- a/flo-server/src/engine/controller/cluster_state/primary_state.rs +++ b/flo-server/src/engine/controller/cluster_state/primary_state.rs @@ -81,7 +81,7 @@ impl PrimaryState { reader_start_segment: current_segment, }) } else { - match controller.get_next_event(start_after_index.saturating_sub(1)) { + match controller.get_next_counter_and_term(start_after_index.saturating_sub(1)) { Some(result) => { let (prev_index, prev_term) = result?; // return error if there is one let (start_segment, start_offset) = controller.get_next_entry(start_after_index).map(|entry| { diff --git a/flo-server/src/engine/controller/controller_state.rs b/flo-server/src/engine/controller/controller_state.rs index c8a4791..0bfc9af 100644 --- a/flo-server/src/engine/controller/controller_state.rs +++ b/flo-server/src/engine/controller/controller_state.rs @@ -2,28 +2,55 @@ use std::sync::{Arc, Mutex}; use std::path::PathBuf; use std::collections::HashMap; use std::io; +use tokio_core::reactor::Remote; use protocol::{Term, FloInstanceId}; -use event::EventCounter; +use event::{OwnedFloEvent, EventCounter, ActorId}; use engine::ConnectionId; -use engine::controller::{ConnectionRef, SystemEvent}; -use engine::event_stream::{EventStreamRef, EventStreamRefMut, EventStreamOptions}; -use engine::event_stream::partition::{SegmentNum, IndexEntry, PersistentEvent, PartitionReader}; +use engine::controller::{ConnectionRef, SystemEvent, SystemEventData}; +use engine::event_stream::{EventStreamRef, EventStreamRefMut, EventStreamOptions, init_new_event_stream}; +use engine::event_stream::partition::{SegmentNum, IndexEntry, PersistentEvent, PartitionReader, ReplicationResult}; use engine::event_stream::partition::controller::PartitionImpl; +use std::borrow::Cow; +#[derive(Debug, PartialEq)] +pub struct SystemEventRef<'a> { + pub counter: EventCounter, + pub data: Cow<'a, SystemEventData> +} pub trait ControllerState { fn add_connection(&mut self, connection: ConnectionRef); fn remove_connection(&mut self, connection_id: ConnectionId); fn get_connection(&self, connection_id: ConnectionId) -> Option<&ConnectionRef>; + fn set_partitions_writable(&mut self, partition_num: ActorId); + fn create_event_stream(&mut self, options: EventStreamOptions) -> io::Result<()>; + fn create_partitions(&mut self, partition_num: ActorId) -> io::Result<()>; + + fn get_commit_index(&self) -> EventCounter; + fn set_commit_index(&mut self, new_index: EventCounter); fn get_last_committed(&mut self) -> io::Result<(EventCounter, Term)>; - fn get_next_event(&mut self, start_after: EventCounter) -> Option>; + fn get_next_event<'a>(&'a mut self, start_after: EventCounter) -> Option>>; + + fn get_next_counter_and_term(&mut self, start_after: EventCounter) -> Option> { + self.get_next_event(start_after).map(|result| { + result.map(|event| { + (event.counter, event.data.term) + }) + }) + } fn get_next_entry(&self, event_counter: EventCounter) -> Option; fn get_current_file_offset(&self) -> (SegmentNum, usize); fn add_system_replication_node(&mut self, peer: FloInstanceId); + + /// Called only by the clusterState and only when this instance is a follower + fn replicate_system_events(&mut self, events: &[OwnedFloEvent]) -> io::Result; + + /// Called only by the clusterState and only when this instance is primary + fn produce_system_event(&mut self, event: SystemEventData) -> io::Result; } pub struct ControllerStateImpl { @@ -47,6 +74,8 @@ pub struct ControllerStateImpl { /// Stores all of the active connections with this instance pub all_connections: HashMap, + + remote: Remote, } impl ControllerStateImpl { @@ -54,11 +83,13 @@ impl ControllerStateImpl { event_streams: HashMap, shared_event_stream_refs: Arc>>, storage_dir: PathBuf, - default_stream_options: EventStreamOptions) -> ControllerStateImpl { + default_stream_options: EventStreamOptions, + remote: Remote) -> ControllerStateImpl { let system_partition_reader = system_partition.create_reader(0, ::engine::event_stream::partition::EventFilter::All, 0); ControllerStateImpl { + remote, shared_event_stream_refs, event_streams, default_stream_options, @@ -69,6 +100,15 @@ impl ControllerStateImpl { } } + fn add_event_stream(&mut self, new_stream: EventStreamRefMut) { + let name = new_stream.get_name().to_owned(); + let shared_ref = new_stream.clone_ref(); + + self.event_streams.insert(name.clone(), new_stream); + let mut lock = self.shared_event_stream_refs.lock().unwrap(); + lock.insert(name, shared_ref); + } + fn read_event(&mut self, entry: IndexEntry) -> io::Result> { self.system_partition_reader.set_to(entry.segment, entry.file_offset)?; let result = self.system_partition_reader.read_next_uncommitted().ok_or_else(|| { @@ -97,38 +137,83 @@ impl ControllerState for ControllerStateImpl { self.all_connections.get(&connection_id) } - fn get_next_entry(&self, previous: EventCounter) -> Option { - self.system_partition.get_next_index_entry(previous) + fn set_partitions_writable(&mut self, partition_num: ActorId) { + for event_stream in self.event_streams.values_mut() { + event_stream.set_writable_partition(partition_num); + } } - fn get_current_file_offset(&self) -> (SegmentNum, usize) { - self.system_partition.get_head_position() + fn create_event_stream(&mut self, options: EventStreamOptions) -> Result<(), io::Error> { + if self.event_streams.contains_key(&options.name) { + return Err(io::Error::new(io::ErrorKind::AlreadyExists, "Event stream already exists")); + } + + let storage_dir = self.storage_dir.clone(); + let remote = self.remote.clone(); + + init_new_event_stream(storage_dir, options, remote, false).map(|stream| { + self.add_event_stream(stream); + }) + } + + fn create_partitions(&mut self, _partition_num: u16) -> Result<(), io::Error> { + unimplemented!() + } + + fn get_commit_index(&self) -> EventCounter { + self.system_partition.get_commit_index() + } + + fn set_commit_index(&mut self, new_index: EventCounter) { + self.system_partition.update_commit_index(new_index); } fn get_last_committed(&mut self) -> io::Result<(EventCounter, Term)> { - let start_after = self.system_partition.get_commit_index(); - if start_after == 0 { + let commit_index = self.system_partition.get_commit_index(); + if commit_index == 0 { return Ok((0, 0)); } - match self.get_next_event(start_after) { - Some(result) => result, + match self.get_next_counter_and_term(commit_index - 1) { + Some(result) => { + result + } None => { - error!("No SystemEvent was found for the commit index: {}. System partition is in an invalid state!", start_after); - Err(io::Error::new(io::ErrorKind::InvalidData, format!("Expected to have a SystemEvent at index: {} since that is the commit index!", start_after))) + error!("No SystemEvent was found for the commit index: {}. System partition is in an invalid state!", commit_index); + Err(io::Error::new(io::ErrorKind::InvalidData, format!("Expected to have a SystemEvent at index: {} since that is the commit index!", commit_index))) } } } - - fn get_next_event(&mut self, start_after: EventCounter) -> Option> { + fn get_next_event<'a>(&'a mut self, start_after: EventCounter) -> Option>> { self.get_next_entry(start_after.saturating_sub(1)).map(|index_entry| { self.read_event(index_entry).map(|system_event| { - (start_after, system_event.term()) + let counter = system_event.counter(); + SystemEventRef{ + counter, + data: Cow::Owned(system_event.system_data().clone()), + } }) }) } + + fn get_next_entry(&self, previous: EventCounter) -> Option { + self.system_partition.get_next_index_entry(previous) + } + + fn get_current_file_offset(&self) -> (SegmentNum, usize) { + self.system_partition.get_head_position() + } + fn add_system_replication_node(&mut self, peer: FloInstanceId) { self.system_partition.add_replication_node(peer); } + + fn replicate_system_events(&mut self, events: &[OwnedFloEvent]) -> io::Result { + self.system_partition.replicate_events(0, events) + } + + fn produce_system_event(&mut self, _event: SystemEventData) -> Result { + unimplemented!() + } } @@ -137,12 +222,14 @@ impl ControllerState for ControllerStateImpl { pub mod mock { use super::*; use std::collections::BTreeMap; + use engine::controller::SystemEventKind; #[derive(Debug)] pub struct MockControllerState { pub commit_index: EventCounter, pub all_connections: HashMap, pub system_events: BTreeMap, + pub writable_partitions: Option, } impl MockControllerState { @@ -151,6 +238,7 @@ pub mod mock { commit_index: 0, all_connections: HashMap::new(), system_events: BTreeMap::new(), + writable_partitions: None, } } @@ -186,42 +274,116 @@ pub mod mock { fn get_connection(&self, connection_id: ConnectionId) -> Option<&ConnectionRef> { self.all_connections.get(&connection_id) } - fn get_next_entry(&self, event_counter: EventCounter) -> Option { - self.system_events.range((event_counter + 1)..).next().map(|(id, sys)| { - IndexEntry::new(*id, sys.segment, sys.file_offset) - }) + fn set_partitions_writable(&mut self, partition_num: u16) { + self.writable_partitions = Some(partition_num); + } + fn create_event_stream(&mut self, _options: EventStreamOptions) -> Result<(), io::Error> { + unimplemented!() + } + fn create_partitions(&mut self, _partition_num: u16) -> Result<(), io::Error> { + unimplemented!() } - fn get_current_file_offset(&self) -> (SegmentNum, usize) { - self.system_events.values().last().map(|sys| { - (sys.segment, sys.file_offset) - }).unwrap_or((SegmentNum::new_unset(), 0)) + fn get_commit_index(&self) -> EventCounter { + self.commit_index + } + + fn set_commit_index(&mut self, new_index: EventCounter) { + self.commit_index = self.commit_index.max(new_index); } fn get_last_committed(&mut self) -> io::Result<(EventCounter, Term)> { let commit_idx = self.commit_index; self.system_events.get(&commit_idx).map(|sys| { - (sys.id, sys.term) + (sys.id, sys.data.term) }).ok_or_else(|| { io::Error::new(io::ErrorKind::Other, format!("Test error because there is no stubbed event for the commit index: {}", commit_idx)) }) } - fn get_next_event(&mut self, start_after: EventCounter) -> Option> { + fn get_next_event<'a>(&'a mut self, start_after: EventCounter) -> Option>> { self.system_events.range((start_after + 1)..).next().map(|(id, sys)| { - Ok((*id, sys.term)) + Ok(SystemEventRef { + counter: *id, + data: Cow::Borrowed(&sys.data), + }) + }) + } + + fn get_next_entry(&self, event_counter: EventCounter) -> Option { + self.system_events.range((event_counter + 1)..).next().map(|(id, sys)| { + IndexEntry::new(*id, sys.segment, sys.file_offset) }) } + fn get_current_file_offset(&self) -> (SegmentNum, usize) { + self.system_events.values().last().map(|sys| { + (sys.segment, sys.file_offset) + }).unwrap_or((SegmentNum::new_unset(), 0)) + } + fn add_system_replication_node(&mut self, _peer: FloInstanceId) { unimplemented!() } + + fn replicate_system_events(&mut self, events: &[OwnedFloEvent]) -> io::Result { + let segment_num = self.system_events.values().last().map(|e| e.segment).unwrap_or(SegmentNum::new(1)); + let mut file_offset = self.system_events.values().last().map(|e| e.file_offset).unwrap_or(0); + let mut highest = 0; + for event in events.iter() { + let counter = event.id.event_counter; + let system_event = SystemEvent::from_event(event).map_err(|se| { + io::Error::new(io::ErrorKind::InvalidData, format!("invalid system event data: {}", se)) + })?; + let data = system_event.deserialized_data; + let mock = MockSystemEvent { + id: counter, + segment: segment_num, + data, + file_offset, + }; + self.add_mock_event(mock); + file_offset += 100; + highest = highest.max(counter); + } + Ok(ReplicationResult { + op_id: 0, + success: true, + highest_event_counter: highest, + }) + } + + fn produce_system_event(&mut self, _event: SystemEventData) -> Result { + unimplemented!() + } } #[derive(Clone, Debug, PartialEq)] pub struct MockSystemEvent { pub id: EventCounter, - pub term: Term, + pub data: SystemEventData, pub segment: SegmentNum, pub file_offset: usize, + + } + + impl MockSystemEvent { + pub fn with_any_data(counter: EventCounter, term: Term, segment: SegmentNum, file_offset: usize) -> MockSystemEvent { + use engine::controller::system_event::ClusterMemberJoining; + + let data = SystemEventKind::NewClusterMemberJoining(ClusterMemberJoining { new_member: 45678 }); + MockSystemEvent::with_data(counter, term, segment, file_offset, data) + } + + pub fn with_data(counter: EventCounter, term: Term, segment: SegmentNum, file_offset: usize, data: SystemEventKind) -> MockSystemEvent { + MockSystemEvent { + id: counter, + segment, + file_offset, + data: SystemEventData { + term, + kind: data + } + } + } } } diff --git a/flo-server/src/engine/controller/initialization.rs b/flo-server/src/engine/controller/initialization.rs index 4aef7b5..a6217d0 100644 --- a/flo-server/src/engine/controller/initialization.rs +++ b/flo-server/src/engine/controller/initialization.rs @@ -51,6 +51,7 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul debug!("Starting Flo Controller with: {:?}", options); let ControllerOptions{storage_dir, default_stream_options, cluster_options} = options; let use_cluster_mode = cluster_options.is_some(); + let start_partitions_as_writable = !use_cluster_mode; let system_primary_status_writer = AtomicBoolWriter::with_value(false); let system_primary_address: SystemPrimaryAddressRef = Arc::new(RwLock::new(None)); @@ -61,7 +62,7 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul debug!("Initialized system partition"); // early return if this fails - let user_streams = init_user_streams(&storage_dir, &default_stream_options, &remote)?; + let user_streams = init_user_streams(&storage_dir, &default_stream_options, &remote, start_partitions_as_writable)?; debug!("Initialized all {} user event streams", user_streams.len()); let shared_stream_refs = user_streams.iter().map(|(key, value)| { (key.to_owned(), value.clone_ref()) @@ -96,7 +97,7 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul let cluster_opts = cluster_options.unwrap(); let system_stream_ref = engine_ref.get_system_stream(); - ::engine::controller::tick_generator::spawn_tick_generator(cluster_opts.heartbeat_interval_millis, remote, system_stream_ref); + ::engine::controller::tick_generator::spawn_tick_generator(cluster_opts.heartbeat_interval_millis, remote.clone(), system_stream_ref); init_cluster_consensus_processor(persistent_state, cluster_opts, @@ -113,7 +114,8 @@ pub fn start_controller(options: ControllerOptions, remote: Remote) -> io::Resul shared_stream_refs, storage_dir, consensus_processor, - default_stream_options); + default_stream_options, + remote); run_controller_impl(flo_controller, system_partition_rx); Ok(engine_ref) @@ -161,7 +163,7 @@ fn create_engine_ref(shared_stream_refs: Arc io::Result> { +fn init_user_streams(storage_dir: &Path, options: &EventStreamOptions, remote: &Remote, start_writable: bool) -> io::Result> { let mut user_streams = HashMap::new(); // all sorts of early returns if there's filesystem failures @@ -173,7 +175,8 @@ fn init_user_streams(storage_dir: &Path, options: &EventStreamOptions, remote: & let stream = init_existing_event_stream( stream_storage, options.clone(), - remote.clone())?; + remote.clone(), + start_writable)?; user_streams.insert(stream.get_name().to_owned(), stream); } } @@ -184,7 +187,8 @@ fn init_user_streams(storage_dir: &Path, options: &EventStreamOptions, remote: & let new_stream = init_new_event_stream( new_stream_dir, options.clone(), - remote.clone())?; + remote.clone(), + start_writable)?; user_streams.insert(options.name.clone(), new_stream); } diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index a4292fb..5e20b42 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -13,6 +13,7 @@ use std::sync::{Arc, Mutex}; use std::collections::HashMap; use std::io; use std::time::Instant; +use tokio_core::reactor::Remote; use protocol::{ProduceEvent, Term}; use engine::ConnectionId; @@ -58,10 +59,11 @@ impl FloController { shared_stream_refs: Arc>>, storage_dir: PathBuf, cluster_state: Box, - default_stream_options: EventStreamOptions) -> FloController { + default_stream_options: EventStreamOptions, + remote: Remote) -> FloController { let controller_state = ControllerStateImpl::new(system_partition, event_streams, shared_stream_refs, - storage_dir, default_stream_options); + storage_dir, default_stream_options, remote); FloController { cluster_state, diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index 701b0f2..18b3a40 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -1,13 +1,13 @@ use rmp_serde::decode::Error; use protocol::{Term, FloInstanceId}; -use event::{FloEvent, EventData, FloEventId, OwnedFloEvent, ActorId, Timestamp}; +use event::{FloEvent, EventData, FloEventId, EventCounter, OwnedFloEvent, ActorId, Timestamp}; use engine::event_stream::partition::PersistentEvent; #[derive(Debug, PartialEq)] pub struct SystemEvent { - wrapped: E, - deserialized_data: SystemEventData + pub wrapped: E, + pub deserialized_data: SystemEventData } impl SystemEvent { @@ -20,6 +20,9 @@ impl SystemEvent { }) } + pub fn counter(&self) -> EventCounter { + self.wrapped.id().event_counter + } pub fn term(&self) -> Term { self.system_data().term @@ -106,20 +109,26 @@ impl FloEvent for SystemEvent { #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub struct ClusterMemberJoining { - new_member: FloInstanceId, + pub new_member: FloInstanceId, } #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub struct ClusterMemberJoined { - new_member: FloInstanceId, - new_member_partition_num: ActorId, + pub new_member: FloInstanceId, + pub new_member_partition_num: ActorId, } +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct ClusterMemberOnline { + pub member: FloInstanceId, + pub member_partition_num: ActorId, +} #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub enum SystemEventKind { NewClusterMemberJoining(ClusterMemberJoining), NewClusterMemberJoined(ClusterMemberJoined), + ExistingMemberOnline(ClusterMemberOnline), } #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] diff --git a/flo-server/src/engine/event_stream/highest_counter.rs b/flo-server/src/engine/event_stream/highest_counter.rs index 174574e..d527c6a 100644 --- a/flo-server/src/engine/event_stream/highest_counter.rs +++ b/flo-server/src/engine/event_stream/highest_counter.rs @@ -3,6 +3,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; +use event::EventCounter; + #[derive(Clone, Debug)] pub struct HighestCounter(Arc); @@ -12,15 +14,15 @@ impl HighestCounter { HighestCounter::starting_at(0) } - pub fn starting_at(count: u64) -> HighestCounter { + pub fn starting_at(count: EventCounter) -> HighestCounter { HighestCounter(Arc::new(AtomicUsize::new(count as usize))) } - pub fn set_if_greater(&self, value: u64) { + pub fn set_if_greater(&self, value: EventCounter) { let mut current = self.0.load(Ordering::SeqCst); let mut attempts = 1; loop { - if value > current as u64 { + if value > current as EventCounter { let prev = self.0.compare_and_swap(current, value as usize, Ordering::SeqCst); if prev == current { break; @@ -40,7 +42,7 @@ impl HighestCounter { } } - pub fn increment_and_get(&self, inc_amount: u64) -> u64 { + pub fn increment_and_get(&self, inc_amount: EventCounter) -> EventCounter { let mut current = self.0.load(Ordering::SeqCst); let mut attempts = 1; loop { @@ -58,7 +60,7 @@ impl HighestCounter { if attempts > 1 { debug!("set_if_greater took: {} attempts", attempts); } - current as u64 + inc_amount + current as EventCounter + inc_amount } pub fn get(&self) -> u64 { diff --git a/flo-server/src/engine/event_stream/mod.rs b/flo-server/src/engine/event_stream/mod.rs index 82e49e0..c6864f0 100644 --- a/flo-server/src/engine/event_stream/mod.rs +++ b/flo-server/src/engine/event_stream/mod.rs @@ -41,9 +41,7 @@ impl EventStreamOptions { } } - - -pub fn init_existing_event_stream(event_stream_storage_dir: PathBuf, options: EventStreamOptions, remote: Remote) -> Result { +pub fn init_existing_event_stream(event_stream_storage_dir: PathBuf, options: EventStreamOptions, remote: Remote, start_writable: bool) -> Result { debug!("Starting initialization of existing event stream with: {:?}", &options); let partition_numbers = determine_existing_partition_dirs(&event_stream_storage_dir)?; @@ -53,7 +51,7 @@ pub fn init_existing_event_stream(event_stream_storage_dir: PathBuf, options: Ev let mut partition_refs = Vec::with_capacity(partition_numbers.len()); for partition_num in partition_numbers { - let partition_ref = initialize_existing_partition(partition_num, &event_stream_storage_dir, &options, highest_counter.clone())?; + let partition_ref = initialize_existing_partition(partition_num, &event_stream_storage_dir, &options, highest_counter.clone(), start_writable)?; partition_refs.push(partition_ref); } @@ -61,8 +59,9 @@ pub fn init_existing_event_stream(event_stream_storage_dir: PathBuf, options: Ev let tick_interval = options.get_tick_interval(); let event_stream = EventStreamRefMut { - name: options.name, + event_stream_options: options, partitions: partition_refs, + highest_counter, }; start_tick_timer(remote, event_stream.clone_ref(), tick_interval); @@ -70,7 +69,7 @@ pub fn init_existing_event_stream(event_stream_storage_dir: PathBuf, options: Ev Ok(event_stream) } -pub fn init_new_event_stream(event_stream_storage_dir: PathBuf, options: EventStreamOptions, remote: Remote) -> Result { +pub fn init_new_event_stream(event_stream_storage_dir: PathBuf, options: EventStreamOptions, remote: Remote, start_writable: bool) -> Result { debug!("Starting initialization of new event stream with: {:?}", &options); let partition_count = options.num_partitions; @@ -80,24 +79,23 @@ pub fn init_new_event_stream(event_stream_storage_dir: PathBuf, options: EventSt let highest_counter = HighestCounter::zero(); for i in 0..partition_count { let partition_num: ActorId = i + 1; - let partition_ref = initialize_new_partition(partition_num, &event_stream_storage_dir, &options, highest_counter.clone())?; + let partition_ref = initialize_new_partition(partition_num, &event_stream_storage_dir, &options, highest_counter.clone(), start_writable)?; // We're appending these in order so that they can be indexed up by partition number later partition_refs.push(partition_ref); } let tick_interval = options.get_tick_interval(); - let EventStreamOptions{name, ..} = options; - debug!("Finished initializing {} partitions for event stream: '{}'", partition_count, &name); + debug!("Finished initializing {} partitions for event stream: '{}'", partition_count, options.name); let event_stream = EventStreamRefMut { - name: name, + event_stream_options: options, partitions: partition_refs, + highest_counter: highest_counter, }; start_tick_timer(remote, event_stream.clone_ref(), tick_interval); Ok(event_stream) } - pub fn get_event_steam_data_dir(server_storage_dir: &Path, event_stream_name: &str) -> PathBuf { server_storage_dir.join(event_stream_name) } @@ -125,17 +123,18 @@ fn determine_existing_partition_dirs(event_stream_dir: &Path) -> io::Result + partitions: Vec, + event_stream_options: EventStreamOptions, + highest_counter: HighestCounter, } impl EventStreamRefMut { pub fn get_name(&self) -> &str { - &self.name + &self.event_stream_options.name } pub fn clone_ref(&self) -> EventStreamRef { - let name = self.name.clone(); + let name = self.get_name().to_owned(); let partitions = self.partitions.iter().map(|part| part.clone_ref()).collect::>(); EventStreamRef { @@ -143,6 +142,36 @@ impl EventStreamRefMut { partitions } } + + pub fn set_writable_partition(&mut self, partition_num: ActorId) { + info!("Setting partition: {} for event_stream: '{}' to writable", partition_num, self.get_name()); + for partition in self.partitions.iter_mut() { + if partition.partition_num() == partition_num { + partition.set_writable(); + } else { + partition.set_read_only(); + } + } + } + + pub fn add_new_partition(&mut self, partition_num: ActorId, event_stream_data: &Path) -> io::Result<()> { + info!("Adding new partition: {} to event_stream: '{}'", partition_num, self.get_name()); + if self.partitions.iter().any(|p| p.partition_num() == partition_num) { + return Err(io::Error::new(io::ErrorKind::AlreadyExists, "Partition number already exists")); + } + if !self.partition_number_is_next_sequential(partition_num) { + return Err(io::Error::new(io::ErrorKind::Other, "Partition number is not the next sequential number")); + } + + let highest_counter = self.highest_counter.clone(); + let partition = initialize_new_partition(partition_num, event_stream_data, &self.event_stream_options, highest_counter, false)?; + self.partitions.push(partition); + Ok(()) + } + + fn partition_number_is_next_sequential(&self, partition_num: ActorId) -> bool { + self.partitions.len() == partition_num as usize + 1 + } } #[derive(Clone, Debug)] diff --git a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs index 1f2be90..d9f8392 100644 --- a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs +++ b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs @@ -60,7 +60,7 @@ impl CommitManager { } } - fn update_commit_index(&mut self, new_index: EventCounter) { + pub fn update_commit_index(&mut self, new_index: EventCounter) { self.commit_index.set_if_greater(new_index as usize); } diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index 593166a..f6c27b5 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -233,7 +233,7 @@ impl PartitionImpl { // TODO: ensure that prev_event_counter and prev_event_term are checked for system event replication // TODO: think about handling empty `events` vec. maybe best to handle that in the connection handler? // TODO: ensure that we are in Follower status and that the events came from the leader - let result = self.replicate_events(op_id, events); + let result = self.replicate_events(op_id, events.as_slice()); let response = result.unwrap_or_else(|io_err| { let head = self.index.greatest_event_counter(); error!("Error handling replicate operation with op_id: {}, io error: '{}', sending fail response with head: {}", op_id, io_err, head); @@ -246,8 +246,12 @@ impl PartitionImpl { let _ = client_sender.send(response); } + pub fn update_commit_index(&mut self, new_commit_index: EventCounter) { + self.commit_manager.update_commit_index(new_commit_index); + } + /// persists events in this partition. Panics if `events` is empty - pub fn replicate_events(&mut self, op_id: u32, events: Vec) -> io::Result { + pub fn replicate_events(&mut self, op_id: u32, events: &[F]) -> io::Result { let commit_index = self.commit_manager.get_commit_index(); // ignore any events with id.event_counter < commit_index @@ -363,7 +367,9 @@ impl PartitionImpl { pub fn handle_produce(&mut self, produce: ProduceOperation) -> io::Result<()> { let ProduceOperation {client, op_id, events} = produce; - let result = self.append_all(events); + let result = self.verify_partition_is_primary().and_then(|()| { + self.append_all(events) + }); match result { Ok(id) => { @@ -383,6 +389,14 @@ impl PartitionImpl { Ok(()) } + fn verify_partition_is_primary(&self) -> io::Result<()> { + if self.is_current_primary() { + Ok(()) + } else { + Err(io::Error::new(io::ErrorKind::Other, "Not primary")) + } + } + fn append_all(&mut self, events: Vec) -> io::Result { let event_count = events.len(); // reserve the range of ids for the events @@ -690,7 +704,7 @@ mod test { ]; let op_id = 567; - let result = partition.replicate_events(op_id, events.clone()).expect("Failed to replicate events"); + let result = partition.replicate_events(op_id, events.as_slice()).expect("Failed to replicate events"); assert!(result.success); assert_eq!(op_id, result.op_id); assert_eq!(4, result.highest_event_counter); diff --git a/flo-server/src/engine/event_stream/partition/mod.rs b/flo-server/src/engine/event_stream/partition/mod.rs index f4c5a4a..aabe6d8 100644 --- a/flo-server/src/engine/event_stream/partition/mod.rs +++ b/flo-server/src/engine/event_stream/partition/mod.rs @@ -190,6 +190,14 @@ pub struct PartitionRefMut { } impl PartitionRefMut { + pub fn set_writable(&mut self) { + self.status_writer.set(true); + } + + pub fn set_read_only(&mut self) { + self.status_writer.set(false); + } + pub fn partition_num(&self) -> ActorId { self.partition_ref.partition_num() } @@ -304,10 +312,14 @@ impl PartitionRef { pub fn initialize_existing_partition(partition_num: ActorId, event_stream_data_dir: &Path, event_stream_options: &EventStreamOptions, - highest_counter: HighestCounter) -> io::Result { - - // TODO: for now we are starting every partition as primary. This will need to change once we have a raft implementation - let status_writer = AtomicBoolWriter::with_value(true); + highest_counter: HighestCounter, + start_writable: bool) -> io::Result { + + // This will be the starting value, which will be used, even prior to determining + // the system primary. So, we only set this to true when running in standalone mode + // (no cluster). Otherwise, we'll start it off as `false` and flip it later, + // once we are in touch with the system primary + let status_writer = AtomicBoolWriter::with_value(start_writable); let primary_addr = Arc::new(RwLock::new(None)); let partition_data_dir = get_partition_data_dir(event_stream_data_dir, partition_num); @@ -324,10 +336,10 @@ pub fn initialize_existing_partition(partition_num: ActorId, pub fn initialize_new_partition(partition_num: ActorId, event_stream_data_dir: &Path, event_stream_options: &EventStreamOptions, - highest_counter: HighestCounter) -> io::Result { + highest_counter: HighestCounter, + start_writable: bool) -> io::Result { - // TODO: for now we are starting every partition as primary. This will need to change once we have a raft implementation - let status_writer = AtomicBoolWriter::with_value(true); + let status_writer = AtomicBoolWriter::with_value(start_writable); let primary_addr = Arc::new(RwLock::new(None)); let partition_data_dir = get_partition_data_dir(event_stream_data_dir, partition_num); diff --git a/flo-server/tests/embedded_tests.rs b/flo-server/tests/embedded_tests.rs index f9daf24..05d6f7a 100644 --- a/flo-server/tests/embedded_tests.rs +++ b/flo-server/tests/embedded_tests.rs @@ -128,7 +128,7 @@ fn oldest_events_are_dropped_from_beginning_of_stream_after_time_based_expiratio } let start_time = ::std::time::Instant::now(); - let until_all_expired = (retention_duration + segment_duration + chrono::Duration::milliseconds(750)).to_std().unwrap(); + let until_all_expired = (retention_duration + segment_duration + chrono::Duration::milliseconds(3000)).to_std().unwrap(); let mut first_event_id = FloEventId::new(1, 0); let mut vv = VersionVector::new(); vv.set(first_event_id); From b8264e1a49e3712b46375e5dfb0333f3c27b8336 Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 12 Jun 2018 20:24:13 -0400 Subject: [PATCH 69/73] adjust system event types in prep for cluster membership rewrite --- .../src/engine/controller/controller_state.rs | 8 +++-- .../src/engine/controller/system_event.rs | 30 +++++++++++-------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/flo-server/src/engine/controller/controller_state.rs b/flo-server/src/engine/controller/controller_state.rs index 0bfc9af..a1a88d0 100644 --- a/flo-server/src/engine/controller/controller_state.rs +++ b/flo-server/src/engine/controller/controller_state.rs @@ -223,6 +223,7 @@ pub mod mock { use super::*; use std::collections::BTreeMap; use engine::controller::SystemEventKind; + use engine::controller::system_event::ClusterMember; #[derive(Debug)] pub struct MockControllerState { @@ -367,9 +368,12 @@ pub mod mock { impl MockSystemEvent { pub fn with_any_data(counter: EventCounter, term: Term, segment: SegmentNum, file_offset: usize) -> MockSystemEvent { - use engine::controller::system_event::ClusterMemberJoining; + use test_utils::addr; - let data = SystemEventKind::NewClusterMemberJoining(ClusterMemberJoining { new_member: 45678 }); + let data = SystemEventKind::NewClusterMemberJoining(ClusterMember { + id: 45678, + address: addr("127.0.0.1:3000") + }); MockSystemEvent::with_data(counter, term, segment, file_offset, data) } diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index 18b3a40..ca20070 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -3,6 +3,7 @@ use rmp_serde::decode::Error; use protocol::{Term, FloInstanceId}; use event::{FloEvent, EventData, FloEventId, EventCounter, OwnedFloEvent, ActorId, Timestamp}; use engine::event_stream::partition::PersistentEvent; +use std::net::SocketAddr; #[derive(Debug, PartialEq)] pub struct SystemEvent { @@ -106,29 +107,29 @@ impl FloEvent for SystemEvent { } } +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct ClusterMember { + pub id: FloInstanceId, + pub address: SocketAddr, +} #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] -pub struct ClusterMemberJoining { - pub new_member: FloInstanceId, +pub struct InitialClusterMembership { + peers: Vec, } #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] -pub struct ClusterMemberJoined { +pub struct PartitionAssigned { pub new_member: FloInstanceId, pub new_member_partition_num: ActorId, } -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] -pub struct ClusterMemberOnline { - pub member: FloInstanceId, - pub member_partition_num: ActorId, -} #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub enum SystemEventKind { - NewClusterMemberJoining(ClusterMemberJoining), - NewClusterMemberJoined(ClusterMemberJoined), - ExistingMemberOnline(ClusterMemberOnline), + ClusterInitialized(InitialClusterMembership), + NewClusterMemberJoining(ClusterMember), + NewClusterMemberJoined(PartitionAssigned), } #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] @@ -159,10 +160,13 @@ mod test { #[test] fn system_event_data_is_serialized_and_deserialized_inside_system_event() { + use ::test_utils::addr; + let data = SystemEventData { term: 33, - kind: SystemEventKind::NewClusterMemberJoining(ClusterMemberJoining { - new_member: 555, + kind: SystemEventKind::NewClusterMemberJoining(ClusterMember { + id: 555, + address: addr("127.0.0.1:3000"), }) }; let id = FloEventId::new(3, 4); From 84ca31b22d799ad710c3e7ed794c5ff3cc26b0b9 Mon Sep 17 00:00:00 2001 From: pfried Date: Tue, 12 Jun 2018 23:08:49 -0400 Subject: [PATCH 70/73] start having system events applied to persistent state --- .../engine/controller/cluster_state/mod.rs | 86 ++++----- .../cluster_state/peer_connections.rs | 15 +- .../{persistent.rs => persistent/file.rs} | 68 +------ .../cluster_state/persistent/mod.rs | 166 ++++++++++++++++++ .../src/engine/controller/controller_state.rs | 5 +- .../src/engine/controller/system_event.rs | 15 +- 6 files changed, 229 insertions(+), 126 deletions(-) rename flo-server/src/engine/controller/cluster_state/{persistent.rs => persistent/file.rs} (58%) create mode 100644 flo-server/src/engine/controller/cluster_state/persistent/mod.rs diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 229a6de..ae4fc8a 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -84,8 +84,8 @@ impl ClusterManager { peer_connection_manager: Box) -> ClusterManager { let mut initialization_peers = starting_peer_addresses.iter().cloned().collect::>(); - for peer in persistent.cluster_members.iter() { - initialization_peers.insert(peer.address); + for peer_address in persistent.get_all_peer_addresses() { + initialization_peers.insert(*peer_address); } ClusterManager { @@ -197,7 +197,7 @@ impl ClusterManager { self.persistent.current_term <= request_term && self.last_applied <= request.last_log_index && self.last_applied_term <= request.last_log_term && - self.persistent.cluster_members.iter().any(|peer| peer.id == request.candidate_id) + self.persistent.contains_peer(request.candidate_id) } fn count_vote_response(&mut self, peer_id: FloInstanceId) -> bool { @@ -206,7 +206,7 @@ impl ClusterManager { } else { trace!("Not counting duplicate vote from peer_id: {:?}", peer_id); } - let peer_count = self.persistent.cluster_members.len() as ActorId; + let peer_count = self.persistent.get_voting_peer_count(); let vote_count = self.votes_received.len() as ActorId; let required_count = ::engine::minimum_required_votes_for_majority(peer_count); let election_won = vote_count >= required_count; @@ -310,9 +310,9 @@ impl ConsensusProcessor for ClusterManager { } let connection = connection.unwrap(); - if !self.persistent.cluster_members.contains(&upgrade.peer) { + if !self.persistent.contains_peer(upgrade.peer.id) { self.persistent.modify(|state| { - state.cluster_members.insert(upgrade.peer.clone()); + state.add_peer(&upgrade.peer); }).expect(STATE_UPDATE_FAILED); let mut lock = self.shared.write().unwrap(); @@ -467,7 +467,7 @@ impl ConsensusProcessor for ClusterManager { let ClusterManager { ref mut primary_state, ref mut connection_manager, ref persistent, ..} = *self; primary_state.as_mut().map(|state| { - let all_peers = persistent.cluster_members.iter().map(|peer| peer.id); + let all_peers = persistent.get_all_peer_ids().cloned(); state.send_append_entries(controller_state, connection_manager.as_mut(), all_peers); }); } @@ -515,7 +515,7 @@ pub fn init_cluster_consensus_processor(persistent_state: FilePersistedState, let ClusterOptions{ peer_addresses, event_loop_handles, election_timeout_millis, this_instance_address, .. } = options; let outgoing_connection_creator = OutgoingConnectionCreatorImpl::new(event_loop_handles, engine_ref); - let peer_connection_manager = PeerConnections::new(peer_addresses.clone(), Box::new(outgoing_connection_creator), &persistent_state.cluster_members); + let peer_connection_manager = PeerConnections::new(peer_addresses.clone(), Box::new(outgoing_connection_creator), persistent_state.get_all_peers()); let state = ClusterManager::new(election_timeout_millis, peer_addresses, @@ -564,8 +564,8 @@ mod test { let subject_id = subject.persistent.this_instance_id; subject.persistent.modify(|state| { - state.cluster_members.insert(peer_1.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&peer_1); + state.add_peer(&peer_2); state.current_term = 5; }).unwrap(); subject.current_primary = Some(subject_id); @@ -609,8 +609,8 @@ mod test { let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); subject.persistent.modify(|state| { - state.cluster_members.insert(peer_1.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&peer_1); + state.add_peer(&peer_2); state.current_term = 5; }).unwrap(); subject.last_applied = 99; @@ -673,8 +673,8 @@ mod test { let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); subject.persistent.modify(|state| { - state.cluster_members.insert(peer_1.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&peer_1); + state.add_peer(&peer_2); state.current_term = 5; }).unwrap(); subject.last_applied = 99; @@ -736,8 +736,8 @@ mod test { subject.persistent.modify(|state| { let this_id = state.this_instance_id; state.voted_for = Some(this_id); - state.cluster_members.insert(peer_1.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&peer_1); + state.add_peer(&peer_2); state.current_term = 5; }).unwrap(); subject.last_applied = 99; @@ -787,8 +787,8 @@ mod test { subject.persistent.modify(|state| { let this_id = state.this_instance_id; state.voted_for = Some(this_id); - state.cluster_members.insert(peer_1.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&peer_1); + state.add_peer(&peer_2); state.current_term = 5; }).unwrap(); subject.state = State::Voted; @@ -836,10 +836,10 @@ mod test { let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); subject.persistent.modify(|state| { - state.cluster_members.insert(peer_1.clone()); - state.cluster_members.insert(peer_2.clone()); - state.cluster_members.insert(peer_3.clone()); - state.cluster_members.insert(peer_4.clone()); + state.add_peer(&peer_1); + state.add_peer(&peer_2); + state.add_peer(&peer_3); + state.add_peer(&peer_4); state.current_term = 5; }).unwrap(); subject.state = State::Voted; @@ -878,8 +878,8 @@ mod test { subject.persistent.modify(|state| { let this_id = state.this_instance_id; state.voted_for = Some(this_id); - state.cluster_members.insert(peer_1.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&peer_1); + state.add_peer(&peer_2); state.current_term = 5; }).unwrap(); subject.state = State::Voted; @@ -927,10 +927,10 @@ mod test { let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); subject.persistent.modify(|state| { - state.cluster_members.insert(peer_1.clone()); - state.cluster_members.insert(peer_2.clone()); - state.cluster_members.insert(peer_3.clone()); - state.cluster_members.insert(peer_4.clone()); + state.add_peer(&peer_1); + state.add_peer(&peer_2); + state.add_peer(&peer_3); + state.add_peer(&peer_4); state.current_term = 5; }).unwrap(); subject.state = State::Follower; @@ -972,8 +972,8 @@ mod test { subject.last_applied = request.last_log_index - 2; subject.persistent.modify(|state| { state.current_term = request.term - 1; - state.cluster_members.insert(candidate.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&candidate); + state.add_peer(&peer_2); }).unwrap(); VoteExpectation { @@ -991,8 +991,8 @@ mod test { subject.last_applied = request.last_log_index; subject.persistent.modify(|state| { state.current_term = request.term + 1; // my current term is greater - state.cluster_members.insert(candidate.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&candidate); + state.add_peer(&peer_2); }).unwrap(); VoteExpectation { @@ -1012,8 +1012,8 @@ mod test { subject.last_applied = request.last_log_index; subject.persistent.modify(|state| { state.current_term = request.term; - state.cluster_members.insert(candidate.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&candidate); + state.add_peer(&peer_2); }).unwrap(); VoteExpectation { @@ -1032,7 +1032,7 @@ mod test { subject.persistent.modify(|state| { state.current_term = request.term - 1; // peer_1 is not a known member - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&peer_2); }).unwrap(); VoteExpectation { @@ -1051,8 +1051,8 @@ mod test { subject.persistent.modify(|state| { state.voted_for = None; state.current_term = request.term; - state.cluster_members.insert(candidate.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&candidate); + state.add_peer(&peer_2); }).unwrap(); VoteExpectation { @@ -1071,8 +1071,8 @@ mod test { subject.persistent.modify(|state| { state.voted_for = Some(peer_2.id); // already voted for peer 2 state.current_term = request.term; - state.cluster_members.insert(candidate.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&candidate); + state.add_peer(&peer_2); }).unwrap(); VoteExpectation { @@ -1090,8 +1090,8 @@ mod test { subject.last_applied = request.last_log_index; subject.persistent.modify(|state| { state.current_term = request.term - 1; - state.cluster_members.insert(candidate.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&candidate); + state.add_peer(&peer_2); }).unwrap(); VoteExpectation { @@ -1110,8 +1110,8 @@ mod test { subject.persistent.modify(|state| { state.current_term = request.term - 1; state.voted_for = Some(state.this_instance_id); - state.cluster_members.insert(candidate.clone()); - state.cluster_members.insert(peer_2.clone()); + state.add_peer(&candidate); + state.add_peer(&peer_2); }).unwrap(); VoteExpectation { diff --git a/flo-server/src/engine/controller/cluster_state/peer_connections.rs b/flo-server/src/engine/controller/cluster_state/peer_connections.rs index a74cec0..97d2d9d 100644 --- a/flo-server/src/engine/controller/cluster_state/peer_connections.rs +++ b/flo-server/src/engine/controller/cluster_state/peer_connections.rs @@ -3,7 +3,7 @@ use engine::ConnectionId; use engine::controller::{ConnectionRef, ControllerState, Peer}; use engine::controller::peer_connection::OutgoingConnectionCreator; use protocol::FloInstanceId; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::fmt::Debug; use std::net::SocketAddr; use std::time::{Duration, Instant}; @@ -28,7 +28,7 @@ pub struct PeerConnections { } impl PeerConnections { - pub fn new(starting_peer_addresses: Vec, outgoing_connection_creator: Box, peers: &HashSet) -> PeerConnections { + pub fn new(starting_peer_addresses: Vec, outgoing_connection_creator: Box, peers: Vec) -> PeerConnections { let known_peers = peers.iter().map(|peer| { let connection = Connection::new(peer.address); (peer.id, connection) @@ -299,7 +299,7 @@ mod test { } fn subject_with_connected_peers(peers: &[Peer], creator: MockOutgoingConnectionCreator) -> (PeerConnections, MockControllerState) { - let mut subject = PeerConnections::new(Vec::new(), creator.boxed(), &peers.iter().cloned().collect()); + let mut subject = PeerConnections::new(Vec::new(), creator.boxed(), peers.to_owned()); let mut controller_state = MockControllerState::new(); subject.establish_connections(Instant::now(), &mut controller_state); for peer in peers { @@ -371,7 +371,7 @@ mod test { let mut creator = MockOutgoingConnectionCreator::new(); let (peer_connection, _rx) = creator.stub(peer_address); - let mut subject = PeerConnections::new(vec![peer_address], creator.boxed(), &HashSet::new()); + let mut subject = PeerConnections::new(vec![peer_address], creator.boxed(), Vec::new()); let mut controller_state = MockControllerState::new(); let time = Instant::now(); @@ -400,9 +400,8 @@ mod test { }; let mut creator = MockOutgoingConnectionCreator::new(); creator.stub(peer_address); - let mut all_peers = HashSet::new(); - all_peers.insert(peer.clone()); - let subject = PeerConnections::new(Vec::new(), creator.boxed(), &all_peers); + let all_peers = vec![peer.clone()]; + let subject = PeerConnections::new(Vec::new(), creator.boxed(), all_peers); assert!(subject.disconnected_peers.contains_key(&peer_address)); } @@ -413,7 +412,7 @@ mod test { let new_peers = vec![peer_address]; let mut creator = MockOutgoingConnectionCreator::new(); creator.stub(peer_address); - let mut subject = PeerConnections::new(new_peers, creator.boxed(), &HashSet::new()); + let mut subject = PeerConnections::new(new_peers, creator.boxed(), Vec::new()); let mut controller_state = MockControllerState::new(); let time = Instant::now(); diff --git a/flo-server/src/engine/controller/cluster_state/persistent.rs b/flo-server/src/engine/controller/cluster_state/persistent/file.rs similarity index 58% rename from flo-server/src/engine/controller/cluster_state/persistent.rs rename to flo-server/src/engine/controller/cluster_state/persistent/file.rs index fa0d1bb..232bcb3 100644 --- a/flo-server/src/engine/controller/cluster_state/persistent.rs +++ b/flo-server/src/engine/controller/cluster_state/persistent/file.rs @@ -1,56 +1,9 @@ -use std::net::SocketAddr; use std::path::PathBuf; use std::fs::{File, OpenOptions}; use std::io::{self, Seek, SeekFrom}; -use std::collections::HashSet; - -use event::ActorId; -use protocol::Term; -use protocol::flo_instance_id::{self, FloInstanceId}; -use engine::controller::controller_messages::Peer; -use super::SharedClusterState; - -/// Holds all the cluster state that we want to survive a reboot. -/// We always persist the `FloInstanceId` because we prefer that to be stable across reboots. We do _not_ want to persist -/// the `SocketAddr` for the server, though, since that may well change after a restart, depending on environment. -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] -pub struct PersistentClusterState { - pub current_term: Term, - pub voted_for: Option, - pub this_instance_id: FloInstanceId, - pub this_partition_num: Option, - pub cluster_members: HashSet, -} -impl PersistentClusterState { - /// called during system startup to initialize the shared cluster state that will be available to all the connection handlers - pub fn initialize_shared_state(&self, this_address: Option) -> SharedClusterState { - - SharedClusterState { - this_partition_num: self.this_partition_num, - this_instance_id: self.this_instance_id, - this_address, - system_primary: None, // we are still starting up, so we have no idea who's primary - peers: self.cluster_members.clone(), - } - } +use super::PersistentClusterState; - pub fn generate_new() -> PersistentClusterState { - /* - TODO: generate new clusterState with a null FloInstanceId and have the primary assign the real one - Technically, there's a _very_ small chance that two separate instances could generate the same instance id. - The way around this is to have the system primary node just generate and assign all instance ids. This would - change the startup procedure significantly to account for that new step before the node enters a normal state - */ - PersistentClusterState { - current_term: 0, - voted_for: None, - this_instance_id: flo_instance_id::generate_new(), - this_partition_num: None, - cluster_members: HashSet::new(), - } - } -} /// Placeholder for a wrapper struct that will take care of persisting the state as it changes #[derive(Debug)] @@ -124,8 +77,10 @@ impl ::std::ops::Deref for FilePersistedState { #[cfg(test)] mod test { use super::*; + use engine::controller::Peer; use tempdir::TempDir; use test_utils::addr; + use protocol::flo_instance_id; #[test] fn modifying_state_persists_changes() { @@ -137,20 +92,9 @@ mod test { let mut subject = FilePersistedState::initialize(path.clone()).unwrap(); subject.modify(|state| { state.current_term = 9; - state.cluster_members = [ - Peer { - id: flo_instance_id::generate_new(), - address: addr("127.0.0.1:3456") - }, - Peer { - id: flo_instance_id::generate_new(), - address: addr("[2001:873::1]:3000") - }, - Peer { - id: flo_instance_id::generate_new(), - address: addr("127.0.0.1:456") - } - ].iter().cloned().collect(); + state.add_peer(&Peer { id: flo_instance_id::generate_new(), address: addr("127.0.0.1:3456") }); + state.add_peer(&Peer { id: flo_instance_id::generate_new(), address: addr("[2001:873::1]:3000") }); + state.add_peer(&Peer { id: flo_instance_id::generate_new(), address: addr("127.0.0.1:456") }); }).unwrap(); let subject2 = FilePersistedState::initialize(path.clone()).unwrap(); diff --git a/flo-server/src/engine/controller/cluster_state/persistent/mod.rs b/flo-server/src/engine/controller/cluster_state/persistent/mod.rs new file mode 100644 index 0000000..63f51dc --- /dev/null +++ b/flo-server/src/engine/controller/cluster_state/persistent/mod.rs @@ -0,0 +1,166 @@ +mod file; + +use std::net::SocketAddr; +use std::collections::HashMap; + +use event::{ActorId, FloEvent, EventCounter}; +use protocol::Term; +use protocol::flo_instance_id::{self, FloInstanceId}; +use engine::controller::controller_messages::Peer; +use engine::controller::system_event::*; +use super::SharedClusterState; + +pub use self::file::FilePersistedState; + +/// Holds all the cluster state that we want to survive a reboot. +/// We always persist the `FloInstanceId` because we prefer that to be stable across reboots. We do _not_ want to persist +/// the `SocketAddr` for the server, though, since that may well change after a restart, depending on environment. +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct PersistentClusterState { + pub current_term: Term, + pub voted_for: Option, + pub this_instance_id: FloInstanceId, + pub this_partition_num: Option, + last_applied: EventCounter, + cluster_members: HashMap, + assigned_partitions: HashMap, +} + +impl PersistentClusterState { + /// called during system startup to initialize the shared cluster state that will be available to all the connection handlers + pub fn initialize_shared_state(&self, this_address: Option) -> SharedClusterState { + + SharedClusterState { + this_partition_num: self.this_partition_num, + this_instance_id: self.this_instance_id, + this_address, + system_primary: None, // we are still starting up, so we have no idea who's primary + peers: self.cluster_members.iter().map(|(id, address)| Peer{id: *id, address: *address}).collect() + } + } + + pub fn generate_new() -> PersistentClusterState { + PersistentClusterState { + last_applied: 0, + current_term: 0, + voted_for: None, + this_instance_id: flo_instance_id::generate_new(), + this_partition_num: None, + cluster_members: HashMap::new(), + assigned_partitions: HashMap::new(), + } + } + + pub fn apply_system_event(&mut self, event: &SystemEvent) { + let counter = event.counter(); + let term = event.term(); + + match event.deserialized_data.kind { + SystemEventKind::ClusterInitialized(ref initial_membership) => { + for peer in initial_membership.peers.iter() { + self.add_peer(peer); + } + } + SystemEventKind::NewClusterMemberJoining(ref new_peer) => { + self.add_peer(new_peer); + } + _ => {} + } + self.last_applied = self.last_applied.max(counter); + self.current_term = self.current_term.max(term); + } + + pub fn add_peer(&mut self, peer: &Peer) { + self.cluster_members.insert(peer.id, peer.address); + } + + pub fn contains_peer(&self, peer_id: FloInstanceId) -> bool { + self.cluster_members.contains_key(&peer_id) + } + + pub fn get_all_peer_ids(&self) -> impl Iterator { + self.cluster_members.keys() + } + + pub fn get_all_peer_addresses(&self) -> impl Iterator { + self.cluster_members.values() + } + + pub fn get_all_peers(&self) -> Vec { + self.cluster_members.iter().map(|(id, address)| { + Peer {id: *id, address: *address} + }).collect() + } + + pub fn get_voting_peer_count(&self) -> ActorId { + self.cluster_members.len() as ActorId + } +} + + +#[cfg(test)] +mod test { + use super::*; + use event::{OwnedFloEvent, FloEventId, time}; + use test_utils::addr; + + #[test] + fn applying_new_member_joining_adds_a_cluster_member() { + let initial_peers = initial_peers(); + let mut subject = subject_with_initial_peers(initial_peers.clone()); + let new_peer = new_peer(4); + let event = sys_event(2, 7, SystemEventKind::NewClusterMemberJoining(new_peer.clone())); + subject.apply_system_event(&event); + + assert_eq!(2, subject.last_applied); + assert_eq!(7, subject.current_term); + + let mut expected_peers = initial_peers; + expected_peers.push(new_peer); + assert_peer_groups_equal(expected_peers, subject.get_all_peers()); + assert!(subject.assigned_partitions.is_empty()); + } + + #[test] + fn applying_initial_membership_sets_cluster_members() { + let peers = initial_peers(); + let subject = subject_with_initial_peers(peers.clone()); + + assert_eq!(1, subject.last_applied); + assert_eq!(1, subject.current_term); + assert_peer_groups_equal(peers, subject.get_all_peers()); + assert!(subject.assigned_partitions.is_empty()); + } + + fn assert_peer_groups_equal, A: IntoIterator>(e: E, a: A) { + let actual = a.into_iter().collect::>(); + let expected = e.into_iter().collect::>(); + assert_eq!(expected, actual); + } + + fn subject_with_initial_peers(peers: Vec) -> PersistentClusterState { + let mut subject = PersistentClusterState::generate_new(); + let event = sys_event(1, 1, SystemEventKind::ClusterInitialized(InitialClusterMembership { + peers + })); + subject.apply_system_event(&event); + subject + } + + fn initial_peers() -> Vec { + vec![ + new_peer(3000), + new_peer(3001), + new_peer(3002), + ] + } + + fn new_peer(port: u16) -> Peer { + Peer { id: flo_instance_id::generate_new(), address: addr(&format!("127.0.0.1:{}", port)) } + } + + fn sys_event(counter: EventCounter, term: Term, kind: SystemEventKind) -> SystemEvent { + let data = SystemEventData { term, kind }; + SystemEvent::new(FloEventId::new(0, counter), None, "/system".to_owned(), time::from_millis_since_epoch(44), data) + } +} diff --git a/flo-server/src/engine/controller/controller_state.rs b/flo-server/src/engine/controller/controller_state.rs index a1a88d0..5847f95 100644 --- a/flo-server/src/engine/controller/controller_state.rs +++ b/flo-server/src/engine/controller/controller_state.rs @@ -222,8 +222,7 @@ impl ControllerState for ControllerStateImpl { pub mod mock { use super::*; use std::collections::BTreeMap; - use engine::controller::SystemEventKind; - use engine::controller::system_event::ClusterMember; + use engine::controller::{SystemEventKind, Peer}; #[derive(Debug)] pub struct MockControllerState { @@ -370,7 +369,7 @@ pub mod mock { pub fn with_any_data(counter: EventCounter, term: Term, segment: SegmentNum, file_offset: usize) -> MockSystemEvent { use test_utils::addr; - let data = SystemEventKind::NewClusterMemberJoining(ClusterMember { + let data = SystemEventKind::NewClusterMemberJoining(Peer { id: 45678, address: addr("127.0.0.1:3000") }); diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index ca20070..443b81c 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -3,7 +3,7 @@ use rmp_serde::decode::Error; use protocol::{Term, FloInstanceId}; use event::{FloEvent, EventData, FloEventId, EventCounter, OwnedFloEvent, ActorId, Timestamp}; use engine::event_stream::partition::PersistentEvent; -use std::net::SocketAddr; +use engine::controller::controller_messages::Peer; #[derive(Debug, PartialEq)] pub struct SystemEvent { @@ -107,18 +107,13 @@ impl FloEvent for SystemEvent { } } -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] -pub struct ClusterMember { - pub id: FloInstanceId, - pub address: SocketAddr, -} #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub struct InitialClusterMembership { - peers: Vec, + pub peers: Vec, } -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)] pub struct PartitionAssigned { pub new_member: FloInstanceId, pub new_member_partition_num: ActorId, @@ -128,7 +123,7 @@ pub struct PartitionAssigned { #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub enum SystemEventKind { ClusterInitialized(InitialClusterMembership), - NewClusterMemberJoining(ClusterMember), + NewClusterMemberJoining(Peer), NewClusterMemberJoined(PartitionAssigned), } @@ -164,7 +159,7 @@ mod test { let data = SystemEventData { term: 33, - kind: SystemEventKind::NewClusterMemberJoining(ClusterMember { + kind: SystemEventKind::NewClusterMemberJoining(Peer { id: 555, address: addr("127.0.0.1:3000"), }) From f910402395f18926a9e73646d89f76f07790f1e1 Mon Sep 17 00:00:00 2001 From: pfried Date: Wed, 13 Jun 2018 22:57:09 -0400 Subject: [PATCH 71/73] apply the remaining system event to persistent state --- .../cluster_state/persistent/mod.rs | 41 ++++++++++++++++++- .../src/engine/controller/system_event.rs | 6 +-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/flo-server/src/engine/controller/cluster_state/persistent/mod.rs b/flo-server/src/engine/controller/cluster_state/persistent/mod.rs index 63f51dc..a3884bc 100644 --- a/flo-server/src/engine/controller/cluster_state/persistent/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/persistent/mod.rs @@ -64,7 +64,16 @@ impl PersistentClusterState { SystemEventKind::NewClusterMemberJoining(ref new_peer) => { self.add_peer(new_peer); } - _ => {} + SystemEventKind::AssignPartition(ref assigned) => { + let peer_id = assigned.peer_id; + if !self.cluster_members.contains_key(&peer_id) { + error!("Fatal error: AssignPartition event refers to an unknown peer_id, which indicates \ + that the system log is inconsistent! Offending event counter: {} term: {}, kind: \ + AssignPartition({:?}), current state: {:?}", counter, term, assigned, self); + panic!("System controller encountered a fatal error"); + } + self.assigned_partitions.insert(assigned.partition_num, peer_id); + } } self.last_applied = self.last_applied.max(counter); self.current_term = self.current_term.max(term); @@ -103,6 +112,36 @@ mod test { use super::*; use event::{OwnedFloEvent, FloEventId, time}; use test_utils::addr; + use std::collections::HashSet; + + #[test] + #[should_panic] + fn applying_partition_assigned_panics_when_peer_is_not_a_cluster_member() { + let mut subject = subject_with_initial_peers(initial_peers()); + let event = sys_event(2, 7, SystemEventKind::AssignPartition(PartitionAssigned { + peer_id: flo_instance_id::generate_new(), + partition_num: 1, + })); + subject.apply_system_event(&event); + } + + #[test] + fn applying_partition_assigned_adds_member_to_the_assigned_members_map() { + let initial_peers = initial_peers(); + let mut subject = subject_with_initial_peers(initial_peers.clone()); + assert!(subject.assigned_partitions.is_empty()); + + let event = sys_event(2, 7, SystemEventKind::AssignPartition(PartitionAssigned { + peer_id: initial_peers[0].id, + partition_num: 3, + })); + subject.apply_system_event(&event); + + assert_eq!(1, subject.assigned_partitions.len()); + assert_eq!(Some(initial_peers[0].id), subject.assigned_partitions.get(&3).cloned()); + assert_eq!(2, subject.last_applied); + assert_eq!(7, subject.current_term); + } #[test] fn applying_new_member_joining_adds_a_cluster_member() { diff --git a/flo-server/src/engine/controller/system_event.rs b/flo-server/src/engine/controller/system_event.rs index 443b81c..db48fa7 100644 --- a/flo-server/src/engine/controller/system_event.rs +++ b/flo-server/src/engine/controller/system_event.rs @@ -115,8 +115,8 @@ pub struct InitialClusterMembership { #[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)] pub struct PartitionAssigned { - pub new_member: FloInstanceId, - pub new_member_partition_num: ActorId, + pub peer_id: FloInstanceId, + pub partition_num: ActorId, } @@ -124,7 +124,7 @@ pub struct PartitionAssigned { pub enum SystemEventKind { ClusterInitialized(InitialClusterMembership), NewClusterMemberJoining(Peer), - NewClusterMemberJoined(PartitionAssigned), + AssignPartition(PartitionAssigned), } #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] From ed17aa2c6cb9a2e73aa4c8dacc2a1a131040de48 Mon Sep 17 00:00:00 2001 From: pfried Date: Fri, 15 Jun 2018 22:01:59 -0400 Subject: [PATCH 72/73] process acknowledgements from peers --- .../engine/controller/cluster_state/mod.rs | 24 +++++++------ .../cluster_state/persistent/mod.rs | 35 ++++++++++--------- .../src/engine/controller/controller_state.rs | 10 ++++++ .../event_stream/partition/controller/mod.rs | 5 +-- 4 files changed, 46 insertions(+), 28 deletions(-) diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index ae4fc8a..3af469e 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -116,9 +116,7 @@ impl ClusterManager { new_commit_index, term, self.last_applied, last_commit_index); controller_state.set_commit_index(new_commit_index); - // while self.last_applied < new_commit_index { - // let event = controller_state.get_next_event(self.last_applied) - // } + // TODO: apply system events } fn transition_state(&mut self, new_state: State) { @@ -310,7 +308,7 @@ impl ConsensusProcessor for ClusterManager { } let connection = connection.unwrap(); - if !self.persistent.contains_peer(upgrade.peer.id) { + if controller_state.get_commit_index() == 0 && !self.persistent.contains_peer(upgrade.peer.id) { self.persistent.modify(|state| { state.add_peer(&upgrade.peer); }).expect(STATE_UPDATE_FAILED); @@ -436,7 +434,7 @@ impl ConsensusProcessor for ClusterManager { })); } - fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, _controller_state: &mut ControllerState) { + fn append_entries_response_received(&mut self, connection_id: ConnectionId, response: AppendEntriesResponse, controller_state: &mut ControllerState) { let from = self.connection_manager.get_peer_id(connection_id); if from.is_none() { error!("Received AppendEntriesResponse from connection_id: {}, which is not a peer connection. Received: {:?}", connection_id, response); @@ -444,8 +442,9 @@ impl ConsensusProcessor for ClusterManager { } let peer_id = from.unwrap(); let response_term = response.term; + let current_term = self.persistent.current_term; - if response_term > self.persistent.current_term { + if response_term > current_term { // Apparently, we've fallen behind! This instance is no longer primary, so we need to step down if we haven't already warn!("Peer: {:?} responded with term: {}, which is higher than the current term: {}. Stepping down as primary", peer_id, response_term, self.persistent.current_term); @@ -455,11 +454,16 @@ impl ConsensusProcessor for ClusterManager { self.persistent.modify(|state| { state.current_term = response_term; }).expect(STATE_UPDATE_FAILED) - } - if response.success.is_some() && self.is_primary() { - let _peer_counter = response.success.unwrap(); - // TODO: confirm entries with partition impl + } else if response.success.is_some() && self.is_primary() { + // cannot be a success if response_term > current_term + let peer_counter = response.success.unwrap(); + let new_commit_index = controller_state.system_event_ack(peer_id, peer_counter); + if let Some(index) = new_commit_index { + self.update_commit_index(index, current_term, controller_state); + } + } else { + error!("Received invalid response from peer_id: {}, on connection_id: {} - {:?}", peer_id, connection_id, response); } } diff --git a/flo-server/src/engine/controller/cluster_state/persistent/mod.rs b/flo-server/src/engine/controller/cluster_state/persistent/mod.rs index a3884bc..90abe79 100644 --- a/flo-server/src/engine/controller/cluster_state/persistent/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/persistent/mod.rs @@ -54,29 +54,32 @@ impl PersistentClusterState { pub fn apply_system_event(&mut self, event: &SystemEvent) { let counter = event.counter(); let term = event.term(); + self.current_term = self.current_term.max(term); - match event.deserialized_data.kind { - SystemEventKind::ClusterInitialized(ref initial_membership) => { - for peer in initial_membership.peers.iter() { - self.add_peer(peer); + if counter > self.last_applied { + match event.deserialized_data.kind { + SystemEventKind::ClusterInitialized(ref initial_membership) => { + for peer in initial_membership.peers.iter() { + self.add_peer(peer); + } } - } - SystemEventKind::NewClusterMemberJoining(ref new_peer) => { - self.add_peer(new_peer); - } - SystemEventKind::AssignPartition(ref assigned) => { - let peer_id = assigned.peer_id; - if !self.cluster_members.contains_key(&peer_id) { - error!("Fatal error: AssignPartition event refers to an unknown peer_id, which indicates \ + SystemEventKind::NewClusterMemberJoining(ref new_peer) => { + self.add_peer(new_peer); + } + SystemEventKind::AssignPartition(ref assigned) => { + let peer_id = assigned.peer_id; + if !self.cluster_members.contains_key(&peer_id) { + error!("Fatal error: AssignPartition event refers to an unknown peer_id, which indicates \ that the system log is inconsistent! Offending event counter: {} term: {}, kind: \ AssignPartition({:?}), current state: {:?}", counter, term, assigned, self); - panic!("System controller encountered a fatal error"); + panic!("System controller encountered a fatal error"); + } + self.assigned_partitions.insert(assigned.partition_num, peer_id); } - self.assigned_partitions.insert(assigned.partition_num, peer_id); } + self.last_applied = counter; } - self.last_applied = self.last_applied.max(counter); - self.current_term = self.current_term.max(term); + } pub fn add_peer(&mut self, peer: &Peer) { diff --git a/flo-server/src/engine/controller/controller_state.rs b/flo-server/src/engine/controller/controller_state.rs index 5847f95..4b93ed8 100644 --- a/flo-server/src/engine/controller/controller_state.rs +++ b/flo-server/src/engine/controller/controller_state.rs @@ -45,6 +45,7 @@ pub trait ControllerState { fn get_current_file_offset(&self) -> (SegmentNum, usize); fn add_system_replication_node(&mut self, peer: FloInstanceId); + fn system_event_ack(&mut self, peer: FloInstanceId, event: EventCounter) -> Option; /// Called only by the clusterState and only when this instance is a follower fn replicate_system_events(&mut self, events: &[OwnedFloEvent]) -> io::Result; @@ -183,6 +184,7 @@ impl ControllerState for ControllerStateImpl { } } } + fn get_next_event<'a>(&'a mut self, start_after: EventCounter) -> Option>> { self.get_next_entry(start_after.saturating_sub(1)).map(|index_entry| { self.read_event(index_entry).map(|system_event| { @@ -207,6 +209,10 @@ impl ControllerState for ControllerStateImpl { self.system_partition.add_replication_node(peer); } + fn system_event_ack(&mut self, peer: FloInstanceId, event: EventCounter) -> Option { + self.system_partition.events_acknowledged(peer, event) + } + fn replicate_system_events(&mut self, events: &[OwnedFloEvent]) -> io::Result { self.system_partition.replicate_events(0, events) } @@ -324,6 +330,10 @@ pub mod mock { unimplemented!() } + fn system_event_ack(&mut self, _peer: u64, _event: u64) -> Option { + unimplemented!() + } + fn replicate_system_events(&mut self, events: &[OwnedFloEvent]) -> io::Result { let segment_num = self.system_events.values().last().map(|e| e.segment).unwrap_or(SegmentNum::new(1)); let mut file_offset = self.system_events.values().last().map(|e| e.file_offset).unwrap_or(0); diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index f6c27b5..18ef719 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -146,11 +146,11 @@ impl PartitionImpl { self.commit_manager.add_member(peer); } - pub fn events_acknowledged(&mut self, peer_id: FloInstanceId, counter: EventCounter) { + pub fn events_acknowledged(&mut self, peer_id: FloInstanceId, counter: EventCounter) -> Option { if !self.is_current_primary() { debug!("partition: {} ignoring events_acknowledged from peer: {} with counter: {} because this instance is no longer primary", self.partition_num, peer_id, counter); - return; + return None; } let new_index = self.commit_manager.acknowledgement_received(peer_id, counter); @@ -160,6 +160,7 @@ impl PartitionImpl { self.pending_produce_operations.commit_success(committed_event); self.consumer_manager.notify_committed(); } + new_index } pub fn event_stream_name(&self) -> &str { From f7c94f0ed557a09bb8d928f19bb87f37707fe286 Mon Sep 17 00:00:00 2001 From: pfried Date: Sun, 24 Jun 2018 10:20:55 -0400 Subject: [PATCH 73/73] backup commit - squash me --- .../engine/controller/cluster_state/mod.rs | 161 +++++++++++------- .../cluster_state/persistent/file.rs | 2 +- .../cluster_state/persistent/mod.rs | 28 +-- .../src/engine/controller/controller_state.rs | 59 +++++-- flo-server/src/engine/controller/mod.rs | 4 +- .../partition/controller/commit_manager.rs | 21 ++- .../event_stream/partition/controller/mod.rs | 4 + 7 files changed, 184 insertions(+), 95 deletions(-) diff --git a/flo-server/src/engine/controller/cluster_state/mod.rs b/flo-server/src/engine/controller/cluster_state/mod.rs index 3af469e..0dc6c18 100644 --- a/flo-server/src/engine/controller/cluster_state/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/mod.rs @@ -36,7 +36,7 @@ pub trait ConsensusProcessor: Send { fn outgoing_connection_failed(&mut self, connection_id: ConnectionId, address: SocketAddr); fn connection_closed(&mut self, connection_id: ConnectionId); - fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote); + fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote, controller_state: &mut ControllerState); fn vote_response_received(&mut self, now: Instant, from: ConnectionId, response: VoteResponse, controller: &mut ControllerState); fn append_entries_received(&mut self, connection_id: ConnectionId, append: ReceiveAppendEntries, controller_state: &mut ControllerState); @@ -54,8 +54,6 @@ pub struct ClusterManager { election_timeout: Duration, last_heartbeat: Instant, primary_status_writer: AtomicBoolWriter, - last_applied: EventCounter, - last_applied_term: Term, persistent: FilePersistedState, votes_received: HashSet, system_partition_primary_address: SystemPrimaryAddressRef, @@ -98,8 +96,6 @@ impl ClusterManager { current_primary: None, this_instance_address, initialization_peers, - last_applied: 0, - last_applied_term: 0, primary_status_writer, persistent, system_partition_primary_address, @@ -107,16 +103,50 @@ impl ClusterManager { } } - fn update_commit_index(&mut self, new_commit_index: EventCounter, term: Term, controller_state: &mut ControllerState) { - if new_commit_index <= self.last_applied { + fn last_applied(&self) -> EventCounter { + self.persistent.get_last_applied().0 + } + + + fn update_commit_index(&mut self, mut new_commit_index: EventCounter, term: Term, controller_state: &mut ControllerState) { + if new_commit_index <= self.last_applied() { + // check against last_applied, since it's possible that events could be committed but not applied, and we'd want to apply those now return; } let last_commit_index = controller_state.get_commit_index(); info!("Applying committed system events new_commit_index: {}, term: {}, last_applied: {}, last_committed: {}", - new_commit_index, term, self.last_applied, last_commit_index); + new_commit_index, term, self.last_applied(), last_commit_index); + + new_commit_index = new_commit_index.max(last_commit_index); controller_state.set_commit_index(new_commit_index); + let result = self.apply_system_events(new_commit_index, term, controller_state); + if let Err(err) = result { + error!("Failed to apply system event! last_applied: {}, err: {:?}", self.last_applied(), err); + } + } + + fn apply_system_events(&mut self, commit_index: EventCounter, _term: Term, controller_state: &mut ControllerState) -> Result<(), io::Error> { + while self.last_applied() < commit_index { + let next_result = controller_state.get_next_event(self.last_applied()); + if next_result.is_none() { + break; + } + let next_sys_event = next_result.unwrap()?; // return early if there's an error reading + self.persistent.modify(|state| { + state.apply_system_event(next_sys_event); + })?; // another early return on failure + } + self.apply_persistent_state(controller_state); + Ok(()) + } + + fn apply_persistent_state(&mut self, controller_state: &mut ControllerState) { + let ClusterManager {state, persistent, shared, ..} = self; + + let shared_lock = shared.lock().unwrap(); + + - // TODO: apply system events } fn transition_state(&mut self, new_state: State) { @@ -157,7 +187,7 @@ impl ClusterManager { (now - self.last_heartbeat) > self.election_timeout } - fn start_new_election(&mut self) { + fn start_new_election(&mut self, controller_state: &mut ControllerState) { info!("Starting new election with term: {}", self.persistent.current_term + 1); self.votes_received.clear(); self.persistent.modify(|state| { @@ -166,11 +196,20 @@ impl ClusterManager { state.voted_for = Some(my_id); }).expect(STATE_UPDATE_FAILED); + let (last_log_index, last_log_term) = match controller_state.get_last_uncommitted_event() { + Some(Ok(sys_event)) => (sys_event.counter, sys_event.data.term), + None => (0, 0), + Some(err) => { + error!("Refusing to start new election due to system partition read error: {:?}", err); + return; + } + }; + let connection_control = ConnectionControl::SendRequestVote(CallRequestVote { term: self.persistent.current_term, candidate_id: self.persistent.this_instance_id, - last_log_index: self.last_applied, - last_log_term: self.last_applied_term, + last_log_index, + last_log_term, }); self.connection_manager.broadcast_to_peers(connection_control); self.transition_state(State::Voted); @@ -178,7 +217,7 @@ impl ClusterManager { self.update_last_heartbeat_to_now(); } - fn can_grant_vote(&self, request: &CallRequestVote) -> bool { + fn can_grant_vote(&self, request: &CallRequestVote, controller_state: &mut ControllerState) -> bool { let request_term = request.term; let my_term = self.persistent.current_term; @@ -192,10 +231,21 @@ impl ClusterManager { } } - self.persistent.current_term <= request_term && - self.last_applied <= request.last_log_index && - self.last_applied_term <= request.last_log_term && - self.persistent.contains_peer(request.candidate_id) + controller_state.get_last_uncommitted_event().map(|read_result| { + match read_result { + Ok(sys_event) => { + // now actually check and return whether the candidate's log is at least as up to date as our own + self.persistent.current_term <= request_term && + sys_event.counter <= request.last_log_index && + sys_event.data.term <= request.last_log_term && + self.persistent.contains_peer(request.candidate_id) + } + Err(err) => { + error!("Refusing to grant vote due to system partition read error: {:?}", err); + false + } + } + }).unwrap_or(true) } fn count_vote_response(&mut self, peer_id: FloInstanceId) -> bool { @@ -284,7 +334,7 @@ impl ConsensusProcessor for ClusterManager { } _ => { if self.election_timed_out(now) { - self.start_new_election(); + self.start_new_election(controller_state); } } } @@ -334,10 +384,10 @@ impl ConsensusProcessor for ClusterManager { self.connection_manager.connection_closed(connection_id); } - fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote) { + fn request_vote_received(&mut self, from: ConnectionId, request: CallRequestVote, controller_state: &mut ControllerState) { let candidate_id = request.candidate_id; - let response = if self.can_grant_vote(&request) { + let response = if self.can_grant_vote(&request, controller_state) { self.persistent.modify(|state| { state.voted_for = Some(candidate_id); state.current_term = request.term; @@ -401,6 +451,7 @@ impl ConsensusProcessor for ClusterManager { self.update_last_heartbeat_to_now(); + // We won't send a response if there's an error let response = match controller_state.get_next_counter_and_term(append.prev_entry_index.saturating_sub(1)) { Some(Ok((actual_prev_index, actual_prev_term))) => { if actual_prev_index == append.prev_entry_index && actual_prev_term == append.prev_entry_term { @@ -416,7 +467,7 @@ impl ConsensusProcessor for ClusterManager { self.create_append_result(append.events.as_slice(), controller_state) } else { debug!("System partition with head at: {} is behind AppendEntries with index: {}, term: {}, returning negative result", - self.last_applied, append.prev_entry_index, append.prev_entry_term); + self.last_applied(), append.prev_entry_index, append.prev_entry_term); None } } @@ -573,8 +624,6 @@ mod test { state.current_term = 5; }).unwrap(); subject.current_primary = Some(subject_id); - subject.last_applied = 99; - subject.last_applied_term = 4; subject.state = State::Primary; subject.primary_state = Some(PrimaryState::new(5)); subject.last_heartbeat = start; @@ -617,8 +666,6 @@ mod test { state.add_peer(&peer_2); state.current_term = 5; }).unwrap(); - subject.last_applied = 99; - subject.last_applied_term = 4; subject.state = State::Primary; subject.primary_state = Some(PrimaryState::new(5)); subject.last_heartbeat = start; @@ -681,8 +728,6 @@ mod test { state.add_peer(&peer_2); state.current_term = 5; }).unwrap(); - subject.last_applied = 99; - subject.last_applied_term = 4; subject.state = State::Primary; subject.primary_state = Some(PrimaryState::new(5)); subject.last_heartbeat = start; @@ -744,12 +789,11 @@ mod test { state.add_peer(&peer_2); state.current_term = 5; }).unwrap(); - subject.last_applied = 99; - subject.last_applied_term = 4; subject.state = State::Voted; subject.last_heartbeat = start; let mut controller_state = MockControllerState::new(); + controller_state.add_new_mock_event(99, 4); subject.tick(t_sec(start, 1), &mut controller_state); connection_manager.verify_in_order(&Invocation::EstablishConnections); @@ -938,10 +982,9 @@ mod test { state.current_term = 5; }).unwrap(); subject.state = State::Follower; - subject.last_applied_term = 5; - subject.last_applied = 9; let mut controller_state = MockControllerState::new(); + controller_state.add_new_mock_event(9, 5); let election_start = t_sec(start, 1); subject.tick(election_start, &mut controller_state); connection_manager.verify_in_order(&Invocation::EstablishConnections); @@ -949,8 +992,8 @@ mod test { connection_control: ConnectionControl::SendRequestVote(CallRequestVote { term: 6, candidate_id: subject.persistent.this_instance_id, - last_log_index: subject.last_applied, - last_log_term: subject.last_applied_term, + last_log_index: 9, + last_log_term: 5, }), }); @@ -971,9 +1014,8 @@ mod test { #[test] fn vote_is_granted_when_candidate_term_and_log_are_more_up_to_date() { - vote_test(|subject, request, candidate, peer_2| { - subject.last_applied_term = request.last_log_term - 1; - subject.last_applied = request.last_log_index - 2; + vote_test(|subject, controller_state, request, candidate, peer_2| { + controller_state.add_new_mock_event(request.last_log_index - 2, request.last_log_term - 1); subject.persistent.modify(|state| { state.current_term = request.term - 1; state.add_peer(&candidate); @@ -990,9 +1032,8 @@ mod test { #[test] fn vote_is_denied_when_candidate_term_is_less_than_current_term() { - vote_test(|subject, request, candidate, peer_2| { - subject.last_applied_term = request.last_log_term; - subject.last_applied = request.last_log_index; + vote_test(|subject, controller_state, request, candidate, peer_2| { + controller_state.add_new_mock_event(request.last_log_index, request.last_log_term); subject.persistent.modify(|state| { state.current_term = request.term + 1; // my current term is greater state.add_peer(&candidate); @@ -1009,11 +1050,10 @@ mod test { #[test] fn vote_is_denied_when_candidate_log_term_is_out_of_date() { - vote_test(|subject, request, candidate, peer_2| { - subject.last_applied_term = request.last_log_term + 1; + vote_test(|subject, controller_state, request, candidate, peer_2| { + controller_state.add_new_mock_event(request.last_log_index, request.last_log_term + 1); // unless we've really screwed something up, the last_log_index should never be the same if the last_log_term is different. // We're doing it this way in the test just to document the behavior in this case - subject.last_applied = request.last_log_index; subject.persistent.modify(|state| { state.current_term = request.term; state.add_peer(&candidate); @@ -1030,9 +1070,8 @@ mod test { #[test] fn vote_is_denied_when_candidate_is_not_a_known_peer() { - vote_test(|subject, request, _candidate, peer_2| { - subject.last_applied_term = request.last_log_term; - subject.last_applied = request.last_log_index; + vote_test(|subject, controller_state, request, _candidate, peer_2| { + controller_state.add_new_mock_event(request.last_log_index, request.last_log_term); subject.persistent.modify(|state| { state.current_term = request.term - 1; // peer_1 is not a known member @@ -1049,9 +1088,8 @@ mod test { #[test] fn vote_is_denied_when_candidate_log_is_out_of_date() { - vote_test(|subject, request, candidate, peer_2| { - subject.last_applied_term = request.last_log_term; - subject.last_applied = request.last_log_index + 1; + vote_test(|subject, controller_state, request, candidate, peer_2| { + controller_state.add_new_mock_event(request.last_log_index + 1, request.last_log_term); subject.persistent.modify(|state| { state.voted_for = None; state.current_term = request.term; @@ -1069,9 +1107,8 @@ mod test { #[test] fn vote_is_denied_when_a_vote_was_already_cast_for_another_member_this_term() { - vote_test(|subject, request, candidate, peer_2| { - subject.last_applied_term = request.last_log_term; - subject.last_applied = request.last_log_index; + vote_test(|subject, controller_state, request, candidate, peer_2| { + controller_state.add_new_mock_event(request.last_log_index, request.last_log_term); subject.persistent.modify(|state| { state.voted_for = Some(peer_2.id); // already voted for peer 2 state.current_term = request.term; @@ -1089,9 +1126,8 @@ mod test { #[test] fn vote_is_granted_when_no_other_vote_was_granted_and_candidate_log_exactly_matches() { - vote_test(|subject, request, candidate, peer_2| { - subject.last_applied_term = request.last_log_term; - subject.last_applied = request.last_log_index; + vote_test(|subject, controller_state, request, candidate, peer_2| { + controller_state.add_new_mock_event(request.last_log_index, request.last_log_term); subject.persistent.modify(|state| { state.current_term = request.term - 1; state.add_peer(&candidate); @@ -1108,9 +1144,8 @@ mod test { #[test] fn vote_is_granted_when_voted_for_is_already_populated_but_candidate_term_is_greater() { - vote_test(|subject, request, candidate, peer_2| { - subject.last_applied_term = request.last_log_term; - subject.last_applied = request.last_log_index; + vote_test(|subject, controller_state, request, candidate, peer_2| { + controller_state.add_new_mock_event(request.last_log_index, request.last_log_term); subject.persistent.modify(|state| { state.current_term = request.term - 1; state.voted_for = Some(state.this_instance_id); @@ -1141,13 +1176,12 @@ mod test { }; let mut subject = create_cluster_manager(vec![peer_1.address, peer_2.address], temp_dir.path(), connection_manager.boxed_ref()); subject.state = State::Follower; - subject.last_applied_term = 7; - subject.last_applied = 9; subject.persistent.modify(|state| { state.current_term = 7; }).unwrap(); let mut controller_state = MockControllerState::new(); + controller_state.add_new_mock_event(9, 7); subject.tick(t_sec(start, 1), &mut controller_state); connection_manager.verify_in_order(&Invocation::EstablishConnections); @@ -1313,7 +1347,7 @@ mod test { granted: bool, } - fn vote_test(setup_fun: F) where F: Fn(&mut ClusterManager, &CallRequestVote, Peer, Peer) -> VoteExpectation { + fn vote_test(setup_fun: F) where F: Fn(&mut ClusterManager, &mut MockControllerState, &CallRequestVote, Peer, Peer) -> VoteExpectation { let temp_dir = TempDir::new("cluster_state_test").unwrap(); let connection_manager = MockPeerConnectionManager::new(); let peer_1 = Peer { @@ -1333,9 +1367,10 @@ mod test { last_log_index: 9, last_log_term: 7, }; - let expectation = setup_fun(&mut subject, &request_vote, peer_1.clone(), peer_2.clone()); + let mut controller_state = MockControllerState::new(); + let expectation = setup_fun(&mut subject, &mut controller_state, &request_vote, peer_1.clone(), peer_2.clone()); - subject.request_vote_received(678, request_vote); + subject.request_vote_received(678, request_vote, &mut controller_state); assert_eq!(expectation.persistent_voted_for, subject.persistent.voted_for); assert_eq!(expectation.term, subject.persistent.current_term); @@ -1372,7 +1407,7 @@ impl ConsensusProcessor for NoOpConsensusProcessor { fn connection_closed(&mut self, _connection_id: ConnectionId) { } - fn request_vote_received(&mut self, _from: ConnectionId, _request: CallRequestVote) { + fn request_vote_received(&mut self, _from: ConnectionId, _request: CallRequestVote, _controller_state: &mut ControllerState) { panic!("invalid operation for a NoOpConsensusProcessor. This should not happen"); } diff --git a/flo-server/src/engine/controller/cluster_state/persistent/file.rs b/flo-server/src/engine/controller/cluster_state/persistent/file.rs index 232bcb3..ac6ac64 100644 --- a/flo-server/src/engine/controller/cluster_state/persistent/file.rs +++ b/flo-server/src/engine/controller/cluster_state/persistent/file.rs @@ -37,7 +37,7 @@ impl FilePersistedState { }) } - pub fn modify(&mut self, fun: F) -> Result<(), io::Error> where F: Fn(&mut PersistentClusterState) { + pub fn modify(&mut self, fun: F) -> Result<(), io::Error> where F: FnOnce(&mut PersistentClusterState) { fun(&mut self.state); self.flush() } diff --git a/flo-server/src/engine/controller/cluster_state/persistent/mod.rs b/flo-server/src/engine/controller/cluster_state/persistent/mod.rs index 90abe79..bc5a788 100644 --- a/flo-server/src/engine/controller/cluster_state/persistent/mod.rs +++ b/flo-server/src/engine/controller/cluster_state/persistent/mod.rs @@ -3,7 +3,7 @@ mod file; use std::net::SocketAddr; use std::collections::HashMap; -use event::{ActorId, FloEvent, EventCounter}; +use event::{ActorId, EventCounter}; use protocol::Term; use protocol::flo_instance_id::{self, FloInstanceId}; use engine::controller::controller_messages::Peer; @@ -11,6 +11,7 @@ use engine::controller::system_event::*; use super::SharedClusterState; pub use self::file::FilePersistedState; +use engine::controller::controller_state::SystemEventRef; /// Holds all the cluster state that we want to survive a reboot. /// We always persist the `FloInstanceId` because we prefer that to be stable across reboots. We do _not_ want to persist @@ -22,6 +23,7 @@ pub struct PersistentClusterState { pub this_instance_id: FloInstanceId, pub this_partition_num: Option, last_applied: EventCounter, + last_applied_term: Term, cluster_members: HashMap, assigned_partitions: HashMap, } @@ -42,6 +44,7 @@ impl PersistentClusterState { pub fn generate_new() -> PersistentClusterState { PersistentClusterState { last_applied: 0, + last_applied_term: 0, current_term: 0, voted_for: None, this_instance_id: flo_instance_id::generate_new(), @@ -51,13 +54,13 @@ impl PersistentClusterState { } } - pub fn apply_system_event(&mut self, event: &SystemEvent) { - let counter = event.counter(); + pub fn apply_system_event(&mut self, event: SystemEventRef) { + let counter = event.counter; let term = event.term(); self.current_term = self.current_term.max(term); if counter > self.last_applied { - match event.deserialized_data.kind { + match event.data.kind { SystemEventKind::ClusterInitialized(ref initial_membership) => { for peer in initial_membership.peers.iter() { self.add_peer(peer); @@ -82,6 +85,10 @@ impl PersistentClusterState { } + pub fn get_last_applied(&self) -> (EventCounter, Term) { + (self.last_applied, self.last_applied_term) + } + pub fn add_peer(&mut self, peer: &Peer) { self.cluster_members.insert(peer.id, peer.address); } @@ -113,7 +120,6 @@ impl PersistentClusterState { #[cfg(test)] mod test { use super::*; - use event::{OwnedFloEvent, FloEventId, time}; use test_utils::addr; use std::collections::HashSet; @@ -125,7 +131,7 @@ mod test { peer_id: flo_instance_id::generate_new(), partition_num: 1, })); - subject.apply_system_event(&event); + subject.apply_system_event(event); } #[test] @@ -138,7 +144,7 @@ mod test { peer_id: initial_peers[0].id, partition_num: 3, })); - subject.apply_system_event(&event); + subject.apply_system_event(event); assert_eq!(1, subject.assigned_partitions.len()); assert_eq!(Some(initial_peers[0].id), subject.assigned_partitions.get(&3).cloned()); @@ -152,7 +158,7 @@ mod test { let mut subject = subject_with_initial_peers(initial_peers.clone()); let new_peer = new_peer(4); let event = sys_event(2, 7, SystemEventKind::NewClusterMemberJoining(new_peer.clone())); - subject.apply_system_event(&event); + subject.apply_system_event(event); assert_eq!(2, subject.last_applied); assert_eq!(7, subject.current_term); @@ -185,7 +191,7 @@ mod test { let event = sys_event(1, 1, SystemEventKind::ClusterInitialized(InitialClusterMembership { peers })); - subject.apply_system_event(&event); + subject.apply_system_event(event); subject } @@ -201,8 +207,8 @@ mod test { Peer { id: flo_instance_id::generate_new(), address: addr(&format!("127.0.0.1:{}", port)) } } - fn sys_event(counter: EventCounter, term: Term, kind: SystemEventKind) -> SystemEvent { + fn sys_event(counter: EventCounter, term: Term, kind: SystemEventKind) -> SystemEventRef { let data = SystemEventData { term, kind }; - SystemEvent::new(FloEventId::new(0, counter), None, "/system".to_owned(), time::from_millis_since_epoch(44), data) + SystemEventRef { counter, data } } } diff --git a/flo-server/src/engine/controller/controller_state.rs b/flo-server/src/engine/controller/controller_state.rs index 4b93ed8..a8d27f8 100644 --- a/flo-server/src/engine/controller/controller_state.rs +++ b/flo-server/src/engine/controller/controller_state.rs @@ -5,18 +5,33 @@ use std::io; use tokio_core::reactor::Remote; use protocol::{Term, FloInstanceId}; -use event::{OwnedFloEvent, EventCounter, ActorId}; +use event::{FloEvent, OwnedFloEvent, EventCounter, ActorId}; use engine::ConnectionId; use engine::controller::{ConnectionRef, SystemEvent, SystemEventData}; use engine::event_stream::{EventStreamRef, EventStreamRefMut, EventStreamOptions, init_new_event_stream}; use engine::event_stream::partition::{SegmentNum, IndexEntry, PersistentEvent, PartitionReader, ReplicationResult}; use engine::event_stream::partition::controller::PartitionImpl; -use std::borrow::Cow; #[derive(Debug, PartialEq)] -pub struct SystemEventRef<'a> { +pub struct SystemEventRef { pub counter: EventCounter, - pub data: Cow<'a, SystemEventData> + pub data: SystemEventData +} + +impl SystemEventRef { + pub fn term(&self) -> Term { + self.data.term + } +} + +impl From> for SystemEventRef { + fn from(evt: SystemEvent) -> Self { + let counter = evt.counter(); + SystemEventRef { + counter, + data: evt.deserialized_data + } + } } pub trait ControllerState { @@ -28,10 +43,11 @@ pub trait ControllerState { fn create_event_stream(&mut self, options: EventStreamOptions) -> io::Result<()>; fn create_partitions(&mut self, partition_num: ActorId) -> io::Result<()>; + fn get_last_uncommitted_event(&mut self) -> Option>; fn get_commit_index(&self) -> EventCounter; fn set_commit_index(&mut self, new_index: EventCounter); fn get_last_committed(&mut self) -> io::Result<(EventCounter, Term)>; - fn get_next_event<'a>(&'a mut self, start_after: EventCounter) -> Option>>; + fn get_next_event(&mut self, start_after: EventCounter) -> Option>; fn get_next_counter_and_term(&mut self, start_after: EventCounter) -> Option> { self.get_next_event(start_after).map(|result| { @@ -161,6 +177,11 @@ impl ControllerState for ControllerStateImpl { unimplemented!() } + fn get_last_uncommitted_event(&mut self) -> Option> { + let counter = self.system_partition.get_highest_uncommitted_event(); + self.get_next_event(counter - 1) + } + fn get_commit_index(&self) -> EventCounter { self.system_partition.get_commit_index() } @@ -185,15 +206,9 @@ impl ControllerState for ControllerStateImpl { } } - fn get_next_event<'a>(&'a mut self, start_after: EventCounter) -> Option>> { + fn get_next_event(&mut self, start_after: EventCounter) -> Option> { self.get_next_entry(start_after.saturating_sub(1)).map(|index_entry| { - self.read_event(index_entry).map(|system_event| { - let counter = system_event.counter(); - SystemEventRef{ - counter, - data: Cow::Owned(system_event.system_data().clone()), - } - }) + self.read_event(index_entry).map(|system_event| system_event.into() ) }) } @@ -265,6 +280,10 @@ pub mod mock { self } + pub fn add_new_mock_event(&mut self, counter: EventCounter, term: Term) { + self.add_mock_event(MockSystemEvent::with_any_data(counter, term, SegmentNum::new(1), 77)) + } + pub fn add_mock_event(&mut self, event: MockSystemEvent) { self.system_events.insert(event.id, event); } @@ -290,6 +309,16 @@ pub mod mock { unimplemented!() } + fn get_last_uncommitted_event(&mut self) -> Option> { + self.system_events.iter().next_back().map(|(id, mock_sys_event)| { + Ok(SystemEventRef { + counter: *id, + data: mock_sys_event.data.clone() + }) + }) + } + + fn get_commit_index(&self) -> EventCounter { self.commit_index } @@ -305,11 +334,11 @@ pub mod mock { io::Error::new(io::ErrorKind::Other, format!("Test error because there is no stubbed event for the commit index: {}", commit_idx)) }) } - fn get_next_event<'a>(&'a mut self, start_after: EventCounter) -> Option>> { + fn get_next_event(&mut self, start_after: EventCounter) -> Option> { self.system_events.range((start_after + 1)..).next().map(|(id, sys)| { Ok(SystemEventRef { counter: *id, - data: Cow::Borrowed(&sys.data), + data: sys.data.clone(), }) }) } diff --git a/flo-server/src/engine/controller/mod.rs b/flo-server/src/engine/controller/mod.rs index 5e20b42..5cf14bf 100644 --- a/flo-server/src/engine/controller/mod.rs +++ b/flo-server/src/engine/controller/mod.rs @@ -100,7 +100,7 @@ impl FloController { cluster_state.tick(op_start_time, controller_state); } SystemOpType::RequestVote(request) => { - cluster_state.request_vote_received(connection_id, request); + cluster_state.request_vote_received(connection_id, request, controller_state); } SystemOpType::VoteResponseReceived(response) => { cluster_state.vote_response_received(op_start_time, connection_id, response, controller_state); @@ -156,7 +156,7 @@ fn produce_system_events(produce_op: partition::ProduceOperation, cluster_state: match validate_system_event(&produce_op.events, term) { Ok(()) => { // hand off the modified operation to the partition, which will complete it - + // This will also handle sending the response back to the client, when appropriate let result = controller_state.system_partition.handle_produce(produce_op); if let Err(partition_err) = result { error!("Partition error creating new system events: {:?}", partition_err); diff --git a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs index d9f8392..bc60557 100644 --- a/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs +++ b/flo-server/src/engine/event_stream/partition/controller/commit_manager.rs @@ -2,7 +2,7 @@ use event::{EventCounter, ActorId}; use protocol::FloInstanceId; use atomics::{AtomicCounterReader, AtomicCounterWriter}; - +use std::collections::HashSet; pub struct CommitManager { @@ -50,6 +50,17 @@ impl CommitManager { } } + pub fn set_peers(&self, peers: &HashSet) { + self.peers.retain(|elem| peers.contains(&elem.0)); + + for new_peer in peers { + if !self.peers.iter().any(|p| p.0 == new_peer) { + self.peers.push((new_peer, 0)); + } + } + self.set_min_required(); + } + pub fn is_standalone(&self) -> bool { self.min_required_for_commit == 0 } @@ -70,14 +81,18 @@ impl CommitManager { pub fn add_member(&mut self, peer_id: FloInstanceId) { self.peers.push((peer_id, 0)); - let new_ack_requirement = self.compute_min_required(); - self.min_required_for_commit = new_ack_requirement; + self.set_min_required(); } pub fn get_commit_index_reader(&self) -> AtomicCounterReader { self.commit_index.reader() } + pub fn set_min_required(&mut self) { + let new_min = self.compute_min_required(); + self.min_required_for_commit = new_min; + } + fn compute_min_required(&self) -> ActorId { let peer_count = self.peers.len() as ActorId; diff --git a/flo-server/src/engine/event_stream/partition/controller/mod.rs b/flo-server/src/engine/event_stream/partition/controller/mod.rs index 18ef719..57387a3 100644 --- a/flo-server/src/engine/event_stream/partition/controller/mod.rs +++ b/flo-server/src/engine/event_stream/partition/controller/mod.rs @@ -167,6 +167,10 @@ impl PartitionImpl { self.event_stream_name.as_str() } + pub fn get_highest_uncommitted_event(&self) -> EventCounter { + self.index.greatest_event_counter() + } + pub fn commit_index_reader(&self) -> AtomicCounterReader { self.commit_manager.get_commit_index_reader() }