diff --git a/CHANGELOG.md b/CHANGELOG.md index 16b024d7..282c1fe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file. ## Version 0.58.4 +- Load cells from storing cells cache by id - Added support for due payment fix - Bump block version diff --git a/Cargo.toml b/Cargo.toml index 318dabbc..b580d06e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,10 +44,12 @@ name = 'zerostate' path = 'bin/zerostate.rs' [dependencies] +adnl = { features = [ 'client', 'node', 'server' ], git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.10.24' } arc-swap = '0.3.11' async-recursion = '0.3.2' async-trait = '0.1.22' bitflags = '1.2.1' +catchain = { path = 'catchain' } chrono = '=0.4.19' clap = '2.33' colored = '1.9.3' @@ -58,12 +60,18 @@ deflate = '1.0.0' dirs = '2.0.2' enum-as-inner = '=0.5.1' env_logger = '0.7.1' +ever_abi = { git = 'https://github.com/everx-labs/ever-abi.git', tag = '2.5.3' } +ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.10.2' } +ever_block_json = { git = 'https://github.com/everx-labs/ever-block-json.git', tag = '0.8.6' } +ever_executor = { git = 'https://github.com/everx-labs/ever-executor.git', tag = '1.17.6' } +ever_vm = { git = 'https://github.com/everx-labs/ever-vm.git', tag = '2.1.5' } failure = '0.1' futures = '0.3.1' futures-timer = '3.0.1' hex = '0.4' inflate = '0.4.5' lazy_static = '1.4.0' +lockfree = { git = 'https://github.com/everx-labs/lockfree.git' } log = '0.4' log4rs = '1.2' log4rs-rolling-file = '0.2.0' @@ -82,19 +90,11 @@ serde_json = '1.0.64' shell-words = '1.0' spin = '0.7.1' statsd = { optional = true, version = '0.15' } +storage = { path = 'storage' } stream-cancel = '0.8.0' string-builder = '^0.2.0' tokio = { features = [ 'rt-multi-thread' ], version = '1.5' } tokio-util = '0.7' -adnl = { features = [ 'client', 'node', 'server' ], git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.10.24' } -catchain = { path = 'catchain' } -ever_abi = { git = 'https://github.com/everx-labs/ever-abi.git', tag = '2.5.3' } -ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.10.2' } -ever_block_json = { git = 'https://github.com/everx-labs/ever-block-json.git', tag = '0.8.6' } -ever_executor = { git = 'https://github.com/everx-labs/ever-executor.git', tag = '1.17.6' } -ever_vm = { git = 'https://github.com/everx-labs/ever-vm.git', tag = '2.1.5' } -lockfree = { git = 'https://github.com/everx-labs/lockfree.git' } -storage = { path = 'storage' } ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.3.79' } validator_session = { path = 'validator-session' } diff --git a/catchain/Cargo.toml b/catchain/Cargo.toml index 5853ce41..fa61670c 100644 --- a/catchain/Cargo.toml +++ b/catchain/Cargo.toml @@ -5,8 +5,10 @@ name = 'catchain' version = '0.1.0' [dependencies] +adnl = { features = [ 'node' ], git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.10.24' } chrono = '0.4.10' crossbeam = '0.7.3' +ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.10.2' } failure = '0.1' futures = '0.3.4' hex = '0.4' @@ -18,10 +20,8 @@ metrics-util = '0.15.0' quanta = '0.11.1' rand = '0.8' regex = '1.3.1' -tokio = { features = [ 'rt-multi-thread' ], version = '1.5' } -adnl = { features = [ 'node' ], git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.10.24' } -ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.10.2' } storage = { path = '../storage' } +tokio = { features = [ 'rt-multi-thread' ], version = '1.5' } ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.3.79' } [dev-dependencies] diff --git a/src/collator_test_bundle.rs b/src/collator_test_bundle.rs index 83e9488f..24c76ffd 100644 --- a/src/collator_test_bundle.rs +++ b/src/collator_test_bundle.rs @@ -39,7 +39,7 @@ use storage::StorageTelemetry; use ever_block::{ BlockIdExt, Message, ShardIdent, Serializable, MerkleUpdate, Deserializable, ValidatorBaseInfo, BlockSignaturesPure, BlockSignatures, HashmapAugType, - TopBlockDescrSet, GlobalCapabilities, OutMsgQueue, + TopBlockDescrSet, OutMsgQueue, }; use ever_block::{ShardStateUnsplit, TopBlockDescr}; use ever_block::{UInt256, fail, error, Result, CellType, read_boc, read_single_root_boc}; @@ -1208,19 +1208,6 @@ impl CollatorTestBundle { pub fn candidate(&self) -> Option<&BlockCandidate> { self.candidate.as_ref() } pub fn set_notes(&mut self, notes: String) { self.index.notes = notes } - - fn get_messages(&self, remp: bool) -> Result, UInt256)>> { - let remp_enabled = self.states - .get(&self.index.last_mc_state) - .ok_or_else(|| error!("Can't load last ms block to read config"))? - .config_params()?.has_capability(GlobalCapabilities::CapRemp); - - if remp_enabled == remp { - Ok(self.external_messages.clone()) - } else { - Ok(vec!()) - } - } } // Is used instead full node's engine for run tests @@ -1346,20 +1333,6 @@ impl EngineOperations for CollatorTestBundle { &self.allocated } - fn get_remp_messages(&self, _shard: &ShardIdent) -> Result, UInt256)>> { - self.get_messages(true) - } - - fn finalize_remp_messages( - &self, - _block: BlockIdExt, - _accepted: Vec, - _rejected: Vec<(UInt256, String)>, - _ignored: Vec, - ) -> Result<()> { - Ok(()) - } - async fn check_remp_duplicate(&self, _message_id: &UInt256) -> Result { Ok(RempDuplicateStatus::Fresh(UInt256::default())) } diff --git a/src/config.rs b/src/config.rs index 54dc35cd..eed0df40 100644 --- a/src/config.rs +++ b/src/config.rs @@ -166,8 +166,7 @@ pub struct TonNodeConfig { port: Option, #[serde(skip)] file_name: String, - #[serde(default = "RempConfig::default")] - remp: RempConfig, + remp: Option, #[serde(default)] restore_db: bool, #[serde(default)] @@ -599,8 +598,11 @@ impl TonNodeConfig { pub fn extensions(&self) -> &NodeExtensions { &self.extensions } - pub fn remp_config(&self) -> &RempConfig { - &self.remp + pub fn remp_config(&self) -> RempConfig { + match &self.remp { + Some(x) => x.clone(), + None => RempConfig::default() + } } pub fn restore_db(&self) -> bool { self.restore_db diff --git a/src/engine_operations.rs b/src/engine_operations.rs index 7d37a0f9..c8786c73 100644 --- a/src/engine_operations.rs +++ b/src/engine_operations.rs @@ -990,18 +990,6 @@ impl EngineOperations for Engine { fn new_remp_message(&self, id: UInt256, message: Arc) -> Result<()> { self.remp_messages()?.new_message(id, message) } - fn get_remp_messages(&self, shard: &ShardIdent) -> Result, UInt256)>> { - self.remp_messages()?.get_messages(shard) - } - fn finalize_remp_messages( - &self, - block: BlockIdExt, - accepted: Vec, - rejected: Vec<(UInt256, String)>, - ignored: Vec, - ) -> Result<()> { - self.remp_messages()?.finalize_messages(block, accepted, rejected, ignored) - } fn finalize_remp_messages_as_ignored(&self, block_id: &BlockIdExt) -> Result<()> { self.remp_messages()?.finalize_remp_messages_as_ignored(block_id) @@ -1013,7 +1001,6 @@ impl EngineOperations for Engine { async fn check_remp_duplicate(&self, message_id: &UInt256) -> Result { self.remp_service() .ok_or_else(|| error!("Can't get message status because remp service was not set"))? - .remp_core_interface()? .check_remp_duplicate(message_id) } diff --git a/src/engine_traits.rs b/src/engine_traits.rs index f0e38300..4d41061b 100644 --- a/src/engine_traits.rs +++ b/src/engine_traits.rs @@ -628,18 +628,6 @@ pub trait EngineOperations : Sync + Send { fn new_remp_message(&self, id: UInt256, message: Arc) -> Result<()> { unimplemented!() } - fn get_remp_messages(&self, shard: &ShardIdent) -> Result, UInt256)>> { - unimplemented!() - } - fn finalize_remp_messages( - &self, - block: BlockIdExt, - accepted: Vec, - rejected: Vec<(UInt256, String)>, - ignored: Vec, - ) -> Result<()> { - unimplemented!() - } fn finalize_remp_messages_as_ignored(&self, block_id: &BlockIdExt) -> Result<()> { unimplemented!() } @@ -968,6 +956,16 @@ pub enum RempDuplicateStatus { #[async_trait::async_trait] pub trait RempCoreInterface: Sync + Send { - async fn process_incoming_message(&self, message_id: UInt256, message: Message, source: Arc) -> Result<()>; + async fn process_incoming_message(&self, message: &RempMessage, source: Arc) -> Result<()>; fn check_remp_duplicate(&self, message_id: &UInt256) -> Result; } + +#[async_trait::async_trait] +pub trait RempQueueCollatorInterface : Send + Sync { + async fn init_queue( + &self, + master_block_id: &BlockIdExt, + prev_blocks_ids: &[&BlockIdExt] + ) -> Result<()>; + async fn get_next_message_for_collation(&self) -> Result, UInt256)>>; +} diff --git a/src/ext_messages.rs b/src/ext_messages.rs index 4380e3dc..30f0f905 100644 --- a/src/ext_messages.rs +++ b/src/ext_messages.rs @@ -27,7 +27,7 @@ const MESSAGE_LIFETIME: u32 = 600; // seconds const MESSAGE_MAX_GENERATIONS: u8 = 3; const MAX_EXTERNAL_MESSAGE_DEPTH: u16 = 512; -const MAX_EXTERNAL_MESSAGE_SIZE: usize = 65535; +pub const MAX_EXTERNAL_MESSAGE_SIZE: usize = 65535; pub const EXT_MESSAGES_TRACE_TARGET: &str = "ext_messages"; @@ -460,6 +460,7 @@ pub fn is_finally_accepted(status: &RempMessageStatus) -> bool { } } +// To be deleted pub struct RempMessagesPool { messages: Map>, statuses_queue: lockfree::queue::Queue<(UInt256, Arc, RempMessageStatus)>, @@ -483,7 +484,7 @@ impl RempMessagesPool { // Important! If call get_messages with same shard two times in row (without finalize_messages between) // the messages returned first call will return second time too. - pub fn get_messages(&self, shard: &ShardIdent) -> Result, UInt256)>> { + /*pub fn get_messages(&self, shard: &ShardIdent) -> Result, UInt256)>> { let mut result = vec!(); let mut ids = String::new(); for guard in self.messages.iter() { @@ -506,7 +507,7 @@ impl RempMessagesPool { ); Ok(result) - } + }*/ pub fn finalize_messages( &self, diff --git a/src/tests/test_helper.rs b/src/tests/test_helper.rs index 5f02e442..a0955892 100644 --- a/src/tests/test_helper.rs +++ b/src/tests/test_helper.rs @@ -697,11 +697,12 @@ impl TestEngine { extra.created_by().clone(), self.clone(), Some(extra.rand_seed().clone()), + None, CollatorSettings::default(), )?; - let (block_candidate, new_state) = collator.collate().await?; - + let (block_candidate, new_state) = collator.collate().await?; + if let Some(res_path) = &self.res_path { let new_block = Block::construct_from_bytes(&block_candidate.data)?; diff --git a/src/validator/collator.rs b/src/validator/collator.rs index f484b182..be67b992 100644 --- a/src/validator/collator.rs +++ b/src/validator/collator.rs @@ -25,7 +25,7 @@ use crate::{ top_block_descr::{cmp_shard_block_descr, Mode as TbdMode, TopBlockDescrStuff}, }, validating_utils::{ - calc_remp_msg_ordering_hash, check_cur_validator_set, check_this_shard_mc_info, + check_cur_validator_set, check_this_shard_mc_info, may_update_shard_block_info, supported_capabilities, supported_version, UNREGISTERED_CHAIN_MAX_LEN, fmt_next_block_descr_from_next_seqno, }, @@ -65,6 +65,7 @@ use ever_executor::{ TransactionExecutor, }; use ever_block::{error, fail, AccountId, Cell, HashmapType, Result, UInt256, UsageTree, SliceData}; +use crate::engine_traits::RempQueueCollatorInterface; use crate::validator::validator_utils::is_remp_enabled; @@ -215,9 +216,6 @@ struct CollatorData { new_messages: BinaryHeap, // using for priority queue accepted_ext_messages: Vec<(UInt256, i32)>, // message id and wokchain id rejected_ext_messages: Vec<(UInt256, String)>, // message id and reject reason - accepted_remp_messages: Vec, - rejected_remp_messages: Vec<(UInt256, String)>, - ignored_remp_messages: Vec, usage_tree: UsageTree, imported_visited: HashSet, @@ -290,9 +288,6 @@ impl CollatorData { new_messages: Default::default(), accepted_ext_messages: Default::default(), rejected_ext_messages: Default::default(), - accepted_remp_messages: Default::default(), - rejected_remp_messages: Default::default(), - ignored_remp_messages: Default::default(), usage_tree, imported_visited: HashSet::new(), gen_utime, @@ -637,18 +632,6 @@ impl CollatorData { std::mem::take(&mut self.rejected_ext_messages)) } - fn withdraw_remp_msg_statuses(&mut self) -> (Vec, Vec<(UInt256, String)>, Vec) { - (std::mem::take(&mut self.accepted_remp_messages), - std::mem::take(&mut self.rejected_remp_messages), - std::mem::take(&mut self.ignored_remp_messages)) - } - - fn set_remp_msg_statuses(&mut self, accepted: Vec, rejected: Vec<(UInt256, String)>, ignored: Vec) { - self.accepted_remp_messages = accepted; - self.rejected_remp_messages = rejected; - self.ignored_remp_messages = ignored; - } - fn estimate_pruned_count(&self) -> usize { if self.enqueue_count != 0 { let total_count = self.dequeue_count + self.enqueue_count + self.remove_count; @@ -988,6 +971,7 @@ impl ExecutionManager { pub struct Collator { engine: Arc, + remp_collator_interface: Option>, shard: ShardIdent, min_mc_seqno: u32, prev_blocks_ids: Vec, @@ -1017,6 +1001,7 @@ impl Collator { created_by: UInt256, engine: Arc, rand_seed: Option, + remp_collator_interface: Option>, collator_settings: CollatorSettings ) -> Result { @@ -1089,6 +1074,7 @@ impl Collator { file_hash: UInt256::default(), }, engine, + remp_collator_interface, shard, min_mc_seqno, prev_blocks_ids, @@ -1338,11 +1324,6 @@ impl Collator { ) -> Result> { log::debug!("{}: do_collate", self.collated_block_descr); - let remp_messages = if is_remp_enabled(self.engine.clone(), mc_data.config()) { - Some(self.engine.get_remp_messages(&self.shard)?) - } else { - None - }; self.check_stop_flag()?; // loads out queues from neighbors and out queue of current shard @@ -1417,15 +1398,13 @@ impl Collator { log::debug!("{}: TIME: process_inbound_internal_messages {}ms;", self.collated_block_descr, now.elapsed().as_millis()); - if let Some(remp_messages) = remp_messages { + if is_remp_enabled(self.engine.clone(), mc_data.config()) { // import remp messages (if space&gas left) let now = std::time::Instant::now(); - let total = remp_messages.len(); - let processed = self.process_remp_messages( - prev_data, collator_data, &mut exec_manager, remp_messages - ).await?; - log::debug!("{}: TIME: process_remp_messages {}ms, processed {}, ignored {}", - self.collated_block_descr, now.elapsed().as_millis(), processed, total - processed); + let (accepted, rejected) = self.process_remp_messages( + prev_data, mc_data, collator_data, &mut exec_manager).await?; + log::debug!("{}: TIME: process_remp_messages {}ms, accepted {}, rejected {}", + self.collated_block_descr, now.elapsed().as_millis(), accepted, rejected); } // import inbound external messages (if space&gas left) @@ -2460,58 +2439,51 @@ impl Collator { async fn process_remp_messages( &self, prev_data: &PrevData, + mc_data: &McData, collator_data: &mut CollatorData, exec_manager: &mut ExecutionManager, - mut remp_messages: Vec<(Arc, UInt256)>, - ) -> Result { - log::trace!("{}: process_remp_messages ({}pcs)", self.collated_block_descr, remp_messages.len()); + ) -> Result<(usize, usize)> { + log::trace!("{}: process_remp_messages", self.collated_block_descr); - remp_messages.sort_by_cached_key(|(_, id)| { - calc_remp_msg_ordering_hash(&id, prev_data.pure_states.iter().map(|s| s.block_id())) - }); - log::trace!("{}: process_remp_messages: sorted {} messages", self.collated_block_descr, remp_messages.len()); + let remp_collator_interface = self.remp_collator_interface.as_ref() + .ok_or_else(|| error!("remp_collator_interface is not set"))?; - let mut ignored = vec!(); - let mut ignore = false; - for (msg, id) in remp_messages.drain(..) { - if ignore { - ignored.push(id); - continue; - } + remp_collator_interface.init_queue( + mc_data.state().block_id(), + &prev_data.pure_states.iter().map(|s| s.block_id()).collect::>() + ).await?; + log::trace!("{}: process_remp_messages: inited queue", self.collated_block_descr); + + while let Some((msg, id)) = remp_collator_interface.get_next_message_for_collation().await? { + log::trace!("{}: process_remp_messages: got next message {:x}", self.collated_block_descr, id); let header = msg.ext_in_header().ok_or_else(|| error!("remp message {:x} \ is not external inbound message", id))?; if self.shard.contains_address(&header.dst)? { let pruned_count = collator_data.estimate_pruned_count(); if !collator_data.block_limit_status.fits_normal(REMP_CUTOFF_LIMIT, pruned_count) { log::trace!("{}: block is loaded enough, stop processing remp messages", self.collated_block_descr); - ignored.push(id); - ignore = true; + break; } else if self.check_cutoff_timeout() { log::warn!("{}: TIMEOUT is elapsed, stop processing remp messages", self.collated_block_descr); - ignored.push(id); - ignore = true; + break; } else { let (_, account_id) = header.dst.extract_std_address(true)?; - log::trace!("{}: remp message {:x} sent to execution", self.collated_block_descr, id); - let msg = AsyncMessage::Ext(msg.deref().clone(), id); + let msg = AsyncMessage::Ext(msg.deref().clone(), id.clone()); exec_manager.execute(account_id, msg, prev_data, collator_data).await?; + log::trace!("{}: remp message {:x} sent to execution", self.collated_block_descr, id); } } else { log::warn!( "{}: process_remp_messages: ignored message {:x} for another shard {}", self.collated_block_descr, id, header.dst ); - ignored.push(id); } self.check_stop_flag()?; } exec_manager.wait_transactions(collator_data).await?; let (accepted, rejected) = collator_data.withdraw_ext_msg_statuses(); - let processed = accepted.len() + rejected.len(); - let accepted = accepted.into_iter().map(|(id, _)| id).collect(); - collator_data.set_remp_msg_statuses(accepted, rejected, ignored); - Ok(processed) + Ok((accepted.len(), rejected.len())) } async fn process_new_messages( @@ -2924,11 +2896,6 @@ impl Collator { // } // !!!! DEBUG !!!! - if is_remp_enabled(self.engine.clone(), mc_data.config()) { - let (accepted, rejected, ignored) = collator_data.withdraw_remp_msg_statuses(); - self.engine.finalize_remp_messages(block_id.clone(), accepted, rejected, ignored)?; - } - self.check_stop_flag()?; let collated_data = if !collator_data.shard_top_block_descriptors.is_empty() { diff --git a/src/validator/fabric.rs b/src/validator/fabric.rs index f79b805f..898e6f2c 100644 --- a/src/validator/fabric.rs +++ b/src/validator/fabric.rs @@ -16,15 +16,16 @@ use std::{ time::SystemTime, }; use super::validator_utils::{ - validator_query_candidate_to_validator_block_candidate, pairvec_to_cryptopair_vec, - get_first_block_seqno_after_prevs + get_first_block_seqno_after_prevs, + pairvec_to_cryptopair_vec, + validator_query_candidate_to_validator_block_candidate, }; use crate::{ - collator_test_bundle::CollatorTestBundle, engine_traits::EngineOperations, - validator::{CollatorSettings, validate_query::ValidateQuery, collator}, - validating_utils::{fmt_next_block_descr_from_next_seqno, fmt_next_block_descr} + collator_test_bundle::CollatorTestBundle, + engine_traits::{EngineOperations, RempQueueCollatorInterface}, + validating_utils::{fmt_next_block_descr_from_next_seqno, fmt_next_block_descr}, + validator::{CollatorSettings, validate_query::ValidateQuery, collator, verification::VerificationManagerPtr} }; -use crate::validator::verification::VerificationManagerPtr; use ever_block::{Block, BlockIdExt, Deserializable, Result, ShardIdent, UInt256, ValidatorSet}; use validator_session::{ValidatorBlockCandidate, BlockPayloadPtr, PublicKeyHash, PublicKey}; @@ -211,6 +212,7 @@ pub async fn run_collate_query ( _min_ts: SystemTime, min_mc_seqno: u32, prev: Vec, + remp_collator_interface: Option>, collator_id: PublicKey, set: ValidatorSet, engine: Arc, @@ -231,6 +233,7 @@ pub async fn run_collate_query ( UInt256::from(collator_id.pub_key()?), engine.clone(), None, + remp_collator_interface, CollatorSettings::default() )?; let collator_result = collator.collate().await; diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 186a664b..dadcc468 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -28,6 +28,7 @@ use adnl::telemetry::Metric; use crate::{ engine_traits::RempDuplicateStatus, ext_messages::{ + create_ext_message, get_level_and_level_change, get_level_numeric_value, is_finally_accepted, is_finally_rejected }, @@ -43,14 +44,18 @@ use ton_api::{ IntoBoxed, ton::ton_node::{ rempmessagestatus::{RempAccepted, RempIgnored}, - RempMessageStatus, RempMessageLevel + RempMessageStatus, RempMessageLevel, + rempcatchainrecordv2::RempCatchainMessageHeaderV2 } }; use ever_block::{ - error, fail, BlockIdExt, Deserializable, ExternalInboundMessageHeader, KeyId, Message, + error, fail, BlockIdExt, Deserializable, ExternalInboundMessageHeader, + GetRepresentationHash, + KeyId, Message, MsgAddressInt, MsgAddrStd, Result, Serializable, SliceData, UInt256, UnixTime32 }; +use ever_block_json::unix_time_to_system_time; #[cfg(test)] #[path = "tests/test_message_cache.rs"] @@ -61,65 +66,42 @@ pub struct RmqMessage { pub message: Arc, pub message_id: UInt256, pub message_uid: UInt256, - pub source_key: Arc, - pub source_idx: u32, - pub timestamp: u32, } impl RmqMessage { - pub fn new(message: Arc, message_id: UInt256, message_uid: UInt256, source_key: Arc, source_idx: u32) -> Result { - return Ok(RmqMessage { message, message_id, message_uid, source_key, source_idx, timestamp: Self::timestamp_now()? }) - } - - pub fn from_rmq_record(record: &ton_api::ton::ton_node::rempcatchainrecord::RempCatchainMessage) -> Result { - let message= Arc::new(Message::construct_from_bytes(&record.message)?); - Ok(RmqMessage { - message: message.clone(), - message_id: record.message_id.clone(), - message_uid: get_message_uid(&message), - source_key: KeyId::from_data(record.source_key_id.as_slice().clone()), - source_idx: record.source_idx as u32, - timestamp: Self::timestamp_now()? - }) + pub fn new(message: Arc) -> Result { + let message_id = message.hash()?; + Self::new_with_id (message, message_id) } - fn timestamp_now() -> Result { - Ok(SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?.as_secs() as u32) + fn new_with_id(message: Arc, message_id: UInt256) -> Result { + let message_uid = get_message_uid(&message); + return Ok(RmqMessage { message, message_id, message_uid }) } - pub fn new_with_updated_source_idx(&self, source_idx: u32) -> Self { - RmqMessage { - message: self.message.clone(), - message_id: self.message_id.clone(), - message_uid: self.message_uid.clone(), - source_key: self.source_key.clone(), - source_idx, - timestamp: self.timestamp - } + pub fn from_raw_message(raw_msg: &ton_api::ton::bytes) -> Result { + let (message_id, message) = create_ext_message(raw_msg)?; + Self::new_with_id (Arc::new(message), message_id) } - pub fn has_no_source_key(&self) -> bool { - self.source_key.data().to_vec().iter().all(|x| *x == 0) + pub fn as_remp_message_body(&self) -> ton_api::ton::ton_node::RempMessageBody { + ton_api::ton::ton_node::rempmessagebody::RempMessageBody { + message: self.message.write_to_bytes().unwrap().into() + }.into_boxed() } - pub fn deserialize(raw: &ton_api::ton::bytes) -> Result { - let rmq_record: ton_api::ton::ton_node::RempCatchainRecord = catchain::utils::deserialize_tl_boxed_object(&raw)?; - Ok(rmq_record) + pub fn serialize_message_body(body: &ton_api::ton::ton_node::RempMessageBody) -> ton_api::ton::bytes { + serialize_tl_boxed_object!(body) } - pub fn as_rmq_record(&self, master_cc: u32) -> ton_api::ton::ton_node::RempCatchainRecord { - ton_api::ton::ton_node::rempcatchainrecord::RempCatchainMessage { - message: self.message.write_to_bytes().unwrap().into(), - message_id: self.message_id.clone().into(), - source_key_id: UInt256::from(self.source_key.data()), - source_idx: self.source_idx as i32, - masterchain_seqno: master_cc as i32 - }.into_boxed() + pub fn deserialize_message_body(raw: &ton_api::ton::bytes) -> Result { + let message_body: ton_api::ton::ton_node::RempMessageBody = catchain::utils::deserialize_tl_boxed_object(&raw)?; + Ok(message_body) } - pub fn serialize(rmq_record: &ton_api::ton::ton_node::RempCatchainRecord) -> Result { - let rmq_record_serialized = serialize_tl_boxed_object!(rmq_record); - return Ok(rmq_record_serialized) + pub fn from_message_body(body: &ton_api::ton::ton_node::RempMessageBody) -> Result { + let (message_id, message) = create_ext_message(body.message())?; + Self::new_with_id (Arc::new(message), message_id) } #[allow(dead_code)] @@ -148,23 +130,15 @@ impl RmqMessage { msg, msg_cell.data(), msg_cell.repr_hash().to_hex_string() ); - let (msg_id, msg_uid, msg) = (msg_cell.repr_hash(), get_message_uid(&msg), msg); - RmqMessage::new (Arc::new(msg), msg_id, msg_uid, KeyId::from_data([0; 32]), 0) + RmqMessage::new (Arc::new(msg)) } } impl fmt::Display for RmqMessage { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let source_key = if self.has_no_source_key() { - "(broadcast)".to_owned() - } - else { - format!("{}", self.source_key) - }; - - write!(f, "id {:x}, uid {:x}, source {}, source_idx {}, ts {}", - self.message_id, self.message_uid, source_key, self.source_idx, self.timestamp + write!(f, "fullmsg, id {:x}, uid {:x}", + self.message_id, self.message_uid ) } } @@ -172,15 +146,72 @@ impl fmt::Display for RmqMessage { #[derive(Debug,PartialEq,Eq)] pub struct RempMessageHeader { pub message_id: UInt256, - pub message_uid: UInt256 + pub message_uid: UInt256, } impl RempMessageHeader { - pub fn new_arc(message_id: &UInt256, message_uid: &UInt256) -> Arc { - Arc::new(RempMessageHeader { + pub fn new(message_id: &UInt256, message_uid: &UInt256) -> Self { + RempMessageHeader { message_id: message_id.clone(), message_uid: message_uid.clone() - }) + } + } + + pub fn new_arc(message_id: &UInt256, message_uid: &UInt256) -> Arc { + Arc::new(Self::new(message_id, message_uid)) + } +/* + pub fn from_rmq_record(record: &ton_api::ton::ton_node::rempcatchainrecordv2::RempCatchainMessageHeaderV2) -> Result<(RempMessageHeader, RempMessageOrigin)> { + let header = RempMessageHeader { + message_id: record.message_id.clone(), + message_uid: record.message_uid.clone(), + }; + let origin = RempMessageOrigin { + source_key: KeyId::from_data(record.source_key_id.as_slice().clone()), + source_idx: record.source_idx as u32, + timestamp: RempMessageOrigin::timestamp_now()? + }; + Ok((header, origin)) + } +*/ + pub fn deserialize(raw: &ton_api::ton::bytes) -> Result { + let rmq_record: ton_api::ton::ton_node::RempCatchainRecordV2 = catchain::utils::deserialize_tl_boxed_object(&raw)?; + Ok(rmq_record) + } +/* + pub fn as_remp_catchain_record(&self, master_cc: u32, origin: &RempMessageOrigin) -> ton_api::ton::ton_node::RempCatchainRecordV2 { + ton_api::ton::ton_node::rempcatchainrecordv2::RempCatchainMessageHeaderV2 { + message_id: self.message_id.clone().into(), + message_uid: self.message_uid.clone().into(), + source_key_id: UInt256::from(origin.source_key.data()), + source_idx: origin.source_idx as i32, + masterchain_seqno: master_cc as i32 + }.into_boxed() + } +*/ + pub fn serialize(rmq_record: &ton_api::ton::ton_node::RempCatchainRecordV2) -> Result { + let rmq_record_serialized = serialize_tl_boxed_object!(rmq_record); + return Ok(rmq_record_serialized) + } + + pub fn from_remp_catchain(record: &RempCatchainMessageHeaderV2) -> Result { + Ok(Self::new(&record.message_id, &record.message_uid)) + } + + pub fn serialize_query(query: &ton_api::ton::ton_node::RempMessageQuery) -> Result { + let query_serialized = serialize_tl_boxed_object!(query); + return Ok(query_serialized) + } + + pub fn deserialize_query(raw: &ton_api::ton::bytes) -> Result { + let query = catchain::utils::deserialize_tl_boxed_object(raw)?; + Ok(query) + } + + pub fn as_remp_message_query(&self) -> ton_api::ton::ton_node::RempMessageQuery { + ton_api::ton::ton_node::rempmessagequery::RempMessageQuery { + message_id: Default::default(), + }.into_boxed() } } @@ -192,6 +223,113 @@ impl Display for RempMessageHeader { } } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RempMessageOrigin { + pub source_key: Arc, + pub source_idx: u32, + pub timestamp: u32, +} + +impl RempMessageOrigin { + pub fn new(source_key: Arc, source_idx: u32) -> Result { + Ok(RempMessageOrigin { + source_key, + source_idx, + timestamp: Self::timestamp_now()? + }) + } + + pub fn new_with_updated_source_idx(&self, source_idx: u32) -> Self { + RempMessageOrigin { + source_key: self.source_key.clone(), + source_idx, + timestamp: self.timestamp + } + } + + pub fn as_remp_catchain_record(&self, message_id: &UInt256, message_uid: &UInt256, master_cc: u32) -> ton_api::ton::ton_node::RempCatchainRecordV2 { + ton_api::ton::ton_node::rempcatchainrecordv2::RempCatchainMessageHeaderV2 { + message_id: message_id.clone().into(), + message_uid: message_uid.clone().into(), + source_key_id: UInt256::from(self.source_key.data()), + source_idx: self.source_idx as i32, + masterchain_seqno: master_cc as i32 + }.into_boxed() + } + + pub fn from_remp_catchain(record: &RempCatchainMessageHeaderV2) -> Result { + Self::new( + KeyId::from_data(record.source_key_id.as_slice().clone()), + record.source_idx as u32 + ) + } + + pub fn has_no_source_key(&self) -> bool { + self.source_key.data().to_vec().iter().all(|x| *x == 0) + } + + fn timestamp_now() -> Result { + Ok(UnixTime32::now().as_u32()) + } + + pub fn system_time(&self) -> Result { + unix_time_to_system_time(UnixTime32::new(self.timestamp).as_u32() as u64) + } + + pub fn create_empty() -> Result { + Ok(RempMessageOrigin { + source_key: KeyId::from_data([0; 32]), + source_idx: 0, + timestamp: RempMessageOrigin::timestamp_now()? + }) + } +} + +impl Display for RempMessageOrigin { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let source_key = if self.has_no_source_key() { + "(broadcast)".to_owned() + } + else { + format!("{}", self.source_key) + }; + + write!(f, "source_key {}, source_idx {}, timestamp {}", + source_key, self.source_idx, self.timestamp + ) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RempMessageWithOrigin { + pub message: RmqMessage, + pub origin: RempMessageOrigin +} + +impl RempMessageWithOrigin { + pub fn get_message_id(&self) -> &UInt256 { + return &self.message.message_id; + } + + pub fn has_no_source_key(&self) -> bool { + return self.origin.has_no_source_key(); + } + + pub fn as_remp_catchain_record(&self, master_cc: u32) -> ton_api::ton::ton_node::RempCatchainRecordV2 { + return self.origin.as_remp_catchain_record(&self.message.message_id, &self.message.message_uid, master_cc) + } + + pub fn as_remp_message_body(&self) -> ton_api::ton::ton_node::RempMessageBody { + return self.message.as_remp_message_body() + } +} + +impl Display for RempMessageWithOrigin { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}, {}", self.message, self.origin) + } +} + pub struct MessageCacheSession { master_cc: u32, @@ -207,9 +345,11 @@ pub struct MessageCacheSession { ids_for_uid: LockfreeMapSet, message_headers: DashMap>, - messages: Map>, + message_origins: DashMap>, + messages: DashMap>, message_events: LockfreeMapSet, //Map>, message_status: DashMap, + message_finally_accepted: DashMap, blocks_processed: DashSet } @@ -221,7 +361,7 @@ impl Display for MessageCacheSession { } impl MessageCacheSession { - fn insert_message_header(&self, msg_id: &UInt256, msg_hdr: Arc) -> Result<()> { + fn insert_message_header(&self, msg_id: &UInt256, msg_hdr: Arc, msg_origin: Option>) -> Result<()> { if msg_id != &msg_hdr.message_id { fail!("Message with id {:x} and its header {} have different ids", msg_id, msg_hdr); } @@ -238,28 +378,99 @@ impl MessageCacheSession { fail!("Message with id {:x} changed its header from {} to {}", msg_id, old_hdr, msg_hdr); } } + + match msg_origin { + Some(origin) => if let Some(old_origin) = self.message_origins.insert(msg_id.clone(), origin.clone()) { + fail!("Message with id {:x} changed its origin from {} to {}", msg_id, origin, old_origin) + }, + None => if let Some(old_origin) = self.message_origins.get(msg_id) { + fail!("Message with id {:x} already has its origin in cache: {}, although we do not know it at insertion", msg_id, old_origin.value()) + } + } + Ok(()) } - fn insert_message(&self, msg: Arc, msg_hdr: Arc) -> Result<()> { + fn insert_message(&self, msg: Arc, msg_hdr: Arc, msg_origin: Option>) -> Result { if msg.message_uid != msg_hdr.message_uid || msg.message_id != msg_hdr.message_id { fail!("Message with id {:x} and uid {:x} and its header {} have different uids or ids", msg.message_id, msg.message_uid, msg_hdr); } - self.insert_message_header(&msg.message_id, msg_hdr)?; + self.insert_message_header(&msg.message_id, msg_hdr, msg_origin)?; match self.messages.insert(msg.message_id.clone(), msg.clone()) { - None => Ok(()), - Some(prev) if *prev.val() == msg => Ok(()), + None => Ok(true), + Some(prev) if prev == msg => Ok(false), Some(p) => fail!("Different messages for same id {:x}, replacing {} with {}", - p.key(), p.val(), msg + msg.message_id, p, msg ) } } + /// Returns body_updated flag + fn update_missing_fields(&self, message_id: &UInt256, message: Option>, origin: Option>) -> Result { + let mut body_updated = false; + if let Some(m1) = &message { + if let Some(m2) = self.messages.get(message_id) { + if m1 != m2.value() { + fail!("Different message body: new body '{}', current cached body '{}'", m1, m2.value()); + } + } + else { + if let Some(m2) = self.messages.insert(message_id.clone(), m1.clone()) { + if m1 != &m2 { + fail!("Parallel updating of message body: new body '{}', another body '{}'", m1, m2) + } + log::warn!(target: "remp", + "Update_missing_fields: body for message {:x} inserted as new, although it is present already in cache, considering non-updated", + message_id + ) + } + else { + body_updated = true + } + } + }; + + // If origin is different from the stored one, that's not a big deal. + if let Some(o1) = &origin { + if let Some(o2) = self.message_origins.get(message_id) { + if o1 != o2.value() { + log::trace!("Different message origin: new origin '{}', current cached origin '{}'", o1, o2.value()); + } + } + else { + if let Some(o2) = self.message_origins.insert(message_id.clone(), o1.clone()) { + if o1 != &o2 { + log::trace!("Different message origin: new origin '{}', current cached origin '{}'", o1, o2); + } + } + } + } + + Ok(body_updated) + } + fn is_message_present(&self, msg_id: &UInt256) -> bool { self.message_headers.contains_key(msg_id) } + fn is_message_fully_known(&self, msg_id: &UInt256) -> Result { + if self.message_headers.contains_key(msg_id) && + self.message_origins.contains_key(msg_id) && + self.messages.contains_key(msg_id) + { + if self.get_message_status(msg_id)?.is_some() { + return Ok(true); + } + else { + fail!("Inconsistent message info in cache: {}", self.message_info(msg_id)) + } + } + else { + return Ok(false); + } + } + fn starts_before_block(&self, blk: &BlockIdExt) -> bool { for inf in &self.inf_shards { if inf.shard().intersect_with(blk.shard()) { @@ -271,14 +482,64 @@ impl MessageCacheSession { return true } - fn is_full_message(&self, msg_id: &UInt256) -> bool { - self.messages.get(msg_id).is_some() + fn all_messages_count(&self) -> (usize,usize,usize) { + (self.message_headers.len(), self.messages.len(), self.message_origins.len()) + } + + fn alter_message_status(&self, message_id: &UInt256, status_updater: F) + -> Result<(RempMessageStatus,RempMessageStatus)> + where F: FnOnce(&RempMessageStatus) -> RempMessageStatus + { + match &mut self.message_status.get_mut(message_id) { + None => { + fail!("Changing status: no message {:x} in message cache session {}", message_id, self) + }, + Some(status) => { + let old_status = status.value().clone(); + let new_status = status_updater(&old_status); + if is_finally_accepted(&new_status) { + if let Some(prev) = self.set_finally_accepted_status(message_id, new_status.clone())? { + Ok((prev, new_status)) + } + else { + Ok((status.value().clone(), new_status)) + } + } + else { + *status.value_mut() = new_status.clone(); + Ok((old_status, status.value().clone())) + } + } + } } - fn all_messages_count(&self) -> usize { - self.message_headers.len() + fn update_message_status(&self, message_id: &UInt256, new_status: RempMessageStatus) -> Result<()> { + if is_finally_accepted (&new_status) { + log::trace!(target: "remp", "Setting finally accepting status for {:x}: {}", message_id, new_status); + self.set_finally_accepted_status(message_id, new_status)?; + } + else { + let old_status = self.message_status.insert(message_id.clone(), new_status.clone()); + log::trace!(target: "remp", "Changing message status for {:x}: {:?} => {}", message_id, old_status, new_status); + + if let Some(actual_status) = self.get_message_status(&message_id)? { + if is_finally_accepted(&actual_status) { + log::error!(target: "remp", "Changing status for {:x} to {}, although it has final status {}", message_id, new_status, actual_status); + } + } + } + Ok(()) } + fn set_finally_accepted_status(&self, message_id: &UInt256, new_status: RempMessageStatus) -> Result> { + if !is_finally_accepted(&new_status) { + fail!("Set finally accepted status for {:x}: status {} is not final.", message_id, new_status) + } + let old = self.message_finally_accepted.insert(message_id.clone(), new_status); + Ok(old) + } + +/* fn update_message_status(&self, message_id: &UInt256, new_status: RempMessageStatus) -> Result { match self.message_status.insert(message_id.clone(), new_status.clone()) { None => fail!("Changing status to {}: no message {:x} in message cache session {}", new_status, message_id, self), @@ -288,30 +549,35 @@ impl MessageCacheSession { } } } - - fn alter_message_status(&self, message_id: &UInt256, status_updater: F) - -> Result<(RempMessageStatus,RempMessageStatus)> - where F: FnOnce(&RempMessageStatus) -> RempMessageStatus - { - match &mut self.message_status.get_mut(message_id) { - None => fail!("Changing status: no message {:x} in message cache session {}", message_id, self), - Some(status) => { - let old_status = status.value().clone(); - *status.value_mut() = status_updater(&old_status); - Ok((old_status, status.value().clone())) + */ + fn get_message_status(&self, message_id: &UInt256) -> Result> { + if let Some(status) = self.message_finally_accepted.get(message_id) { + if !is_finally_accepted(status.value()) { + fail!("Only finally accepted status may be stored in message_finally_accepted, for id {:x} found {}", message_id, status.value()) + } + Ok(Some(status.value().clone())) + } + else if let Some(status) = self.message_status.get(message_id) { + if is_finally_accepted(status.value()) { + fail!("No finally accepted status may be stored in message_status, for id {:x} found {}", message_id, status.value()) } + Ok(Some(status.value().clone())) + } + else { + Ok(None) } } fn message_events_to_string(&self, message_id: &UInt256) -> String { - if let Some(msg) = self.messages.get(message_id) { - let base = msg.val().timestamp; - let events = self.message_events.get_set(message_id); + let events = self.message_events.get_set(message_id); + let result = if let Some(msg) = self.message_origins.get(message_id) { + let base = msg.value().timestamp; events.iter().map(|x| format!("{} ", (*x) as i64 - base as i64)).collect() } else { - "*events: no message base time*".to_string() - } + "*events: no message origin*".to_string() + }; + format!("{} ({} events)", result, events.len()) } fn message_info(&self, message_id: &UInt256) -> String { @@ -319,19 +585,39 @@ impl MessageCacheSession { .get(message_id).map(|x| format!("uid: {:x}", x.value().message_uid)) .unwrap_or_else(|| "*error: no header in cache*".to_owned()); - let status = self.message_status - .get(message_id) - .map(|x| format!("{:?}", x.value())) - .unwrap_or_else(|| "*error: no status in cache*".to_owned()); + let status1 = self.message_status.get(message_id).map(|x| x.value().clone()); + let status2 = self.message_finally_accepted.get(message_id).map(|x| x.value().clone()); + let status = match (status1, status2) { + (Some(x), None) => format!("{:?}/**", x), + (None, Some(x)) => format!("**/{:?}", x), + (Some(x), Some(y)) => format!("{:?}/{:?}", x, y), + (None, None) => "*error: no status in cache*".to_owned() + }; + + let has_additional_info = self.messages.contains_key(message_id) + || self.message_origins.contains_key(message_id) + || !self.message_events.get_set(message_id).is_empty(); + + let additional_info = if has_additional_info { + let body = self.messages + .get(message_id).map(|_| "has body".to_owned()) + .unwrap_or_else(|| "*error: no body*".to_owned()); + + let origin = self.message_origins + .get(message_id).map(|x| x.value().to_string()) + .unwrap_or_else(|| "*error: no origin*".to_owned()); - let collation_history = if self.is_full_message(message_id) { - self.message_events_to_string(message_id) + let collation_history = self.message_events_to_string(message_id); + + format!("{}, {}, {}", body, origin, collation_history) } else { "header only".to_owned() }; - format!("id {:x}, cc {}, {}, status: {}, {}", message_id, self.master_cc, header, status, collation_history) + format!("id {:x}, cc {}, {}, status: {}, {}", + message_id, self.master_cc, header, status, additional_info + ) } fn mark_collation_attempt(&self, msg_id: &UInt256) -> Result<()> { @@ -349,16 +635,25 @@ impl MessageCacheSession { log::debug!(target: "remp", "Removing old message: {}", self.message_info(&id)); - match (self.messages.get(&id), self.message_status.get(&id)) { + let message_status = match self.get_message_status(&id) { + Err(e) => { + log::error!(target: "remp", "Record for message {:?} is incorrect: err: {}", id, e); + stats.incorrect += 1; + continue; + } + Ok(s) => s + }; + + match (self.messages.get(&id), message_status) { (Some(_m),Some(status)) => { - if is_finally_accepted(status.value()) { stats.accepted_in_session += 1 } - else if is_finally_rejected(status.value()) { stats.rejected_in_session += 1 } + if is_finally_accepted(&status) { stats.accepted_in_session += 1 } + else if is_finally_rejected(&status) { stats.rejected_in_session += 1 } }, (None,Some(_status)) => stats.has_only_header += 1, (m, h) => { log::error!(target: "remp", "Record for message {:?} is in incorrect state: msg = {:?}, status = {:?}", - id, m.map(|x| x.val().clone()), h.map(|x| x.value().clone()) + id, m.map(|x| x.value().clone()), h.map(|x| x.clone()) ); stats.incorrect += 1 } @@ -373,9 +668,11 @@ impl MessageCacheSession { start_time, ids_for_uid: LockfreeMapSet::default(), message_headers: DashMap::new(), + message_origins: DashMap::new(), message_events: LockfreeMapSet::default(), - messages: Map::default(), + messages: DashMap::default(), message_status: DashMap::default(), + message_finally_accepted: DashMap::default(), inf_shards: HashSet::from_iter(inf_shards.into_iter()), blocks_processed: DashSet::default(), } @@ -393,40 +690,40 @@ pub struct MessageCache { cache_size_metric: Arc, } -#[allow(dead_code)] impl MessageCache { - pub fn cc_expired(&self, old_cc_seqno: u32) -> bool { - old_cc_seqno < self.master_cc_seqno_lwb.load(Ordering::Relaxed) - } - - pub fn all_messages_count(&self) -> usize { + /// Returns stats about available messages' info in cache, three counts: + /// (total, with_bodies, with_origins). + /// total --- total messages in cache count; + /// with_bodies --- messages that have bodies (that is, broadcasted/received from full node); + /// with_origins --- messages that have origins (that is, received through catchain/from full node). + /// + /// Some messages never have body/origin (those that were taken from accepted master blocks), + /// others may have either body or origin missing due to delays or losses in network. + pub fn all_messages_count(&self) -> (usize,usize,usize) { let range = self.get_master_cc_stored_range(); let mut result: usize = 0; + let mut with_bodies: usize = 0; + let mut with_origins: usize = 0; for cc in range { if let Some(s) = self.sessions.get(&cc) { - result += s.val().all_messages_count(); + let (r,b,o) = s.val().all_messages_count(); + result += r; + with_bodies += b; + with_origins += o; } } - result + (result, with_bodies, with_origins) } - /// Returns new message status, if it worths reporting (final statuses do not need to be reported) - pub fn update_message_status(&self, message_id: &UInt256, new_status: RempMessageStatus) -> Result> { + /// Updates message status: changes status to the given value + pub fn update_message_status(&self, message_id: &UInt256, new_status: RempMessageStatus) -> Result<()> { let session = self.get_session_for_message(message_id).ok_or_else( || error!("Cannot find message {:x} to change its status to {:?}", message_id, new_status) )?; - if let RempMessageStatus::TonNode_RempAccepted(acc_new) = &new_status { - if acc_new.level == RempMessageLevel::TonNode_RempMasterchain { - session.update_message_status(message_id, new_status.clone())?; - return Ok(None) - } - } - - session.update_message_status(message_id, new_status.clone())?; - Ok(Some(new_status)) + session.update_message_status(message_id, new_status) } fn get_session_for_message(&self, message_id: &UInt256) -> Option> { @@ -460,7 +757,7 @@ impl MessageCache { pub fn get_message(&self, message_id: &UInt256) -> Result>> { let msg = self.get_session_for_message(message_id).map( |session| session.messages.get(message_id).map( - |m| m.val().clone() + |m| m.value().clone() ) ); Ok(msg.flatten()) @@ -469,10 +766,7 @@ impl MessageCache { pub fn get_message_status(&self, message_id: &UInt256) -> Result> { match self.get_session_for_message(message_id) { None => Ok(None), - Some(s) => - Ok(Some(s.message_status.get(message_id).ok_or_else(|| error!("No status for message {:x}, {}", - message_id, s - ))?.value().clone())) + Some(s) => s.get_message_status(message_id) } } @@ -486,52 +780,82 @@ impl MessageCache { } } - pub fn get_message_with_status(&self, message_id: &UInt256) -> Result, RempMessageStatus)>> { - self.get_message_with_status_cc(message_id).map(|e| e.map(|(m,s,_c)| (m,s))) - } - - pub fn get_message_with_status_cc(&self, message_id: &UInt256) -> Result, RempMessageStatus, u32)>> { + pub fn get_message_with_origin_status_cc(&self, message_id: &UInt256) -> Result>, Arc, Arc, RempMessageStatus, u32)>> { let session = match self.get_session_for_message(message_id) { None => return Ok(None), Some(s) => s }; - let (msg, status) = ( - session.messages.get(message_id).map(|m| m.val().clone()), - session.message_status.get(message_id).map(|m| m.value().clone()) + let (header, msg, origin, status) = ( + session.message_headers.get(message_id).ok_or_else(|| error!("Message {:x} has no header", message_id))?.value().clone(), + session.messages.get(message_id).map(|m| m.value().clone()), + session.message_origins.get(message_id).map(|m| m.value().clone()), + session.get_message_status(message_id)? ); - match (msg, status) { - (None, Some(_)) => Ok(None), // Bare message info (retrieved from finalized block) - (Some(m), Some (h)) => Ok(Some((m.clone(),h.clone(),session.master_cc))), // Full message info - (m, None) => fail!("Message {:x} has no status, body = {:?}", message_id, m) + match (msg, origin, status) { + (_, None, Some(_)) => Ok(None), // Bare message info (retrieved from finalized block/not received from broadcast) + (m, Some(o), Some (s)) => + Ok(Some((m.clone(), header.clone(), o.clone(), s.clone(), session.master_cc))), // Full message info + (m, o, None) => + fail!("Message {:x} has no status, body = {:?}, origin = {:?}", message_id, m, o), + } + } + + pub fn get_message_origin(&self, message_id: &UInt256) -> Result>> { + match self.get_session_for_message(message_id) { + None => Ok(None), + Some(s) => Ok(s.message_origins.get(message_id).map(|m| m.value().clone())) + } + } + + pub fn get_message_info(&self, message_id: &UInt256) -> Result { + match self.get_session_for_message(message_id) { + None => Ok("absent".to_string()), + Some(s) => Ok(s.message_info(message_id)) + } + } + + pub fn is_message_fully_known(&self, message_id: &UInt256) -> Result { + match self.get_session_for_message(message_id) { + None => Ok(false), + Some(s) => s.is_message_fully_known(message_id) } } - fn insert_message(&self, session: Arc, message: Arc, message_header: Arc, status: &RempMessageStatus) -> Result<()> { + fn insert_message(&self, session: Arc, message: Arc, message_header: Arc, message_origin: Option>, status: &RempMessageStatus) -> Result { if message.message_id != message_header.message_id { fail!("Inconsistent message: message {} and message_header {} have different message_id", message, message_header) } let message_id = message.message_id.clone(); - if session.is_message_present(&message_id) { - fail!("Inconsistent message cache contents: message {} present in cache, although should not", message_id) - } + //if session.is_message_present(&message_id) { + // fail!("Inconsistent message cache contents: message {} present in cache, although should not", message_id) + //} - session.message_status.insert(message_id.clone(), status.clone()); - session.insert_message(message, message_header)?; - Ok(()) + if is_finally_accepted(status) { + session.message_finally_accepted.insert(message_id.clone(), status.clone()); + } + else { + session.message_status.insert(message_id.clone(), status.clone()); + } + session.insert_message(message, message_header, message_origin) } - fn insert_message_header(&self, session: Arc, message_header: Arc, status: &RempMessageStatus) -> Result<()> { + fn insert_message_header(&self, session: Arc, message_header: Arc, message_origin: Option>, status: &RempMessageStatus) -> Result<()> { let message_id = message_header.message_id.clone(); - if session.is_message_present(&message_id) { - fail!("Inconsistent message cache contents: message header {:x} present in cache, although should not", message_id) - } + //if session.is_message_present(&message_id) { + // fail!("Inconsistent message cache contents: message header {:x} present in cache, although should not", message_id) + //} - session.message_status.insert(message_id.clone(), status.clone()); - session.insert_message_header(&message_id, message_header)?; + if is_finally_accepted(status) { + session.message_finally_accepted.insert(message_id.clone(), status.clone()); + } + else { + session.message_status.insert(message_id.clone(), status.clone()); + } + session.insert_message_header(&message_id, message_header, message_origin)?; Ok(()) } @@ -539,12 +863,12 @@ impl MessageCache { /// If we know something about message -- that's more important than anything we discover from RMQ /// If we do not know anything -- TODO: if all reject, then 'Rejected'. Otherwise 'New' /// Actual -- get it as granted ("imprinting") - /// Returns old status and new (added) status - pub async fn add_external_message_status(&self, - message_id: &UInt256, message_uid: &UInt256, message: Option>, + /// Returns old status, new (added) status, and body_updated flag + pub fn add_external_message_status(&self, + message_id: &UInt256, message_uid: &UInt256, message: Option>, message_origin: Option>, status_if_new: RempMessageStatus, status_updater: F, master_cc: u32 - ) -> Result<(Option,RempMessageStatus)> + ) -> Result<(Option,RempMessageStatus,bool)> where F: FnOnce(&RempMessageStatus, &RempMessageStatus) -> RempMessageStatus { match self.get_session_for_message(message_id) { @@ -560,20 +884,54 @@ impl MessageCache { message_uid ); - match message { - None => self.insert_message_header( session, header, &status_if_new)?, - Some(message) => self.insert_message(session, message, header, &status_if_new)? + let body_updated = match message { + None => { + self.insert_message_header( session, header, message_origin, &status_if_new)?; + false + }, + Some(message) => + self.insert_message(session, message, header, message_origin.clone(), &status_if_new)? }; - Ok((None, status_if_new)) + Ok((None, status_if_new, body_updated)) }, Some(session) => { + let body_updated = session + .update_missing_fields(&message_id, message, message_origin) + .unwrap_or_else(|e| { + log::error!(target: "remp", "Different cache contents for external message {}: {}", message_id, e); + false + }); + let (old_status, final_status) = session.alter_message_status(&message_id, |old| status_updater(old,&status_if_new))?; - Ok((Some(old_status), final_status)) + + Ok((Some(old_status), final_status, body_updated)) }, } } + /// Updates message body for the message + /// `data` --- Message body with ids + /// `return` --- Whether the body added or not + pub fn update_message_body(&self, data: Arc) -> Result { + let (old_status, new_status, body_updated) = self.add_external_message_status( + &data.message_id, + &data.message_uid, + Some(data.clone()), + None, + RempMessageStatus::TonNode_RempNew, |old,_new| old.clone(), + self.master_cc_seqno_curr.load(Ordering::Relaxed) + )?; + let info = self.get_message_info(&data.message_id)?; + log::trace!(target: "remp", "Message {}, tried to update message body: old status {}, new status {}, full message cache info {}", + &data, + match old_status { Some(m) => format!("{}",m), None => format!("None") }, + new_status, + info + ); + Ok(body_updated) + } + /// Checks whether message msg_id is accepted by collator; /// if true, changes its status to ignored pub fn change_accepted_by_collator_to_ignored(&self, msg_id: &UInt256) -> Result { @@ -686,7 +1044,8 @@ impl MessageCache { } pub fn message_stats(&self) -> String { - format!("All REMP messages count = {}", self.all_messages_count()) + let (count,with_origins,with_bodies) = self.all_messages_count(); + format!("All REMP messages count = {}, of them: with origins (via catchain) = {}, with bodies (broadcasted) = {}", count, with_origins, with_bodies) } pub fn try_set_master_cc_start_time(&self, master_cc: u32, start_time: UnixTime32, inf_blocks: Vec) -> Result<()> { @@ -881,7 +1240,7 @@ impl MessageCache { stats.add(&session.val().gc_all()); #[cfg(feature = "telemetry")] - self.cache_size_metric.update(self.all_messages_count() as u64); + self.cache_size_metric.update(self.all_messages_count().0 as u64); } self.master_cc_seqno_stored.store(cc_to_remove+1, Relaxed); } diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 62b78ef7..904e3af4 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -12,28 +12,33 @@ */ use std::{ + cmp::Reverse, collections::{BinaryHeap, HashMap}, fmt, fmt::Formatter, + ops::RangeInclusive, sync::Arc, - time::Duration, time::SystemTime + time::{Duration, SystemTime} }; -use std::cmp::Reverse; -use std::ops::RangeInclusive; +use std::sync::atomic::{AtomicUsize, Ordering}; use dashmap::DashMap; use ton_api::ton::ton_node::{ RempMessageStatus, RempMessageLevel, - rempmessagestatus::{RempAccepted, RempIgnored, RempRejected}, RempCatchainRecord + rempmessagestatus::{RempAccepted, RempIgnored, RempRejected}, + RempMessageStatus::TonNode_RempRejected, + RempCatchainRecordV2, + rempcatchainrecordv2::{RempCatchainMessageHeaderV2, RempCatchainMessageDigestV2} }; -use ever_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr}; -use ever_block::{UInt256, Result, fail}; +use ever_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr, Sha256, Serializable}; +use ever_block::{UInt256, Result, fail, gen_random_index, error}; + use catchain::{PrivateKey, PublicKey}; use crate::{ - engine_traits::EngineOperations, - ext_messages::{get_level_and_level_change, is_finally_accepted, is_finally_rejected}, + engine_traits::{EngineOperations, RempDuplicateStatus, RempQueueCollatorInterface}, + ext_messages::{get_level_and_level_change, is_finally_accepted, is_finally_rejected, MAX_EXTERNAL_MESSAGE_SIZE}, validator::{ mutex_wrapper::MutexWrapper, - message_cache::RmqMessage, + message_cache::{RmqMessage, RempMessageHeader, RempMessageOrigin, RempMessageWithOrigin}, remp_manager::RempManager, remp_block_parser::{process_block_messages_by_blockid, BlockProcessor}, remp_catchain::{RempCatchainInfo, RempCatchainInstance}, @@ -42,15 +47,16 @@ use crate::{ } }; use failure::err_msg; -use ton_api::ton::ton_node::rempcatchainrecord::{RempCatchainMessage, RempCatchainMessageDigest}; -use ton_api::ton::ton_node::RempMessageStatus::TonNode_RempRejected; +use storage::db::traits::DbKey; use crate::block::BlockIdExtExtention; -use crate::engine_traits::RempDuplicateStatus; #[derive(Debug,PartialEq,Eq,PartialOrd,Ord,Clone)] enum MessageQueueStatus { Created, Starting, Active, Stopping } const RMQ_STOP_POLLING_INTERVAL: Duration = Duration::from_millis(50); -const RMQ_REQUEST_NEW_BLOCK_INTERVAL: Duration = Duration::from_millis(50); +const RMQ_REQUEST_NEW_BLOCK_INTERVAL: Duration = Duration::from_millis(10); +const RMQ_MAXIMAL_BROADCASTS_IN_PACK: u32 = 1000; +const RMQ_MAXIMAL_QUERIES_IN_PACK: u32 = 1000; +const RMQ_MESSAGE_QUERY_TIMEOUT: Duration = Duration::from_millis(2000); struct MessageQueueImpl { status: MessageQueueStatus, @@ -60,7 +66,10 @@ struct MessageQueueImpl { pending_collation_set: HashMap, /// Messages, waiting for collator invocation - pending_collation_order: BinaryHeap<(Reverse, UInt256)>, + pending_collation_order: BinaryHeap<(Reverse, UInt256)>, + + /// Body sources for the message (actual if current node didn't receive body yet) + body_sources: HashMap> } pub struct MessageQueue { @@ -72,30 +81,46 @@ pub struct MessageQueue { } impl MessageQueueImpl { - pub fn add_to_collation_queue(&mut self, message_id: &UInt256, timestamp: u32, add_if_absent: bool) -> Result<(bool,usize)> { - match self.pending_collation_set.get_mut(message_id) { - Some(waiting_collation) if *waiting_collation => Ok((false, self.pending_collation_set.len())), + fn insert_body_source(&mut self, message_id: &UInt256, body_source: u32) { + match self.body_sources.remove(message_id) { + None => self.body_sources.insert(message_id.clone(), vec!(body_source)), + Some(mut vec) => { + if !vec.contains(&body_source) { + vec.push(body_source); + } + self.body_sources.insert(message_id.clone(), vec) + } + }; + } + + pub fn add_to_collation_queue(&mut self, message_id: &UInt256, timestamp: SystemTime, remp_node_sender: Option, add_if_absent: bool) -> Result<(bool,usize)> { + let added = match self.pending_collation_set.get_mut(message_id) { + Some(waiting_collation) if *waiting_collation => false, Some(waiting_collation) => { self.pending_collation_order.push((Reverse(timestamp), message_id.clone())); *waiting_collation = true; - Ok((true, self.pending_collation_set.len())) + true }, None if add_if_absent => { self.pending_collation_set.insert(message_id.clone(), true); self.pending_collation_order.push((Reverse(timestamp), message_id.clone())); // Max heap --- need earliest message - Ok((true, self.pending_collation_set.len())) + true } None => fail!("Message {} is not present in collation set, cannot return it to collation queue", message_id) + }; + if let Some(ns) = remp_node_sender { + self.insert_body_source(message_id, ns); } + Ok((added, self.pending_collation_set.len())) } - pub fn take_first_for_collation(&mut self) -> Result> { - if let Some((timestamp, id)) = self.pending_collation_order.pop() { + pub fn take_first_for_collation(&mut self) -> Result> { + if let Some((Reverse(timestamp), id)) = self.pending_collation_order.pop() { match self.pending_collation_set.insert(id.clone(), false) { None => fail!("Message {:?}: taken from collation queue, but not found in collation set", id), Some(false) => fail!("Message {:?}: already removed from collation queue and given to collator", id), - Some(true) => Ok(Some((id, timestamp.0))) + Some(true) => Ok(Some((id, timestamp))) } } else { @@ -103,6 +128,10 @@ impl MessageQueueImpl { } } + pub fn get_body_sources(&self, message_id: &UInt256) -> Option<&Vec> { + self.body_sources.get(message_id) + } + pub fn list_pending_for_forwarding(&mut self) -> Result> { return Ok(self.pending_collation_set.keys().cloned().collect()) } @@ -121,10 +150,11 @@ impl MessageQueue { status: MessageQueueStatus::Created, pending_collation_set: HashMap::new(), pending_collation_order: BinaryHeap::new(), + body_sources: HashMap::new(), }, format!("<>", remp_catchain_instance), #[cfg(feature = "telemetry")] - engine.remp_core_telemetry().rmq_catchain_mutex_metric(&remp_catchain_info.general_session_info.shard), + engine.remp_core_telemetry().rmq_catchain_mutex_metric(&remp_catchain_info.general_session_info.shard), ); log::trace!(target: "remp", "Creating MessageQueue {}", remp_catchain_instance); @@ -138,7 +168,7 @@ impl MessageQueue { }); } - pub async fn create_and_start ( + pub async fn create_and_start( engine: Arc, manager: Arc, info: Arc, local_key: PrivateKey ) -> Result> { let queue = Arc::new(Self::create(engine, manager, info)?); @@ -150,61 +180,60 @@ impl MessageQueue { self.queues.execute_sync(|q| { if q.status != required_status { fail!("RMQ {}: MessageQueue status is {:?}, but required to be {:?}", self, q.status, required_status) - } - else { + } else { q.status = new_status; Ok(()) } }).await } - pub fn send_response_to_fullnode(&self, rmq_message: Arc, status: RempMessageStatus) { - log::debug!(target: "remp", "RMQ {}: queueing response to fullnode {}, status {}", - self, rmq_message, status + pub fn send_response_to_fullnode(&self, message_id: &UInt256, origin: Arc, status: RempMessageStatus) { + log::debug!(target: "remp", "RMQ {}: queueing response to fullnode {:x}, status {}", + self, message_id, status ); - if rmq_message.has_no_source_key() { - log::trace!(target: "remp", "RMQ {}: message {} was broadcast and has no source key, no response", self, rmq_message) - } - else if let Err(e) = self.remp_manager.queue_response_to_fullnode( - self.catchain_info.local_key_id.clone(), rmq_message.clone(), status.clone() + if origin.has_no_source_key() { + log::trace!(target: "remp", "RMQ {}: message {:x} was broadcast and has no source key, no response", self, message_id) + } else if let Err(e) = self.remp_manager.queue_response_to_fullnode( + self.catchain_info.local_key_id.clone(), message_id.clone(), origin.clone(), status.clone() ) { - log::error!(target: "remp", "RMQ {}: cannot queue response to fullnode: {}, {}, local key {:x}, error `{}`", - self, rmq_message, status, self.catchain_info.local_key_id, e + log::error!(target: "remp", "RMQ {}: cannot queue response to fullnode: message id {} from {}, {}, local key {:x}, error `{}`", + self, message_id, origin, status, self.catchain_info.local_key_id, e ); } } - pub fn update_status_send_response(&self, msgid: &UInt256, message: Arc, new_status: RempMessageStatus) { - match self.remp_manager.message_cache.update_message_status(&msgid, new_status.clone()) { - Ok(Some(final_status)) => self.send_response_to_fullnode(message.clone(), final_status), - Ok(None) => (), // Send nothing, no status update is requested - Err(e) => log::error!(target: "remp", + pub fn update_status_send_response(&self, msgid: &UInt256, origin: Arc, new_status: RempMessageStatus) { + if let Err(e) = self.remp_manager.message_cache.update_message_status(&msgid, new_status.clone()) { + log::error!(target: "remp", "RMQ {}: Cannot update status for {:x}, new status {}, error {}", self, msgid, new_status, e ) + } else { + if !is_finally_accepted(&new_status) { + self.send_response_to_fullnode(msgid, origin, new_status) + } } } pub async fn update_status_send_response_by_id(&self, msgid: &UInt256, new_status: RempMessageStatus) -> Result> { - let message = self.get_message(msgid)?; - match &message { - Some(rm) => { - self.update_status_send_response(msgid, rm.clone(), new_status.clone()); + match &self.remp_manager.message_cache.get_message_with_origin_status_cc(msgid)? { + Some((Some(rm), _header, origin, _, _)) => { + self.update_status_send_response(msgid, origin.clone(), new_status.clone()); Ok(rm.clone()) }, - None => + Some((None, _, _, _, _)) | None => fail!( - "RMQ {}: cannot find message {:x} in RMQ messages hash! (new status {})", + "RMQ {}: cannot find message {:x} body in RMQ messages hash! (new status {})", self, msgid, new_status ) } } - pub async fn start (self: Arc, local_key: PrivateKey) -> Result<()> { + pub async fn start(self: Arc, local_key: PrivateKey) -> Result<()> { self.set_queue_status(MessageQueueStatus::Created, MessageQueueStatus::Starting).await?; log::trace!(target: "remp", "RMQ {}: starting", self); - + let catchain_instance_res = self.remp_manager.catchain_store.start_catchain( self.engine.clone(), self.remp_manager.clone(), self.catchain_info.clone(), local_key ).await; @@ -228,10 +257,16 @@ impl MessageQueue { loop { let (do_stop, do_break) = self.queues.execute_sync(|q| { match q.status { - MessageQueueStatus::Created => { q.status = MessageQueueStatus::Stopping; Ok((false, true)) }, + MessageQueueStatus::Created => { + q.status = MessageQueueStatus::Stopping; + Ok((false, true)) + }, MessageQueueStatus::Starting => Ok((false, false)), MessageQueueStatus::Stopping => fail!("RMQ {}: cannot stop queue with 'Stopping' status", self), - MessageQueueStatus::Active => { q.status = MessageQueueStatus::Stopping; Ok((true, true)) }, + MessageQueueStatus::Active => { + q.status = MessageQueueStatus::Stopping; + Ok((true, true)) + }, } }).await?; @@ -250,54 +285,66 @@ impl MessageQueue { } } - pub async fn put_message_to_rmq(&self, old_message: Arc) -> Result<()> { - if self.queues.execute_sync(|q| q.pending_collation_set.contains_key(&old_message.message_id)).await { - log::trace!(target: "remp", "Point 3. RMQ {}; computing message {} delay --- already have it in local queue, should be skipped", self, old_message); + pub async fn put_message_to_rmq(&self, msg: Arc) -> Result<()> { + if self.queues.execute_sync(|q| q.pending_collation_set.contains_key(msg.get_message_id())).await { + log::trace!(target: "remp", "Point 3. RMQ {}; computing message {} delay --- already have it in local queue, should be skipped", self, msg); return Ok(()) } - let msg = Arc::new(old_message.new_with_updated_source_idx(self.catchain_info.local_idx as u32)); - log::trace!(target: "remp", "Point 3. Pushing to RMQ {}; message {}", self, msg); - self.catchain_instance.pending_messages_queue_send(msg.as_rmq_record(self.catchain_info.get_master_cc_seqno()))?; + let origin_with_idx = Arc::new(msg.origin.new_with_updated_source_idx(self.catchain_info.local_idx as u32)); + let body_updated = self.remp_manager.message_cache.update_message_body(Arc::new(msg.message.clone()))?; + log::trace!(target: "remp", "Point 3. Pushing to RMQ {}; message {}, {}{}", + self, msg, origin_with_idx, + (if body_updated { " + broadcast" } else { "" }).to_string() + ); + + let msg_body = if body_updated { + Some(msg.as_remp_message_body()) + } else { None }; + + self.catchain_instance.pending_messages_broadcast_send( + msg.as_remp_catchain_record(self.catchain_info.get_master_cc_seqno()), + msg_body + )?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().in_channel_to_catchain( &self.catchain_info.general_session_info.shard, self.catchain_instance.pending_messages_queue_len()?); - if let Some(session) = &self.remp_manager.catchain_store.get_catchain_session(&self.catchain_info.queue_id).await { - log::trace!(target: "remp", "Point 3. Activating RMQ {} processing", self); - session.request_new_block(SystemTime::now() + RMQ_REQUEST_NEW_BLOCK_INTERVAL); + if self.catchain_instance.is_session_active() { + log::trace!(target: "remp", "Activating RMQ {} processing", self); + self.activate_exchange()?; + self.poll_outbound_queues()?; // Temporary status "New" --- message is not registered yet - self.send_response_to_fullnode(msg, RempMessageStatus::TonNode_RempNew); + self.send_response_to_fullnode(msg.get_message_id(), origin_with_idx, RempMessageStatus::TonNode_RempNew); Ok(()) - } - else { + } else { log::error!(target: "remp", "RMQ {} not started", self); Err(failure::err_msg("RMQ is not started")) } } - async fn add_pending_collation(&self, rmq_message: Arc, status_to_send: Option) -> Result<()> { - let (added_to_queue, _len) = self.queues.execute_sync( - |catchain| catchain.add_to_collation_queue( - &rmq_message.message_id, rmq_message.timestamp, true + async fn add_pending_collation(&self, message_id: &UInt256, remp_message_origin: Arc, remp_node_sender: u32, status_to_send: Option) -> Result<()> { + let (added_to_queue, _len) = self.queues.execute_sync(|catchain| + catchain.add_to_collation_queue( + message_id, remp_message_origin.system_time()?, Some(remp_node_sender), true ) ).await?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().pending_collation( - &self.catchain_info.general_session_info.shard, + &self.catchain_info.general_session_info.shard, _len ); if added_to_queue { log::trace!(target: "remp", - "Point 5. RMQ {}: adding message {} to collator queue", self, rmq_message + "Point 5. RMQ {}: adding message {:x} to collator queue", self, message_id ); - self.remp_manager.message_cache.mark_collation_attempt(&rmq_message.message_id)?; + self.remp_manager.message_cache.mark_collation_attempt(message_id)?; if let Some(status) = status_to_send { - self.send_response_to_fullnode(rmq_message.clone(), status); + self.send_response_to_fullnode(message_id, remp_message_origin, status); } } @@ -353,20 +400,20 @@ impl MessageQueue { self.catchain_instance.is_session_active() } - async fn process_pending_remp_catchain_message(&self, rmq_record_message: &RempCatchainMessage) -> Result<()> { - let rmq_message = Arc::new(RmqMessage::from_rmq_record(rmq_record_message)?); - let rmq_message_master_seqno = rmq_record_message.masterchain_seqno as u32; - let forwarded = self.catchain_info.get_master_cc_seqno() > rmq_message_master_seqno; + async fn process_pending_remp_catchain_message(&self, catchain_record: &RempCatchainMessageHeaderV2, remp_node_sender: u32) -> Result<()> { + let remp_message_header = Arc::new(RempMessageHeader::from_remp_catchain(catchain_record)?); + let remp_message_origin = Arc::new(RempMessageOrigin::from_remp_catchain(catchain_record)?); + let message_master_seqno = catchain_record.masterchain_seqno as u32; + let forwarded = self.catchain_info.get_master_cc_seqno() > message_master_seqno; log::trace!(target: "remp", - "Point 4. RMQ {}: inserting pending message {} from RMQ into message_cache, forwarded {}, message_master_cc {}", - self, rmq_message, forwarded, rmq_message_master_seqno + "Point 4. RMQ {}: inserting pending message {}, {} from RMQ node {} into message_cache, forwarded {}, message_master_cc {}", + self, remp_message_header, remp_node_sender, remp_message_origin, forwarded, message_master_seqno ); let status_if_new = if let Some((_msg, status)) = self.is_queue_overloaded().await { status - } - else { + } else { if forwarded { Self::forwarded_ignored_status().clone() } else { RempMessageStatus::TonNode_RempNew } }; @@ -378,15 +425,15 @@ impl MessageQueue { // 1c. We know it with some normal status -- leave in place our knowledge. // 1d. We know only forwarded message status -- replace it with new status. let added = self.remp_manager.message_cache.add_external_message_status( - &rmq_message.message_id, - &rmq_message.message_uid, - Some(rmq_message.clone()), + &remp_message_header.message_id, + &remp_message_header.message_uid, + None, + Some(remp_message_origin.clone()), status_if_new, |old_status, new_status| { if Self::is_forwarded_status(old_status) { return new_status.clone() - } - else if !Self::is_final_status(old_status) && forwarded { + } else if !Self::is_final_status(old_status) && forwarded { // If the message status is non-negative (that is, not reject, timeout, etc) // And the message is non final, then at least we have not rejected it. // So, we specify non-forwarding Ignored status, giving us @@ -398,7 +445,7 @@ impl MessageQueue { None => { log::error!(target: "remp", "RMQ {}: cannot increment level {:?} for message_id {:x}", - self, old_status, rmq_message.message_id + self, old_status, remp_message_header.message_id ); RempMessageLevel::TonNode_RempMasterchain } @@ -412,34 +459,34 @@ impl MessageQueue { } old_status.clone() }, - rmq_message_master_seqno - ).await; + message_master_seqno + ); match added { Err(e) => { log::error!(target: "remp", "Point 4. RMQ {}: cannot insert new message {} into message_cache, error: `{}`", - self, rmq_message, e + self, remp_message_header, e ); }, - Ok((Some(_),new_status)) if Self::is_final_status(&new_status) => { + Ok((Some(_), new_status, _body_updated)) if Self::is_final_status(&new_status) => { log::trace!(target: "remp", "Point 4. RMQ {}. Message {:x} master_cc_seqno {} from validator {} has final status {}, skipping", - self, rmq_message.message_id, rmq_message_master_seqno, rmq_message.source_idx, new_status + self, remp_message_header.message_id, message_master_seqno, remp_message_origin.source_idx, new_status ); #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().add_to_cache_attempt(false); } - Ok((old_status,new_status)) => { + Ok((old_status, new_status, _body_updated)) => { log::trace!(target: "remp", "Point 4. RMQ {}. Message {:x} master_cc_seqno {} from validator {} has non-final status {}{}, will be collated", - self, rmq_message.message_id, rmq_message_master_seqno, rmq_message.source_idx, new_status, + self, remp_message_header.message_id, message_master_seqno, remp_message_origin.source_idx, new_status, match &old_status { None => format!(" (no old status)"), Some(x) => format!(" (old status {})", x) } ); - self.add_pending_collation(rmq_message, Some(new_status)).await?; + self.add_pending_collation(&remp_message_header.message_id, remp_message_origin, remp_node_sender, Some(new_status)).await?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().add_to_cache_attempt(true); } @@ -447,14 +494,13 @@ impl MessageQueue { Ok(()) } - async fn process_pending_remp_catchain_digest(&self, reject_digest: &RempCatchainMessageDigest) -> Result<()> { + async fn process_pending_remp_catchain_digest(&self, reject_digest: &RempCatchainMessageDigestV2) -> Result<()> { if !self.catchain_info.master_cc_range.contains(&(reject_digest.masterchain_seqno as u32)) { log::error!(target: "remp", "Point 4. RMQ {}. Message digest (masterchain_seqno = {}, len = {}) does not fit to RMQ master cc range {}", self, reject_digest.masterchain_seqno, reject_digest.messages.len(), self.catchain_info.master_cc_range_info() ) - } - else { + } else { log::info!(target: "remp", "Point 4. RMQ {}. Message digest (masterchain_seqno = {}, len = {}) received", self, reject_digest.masterchain_seqno, reject_digest.messages.len() @@ -462,34 +508,48 @@ impl MessageQueue { for message_ids in reject_digest.messages.iter() { self.remp_manager.message_cache.add_external_message_status( &message_ids.id, &message_ids.uid, - None, + None, None, Self::forwarded_rejected_status().clone(), // Forwarded reject cannot replace any status: if we know something // to grant re-collation for the message, then this knowledge is // more important than some particular reject opinion of some other // validator. - |old,_new| { old.clone() }, + |old, _new| { old.clone() }, reject_digest.masterchain_seqno as u32 - ).await?; + )?; } } Ok(()) } - async fn process_pending_remp_catchain_record(&self, remp_catchain_record: &RempCatchainRecord) -> Result<()> { + async fn process_pending_remp_catchain_record(&self, remp_catchain_record: &RempCatchainRecordV2, relayer: u32) -> Result<()> { match remp_catchain_record { - RempCatchainRecord::TonNode_RempCatchainMessage(msg) => - self.process_pending_remp_catchain_message(msg).await, - RempCatchainRecord::TonNode_RempCatchainMessageDigest(digest) => + RempCatchainRecordV2::TonNode_RempCatchainMessageHeaderV2(msg) => + self.process_pending_remp_catchain_message(msg, relayer).await, + RempCatchainRecordV2::TonNode_RempCatchainMessageDigestV2(digest) => self.process_pending_remp_catchain_digest(digest).await } } + /// Check pending queues, activate catchain exchange + fn poll_outbound_queues(&self) -> Result<()> { + self.catchain_instance.poll_outbound_queues( + RMQ_MAXIMAL_BROADCASTS_IN_PACK, + RMQ_MAXIMAL_QUERIES_IN_PACK, + RMQ_MESSAGE_QUERY_TIMEOUT, + MAX_EXTERNAL_MESSAGE_SIZE as u64 + 1000 + ) + } + + fn activate_exchange(&self) -> Result<()> { + self.catchain_instance.activate_exchange(RMQ_REQUEST_NEW_BLOCK_INTERVAL) + } + /// Check received messages queue and put all received messages into /// hash map. Check status of all old messages in the hash map. pub async fn poll(&self) -> Result<()> { - log::debug!(target: "remp", "Point 4. RMQ {}: polling; total {} messages in cache, {} messages in rmq_queue", - self, + log::debug!(target: "remp", "RMQ {}: polling; total {} messages in cache, {} messages in rmq_queue", + self, self.received_messages_count().await, self.catchain_instance.rmq_catchain_receiver_len()? ); @@ -511,19 +571,216 @@ impl MessageQueue { log::trace!(target: "remp", "RMQ {}: No more messages in rmq_queue", self); break 'queue_loop; }, - Ok(Some(record)) => self.process_pending_remp_catchain_record(&record).await? + Ok(Some((record, relayer))) => self.process_pending_remp_catchain_record(&record, relayer).await? } } + // Process responses for messages with not received bodies + 'responses_loop: loop { + match self.catchain_instance.query_response_receiver_try_recv() { + Err(e) => { + log::error!(target: "remp", "RMQ {}: error receiving from query_response_receiver queue: `{}`", self, e); + break 'responses_loop; + }, + Ok(None) => { + log::trace!(target: "remp", "RMQ {}: no more query responses", self); + break 'responses_loop; + }, + Ok(Some(message_body)) => { + let rmq_message = Arc::new(RmqMessage::from_message_body(&message_body)?); + self.remp_manager.message_cache.update_message_body(rmq_message.clone())?; + } + } + } + + self.poll_outbound_queues()?; + self.activate_exchange() + } +} + +impl MessageQueue { + /// Check message whether it is good for collation: + /// * Bool result --- should we reinsert it again into collation queue or not + /// * + async fn check_one_message_for_collation (&self, msg_id: &UInt256) -> Result<(bool, Option<(Arc, Arc)>)> { + let (message, origin, status) = match self.remp_manager.message_cache.get_message_with_origin_status_cc(msg_id)? { + Some((None, h, o, s, _cc)) => { + // -1 for absent index seems to be the easiest way to get around borrow/scope checker + let dst_idx = self.queues.execute_sync(|x| + match x.get_body_sources(&msg_id) { + None => -1, + Some(v) if v.len() == 0 => -1, + Some(v) => v.get(gen_random_index(v.len() as u16) as usize).map(|nn| *nn as i32).unwrap_or(-1) + } + ).await; + + if dst_idx >= 0 { + log::trace!( + target: "remp", + "Point 5. RMQ {}: message {:x} with status {:?} from {} found in pending_collation queue, but has no body yet; requesting body again from node {}", + self, msg_id, s, o, dst_idx + ); + + let dst_adnl_id = self.catchain_instance.get_adnl_id(dst_idx as usize).ok_or_else( + || error!("RMQ {}: cannot find adnl id for idx {}", self, dst_idx) + )?; + self.catchain_instance.pending_messages_queries_send( + dst_adnl_id, + h.as_remp_message_query() + )?; + } + else { + log::error!(target: "remp", "Point 5. RMQ {}: message {:x} with status {:?} from {} has no body and no body source; skipping", + self, msg_id, s, o + ) + } + return Ok((true, None)) + }, + None => { + log::warn!( + target: "remp", + "Point 5. RMQ {}: message {:x} found in pending_collation queue, but has no origin; will not be collated", + self, msg_id + ); + return Ok((true, None)) + }, + Some((Some(m), _h, o, s, _cc)) => (m, o, s) + }; + + match status.clone() { + RempMessageStatus::TonNode_RempNew + | RempMessageStatus::TonNode_RempIgnored(_) => (), + _ => { + log::trace!(target: "remp", "Point 5. RMQ {}: Skipping message {:x}, status does not allow it to collate: {}", + self, msg_id, status + ); + return Ok((false, None)) + } + }; + + match self.remp_manager.message_cache.check_message_duplicates(&message.message_id)? { + RempDuplicateStatus::Absent => fail!("Message {:x} is present in cache, but check_message_duplicates = Absent", &message.message_id), + RempDuplicateStatus::Fresh(_) => + log::trace!(target: "remp", "Point 5. RMQ {}: sending message {:x} to collator queue", self, message.message_id), + + d @ RempDuplicateStatus::Duplicate(_, _, _) => { + let duplicate_info = self.remp_manager.message_cache.duplicate_info(&d); + if !Self::is_final_status(&status) { + log::trace!(target: "remp", "Point 5. RMQ {}: rejecting message {:x}: '{}'", + self, message.message_id, duplicate_info + ); + let rejected = RempRejected { + level: RempMessageLevel::TonNode_RempQueue, + block_id: BlockIdExt::default(), + error: duplicate_info + }; + self.update_status_send_response(msg_id, origin.clone(), RempMessageStatus::TonNode_RempRejected(rejected)); + } + else { + log::error!(target: "remp", "Point 5. RMQ {}: message {:x} must not have a final status {:?}", + self, message.message_id, status + ) + } + return Ok((false, None)) + } + } + + Ok((false, Some((message.message.clone(), origin)))) + } + + async fn put_back_to_collation_queue(&self, msgid: &UInt256, timestamp: SystemTime) -> Result<()> { + let (added, _) = self.queues.execute_sync(|x| + x.add_to_collation_queue(&msgid, timestamp, None, false) + ).await?; + + if !added { + fail!("Message {:x} is already waiting for collation while postponing it", msgid); + } + Ok(()) } + pub async fn get_one_message_for_collation(&self, message_deadline: SystemTime) -> Result, Arc)>> { + while let Some((msgid, timestamp)) = self.queues.execute_sync(|x| x.take_first_for_collation()).await? { + if message_deadline <= timestamp { + self.put_back_to_collation_queue(&msgid, timestamp).await?; + return Ok(None) + } + + let (message, origin) = match self.check_one_message_for_collation(&msgid).await { + Err(e) => { + log::error!( + target: "remp", + "Point 5. RMQ {}: message {:x} found in pending_collation queue, error retriving it from messages/message_statuses: {}", + self, msgid, e + ); + continue + }, + Ok((leave_in_queue, None)) => { + if leave_in_queue { + self.put_back_to_collation_queue(&msgid, SystemTime::now()).await?; + } + continue + }, + Ok((_, Some(x))) => x + }; + + let new_status = RempMessageStatus::TonNode_RempAccepted (RempAccepted { + level: RempMessageLevel::TonNode_RempQueue, + block_id: BlockIdExt::default(), + master_id: BlockIdExt::default() + }); + self.update_status_send_response(&msgid, origin.clone(), new_status); + + return Ok(Some((msgid,message,origin))); + } + return Ok(None) + } + + pub async fn prepare_messages_for_collation (&self) -> Result, Arc)>> { + let message_deadline = SystemTime::now(); + let mut prepared_for_collation = Vec::new(); + + while let Some(msg_info) = self.get_one_message_for_collation(message_deadline).await? { + prepared_for_collation.push(msg_info); + } + + Ok(prepared_for_collation) + } +/* /// Prepare messages for collation - to be called just before collator invocation. pub async fn collect_messages_for_collation (&self) -> Result<()> { log::trace!(target: "remp", "RMQ {}: collecting messages for collation", self); let mut cnt = 0; - while let Some((msgid, _timestamp)) = self.queues.execute_sync(|x| x.take_first_for_collation()).await? { - let (status, message) = match self.remp_manager.message_cache.get_message_with_status(&msgid) { + let message_deadline = SystemTime::now(); + + while let Some((msg_id,message,_origin)) = self.get_one_message_for_collation(message_deadline).await? { + //self.queues.execute_sync(|x| x.take_first_for_collation()).await? { + + let (message, origin) = match self.check_one_message_for_collation(&msgid).await { + Err(e) => { + log::error!( + target: "remp", + "Point 5. RMQ {}: message {:x} found in pending_collation queue, error retriving it from messages/message_statuses: {}", + self, msgid, e + ); + continue + }, + Ok((leave_in_queue, None)) => { + if leave_in_queue { + let (added, _) = self.queues.execute_sync(|x| + x.add_to_collation_queue(&msgid, UnixTime32::now().as_u32(), None, false) + ).await?; + if added { + fail!("Message {:x} is already waiting for collation while postponing it", msgid); + } + } + continue + }, + Ok((_, Some(x))) => x + }; + + let (message, origin, status) = match self.remp_manager.message_cache.get_message_with_origin_status_cc(&msgid) { Err(e) => { log::error!( target: "remp", @@ -532,15 +789,47 @@ impl MessageQueue { ); continue } + Ok(Some((None, h, o, s, _cc))) => { + // -1 for absent index seems to be the easiest way to get around borrow/scope checker + let dst_idx = self.queues.execute_sync(|x| + match x.get_body_sources(&msgid) { + None => -1, + Some(v) if v.len() == 0 => -1, + Some(v) => v.get(gen_random_index(v.len() as u16) as usize).map(|nn| *nn as i32).unwrap_or(-1) + } + ).await; + + if dst_idx >= 0 { + log::trace!( + target: "remp", + "Point 5. RMQ {}: message {:x} with status {:?} from {} found in pending_collation queue, but has no body yet; requesting body again from node {}", + self, msgid, s, o, dst_idx + ); + + let dst_adnl_id = self.catchain_instance.get_adnl_id(dst_idx as usize).ok_or_else( + || error!("RMQ {}: cannot find adnl id for idx {}", self, dst_idx) + )?; + self.catchain_instance.pending_messages_queries_send( + dst_adnl_id, + h.as_remp_message_query() + )?; + } + else { + log::error!(target: "remp", "Point 5. RMQ {}: message {:x} with status {:?} from {} has no body and no body source; skipping", + self, msgid, s, o + ) + } + continue + }, Ok(None) => { - log::error!( + log::warn!( target: "remp", - "Point 5. RMQ {}: message {:x} found in pending_collation queue, but not in messages/message_statuses", + "Point 5. RMQ {}: message {:x} found in pending_collation queue, but has no origin; will not be collated", self, msgid ); continue }, - Ok(Some((m, s))) => (s, m) + Ok(Some((Some(m), _h, o, s, _cc))) => (m, o, s) }; match status.clone() { @@ -569,7 +858,7 @@ impl MessageQueue { block_id: BlockIdExt::default(), error: duplicate_info }; - self.update_status_send_response(&msgid, message.clone(), RempMessageStatus::TonNode_RempRejected(rejected)); + self.update_status_send_response(&msgid, origin.clone(), RempMessageStatus::TonNode_RempRejected(rejected)); } else { log::error!(target: "remp", "Point 5. RMQ {}: message {:x} must not have a final status {:?}", @@ -580,32 +869,21 @@ impl MessageQueue { } } - match self.send_message_to_collator(msgid.clone(), message.message.clone()).await { + match self.send_message_to_collator(msg_id.clone(), message.clone()).await { Err(e) => { let error = format!("{}", e); log::error!(target: "remp", - "Point 5. RMQ {}: error sending message {:x} to collator: `{}`; message status is unknown till collation end", - self, msgid, &error + "Point 5. RMQ {}: error sending message {:x} to collator: `{}`", + self, msg_id, &error ); - - // No new status: failure inside collator does not say - // anything about the message. Let's wait till collation end. }, - Ok(()) => { - let new_status = RempMessageStatus::TonNode_RempAccepted (RempAccepted { - level: RempMessageLevel::TonNode_RempQueue, - block_id: BlockIdExt::default(), - master_id: BlockIdExt::default() - }); - self.update_status_send_response(&msgid, message.clone(), new_status); - cnt = cnt + 1; - } + Ok(()) => cnt = cnt + 1 } } log::trace!(target: "remp", "Point 5. RMQ {}: total {} messages for collation", self, cnt); Ok(()) } - +*/ pub async fn all_accepted_by_collator_to_ignored(&self) -> Result> { let mut downgrading = Vec::new(); @@ -639,9 +917,9 @@ impl MessageQueue { /// Returns message to collation queue of the current collator. /// Message must alreaedy present in the queue. pub async fn return_to_collation_queue(&self, message_id: &UInt256) -> Result<()> { - if let Some(message) = self.remp_manager.message_cache.get_message(message_id)? { + if let Some(origin) = self.remp_manager.message_cache.get_message_origin(message_id)? { self.queues.execute_sync( - |catchain| catchain.add_to_collation_queue(message_id, message.timestamp, false) + |catchain| catchain.add_to_collation_queue(message_id, origin.system_time()?, None, false) ).await?; Ok(()) } @@ -726,10 +1004,6 @@ impl MessageQueue { ); } - pub fn get_message(&self, id: &UInt256) -> Result>> { - self.remp_manager.message_cache.get_message(id) - } - async fn get_queue_len(&self) -> usize { self.queues.execute_sync(|q| q.pending_collation_set.len()).await } @@ -764,12 +1038,6 @@ impl MessageQueue { self.queues.execute_sync(|mq| mq.pending_collation_set.len()).await } - async fn send_message_to_collator(&self, message_id: UInt256, message: Arc) -> Result<()> { - self.remp_manager.collator_receipt_dispatcher.queue.send_message_to_collator( - message_id, message - ).await - } - //pub fn pack_payload } @@ -800,14 +1068,15 @@ struct StatusUpdater { #[async_trait::async_trait] impl BlockProcessor for StatusUpdater { async fn process_message(&self, message_id: &UInt256, _message_uid: &UInt256) { - match self.queue.get_message(message_id) { - Err(e) => log::error!(target: "remp", "Cannot get message {:x} from cache: {}", message_id, e), - Ok(None) => log::warn!(target: "remp", "Cannot find message {:x} in cache", message_id), - Ok(Some(message)) => { - log::trace!(target: "remp", "Point 7. RMQ {} shard accepted message {}, new status {}", - self.queue, message, self.new_status + match self.queue.remp_manager.message_cache.get_message_with_origin_status_cc(message_id) { + Err(e) => log::error!(target: "remp", "Point 7. Cannot get message {:x} from cache: {}", message_id, e), + Ok(None) => log::warn!(target: "remp", "Point 7. Message {:x} is not stored in cache (origin is missing)", message_id), + Ok(Some((None, _, _, _, _))) => log::warn!(target: "remp", "Point 7. Message {:x} is not stored in cache (body is missing)", message_id), + Ok(Some((Some(message),_header,origin,_,_))) => { + log::trace!(target: "remp", "Point 7. RMQ {} shard accepted message {}, {}, new status {}", + self.queue, message, origin, self.new_status ); - self.queue.update_status_send_response(message_id, message, self.new_status.clone()) + self.queue.update_status_send_response(message_id, origin, self.new_status.clone()) } } } @@ -954,10 +1223,10 @@ impl RmqQueueManager { } //let mut rejected_message_digests: HashMap = HashMap::default(); - let mut rejected_message_digests: Vec = Vec::new(); + let mut rejected_message_digests: Vec = Vec::new(); for msgid in to_forward.iter() { - let (message, message_status, message_cc) = match self.remp_manager.message_cache.get_message_with_status_cc(msgid) { + let (message, origin, message_status, message_cc) = match self.remp_manager.message_cache.get_message_with_origin_status_cc(msgid) { Err(e) => { log::error!( target: "remp", @@ -969,13 +1238,13 @@ impl RmqQueueManager { Ok(None) => { log::error!( target: "remp", - "Point 5a. RMQ {}: message {:x} found in pending_collation queue, but not in messages/message_statuses", + "Point 5a. RMQ {}: message {:x} found in pending_collation queue, but not in messages/message_statuses, not forwarding", self, msgid ); continue }, - Ok(Some((_m, status, _cc))) if status == RempMessageStatus::TonNode_RempTimeout => continue, - Ok(Some((_m, _status, cc))) if !new_cc_range.contains(&cc) => { + Ok(Some((_m, _header, _origin, status, _cc))) if status == RempMessageStatus::TonNode_RempTimeout => continue, + Ok(Some((_m, _header, _origin, _status, cc))) if !new_cc_range.contains(&cc) => { if *new_cc_range.end() < cc { log::error!(target: "remp", "Point 5a. RMQ {}: message {:x} is younger (cc={}) than next_cc_range end {}..={} -- impossible", self, msgid, cc, new_cc_range.start(), new_cc_range.end() @@ -983,33 +1252,24 @@ impl RmqQueueManager { } continue }, - Ok(Some((m, status, cc))) => (m, status, cc) + Ok(Some((None, _header, _origin, _status, _cc))) => { + log::error!( + target: "remp", + "Point 5a. RMQ {}: message {:x} found in pending_collation queue, but has no message body, not forwarding", + self, msgid + ); + continue + } + Ok(Some((Some(m), _header, origin, status, cc))) => (m, origin, status, cc) }; // Forwarding: - // 1. Rejected messages -- as plain id (message id, message uid) - // 2. Non-final messages -- (full RmqRecord) + // 1. Rejected messages -- MessageDigestV2 (id, uid) + // 2. Non-final messages -- Message (id, uid, origin) + MessageBody as broadcast // All other messages are not forwarded if is_finally_rejected(&message_status) { -/* - if let Some(digest) = rejected_message_digests.get_mut (&message_cc) { - digest.messages.0.push(ton_api::ton::ton_node::rempcatchainmessageids::RempCatchainMessageIds { - id: message.message_id.clone(), - uid: message.message_uid.clone() - }); - } - else { - let mut digest = ton_api::ton::ton_node::rempcatchainrecord::RempCatchainMessageDigest::default(); - digest.masterchain_seqno = message_cc as i32; - digest.messages.0.push(ton_api::ton::ton_node::rempcatchainmessageids::RempCatchainMessageIds { - id: message.message_id.clone(), - uid: message.message_uid.clone() - }); - rejected_message_digests.insert(message_cc, digest); - } -*/ - let mut digest = RempCatchainMessageDigest::default(); + let mut digest = RempCatchainMessageDigestV2::default(); digest.masterchain_seqno = message_cc as i32; digest.messages.0.push(ton_api::ton::ton_node::rempcatchainmessageids::RempCatchainMessageIds { id: message.message_id.clone(), @@ -1026,7 +1286,11 @@ impl RmqQueueManager { } for new in next_queues.iter() { - if let Err(x) = new.catchain_instance.pending_messages_queue_send(message.as_rmq_record(message_cc)) { + if let Err(x) = new.catchain_instance.pending_messages_broadcast_send( + origin.as_remp_catchain_record(&message.message_id, &message.message_uid, message_cc), + Some(message.as_remp_message_body()) + ) + { log::error!(target: "remp", "Point 5a. RMQ {}: message {:x} cannot be put to new queue {}: `{}`", self, msgid, new, x @@ -1039,23 +1303,8 @@ impl RmqQueueManager { } for digest in rejected_message_digests.into_iter() { -/* - if !digest.messages.0.is_empty() { - let digest_len = digest.messages.0.len(); - let msg = ton_api::ton::ton_node::RempCatchainRecord::TonNode_RempCatchainMessageDigest(digest); - for new in next_queues.iter() { - if let Err(x) = new.catchain_instance.pending_messages_queue_send(msg.clone()) { - log::error!(target: "remp", - "Point 5a. RMQ {}: message digest (len={}) cannot be put to new queue {}: `{}`", - self, digest_len, new, x - ) - } else { - sent = sent + 1; - } - } - */ let digest_len = digest.messages.0.len(); - let msg = ton_api::ton::ton_node::RempCatchainRecord::TonNode_RempCatchainMessageDigest(digest); + let msg = ton_api::ton::ton_node::RempCatchainRecordV2::TonNode_RempCatchainMessageDigestV2(digest); for new in next_queues.iter() { if let Err(x) = new.catchain_instance.pending_messages_queue_send(msg.clone()) { log::error!(target: "remp", @@ -1067,11 +1316,6 @@ impl RmqQueueManager { sent_rejects = sent_rejects + 1; } } - /* - else { - log::error!(target: "remp", "RMQ {}: rejected message digest for {} is empty, but present in cache!", self, master_cc) - } - */ } log::info!(target: "remp", "RMQ {}: forwarding messages to new RMQ, total {}, actually sent {} (with {} rejects of them)", @@ -1107,7 +1351,7 @@ impl RmqQueueManager { log::trace!(target: "remp", "Stopping RMQ {} finished, next queues [{}] will be stopped by GC", self, next_queues); } - pub async fn put_message_to_rmq(&self, message: Arc) -> Result<()> { + pub async fn put_message_to_rmq(&self, message: Arc) -> Result<()> { if let Some(cur_queue) = &self.cur_queue { cur_queue.clone().put_message_to_rmq(message).await } @@ -1115,7 +1359,7 @@ impl RmqQueueManager { fail!("Cannot put message to RMQ {}: queue is not started yet", self) } } - +/* pub async fn collect_messages_for_collation (&self) -> Result<()> { if let Some(queue) = &self.cur_queue { queue.collect_messages_for_collation().await?; @@ -1125,6 +1369,16 @@ impl RmqQueueManager { fail!("Collecting messages for collation: RMQ {} is not started", self) } } +*/ + pub async fn prepare_messages_for_collation (&self) -> Result, Arc)>> { + if let Some(queue) = &self.cur_queue { + let messages = queue.prepare_messages_for_collation().await?; + Ok(messages) + } + else { + fail!("Preparing messages for collation: RMQ {} is not started", self) + } + } pub async fn process_collation_result (&self) -> Result<()> { if let Some(queue) = &self.cur_queue { @@ -1177,7 +1431,7 @@ impl RmqQueueManager { } pub async fn poll(&self) { - log::trace!(target: "remp", "Point 2. RMQ {} manager: polling incoming messages", self); + log::trace!(target: "remp", "RMQ {} manager: polling incoming messages", self); if let Some(cur_queue) = &self.cur_queue { if !cur_queue.is_session_active() { log::warn!(target: "remp", "RMQ {} is not active yet, waiting...", self); @@ -1190,17 +1444,17 @@ impl RmqQueueManager { let mut cnt_rejected_overload = 0; 'a: loop { match self.remp_manager.poll_incoming(&self.shard).await { - (Some(rmq_message), _) => { + (Some(msg), _) => { if let Some((overload_message, status)) = cur_queue.is_queue_overloaded().await { - log::warn!(target: "remp", "Point 3. RMQ {}: {}, ignoring incoming message {}", self, overload_message, rmq_message); - cur_queue.send_response_to_fullnode(rmq_message, status); + log::warn!(target: "remp", "Point 3. RMQ {}: {}, ignoring incoming message {}", self, overload_message, msg); + cur_queue.send_response_to_fullnode(&msg.get_message_id(), Arc::new(msg.origin.clone()), status); cnt_rejected_overload+=1; } - else if let Err(e) = self.put_message_to_rmq(rmq_message.clone()).await { + else if let Err(e) = self.put_message_to_rmq(msg.clone()).await { log::warn!(target: "remp", "Point 3. Error sending RMQ {} message {:?}: {}; returning back to incoming queue", - self, rmq_message, e + self, msg, e ); - self.remp_manager.return_to_incoming(rmq_message, &self.shard).await; + self.remp_manager.return_to_incoming(msg, &self.shard).await; break 'a; } else { @@ -1287,6 +1541,144 @@ impl fmt::Display for RmqQueueManager { } } +pub struct RempQueueCollatorInterfaceImpl { + queue: Arc, + messages_to_process: lockfree::queue::Queue<(UInt256, Arc, Arc)>, + messages_count: AtomicUsize +} + +impl RempQueueCollatorInterfaceImpl { + pub fn new(remp_manager: Arc) -> Self { + Self { + queue: remp_manager, + messages_to_process: lockfree::queue::Queue::new(), + messages_count: AtomicUsize::new(0) + } + } + + fn compute_ordering_hash(msg_id: &UInt256, prev_blocks_ids: &[&BlockIdExt]) -> UInt256 { + let mut hasher = Sha256::new(); + for prev_id in prev_blocks_ids { + hasher.update(prev_id.root_hash().as_slice()); + } + hasher.update(msg_id.as_slice()); + UInt256::from(hasher.finalize().as_slice()) + } + + /// All messages that were not requested by collator, are returned back to queue by this function. + /// The function returns (total messages, messages returned back) + pub async fn return_prepared_messages_to_queue(&self) -> Result<(usize, usize)> { + let mut cnt = 0; + let cur_queue = match &self.queue.cur_queue { + Some(q) => q, + None => fail!("No current queue {}: cannot return prepared messages to non-initalized/removed queue", self) + }; + for (msg_id,_,_) in self.messages_to_process.pop_iter() { + cur_queue.return_to_collation_queue(&msg_id).await?; + cnt += 1; + } + Ok((self.messages_count.load(Ordering::Relaxed), cnt)) + } + + pub fn into_interface(self: Arc) -> Arc { + let interface: Arc = self.clone(); + interface + } +} + +impl fmt::Display for RempQueueCollatorInterfaceImpl { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.queue) + } +} + +#[async_trait::async_trait] +impl RempQueueCollatorInterface for RempQueueCollatorInterfaceImpl { + /// Returns true if the messages were prepared, + /// false if the messages could not be prepared (master block is not processed yet). + async fn init_queue( + &self, + master_block_id: &BlockIdExt, + prev_blocks_ids: &[&BlockIdExt] + ) -> Result<()> { + if !self.queue.remp_manager.message_cache.is_block_processed(master_block_id)? { + log::warn!(target: "remp", "Block {} is not ready yet, cannot collect messages relative to it", master_block_id); + return Ok(()); + } + + let prepared_messages = self.queue.prepare_messages_for_collation().await?; + let mut ordered_messages: Vec<(UInt256, UInt256, Arc, Arc)> = prepared_messages.into_iter().map( + |(id,msg,origin)| (Self::compute_ordering_hash(&id, prev_blocks_ids), id, msg, origin) + ).collect(); + ordered_messages.sort_by(|(ordering_id1,_,_,_), (ordering_id2,_,_,_)| ordering_id2.cmp(ordering_id1)); + let cnt = ordered_messages.len(); +/* + log::trace!(target: "remp", "DebugMessageCollationStart: {}", + prev_blocks_ids.iter().map(|x| format!(" {} ", x)).collect::() + ); + for (order, id, msg, origin) in ordered_messages.into_iter() { + let msg_serialized: ton_api::ton::bytes = msg.write_to_bytes().unwrap().into(); + log::trace!(target: "remp", "DebugMessageCollation: {:x} {:x} {:?}", order, id, msg_serialized); + self.messages_to_process.push((id, msg, origin)); + } + log::trace!(target: "remp", "DebugMessageCollationFinish"); +*/ + for (_order, id, _msg, origin) in ordered_messages.into_iter() { + if let Some(x) = &self.queue.cur_queue { + let reject = RempRejected { + level: RempMessageLevel::TonNode_RempCollator, + block_id: BlockIdExt::default(), + error: "Test message rejected".as_string() + }; + x.update_status_send_response(&id, origin, RempMessageStatus::TonNode_RempRejected(reject)) + } + } + + log::info!(target: "remp", "Point 5. RMQ {}: total {} messages for collation", self, cnt); + self.messages_count.store(cnt, Ordering::Relaxed); + + Ok(()) + } + + async fn get_next_message_for_collation(&self) -> Result, UInt256)>> { + let cur_queue = match &self.queue.cur_queue { + Some(q) => q, + None => fail!("No current queue {}: cannot get next message for collation", self) + }; + + Ok(self.messages_to_process.pop().map(|(id,msg,origin)| { + let new_status = RempMessageStatus::TonNode_RempAccepted (RempAccepted { + level: RempMessageLevel::TonNode_RempQueue, + block_id: BlockIdExt::default(), + master_id: BlockIdExt::default() + }); + cur_queue.update_status_send_response(&id, origin.clone(), new_status); + (msg,id) + })) + /*let reason = if self.queue.remp_manager.message_cache.is_block_processed(master_block_id)? { + if let Some(q) = &self.queue.cur_queue { + return Ok(q.get_one_message_for_collation(self.message_deadline, generation_deadline) + .await? + .map(|(_id,m,_o)| m) + ); + } + else { + "no current queue is active" + } + } + else { + "block is not processed yet" + }; + + log::warn!(target: "remp", + "RMQ {}: collator request relative to master block {} cannot be processed: {}", + self, master_block_id, reason + ); + + return Ok(None)*/ + } +} + #[cfg(test)] #[path = "tests/test_rmq_messages.rs"] mod tests; diff --git a/src/validator/remp_block_parser.rs b/src/validator/remp_block_parser.rs index e0a79f3b..e5594b64 100644 --- a/src/validator/remp_block_parser.rs +++ b/src/validator/remp_block_parser.rs @@ -89,9 +89,9 @@ impl BlockProcessor for RempMasterBlockIndexingProcessor { }; if let Err(e) = self.message_cache.add_external_message_status( - message_id, message_uid, None, RempMessageStatus::TonNode_RempAccepted(accepted), + message_id, message_uid, None, None, RempMessageStatus::TonNode_RempAccepted(accepted), |_o,n| n.clone(), self.masterchain_seqno - ).await { + ) { log::warn!(target: "remp", "Update message {:x}, uid {:x} status failed: {}", message_id, message_uid, e); } } diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index 40f81c91..40a0e248 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -18,7 +18,7 @@ use std::ops::RangeInclusive; use crate::{ engine_traits::EngineOperations, validator::{ - catchain_overlay::CatchainOverlayManagerImpl, message_cache::RmqMessage, + catchain_overlay::CatchainOverlayManagerImpl, message_cache::{RmqMessage, RempMessageHeader}, sessions_computing::GeneralSessionInfo, mutex_wrapper::MutexWrapper, remp_manager::RempManager, validator_utils::{ @@ -35,18 +35,18 @@ use catchain::{ PublicKey, PublicKeyHash }; use ton_api::{ - IntoBoxed, ton::ton_node::RempCatchainRecord + IntoBoxed, ton::ton_node::{RempCatchainRecordV2, RempMessageQuery} }; use ever_block::ValidatorDescr; use ever_block::{error, fail, KeyId, Result, UInt256}; const REMP_CATCHAIN_START_POLLING_INTERVAL: Duration = Duration::from_millis(50); -fn get_remp_catchain_record_info(r: &RempCatchainRecord) -> String { +fn get_remp_catchain_record_info(r: &RempCatchainRecordV2) -> String { match r { - RempCatchainRecord::TonNode_RempCatchainMessage(msg) => + RempCatchainRecordV2::TonNode_RempCatchainMessageHeaderV2(msg) => format!("msg_id: {:x}", msg.message_id), - RempCatchainRecord::TonNode_RempCatchainMessageDigest(msg) => + RempCatchainRecordV2::TonNode_RempCatchainMessageDigestV2(msg) => format!("digest, master_seqno={}, len={}", msg.masterchain_seqno, msg.messages.len()) } } @@ -54,25 +54,45 @@ fn get_remp_catchain_record_info(r: &RempCatchainRecord) -> String { pub struct RempCatchainInstanceImpl { pub catchain_ptr: CatchainPtr, - pending_messages_queue_receiver: crossbeam_channel::Receiver, - pub pending_messages_queue_sender: crossbeam_channel::Sender, + pending_messages_queue_receiver: crossbeam_channel::Receiver, + pub pending_messages_queue_sender: crossbeam_channel::Sender, - pub rmq_catchain_receiver: crossbeam_channel::Receiver, - rmq_catchain_sender: crossbeam_channel::Sender + pub rmq_catchain_receiver: crossbeam_channel::Receiver<(RempCatchainRecordV2,u32)>, + rmq_catchain_sender: crossbeam_channel::Sender<(RempCatchainRecordV2,u32)>, + + query_response_receiver: crossbeam_channel::Receiver, + query_response_sender: Arc>, + + pending_messages_queries_receiver: crossbeam_channel::Receiver<(Arc,RempMessageQuery)>, + pending_messages_queries_sender: crossbeam_channel::Sender<(Arc,RempMessageQuery)>, + + pub pending_messages_broadcast_receiver: crossbeam_channel::Receiver, + pending_messages_broadcast_sender: crossbeam_channel::Sender, } impl RempCatchainInstanceImpl { fn new(catchain_ptr: CatchainPtr) -> Self { - let (pending_messages_queue_sender, pending_messages_queue_receiver) = - crossbeam_channel::unbounded(); - let (rmq_catchain_sender, rmq_catchain_receiver) = - crossbeam_channel::unbounded(); + let (pending_messages_queue_sender, pending_messages_queue_receiver) = crossbeam_channel::unbounded(); + let (rmq_catchain_sender, rmq_catchain_receiver) = crossbeam_channel::unbounded(); + let (remp_messages_sender, remp_messages_receiver) = crossbeam_channel::unbounded(); + let (query_response_sender, query_response_receiver) = crossbeam_channel::unbounded(); + let (pending_messages_queries_sender, pending_messages_queries_receiver) = crossbeam_channel::unbounded(); Self { catchain_ptr, pending_messages_queue_sender, pending_messages_queue_receiver, - rmq_catchain_sender, rmq_catchain_receiver + rmq_catchain_sender, rmq_catchain_receiver, + pending_messages_broadcast_sender: remp_messages_sender, + pending_messages_broadcast_receiver: remp_messages_receiver, + query_response_sender: Arc::new(query_response_sender), + query_response_receiver, + pending_messages_queries_sender, + pending_messages_queries_receiver } } + + fn get_query_response_sender(&self) -> Arc> { + self.query_response_sender.clone() + } } pub struct RempCatchainInstance { @@ -106,7 +126,11 @@ impl RempCatchainInstance { self.instance_impl.load().map(|inst| inst.catchain_ptr.clone()) } - pub fn pending_messages_queue_send(&self, msg: RempCatchainRecord) -> Result<()> { + pub fn get_adnl_id(&self, node_idx: usize) -> Option> { + self.info.get_adnl_id(node_idx) + } + + pub fn pending_messages_queue_send(&self, msg: RempCatchainRecordV2) -> Result<()> { let instance = self.get_instance_impl()?; match instance.pending_messages_queue_sender.send(msg) { Ok(()) => Ok(()), @@ -114,13 +138,28 @@ impl RempCatchainInstance { } } + pub fn pending_messages_broadcast_send(&self, msg: RempCatchainRecordV2, msg_body: Option) -> Result<()> { + let instance = self.get_instance_impl()?; + if let Err(e) = instance.pending_messages_queue_sender.send(msg) { + fail!("pending_messages_queue_sender: send error {}", e) + } + + if let Some(body) = msg_body { + if let Err(e) = instance.pending_messages_broadcast_sender.send(body) { + fail!("pending_messages_broadcast_sender: send error {}", e) + } + } + + Ok(()) + } + #[cfg(feature = "telemetry")] pub fn pending_messages_queue_len(&self) -> Result { let instance = self.get_instance_impl()?; Ok(instance.pending_messages_queue_sender.len()) } - pub fn pending_messages_queue_try_recv(&self) -> Result> { + fn pending_messages_queue_try_recv(&self) -> Result> { let instance = self.get_instance_impl()?; match instance.pending_messages_queue_receiver.try_recv() { Ok(x) => Ok(Some(x)), @@ -129,23 +168,58 @@ impl RempCatchainInstance { } } + fn pending_messages_broadcast_try_recv(&self) -> Result> { + let instance = self.get_instance_impl()?; + match instance.pending_messages_broadcast_receiver.try_recv() { + Ok(x) => Ok(Some(x)), + Err(crossbeam_channel::TryRecvError::Empty) => Ok(None), + Err(crossbeam_channel::TryRecvError::Disconnected) => fail!("channel disconnected") + } + } + + pub fn pending_messages_queries_send(&self, dst: Arc, query: RempMessageQuery) -> Result<()> { + let instance = self.get_instance_impl()?; + match instance.pending_messages_queries_sender.send((dst, query)) { + Ok(()) => Ok(()), + Err(e) => fail!("pending_messages_queries_send error: {}", e) + } + } + + fn pending_messages_queries_try_recv(&self) -> Result,RempMessageQuery)>> { + let instance = self.get_instance_impl()?; + match instance.pending_messages_queries_receiver.try_recv() { + Ok(x) => Ok(Some(x)), + Err(crossbeam_channel::TryRecvError::Empty) => Ok(None), + Err(crossbeam_channel::TryRecvError::Disconnected) => fail!("channel disconnected") + } + } + + pub fn query_response_receiver_try_recv(&self) -> Result> { + let instance = self.get_instance_impl()?; + match instance.query_response_receiver.try_recv() { + Ok(x) => Ok(Some(x)), + Err(crossbeam_channel::TryRecvError::Empty) => Ok(None), + Err(crossbeam_channel::TryRecvError::Disconnected) => fail!("query response receiver channel disconnected") + } + } + pub fn rmq_catchain_receiver_len(&self) -> Result { let instance = self.get_instance_impl()?; Ok(instance.rmq_catchain_receiver.len()) } - pub fn rmq_catchain_try_recv(&self) -> Result> { + pub fn rmq_catchain_try_recv(&self) -> Result> { let instance = self.get_instance_impl()?; match instance.rmq_catchain_receiver.try_recv() { - Ok(x) => Ok(Some(x)), + Ok((x,relayer)) => Ok(Some((x,relayer))), Err(crossbeam_channel::TryRecvError::Empty) => Ok(None), Err(crossbeam_channel::TryRecvError::Disconnected) => fail!("channel disconnected") } } - pub fn rmq_catchain_send(&self, msg: RempCatchainRecord) -> Result<()> { + fn rmq_catchain_send(&self, msg: RempCatchainRecordV2, msg_remp_source_idx: u32) -> Result<()> { let instance = self.get_instance_impl()?; - match instance.rmq_catchain_sender.send(msg) { + match instance.rmq_catchain_sender.send((msg, msg_remp_source_idx)) { Ok(()) => Ok(()), Err(e) => fail!("rmq_cathcain_sender: send error {}", e) } @@ -154,6 +228,68 @@ impl RempCatchainInstance { pub fn get_id(&self) -> u128 { self.id.duration_since(UNIX_EPOCH).unwrap().as_micros() } + + fn poll_pending_broadcasts(&self, max_broadcasts: u32) -> Result<()> { + for _msg_count in 0..max_broadcasts { + match self.pending_messages_broadcast_try_recv()? { + None => break, + Some(msg) => { + let block_payload = catchain::CatchainFactory::create_block_payload( + serialize_tl_boxed_object!(&msg), + ); + self.get_instance_impl()?.catchain_ptr.send_broadcast(block_payload) + } + } + } + Ok(()) + } + + fn deserialize_and_put_query_response_to_queue(r: Result, query_response_sender: Arc>) -> Result<()> { + let response_record = RmqMessage::deserialize_message_body(r?.data())?; + match query_response_sender.send(response_record) { + Ok(()) => Ok(()), + Err(e) => fail!("query_response_sender: send error {}", e) + } + } + + fn poll_pending_queries(&self, max_queries: u32, query_timeout: SystemTime, max_answer_size: u64) -> Result<()> { + for _msg_count in 0..max_queries { + match self.pending_messages_queries_try_recv()? { + None => break, + Some((dst,query)) => { + let query_id = query.message_id().clone(); + let query_payload = CatchainFactory::create_block_payload( + RempMessageHeader::serialize_query(&query)? + ); + let query_response_sender = self.get_instance_impl()?.get_query_response_sender(); + self.get_instance_impl()?.catchain_ptr.send_query_via_rldp( + dst.clone(), + "message body request".to_string(), + Box::new (move |r: Result| { + if let Err(e) = Self::deserialize_and_put_query_response_to_queue(r, query_response_sender) { + log::error!(target: "remp", "Cannot deserelialize response to query for message {:x}, error: {}", query_id, e) + } + }), + query_timeout, + query_payload, + max_answer_size + ) + } + } + } + Ok(()) + } + + pub fn poll_outbound_queues(&self, max_broadcasts: u32, max_queries: u32, query_timeout: Duration, max_answer_size: u64) -> Result<()> { + self.poll_pending_broadcasts(max_broadcasts)?; + self.poll_pending_queries(max_queries, SystemTime::now() + query_timeout, max_answer_size) + } + + pub fn activate_exchange(&self, delay: Duration) -> Result<()> { + let session = &self.get_instance_impl()?.catchain_ptr; + session.request_new_block(SystemTime::now() + delay); + Ok(()) + } } impl fmt::Display for RempCatchainInstance { @@ -281,6 +417,10 @@ impl RempCatchainInfo { general_eq } + + pub fn get_adnl_id(&self, index: usize) -> Option> { + self.nodes.get(index).map(|n| n.adnl_id.clone()) + } } impl fmt::Display for RempCatchainInfo { @@ -385,21 +525,23 @@ impl RempCatchain { for msgbx in pld.actions.0.iter() { match msgbx { ::ton_api::ton::validator_session::round::Message::ValidatorSession_Message_Commit(msg) => { - match RmqMessage::deserialize(&msg.signature) { - Ok(unpacked_message) => { + match RempMessageHeader::deserialize(&msg.signature) { + Ok(record) => { #[cfg(feature = "telemetry")] { total += 1; } log::trace!(target: "remp", "Point 4. Message received from RMQ {}: {:?}, decoded {:?}, put to rmq_catchain queue", - self, msg.signature, unpacked_message //catchain.received_messages.len() + self, msg.signature, record //catchain.received_messages.len() ); - if let Err(e) = self.instance.rmq_catchain_send(unpacked_message.clone()) { + + if let Err(e) = self.instance.rmq_catchain_send(record.clone(), source_idx) { log::error!( target: "remp", "Point 4. Cannot put message {:?} from RMQ {} to queue: {}", - unpacked_message, self, e + record, self, e ) } + }, Err(e) => log::error!(target: "remp", "Cannot deserialize message from RMQ {} {:?}: {}", self, msg.signature, e @@ -420,6 +562,29 @@ impl RempCatchain { Err(e) => log::error!(target: "remp", "Cannot deserialize RMQ {} message: {}", self, e) } } + + fn unpack_broadcast(&self, payload: BlockPayloadPtr) -> Result { + let message = RmqMessage::deserialize_message_body(payload.data())?; + let rmqrecord = Arc::new(RmqMessage::from_message_body(&message)?); + self.remp_manager.message_cache.update_message_body(rmqrecord.clone())?; + + Ok(rmqrecord.message_id.clone()) + } + + fn process_query(&self, query_payload: BlockPayloadPtr) -> Result { + let query = RempMessageHeader::deserialize_query(query_payload.data())?; + let message_id = query.message_id(); + if let Some(message_body) = self.remp_manager.message_cache.get_message(message_id)? { + let remp_body_response = message_body.as_remp_message_body(); + let remp_body_response_raw = RmqMessage::serialize_message_body(&remp_body_response); + let response_payload = CatchainFactory::create_block_payload(remp_body_response_raw); + + Ok(response_payload) + } + else { + fail!("Message body {:x} is not present in message cache", message_id) + } + } } impl fmt::Display for RempCatchain { @@ -438,16 +603,17 @@ impl CatchainListener for RempCatchain { } fn process_blocks(&self, blocks: Vec) { - log::trace!(target: "remp", "Processing RMQ {}: new external messages, len = {}", self, blocks.len()); + log::trace!(target: "remp", "Processing RMQ {}: new external messages, len {}", self, blocks.len()); let mut msg_vect: Vec<::ton_api::ton::validator_session::round::Message> = Vec::new(); let mut msg_ids: Vec = Vec::new(); + let MAX_VECT_COUNT = 8; - if let Ok(Some(msg)) = self.instance.pending_messages_queue_try_recv() { + while let Ok(Some(msg)) = self.instance.pending_messages_queue_try_recv() { let msg_body = ::ton_api::ton::validator_session::round::validator_session::message::message::Commit { round: 0, candidate: Default::default(), - signature: RmqMessage::serialize(&msg).unwrap() + signature: RempMessageHeader::serialize(&msg).unwrap() }.into_boxed(); log::trace!(target: "remp", "Point 3. RMQ {} sending message: {:?}, decoded {:?}", self, msg_body, msg @@ -455,6 +621,8 @@ impl CatchainListener for RempCatchain { msg_vect.push(msg_body); msg_ids.push(get_remp_catchain_record_info(&msg)); + + if msg_vect.len() >= MAX_VECT_COUNT { break; } } let payload = ::ton_api::ton::validator_session::blockupdate::BlockUpdate { @@ -470,8 +638,8 @@ impl CatchainListener for RempCatchain { catchain::CatchainFactory::create_block_payload( serialized_payload.clone(), ), false, false); - log::trace!(target: "remp", "Point 3. RMQ {} sent messages: '{:?}'", - self, msg_ids + log::trace!(target: "remp", "Point 3. RMQ {} sent messages, cnt {}, len {}: '{:?}'", + self, msg_ids.len(), serialized_payload.len(), msg_ids ); #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().sent_to_catchain( @@ -493,22 +661,28 @@ impl CatchainListener for RempCatchain { log::trace!(target: "remp", "MessageQueue {} started", self) } - fn process_broadcast(&self, _source_id: PublicKeyHash, _data: BlockPayloadPtr) { - log::trace!(target: "remp", "MessageQueue {} process broadcast", self) + fn process_broadcast(&self, source_id: PublicKeyHash, data: BlockPayloadPtr) { + let len = data.data().len(); + log::trace!(target: "remp", "MessageQueue {} process broadcast from {}, len {}", self, source_id, len); + match self.unpack_broadcast(data) { + Err(e) => log::error!(target: "remp", "MessageQueue {}, error processing broadcast from {}, message body will be ignored: `{}`", self, source_id, e), + Ok(id) => log::trace!(target: "remp", "MessageQueue {} broadcast message: id {:x}, len {}", self, id, len) + } } - fn process_query( - &self, - source_id: PublicKeyHash, - data: BlockPayloadPtr, - _callback: ExternalQueryResponseCallback - ) { - let data = data.data(); - log::trace!( - target: "remp", - "Processing RMQ {} Query {:?} from {}", - self, data.as_slice(), source_id - ); + fn process_query(&self, source_id: PublicKeyHash, data: BlockPayloadPtr, callback: ExternalQueryResponseCallback) { + let raw_data = data.data(); + log::trace!(target: "remp", "Processing RMQ {} Query {:?} from {}", self, raw_data.as_slice(), source_id); + + match self.process_query(data) { + Err(e) => { + log::error!(target: "remp", "MessageQueue {}, error processing query from {}, message query will be ignored: `{}`", self, source_id, e); + callback(Err(error!("RMQ {}: error processing query from {}: {}", self, source_id, e))) + }, + Ok(body) => { + callback(Ok(body)) + } + } } fn set_time(&self, _timestamp: SystemTime) { @@ -682,12 +856,6 @@ impl RempCatchainStore { Ok(()) } - pub async fn get_catchain_session(&self, session_id: &UInt256) -> Option { - self.catchains.execute_sync(|x| { - x.get(session_id).map(|rcw| rcw.info.instance.get_session()).flatten() - }).await - } - pub async fn list_catchain_sessions(&self) -> Vec<(String, UInt256)> { let sessions_to_list = self.catchains.execute_sync(|x| { let mut sessions_to_list = Vec::new(); diff --git a/src/validator/remp_manager.rs b/src/validator/remp_manager.rs index a6221dab..2f308fd3 100644 --- a/src/validator/remp_manager.rs +++ b/src/validator/remp_manager.rs @@ -29,9 +29,9 @@ use crate::{ config::RempConfig, engine_traits::{EngineOperations, RempCoreInterface, RempDuplicateStatus}, validator::{ - message_cache::{RmqMessage, MessageCache}, mutex_wrapper::MutexWrapper, + message_cache::{RmqMessage, RempMessageOrigin, RempMessageWithOrigin, MessageCache}, mutex_wrapper::MutexWrapper, remp_catchain::RempCatchainStore, - validator_utils::{get_message_uid, get_shard_by_message} + validator_utils::get_shard_by_message } }; @@ -42,31 +42,32 @@ use std::time::SystemTime; use adnl::telemetry::Metric; use chrono::{DateTime, Utc}; use crossbeam_channel::TryRecvError; -use rand::Rng; +use rand::{Rng, RngCore}; +use rand::prelude::ThreadRng; pub struct RempInterfaceQueues { message_cache: Arc, runtime: Arc, pub engine: Arc, pub incoming_sender: - crossbeam_channel::Sender>, + crossbeam_channel::Sender>, pub response_receiver: - crossbeam_channel::Receiver<(UInt256, Arc, RempMessageStatus)> + crossbeam_channel::Receiver<(UInt256, UInt256, Arc, RempMessageStatus)> } pub struct RempDelayer { max_incoming_broadcast_delay_millis: u32, random_seed: u64, - pub incoming_receiver: crossbeam_channel::Receiver>, - pub delayed_incoming_sender: crossbeam_channel::Sender>, - delay_heap: MutexWrapper<(BinaryHeap<(Reverse, UInt256)>, HashMap>)>, + pub incoming_receiver: crossbeam_channel::Receiver>, + pub delayed_incoming_sender: crossbeam_channel::Sender>, + delay_heap: MutexWrapper<(BinaryHeap<(Reverse, UInt256)>, HashMap>)>, } impl RempDelayer { pub fn new (random_seed: u64, options: &RempConfig, - incoming_receiver: crossbeam_channel::Receiver>, - delayed_incoming_sender: crossbeam_channel::Sender>) -> Self { + incoming_receiver: crossbeam_channel::Receiver>, + delayed_incoming_sender: crossbeam_channel::Sender>) -> Self { Self { max_incoming_broadcast_delay_millis: options.get_max_incoming_broadcast_delay_millis(), random_seed, @@ -76,21 +77,22 @@ impl RempDelayer { } } - async fn push_delayer(&self, activation_time: SystemTime, msg: Arc) { + async fn push_delayer(&self, activation_time: SystemTime, msg: Arc) { let tm: DateTime = activation_time.into(); self.delay_heap.execute_sync(|(h,m)| { - if !m.contains_key(&msg.message_id) { + let message_id = msg.get_message_id(); + if !m.contains_key(message_id) { log::trace!(target: "remp", "Delaying REMP message {} till {}", msg, tm.format("%T")); - h.push((Reverse(activation_time), msg.message_id.clone())); - m.insert(msg.message_id.clone(), msg); + h.push((Reverse(activation_time), message_id.clone())); + m.insert(message_id.clone(), msg); } else { - log::trace!(target: "remp", "REMP message {} is already waiting", msg); + log::trace!(target: "remp", "Delayer: REMP message {} is already waiting", msg); } }).await } - async fn pop_delayer(&self) -> Option> { + async fn pop_delayer(&self) -> Option> { let now = SystemTime::now(); let res = self.delay_heap.execute_sync(|(h,m)| { match h.peek() { @@ -107,7 +109,7 @@ impl RempDelayer { if let Some((tm,msg)) = res { let tm: DateTime = tm.0.into(); - log::trace!(target: "remp", "Sending further REMP message {}, delayed till {}", msg, tm.format("%T")); + log::trace!(target: "remp", "Resuming REMP message {}, delayed till {}", msg, tm.format("%T")); Some(msg) } else { @@ -123,7 +125,7 @@ impl RempDelayer { max (h,m) } - async fn poll_incoming_once(&self) -> Result>> { + async fn poll_incoming_once(&self) -> Result>> { if let Some(msg) = self.pop_delayer().await { return Ok(Some(msg)) } @@ -132,7 +134,7 @@ impl RempDelayer { match self.incoming_receiver.try_recv() { Ok(msg) => { if msg.has_no_source_key() && self.max_incoming_broadcast_delay_millis > 0 { - let delay = (msg.message_id.first_u64() + self.random_seed) % (self.max_incoming_broadcast_delay_millis as u64); + let delay = (msg.get_message_id().first_u64() + self.random_seed) % (self.max_incoming_broadcast_delay_millis as u64); self.push_delayer(SystemTime::now() + Duration::from_millis(delay), msg).await; } else { @@ -321,21 +323,21 @@ impl> RempQueueDispatcher { pub struct RempIncomingQueue { engine: Arc, - pub incoming_receiver: crossbeam_channel::Receiver> + pub incoming_receiver: crossbeam_channel::Receiver> } impl RempIncomingQueue { pub fn new( engine: Arc, - incoming_receiver: crossbeam_channel::Receiver> + incoming_receiver: crossbeam_channel::Receiver> ) -> Self { RempIncomingQueue { engine, incoming_receiver } } } #[async_trait::async_trait] -impl RempQueue for RempIncomingQueue { - fn receive_message(&self) -> Result>> { +impl RempQueue for RempIncomingQueue { + fn receive_message(&self) -> Result>> { match self.incoming_receiver.try_recv() { Ok(x) => Ok(Some(x)), Err(crossbeam_channel::TryRecvError::Empty) => @@ -345,8 +347,8 @@ impl RempQueue for RempIncomingQueue { } } - async fn compute_shard(&self, msg: Arc) -> Result { - get_shard_by_message(self.engine.clone(), msg.message.clone()).await + async fn compute_shard(&self, msg: Arc) -> Result { + get_shard_by_message(self.engine.clone(), msg.message.message.clone()).await } } @@ -362,10 +364,6 @@ impl CollatorInterfaceWrapper { // message_dispatcher: Mutex::new(HashMap::new()) } } - - pub async fn send_message_to_collator(&self, msg_id: UInt256, msg: Arc) -> Result<()> { - self.engine.new_remp_message(msg_id.clone(), msg) - } } pub struct CollatorResult { @@ -439,9 +437,9 @@ pub struct RempManager { pub catchain_store: Arc, pub message_cache: Arc, incoming_delayer: RempDelayer, - incoming_dispatcher: RempQueueDispatcher, + incoming_dispatcher: RempQueueDispatcher, pub collator_receipt_dispatcher: RempQueueDispatcher, - pub response_sender: crossbeam_channel::Sender<(UInt256, Arc, RempMessageStatus)> + pub response_sender: crossbeam_channel::Sender<(UInt256 /* local_id */, UInt256 /* message_id */, Arc, RempMessageStatus)> } impl RempManager { @@ -503,7 +501,7 @@ impl RempManager { self.collator_receipt_dispatcher.remove_actual_shard(shard).await; } - pub async fn poll_incoming(&self, shard: &ShardIdent) -> (Option>, usize) { + pub async fn poll_incoming(&self, shard: &ShardIdent) -> (Option>, usize) { match self.incoming_delayer.poll_incoming().await { Ok(n) => { log::trace!(target: "remp", "Polling REMP incoming delayer queue: {} messages processed", n); @@ -515,12 +513,12 @@ impl RempManager { return self.incoming_dispatcher.poll(shard).await; } - pub async fn return_to_incoming(&self, message: Arc, shard: &ShardIdent) { + pub async fn return_to_incoming(&self, message: Arc, shard: &ShardIdent) { self.incoming_dispatcher.return_back(message, shard).await; } - pub fn queue_response_to_fullnode(&self, local_key_id: UInt256, rmq_message: Arc, status: RempMessageStatus) -> Result<()> { - self.response_sender.send((local_key_id, rmq_message, status))?; + pub fn queue_response_to_fullnode(&self, local_key_id: UInt256, message_id: UInt256, origin: Arc, status: RempMessageStatus) -> Result<()> { + self.response_sender.send((local_key_id, message_id, origin, status))?; Ok(()) } @@ -553,8 +551,14 @@ impl RempManager { #[allow(dead_code)] impl RempInterfaceQueues { - pub fn make_test_message(&self) -> Result { - RmqMessage::make_test_message(&SliceData::new_empty()) + pub fn make_test_message(&self, thread_rng: &mut ThreadRng) -> Result { + let mut data_array = [0 as u8; 128]; + thread_rng.fill_bytes(&mut data_array); + let data = SliceData::new(data_array.to_vec()); + Ok(RempMessageWithOrigin { + message: RmqMessage::make_test_message(&data)?, + origin: RempMessageOrigin::create_empty()? + }) } /** @@ -563,34 +567,35 @@ impl RempInterfaceQueues { * 1. random messages are generated each second, and sent to the REMP input queue * 2. responses after REMP processing are taken from REMP response queue and printed */ - pub async fn test_remp_messages_loop(&self) { + pub async fn test_remp_messages_loop(&self, pause: Duration, batch_size: usize) { log::info!(target: "remp", "Test REMP messages loop is started"); loop { - match self.make_test_message() { - Err(e) => log::error!(target: "remp", "Cannot make test REMP message: `{}`", e), - Ok(test_message) => { - if let Err(x) = self.incoming_sender.send(Arc::new(test_message)) { - log::error!(target: "remp", "Cannot send test REMP message to RMQ: {}", - x - ); - } - - while let Ok(msg) = self.response_receiver.try_recv() { - log::trace!(target: "remp", "Received test REMP response: {:?}", msg); + for _i in 0..batch_size { + let mut thread_rng = ThreadRng::default(); + match self.make_test_message(&mut thread_rng) { + Err(e) => log::error!(target: "remp", "Cannot make test REMP message: `{}`", e), + Ok(msg) => { + if let Err(x) = self.incoming_sender.send(Arc::new(msg)) { + log::error!(target: "remp", "Cannot send test REMP message to RMQ: {}", x); + } + + while let Ok(msg) = self.response_receiver.try_recv() { + log::trace!(target: "remp", "Received test REMP response: {:?}", msg); + } } } } - tokio::time::sleep(std::time::Duration::from_secs(1)).await; + tokio::time::sleep(pause).await; } } pub async fn send_response_to_fullnode( - &self, local_key_id: UInt256, rmq_message: Arc, status: RempMessageStatus + &self, local_key_id: UInt256, message_id: UInt256, origin: Arc, status: RempMessageStatus ) { let receipt = ton_api::ton::ton_node::RempReceipt::TonNode_RempReceipt ( ton_api::ton::ton_node::rempreceipt::RempReceipt { - message_id: rmq_message.message_id.clone().into(), + message_id: message_id.clone().into(), status: status.clone(), timestamp: 0, source_id: local_key_id.into() @@ -599,15 +604,15 @@ impl RempInterfaceQueues { let (engine,runtime) = (self.engine.clone(), self.runtime.clone()); runtime.clone().spawn(async move { - if let Err(e) = engine.send_remp_receipt(rmq_message.source_key.clone(), receipt).await { + if let Err(e) = engine.send_remp_receipt(origin.source_key.clone(), receipt).await { log::error!(target: "remp", "Cannot send {} response message {:x} to {}: {}", - status, rmq_message.message_id, rmq_message.source_key, e + status, message_id, origin.source_key, e ) } else { log::trace!(target: "remp", "Sending {} response for message {:x} to {}", - status, rmq_message.message_id, rmq_message.source_idx + status, message_id, origin.source_idx ) } }); @@ -616,8 +621,8 @@ impl RempInterfaceQueues { pub async fn poll_responses_loop(&self) { loop { match self.response_receiver.try_recv() { - Ok((local_key_id, msg,status)) => - self.send_response_to_fullnode(local_key_id, msg, status).await, + Ok((local_key_id, hdr, origin, status)) => + self.send_response_to_fullnode(local_key_id, hdr, origin, status).await, Err(crossbeam_channel::TryRecvError::Empty) => tokio::time::sleep(Duration::from_millis(1)).await, Err(crossbeam_channel::TryRecvError::Disconnected) => return @@ -628,27 +633,34 @@ impl RempInterfaceQueues { #[async_trait::async_trait] impl RempCoreInterface for RempInterfaceQueues { - async fn process_incoming_message(&self, message_id: UInt256, message: Message, source: Arc) -> Result<()> { - let arc_message = Arc::new(message.clone()); - + async fn process_incoming_message(&self, message: &ton_api::ton::ton_node::RempMessage, source: Arc) -> Result<()> { // build message - let remp_message = Arc::new(RmqMessage::new ( - arc_message, - message_id.clone(), - get_message_uid(&message), - source, - 0 - )?); - - if self.message_cache.get_message(&message_id)?.is_some() { + let remp_message = RmqMessage::from_raw_message(message.message())?; + if message.id() != &remp_message.message_id { + fail!("Message with computed id {:x} has different id {:x} in RempMessage struct, message will be ignored", + remp_message.message_id, message.id() + ); + } + if self.message_cache.is_message_fully_known(&remp_message.message_id)? { log::trace!(target: "remp", - "Point 1. We already know about message {:x}, no forwarding to incoming queue is necessary", - message_id + "Point 1. Message {:x} is fully known, no forwarding to incoming queue is necessary", + remp_message.message_id ); } else { - log::trace!(target: "remp", "Point 1. Adding incoming message {} to incoming queue", remp_message); - self.incoming_sender.send(remp_message)?; + let remp_message_origin = RempMessageOrigin::new ( + source, + 0 + )?; + + log::trace!(target: "remp", + "Point 1. Adding incoming message {} to incoming queue, known message info {}", + remp_message, self.message_cache.get_message_info(&remp_message.message_id)? + ); + self.incoming_sender.send(Arc::new(RempMessageWithOrigin { + message: remp_message, + origin: remp_message_origin + }))?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().in_channel_from_fullnode(self.incoming_sender.len()); } diff --git a/src/validator/remp_service.rs b/src/validator/remp_service.rs index 4efe4081..c13f0234 100644 --- a/src/validator/remp_service.rs +++ b/src/validator/remp_service.rs @@ -12,18 +12,17 @@ */ use crate::{ - engine_traits::{EngineOperations, RempCoreInterface}, ext_messages::create_ext_message, - network::remp::RempMessagesSubscriber, + engine_traits::{EngineOperations, RempCoreInterface, RempDuplicateStatus}, + network::remp::RempMessagesSubscriber }; -use std::{ops::Deref, sync::Arc}; -use ton_api::ton::ton_node::RempMessage; -use ever_block::{error, fail, KeyId, Result}; +use std::sync::{Arc, Weak}; +use ever_block::{error, fail, KeyId, Result, UInt256}; #[derive(Default)] pub struct RempService { - engine: tokio::sync::OnceCell>, - remp_core: tokio::sync::OnceCell>, + engine: tokio::sync::OnceCell>, + remp_core: tokio::sync::OnceCell>, } impl RempService { @@ -32,21 +31,30 @@ impl RempService { } pub fn set_engine(&self, engine: Arc) -> Result<()> { - self.engine.set(engine).map_err(|_| error!("Attempt to set engine twice")) + self.engine.set(Arc::downgrade(&engine)).map_err(|_| error!("Attempt to set engine twice")) } pub fn set_remp_core_interface(&self, remp_core: Arc) -> Result<()> { - self.remp_core.set(remp_core).map_err(|_| error!("Attempt to set remp_core twice")) + self.remp_core.set(Arc::downgrade(&remp_core)).map_err(|_| error!("Attempt to set remp_core twice")) } - pub fn remp_core_interface(&self) -> Result<&dyn RempCoreInterface> { - self.remp_core.get().ok_or_else(|| error!("remp_core was not set")).map(|rci| rci.deref()) + fn get_core_interface(&self) -> Result> { + self.remp_core.get() + .ok_or_else(|| error!("remp_core was not set"))? + .upgrade().ok_or_else(|| error!("remp_core weak reference is null")) } - async fn new_remp_message(&self, message: RempMessage, source: &Arc) -> Result<()> { + pub fn check_remp_duplicate(&self, id: &UInt256) -> Result { + self.get_core_interface()?.check_remp_duplicate(id) + } + + async fn process_incoming_message(&self, message: &ton_api::ton::ton_node::RempMessage, source: &Arc) -> Result<()> { // TODO send error receipt in case of any error - let engine = self.engine.get().ok_or_else(|| error!("engine was not set"))?; - let remp_core = self.remp_core_interface()?; + let engine = self.engine + .get().ok_or_else(|| error!("engine was not set"))? + .upgrade().ok_or_else(|| error!("engine weak reference is null"))?; + + let remp_core = self.get_core_interface()?; #[cfg(feature = "telemetry")] engine.remp_core_telemetry().message_from_fullnode(); @@ -55,32 +63,19 @@ impl RempService { fail!("Can't process REMP message because validator is out of sync"); } - // deserialise message - let id = message.id().clone(); - let (real_id, message) = create_ext_message(&message.message())?; - if real_id != id { - fail!("Given message id {:x} is not equal calculated one {:x}", id, real_id); - } - - log::trace!(target: "remp", "Point 0. Incoming REMP message {:x} received from {}: {:?}", - id, source, message - ); - // push into remp catchain - remp_core.process_incoming_message(id, message, source.clone()).await?; - - Ok(()) + remp_core.process_incoming_message(message, source.clone()).await } } #[async_trait::async_trait] impl RempMessagesSubscriber for RempService { - async fn new_remp_message(&self, message: RempMessage, source: &Arc) -> Result<()> { + async fn new_remp_message(&self, message: ton_api::ton::ton_node::RempMessage, source: &Arc) -> Result<()> { // TODO send error receipt in case of any error let id = message.id().clone(); log::trace!(target: "remp", "Point 0. Processing incoming REMP message {:x}", id); - match self.new_remp_message(message, source).await { + match self.process_incoming_message(&message, source).await { Ok(_) => log::trace!(target: "remp", "Point 0. Processed incoming REMP message {:x}", id), Err(e) => log::error!(target: "remp", "Point 0. Error processing incoming REMP message {:x}: {}", id, e) } diff --git a/src/validator/tests/test_collator.rs b/src/validator/tests/test_collator.rs index 7560ce27..6b710912 100644 --- a/src/validator/tests/test_collator.rs +++ b/src/validator/tests/test_collator.rs @@ -84,6 +84,7 @@ async fn try_collate_by_engine( created_by_opt.unwrap_or_default(), engine.clone(), rand_seed_opt, + None, CollatorSettings::default(), )?; let (block_candidate, new_state) = collator.collate().await?; diff --git a/src/validator/tests/test_message_cache.rs b/src/validator/tests/test_message_cache.rs index fec7d46d..51b15325 100644 --- a/src/validator/tests/test_message_cache.rs +++ b/src/validator/tests/test_message_cache.rs @@ -22,7 +22,7 @@ use ever_block::{BlockIdExt, ShardIdent}; use ever_block::{Result, SliceData, error, UInt256}; use crate::engine_traits::RempDuplicateStatus; use crate::ext_messages::get_level_and_level_change; -use crate::validator::message_cache::{MessageCache, RmqMessage}; +use crate::validator::message_cache::{MessageCache, RmqMessage, RempMessageOrigin}; use crate::validator::reliable_message_queue::MessageQueue; //use crate::test_helper::init_test_log; @@ -170,10 +170,11 @@ fn do_test_message_cache_add_remove(check_uids: bool) -> Result<()> { tb.cache.add_external_message_status( &msg.message_id, &msg.message_uid, Some(msg.clone()), + Some(Arc::new(RempMessageOrigin::create_empty()?)), RempMessageStatus::TonNode_RempNew, |_old,new| new.clone(), master_seq as u32 - ).await?; + )?; let mut should_be_duplicate = false; if check_uids { @@ -216,7 +217,7 @@ fn do_test_message_cache_add_remove(check_uids: bool) -> Result<()> { let lwb = tb.cache.update_master_cc_ranges(ms3, Duration::from_secs(1))?; tb.cache.gc_old_messages(*lwb.start()).await; assert_eq!(tb.cache.gc_old_messages(*lwb.start()).await.total, 0); - assert_eq!(tb.cache.all_messages_count(), 0); + assert_eq!(tb.cache.all_messages_count(), (0,0,0)); Ok(()) }) @@ -243,18 +244,20 @@ fn do_test_message_cache_uids_same_block() -> Result<()> { tb.cache.add_external_message_status( &msg2.message_id, &msg2.message_uid, Some(msg2.clone()), + Some(Arc::new(RempMessageOrigin::create_empty()?)), RempMessageStatus::TonNode_RempNew, |_old,new| new.clone(), seq - ).await?; + )?; tb.cache.add_external_message_status( &msg1.message_id, &msg1.message_uid, Some(msg1.clone()), + Some(Arc::new(RempMessageOrigin::create_empty()?)), RempMessageStatus::TonNode_RempNew, |_old,new| new.clone(), seq - ).await?; + )?; let dup1 = tb.cache.check_message_duplicates(&msg1.message_id)?; let dup2 = tb.cache.check_message_duplicates(&msg2.message_id)?; @@ -303,10 +306,12 @@ fn do_message_cache_simple_random_test() -> Result<()> { if !tb.simple_cache.uid_has_final_status(&msg.message_uid) { tb.cache.add_external_message_status( &msg.message_id, &msg.message_uid, - Some(msg.clone()), status.clone(), + Some(msg.clone()), + Some(Arc::new(RempMessageOrigin::create_empty()?)), + status.clone(), |_old,new| new.clone(), seq - ).await?; + )?; tb.simple_cache.add_external_message_status( &msg.message_id, &msg.message_uid, seq as i32, &status @@ -346,7 +351,7 @@ fn do_message_cache_simple_random_test() -> Result<()> { tb.cache.gc_old_messages(*lwb.start()).await; tb.simple_cache.remove_old_messages(max_seqs as i32 + 1); - assert_eq!(tb.cache.all_messages_count(), tb.simple_cache.all_messages_count()); + assert_eq!(tb.cache.all_messages_count().0, tb.simple_cache.all_messages_count()); Ok(()) }) } @@ -381,23 +386,25 @@ pub fn test_message_cache_cc_ranges_and_gc() -> Result<()> { tb.cache.add_external_message_status( &msg.message_id, &msg.message_uid, - Some(msg.clone()), RempMessageStatus::TonNode_RempNew, + Some(msg.clone()), + Some(Arc::new(RempMessageOrigin::create_empty()?)), + RempMessageStatus::TonNode_RempNew, |_old,new| new.clone(), 1 - ).await?; + )?; tb.cache.try_set_master_cc_start_time(2 as u32,2.into(), vec!())?; let range = tb.cache.update_master_cc_ranges(2, Duration::from_secs(1))?; - assert!(tb.cache.get_message_with_status_cc(&msg.message_id)?.is_some()); + assert!(tb.cache.get_message_with_origin_status_cc(&msg.message_id)?.is_some()); tb.cache.gc_old_messages(*range.start()).await; tb.cache.try_set_master_cc_start_time(3 as u32,3.into(), vec!())?; let range = tb.cache.update_master_cc_ranges(3, Duration::from_secs(1))?; - assert!(tb.cache.get_message_with_status_cc(&msg.message_id)?.is_some()); + assert!(tb.cache.get_message_with_origin_status_cc(&msg.message_id)?.is_some()); tb.cache.gc_old_messages(*range.start()).await; - assert!(tb.cache.get_message_with_status_cc(&msg.message_id)?.is_none()); + assert!(tb.cache.get_message_with_origin_status_cc(&msg.message_id)?.is_none()); Ok(()) }) diff --git a/src/validator/tests/test_rmq_messages.rs b/src/validator/tests/test_rmq_messages.rs index 038fce89..f4fa1e0a 100644 --- a/src/validator/tests/test_rmq_messages.rs +++ b/src/validator/tests/test_rmq_messages.rs @@ -11,3 +11,604 @@ * limitations under the License. */ +use std::{collections::HashSet, str::FromStr, sync::Arc, time::Duration}; +use std::ops::RangeInclusive; +use std::thread::sleep; + +use ton_api::ton::ton_node::{RempMessageLevel, RempMessageLevel::TonNode_RempMasterchain, RempMessageStatus, rempmessagestatus::{RempAccepted, RempIgnored}}; + +use ever_block::{ + Message, Serializable, Deserializable, ExternalInboundMessageHeader, + MsgAddressInt, Grams, ShardIdent, ValidatorDescr, SigPubKey, BlockIdExt, + MsgAddressExt::AddrNone, UnixTime32, GetRepresentationHash, + error, fail, Result, SliceData, UInt256 +}; + +use crate::{ + config::RempConfig, + engine_traits::{EngineOperations, RempCoreInterface, RempDuplicateStatus}, + validator::{ + reliable_message_queue::{MessageQueue, RmqMessage}, + remp_block_parser::{BlockProcessor, RempMasterBlockIndexingProcessor}, + remp_manager::{RempInterfaceQueues, RempManager, RempSessionStats}, + sessions_computing::GeneralSessionInfo, + validator_utils::{ + get_message_uid, sigpubkey_to_publickey, ValidatorListHash + } + } +}; +#[cfg(feature = "telemetry")] +use crate::validator::telemetry::RempCoreTelemetry; + +use catchain::PublicKey; +use crate::validator::message_cache::{RempMessageOrigin, RempMessageWithOrigin}; +use crate::validator::remp_catchain::RempCatchainInfo; + +#[test] +fn test_rmq_message_serialize() -> Result<()> { + let pre_message = Message::with_ext_in_header_and_body( + ExternalInboundMessageHeader { + src: AddrNone, + dst: MsgAddressInt::from_str( + "-1:7777777777777777777777777777777777777777777777777777777777777777" + ).unwrap(), + import_fee: Grams::zero() + }, + SliceData::from_string( + "e32fb2df7d15a1266c0b4e949607b67f69f2edfa8dc401bec2df9e985e69eecf8e5fb3903cfc3d19d9910077f0e83c4516cf544c9ca65d33a42c71f6b74ed281d54fe4d47282105f230ce104f0bf52cc3adfdb4d5a538f86e33acf55ca68d1480000005f43e9ca6118634e4fdb079a4f00000000603_" + ).unwrap() + ); + let message = Arc::new(pre_message); + let rmq_message_test0: Arc = Arc::new(RmqMessage::new(message)?); + + // Testing proper message serialization + let data: ton_api::ton::bytes = rmq_message_test0.message.write_to_bytes().unwrap().into(); + let msg_buffer = Message::construct_from_bytes(&data).unwrap(); + assert_eq!(*rmq_message_test0.message, msg_buffer); + + // Testing RempMessageBody structure serialization/deserailization + let test0_serialized = RmqMessage::serialize_message_body(&rmq_message_test0.as_remp_message_body()); + println!("Serialized message data: {:?}", data); + println!("Serialized remp_message_body: {:?}", test0_serialized); + let msg = RmqMessage::deserialize_message_body(&test0_serialized).unwrap(); + println!("Deserialzed message data: {:?}", msg.message()); + let rmq_message = RmqMessage::from_message_body(&msg)?; + + // Testing message construction from header and body + let mut message_without_params = Message::default(); + message_without_params.set_header(rmq_message.message.header().clone()); + message_without_params.set_body(rmq_message.message.body().unwrap()); + assert_eq!(*rmq_message_test0.message, message_without_params); + assert_eq!(rmq_message_test0.message_id, rmq_message.message_id, + "Message ids differ: {:x} /= {:x}", rmq_message_test0.message_id, rmq_message.message_id); + assert_eq!(rmq_message_test0.message_uid, rmq_message.message_uid, + "Message uids differ: {:x} /= {:x}", rmq_message_test0.message_uid, rmq_message.message_uid); + + Ok(()) +} + +#[test] +fn test_rmq_message_id() -> Result<()> { + let message_id = UInt256::from_str("c80d5e0e0ada0f4c74ecb03c97947e09204c4d989ac720e1754aeec6f01a4acf")?; + let message_uid = UInt256::from_str("e4089ffa779a932656bd91a9e5b7323f714fb1782c052e8755fb56312be02c83")?; + let message_bytes = [ + 181, 238, 156, 114, 1, 1, 4, 1, 0, 209, 0, 1, 69, 137, 254, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, + 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, + 12, 1, 1, 225, 138, 38, 33, 171, 175, 201, 185, 142, 85, 15, 44, 87, 151, 128, 86, 56, 56, 18, 59, 118, 138, + 154, 35, 57, 76, 24, 106, 28, 34, 58, 59, 105, 58, 38, 145, 18, 99, 38, 183, 252, 21, 167, 111, 232, 24, 140, + 227, 54, 240, 143, 185, 206, 53, 134, 220, 228, 17, 238, 112, 57, 5, 129, 96, 1, 242, 26, 212, 19, 110, 243, 200, + 152, 241, 181, 102, 21, 208, 252, 108, 146, 59, 40, 199, 102, 184, 223, 73, 204, 64, 183, 116, 173, 179, 54, + 87, 34, 128, 0, 0, 99, 176, 207, 135, 51, 25, 133, 83, 224, 68, 199, 96, 179, 96, 2, 1, 99, 128, 9, 75, 218, + 166, 71, 170, 33, 189, 182, 153, 177, 109, 126, 100, 87, 19, 110, 125, 93, 129, 102, 179, 156, 141, 198, 170, + 248, 140, 147, 62, 90, 201, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 171, 102, 80, 210, 228, 3, 0, 0 + ].to_vec(); + + let rmq_message = RmqMessage::from_raw_message(&message_bytes)?; + assert_eq!(message_id, rmq_message.message_id); + assert_eq!(message_uid, rmq_message.message_uid); + assert_eq!(message_id, rmq_message.message.hash()?); + assert_eq!(message_uid, get_message_uid(&rmq_message.message)); + Ok(()) +} + +struct RmqTestEngine { + #[cfg(feature = "telemetry")] + remp_core_telemetry: RempCoreTelemetry, + collator_queue: lockfree::queue::Queue<(UInt256, Arc)> +} + +impl EngineOperations for RmqTestEngine { + fn new_remp_message(&self, id: UInt256, message: Arc) -> Result<()> { + println!("New message received for collation: {:x}", id); + self.collator_queue.push((id, message)); + Ok(()) + } + + #[cfg(feature = "telemetry")] + fn remp_core_telemetry(&self) -> &RempCoreTelemetry { + &self.remp_core_telemetry + } +} + +impl RmqTestEngine { + pub fn new() -> Self { + Self { + #[cfg(feature = "telemetry")] + remp_core_telemetry : RempCoreTelemetry::new(10), + collator_queue: lockfree::queue::Queue::new() + } + } +} + +struct RmqTestbench { + engine: Arc, + remp_manager: Arc, + remp_interface_queues: RempInterfaceQueues, + params: Arc, + local_key: PublicKey, + curr_validators: Vec, + next_validators: Vec, + node_list_id: UInt256, + rp_guarantee: Duration, + message_queue: MessageQueue +} + +impl RmqTestbench { + fn create_runtime() -> Result { + let runtime = match tokio::runtime::Runtime::new() { + Ok(r) => r, + Err(e) => fail!(e) + }; + return Ok(runtime); + } + + async fn new(runtime_handle: &tokio::runtime::Handle, masterchain_seqno: u32, rp_guarantee: Duration) -> Result { + let engine = Arc::new(RmqTestEngine::new()); + + let remp_config = RempConfig::create_empty(); + let (remp_manager_value, remp_interface_queues) = RempManager::create_with_options( + engine.clone(), remp_config.clone(), Arc::new(runtime_handle.clone()) + ); + let remp_manager = Arc::new(remp_manager_value); + let local_validator = ValidatorDescr::with_params ( + SigPubKey::from_bytes(UInt256::rand().as_slice())?, + 1, None, None + ); + let local_key = sigpubkey_to_publickey(&local_validator.public_key); + let curr_validators = vec!(local_validator.clone()); + let next_validators = vec!(local_validator.clone()); + let params = Arc::new(GeneralSessionInfo { + shard: ShardIdent::with_tagged_prefix(0,0xc000_0000_0000_0000)?, + opts_hash: Default::default(), + catchain_seqno: 2, + key_seqno: 1, + max_vertical_seqno: 0 + }); + let node_list_id = ValidatorListHash::rand(); + + for cc in 1..=masterchain_seqno { + remp_manager.create_master_cc_session(cc, 0.into(), vec!())?; + } + let masterchain_range = remp_manager.advance_master_cc(masterchain_seqno, rp_guarantee.clone())?; + + let queue_info = Arc::new(RempCatchainInfo::create( + params.clone(), &masterchain_range, + &curr_validators, &next_validators, + &local_key, node_list_id.clone(), + )?); + let message_queue = MessageQueue::create( + engine.clone(), remp_manager.clone(), queue_info + )?; + + Ok(RmqTestbench { + engine, + remp_manager, + remp_interface_queues, + params, + local_key, + curr_validators, + next_validators, + node_list_id, + rp_guarantee, + message_queue + }) + } + + async fn send_pending_message(&self, msg: &RempMessageWithOrigin, masterchain_seqno: u32) -> Result { + self.message_queue.process_pending_remp_catchain_record( + &msg.as_remp_catchain_record(masterchain_seqno), + 0 + ).await?; + self.remp_manager.message_cache.update_message_body(Arc::new(msg.message.clone())) + } + + fn replace_message_queue(&mut self, masterchain_range: &RangeInclusive) -> Result<()> { + let info = Arc::new(RempCatchainInfo::create( + self.params.clone(), masterchain_range, + &self.curr_validators, &self.next_validators, + &self.local_key, self.node_list_id.clone())?); + + self.message_queue = MessageQueue::create( + self.engine.clone(), self.remp_manager.clone(), info + )?; + Ok(()) + } + + async fn advance_master_cc(&mut self, masterchain_seqno: u32, mc_time: UnixTime32) -> Result { + self.remp_manager.create_master_cc_session(masterchain_seqno, mc_time, vec!())?; + let new_range = self.remp_manager.advance_master_cc(masterchain_seqno, self.rp_guarantee)?; + self.replace_message_queue(&new_range)?; + Ok(self.remp_manager.gc_old_messages(*new_range.start()).await) + } +} + +fn random_message() -> Result { + RmqMessage::make_test_message(&SliceData::from(UInt256::rand())) +} + +fn make_test_message_with_origin(body: &SliceData) -> Result { + let m = RmqMessage::make_test_message(body)?; + let o = RempMessageOrigin::create_empty()?; + Ok(RempMessageWithOrigin { message: m, origin: o }) +} + +fn make_test_random_message_with_origin() -> Result { + make_test_message_with_origin(&SliceData::from(UInt256::rand())) +} + +#[test] +fn remp_simple_forwarding_test() -> Result<()> { + // -- seq=0/seq=1,forwarding + // 5k -- new/(k=0/1rejected_l/2rejected_r/3rejected_lr) + // 5k+1 -- ignored/() + // 5k+2 -- duplicate/() + // 5k+3 -- accepted/() + // 5k+4 -- rejected/() + + //let mut msgs = Vec::new(); + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let testbench = RmqTestbench::new(&runtime_handle, 2, Duration::from_secs(10)).await?; + + let blk1 = BlockIdExt::with_params( + testbench.params.shard.clone(), + 5129, + UInt256::rand(), + UInt256::rand() + ); + + let m1 = make_test_random_message_with_origin()?; + let m2 = make_test_random_message_with_origin()?; + + testbench.remp_manager.message_cache.add_external_message_status( + m1.get_message_id(), + &m1.message.message_uid, + Some(Arc::new(m1.message.clone())), + Some(Arc::new(m1.origin.clone())), + RempMessageStatus::TonNode_RempAccepted(RempAccepted { + level: RempMessageLevel::TonNode_RempShardchain, + block_id: blk1.clone(), + master_id: Default::default() + }), + |_old,new| new.clone(), + 1 + )?; + + //testbench.remp_manager.options + println!("{:?}", testbench.remp_manager.message_cache.get_message_with_origin_status_cc(m1.get_message_id())); + + assert_eq!(testbench.send_pending_message(&m1, 1).await?, false); // false: we've added m1, but not m2 + assert_eq!(testbench.send_pending_message(&m2, 1).await?, true); + + let ign = RempIgnored { level: RempMessageLevel::TonNode_RempMasterchain, block_id: blk1 }; + assert_eq!(testbench.remp_manager.message_cache.get_message_status(m1.get_message_id())?.unwrap(), RempMessageStatus::TonNode_RempIgnored(ign)); + assert_eq!(testbench.remp_manager.message_cache.get_message_status(m2.get_message_id())?.unwrap(), MessageQueue::forwarded_ignored_status()); + + println!("Remp interface queues: {} repsonses", testbench.remp_interface_queues.response_receiver.len()); + Ok(()) + }) +} + +#[test] +fn remp_simple_collation_equal_uids_test() -> Result<()> { + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let testbench = RmqTestbench::new(&runtime_handle, 2, Duration::from_secs(10)).await?; + + let body1 = SliceData::from(UInt256::rand()); + let body2 = SliceData::from(UInt256::rand()); + + let mut msgs = Vec::new(); + for _cnt in 0..5 { + msgs.push(make_test_message_with_origin(&body1)?); + msgs.push(make_test_message_with_origin(&body2)?); + } + + let (_min_id,uid_for_min_id) = msgs.iter().map(|a| (a.get_message_id(), &a.message.message_uid)).min().unwrap(); + let msg_to_collate = msgs.iter().filter(|x| &x.message.message_uid != uid_for_min_id).map(|x| x.get_message_id()).min().unwrap(); + let (acc_id,acc_uid) = msgs.iter().filter(|x| &x.message.message_uid == uid_for_min_id).map(|x| (x.get_message_id(),&x.message.message_uid)).max().unwrap(); + + let mut must_be_collated = HashSet::::new(); + must_be_collated.insert(msg_to_collate.clone()); + let must_be_rejected : Vec = msgs.iter() + .filter(|a| !must_be_collated.contains(a.get_message_id()) && a.get_message_id() != acc_id) + .map(|a| a.clone()) + .collect(); + + for m in msgs.iter() { + let pc = if must_be_collated.contains(m.get_message_id()) { "C" } else { "" }.to_string(); + let pa = if m.get_message_id() == acc_id { "A" } else { "" }.to_string(); + let pr = if must_be_rejected.contains(&m) { "R" } else { "" }.to_string(); + println!("Pending msg: {:x} {}{}{}", m.get_message_id(), pc, pa, pr); + + // All msg ids are different, therefore body must be added + assert!(testbench.send_pending_message(m, testbench.message_queue.catchain_info.get_master_cc_seqno()).await?); + } + + let accepted = RempAccepted { + level: RempMessageLevel::TonNode_RempMasterchain, + block_id: Default::default(), + master_id: Default::default() + }; + testbench.remp_manager.message_cache.add_external_message_status( + &acc_id, &acc_uid, None, None, + RempMessageStatus::TonNode_RempAccepted(accepted), + |_o,n| n.clone(), + 2 + )?; + + println!("Collecting messages for collation"); + sleep(Duration::from_millis(10)); // To overcome SystemTime inconsistency and make tests reproducible. + let messages = testbench.message_queue.prepare_messages_for_collation().await?; + for (msg_id, msg, _order) in messages.into_iter() { + testbench.engine.new_remp_message(msg_id.clone(), msg)?; + } + + for (id, _msg) in testbench.engine.collator_queue.pop_iter() { + println!("collated: {:x}", id); + assert!(must_be_collated.remove(&id)); + } + assert!(must_be_collated.is_empty()); + + for id in must_be_rejected.iter() { + let status = testbench.remp_manager.message_cache.get_message_status(id.get_message_id())?; + if let Some(RempMessageStatus::TonNode_RempRejected(rejected)) = &status { + assert_eq!(rejected.level, RempMessageLevel::TonNode_RempQueue); + println!("rejected: {}, '{:?}'", id, status) + } + else { + panic!("Expected rejected status, found {:?}", status); + } + } + + Ok(()) + }) +} + +#[test] +fn remp_simple_expiration_test() -> Result<()> { + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let mut testbench = RmqTestbench::new(&runtime_handle, 2, Duration::from_secs(10)).await?; + + let mut bodies = Vec::new(); + for _ in 0..5 { + bodies.push(SliceData::from(UInt256::rand())); + } + + for b in bodies.iter() { + let m = make_test_message_with_origin(b)?; + println!("Pending msg 2: {:x}", m.get_message_id()); + testbench.send_pending_message(&m, testbench.message_queue.catchain_info.get_master_cc_seqno()).await?; + } + println!("Collected old messages 3: {}", testbench.advance_master_cc(3, 30.into()).await?); + println!("Collected old messages 4: {}", testbench.advance_master_cc(4, 40.into()).await?); + println!("Collected old messages 5: {}", testbench.advance_master_cc(5, 50.into()).await?); + + let mut msgs = Vec::new(); + for b in bodies.iter() { + let m = make_test_message_with_origin(b)?; + println!("Pending msg 5: {:x}", m.get_message_id()); + testbench.send_pending_message(&m, testbench.message_queue.catchain_info.get_master_cc_seqno()).await?; + msgs.push(m); + } + + for m in msgs.iter() { + assert_eq!( + testbench.remp_interface_queues.check_remp_duplicate(m.get_message_id())?, + RempDuplicateStatus::Fresh(m.message.message_uid.clone()) + ); + testbench.remp_manager.message_cache.add_external_message_status( + m.get_message_id(), + &m.message.message_uid, + Some(Arc::new(m.message.clone())), + Some(Arc::new(m.origin.clone())), + RempMessageStatus::TonNode_RempAccepted(RempAccepted { + level: TonNode_RempMasterchain, + block_id: Default::default(), + master_id: Default::default() + }), + |_o,n| n.clone(), + 5 + )?; + } + msgs.clear(); + + println!("Collected old messages 6: {}", testbench.advance_master_cc(6, 60.into()).await?); + + for b in bodies.iter() { + let m = make_test_message_with_origin(b)?; + println!("Pending msg 6: {:x}", m.get_message_id()); + testbench.send_pending_message(&m, testbench.message_queue.catchain_info.get_master_cc_seqno()).await?; + msgs.push(m); + } + + for m in msgs.iter() { + match testbench.remp_interface_queues.check_remp_duplicate(&m.get_message_id())? { + RempDuplicateStatus::Absent => panic!("Message {} must present", m), + RempDuplicateStatus::Fresh(uid) => panic!("Message {} must not be fresh with uid {:x}", m, uid), + RempDuplicateStatus::Duplicate(_, uid, _) => assert_eq!(m.message.message_uid, uid) + } + } + Ok(()) + }) +} + +#[test] +fn remp_simple_status_updating_test() -> Result<()> { + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let mut testbench = RmqTestbench::new(&runtime_handle, 1, Duration::from_secs(10)).await?; + let blk1 = BlockIdExt::with_params( + testbench.params.shard.clone(), + 5129, + UInt256::rand(), + UInt256::rand() + ); + + let common_body = SliceData::from(UInt256::rand()); + + let m1 = make_test_message_with_origin(&common_body)?; + let m2 = make_test_message_with_origin(&common_body)?; + + let proc = RempMasterBlockIndexingProcessor::new( + blk1.clone(), blk1.clone(), + testbench.remp_manager.message_cache.clone(), + 1 + ); + proc.process_message(m1.get_message_id(), &m1.message.message_uid).await; + + testbench.advance_master_cc(2, 10.into()).await?; + + testbench.remp_manager.message_cache.add_external_message_status( + m2.get_message_id(), + &m2.message.message_uid, + Some(Arc::new(m2.message.clone())), + Some(Arc::new(m2.origin.clone())), + RempMessageStatus::TonNode_RempNew, + |_old,new| new.clone(), + 2 + )?; + + assert_eq!( + testbench.remp_interface_queues.check_remp_duplicate(m2.get_message_id())?, + RempDuplicateStatus::Duplicate( + blk1.clone(), m1.message.message_uid.clone(), m1.get_message_id().clone() + ) + ); + + testbench.advance_master_cc(3, 20.into()).await?; + + assert_eq!( + testbench.remp_interface_queues.check_remp_duplicate(m2.get_message_id())?, + RempDuplicateStatus::Fresh(m2.message.message_uid.clone()) + ); + + Ok(()) + }) +} + +async fn push_random_msgs(testbench: &RmqTestbench, count: usize) -> Result>> { + let blk1 = BlockIdExt::with_params( + testbench.params.shard.clone(), + 5129, + UInt256::rand(), + UInt256::rand() + ); + + let mut msgs = Vec::new(); + for _ in 0..count { + msgs.push(Arc::new(random_message()?)); + } + + let proc = RempMasterBlockIndexingProcessor::new( + blk1.clone(), blk1.clone(), + testbench.remp_manager.message_cache.clone(), + testbench.message_queue.catchain_info.get_master_cc_seqno() + ); + + for m in msgs.iter() { + proc.process_message(&m.message_id, &m.message_uid).await; + } + + Ok(msgs) +} + +#[test] +fn remp_simple_gc_range_test() -> Result<()> { + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let mut testbench = RmqTestbench::new(&runtime_handle, 1, Duration::from_secs(10)).await?; + + let msgs = push_random_msgs(&testbench, 20).await?; + + let stats = testbench.advance_master_cc(2, 5.into()).await?; + assert_eq!(stats.total, 0); + + let msgs2 = push_random_msgs(&testbench, 24).await?; + + let stats = testbench.advance_master_cc(3, 7.into()).await?; + assert_eq!(stats.total, 0); + let stats = testbench.advance_master_cc(4, 9.into()).await?; + assert_eq!(stats.total, 0); + let stats = testbench.advance_master_cc(5, 14.into()).await?; + assert_eq!(stats.total, 0); + let stats = testbench.advance_master_cc(6, 15.into()).await?; + assert_eq!(stats.has_only_header, msgs.len()); + let stats = testbench.advance_master_cc(7, 17.into()).await?; + assert_eq!(stats.has_only_header, msgs2.len()); + let stats = testbench.advance_master_cc(8, 18.into()).await?; + assert_eq!(stats.total, 0); + + let msgs3 = push_random_msgs(&testbench, 17).await?; + let stats = testbench.advance_master_cc(9, 100.into()).await?; + assert_eq!(stats.total, 0); + let _msgs4 = push_random_msgs(&testbench, 417).await?; + let stats = testbench.advance_master_cc(10, 218.into()).await?; + assert_eq!(stats.has_only_header, msgs3.len()); + let stats = testbench.advance_master_cc(10, 218.into()).await?; + assert_eq!(stats.total, 0); + + Ok(()) + }) +} + +#[test] +fn remp_simple_advance_special_cases() -> Result<()> { + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let mut testbench = RmqTestbench::new(&runtime_handle, 1, Duration::from_secs(10)).await?; + let msgs = push_random_msgs(&testbench, 20).await?; + let stat = testbench.advance_master_cc(2, 20.into()).await?; + assert_eq!(stat.total, 0); + let msgs2 = push_random_msgs(&testbench, 20).await?; + let stat = testbench.advance_master_cc(3, 30.into()).await?; + assert_eq!(stat.total, msgs.len()); + assert!(testbench.advance_master_cc(3, 40.into()).await.is_err()); + assert!(testbench.advance_master_cc(3, 29.into()).await.is_err()); + let stat = testbench.advance_master_cc(3, 30.into()).await?; + assert_eq!(stat.total, 0); + let stat = testbench.advance_master_cc(4, 40.into()).await?; + assert_eq!(stat.total, msgs2.len()); + assert!(testbench.advance_master_cc(5, 39.into()).await.is_err()); + Ok(()) + }) +} diff --git a/src/validator/validator_group.rs b/src/validator/validator_group.rs index 94ead864..9b8eaf36 100644 --- a/src/validator/validator_group.rs +++ b/src/validator/validator_group.rs @@ -16,7 +16,7 @@ use std::ops::RangeInclusive; use crossbeam_channel::Receiver; use catchain::utils::get_hash; -use ever_block::{BlockIdExt, ShardIdent, ValidatorSet, ValidatorDescr}; +use ever_block::{BlockIdExt, ShardIdent, ValidatorSet, ValidatorDescr, UnixTime32}; use ever_block::{fail, error, Result, UInt256}; use validator_session::{ BlockHash, BlockPayloadPtr, CatchainOverlayManagerPtr, @@ -53,6 +53,7 @@ use crate::{ } }, validating_utils::{fmt_next_block_descr_from_next_seqno, append_rh_to_next_block_descr} }; +use crate::validator::reliable_message_queue::RempQueueCollatorInterfaceImpl; #[cfg(feature = "slashing")] use crate::validator::slashing::SlashingManagerPtr; @@ -582,6 +583,13 @@ impl ValidatorGroup { self.group_impl.execute_sync(|group_impl| group_impl.reliable_queue.clone()).await } + pub async fn get_remp_queue_collator_interface(&self) -> Option> { + let queue_manager = self.get_reliable_message_queue().await; + queue_manager.map(|x| { + Arc::new(RempQueueCollatorInterfaceImpl::new(x)) + }) + } + pub async fn info_round(&self, round: u32) -> String { self.group_impl.execute_sync(|group_impl| group_impl.info_round(round)).await } @@ -652,7 +660,7 @@ impl ValidatorGroup { let (_lk_round, prev_block_ids, mm_block_id, min_ts) = self.group_impl.execute_sync(|group_impl| group_impl.update_round (round)).await; - let include_external_messages = match self.check_in_sync(&prev_block_ids).await { + let remp_queue_collator_interface_impl = match self.check_in_sync(&prev_block_ids).await { Err(e) => { log::warn!(target: "validator", "({}): Error checking sync for {}: `{}`", next_block_descr, self.info_round(round).await, e @@ -660,17 +668,21 @@ impl ValidatorGroup { callback(Err(e)); return; } - Ok(external_messages) => external_messages + Ok(false) => None, + Ok(true) => self.get_remp_queue_collator_interface().await }; + /* + // To be removed after moving message queue processing into collator if let Some(rmq) = self.get_reliable_message_queue().await { log::info!( target: "validator", "ValidatorGroup::on_generate_slot: ({}) collecting REMP messages \ for {} for collation: {}", - next_block_descr, self.info_round(round).await, include_external_messages + next_block_descr, self.info_round(round).await, remp_queue_collator_interface_impl.is_some() ); - if include_external_messages { + if let Some(interface_impl) = remp_queue_collator_interface_impl { + let interface if let Err(e) = rmq.collect_messages_for_collation().await { log::error!( target: "validator", @@ -682,6 +694,7 @@ impl ValidatorGroup { } } } + */ let result = match mm_block_id { Some(mc) => { @@ -690,6 +703,7 @@ impl ValidatorGroup { min_ts, mc.seq_no, prev_block_ids, + remp_queue_collator_interface_impl.clone().map(|x| x.into_interface()), self.local_key.clone(), self.validator_set.clone(), self.engine.clone(), @@ -701,6 +715,18 @@ impl ValidatorGroup { None => Err(error!("Min masterchain block id missing")), }; + let return_result_message = if let Some(x) = remp_queue_collator_interface_impl { + match x.return_prepared_messages_to_queue().await { + Ok((total, returned)) => format!("total external messages {}, processed {}, returned to queue {}", + total, total as i64 - returned as i64, returned + ), + Err(e) => format!("error returning non-processed external messages to queue `{}`", e) + } + } + else { + format!("no external messages processed") + }; + let candidate = match self.verification_manager.clone() { Some(_) => match &result { Ok(candidate) => Some(candidate.clone()), @@ -711,8 +737,7 @@ impl ValidatorGroup { let result_message = match &result { Ok(_) => { - let now = std::time::SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs(); + let now = UnixTime32::now().as_u32() as u64; self.last_collation_time.fetch_max(now, Ordering::Relaxed); format!("Collation successful") @@ -744,6 +769,8 @@ impl ValidatorGroup { } }; + let result_message = format!("{}{}", result_message, return_result_message); + if let Some(rmq) = self.get_reliable_message_queue().await { if let Err(e) = rmq.process_collation_result().await { log::error!(target: "validator", "({}): Error processing collation results for {}: `{}`", diff --git a/src/validator/validator_manager.rs b/src/validator/validator_manager.rs index 5299ca47..f19e4528 100644 --- a/src/validator/validator_manager.rs +++ b/src/validator/validator_manager.rs @@ -1486,7 +1486,7 @@ impl ValidatorManagerImpl { } if let Some(rm) = &self.remp_manager { - metrics::gauge!("remp_message_cache_size", rm.message_cache.all_messages_count() as f64); + metrics::gauge!("remp_message_cache_size", rm.message_cache.all_messages_count().0 as f64); log::info!(target: "validator_manager", "Remp message cache stats: {}", rm.message_cache.message_stats()); for (s, shard_ident) in rm.catchain_store.list_catchain_sessions().await.iter() { @@ -1726,11 +1726,19 @@ pub fn start_validator_manager( log::error!(target: "validator_manager", "set_remp_core_interface: {}", e); } + let iface = remp_iface.clone(); runtime.clone().spawn(async move { log::info!(target: "remp", "Starting REMP responses polling loop"); - remp_iface.poll_responses_loop().await; + iface.poll_responses_loop().await; log::info!(target: "remp", "Finishing REMP responses polling loop"); }); + + let iface = remp_iface.clone(); + runtime.clone().spawn(async move { + log::info!(target: "remp", "Starting REMP testing loop"); + iface.test_remp_messages_loop(Duration::from_millis(300), 100).await; + log::info!(target: "remp", "Finishing REMP testing loop"); + }); } if let Err(e) = manager.invoke().await { diff --git a/src/validator/validator_utils.rs b/src/validator/validator_utils.rs index 9df997d1..cb603484 100644 --- a/src/validator/validator_utils.rs +++ b/src/validator/validator_utils.rs @@ -17,13 +17,7 @@ use crate::{ }; use catchain::{BlockPayloadPtr, CatchainNode, PublicKey, PublicKeyHash}; -use ever_block::{ - error, fail, BlockIdExt, BlockInfo, BlockSignatures, BlockSignaturesPure, BuilderData, - ConfigParams, CryptoSignature, CryptoSignaturePair, Deserializable, Ed25519KeyOption, - GlobalCapabilities, HashmapType, KeyId, Message, Result, Serializable, Sha256, ShardIdent, - SigPubKey, UInt256, UnixTime32, ValidatorBaseInfo, ValidatorDescr, ValidatorSet, Workchains, - WorkchainDescr -}; +use ever_block::{error, fail, BlockIdExt, BlockInfo, BlockSignatures, BlockSignaturesPure, BuilderData, ConfigParams, CryptoSignature, CryptoSignaturePair, Deserializable, Ed25519KeyOption, GlobalCapabilities, HashmapType, KeyId, Message, Result, Serializable, Sha256, ShardIdent, SigPubKey, UInt256, UnixTime32, ValidatorBaseInfo, ValidatorDescr, ValidatorSet, Workchains, WorkchainDescr, AccountIdPrefixFull}; use ever_block::CatchainConfig; use std::{collections::HashMap, fmt::Debug, hash::Hash, sync::Arc}; use ton_api::ton::engine::validator::validator::groupmember::GroupMember; @@ -418,12 +412,29 @@ pub fn calc_subset_for_workchain_standard( pub async fn get_shard_by_message(engine: Arc, message: Arc) -> Result { let dst_wc = message.dst_workchain_id() .ok_or_else(|| error!("Can't get workchain id from message"))?; - let dst_address = message.int_dst_account_id() + let mut dst_address = message.int_dst_account_id() .ok_or_else(|| error!("Can't get standart destination address from message"))?; + let last_mc_state = engine.load_last_applied_mc_state().await?; + let mut shards_vec = Vec::new(); + last_mc_state.shards()?.iterate_shards_for_workchain( + dst_wc, |s, _| { shards_vec.push(s); Ok(true) } + )?; + + if shards_vec.len() == 0 { + return Ok(ShardIdent::masterchain()) + } + + let shard_id = shards_vec + .get(dst_address.hash(0).first_u64() as usize % shards_vec.len()) + .ok_or_else(|| error!("No shards"))?; + // find account and related shard - let (_account, shard) = engine.load_account(dst_wc, dst_address.clone()).await?; - Ok(shard) + //let (_account, shard) = engine.load_account(dst_wc, dst_address.clone()).await?; + + log::trace!(target: "remp", "get_shard_by_message: message {:x}: shard {}", dst_address, shard_id); + + Ok(shard_id.clone()) } pub fn get_first_block_seqno_after_prevs(prevs: &Vec) -> Option { diff --git a/storage/Cargo.toml b/storage/Cargo.toml index eee6b611..5118f8b4 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -4,14 +4,17 @@ name = 'storage' version = '0.5.0' [dependencies] +adnl = { git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.10.24' } ahash = '0.8' async-trait = '0.1.31' bytes = '1.1.0' +ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.10.2' } failure = '0.1' fnv = '1.0.6' futures = '0.3.4' hex = '0.4' lazy_static = '1.4.0' +lockfree = { git = 'https://github.com/everx-labs/lockfree.git' } log = '0.4' log4rs = '1.2' lru = '0.11.0' @@ -26,9 +29,6 @@ serde_derive = '1.0.114' strum = '0.18.0' strum_macros = '0.18.0' tokio = { features = [ 'fs', 'rt-multi-thread' ], version = '1.5' } -adnl = { git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.10.24' } -ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.10.2' } -lockfree = { git = 'https://github.com/everx-labs/lockfree.git' } ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.3.79' } [build-dependencies] diff --git a/storage/src/dynamic_boc_rc_db.rs b/storage/src/dynamic_boc_rc_db.rs index c1edd70e..fb368cd1 100644 --- a/storage/src/dynamic_boc_rc_db.rs +++ b/storage/src/dynamic_boc_rc_db.rs @@ -301,9 +301,7 @@ impl DynamicBocDb { // Is thread-safe pub fn load_boc(self: &Arc, root_cell_id: &UInt256, use_cache: bool) -> Result { - let storage_cell = self.load_cell(root_cell_id, use_cache)?; - - Ok(Cell::with_cell_impl_arc(storage_cell)) + self.load_cell(root_cell_id, use_cache) } pub fn check_and_update_cells(&mut self) -> Result<()> { @@ -454,7 +452,7 @@ impl DynamicBocDb { self: &Arc, cell_id: &UInt256, use_cache: bool, - ) -> Result> { + ) -> Result { let deserialize = |value: &[u8]| { StorageCell::deserialize(self, value, use_cache, false).or_else( @@ -469,7 +467,7 @@ impl DynamicBocDb { target: TARGET, "DynamicBocDb::load_cell from cache id {cell_id:x}" ); - return Ok(Arc::new(cell)) + return Ok(Cell::with_cell_impl(cell)) }, Err(e) => log::trace!( target: TARGET, @@ -481,6 +479,15 @@ impl DynamicBocDb { let storage_cell_data = match self.db.get(cell_id) { Ok(data) => data, Err(e) => { + + if let Some(guard) = self.storing_cells.get(cell_id) { + log::error!( + target: TARGET, + "DynamicBocDb::load_cell from storing_cells by id {cell_id:x}", + ); + return Ok(guard.val().clone()); + } + log::error!("FATAL!"); log::error!("FATAL! Can't load cell {:x} from db, error: {:?}", cell_id, e); log::error!("FATAL!"); @@ -519,7 +526,7 @@ impl DynamicBocDb { "DynamicBocDb::load_cell from DB id {cell_id:x} use_cache {use_cache}" ); - Ok(storage_cell) + Ok(Cell::with_cell_impl_arc(storage_cell)) } pub(crate) fn allocated(&self) -> &StorageAlloc { @@ -819,7 +826,7 @@ impl DoneCellsStorage for DoneCellsStorageAdapter { fn get(&self, index: u32) -> Result { let id = UInt256::from_slice(self.index.get(&index.into())?.as_ref()).into(); - Ok(Cell::with_cell_impl_arc(self.boc_db.clone().load_cell(&id, false)?)) + Ok(self.boc_db.clone().load_cell(&id, false)?) } fn cleanup(&mut self) -> Result<()> { @@ -849,8 +856,7 @@ impl CellByHashStorageAdapter { impl CellByHashStorage for CellByHashStorageAdapter { fn get_cell_by_hash(&self, hash: &UInt256) -> Result { - let cell = Cell::with_cell_impl_arc(self.boc_db.clone().load_cell(&hash, self.use_cache)?); - Ok(cell) + self.boc_db.clone().load_cell(&hash, self.use_cache) } } @@ -920,7 +926,7 @@ impl OrderedCellsStorage for OrderedCellsStorageAdapter { fn get_cell_by_index(&self, index: u32) -> Result { let id = UInt256::from_slice(self.index1.get(&index.into())?.as_ref()).into(); - let cell = Cell::with_cell_impl_arc(self.boc_db.clone().load_cell(&id, false)?); + let cell = self.boc_db.clone().load_cell(&id, false)?; let slowdown = self.slowdown(); if index % 1000 == 0 { @@ -1011,4 +1017,4 @@ impl RawCellsCache { Ok(value) } -} +} \ No newline at end of file diff --git a/storage/src/types/storage_cell.rs b/storage/src/types/storage_cell.rs index 66707572..81a24e48 100644 --- a/storage/src/types/storage_cell.rs +++ b/storage/src/types/storage_cell.rs @@ -198,17 +198,17 @@ impl StorageCell { }; let boc_db = self.boc_db.upgrade().ok_or_else(|| error!("BocDb is dropped"))?; - let storage_cell = boc_db.load_cell( + let cell = boc_db.load_cell( &hash, self.use_cache )?; if self.use_cache { self.references.write()[index].cell = - Some(Arc::downgrade(&storage_cell) as Weak); + Some(Arc::downgrade(cell.cell_impl()) as Weak); } - Ok(storage_cell) + Ok(cell.cell_impl().clone()) } } @@ -287,4 +287,4 @@ impl PartialEq for StorageCell { fn eq(&self, other: &Self) -> bool { self.cell_data.raw_hash(MAX_LEVEL) == other.cell_data.raw_hash(MAX_LEVEL) } -} +} \ No newline at end of file diff --git a/tests/setup_test_paths.sh b/tests/setup_test_paths.sh new file mode 100755 index 00000000..f2fd4115 --- /dev/null +++ b/tests/setup_test_paths.sh @@ -0,0 +1,5 @@ +TESTS_DIR=$(realpath $(dirname "${BASH_SOURCE:-$0}")) +BASE_DIR=${TESTS_DIR}/../../ +TOOLS_DIR=${BASE_DIR}/ever-node-tools/target/release/ +CONFIG_PATH=${BASE_DIR}/ever-node/target/release/ +CONSOLE_CONFIG_0=${CONFIG_PATH}configs_0/console.json diff --git a/tests/test_high_load/.gitignore b/tests/test_high_load/.gitignore new file mode 100644 index 00000000..01380c91 --- /dev/null +++ b/tests/test_high_load/.gitignore @@ -0,0 +1,2 @@ +*.boc +logs/* \ No newline at end of file diff --git a/tests/test_high_load/Brazil.abi.json b/tests/test_high_load/Brazil.abi.json new file mode 100644 index 00000000..8ea7e666 --- /dev/null +++ b/tests/test_high_load/Brazil.abi.json @@ -0,0 +1,126 @@ +{ + "ABI version": 2, + "header": ["time"], + "functions": [ + { + "name": "constructor", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "setC", + "inputs": [ + {"name":"counter","type":"uint128"} + ], + "outputs": [ + ] + }, + { + "name": "deployWorker", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "stopWorker", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "NeedMoney", + "inputs": [ + {"name":"id","type":"uint128"} + ], + "outputs": [ + ] + }, + { + "name": "build", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "init", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "setNumber", + "inputs": [ + {"name":"number","type":"uint128"} + ], + "outputs": [ + ] + }, + { + "name": "setCode2", + "inputs": [ + {"name":"c","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "setData2", + "inputs": [ + {"name":"c","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "setContract2", + "inputs": [ + {"name":"c","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "setKey", + "inputs": [ + {"name":"key","type":"uint256"} + ], + "outputs": [ + ] + }, + { + "name": "setdGrant", + "inputs": [ + {"name":"dgrant","type":"uint128"} + ], + "outputs": [ + ] + }, + { + "name": "upgrade", + "inputs": [ + {"name":"newcode","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "grant", + "inputs": [ + {"name":"addr","type":"address"}, + {"name":"value","type":"uint128"} + ], + "outputs": [ + ] + } + ], + "data": [ + ], + "events": [ + ] +} diff --git a/tests/test_high_load/Brazil.sol b/tests/test_high_load/Brazil.sol new file mode 100644 index 00000000..03da1b5b --- /dev/null +++ b/tests/test_high_load/Brazil.sol @@ -0,0 +1,173 @@ +/* */ +pragma solidity >= 0.6.0; + +import "Worker.sol"; + +contract Brazil is IBrazil { + + uint128 _counter = 0; + uint128 _deployed = 0; + uint128 _number = 10; + TvmCell _contract2; + TvmCell _code2; + TvmCell _data2; + + uint _key; + uint128 _dgrant = 2e9; + uint128 _giveMoney = 3e9; + + address _one; + address _two; + address _three; + + address[] _DeployedArray; + address[] _WorkerHelps; + + modifier alwaysAccept { + tvm.accept(); + _; + } + + constructor() public alwaysAccept { + } + + + function setC(uint128 counter) public alwaysAccept { + if (counter < 0) { + counter = 0; + } + _counter = counter; + while (_counter > _deployed){ + deployWorker(); + } + while (_deployed > _counter){ + stopWorker(); + } + } + + function _isSameShard(address a1, address a2) private pure returns (bool) { + uint sh1 = a1.value >> 252; + uint sh2 = a2.value >> 252; + return (sh1 == sh2); + } + + function deployWorker() public alwaysAccept { + if (_DeployedArray.length > _counter){ + while (_deployed < _counter){ + Worker(_DeployedArray[_deployed]).start(); + Worker(_WorkerHelps[_deployed]).start(); + _deployed++; + } + return; + } + if (_DeployedArray.length > _deployed){ + while (_deployed < _DeployedArray.length){ + Worker(_DeployedArray[_deployed]).start(); + Worker(_WorkerHelps[_deployed]).start(); + _deployed++; + } + return; + } + address a1; + address a2; + TvmCell s1; + TvmCell s2; + + _key = _key + rnd.next(3); + s1 = tvm.insertPubkey(_contract2, _key); + a1 = address(tvm.hash(s1)); + + a2 = a1; + while (_isSameShard(a2, a1)) { + _key++; + s2 = tvm.insertPubkey(_contract2, _key); + a2 = address(tvm.hash(s2)); + } + _DeployedArray.push(a1); + _WorkerHelps.push(a2); + _deployed++; + new Worker {stateInit:s1, value:_dgrant} (a2, _deployed); + new Worker {stateInit:s2, value:_dgrant} (a1, _deployed); + } + + function stopWorker() public alwaysAccept { + while (_deployed > _counter){ + Worker(_DeployedArray[_deployed - 1]).stop(); + Worker(_WorkerHelps[_deployed- 1]).stop(); + _deployed--; + } + } + + function NeedMoney(uint128 id) external override alwaysAccept { +// if (_DeployedArray[id - 1].balance() > 1e9){ +// return; +// } + if (id <= _deployed){ + _DeployedArray[id - 1].transfer(_giveMoney); + _WorkerHelps[id - 1].transfer(_giveMoney); + } + } + + + function build() public alwaysAccept { + _contract2 = tvm.buildStateInit(_code2, _data2); + } + + function init() public alwaysAccept { + _counter = 0; + _deployed = 0; + _key = rnd.next(3); + _contract2 = tvm.buildStateInit(_code2, _data2); + } + + + /* Setters */ + + function setNumber(uint128 number) public alwaysAccept { + _number = number; + } + + + function setCode2(TvmCell c) public alwaysAccept { + _code2 = c; + } + + function setData2(TvmCell c) public alwaysAccept { + _data2 = c; + } + + function setContract2(TvmCell c) public alwaysAccept { + _contract2 = c; + } + + function setKey(uint key) public alwaysAccept { + _key = key; + } + + function setdGrant(uint128 dgrant) public alwaysAccept { + _dgrant = dgrant; + } + + function setgMoney(uint128 money) public alwaysAccept { + _giveMoney = money; + } + + /* fallback/receive */ + receive() external { + + } + + function upgrade(TvmCell newcode) public { +// require(msg.pubkey() == tvm.pubkey(), 101); + tvm.accept(); + tvm.commit(); + tvm.setcode(newcode); + tvm.setCurrentCode(newcode); +// onCodeUpgrade(); + } + + function grant(address addr, uint128 value) external { + tvm.accept(); + addr.transfer(value, false, 3); + } +} diff --git a/tests/test_high_load/Brazil.tvc b/tests/test_high_load/Brazil.tvc new file mode 100644 index 00000000..95f1b080 Binary files /dev/null and b/tests/test_high_load/Brazil.tvc differ diff --git a/tests/test_high_load/SafeMultisigWallet.abi.json b/tests/test_high_load/SafeMultisigWallet.abi.json new file mode 100644 index 00000000..61f8e5cc --- /dev/null +++ b/tests/test_high_load/SafeMultisigWallet.abi.json @@ -0,0 +1,124 @@ +{ + "ABI version": 2, + "header": ["pubkey", "time", "expire"], + "functions": [ + { + "name": "constructor", + "inputs": [ + {"name":"owners","type":"uint256[]"}, + {"name":"reqConfirms","type":"uint8"} + ], + "outputs": [ + ] + }, + { + "name": "acceptTransfer", + "inputs": [ + {"name":"payload","type":"bytes"} + ], + "outputs": [ + ] + }, + { + "name": "sendTransaction", + "inputs": [ + {"name":"dest","type":"address"}, + {"name":"value","type":"uint128"}, + {"name":"bounce","type":"bool"}, + {"name":"flags","type":"uint8"}, + {"name":"payload","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "submitTransaction", + "inputs": [ + {"name":"dest","type":"address"}, + {"name":"value","type":"uint128"}, + {"name":"bounce","type":"bool"}, + {"name":"allBalance","type":"bool"}, + {"name":"payload","type":"cell"} + ], + "outputs": [ + {"name":"transId","type":"uint64"} + ] + }, + { + "name": "confirmTransaction", + "inputs": [ + {"name":"transactionId","type":"uint64"} + ], + "outputs": [ + ] + }, + { + "name": "isConfirmed", + "inputs": [ + {"name":"mask","type":"uint32"}, + {"name":"index","type":"uint8"} + ], + "outputs": [ + {"name":"confirmed","type":"bool"} + ] + }, + { + "name": "getParameters", + "inputs": [ + ], + "outputs": [ + {"name":"maxQueuedTransactions","type":"uint8"}, + {"name":"maxCustodianCount","type":"uint8"}, + {"name":"expirationTime","type":"uint64"}, + {"name":"minValue","type":"uint128"}, + {"name":"requiredTxnConfirms","type":"uint8"} + ] + }, + { + "name": "getTransaction", + "inputs": [ + {"name":"transactionId","type":"uint64"} + ], + "outputs": [ + {"components":[{"name":"id","type":"uint64"},{"name":"confirmationsMask","type":"uint32"},{"name":"signsRequired","type":"uint8"},{"name":"signsReceived","type":"uint8"},{"name":"creator","type":"uint256"},{"name":"index","type":"uint8"},{"name":"dest","type":"address"},{"name":"value","type":"uint128"},{"name":"sendFlags","type":"uint16"},{"name":"payload","type":"cell"},{"name":"bounce","type":"bool"}],"name":"trans","type":"tuple"} + ] + }, + { + "name": "getTransactions", + "inputs": [ + ], + "outputs": [ + {"components":[{"name":"id","type":"uint64"},{"name":"confirmationsMask","type":"uint32"},{"name":"signsRequired","type":"uint8"},{"name":"signsReceived","type":"uint8"},{"name":"creator","type":"uint256"},{"name":"index","type":"uint8"},{"name":"dest","type":"address"},{"name":"value","type":"uint128"},{"name":"sendFlags","type":"uint16"},{"name":"payload","type":"cell"},{"name":"bounce","type":"bool"}],"name":"transactions","type":"tuple[]"} + ] + }, + { + "name": "getTransactionIds", + "inputs": [ + ], + "outputs": [ + {"name":"ids","type":"uint64[]"} + ] + }, + { + "name": "getCustodians", + "inputs": [ + ], + "outputs": [ + {"components":[{"name":"index","type":"uint8"},{"name":"pubkey","type":"uint256"}],"name":"custodians","type":"tuple[]"} + ] + } + ], + "data": [ + ], + "events": [ + { + "name": "TransferAccepted", + "inputs": [ + {"name":"payload","type":"bytes"} + ], + "outputs": [ + ] + } + ] +} + diff --git a/tests/test_high_load/SetcodeMultisigWallet.abi.json b/tests/test_high_load/SetcodeMultisigWallet.abi.json new file mode 100644 index 00000000..60e8fb3e --- /dev/null +++ b/tests/test_high_load/SetcodeMultisigWallet.abi.json @@ -0,0 +1,160 @@ +{ + "ABI version": 2, + "header": ["pubkey", "time", "expire"], + "functions": [ + { + "name": "constructor", + "inputs": [ + {"name":"owners","type":"uint256[]"}, + {"name":"reqConfirms","type":"uint8"} + ], + "outputs": [ + ] + }, + { + "name": "acceptTransfer", + "inputs": [ + {"name":"payload","type":"bytes"} + ], + "outputs": [ + ] + }, + { + "name": "sendTransaction", + "inputs": [ + {"name":"dest","type":"address"}, + {"name":"value","type":"uint128"}, + {"name":"bounce","type":"bool"}, + {"name":"flags","type":"uint8"}, + {"name":"payload","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "submitTransaction", + "inputs": [ + {"name":"dest","type":"address"}, + {"name":"value","type":"uint128"}, + {"name":"bounce","type":"bool"}, + {"name":"allBalance","type":"bool"}, + {"name":"payload","type":"cell"} + ], + "outputs": [ + {"name":"transId","type":"uint64"} + ] + }, + { + "name": "confirmTransaction", + "inputs": [ + {"name":"transactionId","type":"uint64"} + ], + "outputs": [ + ] + }, + { + "name": "isConfirmed", + "inputs": [ + {"name":"mask","type":"uint32"}, + {"name":"index","type":"uint8"} + ], + "outputs": [ + {"name":"confirmed","type":"bool"} + ] + }, + { + "name": "getParameters", + "inputs": [ + ], + "outputs": [ + {"name":"maxQueuedTransactions","type":"uint8"}, + {"name":"maxCustodianCount","type":"uint8"}, + {"name":"expirationTime","type":"uint64"}, + {"name":"minValue","type":"uint128"}, + {"name":"requiredTxnConfirms","type":"uint8"}, + {"name":"requiredUpdConfirms","type":"uint8"} + ] + }, + { + "name": "getTransaction", + "inputs": [ + {"name":"transactionId","type":"uint64"} + ], + "outputs": [ + {"components":[{"name":"id","type":"uint64"},{"name":"confirmationsMask","type":"uint32"},{"name":"signsRequired","type":"uint8"},{"name":"signsReceived","type":"uint8"},{"name":"creator","type":"uint256"},{"name":"index","type":"uint8"},{"name":"dest","type":"address"},{"name":"value","type":"uint128"},{"name":"sendFlags","type":"uint16"},{"name":"payload","type":"cell"},{"name":"bounce","type":"bool"}],"name":"trans","type":"tuple"} + ] + }, + { + "name": "getTransactions", + "inputs": [ + ], + "outputs": [ + {"components":[{"name":"id","type":"uint64"},{"name":"confirmationsMask","type":"uint32"},{"name":"signsRequired","type":"uint8"},{"name":"signsReceived","type":"uint8"},{"name":"creator","type":"uint256"},{"name":"index","type":"uint8"},{"name":"dest","type":"address"},{"name":"value","type":"uint128"},{"name":"sendFlags","type":"uint16"},{"name":"payload","type":"cell"},{"name":"bounce","type":"bool"}],"name":"transactions","type":"tuple[]"} + ] + }, + { + "name": "getTransactionIds", + "inputs": [ + ], + "outputs": [ + {"name":"ids","type":"uint64[]"} + ] + }, + { + "name": "getCustodians", + "inputs": [ + ], + "outputs": [ + {"components":[{"name":"index","type":"uint8"},{"name":"pubkey","type":"uint256"}],"name":"custodians","type":"tuple[]"} + ] + }, + { + "name": "submitUpdate", + "inputs": [ + {"name":"codeHash","type":"uint256"}, + {"name":"owners","type":"uint256[]"}, + {"name":"reqConfirms","type":"uint8"} + ], + "outputs": [ + {"name":"updateId","type":"uint64"} + ] + }, + { + "name": "confirmUpdate", + "inputs": [ + {"name":"updateId","type":"uint64"} + ], + "outputs": [ + ] + }, + { + "name": "executeUpdate", + "inputs": [ + {"name":"updateId","type":"uint64"}, + {"name":"code","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "getUpdateRequests", + "inputs": [ + ], + "outputs": [ + {"components":[{"name":"id","type":"uint64"},{"name":"index","type":"uint8"},{"name":"signs","type":"uint8"},{"name":"confirmationsMask","type":"uint32"},{"name":"creator","type":"uint256"},{"name":"codeHash","type":"uint256"},{"name":"custodians","type":"uint256[]"},{"name":"reqConfirms","type":"uint8"}],"name":"updates","type":"tuple[]"} + ] + } + ], + "data": [ + ], + "events": [ + { + "name": "TransferAccepted", + "inputs": [ + {"name":"payload","type":"bytes"} + ], + "outputs": [ + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_high_load/addrBrazil b/tests/test_high_load/addrBrazil new file mode 100644 index 00000000..b1cce052 --- /dev/null +++ b/tests/test_high_load/addrBrazil @@ -0,0 +1,3 @@ +0:2fbb0d78c8428f1770f19b622a8e2426daeb347d7b3f26d4e1e2888c55b6b1c2 +0:95235ccbcc3dce4e5cb2e748fccd6745a4271ed325df7364c8778b0e8bd597e2 +0:4a5ed5323d510dedb4cd8b6bf322b89b73eaec0b359ce46e3557c46499f2d64c diff --git a/tests/test_high_load/deploy.keys.2.json b/tests/test_high_load/deploy.keys.2.json new file mode 100644 index 00000000..4d38f6c1 --- /dev/null +++ b/tests/test_high_load/deploy.keys.2.json @@ -0,0 +1,4 @@ +{ + "public": "553f9351ca08417c8c338413c2fd4b30eb7f6d35694e3e1b8ceb3d5729a34520", + "secret": "4211ad17d7ae93c0e9df877bff1bebea86d2ea4aca39309a967341c5e2c854a4" + } \ No newline at end of file diff --git a/tests/test_high_load/deploy.keys.json b/tests/test_high_load/deploy.keys.json new file mode 100644 index 00000000..2fedf1d4 --- /dev/null +++ b/tests/test_high_load/deploy.keys.json @@ -0,0 +1,4 @@ +{ + "public": "21e0e3a0302b3d6b3e70802ca20af6b9901e6007afcd3087e16db1474c2dcf62", + "secret": "69b0b784c01a4eec1d5edaa334d741ad4e9f590bcecdf5f30743fc19cd6e0ddf" +} \ No newline at end of file diff --git a/tests/test_high_load/msig.keys.json b/tests/test_high_load/msig.keys.json new file mode 100644 index 00000000..b4453769 --- /dev/null +++ b/tests/test_high_load/msig.keys.json @@ -0,0 +1,4 @@ +{ + "public": "c86b504dbbcf2263c6d5985743f1b248eca31d9ae37d273102ddd2b6ccd95c8a", + "secret": "9e8148716bca7aa1c33266eedcbaa56c3d00b6f58caaa78561bab54037f717e8" + } \ No newline at end of file diff --git a/tests/test_high_load/remove_junk.sh b/tests/test_high_load/remove_junk.sh new file mode 100755 index 00000000..7aaabfb4 --- /dev/null +++ b/tests/test_high_load/remove_junk.sh @@ -0,0 +1 @@ +rm msg*.boc diff --git a/tests/test_high_load/seedphrase b/tests/test_high_load/seedphrase new file mode 100644 index 00000000..6a52d1f6 --- /dev/null +++ b/tests/test_high_load/seedphrase @@ -0,0 +1,3 @@ +Config: /shared/Brazil/tonos-cli/target/release/tonos-cli.conf.json +Succeeded. +Seed phrase: "rifle normal planet still modify broom unique axis marine gallery hurry parent" diff --git a/tests/test_high_load/test_console.sh b/tests/test_high_load/test_console.sh new file mode 100755 index 00000000..3177e1a4 --- /dev/null +++ b/tests/test_high_load/test_console.sh @@ -0,0 +1,2 @@ +source ../setup_test_paths.sh +$TOOLS_DIR/console -C ${CONSOLE_CONFIG_0} -c "getstats" diff --git a/tests/test_high_load/test_high_load.sh b/tests/test_high_load/test_high_load.sh new file mode 100755 index 00000000..6421a827 --- /dev/null +++ b/tests/test_high_load/test_high_load.sh @@ -0,0 +1,169 @@ +TOOLS=../../../ton-labs-node-tools/target/release/ +RUN_TEST_ROOT=../test_run_net/ +TEST_ROOT=$(pwd) +DELAY=20 + +run net +cd $RUN_TEST_ROOT +if ! $RUN_TEST_ROOT/test_run_net.sh +then + exit 1 +fi + +cd ../../../ +if ! [ -d "tonos-cli" ] +then + git clone "git@github.com:tonlabs/tonos-cli.git" +fi +cd tonos-cli +cargo build --release +cd target/release +CLI=$(pwd) + +cd $TEST_ROOT + +echo "Preparation (0/6) deploing giver" +$CLI/tonos-cli \ + message "-1:7777777777777777777777777777777777777777777777777777777777777777" \ + constructor '{ + "owners":["0xc86b504dbbcf2263c6d5985743f1b248eca31d9ae37d273102ddd2b6ccd95c8a"], + "reqConfirms":1 + }' \ + --abi SetcodeMultisigWallet.abi.json \ + --sign deploy.keys.2.json \ + --output "$TEST_ROOT/msg0.boc" \ + --raw +$TOOLS/console -C $RUN_TEST_ROOT/console1.json -c "sendmessage $TEST_ROOT/msg0.boc" +echo " waiting for $DELAY sec" +sleep $DELAY + +echo "Preparation (1/6) sending money to Brazil contract" +$CLI/tonos-cli \ + message "-1:7777777777777777777777777777777777777777777777777777777777777777" \ + submitTransaction '{ + "dest" : "0:4a5ed5323d510dedb4cd8b6bf322b89b73eaec0b359ce46e3557c46499f2d64c", + "value":"3000000000000000", + "bounce":false, + "allBalance":false, + "payload":"" + }' \ + --abi SafeMultisigWallet.abi.json \ + --sign msig.keys.json \ + --output "$TEST_ROOT/msg1.boc" \ + --raw +$TOOLS/console -C $RUN_TEST_ROOT/console1.json -c "sendmessage $TEST_ROOT/msg1.boc" +echo " waiting for $DELAY sec" +sleep $DELAY + +echo "Preparation (2/6) deploing Brazil contract" +$CLI/tonos-cli \ + deploy_message \ + Brazil.tvc \ + --abi Brazil.abi.json \ + --sign deploy.keys.json {} \ + --raw \ + --output "$TEST_ROOT/msg2.boc" +$TOOLS/console -C $RUN_TEST_ROOT/console1.json -c "sendmessage $TEST_ROOT/msg2.boc" +echo " waiting for $DELAY sec" +sleep $DELAY + +echo "Preparation (3/6) setting code" +$CLI/tonos-cli \ + message '0:4a5ed5323d510dedb4cd8b6bf322b89b73eaec0b359ce46e3557c46499f2d64c' --abi Brazil.abi.json \ + setCode2 '{"c":"te6ccgECGgEABBUAAib/APSkICLAAZL0oOGK7VNYMPShBwEBCvSkIPShAgIDzsAGAwIBSAUEAFU7UTQ0//TP9MA1fpA+G74bNJ/+kDTf9cLf/hv+G34a/hqf/hh+Gb4Y/higAFs+ELIy//4Q88LP/hGzwsAyPhM+E4Czs74SvhL+E34T15AzxHKf87Lf8t/ye1UgAPNn4SpLbMOH4J28QghA7msoAuY5A+E+ktX/4b/hPwg+OMnD4b/hOyM+FiM6NBA5iWgAAAAAAAAAAAAAAAAABzxbPgc+Bz5EVvcFC+E3PC3/JcfsA3t74S8jPhQjOjQRQFXUqAAAAAAAAAAAAAAAAAAHPFs+Bz4HJc/sAgIBIAoIAbT/f40IYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABPhpIe1E0CDXScIBjifT/9M/0wDV+kD4bvhs0n/6QNN/1wt/+G/4bfhr+Gp/+GH4Zvhj+GIJAbKOgOLTAAGfgQIA1xgg+QFY+EL5EPKo3tM/AY4e+EMhuSCfMCD4I4ED6KiCCBt3QKC53pL4Y+CANPI02NMfIcEDIoIQ/////byxk1vyPOAB8AH4R26TMPI83hECASATCwIBIA0MACm6YBoLL4QW6S8AXe0XD4avAEf/hngBD7oh5WFfhBboDgFsjoDe+Ebyc3H4ZvpA1w1/ldTR0NN/39Eg+G34ACH4a/hJ+G5wkyDBCpXwAaS1f+gwW/AEf/hnDwFi7UTQINdJwgGOJ9P/0z/TANX6QPhu+GzSf/pA03/XC3/4b/ht+Gv4an/4Yfhm+GP4YhABBo6A4hEB/vQFcPhqjQhgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAE+GuNCGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT4bHD4bY0IYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABPhucPhvcAESADSAQPQO8r3XC//4YnD4Y3D4Zn/4YXH4anD4bwIBIBcUARm7eoGUD4QW6S8AXe0YFQH+jnn4SpLbMOH4J28QghA7msoAuY5A+E+ktX/4b/hPwg+OMnD4b/hOyM+FiM6NBA5iWgAAAAAAAAAAAAAAAAABzxbPgc+Bz5EVvcFC+E3PC3/JcfsA3t74S8jPhQjOjQRQFXUqAAAAAAAAAAAAAAAAAAHPFs+Bz4HJc/sA2PAEfxYABPhnAgFiGRgAQ7TOBrf8ILdJeALvaLj8NThJkGCFSvgA0lq/9Bh4Aj/8M8AAotpwItDTA/pAMPhpqTgA3CHHACCcMCHTHyHAACCSbCHe344RcfAB8AX4SfhLxwWS8AHe8ATgIcEDIoIQ/////byxk1vyPOAB8AH4R26TMPI83g=="}' \ + --output "$TEST_ROOT/msg3.boc" \ + --raw +$TOOLS/console -C $RUN_TEST_ROOT/console1.json -c "sendmessage $TEST_ROOT/msg3.boc" +echo " waiting for $DELAY sec" +sleep $DELAY + +echo "Preparation (4/6) setting data" +$CLI/tonos-cli \ + message '0:4a5ed5323d510dedb4cd8b6bf322b89b73eaec0b359ce46e3557c46499f2d64c' --abi Brazil.abi.json \ + setData2 '{"c":"te6ccgEBAgEAKAABAcABAEPQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAg"}' \ + --output "$TEST_ROOT/msg4.boc" \ + --raw +$TOOLS/console -C $RUN_TEST_ROOT/console1.json -c "sendmessage $TEST_ROOT/msg4.boc" +echo " waiting for $DELAY sec" +sleep $DELAY + +echo "Preparation (5/6) building" +$CLI/tonos-cli \ + message '0:4a5ed5323d510dedb4cd8b6bf322b89b73eaec0b359ce46e3557c46499f2d64c' --abi Brazil.abi.json \ + build '{}' \ + --output "$TEST_ROOT/msg5.boc" \ + --raw +$TOOLS/console -C $RUN_TEST_ROOT/console1.json -c "sendmessage $TEST_ROOT/msg5.boc" +echo " waiting for $DELAY sec" +sleep $DELAY + +echo "Preparation (6/6) setting grant" +$CLI/tonos-cli \ + message '0:4a5ed5323d510dedb4cd8b6bf322b89b73eaec0b359ce46e3557c46499f2d64c' --abi Brazil.abi.json \ + setdGrant '{"dgrant":3000000000}' \ + --output "$TEST_ROOT/msg6.boc" \ + --raw +$TOOLS/console -C $RUN_TEST_ROOT/console1.json -c "sendmessage $TEST_ROOT/msg6.boc" +echo " waiting for $DELAY sec" +sleep $DELAY + +echo "Starting load..." +COUNTER=20 +N=0 +MAX=30 +DELAY=10 +while [ $N -lt $(($MAX + 1)) ] +do + echo "($N/$MAX) senging message" + $CLI/tonos-cli \ + message '0:4a5ed5323d510dedb4cd8b6bf322b89b73eaec0b359ce46e3557c46499f2d64c' \ + --abi Brazil.abi.json \ + setC {\"counter\":$COUNTER} \ + --output "$TEST_ROOT/msg.boc" \ + --raw + $TOOLS/console -C $RUN_TEST_ROOT/console1.json -c "sendmessage $TEST_ROOT/msg.boc" + sleep $DELAY + (( N++ )) + COUNTER=$(($COUNTER + 20)) +done + +HILOAD_DURATION=30 +echo "Working $HILOAD_DURATION mins under hiload" +N=0 +while [ $N -lt $HILOAD_DURATION ] +do + sleep 60 + (( N++ )) + tail -n 1000 "/shared/output_1.log" | grep COLLATED + echo "Working $HILOAD_DURATION mins under hiload ($N mins done)" +done + +echo "Checking if the blockchain is alive" + +function find_new_block_of_shard { + N=1 + while [ $N -lt 6 ] + do + if tail -n 2000 "/shared/output_$N.log" | grep -q "Applied block ($1" + then + echo "New applied block of shard ($1) - FOUND on node #$N!" + else + echo "ERROR: Can't find new applied block of shard ($1) on node #$N!" + exit 1 + fi + (( N++ )) + done +} + +find_new_block_of_shard "0\:1000000000000000" +find_new_block_of_shard "0\:3000000000000000" +find_new_block_of_shard "0\:5000000000000000" +find_new_block_of_shard "0\:7000000000000000" +find_new_block_of_shard "0\:9000000000000000" +find_new_block_of_shard "0\:b000000000000000" +find_new_block_of_shard "0\:d000000000000000" +find_new_block_of_shard "0\:f000000000000000" + +echo "TEST PASSED. Killing nodes" + +pkill ton_node diff --git a/tests/test_high_load/test_one_message.sh b/tests/test_high_load/test_one_message.sh new file mode 100755 index 00000000..ce7a4e73 --- /dev/null +++ b/tests/test_high_load/test_one_message.sh @@ -0,0 +1,66 @@ +source ../setup_test_paths.sh + +RUN_TEST_ROOT=../test_run_net/ +TEST_ROOT=$(pwd) +mkdir logs +export ROOT=$(pwd)/logs +export REMP_TEST=true +DELAY=20 + +cd $RUN_TEST_ROOT +if ! $RUN_TEST_ROOT/test_run_net.sh +then + exit 1 +fi + +cd ../../../ +if ! [ -d "tonos-cli" ] +then + git clone "git@github.com:tonlabs/tonos-cli.git" +fi +cd tonos-cli +cargo build --release +cd target/release +CLI=$(pwd) + +cd $TEST_ROOT + +echo "Press ENTER to start transactions" +read -r input + +echo "Preparation (0/6) deploing giver" +$CLI/tonos-cli \ + message "-1:7777777777777777777777777777777777777777777777777777777777777777" \ + constructor '{ + "owners":["0xc86b504dbbcf2263c6d5985743f1b248eca31d9ae37d273102ddd2b6ccd95c8a"], + "reqConfirms":1 + }' \ + --abi SetcodeMultisigWallet.abi.json \ + --sign deploy.keys.2.json \ + --output "$TEST_ROOT/msg0.boc" \ + --raw + +echo $TOOLS_DIR/console -C ${CONSOLE_CONFIG_0} -c "sendmessage $TEST_ROOT/msg0.boc" +$TOOLS_DIR/console -C ${CONSOLE_CONFIG_0} -c "sendmessage $TEST_ROOT/msg0.boc" + +while true; do + echo "Enter amount of money to transfer: " + read -r input + echo "Preparation (1/6) sending money to Brazil contract" + $CLI/tonos-cli \ + message "-1:7777777777777777777777777777777777777777777777777777777777777777" \ + submitTransaction "{ + \"dest\" : \"0:4a5ed5323d510dedb4cd8b6bf322b89b73eaec0b359ce46e3557c46499f2d64c\", + \"value\":\"$input\", + \"bounce\":false, + \"allBalance\":false, + \"payload\":\"\" + }" \ + --abi SafeMultisigWallet.abi.json \ + --sign msig.keys.json \ + --output "$TEST_ROOT/msg1.boc" \ + --raw + $TOOLS_DIR/console -C ${CONSOLE_CONFIG_0} -c "sendmessage $TEST_ROOT/msg1.boc" +done + +pkill ton_node diff --git a/tests/test_remp/.gitignore b/tests/test_remp/.gitignore new file mode 100644 index 00000000..205f950d --- /dev/null +++ b/tests/test_remp/.gitignore @@ -0,0 +1,6 @@ +*.log +*__address +*.boc +*.tvc +*.json +*.code \ No newline at end of file diff --git a/tests/test_remp/no_replay.sol b/tests/test_remp/no_replay.sol new file mode 100644 index 00000000..f0c628d6 --- /dev/null +++ b/tests/test_remp/no_replay.sol @@ -0,0 +1,27 @@ +pragma ton-solidity >= 0.55.0; +//pragma AbiHeader time; + +// This contract demonstrates custom replay protection functionality. +contract CustomReplaySample { + + uint public value; + + function addValue(uint num) public { + tvm.accept(); + value += num; + } + + function getValue() public view returns (uint) { + return value; + } + + // Function with predefined name which is used to replace custom replay protection. + function afterSignatureCheck(TvmSlice body, TvmCell) private pure inline returns (TvmSlice) { + // Via TvmSlice methods we read header fields from the message body + + body.decode(uint64); // The first 64 bits contain timestamp which is usually used to differentiate messages. + + // After reading message headers this function must return the rest of the body slice. + return body; + } +} \ No newline at end of file diff --git a/tests/test_remp/test_remp.sh b/tests/test_remp/test_remp.sh new file mode 100755 index 00000000..2ea5c62b --- /dev/null +++ b/tests/test_remp/test_remp.sh @@ -0,0 +1,422 @@ +source "../setup_test_paths.sh" + +contracts_count=20 +messages_count=50 +timeout_sec=0.5 +prime_numbers=( + 1091 1093 1097 1103 1109 1117 1123 1129 1151 1153 1163 1171 1181 1187 1193 1201 1213 1217 1223 + 1231 1237 1249 1259 1277 1279 1283 1289 1291 1297 1301 1303 1307 1319 1321 1327 1361 1367 1373 + 1399 1409 1423 1427 1429 1433 1439 1447 1451 1453 1459 1471 1481 1483 1487 1489 1493 1499 1511 + 1531 1543 1549 1553 1559 1567 1571 1579 1583 1597 1601 1607 1609 1613 1619 1621 1627 1637 1657 + 1667 1669 1693 1697 1699 1709 1721 1723 1733 1741 1747 1753 1759 1777 1783 1787 1789 1801 1811 + 1831 1847 1861 1867 1871 1873 1877 1879 1889 1901 1907 1913 1931 1933 1949 1951 1973 1979 1987 + 1997 1999 2003 2011 2017 2027 2029 2039 2053 2063 2069 2081 2083 2087 2089 2099 2111 2113 2129 + 2137 2141 2143 2153 2161 2179 2203 2207 2213 2221 2237 2239 2243 2251 2267 2269 2273 2281 2287 + 2297 2309 2311 2333 2339 2341 2347 2351 2357 2371 2377 2381 2383 2389 2393 2399 2411 2417 2423 + 2441 2447 2459 2467 2473 2477 2503 2521 2531 2539 2543 2549 2551 2557 2579 2591 2593 2609 2617 + 2633 2647 2657 2659 2663 2671 2677 2683 2687 2689 2693 2699 2707 2711 2713 2719 2729 2731 2741 + 2753 2767 2777 2789 2791 2797 2801 2803 2819 2833 2837 2843 2851 2857 2861 2879 2887 2897 2903 + 2917 2927 2939 2953 2957 2963 2969 2971 2999 3001 3011 3019 3023 3037 3041 3049 3061 3067 3079 + 3089 3109 3119 3121 3137 3163 3167 3169 3181 3187 3191 3203 3209 3217 3221 3229 3251 3253 3257 + 3271 3299 3301 3307 3313 3319 3323 3329 3331 3343 3347 3359 3361 3371 3373 3389 3391 3407 3413 + 3449 3457 3461 3463 3467 3469 3491 3499 3511 3517 3527 3529 3533 3539 3541 3547 3557 3559 3571 + 3583 3593 3607 3613 3617 3623 3631 3637 3643 3659 3671 3673 3677 3691 3697 3701 3709 3719 3727 + 3739 3761 3767 3769 3779 3793 3797 3803 3821 3823 3833 3847 3851 3853 3863 3877 3881 3889 3907 + 3917 3919 3923 3929 3931 3943 3947 3967 3989 4001 4003 4007 4013 4019 4021 4027 4049 4051 4057 + 4079 4091 4093 4099 4111 4127 4129 4133 4139 4153 4157 4159 4177 4201 4211 4217 4219 4229 4231 + 4243 4253 4259 4261 4271 4273 4283 4289 4297 4327 4337 4339 4349 4357 4363 4373 4391 4397 4409 + 4423 4441 4447 4451 4457 4463 4481 4483 4493 4507 4513 4517 4519 4523 4547 4549 4561 4567 4583 + 4597 4603 4621 4637 4639 4643 4649 4651 4657 4663 4673 4679 4691 4703 4721 4723 4729 4733 4751 + 4783 4787 4789 4793 4799 4801 4813 4817 4831 4861 4871 4877 4889 4903 4909 4919 4931 4933 4937 + 4951 4957 4967 4969 4973 4987 4993 4999 5003 5009 5011 5021 5023 5039 5051 5059 5077 5081 5087 + 5101 5107 5113 5119 5147 5153 5167 5171 5179 5189 5197 5209 5227 5231 5233 5237 5261 5273 5279 + 5297 5303 5309 5323 5333 5347 5351 5381 5387 5393 5399 5407 5413 5417 5419 5431 5437 5441 5443 + 5471 5477 5479 5483 5501 5503 5507 5519 5521 5527 5531 5557 5563 5569 5573 5581 5591 5623 5639 + 5647 5651 5653 5657 5659 5669 5683 5689 5693 5701 5711 5717 5737 5741 5743 5749 5779 5783 5791 + 5807 5813 5821 5827 5839 5843 5849 5851 5857 5861 5867 5869 5879 5881 5897 5903 5923 5927 5939 + 5981 5987 6007 6011 6029 6037 6043 6047 6053 6067 6073 6079 6089 6091 6101 6113 6121 6131 6133 + 6151 6163 6173 6197 6199 6203 6211 6217 6221 6229 6247 6257 6263 6269 6271 6277 6287 6299 6301 + 6317 6323 6329 6337 6343 6353 6359 6361 6367 6373 6379 6389 6397 6421 6427 6449 6451 6469 6473 + 6491 6521 6529 6547 6551 6553 6563 6569 6571 6577 6581 6599 6607 6619 6637 6653 6659 6661 6673 + 6689 6691 6701 6703 6709 6719 6733 6737 6761 6763 6779 6781 6791 6793 6803 6823 6827 6829 6833 + 6857 6863 6869 6871 6883 6899 6907 6911 6917 6947 6949 6959 6961 6967 6971 6977 6983 6991 6997 + 7013 7019 7027 7039 7043 7057 7069 7079 7103 7109 7121 7127 7129 7151 7159 7177 7187 7193 7207 + 7213 7219 7229 7237 7243 7247 7253 7283 7297 7307 7309 7321 7331 7333 7349 7351 7369 7393 7411 + 7433 7451 7457 7459 7477 7481 7487 7489 7499 7507 7517 7523 7529 7537 7541 7547 7549 7559 7561 + 7577 7583 7589 7591 7603 7607 7621 7639 7643 7649 7669 7673 7681 7687 7691 7699 7703 7717 7723 + 7727 7741 7753 7757 7759 7789 7793 7817 7823 7829 7841 7853 7867 7873 7877 7879 7883 7901 7907 +) +sold=${BASE_DIR}/TVM-Solidity-Compiler/target/release/sold +tvm_linker=${BASE_DIR}/TVM-linker/target/release/tvm_linker +lib="${BASE_DIR}/TVM-Solidity-Compiler/lib" +tonos_cli="${BASE_DIR}/tonos-cli/target/release/tonos-cli" +giver_keys=${TESTS_DIR}/test_high_load/msig.keys.json +giver_abi=${TESTS_DIR}/test_high_load/SafeMultisigWallet.abi.json +console=${TOOLS_DIR}/console +console_config=${CONSOLE_CONFIG_0} +contract_src="no_replay.sol" +abi_file="${contract_src%.sol}.abi.json" +node_log="/shared/output_0.log" + +# $1 number of contract $2 tvc_file +function pre_deploy { + local n=$1 + local tvc_file=$2 + local msg_boc="msg.$n.boc" + + # Generate address, keys and seed phrase + echo "Generating address, keys and seed phrase..." + local keys_file="${contract_src%.sol}.$n.keys.json" + if ! output="$($tonos_cli genaddr $tvc_file --abi $abi_file --genkey $keys_file)" ; then + echo "ERROR: $output" + return 1 + fi + local address=$(echo "$output" | grep "Raw address" | cut -c 14-) + echo $address > "${n}__address" + + # Send money to contract's address + # prepare message + echo "Preparing message to send money..." + if ! output="$( $tonos_cli \ + message "-1:7777777777777777777777777777777777777777777777777777777777777777" \ + submitTransaction '{ + "dest" : "'$address'", + "value":"10000000000", + "bounce":false, + "allBalance":false, + "payload":"" + }' \ + --abi "$giver_abi" \ + --sign "$giver_keys" \ + --output "$msg_boc" \ + --raw )" + then + echo "ERROR: $output" + return 1 + fi + local message_id=$( echo "$output" | grep "MessageId" | cut -c 12- ) + echo "Message id: " $message_id + # send it + echo "Sending money to $address..." + if ! output="$($console -C $console_config -c "sendmessage $msg_boc")" ; then + echo "ERROR: $output" + return 1 + fi + look_after_message $message_id & + sleep 20 + + # Ask contract's status to check have it got money + for (( attempt=0; attempt < 10; attempt++ )); do + echo "Ask contract's status to check have it got money..." + if ! output="$( $console -C $console_config -c "getaccount $address" )" ; then + echo "ERROR: $output" + return 1 + fi + # echo "$output" + if [ $(echo "$output" | grep "Uninit" | wc -l) != "1" ] ; then + if [ $attempt -eq 9 ] ; then + echo "ERROR can't find uninit account with money." + return 1 + else + echo "WARN can't find uninit account with money." + fi + else + break + fi + sleep 5 + done + + echo "$address successfully got money to future deploy." +} + +# $1 number of contract $2 tvc_file $3 address +function deploy { + + local n=$1 + local tvc_file=$2 + local address=$3 + local msg_boc="msg.$n.boc" + local keys_file="${contract_src%.sol}.$n.keys.json" + + # Deploy contract + # prepare message + echo "Preparing message to deploy contract..." + if ! output="$( $tonos_cli \ + deploy_message "$tvc_file" \ + --abi "$abi_file" \ + --sign "$keys_file" {} \ + --raw \ + --output "$msg_boc" )" + then + echo "ERROR: $output" + return 1 + fi + local message_id=$( echo "$output" | grep "MessageId" | cut -c 12- ) + echo "Message id: " $message_id + # send it + echo "Deploying $address..." + if ! output="$( $console -C $console_config -c "sendmessage $msg_boc" )" ; then + echo "ERROR: $output" + return 1 + fi + look_after_message $message_id & + sleep 20 + + # Ask contract's status to check have it been deployed + echo "Ask contract's status to check have it been deployed..." + for (( attempt=0; attempt < 10; attempt++ )); do + echo "Ask contract's status to check have it got money..." + if ! output="$( $console -C $console_config -c "getaccount $address" )" ; then + echo "ERROR: $output" + return 1 + fi + echo "$output" + if [ $(echo "$output" | grep "Active" | wc -l) != "1" ] ; then + if [ $attempt -eq 9 ] ; then + echo "ERROR can't find uninit account with money." + return 1 + else + echo "WARN can't find uninit account with money." + fi + else + break + fi + sleep 5 + done + + echo "$address successfully deployed." +} + +# $1 message_id +function look_after_message { + local message_id=$1 + + for (( attempt=0; attempt < 25; attempt++ )); do + sleep 10 + + local output="$( cat $node_log | grep "New processing stage for external message $message_id" )" + if [ $(echo "$output" | grep "TonNode_RempAccepted(RempAccepted { level: TonNode_RempMasterchain" | wc -l) != "1" ] ; then + if [ $attempt -eq 24 ] ; then + echo "ERROR message $message_id was not accepted yet! Current statuses: " + echo "$output" + return 1 + else + echo "WARN message $message_id was not accepted yet! Current statuses: " + echo "$output" + fi + else + echo "Message $message_id was accepted! Statuses: " + echo "$output" + break + fi + done +} + +# $1 address $2 keys $3 number of message +function send_message { + local address=$1 + local keys_file=$2 + local message_number=$3 + local abi_file="${contract_src%.sol}.abi.json" + local msg_boc="msg.$n.boc" + local n=${prime_numbers[$message_number]} + + # prepare message + echo "Preparing message..." + if ! output="$( $tonos_cli \ + message "$address" \ + addValue '{ "num": '$n'}' \ + --abi "$abi_file" \ + --sign "$keys_file" \ + --output "$msg_boc" \ + --raw )" + then + echo "ERROR: $output" + return 1 + fi + local message_id=$( echo "$output" | grep "MessageId" | cut -c 12- ) + echo "Message id: " $message_id + + # send it + echo "Send to $address..." + if ! output="$($console -C $console_config -c "sendmessage $msg_boc")" ; then + echo "ERROR: $output" + return 1 + fi + + look_after_message $message_id & +} + +# 1 address $2 number of contract +function check_contract { + local address=$1 + local n=$2 + + # Calculate expected value + local expacted_sum=0 + for (( n=0; n < messages_count; n++ )); do + let "expacted_sum = expacted_sum + ${prime_numbers[$n]}" + done + + # local call (get method) + for (( attempt=0; attempt < 10; attempt++ )); do + echo "Obtaining contract's boc..." + if ! output="$( $console -C $console_config -c "getaccountstate $address $n.boc" )" ; then + echo "ERROR: $output" + return 1 + fi + echo "Calling contract's method locally..." + if ! output="$( $tonos_cli \ + run $n.boc \ + --abi "$abi_file" \ + --boc \ + getValue '{ }' )" + then + echo "ERROR $output" + return 1 + fi + # echo "$output" + + local hex=$(printf "%x" $expacted_sum) + if [ $(echo "$output" | grep $hex | wc -l) != "1" ] ; then + echo "$output" + echo "expacted_sum: "$expacted_sum" hex: "$hex + if [ $attempt -eq 9 ] ; then + echo "ERROR contract's value != expected one." + return 1 + else + echo "WARN contract's value != expected one." + fi + else + break + fi + sleep 10 + done +} + +# $1 number of contract $2 tvc_file +function one_contract_scenario { + local n=$1 + local tvc_file=$2 + local keys_file="${contract_src%.sol}.$n.keys.json" + local address=$(cat ${n}__address) + + if ! deploy $n $tvc_file $address ; then + return 1 + fi + + for (( mn=0; mn < messages_count; mn++ )); do + # $1 address $2 keys $3 number of message + if ! send_message $address $keys_file $mn ; then + return 1 + fi + sleep $timeout_sec + done + + echo "waiting..." + sleep 30 + + if ! check_contract $address $n ; then + return 1 + fi + + echo "$address succesfully finished their scenario!" + + exit 0 +} + +function cleanup { + rm *.log + rm *.boc + rm *__address + rm *.json + rm *.code + rm *.tvc +} + +# +# TEST REMP +# Pre deploy: send money to contracts (one by one) +# In parallel: +# - deploy contract +# - send messages - each message contains next prime number, +# contract stores sum of all got numbers. +# - check contract state - check if all messages was delivered exactly ones. +# We will check if contract's sum is equal to sum of sent numbers. + +cleanup + +if [ $messages_count -gt ${#prime_numbers[*]} ]; then + echo "messages_count value is too big, max=${#prime_numbers[*]}" + exit 1 +fi + +# Don't forget to deploy giver first! + +# compile & link test contract +echo "Compile..." +if ! $sold $contract_src ; then exit 1 ; fi +#code_file="${contract_src%.sol}.tvc" + +#echo "Linking..." +#if ! output="$($tvm_linker compile $code_file --lib $lib/stdlib_sol.tvm)" ; then +# echo "ERROR $output" +# exit 1 +#fi +#tvc_file=$(echo "$output" | grep "Saved contract to file" | cut -c 24-) +tvc_file="${contract_src%.sol}.tvc" + +# +echo "Obtaining blockchain's config..." +if ! output="$( $console -C $console_config -c "getblockchainconfig" )" ; then + echo "ERROR $output" + exit 1 +fi +echo $output | grep b5ee9c72 | xxd -r -p > config.boc + +# +echo "Send money to contracts (one by one)" +rm pre_deploy.log +for (( n=0; n < contracts_count; n++ )); do + echo "Sending money to #$n..." + echo " + + + Sending money to #$n... + + " >> pre_deploy.log + if ! pre_deploy $n $tvc_file >> pre_deploy.log ; then + # if ! output="$( pre_deploy $n $tvc_file )" ; then + # echo "ERROR $output" + exit 1 + else + echo "Sent money to #$n" + fi +done + +# run test scenario +echo "Run test scenario..." +for (( n=0; n < contracts_count; n++ )); do + one_contract_scenario $n $tvc_file > $n.log & +done + +wait + +# check log files of each contract +result=0 +for (( n=0; n < contracts_count; n++ )); do + if err="$( cat $n.log | grep ERROR )"; then + echo "Contract #$n has errors in their test scenario" + echo err + echo + result=1 + else + echo "$n succesfully finished their scenario!" + fi +done + +if [ $result -eq 0 ]; then + echo "Succesfully finished!" +fi + +exit $result diff --git a/tests/test_run_net/test_run_net.sh b/tests/test_run_net/test_run_net.sh old mode 100644 new mode 100755 index 8ae7827e..630f2846 --- a/tests/test_run_net/test_run_net.sh +++ b/tests/test_run_net/test_run_net.sh @@ -1,6 +1,6 @@ #!/bin/bash -NODES=6 +NODES=3 WORKCHAINS=false REMP_TEST=false CURRENT_BRANCH="$(git branch --show-current)" @@ -39,6 +39,27 @@ then exit 1 fi +cd ../../../ +if ! [ -d "ever-node-tools" ] +then + git clone "https://github.com/tonlabs/ever-node-tools" + cd ever-node-tools + git checkout "$CURRENT_BRANCH" || echo "Use default branch" + git submodule init + git submodule update +else + cd ever-node-tools +fi +TOOLS_ROOT=$(pwd) + +# cargo update +echo "Building $(pwd)" +if ! cargo build --release +then + exit 1 +fi +cd target/release/ + cd $TEST_ROOT NOWDATE=$(date +"%s") # NOWIP=$(curl ifconfig.me) diff --git a/tests/test_run_net/zero_state_blanc_1.json b/tests/test_run_net/zero_state_blanc_1.json index 61d536ec..bde11397 100644 --- a/tests/test_run_net/zero_state_blanc_1.json +++ b/tests/test_run_net/zero_state_blanc_1.json @@ -128,7 +128,7 @@ "p16": { "max_validators": 1000, "max_main_validators": 100, - "min_validators": 5 + "min_validators": 3 }, "p17": { "min_stake": "10000000000000", @@ -248,6 +248,6 @@ "utime_since": nowdate, "utime_until": 2000242087, "total": p34_total, - "main": 5, + "main": 3, "total_weight": p34_total_weight, "list": [ diff --git a/validator-session/Cargo.toml b/validator-session/Cargo.toml index 0dfe71f0..ddf63d41 100644 --- a/validator-session/Cargo.toml +++ b/validator-session/Cargo.toml @@ -5,9 +5,12 @@ name = 'validator_session' version = '0.0.2' [dependencies] +adnl = { git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.10.24' } backtrace = '0.3' +catchain = { path = '../catchain' } crc = '3.0' crossbeam = '0.7' +ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.10.2' } failure = '0.1' hex = '0.4' lazy_static = '1.4' @@ -15,9 +18,6 @@ log = '0.4' metrics = '0.21.0' metrics-core = '0.5' rand = '0.8' -adnl = { git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.10.24' } -catchain = { path = '../catchain' } -ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.10.2' } storage = { path = '../storage' } ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.3.79' }