From fadbc2a5244402d2096a23266dc299ea919f96c1 Mon Sep 17 00:00:00 2001 From: Vale McUrchin Date: Wed, 12 Feb 2025 16:47:30 -0500 Subject: [PATCH 01/13] test excluding unsuccessful transactions --- src/extract_transactions.rs | 26 +++++++------------------- src/json_rescue_v5_extract.rs | 5 ++++- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/extract_transactions.rs b/src/extract_transactions.rs index 6055ce2..c066367 100644 --- a/src/extract_transactions.rs +++ b/src/extract_transactions.rs @@ -37,26 +37,8 @@ pub async fn extract_current_transactions( let chunk = load_chunk(archive_path, each_chunk_manifest).await?; for (i, tx) in chunk.txns.iter().enumerate() { - // TODO: unsure if this is off by one - // perhaps reverse the vectors before transforming - - // first increment the block metadata. This assumes the vector is sequential. + // first collect the block metadata. This assumes the vector is sequential. if let Some(block) = tx.try_as_block_metadata() { - // // check the epochs are incrementing or not - // if epoch > block.epoch() - // && round > block.round() - // && timestamp > block.timestamp_usecs() - // { - // dbg!( - // epoch, - // block.epoch(), - // round, - // block.round(), - // timestamp, - // block.timestamp_usecs() - // ); - // } - epoch = block.epoch(); round = block.round(); timestamp = block.timestamp_usecs(); @@ -66,6 +48,12 @@ pub async fn extract_current_transactions( .txn_infos .get(i) .expect("could not index on tx_info chunk, vectors may not be same length"); + + // only process successful transactions + if !tx_info.status().is_success() { + continue; + }; + let tx_hash_info = tx_info.transaction_hash(); let tx_events = chunk diff --git a/src/json_rescue_v5_extract.rs b/src/json_rescue_v5_extract.rs index 763ca37..91d4c77 100644 --- a/src/json_rescue_v5_extract.rs +++ b/src/json_rescue_v5_extract.rs @@ -32,9 +32,12 @@ pub fn extract_v5_json_rescue( ) -> Result<(Vec, Vec, Vec)> { let json = std::fs::read_to_string(one_json_file).context("could not read file")?; - let txs: Vec = serde_json::from_str(&json) + let mut txs: Vec = serde_json::from_str(&json) .map_err(|e| anyhow!("could not parse JSON to TransactionViewV5, {:?}", e))?; + // remove any aborted txs + txs.retain(|t| t.vm_status.is_executed()); + decode_transaction_dataview_v5(&txs) } From ad0d90a617434b96aaf1db86fb436dd88cb95170 Mon Sep 17 00:00:00 2001 From: Isa McAccelerando Date: Wed, 12 Feb 2025 16:55:13 -0500 Subject: [PATCH 02/13] info prints --- src/extract_transactions.rs | 5 +++++ src/json_rescue_v5_extract.rs | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/extract_transactions.rs b/src/extract_transactions.rs index c066367..6af9e99 100644 --- a/src/extract_transactions.rs +++ b/src/extract_transactions.rs @@ -33,6 +33,8 @@ pub async fn extract_current_transactions( let mut user_txs: Vec = vec![]; let mut events: Vec = vec![]; + let mut count_excluded = 0; + for each_chunk_manifest in manifest.chunks { let chunk = load_chunk(archive_path, each_chunk_manifest).await?; @@ -51,6 +53,7 @@ pub async fn extract_current_transactions( // only process successful transactions if !tx_info.status().is_success() { + count_excluded += 1; continue; }; @@ -92,6 +95,8 @@ pub async fn extract_current_transactions( } } + info!("Excluding {} unsuccessful transactions", count_excluded); + Ok((user_txs, events)) } diff --git a/src/json_rescue_v5_extract.rs b/src/json_rescue_v5_extract.rs index 91d4c77..a01d336 100644 --- a/src/json_rescue_v5_extract.rs +++ b/src/json_rescue_v5_extract.rs @@ -21,7 +21,7 @@ use libra_backwards_compatibility::{ use anyhow::{anyhow, Context, Result}; use diem_temppath::TempPath; use diem_types::account_address::AccountAddress; -use log::trace; +use log::{info, trace}; use std::path::{Path, PathBuf}; /// The canonical transaction archives for V5 were kept in a different format as in v6 and v7. @@ -36,7 +36,12 @@ pub fn extract_v5_json_rescue( .map_err(|e| anyhow!("could not parse JSON to TransactionViewV5, {:?}", e))?; // remove any aborted txs + let orig_len = txs.len(); txs.retain(|t| t.vm_status.is_executed()); + let new_len = txs.len(); + if orig_len > new_len { + info!("Excluding {} unsuccessful transactions", orig_len - new_len); + }; decode_transaction_dataview_v5(&txs) } From a16f5e5e7d60b9d96789ae42969035e9d50419b6 Mon Sep 17 00:00:00 2001 From: Gabi Sforzando Date: Thu, 13 Feb 2025 15:51:26 -0500 Subject: [PATCH 03/13] patch tests --- tests/test_extract_transactions.rs | 3 +-- tests/test_json_rescue_v5_load.rs | 42 +++++++++++++++++++++++++++--- tests/test_json_rescue_v5_parse.rs | 11 ++++---- tests/test_load_tx.rs | 7 ++--- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/tests/test_extract_transactions.rs b/tests/test_extract_transactions.rs index d4274db..4050319 100644 --- a/tests/test_extract_transactions.rs +++ b/tests/test_extract_transactions.rs @@ -18,8 +18,7 @@ async fn test_extract_tx_from_archive() -> anyhow::Result<()> { async fn test_extract_v6_tx_from_archive() -> anyhow::Result<()> { let archive_path = support::fixtures::v6_tx_manifest_fixtures_path(); let list = extract_current_transactions(&archive_path, &FrameworkVersion::V6).await?; - - assert!(list.0.len() == 27); + assert!(list.0.len() == 25); assert!(list.1.len() == 52); Ok(()) diff --git a/tests/test_json_rescue_v5_load.rs b/tests/test_json_rescue_v5_load.rs index b2f0472..80b7c8b 100644 --- a/tests/test_json_rescue_v5_load.rs +++ b/tests/test_json_rescue_v5_load.rs @@ -25,7 +25,7 @@ async fn test_load_all_tgz() -> anyhow::Result<()> { let tx_count = json_rescue_v5_load::single_thread_decompress_extract(&path, &pool).await?; - assert!(tx_count == 13); + assert!(tx_count == 12); Ok(()) } @@ -46,7 +46,7 @@ async fn test_load_entrypoint() -> anyhow::Result<()> { let path = fixtures::v5_json_tx_path(); let tx_count = json_rescue_v5_load::rip_concurrent_limited(&path, &pool, None).await?; - assert!(tx_count == 13); + assert!(tx_count == 12); Ok(()) } @@ -68,7 +68,7 @@ async fn test_load_queue() -> anyhow::Result<()> { let tx_count = json_rescue_v5_load::rip_concurrent_limited(&path, &pool, None).await?; - assert!(tx_count == 13); + assert!(tx_count == 12); let tx_count = json_rescue_v5_load::rip_concurrent_limited(&path, &pool, None).await?; assert!(tx_count == 0); @@ -116,3 +116,39 @@ async fn test_rescue_v5_parse_set_wallet_tx() -> anyhow::Result<()> { Ok(()) } + +// #[tokio::test] +// fn test_stream() { +// async fn process_files(paths: Vec<&str>) { +// let mut stream = stream::iter(paths) +// .then(|path| async move { +// match read_json_file(path).await { +// Ok(data) => Some(data), +// Err(_) => None, +// } +// }) +// .filter_map(|x| async { x }) +// .flat_map(|data| stream::iter(data)); + +// let mut batch: VecDeque = VecDeque::new(); + +// while let Some(item) = stream.next().await { +// batch.push_back(item); + +// if batch.len() >= 100 { +// // Batch is large enough, process it +// let mut batch_to_process: Vec = Vec::new(); +// while let Some(_) = batch.pop_front() { +// batch_to_process.push(batch.pop_front().unwrap()); +// } +// process_batch(batch_to_process).await; +// } +// } + +// // Process any remaining items in the batch +// if !batch.is_empty() { +// let mut batch_to_process: Vec = batch.into(); +// process_batch(batch_to_process).await; +// } +// } +// } diff --git a/tests/test_json_rescue_v5_parse.rs b/tests/test_json_rescue_v5_parse.rs index efb86b5..ded96f7 100644 --- a/tests/test_json_rescue_v5_parse.rs +++ b/tests/test_json_rescue_v5_parse.rs @@ -81,10 +81,11 @@ fn test_json_full_file() -> anyhow::Result<()> { let p = fixtures::v5_json_tx_path().join("10000-10999.json"); let (tx, _, _) = extract_v5_json_rescue(&p)?; - assert!(tx.len() == 4); - let first = tx.first().unwrap(); - assert!(first.sender.to_hex_literal() == "0xb31bd7796bc113013a2bf6c3953305fd"); + assert!(tx.len() == 3); + let first = tx.first().unwrap(); + dbg!(&first); + assert!(first.sender.to_hex_literal() == "0xecaf65add1b785b0495e3099f4045ec0"); if let Some(EntryFunctionArgs::V5(ScriptFunctionCall::CreateUserByCoinTx { account, .. })) = first.entry_function @@ -103,8 +104,8 @@ fn decompress_and_read() { // get an advanced record let first_file = temp_dir.path().join("10000-10999.json"); let (tx, _, _) = extract_v5_json_rescue(&first_file).unwrap(); - assert!(tx.len() == 4); + assert!(tx.len() == 3); let first = tx.first().unwrap(); - assert!(first.sender.to_hex_literal() == "0xb31bd7796bc113013a2bf6c3953305fd"); + assert!(first.sender.to_hex_literal() == "0xecaf65add1b785b0495e3099f4045ec0"); } diff --git a/tests/test_load_tx.rs b/tests/test_load_tx.rs index a8bd786..f2ea441 100644 --- a/tests/test_load_tx.rs +++ b/tests/test_load_tx.rs @@ -19,7 +19,7 @@ async fn test_tx_batch() -> anyhow::Result<()> { libra_forensic_db::log_setup(); let archive_path = support::fixtures::v6_tx_manifest_fixtures_path(); let (txs, _events) = extract_current_transactions(&archive_path, &FrameworkVersion::V6).await?; - assert!(txs.len() == 27); + assert!(txs.len() == 25); let c = start_neo4j_container(); let port = c.get_host_port_ipv4(7687); @@ -79,7 +79,7 @@ async fn test_tx_batch() -> anyhow::Result<()> { let row = result.next().await?.unwrap(); let total_tx_count: i64 = row.get("total_tx_count").unwrap(); - assert!(total_tx_count == 24); + assert!(total_tx_count == 22); Ok(()) } @@ -105,7 +105,8 @@ async fn test_load_entry_point_tx() -> anyhow::Result<()> { assert!(res.created_accounts == 25); assert!(res.modified_accounts == 6); assert!(res.unchanged_accounts == 0); - assert!(res.created_tx == 27); + dbg!(&res.created_tx); + assert!(res.created_tx == 25); Ok(()) } From b3a17a94de48f7cbedc226eca04f3d54b53a3efe Mon Sep 17 00:00:00 2001 From: 0o-de-lally <> Date: Thu, 5 Feb 2026 18:13:24 +0000 Subject: [PATCH 04/13] fix: vendor read_tx_chunk to resolve CI clippy failure --- Cargo.toml | 1 + src/extract_transactions.rs | 2 +- src/lib.rs | 1 + src/read_tx_chunk.rs | 83 +++++++++++++++++++++++++++++++++++++ src/unzip_temp.rs | 2 +- tests/test_unzip.rs | 2 +- 6 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 src/read_tx_chunk.rs diff --git a/Cargo.toml b/Cargo.toml index a9ae316..6c0dce2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ chrono = { version = "0.4.19", features = ["clock", "serde"] } clap = { version = "4.3.5", features = ["derive", "unstable-styles"] } diem-temppath = { git = "https://github.com/0LNetworkCommunity/diem.git", branch = "release" } diem-types = { git = "https://github.com/0LNetworkCommunity/diem.git", branch = "release" } +diem-backup-cli = { git = "https://github.com/0LNetworkCommunity/diem.git", branch = "release" } diem-crypto = { git = "https://github.com/0LNetworkCommunity/diem.git", branch = "release" } env_logger = "^0.11" flate2 = "^1.0" diff --git a/src/extract_transactions.rs b/src/extract_transactions.rs index 6af9e99..a4170cf 100644 --- a/src/extract_transactions.rs +++ b/src/extract_transactions.rs @@ -7,7 +7,7 @@ use diem_crypto::HashValue; use diem_types::account_config::{NewBlockEvent, WithdrawEvent}; use diem_types::contract_event::ContractEvent; use diem_types::{account_config::DepositEvent, transaction::SignedTransaction}; -use libra_storage::read_tx_chunk::{load_chunk, load_tx_chunk_manifest}; +use crate::read_tx_chunk::{load_chunk, load_tx_chunk_manifest}; use libra_types::move_resource::coin_register_event::CoinRegisterEvent; use log::{error, info, warn}; use serde_json::json; diff --git a/src/lib.rs b/src/lib.rs index a1536ca..cc677bc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ pub mod load_exchange_orders; pub mod load_tx_cypher; pub mod neo4j_init; pub mod queue; +pub mod read_tx_chunk; pub mod scan; pub mod schema_account_state; pub mod schema_exchange_orders; diff --git a/src/read_tx_chunk.rs b/src/read_tx_chunk.rs new file mode 100644 index 0000000..f55c73f --- /dev/null +++ b/src/read_tx_chunk.rs @@ -0,0 +1,83 @@ +use std::path::Path; + +use anyhow::{anyhow, Context, Result}; + +use diem_backup_cli::backup_types::transaction::manifest::TransactionBackup; +use diem_backup_cli::backup_types::transaction::manifest::TransactionChunk; +use diem_backup_cli::utils::read_record_bytes::ReadRecordBytes; +use diem_types::contract_event::ContractEvent; +use diem_types::transaction::Transaction; +use diem_types::transaction::TransactionInfo; +use diem_types::write_set::WriteSet; +use libra_backwards_compatibility::version_five::state_snapshot_v5::open_for_read; + +/// read snapshot manifest file into object +pub fn load_tx_chunk_manifest(path: &Path) -> anyhow::Result { + let s = + std::fs::read_to_string(path).context(format!("Error: cannot read file at {:?}", path))?; + + let map: TransactionBackup = serde_json::from_str(&s)?; + + Ok(map) +} + +// similar to Loaded Chunk +// diem/storage/backup/backup-cli/src/backup_types/transaction/restore.rs +// The vectors below are OF THE SAME LENGTH +// It is a table where for example, a tx without events will be an empty slot in the vector. +pub struct TransactionArchiveChunk { + pub manifest: TransactionChunk, + pub txns: Vec, + pub txn_infos: Vec, + pub event_vecs: Vec>, + pub write_sets: Vec, +} + +pub async fn load_chunk( + archive_path: &Path, + manifest: TransactionChunk, +) -> Result { + let full_handle = archive_path + .parent() + .expect("could not read archive path") + .join(&manifest.transactions); + let handle_str = full_handle.to_str().unwrap(); + assert!(full_handle.exists(), "file does not exist"); + + let mut file = open_for_read(handle_str) + .await + .map_err(|e| anyhow!("snapshot chunk {:?}, {:?}", &handle_str, e))?; + + let mut txns = Vec::new(); + let mut txn_infos = Vec::new(); + let mut event_vecs = Vec::new(); + let mut write_sets = Vec::new(); + + while let Some(record_bytes) = file.read_record_bytes().await? { + let (txn, txn_info, events, write_set): (_, _, _, WriteSet) = + bcs::from_bytes(&record_bytes)?; + txns.push(txn); + txn_infos.push(txn_info); + event_vecs.push(events); + write_sets.push(write_set); + } + + // the chunk is a table implements with vectors, + // they should have the same length + assert!( + txns.len() == txn_infos.len() + && txn_infos.len() == event_vecs.len() + && event_vecs.len() == write_sets.len(), + "transactions chunk have different vector length for txs, events, and writesets" + ); + + // TODO: for purposes of explorer/warehouse do we want to do the full tx restore controller verifications + + Ok(TransactionArchiveChunk { + manifest, + txns, + txn_infos, + event_vecs, + write_sets, + }) +} diff --git a/src/unzip_temp.rs b/src/unzip_temp.rs index c1a6262..8a8b187 100644 --- a/src/unzip_temp.rs +++ b/src/unzip_temp.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result}; use diem_temppath::TempPath; use flate2::read::GzDecoder; use glob::glob; -// use libra_storage::read_tx_chunk::load_tx_chunk_manifest; +// use crate::read_tx_chunk::load_tx_chunk_manifest; use log::{info, warn}; use std::{ fs::File, diff --git a/tests/test_unzip.rs b/tests/test_unzip.rs index 01964e5..830d040 100644 --- a/tests/test_unzip.rs +++ b/tests/test_unzip.rs @@ -1,6 +1,6 @@ mod support; use libra_forensic_db::unzip_temp; -use libra_storage::read_tx_chunk::load_tx_chunk_manifest; +use libra_forensic_db::read_tx_chunk::load_tx_chunk_manifest; #[ignore] #[test] From 6cc3aabbf3d9f04be2b960cb6039b9fd9a136c55 Mon Sep 17 00:00:00 2001 From: 0o-de-lally <> Date: Thu, 5 Feb 2026 18:15:04 +0000 Subject: [PATCH 05/13] fix: run rust-tests on all PR branches and include reopened event --- .github/workflows/rust-tests.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust-tests.yaml b/.github/workflows/rust-tests.yaml index 09be3c6..a71f63e 100644 --- a/.github/workflows/rust-tests.yaml +++ b/.github/workflows/rust-tests.yaml @@ -9,9 +9,8 @@ on: types: - opened - synchronize - branches: - - "release**" - - "main**" + - reopened + branches: ["**"] schedule: - cron: "30 00 * * *" From e1b0dd9098f32c0e3aa35b526102a4ecb99f6eb1 Mon Sep 17 00:00:00 2001 From: 0o-de-lally <> Date: Thu, 5 Feb 2026 18:16:28 +0000 Subject: [PATCH 06/13] fix: enable workflows on all pull requests --- .github/workflows/cleanliness.yaml | 8 +------- .github/workflows/rust-tests.yaml | 5 ----- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/cleanliness.yaml b/.github/workflows/cleanliness.yaml index 40a6613..e70c922 100644 --- a/.github/workflows/cleanliness.yaml +++ b/.github/workflows/cleanliness.yaml @@ -5,13 +5,7 @@ on: - "[0-9]+.[0-9]+.[0-9]+" branches: ["**"] # glob pattern to allow slash / pull_request: - types: - - opened - - synchronize - branches: - - "release**" - - "main**" -env: + env: DIEM_FORGE_NODE_BIN_PATH: ${{github.workspace}}/diem-node LIBRA_CI: 1 MODE_0L: "TESTNET" diff --git a/.github/workflows/rust-tests.yaml b/.github/workflows/rust-tests.yaml index a71f63e..1b37b87 100644 --- a/.github/workflows/rust-tests.yaml +++ b/.github/workflows/rust-tests.yaml @@ -6,11 +6,6 @@ on: - "[0-9]+.[0-9]+.[0-9]+" branches: ["**"] # glob pattern to allow slash / pull_request: - types: - - opened - - synchronize - - reopened - branches: ["**"] schedule: - cron: "30 00 * * *" From 00574f70b24fb3e8381a0de54f0ce9dacfb37d09 Mon Sep 17 00:00:00 2001 From: 0o-de-lally <> Date: Thu, 5 Feb 2026 18:18:06 +0000 Subject: [PATCH 07/13] fix: restore env indentation in cleanliness workflow --- .github/workflows/cleanliness.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cleanliness.yaml b/.github/workflows/cleanliness.yaml index e70c922..5041352 100644 --- a/.github/workflows/cleanliness.yaml +++ b/.github/workflows/cleanliness.yaml @@ -5,7 +5,8 @@ on: - "[0-9]+.[0-9]+.[0-9]+" branches: ["**"] # glob pattern to allow slash / pull_request: - env: + +env: DIEM_FORGE_NODE_BIN_PATH: ${{github.workspace}}/diem-node LIBRA_CI: 1 MODE_0L: "TESTNET" From cce1d96d40c4a4e997841f645f5b41ce941757c1 Mon Sep 17 00:00:00 2001 From: 0o-de-lally <> Date: Thu, 5 Feb 2026 18:19:05 +0000 Subject: [PATCH 08/13] fix: restrict PR workflows to main branch on open/sync --- .github/workflows/cleanliness.yaml | 2 ++ .github/workflows/rust-tests.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/cleanliness.yaml b/.github/workflows/cleanliness.yaml index 5041352..264faef 100644 --- a/.github/workflows/cleanliness.yaml +++ b/.github/workflows/cleanliness.yaml @@ -5,6 +5,8 @@ on: - "[0-9]+.[0-9]+.[0-9]+" branches: ["**"] # glob pattern to allow slash / pull_request: + types: [opened, synchronize] + branches: [main] env: DIEM_FORGE_NODE_BIN_PATH: ${{github.workspace}}/diem-node diff --git a/.github/workflows/rust-tests.yaml b/.github/workflows/rust-tests.yaml index 1b37b87..391229f 100644 --- a/.github/workflows/rust-tests.yaml +++ b/.github/workflows/rust-tests.yaml @@ -6,6 +6,8 @@ on: - "[0-9]+.[0-9]+.[0-9]+" branches: ["**"] # glob pattern to allow slash / pull_request: + types: [opened, synchronize] + branches: [main] schedule: - cron: "30 00 * * *" From df507c6afd39aa95a40dd467d1b091dc50cf1e5f Mon Sep 17 00:00:00 2001 From: 0o-de-lally <> Date: Thu, 5 Feb 2026 18:21:09 +0000 Subject: [PATCH 09/13] fix: apply cargo fmt --- src/extract_transactions.rs | 2 +- tests/test_unzip.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extract_transactions.rs b/src/extract_transactions.rs index a4170cf..a87da8c 100644 --- a/src/extract_transactions.rs +++ b/src/extract_transactions.rs @@ -1,4 +1,5 @@ use crate::decode_entry_function::decode_entry_function_all_versions; +use crate::read_tx_chunk::{load_chunk, load_tx_chunk_manifest}; use crate::scan::FrameworkVersion; use crate::schema_transaction::{RelationLabel, UserEventTypes, WarehouseEvent, WarehouseTxMaster}; use anyhow::Result; @@ -7,7 +8,6 @@ use diem_crypto::HashValue; use diem_types::account_config::{NewBlockEvent, WithdrawEvent}; use diem_types::contract_event::ContractEvent; use diem_types::{account_config::DepositEvent, transaction::SignedTransaction}; -use crate::read_tx_chunk::{load_chunk, load_tx_chunk_manifest}; use libra_types::move_resource::coin_register_event::CoinRegisterEvent; use log::{error, info, warn}; use serde_json::json; diff --git a/tests/test_unzip.rs b/tests/test_unzip.rs index 830d040..0eb59b1 100644 --- a/tests/test_unzip.rs +++ b/tests/test_unzip.rs @@ -1,6 +1,6 @@ mod support; -use libra_forensic_db::unzip_temp; use libra_forensic_db::read_tx_chunk::load_tx_chunk_manifest; +use libra_forensic_db::unzip_temp; #[ignore] #[test] From 0cc53d502666e59bbb4bc8ecb6c3d08ad886eebf Mon Sep 17 00:00:00 2001 From: 0o-de-lally <> Date: Thu, 5 Feb 2026 18:22:21 +0000 Subject: [PATCH 10/13] feat: separate cargo check into distinct job before clippy --- .github/workflows/cleanliness.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/cleanliness.yaml b/.github/workflows/cleanliness.yaml index 264faef..70b26ba 100644 --- a/.github/workflows/cleanliness.yaml +++ b/.github/workflows/cleanliness.yaml @@ -14,7 +14,28 @@ env: MODE_0L: "TESTNET" jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: setup env + uses: ./.github/actions/build_env + + - uses: Swatinem/rust-cache@v2 + with: + shared-key: "forensic-db" + cache-all-crates: true + cache-on-failure: true + + - name: check + uses: actions-rs/cargo@v1 + with: + command: check + args: --workspace --tests + clippy: + needs: check runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 60057946cd16c93cf4ae5a0de4ecb1f6c4b2454b Mon Sep 17 00:00:00 2001 From: 0o-de-lally <> Date: Thu, 5 Feb 2026 18:23:37 +0000 Subject: [PATCH 11/13] refactor: run format and clippy jobs in parallel --- .github/workflows/cleanliness.yaml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cleanliness.yaml b/.github/workflows/cleanliness.yaml index 70b26ba..b3a33be 100644 --- a/.github/workflows/cleanliness.yaml +++ b/.github/workflows/cleanliness.yaml @@ -14,7 +14,7 @@ env: MODE_0L: "TESTNET" jobs: - check: + format: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -28,14 +28,13 @@ jobs: cache-all-crates: true cache-on-failure: true - - name: check + - name: format uses: actions-rs/cargo@v1 with: - command: check - args: --workspace --tests + command: fmt + args: --all -- --check clippy: - needs: check runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -49,12 +48,6 @@ jobs: cache-all-crates: true cache-on-failure: true - - name: format - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check - # TODO: clippy can share cache if build for tests is done prior # - name: build for cache # run: cargo build --tests --workspace From cc990c411191b2e6c15bd205827c0d7193a80712 Mon Sep 17 00:00:00 2001 From: 0o-de-lally <> Date: Thu, 5 Feb 2026 19:03:09 +0000 Subject: [PATCH 12/13] fix: resolve clippy errors and limit workflows to main branch --- .github/workflows/cleanliness.yaml | 2 +- .github/workflows/rust-tests.yaml | 2 +- src/warehouse_cli.rs | 22 +++++----------------- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/.github/workflows/cleanliness.yaml b/.github/workflows/cleanliness.yaml index b3a33be..5a36f0e 100644 --- a/.github/workflows/cleanliness.yaml +++ b/.github/workflows/cleanliness.yaml @@ -3,7 +3,7 @@ on: push: tags: # only on releases, not RC, since we've tested already - "[0-9]+.[0-9]+.[0-9]+" - branches: ["**"] # glob pattern to allow slash / + branches: ["main"] pull_request: types: [opened, synchronize] branches: [main] diff --git a/.github/workflows/rust-tests.yaml b/.github/workflows/rust-tests.yaml index 391229f..a36a296 100644 --- a/.github/workflows/rust-tests.yaml +++ b/.github/workflows/rust-tests.yaml @@ -4,7 +4,7 @@ on: push: tags: # only on releases, not RC, since we've tested already - "[0-9]+.[0-9]+.[0-9]+" - branches: ["**"] # glob pattern to allow slash / + branches: ["main"] pull_request: types: [opened, synchronize] branches: [main] diff --git a/src/warehouse_cli.rs b/src/warehouse_cli.rs index f40b2d9..9e9eba5 100644 --- a/src/warehouse_cli.rs +++ b/src/warehouse_cli.rs @@ -12,7 +12,7 @@ use crate::{ json_rescue_v5_load, load::{ingest_all, try_load_one_archive}, load_exchange_orders, - neo4j_init::{self, get_credentials_from_env, PASS_ENV, URI_ENV, USER_ENV}, + neo4j_init::{self, get_credentials_from_env}, scan::{scan_dir_archive, BundleContent, ManifestInfo}, unzip_temp, util, }; @@ -294,22 +294,10 @@ pub async fn try_db_connection_pool(cli: &WarehouseCli) -> Result { let db = match get_credentials_from_env() { Ok((uri, user, password)) => Graph::new(uri, user, password).await?, Err(_) => { - if cli.db_uri.is_some() && cli.db_username.is_some() && cli.db_password.is_some() { - Graph::new( - cli.db_uri.as_ref().unwrap(), - cli.db_username.as_ref().unwrap(), - cli.db_password.as_ref().unwrap(), - ) - .await? - } else { - println!("Must pass DB credentials, either with CLI args or environment variable"); - println!("call with --db-uri, --db-user, and --db-password"); - println!( - "Alternatively export credentials to env variables: {}, {}, {}", - URI_ENV, USER_ENV, PASS_ENV - ); - bail!("could not get a db instance with credentials"); - } + let uri = cli.db_uri.as_ref().expect("Must pass --db-uri or set URI_ENV"); + let user = cli.db_username.as_ref().expect("Must pass --db-user or set USER_ENV"); + let password = cli.db_password.as_ref().expect("Must pass --db-password or set PASS_ENV"); + Graph::new(uri, user, password).await? } }; Ok(db) From cdf7287cab1a47f5869972cc60672686995309e1 Mon Sep 17 00:00:00 2001 From: 0o-de-lally <> Date: Thu, 5 Feb 2026 19:18:53 +0000 Subject: [PATCH 13/13] fix: apply cargo fmt --- src/warehouse_cli.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/warehouse_cli.rs b/src/warehouse_cli.rs index 9e9eba5..b3d1bc1 100644 --- a/src/warehouse_cli.rs +++ b/src/warehouse_cli.rs @@ -294,9 +294,18 @@ pub async fn try_db_connection_pool(cli: &WarehouseCli) -> Result { let db = match get_credentials_from_env() { Ok((uri, user, password)) => Graph::new(uri, user, password).await?, Err(_) => { - let uri = cli.db_uri.as_ref().expect("Must pass --db-uri or set URI_ENV"); - let user = cli.db_username.as_ref().expect("Must pass --db-user or set USER_ENV"); - let password = cli.db_password.as_ref().expect("Must pass --db-password or set PASS_ENV"); + let uri = cli + .db_uri + .as_ref() + .expect("Must pass --db-uri or set URI_ENV"); + let user = cli + .db_username + .as_ref() + .expect("Must pass --db-user or set USER_ENV"); + let password = cli + .db_password + .as_ref() + .expect("Must pass --db-password or set PASS_ENV"); Graph::new(uri, user, password).await? } };