From 617d6d146fdf0b38f645447a6a78bfc693e08f4c Mon Sep 17 00:00:00 2001 From: Nishant Bhagat Date: Wed, 24 Jun 2026 01:38:36 +0530 Subject: [PATCH 1/2] Fix for `SQLite3::BusyException: database is locked` during parallel platform sync ## Summary (generated via claude) **Problem** `SyncManager#execute_sync` runs all platforms concurrently (`Concurrent::Promises.future`), and each writes to the same SQLite file via `DiffEngine`. With the default rollback journal, a reader blocks a writer, so under contention a write fails within the 5s timeout and raises `database is locked` - silently skipping that program for the cycle. Example: `ERROR [hackerone] Failed to process program xyz_bbp: SQLite3::BusyException: database is locked` **Fix** Configure SQLite for concurrent access in `Database.connect`: - WAL journal mode so readers no longer block the writer (the actual fix) - Busy timeout raised to 10s so writers wait instead of erroring - `synchronous = NORMAL` (recommended WAL pairing) WAL persists in the DB header, so one call covers all pooled connections; `:timeout` is applied per-connection by Sequel. --- lib/scopes_extractor/database.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/scopes_extractor/database.rb b/lib/scopes_extractor/database.rb index d0e87b0..c492e36 100644 --- a/lib/scopes_extractor/database.rb +++ b/lib/scopes_extractor/database.rb @@ -7,7 +7,8 @@ def connect database_path = resolve_database_path ensure_db_directory(database_path) - ScopesExtractor.db = Sequel.sqlite(database_path) + ScopesExtractor.db = Sequel.sqlite(database_path, timeout: 10_000) + configure_concurrency(ScopesExtractor.db) ScopesExtractor.logger.info "Connected to database: #{database_path}".green ScopesExtractor.db @@ -50,6 +51,11 @@ def reset private + def configure_concurrency(db) + db.run('PRAGMA journal_mode = WAL') + db.run('PRAGMA synchronous = NORMAL') + end + def resolve_database_path path = Config.database_path path.start_with?('/') ? path : File.join(ScopesExtractor.root, path) From 10cd2238b20956aed2844bf419af00d6160fd2d2 Mon Sep 17 00:00:00 2001 From: Joshua MARTINELLE Date: Wed, 24 Jun 2026 08:04:51 +0200 Subject: [PATCH 2/2] refactor(database): use after_connect hook so synchronous pragma applies pool-wide --- .gitignore | 2 ++ lib/scopes_extractor/database.rb | 14 ++++++++----- spec/scopes_extractor/database_spec.rb | 27 +++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index e26b420..82902c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .env /coverage db/scopes.db +db/scopes.db-wal +db/scopes.db-shm spec/examples.txt diff --git a/lib/scopes_extractor/database.rb b/lib/scopes_extractor/database.rb index c492e36..537d2ef 100644 --- a/lib/scopes_extractor/database.rb +++ b/lib/scopes_extractor/database.rb @@ -7,8 +7,8 @@ def connect database_path = resolve_database_path ensure_db_directory(database_path) - ScopesExtractor.db = Sequel.sqlite(database_path, timeout: 10_000) - configure_concurrency(ScopesExtractor.db) + ScopesExtractor.db = Sequel.sqlite(database_path, timeout: 10_000, + after_connect: method(:configure_concurrency)) ScopesExtractor.logger.info "Connected to database: #{database_path}".green ScopesExtractor.db @@ -51,9 +51,13 @@ def reset private - def configure_concurrency(db) - db.run('PRAGMA journal_mode = WAL') - db.run('PRAGMA synchronous = NORMAL') + # Applied to every connection in Sequel's pool via :after_connect. + # +conn+ is the raw SQLite3::Database driver connection, so use #execute. + # WAL persists in the DB header (one writer, non-blocking readers); + # synchronous = NORMAL is per-connection and must be set on each one. + def configure_concurrency(conn) + conn.execute('PRAGMA journal_mode = WAL') + conn.execute('PRAGMA synchronous = NORMAL') end def resolve_database_path diff --git a/spec/scopes_extractor/database_spec.rb b/spec/scopes_extractor/database_spec.rb index 7efdda4..8835609 100644 --- a/spec/scopes_extractor/database_spec.rb +++ b/spec/scopes_extractor/database_spec.rb @@ -12,7 +12,10 @@ end after do - FileUtils.rm_f(test_db_path) + ScopesExtractor.db&.disconnect + ScopesExtractor.db = nil + # WAL mode leaves -wal/-shm side-files alongside the main database file. + FileUtils.rm_f([test_db_path, "#{test_db_path}-wal", "#{test_db_path}-shm"]) end describe '.connect' do @@ -31,6 +34,28 @@ expect(ScopesExtractor.logger).to receive(:info).with(/Connected to database/) described_class.connect end + + it 'enables WAL journal mode for concurrent access' do + db = described_class.connect + expect(db['PRAGMA journal_mode'].first[:journal_mode]).to eq('wal') + end + + it 'sets synchronous to NORMAL on every pooled connection' do + db = described_class.connect + + # NORMAL = 1, and it is a per-connection pragma, so exercise several + # concurrent connections to ensure after_connect ran on each. + results = Array.new(5).map do + Thread.new { db['PRAGMA synchronous'].first[:synchronous] }.value + end + + expect(results).to all(eq(1)) + end + + it 'raises the busy timeout above the 5s default' do + db = described_class.connect + expect(db['PRAGMA busy_timeout'].first[:timeout]).to eq(10_000) + end end describe '.migrate' do