From 8f2f5a5ec610cccd983c8ac93acd75731cb154d8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 25 Sep 2016 11:41:48 -0500 Subject: [PATCH 001/591] Copy from event_stream-postgres --- .gitignore | 11 ++ MIT-License.txt | 21 +++ README.md | 3 + bin/clear-events-table | 3 + bin/install-event-stream-database | 3 + bin/list-events | 3 + bin/uninstall-event-stream-database | 3 + database/clear-events-table.sh | 34 ++++ database/functions/category.sql | 10 + database/functions/stream-version.sql | 13 ++ database/functions/write-event.sql | 49 +++++ database/indexes/events-indexes.sql | 7 + database/install.sh | 78 ++++++++ database/list-events.sh | 34 ++++ database/tables/events-table.sql | 19 ++ .../test/write-event-expected-version.sql | 1 + database/test/write-event.sql | 1 + database/uninstall.sh | 60 ++++++ event_source-postgres.gemspec | 34 ++++ init.rb | 10 + install-gems.sh | 79 ++++++++ lib/event_source/postgres.rb | 35 ++++ lib/event_source/postgres/controls.rb | 10 + .../postgres/controls/category.rb | 25 +++ .../postgres/controls/event_data.rb | 26 +++ .../postgres/controls/event_data/hash.rb | 30 +++ .../postgres/controls/event_data/metadata.rb | 26 +++ .../postgres/controls/event_data/write.rb | 42 +++++ lib/event_source/postgres/controls/put.rb | 19 ++ lib/event_source/postgres/controls/stream.rb | 19 ++ .../postgres/controls/stream_name.rb | 14 ++ lib/event_source/postgres/event_data.rb | 11 ++ lib/event_source/postgres/event_data/hash.rb | 33 ++++ lib/event_source/postgres/event_data/read.rb | 18 ++ lib/event_source/postgres/event_data/write.rb | 8 + lib/event_source/postgres/get.rb | 93 ++++++++++ .../postgres/get/select_statement.rb | 94 ++++++++++ lib/event_source/postgres/iterator.rb | 106 +++++++++++ lib/event_source/postgres/iterator/cycle.rb | 174 ++++++++++++++++++ lib/event_source/postgres/no_stream.rb | 13 ++ lib/event_source/postgres/put.rb | 130 +++++++++++++ lib/event_source/postgres/read.rb | 59 ++++++ lib/event_source/postgres/session.rb | 109 +++++++++++ lib/event_source/postgres/settings.rb | 30 +++ lib/event_source/postgres/stream.rb | 43 +++++ lib/event_source/postgres/stream_name.rb | 25 +++ lib/event_source/postgres/write.rb | 7 + library-symlinks.sh | 88 +++++++++ materials/nathans_sketch.rb | 33 ++++ materials/new_school_sketch.rb | 82 +++++++++ materials/poll.rb | 21 +++ materials/retry_batch_message_sketch.rb | 64 +++++++ materials/subscription_sketch.rb | 13 ++ remove-lib-symlinks.sh | 3 + set-local-gem-path.sh | 14 ++ settings/event_stream_postgres.json | 14 ++ symlink-lib.sh | 3 + test.rb | 1 + test/automated.rb | 6 + test/automated/automated_init.rb | 1 + test/automated/event_data/hash.rb | 23 +++ .../event_data/read_data_category.rb | 16 ++ test/automated/get/batch_size.rb | 17 ++ test/automated/get/category.rb | 18 ++ test/automated/get/get.rb | 16 ++ test/automated/get/no_events.rb | 15 ++ test/automated/get/precedence.rb | 29 +++ .../cycle/retry_when_no_further_event_data.rb | 28 +++ test/automated/iterator/cycle/telemetry.rb | 65 +++++++ test/automated/iterator/next.rb | 22 +++ .../iterator/no_further_event_data.rb | 19 ++ test/automated/iterator_tests.rb | 5 + test/automated/put/category_as_stream_name.rb | 18 ++ test/automated/put/expected_version_error.rb | 20 ++ .../put/no_stream/existing_stream.rb | 24 +++ test/automated/put/no_stream/no_stream.rb | 18 ++ test/automated/put/put.rb | 16 ++ test/automated/put/returns_stream_position.rb | 15 ++ .../put/stream_position_increases.rb | 15 ++ test/automated/read/missing_block_error.rb | 13 ++ test/automated/read/read.rb | 17 ++ .../read/retry_when_no_further_event_data.rb | 12 ++ .../automated/read/synchronous_result.skip.rb | 13 ++ .../select_statement/category_query.rb | 20 ++ test/automated/select_statement/defaults.rb | 32 ++++ .../select_statement/stream_name_query.rb | 20 ++ test/automated/session.rb | 28 +++ test/automated/settings.rb | 17 ++ test/automated/stream.rb | 65 +++++++ .../stream_name/category_stream_name.rb | 8 + test/automated/stream_name/stream_id.rb | 16 ++ test/automated/stream_name/stream_name.rb | 9 + test/script/reader_does_not_terminate.rb | 3 + test/test_init.rb | 25 +++ 94 files changed, 2715 insertions(+) create mode 100644 .gitignore create mode 100644 MIT-License.txt create mode 100644 README.md create mode 100755 bin/clear-events-table create mode 100755 bin/install-event-stream-database create mode 100755 bin/list-events create mode 100755 bin/uninstall-event-stream-database create mode 100755 database/clear-events-table.sh create mode 100644 database/functions/category.sql create mode 100644 database/functions/stream-version.sql create mode 100644 database/functions/write-event.sql create mode 100644 database/indexes/events-indexes.sql create mode 100755 database/install.sh create mode 100755 database/list-events.sh create mode 100644 database/tables/events-table.sql create mode 100644 database/test/write-event-expected-version.sql create mode 100644 database/test/write-event.sql create mode 100755 database/uninstall.sh create mode 100644 event_source-postgres.gemspec create mode 100644 init.rb create mode 100755 install-gems.sh create mode 100644 lib/event_source/postgres.rb create mode 100644 lib/event_source/postgres/controls.rb create mode 100644 lib/event_source/postgres/controls/category.rb create mode 100644 lib/event_source/postgres/controls/event_data.rb create mode 100644 lib/event_source/postgres/controls/event_data/hash.rb create mode 100644 lib/event_source/postgres/controls/event_data/metadata.rb create mode 100644 lib/event_source/postgres/controls/event_data/write.rb create mode 100644 lib/event_source/postgres/controls/put.rb create mode 100644 lib/event_source/postgres/controls/stream.rb create mode 100644 lib/event_source/postgres/controls/stream_name.rb create mode 100644 lib/event_source/postgres/event_data.rb create mode 100644 lib/event_source/postgres/event_data/hash.rb create mode 100644 lib/event_source/postgres/event_data/read.rb create mode 100644 lib/event_source/postgres/event_data/write.rb create mode 100644 lib/event_source/postgres/get.rb create mode 100644 lib/event_source/postgres/get/select_statement.rb create mode 100644 lib/event_source/postgres/iterator.rb create mode 100644 lib/event_source/postgres/iterator/cycle.rb create mode 100644 lib/event_source/postgres/no_stream.rb create mode 100644 lib/event_source/postgres/put.rb create mode 100644 lib/event_source/postgres/read.rb create mode 100644 lib/event_source/postgres/session.rb create mode 100644 lib/event_source/postgres/settings.rb create mode 100644 lib/event_source/postgres/stream.rb create mode 100644 lib/event_source/postgres/stream_name.rb create mode 100644 lib/event_source/postgres/write.rb create mode 100755 library-symlinks.sh create mode 100644 materials/nathans_sketch.rb create mode 100644 materials/new_school_sketch.rb create mode 100644 materials/poll.rb create mode 100644 materials/retry_batch_message_sketch.rb create mode 100644 materials/subscription_sketch.rb create mode 100755 remove-lib-symlinks.sh create mode 100755 set-local-gem-path.sh create mode 100644 settings/event_stream_postgres.json create mode 100755 symlink-lib.sh create mode 100644 test.rb create mode 100644 test/automated.rb create mode 100644 test/automated/automated_init.rb create mode 100644 test/automated/event_data/hash.rb create mode 100644 test/automated/event_data/read_data_category.rb create mode 100644 test/automated/get/batch_size.rb create mode 100644 test/automated/get/category.rb create mode 100644 test/automated/get/get.rb create mode 100644 test/automated/get/no_events.rb create mode 100644 test/automated/get/precedence.rb create mode 100644 test/automated/iterator/cycle/retry_when_no_further_event_data.rb create mode 100644 test/automated/iterator/cycle/telemetry.rb create mode 100644 test/automated/iterator/next.rb create mode 100644 test/automated/iterator/no_further_event_data.rb create mode 100644 test/automated/iterator_tests.rb create mode 100644 test/automated/put/category_as_stream_name.rb create mode 100644 test/automated/put/expected_version_error.rb create mode 100644 test/automated/put/no_stream/existing_stream.rb create mode 100644 test/automated/put/no_stream/no_stream.rb create mode 100644 test/automated/put/put.rb create mode 100644 test/automated/put/returns_stream_position.rb create mode 100644 test/automated/put/stream_position_increases.rb create mode 100644 test/automated/read/missing_block_error.rb create mode 100644 test/automated/read/read.rb create mode 100644 test/automated/read/retry_when_no_further_event_data.rb create mode 100644 test/automated/read/synchronous_result.skip.rb create mode 100644 test/automated/select_statement/category_query.rb create mode 100644 test/automated/select_statement/defaults.rb create mode 100644 test/automated/select_statement/stream_name_query.rb create mode 100644 test/automated/session.rb create mode 100644 test/automated/settings.rb create mode 100644 test/automated/stream.rb create mode 100644 test/automated/stream_name/category_stream_name.rb create mode 100644 test/automated/stream_name/stream_id.rb create mode 100644 test/automated/stream_name/stream_name.rb create mode 100644 test/script/reader_does_not_terminate.rb create mode 100644 test/test_init.rb diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d86689 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +.DS_store +.bundle/ +.ruby-version +.ruby-gemset +.rvmrc +Gemfile.lock +*.log +*.gem +gems +gems_backup +*scratch* diff --git a/MIT-License.txt b/MIT-License.txt new file mode 100644 index 0000000..24cc64e --- /dev/null +++ b/MIT-License.txt @@ -0,0 +1,21 @@ +Original work Copyright (c) 2015-2016 Obsidian Software, Inc. +Modified work Copyright (c) 2016 Scott Bellware + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b23d82c --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +## License + +The `event_stream-postgres` library is released under the [MIT License](https://github.com/eventide-project/event-stream-postgres/blob/master/MIT-License.txt). diff --git a/bin/clear-events-table b/bin/clear-events-table new file mode 100755 index 0000000..0108fa8 --- /dev/null +++ b/bin/clear-events-table @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +database/clear-events-table.sh diff --git a/bin/install-event-stream-database b/bin/install-event-stream-database new file mode 100755 index 0000000..7e57268 --- /dev/null +++ b/bin/install-event-stream-database @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +database/install.sh diff --git a/bin/list-events b/bin/list-events new file mode 100755 index 0000000..c75c2d9 --- /dev/null +++ b/bin/list-events @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +database/list-events.sh diff --git a/bin/uninstall-event-stream-database b/bin/uninstall-event-stream-database new file mode 100755 index 0000000..3c238f9 --- /dev/null +++ b/bin/uninstall-event-stream-database @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +database/uninstall.sh diff --git a/database/clear-events-table.sh b/database/clear-events-table.sh new file mode 100755 index 0000000..a6156d4 --- /dev/null +++ b/database/clear-events-table.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -e + +clear + +echo +echo "Clearing Events Table" +echo "= = =" +echo + +if [ -z ${DATABASE_USER+x} ]; then + echo "(DATABASE_USER is not set)" + user=eventstream +else + user=$DATABASE_USER +fi +echo "Database user is: $user" + +if [ -z ${DATABASE_NAME+x} ]; then + echo "(DATABASE_NAME is not set)" + database=eventstream +else + database=$DATABASE_NAME +fi +echo "Database name is: $database" + +echo + +function truncate-events-table { + psql $database -c "TRUNCATE events RESTART IDENTITY;" +} + +truncate-events-table diff --git a/database/functions/category.sql b/database/functions/category.sql new file mode 100644 index 0000000..014fc74 --- /dev/null +++ b/database/functions/category.sql @@ -0,0 +1,10 @@ +CREATE OR REPLACE FUNCTION category( + _stream_name varchar +) +RETURNS varchar +AS $$ +BEGIN + return split_part(_stream_name, '-', 1); +END; +$$ LANGUAGE plpgsql +IMMUTABLE; diff --git a/database/functions/stream-version.sql b/database/functions/stream-version.sql new file mode 100644 index 0000000..1b4c47b --- /dev/null +++ b/database/functions/stream-version.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE FUNCTION stream_version( + _stream_name varchar +) +RETURNS int +AS $$ +DECLARE + stream_version int; +BEGIN + select max(stream_position) into stream_version from events where stream_name = _stream_name; + + return stream_version; +END; +$$ LANGUAGE plpgsql; diff --git a/database/functions/write-event.sql b/database/functions/write-event.sql new file mode 100644 index 0000000..48af008 --- /dev/null +++ b/database/functions/write-event.sql @@ -0,0 +1,49 @@ +CREATE OR REPLACE FUNCTION write_event( + _stream_name varchar, + _type varchar, + _data jsonb, + _metadata jsonb DEFAULT NULL, + _expected_version int DEFAULT NULL +) +RETURNS int +AS $$ +DECLARE + stream_version int; + stream_position int; + category varchar; +BEGIN + stream_version := stream_version(_stream_name); + + if stream_version is null then + stream_version := -1; + end if; + + if _expected_version is not null then + if _expected_version != stream_version then + raise exception 'Wrong expected version: % (Stream: %, Stream Version: %)', _expected_version, _stream_name, stream_version; + end if; + end if; + + stream_position := stream_version + 1; + + insert into "events" + ( + "stream_name", + "stream_position", + "type", + "data", + "metadata" + ) + values + ( + _stream_name, + stream_position, + _type, + _data, + _metadata + ) + ; + + return stream_position; +END; +$$ LANGUAGE plpgsql; diff --git a/database/indexes/events-indexes.sql b/database/indexes/events-indexes.sql new file mode 100644 index 0000000..5ffbda6 --- /dev/null +++ b/database/indexes/events-indexes.sql @@ -0,0 +1,7 @@ +-- ---------------------------- +-- Indexes structure for table events +-- ---------------------------- +CREATE INDEX CONCURRENTLY "events_category_global_position_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "global_position" "pg_catalog"."int8_ops" ASC NULLS LAST); +CREATE INDEX CONCURRENTLY "events_category_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); +CREATE INDEX CONCURRENTLY "events_stream_name_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); +CREATE UNIQUE INDEX CONCURRENTLY "events_stream_name_stream_position_uniq_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "stream_position" "pg_catalog"."int4_ops" ASC NULLS LAST); diff --git a/database/install.sh b/database/install.sh new file mode 100755 index 0000000..85cc623 --- /dev/null +++ b/database/install.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +set -e + +clear + +echo +echo "Installing Database" +echo "= = =" +echo + +if [ -z ${DATABASE_USER+x} ]; then + echo "(DATABASE_USER is not set)" + user=eventstream +else + user=$DATABASE_USER +fi + +if [ -z ${DATABASE_NAME+x} ]; then + echo "(DATABASE_NAME is not set)" + database=eventstream +else + database=$DATABASE_NAME +fi + +echo + +function create-user { + echo "Database user is: $user" + + user_exists=`psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` + + if [ "$user_exists" = "1" ]; then + echo "Database user \"$user\" was previously created. Not creating again." + else + echo "Database user \"$user\" has not yet been created" + echo "Creating database user \"$user\"" + createuser -s $user + fi + + echo +} + +function create-database { + echo "Database name is: $database" + + database_exists=`psql -tAc "SELECT 1 FROM pg_database WHERE datname='$database'"` + + if [ "$database_exists" = "1" ]; then + echo "Database \"$database\" was previously created. Not creating again." + else + echo "Database \"$database\" has not yet been created" + echo "Creating database \"$database\"" + createdb $database || true + fi + + echo +} + +function create-tables { + psql $database -f database/tables/events-table.sql +} + +function create-functions { + psql $database -f database/functions/category.sql + psql $database -f database/functions/stream-version.sql + psql $database -f database/functions/write-event.sql +} + +function create-indexes { + psql $database -f database/indexes/events-indexes.sql +} + +create-user +create-database +create-tables +create-functions +create-indexes diff --git a/database/list-events.sh b/database/list-events.sh new file mode 100755 index 0000000..bebd8af --- /dev/null +++ b/database/list-events.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -e + +clear + +echo +echo "Listing Events" +echo "= = =" +echo + +if [ -z ${DATABASE_USER+x} ]; then + echo "(DATABASE_USER is not set)" + user=eventstream +else + user=$DATABASE_USER +fi +echo "Database user is: $user" + +if [ -z ${DATABASE_NAME+x} ]; then + echo "(DATABASE_NAME is not set)" + database=eventstream +else + database=$DATABASE_NAME +fi +echo "Database name is: $database" + +echo + +function select-all-events { + psql $database -c "SELECT * FROM events" +} + +select-all-events diff --git a/database/tables/events-table.sql b/database/tables/events-table.sql new file mode 100644 index 0000000..f4d838b --- /dev/null +++ b/database/tables/events-table.sql @@ -0,0 +1,19 @@ +-- ---------------------------- +-- Table structure for events +-- ---------------------------- +DROP TABLE IF EXISTS "public"."events"; +CREATE TABLE "public"."events" ( + "stream_name" varchar(255) NOT NULL COLLATE "default", + "stream_position" int4 NOT NULL, + "type" varchar(255) NOT NULL COLLATE "default", + "global_position" bigserial NOT NULL , + "data" jsonb NOT NULL, + "metadata" jsonb, + "created_time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL +) +WITH (OIDS=FALSE); + +-- ---------------------------- +-- Primary key structure for table events +-- ---------------------------- +ALTER TABLE "public"."events" ADD PRIMARY KEY ("global_position") NOT DEFERRABLE INITIALLY IMMEDIATE; diff --git a/database/test/write-event-expected-version.sql b/database/test/write-event-expected-version.sql new file mode 100644 index 0000000..4ac2c20 --- /dev/null +++ b/database/test/write-event-expected-version.sql @@ -0,0 +1 @@ +SELECT write_event('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}', 1); diff --git a/database/test/write-event.sql b/database/test/write-event.sql new file mode 100644 index 0000000..46109b9 --- /dev/null +++ b/database/test/write-event.sql @@ -0,0 +1 @@ +SELECT write_event('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}'); diff --git a/database/uninstall.sh b/database/uninstall.sh new file mode 100755 index 0000000..ffb39b8 --- /dev/null +++ b/database/uninstall.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +set -e + +clear + +echo +echo "Uninstalling Database" +echo "= = =" +echo + +if [ -z ${DATABASE_USER+x} ]; then + echo "(DATABASE_USER is not set)" + user=eventstream +else + user=$DATABASE_USER +fi + +if [ -z ${DATABASE_NAME+x} ]; then + echo "(DATABASE_NAME is not set)" + database=eventstream +else + database=$DATABASE_NAME +fi + +echo + +function delete-user { + echo "Database user is: $user" + + user_exists=`psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` + + if [ "$user_exists" = "1" ]; then + echo "Deleting database user \"$user\"" + dropuser $user + else + echo "Database user \"$user\" does not exist. Not deleting." + fi + + echo +} + +function delete-database { + echo "Database name is: $database" + + database_exists=`psql -tAc "SELECT 1 FROM pg_database WHERE datname='$database'"` + + if [ "$database_exists" = "1" ]; then + echo "Deleting database \"$database\"" + dropdb $database + else + echo "Database \"$database\" does not exist. Not deleting." + fi + + echo +} + + +delete-database +delete-user diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec new file mode 100644 index 0000000..29575bd --- /dev/null +++ b/event_source-postgres.gemspec @@ -0,0 +1,34 @@ +# -*- encoding: utf-8 -*- +Gem::Specification.new do |s| + s.name = 'event_stream-postgres' + s.version = '0.0.0.0' + s.summary = 'Event stream client for PostgreSQL' + s.description = ' ' + + s.authors = ['The Eventide Project'] + s.email = 'opensource@eventide-project.org' + s.homepage = 'https://github.com/eventide-project/event-stream-postgres' + s.licenses = ['MIT'] + + s.require_paths = ['lib'] + s.files = Dir.glob('{lib}/**/*') + s.platform = Gem::Platform::RUBY + s.required_ruby_version = '>= 2.2.3' + + s.executables = ['install-event-stream-database', 'uninstall-event-stream-database'] + s.bindir = 'bin' + + s.add_runtime_dependency 'telemetry' + s.add_runtime_dependency 'telemetry-logger' + s.add_runtime_dependency 'casing' + s.add_runtime_dependency 'schema' + s.add_runtime_dependency 'initializer' + s.add_runtime_dependency 'serialize' + s.add_runtime_dependency 'settings' + s.add_runtime_dependency 'async_invocation' + s.add_runtime_dependency 'controls' + + s.add_runtime_dependency 'pg' + + s.add_development_dependency 'test_bench' +end diff --git a/init.rb b/init.rb new file mode 100644 index 0000000..4ea7e08 --- /dev/null +++ b/init.rb @@ -0,0 +1,10 @@ +lib_dir = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift lib_dir unless $LOAD_PATH.include?(lib_dir) + +libraries_dir = ENV['LIBRARIES_HOME'] +unless libraries_dir.nil? + libraries_dir = File.expand_path(libraries_dir) + $LOAD_PATH.unshift libraries_dir unless $LOAD_PATH.include?(libraries_dir) +end + +require 'event_stream/postgres' diff --git a/install-gems.sh b/install-gems.sh new file mode 100755 index 0000000..2ea4b74 --- /dev/null +++ b/install-gems.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +set -e + +if [ -z ${GEM_AUTHORITY_PATH+x} ]; then + echo "GEM_AUTHORITY_PATH is not set" + exit +fi + +echo +echo 'Installing local gems' +echo '= = =' + +source ./set-local-gem-path.sh + +echo +echo 'Removing gem files' +echo '- - -' +if test -n "$(find . -maxdepth 1 -name '*.gem' -print -quit)"; then + for gem in *.gem; do + echo "- $gem" + rm $gem + done +else + echo "(No gem files found)" +fi + +echo +echo 'Building gems' +echo '- - -' +for gemspec in *.gemspec; do + echo "- $gemspec" + gem build $gemspec +done + +if [ -z ${POSTURE+x} ]; then + echo "(POSTURE is not set. Using \"operational\" by default.)" + posture="operational" +else + posture=$POSTURE +fi + +scheme="https:" +gem_repo_authority_path=$GEM_AUTHORITY_PATH +public_gem_repo_uri="$scheme//$gem_repo_authority_path" + +gemfury_token="" +if [ ! -z ${GEMFURY_TOKEN+x} ]; then + gemfury_token=$GEMFURY_TOKEN +fi + +private_source="" +if [ ! $gemfury_token = "" ]; then + private_gem_repo_uri="$scheme//$gemfury_token@$gem_repo_authority_path" + private_source="--source $private_gem_repo_uri" +fi + +public_source="--source $public_gem_repo_uri" + +ruby_gems_source="--source https://rubygems.org" + +echo +echo "Installing gems locally (posture: $posture)" +echo '- - -' +for gem in *.gem; do + echo "($gem)" + cmd="gem install $gem --clear-sources $private_source $public_source $ruby_gems_source --install-dir ./gems" + + if [ operational != "$posture" ]; then + cmd="$cmd --development" + fi + + echo $cmd + ($cmd) || exit 1 +done + +echo '= = =' +echo '(done)' +echo diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb new file mode 100644 index 0000000..9d1709e --- /dev/null +++ b/lib/event_source/postgres.rb @@ -0,0 +1,35 @@ +require 'pg' + +require 'casing' +require 'clock' +require 'dependency'; Dependency.activate +require 'schema' +require 'initializer'; Initializer.activate +require 'serialize' +require 'settings'; Settings.activate +require 'telemetry' +require 'telemetry/logger' +require 'async_invocation' + +require 'event_stream/postgres/no_stream' + +require 'event_stream/postgres/stream_name' +require 'event_stream/postgres/stream' +require 'event_stream/postgres/event_data' +require 'event_stream/postgres/event_data/hash' +require 'event_stream/postgres/event_data/write' +require 'event_stream/postgres/event_data/read' + +require 'event_stream/postgres/settings' +require 'event_stream/postgres/session' + +require 'event_stream/postgres/put' +require 'event_stream/postgres/write' + +require 'event_stream/postgres/get/select_statement' +require 'event_stream/postgres/get' +require 'event_stream/postgres/iterator/cycle' +require 'event_stream/postgres/iterator' +require 'event_stream/postgres/read' + + diff --git a/lib/event_source/postgres/controls.rb b/lib/event_source/postgres/controls.rb new file mode 100644 index 0000000..69b8efa --- /dev/null +++ b/lib/event_source/postgres/controls.rb @@ -0,0 +1,10 @@ +require 'controls' + +require 'event_stream/postgres/controls/category' +require 'event_stream/postgres/controls/stream_name' +require 'event_stream/postgres/controls/stream' +require 'event_stream/postgres/controls/event_data' +require 'event_stream/postgres/controls/event_data/hash' +require 'event_stream/postgres/controls/event_data/metadata' +require 'event_stream/postgres/controls/event_data/write' +require 'event_stream/postgres/controls/put' diff --git a/lib/event_source/postgres/controls/category.rb b/lib/event_source/postgres/controls/category.rb new file mode 100644 index 0000000..0140b65 --- /dev/null +++ b/lib/event_source/postgres/controls/category.rb @@ -0,0 +1,25 @@ +module EventStream + module Postgres + module Controls + module Category + def self.example(category: nil, randomize_category: nil) + if randomize_category.nil? + if !category.nil? + randomize_category = false + end + end + + randomize_category = true if randomize_category.nil? + + category ||= 'Test' + + if randomize_category + category = "#{category}#{Identifier::UUID.random.gsub('-', '')}" + end + + category + end + end + end + end +end diff --git a/lib/event_source/postgres/controls/event_data.rb b/lib/event_source/postgres/controls/event_data.rb new file mode 100644 index 0000000..b807c30 --- /dev/null +++ b/lib/event_source/postgres/controls/event_data.rb @@ -0,0 +1,26 @@ +module EventStream + module Postgres + module Controls + module EventData + def self.type + 'SomeType' + end + + def self.data + { :some_attribute => 'some value' } + end + + module JSON + def self.data(id=nil) + data = EventData.data + Casing::Camel.(data, symbol_to_string: true) + end + + def self.text + data.to_json + end + end + end + end + end +end diff --git a/lib/event_source/postgres/controls/event_data/hash.rb b/lib/event_source/postgres/controls/event_data/hash.rb new file mode 100644 index 0000000..627a8a6 --- /dev/null +++ b/lib/event_source/postgres/controls/event_data/hash.rb @@ -0,0 +1,30 @@ +module EventStream + module Postgres + module Controls + module EventData + module Hash + def self.data + { + some_attribute: 'some value' + } + end + + def self.example + EventStream::Postgres::EventData::Hash[data] + end + + module JSON + def self.data(id=nil) + data = Hash.data + Casing::Camel.(data, symbol_to_string: true) + end + + def self.text + ::JSON.generate(data) + end + end + end + end + end + end +end diff --git a/lib/event_source/postgres/controls/event_data/metadata.rb b/lib/event_source/postgres/controls/event_data/metadata.rb new file mode 100644 index 0000000..d005dfe --- /dev/null +++ b/lib/event_source/postgres/controls/event_data/metadata.rb @@ -0,0 +1,26 @@ +module EventStream + module Postgres + module Controls + module EventData + module Metadata + def self.data + { + some_meta_attribute: 'some meta value' + } + end + + module JSON + def self.data(id=nil) + data = Metadata.data + Casing::Camel.(data, symbol_to_string: true) + end + + def self.text + data.to_json + end + end + end + end + end + end +end diff --git a/lib/event_source/postgres/controls/event_data/write.rb b/lib/event_source/postgres/controls/event_data/write.rb new file mode 100644 index 0000000..d8b956d --- /dev/null +++ b/lib/event_source/postgres/controls/event_data/write.rb @@ -0,0 +1,42 @@ +module EventStream + module Postgres + module Controls + module EventData + module Write + def self.data + { + :type => EventData.type, + :data => EventData.data, + :metadata => EventData::Metadata.data + } + end + + def self.example(type: nil, data: nil, metadata: nil) + type ||= EventData.type + data ||= EventData.data + metadata ||= EventData::Metadata.data + + event_data = EventStream::Postgres::EventData::Write.build + + event_data.type = type + event_data.data = data + event_data.metadata = metadata + + event_data + end + + module JSON + def self.data + data = Write.data + Casing::Camel.(data, symbol_to_string: true) + end + + def self.text + data.to_json + end + end + end + end + end + end +end diff --git a/lib/event_source/postgres/controls/put.rb b/lib/event_source/postgres/controls/put.rb new file mode 100644 index 0000000..e8eb823 --- /dev/null +++ b/lib/event_source/postgres/controls/put.rb @@ -0,0 +1,19 @@ +module EventStream + module Postgres + module Controls + module Put + def self.call(instances: nil, stream_name: nil, event: nil, category: nil) + instances ||= 1 + stream_name ||= StreamName.example(category: category) + event ||= EventData::Write.example + + instances.times do + EventStream::Postgres::Put.(stream_name, event) + end + + stream_name + end + end + end + end +end diff --git a/lib/event_source/postgres/controls/stream.rb b/lib/event_source/postgres/controls/stream.rb new file mode 100644 index 0000000..2f8d6fe --- /dev/null +++ b/lib/event_source/postgres/controls/stream.rb @@ -0,0 +1,19 @@ +module EventStream + module Postgres + module Controls + module Stream + def self.example(stream_name: nil, category: nil, id: nil, randomize_category: nil) + stream_name ||= StreamName.example category: category, id: id, randomize_category: randomize_category + ::Stream.build stream_name: stream_name + end + + module Category + def self.example(category: nil, randomize_category: nil) + category ||= Controls::Category.example category: category, randomize_category: randomize_category + ::Stream.build category: category + end + end + end + end + end +end diff --git a/lib/event_source/postgres/controls/stream_name.rb b/lib/event_source/postgres/controls/stream_name.rb new file mode 100644 index 0000000..949ee8d --- /dev/null +++ b/lib/event_source/postgres/controls/stream_name.rb @@ -0,0 +1,14 @@ +module EventStream + module Postgres + module Controls + module StreamName + def self.example(category: nil, id: nil, randomize_category: nil) + category ||= Category.example category: category, randomize_category: randomize_category + id ||= Identifier::UUID.random + + "#{category}-#{id}" + end + end + end + end +end diff --git a/lib/event_source/postgres/event_data.rb b/lib/event_source/postgres/event_data.rb new file mode 100644 index 0000000..1f7d29e --- /dev/null +++ b/lib/event_source/postgres/event_data.rb @@ -0,0 +1,11 @@ +module EventStream + module Postgres + class EventData + include Schema::DataStructure + + attribute :type + attribute :data + attribute :metadata + end + end +end diff --git a/lib/event_source/postgres/event_data/hash.rb b/lib/event_source/postgres/event_data/hash.rb new file mode 100644 index 0000000..08dbbef --- /dev/null +++ b/lib/event_source/postgres/event_data/hash.rb @@ -0,0 +1,33 @@ +module EventStream + module Postgres + class EventData + class Hash < ::Hash + module Serializer + def self.json + JSON + end + + def self.instance(raw_data) + Hash[raw_data] + end + + def self.raw_data(instance) + Hash[instance] + end + + module JSON + def self.serialize(raw_hash_data) + json_formatted_data = Casing::Camel.(raw_hash_data) + ::JSON.generate(json_formatted_data) + end + + def self.deserialize(text) + json_formatted_data = ::JSON.parse(text, :symbolize_names => true) + Casing::Underscore.(json_formatted_data) + end + end + end + end + end + end +end diff --git a/lib/event_source/postgres/event_data/read.rb b/lib/event_source/postgres/event_data/read.rb new file mode 100644 index 0000000..5a1c381 --- /dev/null +++ b/lib/event_source/postgres/event_data/read.rb @@ -0,0 +1,18 @@ +module EventStream + module Postgres + class EventData + class Read < EventData + include Schema::DataStructure + + attribute :stream_name + attribute :stream_position + attribute :global_position + attribute :created_time + + def category + StreamName.category(stream_name) + end + end + end + end +end diff --git a/lib/event_source/postgres/event_data/write.rb b/lib/event_source/postgres/event_data/write.rb new file mode 100644 index 0000000..c989c33 --- /dev/null +++ b/lib/event_source/postgres/event_data/write.rb @@ -0,0 +1,8 @@ +module EventStream + module Postgres + class EventData + class Write < EventData + end + end + end +end diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb new file mode 100644 index 0000000..8e047c2 --- /dev/null +++ b/lib/event_source/postgres/get.rb @@ -0,0 +1,93 @@ +module EventStream + module Postgres + class Get + initializer :stream_name, :category, :batch_size, :precedence + + dependency :session, Session + dependency :logger, Telemetry::Logger + + def self.build(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, session: nil) + new(stream_name, category, batch_size, precedence).tap do |instance| + instance.configure(session: session) + end + end + + def self.configure(receiver, attr_name: nil, stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, session: nil) + attr_name ||= :get + instance = build(stream_name: stream_name, category: category, batch_size: batch_size, precedence: precedence, session: session) + receiver.public_send "#{attr_name}=", instance + end + + def self.call(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, session: nil) + instance = build(stream_name: stream_name, category: category, batch_size: batch_size, precedence: precedence, session: session) + instance.(stream_position: stream_position) + end + + def configure(session: nil) + Session.configure self, session: session + Telemetry::Logger.configure self + end + + def call(stream_position: nil) + logger.opt_trace "Getting event data (Stream Position: #{stream_position.inspect}, Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + + stream = Stream.build stream_name: stream_name, category: category + records = get_records(stream, stream_position) + + events = convert(records) + + logger.opt_debug "Finished getting event data (Count: #{events.length}, Stream Position: #{stream_position.inspect}, Stream: #{stream.name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + + events + end + + def get_records(stream, stream_position) + logger.opt_trace "Getting records (Stream: #{stream.name}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + + select_statement = SelectStatement.build(stream, offset: stream_position, batch_size: batch_size, precedence: precedence) + + records = session.connection.exec(select_statement.sql) + + logger.opt_debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + + records + end + + def convert(records) + logger.opt_trace "Converting records to event data (Records Count: #{records.ntuples})" + + events = records.map do |record| + record['data'] = Deserialize.data(record['data']) + record['metadata'] = Deserialize.metadata(record['metadata']) + record['created_time'] = Time.utc_coerced(record['created_time']) + + EventData::Read.build record + end + + logger.opt_debug "Converted records to event data (Event Data Count: #{events.length})" + + events + end + + module Deserialize + def self.data(serialized_data) + Serialize::Read.(serialized_data, EventData::Hash, :json) + end + + def self.metadata(serialized_metadata) + if serialized_metadata.nil? + nil + else + Serialize::Read.(serialized_metadata, EventData::Hash, :json) + end + end + end + + module Time + def self.utc_coerced(local_time) + Clock::UTC.coerce(local_time) + end + end + end + end +end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb new file mode 100644 index 0000000..4a19476 --- /dev/null +++ b/lib/event_source/postgres/get/select_statement.rb @@ -0,0 +1,94 @@ +module EventStream + module Postgres + class Get + class SelectStatement + initializer :stream, w(:offset), w(:batch_size), w(:precedence) + + dependency :logger, Telemetry::Logger + + def offset + @offset ||= Defaults.offset + end + + def batch_size + @batch_size ||= Defaults.batch_size + end + + def precedence + @precedence ||= Defaults.precedence + end + + def stream_name + stream.name + end + + def stream_type + stream.type + end + + def self.build(stream, offset: nil, batch_size: nil, precedence: nil) + new(stream, offset, batch_size, precedence).tap do |instance| + instance.configure + end + end + + def configure + Telemetry::Logger.configure self + end + + def sql + logger.opt_trace "Composing select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" + + statement = <<-SQL + SELECT + stream_name::varchar, + stream_position::int, + type::varchar, + global_position::bigint, + data::varchar, + metadata::varchar, + created_time::timestamp + FROM + events + WHERE + #{where_clause_field} = '#{stream_name}' + ORDER BY + global_position #{precedence.to_s.upcase} + LIMIT + #{batch_size} + OFFSET + #{offset} + ; + SQL + + logger.opt_debug "Composed select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" + logger.opt_data "Statement: #{statement}" + + statement + end + + def where_clause_field + if stream.type == :stream + 'stream_name' + else + 'category(stream_name)' + end + end + + module Defaults + def self.offset + 0 + end + + def self.batch_size + 1000 + end + + def self.precedence + :asc + end + end + end + end + end +end diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb new file mode 100644 index 0000000..7b69861 --- /dev/null +++ b/lib/event_source/postgres/iterator.rb @@ -0,0 +1,106 @@ +module EventStream + module Postgres + class Iterator + class Error < RuntimeError; end + + attr_accessor :batch + attr_writer :batch_position + attr_writer :stream_offset + + def batch_position + @batch_position ||= 0 + end + + def stream_offset + @stream_offset ||= (stream_position || 0) + end + + dependency :get, Get + dependency :cycle, Cycle + dependency :logger, Telemetry::Logger + + initializer :stream_name, :category, a(:stream_position, 0), :batch_size, :precedence + + def self.build(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + new(stream_name, category, stream_position, batch_size, precedence).tap do |instance| + Get.configure instance, stream_name: stream_name, category: category, batch_size: batch_size, precedence: precedence, session: session + Telemetry::Logger.configure instance + Iterator::Cycle.configure instance, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle + end + end + + def self.configure(receiver, attr_name: nil, stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + attr_name ||= :iterator + instance = build(stream_name: stream_name, category: category, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) + receiver.public_send "#{attr_name}=", instance + end + + def next + logger.opt_trace "Getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" + + resupply(batch) + + event_data = batch[batch_position] + + logger.opt_debug "Finished getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" + logger.opt_data "Event Data: #{event_data.inspect}" + + advance_positions + + event_data + end + + def resupply(batch) + logger.opt_trace "Resupplying batch" + + if batch.nil? + batch_log_text = "Batch: #{batch.inspect}" + else + batch_log_text = "Batch Length: #{batch.length}" + end + + if batch.nil? || batch_position == batch.length + logger.debug "Current batch is depleted (#{batch_log_text}, Batch Position: #{batch_position})" + + batch = get_batch + reset(batch) + else + logger.debug "Current batch not depleted (#{batch_log_text}, Batch Position: #{batch_position})" + end + + logger.opt_debug "Finished resupplying batch" + end + + def reset(batch) + logger.opt_trace "Resetting batch" + + self.batch = batch + self.batch_position = 0 + + logger.opt_debug "Reset batch" + logger.opt_data ("Batch: #{batch.inspect}") + logger.opt_data ("Batch Position: #{batch_position.inspect}") + end + + def get_batch + logger.opt_trace "Getting batch" + + batch = nil + cycle.() do + batch = get.(stream_position: stream_offset) + end + + logger.opt_debug "Finished getting batch (Count: #{batch.length})" + + batch + end + + def advance_positions + logger.opt_trace "Advancing positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" + self.batch_position += 1 + self.stream_offset += 1 + logger.opt_debug "Advanced positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" + end + end + end +end diff --git a/lib/event_source/postgres/iterator/cycle.rb b/lib/event_source/postgres/iterator/cycle.rb new file mode 100644 index 0000000..6dc4e08 --- /dev/null +++ b/lib/event_source/postgres/iterator/cycle.rb @@ -0,0 +1,174 @@ +module EventStream + module Postgres + class Iterator + class Cycle + dependency :clock, Clock::UTC + dependency :telemetry, Telemetry + dependency :logger, Telemetry::Logger + + def delay_milliseconds + @delay_milliseconds ||= Defaults.delay_milliseconds + end + + def delay_condition + @delay_condition ||= Defaults.delay_condition + end + + initializer na(:delay_milliseconds), :timeout_milliseconds, na(:delay_condition) + + def self.build(delay_milliseconds: nil, timeout_milliseconds: nil, delay_condition: nil) + new(delay_milliseconds, timeout_milliseconds, delay_condition).tap do |instance| + instance.configure + end + end + + def self.configure(receiver, attr_name: nil, delay_milliseconds: nil, timeout_milliseconds: nil, delay_condition: nil, cycle: nil) + attr_name ||= :cycle + + if !cycle.nil? + instance = cycle + else + if delay_milliseconds.nil? && timeout_milliseconds.nil? + instance = None.build + else + instance = build(delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, delay_condition: delay_condition) + end + end + + receiver.public_send "#{attr_name}=", instance + end + + def configure + Clock::UTC.configure self + ::Telemetry.configure self + ::Telemetry::Logger.configure self + end + + def self.call(delay_milliseconds: nil, timeout_milliseconds: nil, delay_condition: nil, &action) + instance = build(delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, delay_condition: delay_condition) + instance.call(&action) + end + + def call(&action) + stop_time = nil + if !timeout_milliseconds.nil? + stop_time = clock.now + (timeout_milliseconds.to_f / 1000.0) + end + + logger.opt_trace "Cycling (Delay Milliseconds: #{delay_milliseconds}, Timeout Milliseconds: #{timeout_milliseconds.inspect}, Stop Time: #{clock.iso8601(stop_time)})" + + iteration = -1 + result = nil + loop do + iteration += 1 + telemetry.record :cycle, iteration + + result = invoke(iteration, &action) + + if delay_condition.(result) + logger.opt_debug "No results (Iteration: #{iteration})" + delay + else + logger.opt_debug "Got results (Iteration: #{iteration})" + telemetry.record :got_result + break + end + + if !timeout_milliseconds.nil? + now = clock.now + if now >= stop_time + logger.opt_debug "Timeout has lapsed (Iteration: #{iteration}, Stop Time: #{clock.iso8601(stop_time)}, Timeout Milliseconds: #{timeout_milliseconds})" + telemetry.record :timed_out, now + break + end + end + end + + logger.opt_trace "Cycled (Iterations: #{iteration}, Delay Milliseconds: #{delay_milliseconds}, Timeout Milliseconds: #{timeout_milliseconds.inspect}, Stop Time: #{clock.iso8601(stop_time)})" + + return result + end + + def invoke(iteration, &action) + logger.opt_trace "Invoking action (Iteration: #{iteration})" + + result = action.call + telemetry.record :invoked_action + + logger.opt_debug "Invoked action (Iteration: #{iteration})" + + result + end + + def delay + logger.opt_trace "Delaying (Milliseconds: #{delay_milliseconds})" + + delay_seconds = (delay_milliseconds.to_f / 1000.0) + + sleep delay_seconds + + telemetry.record :delayed, delay_milliseconds + + logger.opt_debug "Finished delaying (Milliseconds: #{delay_milliseconds})" + end + + def self.register_telemetry_sink(writer) + sink = Telemetry.sink + writer.telemetry.register sink + sink + end + + module Telemetry + class Sink + include ::Telemetry::Sink + + record :cycle + record :invoked_action + record :got_result + record :delayed + record :timed_out + end + + Data = Struct.new :milliseconds + + def self.sink + Sink.new + end + end + + module Substitute + def self.build + Cycle::None.build.tap do |instance| + sink = Cycle.register_telemetry_sink(instance) + instance.sink = sink + end + end + end + + class None < Cycle + attr_accessor :sink + + def call(&action) + action.call + end + end + + module Defaults + def self.delay_milliseconds + 200 + end + + def self.delay_condition + lambda do |result| + if result.respond_to? :empty? + result.empty? + else + result.nil? + end + end + end + end + end + end + end +end diff --git a/lib/event_source/postgres/no_stream.rb b/lib/event_source/postgres/no_stream.rb new file mode 100644 index 0000000..2d56eb5 --- /dev/null +++ b/lib/event_source/postgres/no_stream.rb @@ -0,0 +1,13 @@ +module EventStream + module Postgres + module NoStream + def self.name + :no_stream + end + + def self.version + -1 + end + end + end +end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb new file mode 100644 index 0000000..fe90292 --- /dev/null +++ b/lib/event_source/postgres/put.rb @@ -0,0 +1,130 @@ +module EventStream + module Postgres + class Put + attr_reader :stream_name + + dependency :session, Session + dependency :logger, Telemetry::Logger + + def initialize(stream_name) + @stream_name = stream_name + end + + def self.build(stream_name, session: nil) + new(stream_name).tap do |instance| + instance.configure(session: session) + end + end + + def configure(session: nil) + Session.configure(self, session: session) + Telemetry::Logger.configure(self) + end + + def self.call(stream_name, write_event, expected_version: nil, session: nil) + instance = build(stream_name, session: session) + instance.(write_event, expected_version: expected_version) + end + + def call(write_event, expected_version: nil) + logger.opt_trace "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" + + type, data, metadata = destructure_event(write_event) + expected_version = canonize_expected_version(expected_version) + + stream_position = insert_event(type, data, metadata, expected_version) + + logger.opt_debug "Put event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" + + stream_position + end + + def destructure_event(write_event) + type = write_event.type + data = write_event.data + metadata = write_event.metadata + + logger.opt_data "Data: #{data.inspect}" + logger.opt_data "Metadata: #{metadata.inspect}" + + return type, data, metadata + end + + def canonize_expected_version(expected_version) + return expected_version unless expected_version == NoStream.name + + logger.opt_trace "Canonizing expected version (Expected Version: #{expected_version})" + expected_version = NoStream.version + logger.opt_debug "Canonized expected version (Expected Version: #{expected_version})" + expected_version + end + + def insert_event(type, data, metadata, expected_version) + serialized_data = serialized_data(data) + serialized_metadata = serialized_metadata(metadata) + records = execute_query(type, serialized_data, serialized_metadata, expected_version) + stream_position(records) + end + + def execute_query(type, serialized_data, serialized_metadata, expected_version) + logger.opt_trace "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" + + sql_args = [ + stream_name, + type, + serialized_data, + serialized_metadata, + expected_version + ] + + begin + records = session.connection.exec_params(statement, sql_args) + rescue PG::RaiseException => e + raise_error e + end + + logger.opt_debug "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" + + records + end + + def statement + "SELECT write_event($1::varchar, $2::varchar, $3::jsonb, $4::jsonb, $5::int);" + end + + def serialized_data(data) + serializable_data = EventData::Hash[data] + serialized_data = Serialize::Write.(serializable_data, :json) + logger.opt_data "Serialized Data: #{serialized_data.inspect}" + serialized_data + end + + def serialized_metadata(metadata) + serializable_metadata = EventData::Hash[metadata] + serialized_metadata = nil + unless metadata.nil? + serialized_metadata = Serialize::Write.(serializable_metadata, :json) + end + logger.opt_data "Serialized Metadata: #{serialized_metadata.inspect}" + serialized_metadata + end + + def stream_position(records) + stream_position = nil + unless records[0].nil? + stream_position = records[0].values[0] + end + stream_position + end + + def raise_error(pg_error) + error_message = pg_error.message + if error_message.include? 'Wrong expected version' + error_message.gsub!('ERROR:', '').strip! + raise Write::ExpectedVersionError, error_message + end + raise pg_error + end + end + end +end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb new file mode 100644 index 0000000..2e651cc --- /dev/null +++ b/lib/event_source/postgres/read.rb @@ -0,0 +1,59 @@ +module EventStream + module Postgres + class Read + class Error < RuntimeError; end + + initializer :stream_name, :category, :stream_position, :batch_size, :precedence + + dependency :session, Session + dependency :iterator, Iterator + dependency :logger, Telemetry::Logger + + def self.build(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + new(stream_name, category, stream_position, batch_size, precedence).tap do |instance| + Iterator.configure instance, stream_name: stream_name, category: category, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session + Telemetry::Logger.configure instance + end + end + + def self.call(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil, &action) + instance = build(stream_name: stream_name, category: category, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) + instance.(&action) + end + + def self.configure(receiver, attr_name: nil, stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + attr_name ||= :reader + instance = build(stream_name: stream_name, category: category, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) + receiver.public_send "#{attr_name}=", instance + end + + def call(&action) + if action.nil? + error_message = "Reader must be actuated with a block" + logger.error error_message + raise Error, error_message + end + + enumerate_event_data(&action) + + return AsyncInvocation::Incorrect + end + + def enumerate_event_data(&action) + logger.opt_trace "Reading event data (Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + + event_data = nil + + loop do + event_data = iterator.next + + break if event_data.nil? + + action.(event_data) + end + + logger.opt_debug "Finished reading event data (Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + end + end + end +end diff --git a/lib/event_source/postgres/session.rb b/lib/event_source/postgres/session.rb new file mode 100644 index 0000000..286e6a5 --- /dev/null +++ b/lib/event_source/postgres/session.rb @@ -0,0 +1,109 @@ +module EventStream + module Postgres + class Session + def self.settings + Settings.names + end + + settings.each do |s| + setting s + end + + attr_accessor :connection + + dependency :logger, Telemetry::Logger + + def self.build(connection: nil, settings: nil) + new.tap do |instance| + Telemetry::Logger.configure instance + + settings ||= Settings.instance + + settings.set(instance) + + connect(instance, connection) + end + end + + def self.configure(receiver, session: nil, attr_name: nil) + attr_name ||= :session + + instance = session || build + receiver.public_send "#{attr_name}=", instance + instance + end + + def self.connect(instance, connection=nil) + logger.trace "Connecting to database" + + if connection.nil? + logger.debug "No connection. A new one will be built." + connection = build_connection(instance) + else + logger.debug "Reusing existing connection" + end + + instance.connection = connection + + logger.debug "Connected to database" + + connection + end + + def self.build_connection(instance) + settings = instance.settings + logger.trace "Building new connection to database (Settings: #{LogText.settings(settings).inspect})" + + connection = PG::Connection.open(settings) + connection.type_map_for_results = PG::BasicTypeMapForResults.new(connection) + + logger.trace "Built new connection to database (Settings: #{LogText.settings(settings).inspect})" + + connection + end + + def connect + self.class.connect(self) + end + + def connected? + !connection.nil? && connection.status == PG::CONNECTION_OK + end + alias :open? :connected? + + def close + connection.close + connection = nil + end + + def reset + connection.reset + end + + def settings + settings = {} + self.class.settings.each do |s| + val = public_send(s) + settings[s] = val unless val.nil? + end + settings + end + + def self.logger + @logger ||= Telemetry::Logger.get self + end + + module LogText + def self.settings(settings) + s = settings.dup + + if s.has_key?(:password) + s[:password] = '(hidden)' + end + + s + end + end + end + end +end diff --git a/lib/event_source/postgres/settings.rb b/lib/event_source/postgres/settings.rb new file mode 100644 index 0000000..e58e07e --- /dev/null +++ b/lib/event_source/postgres/settings.rb @@ -0,0 +1,30 @@ +module EventStream + module Postgres + class Settings < ::Settings + def self.instance + @instance ||= build + end + + def self.data_source + 'settings/event_stream_postgres.json' + end + + def self.names + [ + :dbname, + :host, + :hostaddr, + :port, + :user, + :password, + :connect_timeout, + :options, + :sslmode, + :krbsrvname, + :gsslib, + :service + ] + end + end + end +end diff --git a/lib/event_source/postgres/stream.rb b/lib/event_source/postgres/stream.rb new file mode 100644 index 0000000..f2e0534 --- /dev/null +++ b/lib/event_source/postgres/stream.rb @@ -0,0 +1,43 @@ +module EventStream + module Postgres + class Stream + class NameError < RuntimeError; end + + initializer :stream_name, :category, :type + + def name + if type == :stream + return stream_name + else + return category + end + end + + def self.build(stream_name: nil, category: nil) + stream_name, category, type = get_name(stream_name, category) + new stream_name, category, type + end + + def self.get_name(stream_name, category) + if !stream_name.nil? && !category.nil? + raise NameError, "Both stream stream_name and category are specified. Specify one or the other." + end + + type = nil + if !stream_name.nil? + category = StreamName.category(stream_name) + type = :stream + else + stream_name = category + type = :category + end + + if stream_name.nil? && category.nil? + raise NameError, "Neither stream name nor category are specified. Specify one or the other." + end + + return stream_name, category, type + end + end + end +end diff --git a/lib/event_source/postgres/stream_name.rb b/lib/event_source/postgres/stream_name.rb new file mode 100644 index 0000000..ded9f8f --- /dev/null +++ b/lib/event_source/postgres/stream_name.rb @@ -0,0 +1,25 @@ +module EventStream + module Postgres + module StreamName + extend self + + def stream_name(category_name, id=nil) + id ||= Identifier::UUID.random + + "#{category_name}-#{id}" + end + + def category_stream_name(category_name) + category_name + end + + def self.get_id(stream_name) + Identifier::UUID.parse(stream_name) + end + + def self.category(stream_name) + stream_name.split('-').first + end + end + end +end diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb new file mode 100644 index 0000000..9782277 --- /dev/null +++ b/lib/event_source/postgres/write.rb @@ -0,0 +1,7 @@ +module EventStream + module Postgres + class Write + class ExpectedVersionError < RuntimeError; end + end + end +end diff --git a/library-symlinks.sh b/library-symlinks.sh new file mode 100755 index 0000000..7d969d9 --- /dev/null +++ b/library-symlinks.sh @@ -0,0 +1,88 @@ +set -e + +if [ -z ${LIBRARIES_DIR+x} ]; then + echo "LIBRARIES_DIR must be set to the libraries directory path... exiting" + exit 1 +fi + +if [ ! -d "$LIBRARIES_DIR" ]; then + echo "$LIBRARIES_DIR does not exit... exiting" + exit 1 +fi + +# function make_directories { +# directories=("$@") + +# for directory in "${directories[@]}"; do +# make_directory $directory +# done +# } + +function make_directory { + directory=$1 + + lib_directory="$LIBRARIES_DIR/$directory" + + if [ ! -d "$lib_directory" ]; then + echo "- making directory $lib_directory" + mkdir -p "$lib_directory" + fi +} + +function remove_lib_symlinks { + name=$1 + directory=$2 + + dest="$LIBRARIES_DIR" + if [ ! -z "$directory" ]; then + dest="$dest/$directory" + fi + dest="$dest/$name" + + for entry in $dest*; do + if [ -h "$entry" ]; then + echo "- removing symlink: $entry" + rm $entry + fi + done +} + +function symlink_lib { + name=$1 + directory=$2 + + echo + echo "Symlinking $name" + echo "- - -" + + remove_lib_symlinks $name $directory + + src="$(PWD)/lib" + dest="$LIBRARIES_DIR" + if [ ! -z "$directory" ]; then + src="$src/$directory" + dest="$dest/$directory" + + make_directory $directory + fi + src="$src/$name" + + echo "- destination is $dest" + + full_name=$directory/$name + + for entry in $src*; do + entry_basename=$(basename $entry) + dest_item="$dest/$entry_basename" + + echo "- symlinking $entry_basename to $dest_item" + + cmd="ln -s $entry $dest_item" + echo $cmd + ($cmd) + done + + echo "- - -" + echo "($name done)" + echo +} diff --git a/materials/nathans_sketch.rb b/materials/nathans_sketch.rb new file mode 100644 index 0000000..972b012 --- /dev/null +++ b/materials/nathans_sketch.rb @@ -0,0 +1,33 @@ +class PostgresqlReader + attr_writer :batch + attr_writer :position + + def initialize(category) + @category = category + end + + # Implementing the reader as a pull system + def next + row = pg.execute "SELECT * FROM events WHERE position = ? AND category = ?", position, category + + position += 1 + + event_data = Serialize::Read.(row, :postgresql, EventData) + + event_data + end + + # the pull system (actuated via #next) easily can become a push system. + # The inverse, going from push to pull, requires synchronization via + # eg fibers, Enumerable::Lazy. etc. + def start(&block) + loop do + event_data = self.next + block.(event_data) + end + end + + def position + @position ||= 0 + end +end diff --git a/materials/new_school_sketch.rb b/materials/new_school_sketch.rb new file mode 100644 index 0000000..bfc2a43 --- /dev/null +++ b/materials/new_school_sketch.rb @@ -0,0 +1,82 @@ +# calls the handle method even if it doesn't exist +handler.send(message) + +# only calls the handle method if it exists +handler.handle(message) + +# Proposal: Just the reader and writer in the client library +# No messaging reader, subscription, or writer in the +# messaging library + +Read.(stream: stream_name) do |event_data| + dispatcher.(event_data) +end + +# or the instance + +reader.(stream: stream_name) do |event_data| + dispatcher.(event_data) +end + +# Note: no need for messaging reader above. The dispatcher +# is the place where deserialization is done right now. +# So there's no need for a Messaging::MessageReader. + +# the handler is still in the dispatcher + +class SomeDispatcher + handler SomeHandler +end + +# This is a subscription + +Read.(category: category_stream_name, poll_milliseconds: 500) do |event_data| + # ... +end + +# Projection + +# (from within the projection's instance code) +reader.(stream_name: stream_name) do |event_data| + apply(event_data) +end + +# In order to convert to/from event_data and message +# types, the reader needs to get a list of types +# So, message types can be had from anything that +# implements a message registry method, returning +# the existing implementation of a Message::Registry + +message_classes = [ + SomeMessage, + SomeOtherMessage +] + +Read.(stream_name: stream_name) do |event_data| + handler.(event_data) + # handler has enough info about message classes that it can + # xvert them +end + +# The writer +# This converts to event_data +writer.(message) + +# This doesn't need to convert to event_data +writer.(event_data) + +# Or with the optional block +writer.(message) do |event_data| + # do something with the event_data if wanted + # this block is for transparency to the opaque event_data before it's saved +end + +# Writer returns event_data, either way +event_data = writer.(message) +event_data = writer.(event_data) + + +# The writer's class actuator +Write.(message, message_classes) + # why does this take message classes? + # it does not de-serialize diff --git a/materials/poll.rb b/materials/poll.rb new file mode 100644 index 0000000..1e44be4 --- /dev/null +++ b/materials/poll.rb @@ -0,0 +1,21 @@ +Read.(category: category_stream_name, poll_milliseconds: 500) do |event_data| + # ... +end + + +Poll.(delay_milliseconds: 500, timeout_milliseconds: 5000) do |poll_controller| + Read.(category: category_stream_name) + # has to decide how and whether to poll again + # if result is a nil, then it has to poll + # internally, poll tracks iterations, timeout. does reset of timeout, etc + +end + + +# Maybe an internal poll control? + + res = get_events ... + + if res.nil? && poll? # <- poll milliseconds are set + end + diff --git a/materials/retry_batch_message_sketch.rb b/materials/retry_batch_message_sketch.rb new file mode 100644 index 0000000..e309ef6 --- /dev/null +++ b/materials/retry_batch_message_sketch.rb @@ -0,0 +1,64 @@ +require 'securerandom' +require 'http_eventstore' +require 'active_support/core_ext/array' + +# Process message +class MessageProcessor + + def initialize + @client = HttpEventstore::Connection.new + end + + def process_message(message) + puts "Receieved message #{message.id}" + product_ids = get_product_ids(message) + + product_ids.in_groups_of(5, false).each_with_index do |group, index| + if index < 2 + stream_name = "inventory" + event_data = { event_type: "InventoryItemUpdated", + data: { product_ids: group }, + metadata: { "$correlationId": message.id, + "$causationId": message.id }, + event_id: SecureRandom.uuid } + puts "Processed ids #{group.join(',')}" + puts "Publishing event #{event_data}" + client.append_to_stream(stream_name, event_data) + else + puts "Could not process ids #{group.join(',')}" + raise "Some Error" + end + end + + end + + private + + attr_reader :client + + def get_product_ids(message) + # Here we would query the event store by the message id + # to find all events that may have completed. + # the completed event product_ids would be removed + # from the original message, to resume the updates + message.product_ids + end +end + +PRODUCT_IDS = (0..20).to_a +MESSAGE_ID = SecureRandom.uuid + +message_processor = MessageProcessor.new +Message = Struct.new(:id, :product_ids) +retry_count = 0 + +begin + message_processor.process_message(message = Message.new(MESSAGE_ID, PRODUCT_IDS)) +rescue RuntimeError => e + if retry_count > 3 + puts "Dead letter message, couldnt process message #{message.id}" + else + retry_count += 1 + retry + end +end diff --git a/materials/subscription_sketch.rb b/materials/subscription_sketch.rb new file mode 100644 index 0000000..8257eab --- /dev/null +++ b/materials/subscription_sketch.rb @@ -0,0 +1,13 @@ +# Timeout and retry + +retry.() do + loop do + get_batch + + if no_batch + break unless timeout.nil? # maybe raise Timeout::Error + + sleep timeout + end + end +end diff --git a/remove-lib-symlinks.sh b/remove-lib-symlinks.sh new file mode 100755 index 0000000..2227b7c --- /dev/null +++ b/remove-lib-symlinks.sh @@ -0,0 +1,3 @@ +source ./library-symlinks.sh + +remove_lib_symlinks 'event_stream' 'postgres' diff --git a/set-local-gem-path.sh b/set-local-gem-path.sh new file mode 100755 index 0000000..f19a99e --- /dev/null +++ b/set-local-gem-path.sh @@ -0,0 +1,14 @@ +unchanged_gem_path=$GEM_PATH + +if [[ ! $GEM_PATH == *"./gems"* ]]; then + export GEM_PATH=./gems:$GEM_PATH + + echo "Gem path was changed" + echo " from: $unchanged_gem_path" + echo " to: $GEM_PATH" +else + echo "Gem path was unchanged" + echo " from: $unchanged_gem_path" +fi + +echo diff --git a/settings/event_stream_postgres.json b/settings/event_stream_postgres.json new file mode 100644 index 0000000..e20c94f --- /dev/null +++ b/settings/event_stream_postgres.json @@ -0,0 +1,14 @@ +{ + "dbname": "eventstream", + "host": "localhost", + "hostaddr": "127.0.0.1", + "port": 5432, + "user": "eventstream", + "password": null, + "connect_timeout": null, + "options": null, + "sslmode": null, + "krbsrvname": null, + "gsslib": null, + "service": null +} diff --git a/symlink-lib.sh b/symlink-lib.sh new file mode 100755 index 0000000..d940f25 --- /dev/null +++ b/symlink-lib.sh @@ -0,0 +1,3 @@ +source ./library-symlinks.sh + +symlink_lib 'event_stream' 'postgres' diff --git a/test.rb b/test.rb new file mode 100644 index 0000000..9efd53e --- /dev/null +++ b/test.rb @@ -0,0 +1 @@ +require_relative 'test/bench' diff --git a/test/automated.rb b/test/automated.rb new file mode 100644 index 0000000..62cd68b --- /dev/null +++ b/test/automated.rb @@ -0,0 +1,6 @@ +require_relative 'test_init' + +TestBench::Runner.( + 'bench/**/*.rb', + exclude_pattern: %r{/^skip_|(?:_init\.rb|\.sketch\.rb|_sketch\.rb|sketch\.rb|\.skip\.rb|_tests\.rb)\z} +) or exit 1 diff --git a/test/automated/automated_init.rb b/test/automated/automated_init.rb new file mode 100644 index 0000000..faa8577 --- /dev/null +++ b/test/automated/automated_init.rb @@ -0,0 +1 @@ +require_relative '../test_init' diff --git a/test/automated/event_data/hash.rb b/test/automated/event_data/hash.rb new file mode 100644 index 0000000..5e68d84 --- /dev/null +++ b/test/automated/event_data/hash.rb @@ -0,0 +1,23 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "EventData Hash" do + context "JSON serialize" do + example_hash = controls::EventData::Hash.example + control_serialized_text = controls::EventData::Hash::JSON.text + + serialized_text = Serialize::Write.(example_hash, :json) + + assert(serialized_text == control_serialized_text) + end + + context "JSON deserialize" do + example_serialized_text = controls::EventData::Hash::JSON.text + control_deserialized_hash = controls::EventData::Hash.example + + deserialized_hash = Serialize::Read.(example_serialized_text, EventData::Hash, :json) + + assert(deserialized_hash == control_deserialized_hash) + end +end diff --git a/test/automated/event_data/read_data_category.rb b/test/automated/event_data/read_data_category.rb new file mode 100644 index 0000000..2e6fb0f --- /dev/null +++ b/test/automated/event_data/read_data_category.rb @@ -0,0 +1,16 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Read Data" do + context "Category" do + read_data = EventData::Read.new + read_data.stream_name = 'some_stream-some_id' + + category = read_data.category + + test "Is the first segment of the stream name" do + assert(category == 'some_stream') + end + end +end diff --git a/test/automated/get/batch_size.rb b/test/automated/get/batch_size.rb new file mode 100644 index 0000000..ece85e1 --- /dev/null +++ b/test/automated/get/batch_size.rb @@ -0,0 +1,17 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Get" do + context "Batch Size" do + stream_name = controls::Put.(instances: 3) + + events = Get.(stream_name: stream_name, batch_size: 2) + + number_of_events = events.length + + test "Number of events retrieved is the specified batch size" do + assert(number_of_events == 2) + end + end +end diff --git a/test/automated/get/category.rb b/test/automated/get/category.rb new file mode 100644 index 0000000..15c9bb1 --- /dev/null +++ b/test/automated/get/category.rb @@ -0,0 +1,18 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Get" do + context "Category" do + category = controls::Category.example + + controls::Put.(instances: 2, category: category) + + events = Get.(category: category) + + test "Number of events retrieved is the number written to the category" do + number_of_events = events.length + assert(number_of_events == 2) + end + end +end diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb new file mode 100644 index 0000000..b3f06f8 --- /dev/null +++ b/test/automated/get/get.rb @@ -0,0 +1,16 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Put and Get" do + stream_name = controls::StreamName.example + write_event = controls::EventData::Write.example + + written_stream_position = Put.(stream_name, write_event) + + read_event = Get.(stream_name: stream_name, stream_position: written_stream_position)[0] + + test "Got the event that was written" do + assert(read_event.stream_position == written_stream_position) + end +end diff --git a/test/automated/get/no_events.rb b/test/automated/get/no_events.rb new file mode 100644 index 0000000..8715a98 --- /dev/null +++ b/test/automated/get/no_events.rb @@ -0,0 +1,15 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Get" do + context "No Events" do + stream_name = controls::StreamName.example + + batch = Get.(stream_name: stream_name) + + test "Empty array" do + assert(batch == []) + end + end +end diff --git a/test/automated/get/precedence.rb b/test/automated/get/precedence.rb new file mode 100644 index 0000000..f9b82bf --- /dev/null +++ b/test/automated/get/precedence.rb @@ -0,0 +1,29 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Get" do + context "Precedence" do + stream_name = controls::Put.(instances: 3) + + context "Ascending" do + events = Get.(stream_name: stream_name, precedence: :asc) + + first_event_postition = events.first.stream_position + + test "First event written is first in the list of results" do + assert(first_event_postition == 0) + end + end + + context "Descending" do + events = Get.(stream_name: stream_name, precedence: :desc) + + first_event_postition = events.first.stream_position + + test "Last event written is first in the list of results" do + assert(first_event_postition == 2) + end + end + end +end diff --git a/test/automated/iterator/cycle/retry_when_no_further_event_data.rb b/test/automated/iterator/cycle/retry_when_no_further_event_data.rb new file mode 100644 index 0000000..832cd31 --- /dev/null +++ b/test/automated/iterator/cycle/retry_when_no_further_event_data.rb @@ -0,0 +1,28 @@ +require_relative '../../automated_init' + +controls = EventStream::Postgres::Controls + +context "Iterator" do + context "Cycle" do + context "Retry when no further event data" do + cycle = Iterator::Cycle.build(delay_milliseconds: 10, timeout_milliseconds: 100) + sink = Iterator::Cycle.register_telemetry_sink(cycle) + + iterator = Iterator.build(stream_name: 'some_stream', cycle: cycle) + + iterator.next + + test "Didn't get result" do + refute(sink.recorded_got_result?) + end + + test "Delayed before retrying" do + assert(sink.recorded_delayed?) + end + + test "Timed out" do + assert(sink.recorded_timed_out?) + end + end + end +end diff --git a/test/automated/iterator/cycle/telemetry.rb b/test/automated/iterator/cycle/telemetry.rb new file mode 100644 index 0000000..d77705c --- /dev/null +++ b/test/automated/iterator/cycle/telemetry.rb @@ -0,0 +1,65 @@ +require_relative '../../automated_init' + +context "Iterator" do + context "Cycle" do + context "Telemetry" do + context "Got Result" do + cycle = Iterator::Cycle.build + sink = Iterator::Cycle.register_telemetry_sink(cycle) + + cycle.() do + 'some result' + end + + test "Recorded cycle" do + iteration = 0 + assert(sink.recorded_cycle? { |r| r.data == iteration }) + end + + test "Recorded invoked action" do + assert(sink.recorded_invoked_action?) + end + + test "Recorded got result" do + assert(sink.recorded_got_result?) + end + + test "Didn't record delayed" do + refute(sink.recorded_delayed?) + end + + test "Didn't record timed out" do + refute(sink.recorded_timed_out?) + end + end + + context "Got No Result" do + cycle = Iterator::Cycle.build(delay_milliseconds: 50, timeout_milliseconds: 100) + sink = Iterator::Cycle.register_telemetry_sink(cycle) + cycle.() do + nil + end + + test "Recorded cycle" do + assert(sink.recorded_cycle?) + end + + test "Recorded invoked action" do + assert(sink.recorded_invoked_action?) + end + + test "Recorded delayed" do + assert(sink.recorded_delayed?) + end + + test "Recorded timed out" do + assert(sink.recorded_timed_out?) + end + + test "Didn't record got result" do + refute(sink.recorded_got_result?) + end + end + end + end +end diff --git a/test/automated/iterator/next.rb b/test/automated/iterator/next.rb new file mode 100644 index 0000000..34be91f --- /dev/null +++ b/test/automated/iterator/next.rb @@ -0,0 +1,22 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Iterator" do + context "Next" do + stream_name = controls::Put.(instances: 2) + + iterator = Iterator.build(stream_name: stream_name, batch_size: 1) + + batch = [] + + 2.times do + event_data = iterator.next + batch << event_data unless event_data.nil? + end + + test "Gets each event" do + assert(batch.length == 2) + end + end +end diff --git a/test/automated/iterator/no_further_event_data.rb b/test/automated/iterator/no_further_event_data.rb new file mode 100644 index 0000000..97737ad --- /dev/null +++ b/test/automated/iterator/no_further_event_data.rb @@ -0,0 +1,19 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Iterator" do + context "No further event data" do + stream_name = controls::Put.(instances: 2) + + iterator = Iterator.build(stream_name: stream_name) + + 2.times { iterator.next } + + last = iterator.next + + test "Results in nil" do + assert(last.nil?) + end + end +end diff --git a/test/automated/iterator_tests.rb b/test/automated/iterator_tests.rb new file mode 100644 index 0000000..adf8d20 --- /dev/null +++ b/test/automated/iterator_tests.rb @@ -0,0 +1,5 @@ +require_relative '../test_init' + +TestBench::Runner.( + 'iterator/**/*.rb' +) or exit 1 diff --git a/test/automated/put/category_as_stream_name.rb b/test/automated/put/category_as_stream_name.rb new file mode 100644 index 0000000..66136b5 --- /dev/null +++ b/test/automated/put/category_as_stream_name.rb @@ -0,0 +1,18 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Put" do + context "Category as Stream Name" do + category = controls::Category.example + write_event = controls::EventData::Write.example + + Put.(category, write_event) + + read_event = Get.(stream_name: category).first + + test "Got the event that was written" do + assert(read_event.stream_name == category) + end + end +end diff --git a/test/automated/put/expected_version_error.rb b/test/automated/put/expected_version_error.rb new file mode 100644 index 0000000..34bcf97 --- /dev/null +++ b/test/automated/put/expected_version_error.rb @@ -0,0 +1,20 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Expected Version" do + context "Expected version does not match the stream version" do + stream_name = controls::StreamName.example + write_event = controls::EventData::Write.example + + stream_position = Put.(stream_name, write_event) + + incorrect_stream_version = stream_position + 1 + + test "Is an error" do + assert proc { Put.(stream_name, write_event, expected_version: incorrect_stream_version ) } do + raises_error? Write::ExpectedVersionError + end + end + end +end diff --git a/test/automated/put/no_stream/existing_stream.rb b/test/automated/put/no_stream/existing_stream.rb new file mode 100644 index 0000000..b4887f7 --- /dev/null +++ b/test/automated/put/no_stream/existing_stream.rb @@ -0,0 +1,24 @@ +require_relative '../../automated_init' + +controls = EventStream::Postgres::Controls + +context "Put" do + context "No Stream" do + context "Existing Stream" do + stream_name = controls::StreamName.example + + write_event_1 = controls::EventData::Write.example(data: {:some_attribute => 'first'}) + write_event_2 = controls::EventData::Write.example(data: {:some_attribute => 'second'}) + + Put.(stream_name, write_event_1) + + erroneous = proc { Put.(stream_name, write_event_2, expected_version: NoStream.name) } + + test "Is an error" do + assert erroneous do + raises_error? Write::ExpectedVersionError + end + end + end + end +end diff --git a/test/automated/put/no_stream/no_stream.rb b/test/automated/put/no_stream/no_stream.rb new file mode 100644 index 0000000..87722f4 --- /dev/null +++ b/test/automated/put/no_stream/no_stream.rb @@ -0,0 +1,18 @@ +require_relative '../../automated_init' + +controls = EventStream::Postgres::Controls + +context "Put" do + context "No Stream" do + context "For a stream that doesn't exist" do + stream_name = controls::StreamName.example + write_event = controls::EventData::Write.example + + stream_position = Put.(stream_name, write_event) + + test "Ensures that the event written is the first event in the stream" do + assert(stream_position == 0) + end + end + end +end diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb new file mode 100644 index 0000000..b3f06f8 --- /dev/null +++ b/test/automated/put/put.rb @@ -0,0 +1,16 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Put and Get" do + stream_name = controls::StreamName.example + write_event = controls::EventData::Write.example + + written_stream_position = Put.(stream_name, write_event) + + read_event = Get.(stream_name: stream_name, stream_position: written_stream_position)[0] + + test "Got the event that was written" do + assert(read_event.stream_position == written_stream_position) + end +end diff --git a/test/automated/put/returns_stream_position.rb b/test/automated/put/returns_stream_position.rb new file mode 100644 index 0000000..584acdb --- /dev/null +++ b/test/automated/put/returns_stream_position.rb @@ -0,0 +1,15 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Write Event Data" do + stream_name = controls::StreamName.example + + write_event = controls::EventData::Write.example + + stream_position = Put.(stream_name, write_event) + + test "Result is stream version" do + refute(stream_position.nil?) + end +end diff --git a/test/automated/put/stream_position_increases.rb b/test/automated/put/stream_position_increases.rb new file mode 100644 index 0000000..c83471c --- /dev/null +++ b/test/automated/put/stream_position_increases.rb @@ -0,0 +1,15 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Stream Version Increases with Subsequent Writes" do + stream_name = controls::StreamName.example + write_event = controls::EventData::Write.example + + stream_position_1 = Put.(stream_name, write_event) + stream_position_2 = Put.(stream_name, write_event) + + test "First version is one less than the second version" do + assert(stream_position_1 + 1 == stream_position_2) + end +end diff --git a/test/automated/read/missing_block_error.rb b/test/automated/read/missing_block_error.rb new file mode 100644 index 0000000..9eb7f74 --- /dev/null +++ b/test/automated/read/missing_block_error.rb @@ -0,0 +1,13 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Read" do + context "No block supplied" do + test "Is incorrect" do + assert proc { Read.(stream_name: 'some_stream_name') } do + raises_error? Read::Error + end + end + end +end diff --git a/test/automated/read/read.rb b/test/automated/read/read.rb new file mode 100644 index 0000000..f76f576 --- /dev/null +++ b/test/automated/read/read.rb @@ -0,0 +1,17 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Read" do + stream_name = controls::Put.(instances: 2) + + batch = [] + + Read.(stream_name: stream_name, batch_size: 1) do |event_data| + batch << event_data + end + + test "Reads batches of events" do + assert(batch.length == 2) + end +end diff --git a/test/automated/read/retry_when_no_further_event_data.rb b/test/automated/read/retry_when_no_further_event_data.rb new file mode 100644 index 0000000..e8c0203 --- /dev/null +++ b/test/automated/read/retry_when_no_further_event_data.rb @@ -0,0 +1,12 @@ +require_relative '../automated_init' + +context "Read" do + cycle = Iterator::Cycle.build(delay_milliseconds: 10, timeout_milliseconds: 100) + sink = Iterator::Cycle.register_telemetry_sink(cycle) + + Read.(stream_name: 'some_stream', batch_size: 1, cycle: cycle) { |event_data| } + + test "Timed out" do + assert(sink.recorded_timed_out?) + end +end diff --git a/test/automated/read/synchronous_result.skip.rb b/test/automated/read/synchronous_result.skip.rb new file mode 100644 index 0000000..fec47b8 --- /dev/null +++ b/test/automated/read/synchronous_result.skip.rb @@ -0,0 +1,13 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Read" do + context "Synchronous Result" do + res = Read.(stream_name: 'some_stream_name') { } + + test "Returns a result that fails if actuated" do + assert(res == AsyncInvocation::Incorrect) + end + end +end diff --git a/test/automated/select_statement/category_query.rb b/test/automated/select_statement/category_query.rb new file mode 100644 index 0000000..91a05be --- /dev/null +++ b/test/automated/select_statement/category_query.rb @@ -0,0 +1,20 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Select Statement" do + context "Category Query" do + stream = controls::Stream::Category.example + + select_statement = Get::SelectStatement.build stream + + sql = select_statement.sql + sql.gsub!(/\s+/, ' ') + + context "Where Clause" do + test "Filters on stream name" do + assert(sql.include? 'WHERE category(stream_name) =') + end + end + end +end diff --git a/test/automated/select_statement/defaults.rb b/test/automated/select_statement/defaults.rb new file mode 100644 index 0000000..6a3c4ec --- /dev/null +++ b/test/automated/select_statement/defaults.rb @@ -0,0 +1,32 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Select Statement" do + context "Defaults" do + stream = controls::Stream.example + + select_statement = Get::SelectStatement.build stream + + context "Offset" do + default_offset = Get::SelectStatement::Defaults.offset + test "#{default_offset}" do + assert(select_statement.offset == default_offset) + end + end + + context "Batch Size" do + default_batch_size = Get::SelectStatement::Defaults.batch_size + test "#{default_batch_size}" do + assert(select_statement.batch_size == default_batch_size) + end + end + + context "Precedence" do + default_precedence = Get::SelectStatement::Defaults.precedence + test "#{default_precedence}" do + assert(select_statement.precedence == default_precedence) + end + end + end +end diff --git a/test/automated/select_statement/stream_name_query.rb b/test/automated/select_statement/stream_name_query.rb new file mode 100644 index 0000000..0e925a0 --- /dev/null +++ b/test/automated/select_statement/stream_name_query.rb @@ -0,0 +1,20 @@ +require_relative '../automated_init' + +controls = EventStream::Postgres::Controls + +context "Select Statement" do + context "Stream Name Query" do + stream = controls::Stream.example + + select_statement = Get::SelectStatement.build stream + + sql = select_statement.sql + sql.gsub!(/\s+/, " ") + + context "Where Clause" do + test "Filters on stream name" do + assert(sql.include? 'WHERE stream_name =') + end + end + end +end diff --git a/test/automated/session.rb b/test/automated/session.rb new file mode 100644 index 0000000..9a97756 --- /dev/null +++ b/test/automated/session.rb @@ -0,0 +1,28 @@ +require_relative 'automated_init' + +context "Session" do + test "Connected" do + connected_session = Session.build + connected = connected_session.connected? + + assert(connected) + end + + context "Settings" do + session = Session.build + + settings = Postgres::Settings.build + settings_hash = settings.get.to_h + + names = Postgres::Settings.names + + names.each do |name| + test "#{name}" do + session_val = session.public_send name + settings_val = settings_hash[name] + + assert(session_val == settings_val) + end + end + end +end diff --git a/test/automated/settings.rb b/test/automated/settings.rb new file mode 100644 index 0000000..5e90030 --- /dev/null +++ b/test/automated/settings.rb @@ -0,0 +1,17 @@ +require_relative 'automated_init' + +context "Settings" do + settings = Postgres::Settings.build + + context "Names" do + settings_hash = settings.get.to_h + + names = Postgres::Settings.names + + names.each do |name| + test "#{name}" do + assert(settings_hash.has_key? name) + end + end + end +end diff --git a/test/automated/stream.rb b/test/automated/stream.rb new file mode 100644 index 0000000..43128c6 --- /dev/null +++ b/test/automated/stream.rb @@ -0,0 +1,65 @@ +require_relative 'automated_init' + +controls = EventStream::Postgres::Controls + +context "Stream" do + context "Stream Name" do + category = 'some_stream' + stream_name = controls::StreamName.example category: category + stream = Stream.build stream_name: stream_name + + test "Name" do + assert(stream.name == stream_name) + end + + test "Type" do + assert(stream.type == :stream) + end + + test "Stream Name" do + assert(stream.stream_name == stream_name) + end + + test "Category" do + assert(stream.category == category) + end + end + + context "Category" do + category = 'some_stream' + + stream = Stream.build category: category + + test "Name" do + assert(stream.name == category) + end + + test "Type" do + assert(stream.type == :category) + end + + test "Stream Name" do + assert(stream.stream_name == category) + end + + test "Category" do + assert(stream.category == category) + end + end + + context "Neither Stream Name or Category" do + test "Is an error" do + assert proc { Stream.build } do + raises_error? Stream::NameError + end + end + end + + context "Both Stream Name and Category" do + test "Is an error" do + assert proc { Stream.build stream_name: 'some_stream', category: 'some_category' } do + raises_error? Stream::NameError + end + end + end +end diff --git a/test/automated/stream_name/category_stream_name.rb b/test/automated/stream_name/category_stream_name.rb new file mode 100644 index 0000000..4cec996 --- /dev/null +++ b/test/automated/stream_name/category_stream_name.rb @@ -0,0 +1,8 @@ +require_relative '../automated_init' + +context "Category Stream Name" do + test "Is the category name" do + category_stream_name = StreamName.category_stream_name 'SomeCategory' + assert(category_stream_name == 'SomeCategory') + end +end diff --git a/test/automated/stream_name/stream_id.rb b/test/automated/stream_name/stream_id.rb new file mode 100644 index 0000000..84cbfeb --- /dev/null +++ b/test/automated/stream_name/stream_id.rb @@ -0,0 +1,16 @@ +require_relative '../automated_init' + +context "Stream ID" do + test "Is the UUID portion of a full stream name" do + id = Identifier::UUID.random + stream_name = "someStream-#{id}" + + stream_id = StreamName.get_id stream_name + assert(stream_id == id) + end + + test "Is nil if there is no type 4 UUID in the stream name" do + stream_id = StreamName.get_id 'someStream' + assert(stream_id.nil?) + end +end diff --git a/test/automated/stream_name/stream_name.rb b/test/automated/stream_name/stream_name.rb new file mode 100644 index 0000000..ad9d946 --- /dev/null +++ b/test/automated/stream_name/stream_name.rb @@ -0,0 +1,9 @@ +require_relative '../automated_init' + +context "Stream Name" do + stream_name = StreamName.stream_name 'someCategory', 'some_id' + + test "Composes the stream name from the category name and an ID" do + assert(stream_name == 'someCategory-some_id') + end +end diff --git a/test/script/reader_does_not_terminate.rb b/test/script/reader_does_not_terminate.rb new file mode 100644 index 0000000..951f590 --- /dev/null +++ b/test/script/reader_does_not_terminate.rb @@ -0,0 +1,3 @@ +require_relative '../test_init' + +Read.(stream_name: 'some_stream', batch_size: 1, delay_milliseconds: 200) { |event_data| } diff --git a/test/test_init.rb b/test/test_init.rb new file mode 100644 index 0000000..d641462 --- /dev/null +++ b/test/test_init.rb @@ -0,0 +1,25 @@ +ENV['CONSOLE_DEVICE'] ||= 'stdout' +ENV['LOG_COLOR'] ||= 'on' + +if ENV['LOG_LEVEL'] + ENV['LOGGER'] ||= 'on' +else + ENV['LOG_LEVEL'] ||= 'trace' +end + +ENV['LOGGER'] ||= 'off' +ENV['LOG_OPTIONAL'] ||= 'on' + +puts RUBY_DESCRIPTION + +require_relative '../init.rb' +require 'event_stream/postgres/controls' +controls = EventStream::Postgres::Controls + +require 'test_bench'; TestBench.activate + +Telemetry::Logger::AdHoc.activate +require 'pp' + +include EventStream +include EventStream::Postgres From ef41e8880412b115516d85d4d465c59896c33803 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 25 Sep 2016 20:13:25 -0500 Subject: [PATCH 002/591] eventsource, rather than eventstream --- database/clear-events-table.sh | 6 ++- database/install.sh | 6 ++- database/list-events.sh | 6 ++- database/uninstall.sh | 6 ++- event_source-postgres.gemspec | 4 +- init.rb | 2 +- lib/event_source/postgres.rb | 40 +++++++++---------- lib/event_source/postgres/controls.rb | 16 ++++---- .../postgres/controls/category.rb | 2 +- .../postgres/controls/event_data.rb | 2 +- .../postgres/controls/event_data/hash.rb | 4 +- .../postgres/controls/event_data/metadata.rb | 2 +- .../postgres/controls/event_data/write.rb | 4 +- lib/event_source/postgres/controls/put.rb | 4 +- lib/event_source/postgres/controls/stream.rb | 2 +- .../postgres/controls/stream_name.rb | 2 +- lib/event_source/postgres/event_data.rb | 2 +- lib/event_source/postgres/event_data/hash.rb | 2 +- lib/event_source/postgres/event_data/read.rb | 2 +- lib/event_source/postgres/event_data/write.rb | 2 +- lib/event_source/postgres/get.rb | 2 +- .../postgres/get/select_statement.rb | 2 +- lib/event_source/postgres/iterator.rb | 2 +- lib/event_source/postgres/iterator/cycle.rb | 2 +- lib/event_source/postgres/no_stream.rb | 2 +- lib/event_source/postgres/put.rb | 2 +- lib/event_source/postgres/read.rb | 2 +- lib/event_source/postgres/session.rb | 2 +- lib/event_source/postgres/settings.rb | 4 +- lib/event_source/postgres/stream.rb | 2 +- lib/event_source/postgres/stream_name.rb | 2 +- lib/event_source/postgres/write.rb | 2 +- ...stgres.json => event_source_postgres.json} | 4 +- test/automated.rb | 2 +- test/automated/event_data/hash.rb | 2 +- .../event_data/read_data_category.rb | 2 +- test/automated/get/batch_size.rb | 2 +- test/automated/get/category.rb | 2 +- test/automated/get/get.rb | 2 +- test/automated/get/no_events.rb | 2 +- test/automated/get/precedence.rb | 2 +- .../cycle/retry_when_no_further_event_data.rb | 2 +- test/automated/iterator/next.rb | 2 +- .../iterator/no_further_event_data.rb | 2 +- test/automated/put/category_as_stream_name.rb | 2 +- test/automated/put/expected_version_error.rb | 2 +- .../put/no_stream/existing_stream.rb | 2 +- test/automated/put/no_stream/no_stream.rb | 2 +- test/automated/put/put.rb | 2 +- test/automated/put/returns_stream_position.rb | 2 +- .../put/stream_position_increases.rb | 2 +- test/automated/read/missing_block_error.rb | 2 +- test/automated/read/read.rb | 2 +- .../automated/read/synchronous_result.skip.rb | 2 +- .../select_statement/category_query.rb | 2 +- test/automated/select_statement/defaults.rb | 2 +- .../select_statement/stream_name_query.rb | 2 +- test/automated/stream.rb | 2 +- test/test_init.rb | 8 ++-- 59 files changed, 106 insertions(+), 98 deletions(-) rename settings/{event_stream_postgres.json => event_source_postgres.json} (80%) diff --git a/database/clear-events-table.sh b/database/clear-events-table.sh index a6156d4..0f877eb 100755 --- a/database/clear-events-table.sh +++ b/database/clear-events-table.sh @@ -9,9 +9,11 @@ echo "Clearing Events Table" echo "= = =" echo +default_name=eventsource + if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" - user=eventstream + user=$default_name else user=$DATABASE_USER fi @@ -19,7 +21,7 @@ echo "Database user is: $user" if [ -z ${DATABASE_NAME+x} ]; then echo "(DATABASE_NAME is not set)" - database=eventstream + database=$default_name else database=$DATABASE_NAME fi diff --git a/database/install.sh b/database/install.sh index 85cc623..60c8f6f 100755 --- a/database/install.sh +++ b/database/install.sh @@ -9,16 +9,18 @@ echo "Installing Database" echo "= = =" echo +default_name=eventsource + if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" - user=eventstream + user=$default_name else user=$DATABASE_USER fi if [ -z ${DATABASE_NAME+x} ]; then echo "(DATABASE_NAME is not set)" - database=eventstream + database=$default_name else database=$DATABASE_NAME fi diff --git a/database/list-events.sh b/database/list-events.sh index bebd8af..6d43a7a 100755 --- a/database/list-events.sh +++ b/database/list-events.sh @@ -9,9 +9,11 @@ echo "Listing Events" echo "= = =" echo +default_name=eventsource + if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" - user=eventstream + user=$default_name else user=$DATABASE_USER fi @@ -19,7 +21,7 @@ echo "Database user is: $user" if [ -z ${DATABASE_NAME+x} ]; then echo "(DATABASE_NAME is not set)" - database=eventstream + database=$default_name else database=$DATABASE_NAME fi diff --git a/database/uninstall.sh b/database/uninstall.sh index ffb39b8..b2f5a9c 100755 --- a/database/uninstall.sh +++ b/database/uninstall.sh @@ -9,16 +9,18 @@ echo "Uninstalling Database" echo "= = =" echo +default_name=eventsource + if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" - user=eventstream + user=$default_name else user=$DATABASE_USER fi if [ -z ${DATABASE_NAME+x} ]; then echo "(DATABASE_NAME is not set)" - database=eventstream + database=$default_name else database=$DATABASE_NAME fi diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 29575bd..c6eb186 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,8 +1,8 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| - s.name = 'event_stream-postgres' + s.name = 'event_source-postgres' s.version = '0.0.0.0' - s.summary = 'Event stream client for PostgreSQL' + s.summary = 'Event source client for PostgreSQL' s.description = ' ' s.authors = ['The Eventide Project'] diff --git a/init.rb b/init.rb index 4ea7e08..5c945fe 100644 --- a/init.rb +++ b/init.rb @@ -7,4 +7,4 @@ $LOAD_PATH.unshift libraries_dir unless $LOAD_PATH.include?(libraries_dir) end -require 'event_stream/postgres' +require 'event_source/postgres' diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index 9d1709e..90d8824 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -11,25 +11,25 @@ require 'telemetry/logger' require 'async_invocation' -require 'event_stream/postgres/no_stream' - -require 'event_stream/postgres/stream_name' -require 'event_stream/postgres/stream' -require 'event_stream/postgres/event_data' -require 'event_stream/postgres/event_data/hash' -require 'event_stream/postgres/event_data/write' -require 'event_stream/postgres/event_data/read' - -require 'event_stream/postgres/settings' -require 'event_stream/postgres/session' - -require 'event_stream/postgres/put' -require 'event_stream/postgres/write' - -require 'event_stream/postgres/get/select_statement' -require 'event_stream/postgres/get' -require 'event_stream/postgres/iterator/cycle' -require 'event_stream/postgres/iterator' -require 'event_stream/postgres/read' +require 'event_source/postgres/no_stream' + +require 'event_source/postgres/stream_name' +require 'event_source/postgres/stream' +require 'event_source/postgres/event_data' +require 'event_source/postgres/event_data/hash' +require 'event_source/postgres/event_data/write' +require 'event_source/postgres/event_data/read' + +require 'event_source/postgres/settings' +require 'event_source/postgres/session' + +require 'event_source/postgres/put' +require 'event_source/postgres/write' + +require 'event_source/postgres/get/select_statement' +require 'event_source/postgres/get' +require 'event_source/postgres/iterator/cycle' +require 'event_source/postgres/iterator' +require 'event_source/postgres/read' diff --git a/lib/event_source/postgres/controls.rb b/lib/event_source/postgres/controls.rb index 69b8efa..1cd3df0 100644 --- a/lib/event_source/postgres/controls.rb +++ b/lib/event_source/postgres/controls.rb @@ -1,10 +1,10 @@ require 'controls' -require 'event_stream/postgres/controls/category' -require 'event_stream/postgres/controls/stream_name' -require 'event_stream/postgres/controls/stream' -require 'event_stream/postgres/controls/event_data' -require 'event_stream/postgres/controls/event_data/hash' -require 'event_stream/postgres/controls/event_data/metadata' -require 'event_stream/postgres/controls/event_data/write' -require 'event_stream/postgres/controls/put' +require 'event_source/postgres/controls/category' +require 'event_source/postgres/controls/stream_name' +require 'event_source/postgres/controls/stream' +require 'event_source/postgres/controls/event_data' +require 'event_source/postgres/controls/event_data/hash' +require 'event_source/postgres/controls/event_data/metadata' +require 'event_source/postgres/controls/event_data/write' +require 'event_source/postgres/controls/put' diff --git a/lib/event_source/postgres/controls/category.rb b/lib/event_source/postgres/controls/category.rb index 0140b65..3080ba8 100644 --- a/lib/event_source/postgres/controls/category.rb +++ b/lib/event_source/postgres/controls/category.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres module Controls module Category diff --git a/lib/event_source/postgres/controls/event_data.rb b/lib/event_source/postgres/controls/event_data.rb index b807c30..2455b93 100644 --- a/lib/event_source/postgres/controls/event_data.rb +++ b/lib/event_source/postgres/controls/event_data.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres module Controls module EventData diff --git a/lib/event_source/postgres/controls/event_data/hash.rb b/lib/event_source/postgres/controls/event_data/hash.rb index 627a8a6..5396035 100644 --- a/lib/event_source/postgres/controls/event_data/hash.rb +++ b/lib/event_source/postgres/controls/event_data/hash.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres module Controls module EventData @@ -10,7 +10,7 @@ def self.data end def self.example - EventStream::Postgres::EventData::Hash[data] + EventSource::Postgres::EventData::Hash[data] end module JSON diff --git a/lib/event_source/postgres/controls/event_data/metadata.rb b/lib/event_source/postgres/controls/event_data/metadata.rb index d005dfe..865d119 100644 --- a/lib/event_source/postgres/controls/event_data/metadata.rb +++ b/lib/event_source/postgres/controls/event_data/metadata.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres module Controls module EventData diff --git a/lib/event_source/postgres/controls/event_data/write.rb b/lib/event_source/postgres/controls/event_data/write.rb index d8b956d..93ca6f0 100644 --- a/lib/event_source/postgres/controls/event_data/write.rb +++ b/lib/event_source/postgres/controls/event_data/write.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres module Controls module EventData @@ -16,7 +16,7 @@ def self.example(type: nil, data: nil, metadata: nil) data ||= EventData.data metadata ||= EventData::Metadata.data - event_data = EventStream::Postgres::EventData::Write.build + event_data = EventSource::Postgres::EventData::Write.build event_data.type = type event_data.data = data diff --git a/lib/event_source/postgres/controls/put.rb b/lib/event_source/postgres/controls/put.rb index e8eb823..84c20fd 100644 --- a/lib/event_source/postgres/controls/put.rb +++ b/lib/event_source/postgres/controls/put.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres module Controls module Put @@ -8,7 +8,7 @@ def self.call(instances: nil, stream_name: nil, event: nil, category: nil) event ||= EventData::Write.example instances.times do - EventStream::Postgres::Put.(stream_name, event) + EventSource::Postgres::Put.(stream_name, event) end stream_name diff --git a/lib/event_source/postgres/controls/stream.rb b/lib/event_source/postgres/controls/stream.rb index 2f8d6fe..fd38b03 100644 --- a/lib/event_source/postgres/controls/stream.rb +++ b/lib/event_source/postgres/controls/stream.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres module Controls module Stream diff --git a/lib/event_source/postgres/controls/stream_name.rb b/lib/event_source/postgres/controls/stream_name.rb index 949ee8d..cf1c004 100644 --- a/lib/event_source/postgres/controls/stream_name.rb +++ b/lib/event_source/postgres/controls/stream_name.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres module Controls module StreamName diff --git a/lib/event_source/postgres/event_data.rb b/lib/event_source/postgres/event_data.rb index 1f7d29e..9eea289 100644 --- a/lib/event_source/postgres/event_data.rb +++ b/lib/event_source/postgres/event_data.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class EventData include Schema::DataStructure diff --git a/lib/event_source/postgres/event_data/hash.rb b/lib/event_source/postgres/event_data/hash.rb index 08dbbef..1380b18 100644 --- a/lib/event_source/postgres/event_data/hash.rb +++ b/lib/event_source/postgres/event_data/hash.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class EventData class Hash < ::Hash diff --git a/lib/event_source/postgres/event_data/read.rb b/lib/event_source/postgres/event_data/read.rb index 5a1c381..2449595 100644 --- a/lib/event_source/postgres/event_data/read.rb +++ b/lib/event_source/postgres/event_data/read.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class EventData class Read < EventData diff --git a/lib/event_source/postgres/event_data/write.rb b/lib/event_source/postgres/event_data/write.rb index c989c33..69fee84 100644 --- a/lib/event_source/postgres/event_data/write.rb +++ b/lib/event_source/postgres/event_data/write.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class EventData class Write < EventData diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 8e047c2..301af6a 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class Get initializer :stream_name, :category, :batch_size, :precedence diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 4a19476..c1633ae 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class Get class SelectStatement diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index 7b69861..35c5e2d 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class Iterator class Error < RuntimeError; end diff --git a/lib/event_source/postgres/iterator/cycle.rb b/lib/event_source/postgres/iterator/cycle.rb index 6dc4e08..61f86eb 100644 --- a/lib/event_source/postgres/iterator/cycle.rb +++ b/lib/event_source/postgres/iterator/cycle.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class Iterator class Cycle diff --git a/lib/event_source/postgres/no_stream.rb b/lib/event_source/postgres/no_stream.rb index 2d56eb5..caac91b 100644 --- a/lib/event_source/postgres/no_stream.rb +++ b/lib/event_source/postgres/no_stream.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres module NoStream def self.name diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index fe90292..254f935 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class Put attr_reader :stream_name diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 2e651cc..ab4df61 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class Read class Error < RuntimeError; end diff --git a/lib/event_source/postgres/session.rb b/lib/event_source/postgres/session.rb index 286e6a5..6a921d4 100644 --- a/lib/event_source/postgres/session.rb +++ b/lib/event_source/postgres/session.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class Session def self.settings diff --git a/lib/event_source/postgres/settings.rb b/lib/event_source/postgres/settings.rb index e58e07e..b52705a 100644 --- a/lib/event_source/postgres/settings.rb +++ b/lib/event_source/postgres/settings.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class Settings < ::Settings def self.instance @@ -6,7 +6,7 @@ def self.instance end def self.data_source - 'settings/event_stream_postgres.json' + 'settings/event_source_postgres.json' end def self.names diff --git a/lib/event_source/postgres/stream.rb b/lib/event_source/postgres/stream.rb index f2e0534..198b854 100644 --- a/lib/event_source/postgres/stream.rb +++ b/lib/event_source/postgres/stream.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class Stream class NameError < RuntimeError; end diff --git a/lib/event_source/postgres/stream_name.rb b/lib/event_source/postgres/stream_name.rb index ded9f8f..fcff12c 100644 --- a/lib/event_source/postgres/stream_name.rb +++ b/lib/event_source/postgres/stream_name.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres module StreamName extend self diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index 9782277..3a577e4 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -1,4 +1,4 @@ -module EventStream +module EventSource module Postgres class Write class ExpectedVersionError < RuntimeError; end diff --git a/settings/event_stream_postgres.json b/settings/event_source_postgres.json similarity index 80% rename from settings/event_stream_postgres.json rename to settings/event_source_postgres.json index e20c94f..4f17e59 100644 --- a/settings/event_stream_postgres.json +++ b/settings/event_source_postgres.json @@ -1,9 +1,9 @@ { - "dbname": "eventstream", + "dbname": "eventsource", "host": "localhost", "hostaddr": "127.0.0.1", "port": 5432, - "user": "eventstream", + "user": "eventsource", "password": null, "connect_timeout": null, "options": null, diff --git a/test/automated.rb b/test/automated.rb index 62cd68b..e36d314 100644 --- a/test/automated.rb +++ b/test/automated.rb @@ -1,6 +1,6 @@ require_relative 'test_init' TestBench::Runner.( - 'bench/**/*.rb', + 'automated/**/*.rb', exclude_pattern: %r{/^skip_|(?:_init\.rb|\.sketch\.rb|_sketch\.rb|sketch\.rb|\.skip\.rb|_tests\.rb)\z} ) or exit 1 diff --git a/test/automated/event_data/hash.rb b/test/automated/event_data/hash.rb index 5e68d84..9472734 100644 --- a/test/automated/event_data/hash.rb +++ b/test/automated/event_data/hash.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "EventData Hash" do context "JSON serialize" do diff --git a/test/automated/event_data/read_data_category.rb b/test/automated/event_data/read_data_category.rb index 2e6fb0f..7a8d9fa 100644 --- a/test/automated/event_data/read_data_category.rb +++ b/test/automated/event_data/read_data_category.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Read Data" do context "Category" do diff --git a/test/automated/get/batch_size.rb b/test/automated/get/batch_size.rb index ece85e1..35db4a0 100644 --- a/test/automated/get/batch_size.rb +++ b/test/automated/get/batch_size.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Get" do context "Batch Size" do diff --git a/test/automated/get/category.rb b/test/automated/get/category.rb index 15c9bb1..1b505d6 100644 --- a/test/automated/get/category.rb +++ b/test/automated/get/category.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Get" do context "Category" do diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index b3f06f8..0269264 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Put and Get" do stream_name = controls::StreamName.example diff --git a/test/automated/get/no_events.rb b/test/automated/get/no_events.rb index 8715a98..e3d721f 100644 --- a/test/automated/get/no_events.rb +++ b/test/automated/get/no_events.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Get" do context "No Events" do diff --git a/test/automated/get/precedence.rb b/test/automated/get/precedence.rb index f9b82bf..2ae48f5 100644 --- a/test/automated/get/precedence.rb +++ b/test/automated/get/precedence.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Get" do context "Precedence" do diff --git a/test/automated/iterator/cycle/retry_when_no_further_event_data.rb b/test/automated/iterator/cycle/retry_when_no_further_event_data.rb index 832cd31..847780c 100644 --- a/test/automated/iterator/cycle/retry_when_no_further_event_data.rb +++ b/test/automated/iterator/cycle/retry_when_no_further_event_data.rb @@ -1,6 +1,6 @@ require_relative '../../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Iterator" do context "Cycle" do diff --git a/test/automated/iterator/next.rb b/test/automated/iterator/next.rb index 34be91f..0442cad 100644 --- a/test/automated/iterator/next.rb +++ b/test/automated/iterator/next.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Iterator" do context "Next" do diff --git a/test/automated/iterator/no_further_event_data.rb b/test/automated/iterator/no_further_event_data.rb index 97737ad..a408be4 100644 --- a/test/automated/iterator/no_further_event_data.rb +++ b/test/automated/iterator/no_further_event_data.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Iterator" do context "No further event data" do diff --git a/test/automated/put/category_as_stream_name.rb b/test/automated/put/category_as_stream_name.rb index 66136b5..4d0d578 100644 --- a/test/automated/put/category_as_stream_name.rb +++ b/test/automated/put/category_as_stream_name.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Put" do context "Category as Stream Name" do diff --git a/test/automated/put/expected_version_error.rb b/test/automated/put/expected_version_error.rb index 34bcf97..98eaf88 100644 --- a/test/automated/put/expected_version_error.rb +++ b/test/automated/put/expected_version_error.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Expected Version" do context "Expected version does not match the stream version" do diff --git a/test/automated/put/no_stream/existing_stream.rb b/test/automated/put/no_stream/existing_stream.rb index b4887f7..468c398 100644 --- a/test/automated/put/no_stream/existing_stream.rb +++ b/test/automated/put/no_stream/existing_stream.rb @@ -1,6 +1,6 @@ require_relative '../../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Put" do context "No Stream" do diff --git a/test/automated/put/no_stream/no_stream.rb b/test/automated/put/no_stream/no_stream.rb index 87722f4..3f0d0fc 100644 --- a/test/automated/put/no_stream/no_stream.rb +++ b/test/automated/put/no_stream/no_stream.rb @@ -1,6 +1,6 @@ require_relative '../../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Put" do context "No Stream" do diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index b3f06f8..0269264 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Put and Get" do stream_name = controls::StreamName.example diff --git a/test/automated/put/returns_stream_position.rb b/test/automated/put/returns_stream_position.rb index 584acdb..d826480 100644 --- a/test/automated/put/returns_stream_position.rb +++ b/test/automated/put/returns_stream_position.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Write Event Data" do stream_name = controls::StreamName.example diff --git a/test/automated/put/stream_position_increases.rb b/test/automated/put/stream_position_increases.rb index c83471c..cb60332 100644 --- a/test/automated/put/stream_position_increases.rb +++ b/test/automated/put/stream_position_increases.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Stream Version Increases with Subsequent Writes" do stream_name = controls::StreamName.example diff --git a/test/automated/read/missing_block_error.rb b/test/automated/read/missing_block_error.rb index 9eb7f74..c48d50a 100644 --- a/test/automated/read/missing_block_error.rb +++ b/test/automated/read/missing_block_error.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Read" do context "No block supplied" do diff --git a/test/automated/read/read.rb b/test/automated/read/read.rb index f76f576..3259b61 100644 --- a/test/automated/read/read.rb +++ b/test/automated/read/read.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Read" do stream_name = controls::Put.(instances: 2) diff --git a/test/automated/read/synchronous_result.skip.rb b/test/automated/read/synchronous_result.skip.rb index fec47b8..ddac581 100644 --- a/test/automated/read/synchronous_result.skip.rb +++ b/test/automated/read/synchronous_result.skip.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Read" do context "Synchronous Result" do diff --git a/test/automated/select_statement/category_query.rb b/test/automated/select_statement/category_query.rb index 91a05be..5599549 100644 --- a/test/automated/select_statement/category_query.rb +++ b/test/automated/select_statement/category_query.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Select Statement" do context "Category Query" do diff --git a/test/automated/select_statement/defaults.rb b/test/automated/select_statement/defaults.rb index 6a3c4ec..4832f09 100644 --- a/test/automated/select_statement/defaults.rb +++ b/test/automated/select_statement/defaults.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Select Statement" do context "Defaults" do diff --git a/test/automated/select_statement/stream_name_query.rb b/test/automated/select_statement/stream_name_query.rb index 0e925a0..04a6e1c 100644 --- a/test/automated/select_statement/stream_name_query.rb +++ b/test/automated/select_statement/stream_name_query.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Select Statement" do context "Stream Name Query" do diff --git a/test/automated/stream.rb b/test/automated/stream.rb index 43128c6..07ad410 100644 --- a/test/automated/stream.rb +++ b/test/automated/stream.rb @@ -1,6 +1,6 @@ require_relative 'automated_init' -controls = EventStream::Postgres::Controls +controls = EventSource::Postgres::Controls context "Stream" do context "Stream Name" do diff --git a/test/test_init.rb b/test/test_init.rb index d641462..08399e2 100644 --- a/test/test_init.rb +++ b/test/test_init.rb @@ -13,13 +13,13 @@ puts RUBY_DESCRIPTION require_relative '../init.rb' -require 'event_stream/postgres/controls' -controls = EventStream::Postgres::Controls +require 'event_source/postgres/controls' +controls = EventSource::Postgres::Controls require 'test_bench'; TestBench.activate Telemetry::Logger::AdHoc.activate require 'pp' -include EventStream -include EventStream::Postgres +include EventSource +include EventSource::Postgres From 88e7d036ebf27ef1b62bfc471d0ee5dd30596c56 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Oct 2016 11:45:04 -0500 Subject: [PATCH 003/591] Script files reflect event source naming --- ...nstall-event-stream-database => install-event-source-database} | 0 ...tall-event-stream-database => uninstall-event-source-database} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename bin/{install-event-stream-database => install-event-source-database} (100%) rename bin/{uninstall-event-stream-database => uninstall-event-source-database} (100%) diff --git a/bin/install-event-stream-database b/bin/install-event-source-database similarity index 100% rename from bin/install-event-stream-database rename to bin/install-event-source-database diff --git a/bin/uninstall-event-stream-database b/bin/uninstall-event-source-database similarity index 100% rename from bin/uninstall-event-stream-database rename to bin/uninstall-event-source-database From ac2cec1a62e7ec9b10b89380d1ed0ee76ceabedd Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Oct 2016 18:48:46 -0500 Subject: [PATCH 004/591] Symlink scripts are operational --- library-symlinks.sh | 24 ++++++++---------------- symlink-lib.sh | 2 +- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/library-symlinks.sh b/library-symlinks.sh index 7d969d9..b68a396 100755 --- a/library-symlinks.sh +++ b/library-symlinks.sh @@ -1,27 +1,19 @@ set -e -if [ -z ${LIBRARIES_DIR+x} ]; then - echo "LIBRARIES_DIR must be set to the libraries directory path... exiting" +if [ -z ${LIBRARIES_HOME+x} ]; then + echo "LIBRARIES_HOME must be set to the libraries directory path... exiting" exit 1 fi -if [ ! -d "$LIBRARIES_DIR" ]; then - echo "$LIBRARIES_DIR does not exit... exiting" +if [ ! -d "$LIBRARIES_HOME" ]; then + echo "$LIBRARIES_HOME does not exist... exiting" exit 1 fi -# function make_directories { -# directories=("$@") - -# for directory in "${directories[@]}"; do -# make_directory $directory -# done -# } - function make_directory { directory=$1 - lib_directory="$LIBRARIES_DIR/$directory" + lib_directory="$LIBRARIES_HOME/$directory" if [ ! -d "$lib_directory" ]; then echo "- making directory $lib_directory" @@ -33,7 +25,7 @@ function remove_lib_symlinks { name=$1 directory=$2 - dest="$LIBRARIES_DIR" + dest="$LIBRARIES_HOME" if [ ! -z "$directory" ]; then dest="$dest/$directory" fi @@ -57,8 +49,8 @@ function symlink_lib { remove_lib_symlinks $name $directory - src="$(PWD)/lib" - dest="$LIBRARIES_DIR" + src="$(pwd)/lib" + dest="$LIBRARIES_HOME" if [ ! -z "$directory" ]; then src="$src/$directory" dest="$dest/$directory" diff --git a/symlink-lib.sh b/symlink-lib.sh index d940f25..0be5c40 100755 --- a/symlink-lib.sh +++ b/symlink-lib.sh @@ -1,3 +1,3 @@ source ./library-symlinks.sh -symlink_lib 'event_stream' 'postgres' +symlink_lib 'event_source' 'postgres' From c4112f189b37cad7470822487c78882d77ab12a3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Oct 2016 18:57:53 -0500 Subject: [PATCH 005/591] Package dependencies accounted for by the event-source dependency are removed --- event_source-postgres.gemspec | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index c6eb186..a83ac18 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -15,17 +15,16 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.required_ruby_version = '>= 2.2.3' - s.executables = ['install-event-stream-database', 'uninstall-event-stream-database'] + s.executables = ['install-event-source-database', 'uninstall-event-source-database'] s.bindir = 'bin' + s.add_runtime_dependency 'event_source' + s.add_runtime_dependency 'telemetry' s.add_runtime_dependency 'telemetry-logger' - s.add_runtime_dependency 'casing' - s.add_runtime_dependency 'schema' - s.add_runtime_dependency 'initializer' - s.add_runtime_dependency 'serialize' s.add_runtime_dependency 'settings' s.add_runtime_dependency 'async_invocation' + s.add_runtime_dependency 'controls' s.add_runtime_dependency 'pg' From ce0c6a7db36f355c376ac3c324dbc375d5655cbb Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Oct 2016 19:03:59 -0500 Subject: [PATCH 006/591] Superfluous package loading is removed --- lib/event_source/postgres.rb | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index 90d8824..c6c8c8f 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -1,14 +1,10 @@ require 'pg' -require 'casing' -require 'clock' -require 'dependency'; Dependency.activate -require 'schema' -require 'initializer'; Initializer.activate -require 'serialize' -require 'settings'; Settings.activate -require 'telemetry' +require 'event_source' + require 'telemetry/logger' +require 'telemetry' +require 'settings'; Settings.activate require 'async_invocation' require 'event_source/postgres/no_stream' From 9d2fa090b882959f1c18ffd6cc9cf78333ef3479 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Oct 2016 19:10:04 -0500 Subject: [PATCH 007/591] Elements accounted for by the event-source library are removed --- lib/event_source/postgres.rb | 9 --- lib/event_source/postgres/controls.rb | 4 ++ .../postgres/controls/category.rb | 20 +----- .../postgres/controls/event_data.rb | 21 +----- .../postgres/controls/event_data/hash.rb | 23 +------ .../postgres/controls/event_data/metadata.rb | 19 +----- .../postgres/controls/event_data/write.rb | 35 +--------- lib/event_source/postgres/controls/stream.rb | 14 +--- .../postgres/controls/stream_name.rb | 9 +-- lib/event_source/postgres/event_data.rb | 11 ---- lib/event_source/postgres/event_data/hash.rb | 33 ---------- lib/event_source/postgres/event_data/read.rb | 18 ----- lib/event_source/postgres/event_data/write.rb | 8 --- lib/event_source/postgres/no_stream.rb | 13 ---- lib/event_source/postgres/stream.rb | 43 ------------ lib/event_source/postgres/stream_name.rb | 25 ------- test/automated/event_data/hash.rb | 23 ------- .../event_data/read_data_category.rb | 16 ----- test/automated/stream.rb | 65 ------------------- .../stream_name/category_stream_name.rb | 8 --- test/automated/stream_name/stream_id.rb | 16 ----- test/automated/stream_name/stream_name.rb | 9 --- 22 files changed, 11 insertions(+), 431 deletions(-) delete mode 100644 lib/event_source/postgres/event_data.rb delete mode 100644 lib/event_source/postgres/event_data/hash.rb delete mode 100644 lib/event_source/postgres/event_data/read.rb delete mode 100644 lib/event_source/postgres/event_data/write.rb delete mode 100644 lib/event_source/postgres/no_stream.rb delete mode 100644 lib/event_source/postgres/stream.rb delete mode 100644 lib/event_source/postgres/stream_name.rb delete mode 100644 test/automated/event_data/hash.rb delete mode 100644 test/automated/event_data/read_data_category.rb delete mode 100644 test/automated/stream.rb delete mode 100644 test/automated/stream_name/category_stream_name.rb delete mode 100644 test/automated/stream_name/stream_id.rb delete mode 100644 test/automated/stream_name/stream_name.rb diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index c6c8c8f..8518844 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -7,15 +7,6 @@ require 'settings'; Settings.activate require 'async_invocation' -require 'event_source/postgres/no_stream' - -require 'event_source/postgres/stream_name' -require 'event_source/postgres/stream' -require 'event_source/postgres/event_data' -require 'event_source/postgres/event_data/hash' -require 'event_source/postgres/event_data/write' -require 'event_source/postgres/event_data/read' - require 'event_source/postgres/settings' require 'event_source/postgres/session' diff --git a/lib/event_source/postgres/controls.rb b/lib/event_source/postgres/controls.rb index 1cd3df0..aa30abe 100644 --- a/lib/event_source/postgres/controls.rb +++ b/lib/event_source/postgres/controls.rb @@ -1,10 +1,14 @@ require 'controls' +require 'event_source/controls' + require 'event_source/postgres/controls/category' require 'event_source/postgres/controls/stream_name' require 'event_source/postgres/controls/stream' + require 'event_source/postgres/controls/event_data' require 'event_source/postgres/controls/event_data/hash' require 'event_source/postgres/controls/event_data/metadata' require 'event_source/postgres/controls/event_data/write' + require 'event_source/postgres/controls/put' diff --git a/lib/event_source/postgres/controls/category.rb b/lib/event_source/postgres/controls/category.rb index 3080ba8..f5c2f34 100644 --- a/lib/event_source/postgres/controls/category.rb +++ b/lib/event_source/postgres/controls/category.rb @@ -1,25 +1,7 @@ module EventSource module Postgres module Controls - module Category - def self.example(category: nil, randomize_category: nil) - if randomize_category.nil? - if !category.nil? - randomize_category = false - end - end - - randomize_category = true if randomize_category.nil? - - category ||= 'Test' - - if randomize_category - category = "#{category}#{Identifier::UUID.random.gsub('-', '')}" - end - - category - end - end + Category = EventSource::Controls::Category end end end diff --git a/lib/event_source/postgres/controls/event_data.rb b/lib/event_source/postgres/controls/event_data.rb index 2455b93..de77de3 100644 --- a/lib/event_source/postgres/controls/event_data.rb +++ b/lib/event_source/postgres/controls/event_data.rb @@ -1,26 +1,7 @@ module EventSource module Postgres module Controls - module EventData - def self.type - 'SomeType' - end - - def self.data - { :some_attribute => 'some value' } - end - - module JSON - def self.data(id=nil) - data = EventData.data - Casing::Camel.(data, symbol_to_string: true) - end - - def self.text - data.to_json - end - end - end + EventData = EventSource::Controls::EventData end end end diff --git a/lib/event_source/postgres/controls/event_data/hash.rb b/lib/event_source/postgres/controls/event_data/hash.rb index 5396035..70b28f7 100644 --- a/lib/event_source/postgres/controls/event_data/hash.rb +++ b/lib/event_source/postgres/controls/event_data/hash.rb @@ -2,28 +2,7 @@ module EventSource module Postgres module Controls module EventData - module Hash - def self.data - { - some_attribute: 'some value' - } - end - - def self.example - EventSource::Postgres::EventData::Hash[data] - end - - module JSON - def self.data(id=nil) - data = Hash.data - Casing::Camel.(data, symbol_to_string: true) - end - - def self.text - ::JSON.generate(data) - end - end - end + Hash = EventSource::Controls::EventData::Hash end end end diff --git a/lib/event_source/postgres/controls/event_data/metadata.rb b/lib/event_source/postgres/controls/event_data/metadata.rb index 865d119..255f5ac 100644 --- a/lib/event_source/postgres/controls/event_data/metadata.rb +++ b/lib/event_source/postgres/controls/event_data/metadata.rb @@ -2,24 +2,7 @@ module EventSource module Postgres module Controls module EventData - module Metadata - def self.data - { - some_meta_attribute: 'some meta value' - } - end - - module JSON - def self.data(id=nil) - data = Metadata.data - Casing::Camel.(data, symbol_to_string: true) - end - - def self.text - data.to_json - end - end - end + Metadata = EventSource::Controls::EventData::Metadata end end end diff --git a/lib/event_source/postgres/controls/event_data/write.rb b/lib/event_source/postgres/controls/event_data/write.rb index 93ca6f0..ca26574 100644 --- a/lib/event_source/postgres/controls/event_data/write.rb +++ b/lib/event_source/postgres/controls/event_data/write.rb @@ -2,40 +2,7 @@ module EventSource module Postgres module Controls module EventData - module Write - def self.data - { - :type => EventData.type, - :data => EventData.data, - :metadata => EventData::Metadata.data - } - end - - def self.example(type: nil, data: nil, metadata: nil) - type ||= EventData.type - data ||= EventData.data - metadata ||= EventData::Metadata.data - - event_data = EventSource::Postgres::EventData::Write.build - - event_data.type = type - event_data.data = data - event_data.metadata = metadata - - event_data - end - - module JSON - def self.data - data = Write.data - Casing::Camel.(data, symbol_to_string: true) - end - - def self.text - data.to_json - end - end - end + Write = EventSource::Controls::EventData::Write end end end diff --git a/lib/event_source/postgres/controls/stream.rb b/lib/event_source/postgres/controls/stream.rb index fd38b03..9ba4d2f 100644 --- a/lib/event_source/postgres/controls/stream.rb +++ b/lib/event_source/postgres/controls/stream.rb @@ -1,19 +1,7 @@ module EventSource module Postgres module Controls - module Stream - def self.example(stream_name: nil, category: nil, id: nil, randomize_category: nil) - stream_name ||= StreamName.example category: category, id: id, randomize_category: randomize_category - ::Stream.build stream_name: stream_name - end - - module Category - def self.example(category: nil, randomize_category: nil) - category ||= Controls::Category.example category: category, randomize_category: randomize_category - ::Stream.build category: category - end - end - end + Stream = EventSource::Controls::Stream end end end diff --git a/lib/event_source/postgres/controls/stream_name.rb b/lib/event_source/postgres/controls/stream_name.rb index cf1c004..4f70ad6 100644 --- a/lib/event_source/postgres/controls/stream_name.rb +++ b/lib/event_source/postgres/controls/stream_name.rb @@ -1,14 +1,7 @@ module EventSource module Postgres module Controls - module StreamName - def self.example(category: nil, id: nil, randomize_category: nil) - category ||= Category.example category: category, randomize_category: randomize_category - id ||= Identifier::UUID.random - - "#{category}-#{id}" - end - end + StreamName = EventSource::Controls::StreamName end end end diff --git a/lib/event_source/postgres/event_data.rb b/lib/event_source/postgres/event_data.rb deleted file mode 100644 index 9eea289..0000000 --- a/lib/event_source/postgres/event_data.rb +++ /dev/null @@ -1,11 +0,0 @@ -module EventSource - module Postgres - class EventData - include Schema::DataStructure - - attribute :type - attribute :data - attribute :metadata - end - end -end diff --git a/lib/event_source/postgres/event_data/hash.rb b/lib/event_source/postgres/event_data/hash.rb deleted file mode 100644 index 1380b18..0000000 --- a/lib/event_source/postgres/event_data/hash.rb +++ /dev/null @@ -1,33 +0,0 @@ -module EventSource - module Postgres - class EventData - class Hash < ::Hash - module Serializer - def self.json - JSON - end - - def self.instance(raw_data) - Hash[raw_data] - end - - def self.raw_data(instance) - Hash[instance] - end - - module JSON - def self.serialize(raw_hash_data) - json_formatted_data = Casing::Camel.(raw_hash_data) - ::JSON.generate(json_formatted_data) - end - - def self.deserialize(text) - json_formatted_data = ::JSON.parse(text, :symbolize_names => true) - Casing::Underscore.(json_formatted_data) - end - end - end - end - end - end -end diff --git a/lib/event_source/postgres/event_data/read.rb b/lib/event_source/postgres/event_data/read.rb deleted file mode 100644 index 2449595..0000000 --- a/lib/event_source/postgres/event_data/read.rb +++ /dev/null @@ -1,18 +0,0 @@ -module EventSource - module Postgres - class EventData - class Read < EventData - include Schema::DataStructure - - attribute :stream_name - attribute :stream_position - attribute :global_position - attribute :created_time - - def category - StreamName.category(stream_name) - end - end - end - end -end diff --git a/lib/event_source/postgres/event_data/write.rb b/lib/event_source/postgres/event_data/write.rb deleted file mode 100644 index 69fee84..0000000 --- a/lib/event_source/postgres/event_data/write.rb +++ /dev/null @@ -1,8 +0,0 @@ -module EventSource - module Postgres - class EventData - class Write < EventData - end - end - end -end diff --git a/lib/event_source/postgres/no_stream.rb b/lib/event_source/postgres/no_stream.rb deleted file mode 100644 index caac91b..0000000 --- a/lib/event_source/postgres/no_stream.rb +++ /dev/null @@ -1,13 +0,0 @@ -module EventSource - module Postgres - module NoStream - def self.name - :no_stream - end - - def self.version - -1 - end - end - end -end diff --git a/lib/event_source/postgres/stream.rb b/lib/event_source/postgres/stream.rb deleted file mode 100644 index 198b854..0000000 --- a/lib/event_source/postgres/stream.rb +++ /dev/null @@ -1,43 +0,0 @@ -module EventSource - module Postgres - class Stream - class NameError < RuntimeError; end - - initializer :stream_name, :category, :type - - def name - if type == :stream - return stream_name - else - return category - end - end - - def self.build(stream_name: nil, category: nil) - stream_name, category, type = get_name(stream_name, category) - new stream_name, category, type - end - - def self.get_name(stream_name, category) - if !stream_name.nil? && !category.nil? - raise NameError, "Both stream stream_name and category are specified. Specify one or the other." - end - - type = nil - if !stream_name.nil? - category = StreamName.category(stream_name) - type = :stream - else - stream_name = category - type = :category - end - - if stream_name.nil? && category.nil? - raise NameError, "Neither stream name nor category are specified. Specify one or the other." - end - - return stream_name, category, type - end - end - end -end diff --git a/lib/event_source/postgres/stream_name.rb b/lib/event_source/postgres/stream_name.rb deleted file mode 100644 index fcff12c..0000000 --- a/lib/event_source/postgres/stream_name.rb +++ /dev/null @@ -1,25 +0,0 @@ -module EventSource - module Postgres - module StreamName - extend self - - def stream_name(category_name, id=nil) - id ||= Identifier::UUID.random - - "#{category_name}-#{id}" - end - - def category_stream_name(category_name) - category_name - end - - def self.get_id(stream_name) - Identifier::UUID.parse(stream_name) - end - - def self.category(stream_name) - stream_name.split('-').first - end - end - end -end diff --git a/test/automated/event_data/hash.rb b/test/automated/event_data/hash.rb deleted file mode 100644 index 9472734..0000000 --- a/test/automated/event_data/hash.rb +++ /dev/null @@ -1,23 +0,0 @@ -require_relative '../automated_init' - -controls = EventSource::Postgres::Controls - -context "EventData Hash" do - context "JSON serialize" do - example_hash = controls::EventData::Hash.example - control_serialized_text = controls::EventData::Hash::JSON.text - - serialized_text = Serialize::Write.(example_hash, :json) - - assert(serialized_text == control_serialized_text) - end - - context "JSON deserialize" do - example_serialized_text = controls::EventData::Hash::JSON.text - control_deserialized_hash = controls::EventData::Hash.example - - deserialized_hash = Serialize::Read.(example_serialized_text, EventData::Hash, :json) - - assert(deserialized_hash == control_deserialized_hash) - end -end diff --git a/test/automated/event_data/read_data_category.rb b/test/automated/event_data/read_data_category.rb deleted file mode 100644 index 7a8d9fa..0000000 --- a/test/automated/event_data/read_data_category.rb +++ /dev/null @@ -1,16 +0,0 @@ -require_relative '../automated_init' - -controls = EventSource::Postgres::Controls - -context "Read Data" do - context "Category" do - read_data = EventData::Read.new - read_data.stream_name = 'some_stream-some_id' - - category = read_data.category - - test "Is the first segment of the stream name" do - assert(category == 'some_stream') - end - end -end diff --git a/test/automated/stream.rb b/test/automated/stream.rb deleted file mode 100644 index 07ad410..0000000 --- a/test/automated/stream.rb +++ /dev/null @@ -1,65 +0,0 @@ -require_relative 'automated_init' - -controls = EventSource::Postgres::Controls - -context "Stream" do - context "Stream Name" do - category = 'some_stream' - stream_name = controls::StreamName.example category: category - stream = Stream.build stream_name: stream_name - - test "Name" do - assert(stream.name == stream_name) - end - - test "Type" do - assert(stream.type == :stream) - end - - test "Stream Name" do - assert(stream.stream_name == stream_name) - end - - test "Category" do - assert(stream.category == category) - end - end - - context "Category" do - category = 'some_stream' - - stream = Stream.build category: category - - test "Name" do - assert(stream.name == category) - end - - test "Type" do - assert(stream.type == :category) - end - - test "Stream Name" do - assert(stream.stream_name == category) - end - - test "Category" do - assert(stream.category == category) - end - end - - context "Neither Stream Name or Category" do - test "Is an error" do - assert proc { Stream.build } do - raises_error? Stream::NameError - end - end - end - - context "Both Stream Name and Category" do - test "Is an error" do - assert proc { Stream.build stream_name: 'some_stream', category: 'some_category' } do - raises_error? Stream::NameError - end - end - end -end diff --git a/test/automated/stream_name/category_stream_name.rb b/test/automated/stream_name/category_stream_name.rb deleted file mode 100644 index 4cec996..0000000 --- a/test/automated/stream_name/category_stream_name.rb +++ /dev/null @@ -1,8 +0,0 @@ -require_relative '../automated_init' - -context "Category Stream Name" do - test "Is the category name" do - category_stream_name = StreamName.category_stream_name 'SomeCategory' - assert(category_stream_name == 'SomeCategory') - end -end diff --git a/test/automated/stream_name/stream_id.rb b/test/automated/stream_name/stream_id.rb deleted file mode 100644 index 84cbfeb..0000000 --- a/test/automated/stream_name/stream_id.rb +++ /dev/null @@ -1,16 +0,0 @@ -require_relative '../automated_init' - -context "Stream ID" do - test "Is the UUID portion of a full stream name" do - id = Identifier::UUID.random - stream_name = "someStream-#{id}" - - stream_id = StreamName.get_id stream_name - assert(stream_id == id) - end - - test "Is nil if there is no type 4 UUID in the stream name" do - stream_id = StreamName.get_id 'someStream' - assert(stream_id.nil?) - end -end diff --git a/test/automated/stream_name/stream_name.rb b/test/automated/stream_name/stream_name.rb deleted file mode 100644 index ad9d946..0000000 --- a/test/automated/stream_name/stream_name.rb +++ /dev/null @@ -1,9 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - stream_name = StreamName.stream_name 'someCategory', 'some_id' - - test "Composes the stream name from the category name and an ID" do - assert(stream_name == 'someCategory-some_id') - end -end From 0dbc28db983e29a0be8537c8ecef38400742ff89 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Oct 2016 20:28:26 -0500 Subject: [PATCH 008/591] Controls namespace in tests refers to this project's controls --- test/automated/get/batch_size.rb | 4 +--- test/automated/get/category.rb | 6 +++--- test/automated/get/get.rb | 6 +++--- test/automated/get/no_events.rb | 4 ++-- test/automated/get/precedence.rb | 4 ++-- .../iterator/cycle/retry_when_no_further_event_data.rb | 2 +- test/automated/iterator/next.rb | 4 ++-- test/automated/iterator/no_further_event_data.rb | 4 ++-- test/automated/put/category_as_stream_name.rb | 6 +++--- test/automated/put/expected_version_error.rb | 6 +++--- test/automated/put/no_stream/existing_stream.rb | 8 ++++---- test/automated/put/no_stream/no_stream.rb | 6 +++--- test/automated/put/put.rb | 6 +++--- test/automated/put/returns_stream_position.rb | 6 +++--- test/automated/put/stream_position_increases.rb | 6 +++--- test/automated/read/missing_block_error.rb | 2 +- test/automated/read/read.rb | 4 ++-- test/automated/read/synchronous_result.skip.rb | 2 +- test/automated/select_statement/category_query.rb | 4 ++-- test/automated/select_statement/defaults.rb | 4 ++-- test/automated/select_statement/stream_name_query.rb | 4 ++-- test/test_init.rb | 1 - 22 files changed, 48 insertions(+), 51 deletions(-) diff --git a/test/automated/get/batch_size.rb b/test/automated/get/batch_size.rb index 35db4a0..661f11d 100644 --- a/test/automated/get/batch_size.rb +++ b/test/automated/get/batch_size.rb @@ -1,10 +1,8 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls - context "Get" do context "Batch Size" do - stream_name = controls::Put.(instances: 3) + stream_name = Controls::Put.(instances: 3) events = Get.(stream_name: stream_name, batch_size: 2) diff --git a/test/automated/get/category.rb b/test/automated/get/category.rb index 1b505d6..e67c5ad 100644 --- a/test/automated/get/category.rb +++ b/test/automated/get/category.rb @@ -1,12 +1,12 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Get" do context "Category" do - category = controls::Category.example + category = Controls::Category.example - controls::Put.(instances: 2, category: category) + Controls::Put.(instances: 2, category: category) events = Get.(category: category) diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index 0269264..65e6de7 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -1,10 +1,10 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Put and Get" do - stream_name = controls::StreamName.example - write_event = controls::EventData::Write.example + stream_name = Controls::StreamName.example + write_event = Controls::EventData::Write.example written_stream_position = Put.(stream_name, write_event) diff --git a/test/automated/get/no_events.rb b/test/automated/get/no_events.rb index e3d721f..cef880e 100644 --- a/test/automated/get/no_events.rb +++ b/test/automated/get/no_events.rb @@ -1,10 +1,10 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Get" do context "No Events" do - stream_name = controls::StreamName.example + stream_name = Controls::StreamName.example batch = Get.(stream_name: stream_name) diff --git a/test/automated/get/precedence.rb b/test/automated/get/precedence.rb index 2ae48f5..854b409 100644 --- a/test/automated/get/precedence.rb +++ b/test/automated/get/precedence.rb @@ -1,10 +1,10 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Get" do context "Precedence" do - stream_name = controls::Put.(instances: 3) + stream_name = Controls::Put.(instances: 3) context "Ascending" do events = Get.(stream_name: stream_name, precedence: :asc) diff --git a/test/automated/iterator/cycle/retry_when_no_further_event_data.rb b/test/automated/iterator/cycle/retry_when_no_further_event_data.rb index 847780c..25d1614 100644 --- a/test/automated/iterator/cycle/retry_when_no_further_event_data.rb +++ b/test/automated/iterator/cycle/retry_when_no_further_event_data.rb @@ -1,6 +1,6 @@ require_relative '../../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Iterator" do context "Cycle" do diff --git a/test/automated/iterator/next.rb b/test/automated/iterator/next.rb index 0442cad..f3a7dcf 100644 --- a/test/automated/iterator/next.rb +++ b/test/automated/iterator/next.rb @@ -1,10 +1,10 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Iterator" do context "Next" do - stream_name = controls::Put.(instances: 2) + stream_name = Controls::Put.(instances: 2) iterator = Iterator.build(stream_name: stream_name, batch_size: 1) diff --git a/test/automated/iterator/no_further_event_data.rb b/test/automated/iterator/no_further_event_data.rb index a408be4..6f8513e 100644 --- a/test/automated/iterator/no_further_event_data.rb +++ b/test/automated/iterator/no_further_event_data.rb @@ -1,10 +1,10 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Iterator" do context "No further event data" do - stream_name = controls::Put.(instances: 2) + stream_name = Controls::Put.(instances: 2) iterator = Iterator.build(stream_name: stream_name) diff --git a/test/automated/put/category_as_stream_name.rb b/test/automated/put/category_as_stream_name.rb index 4d0d578..321c30c 100644 --- a/test/automated/put/category_as_stream_name.rb +++ b/test/automated/put/category_as_stream_name.rb @@ -1,11 +1,11 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Put" do context "Category as Stream Name" do - category = controls::Category.example - write_event = controls::EventData::Write.example + category = Controls::Category.example + write_event = Controls::EventData::Write.example Put.(category, write_event) diff --git a/test/automated/put/expected_version_error.rb b/test/automated/put/expected_version_error.rb index 98eaf88..1452b1d 100644 --- a/test/automated/put/expected_version_error.rb +++ b/test/automated/put/expected_version_error.rb @@ -1,11 +1,11 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Expected Version" do context "Expected version does not match the stream version" do - stream_name = controls::StreamName.example - write_event = controls::EventData::Write.example + stream_name = Controls::StreamName.example + write_event = Controls::EventData::Write.example stream_position = Put.(stream_name, write_event) diff --git a/test/automated/put/no_stream/existing_stream.rb b/test/automated/put/no_stream/existing_stream.rb index 468c398..15e93fc 100644 --- a/test/automated/put/no_stream/existing_stream.rb +++ b/test/automated/put/no_stream/existing_stream.rb @@ -1,14 +1,14 @@ require_relative '../../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Put" do context "No Stream" do context "Existing Stream" do - stream_name = controls::StreamName.example + stream_name = Controls::StreamName.example - write_event_1 = controls::EventData::Write.example(data: {:some_attribute => 'first'}) - write_event_2 = controls::EventData::Write.example(data: {:some_attribute => 'second'}) + write_event_1 = Controls::EventData::Write.example(data: {:some_attribute => 'first'}) + write_event_2 = Controls::EventData::Write.example(data: {:some_attribute => 'second'}) Put.(stream_name, write_event_1) diff --git a/test/automated/put/no_stream/no_stream.rb b/test/automated/put/no_stream/no_stream.rb index 3f0d0fc..f668880 100644 --- a/test/automated/put/no_stream/no_stream.rb +++ b/test/automated/put/no_stream/no_stream.rb @@ -1,12 +1,12 @@ require_relative '../../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Put" do context "No Stream" do context "For a stream that doesn't exist" do - stream_name = controls::StreamName.example - write_event = controls::EventData::Write.example + stream_name = Controls::StreamName.example + write_event = Controls::EventData::Write.example stream_position = Put.(stream_name, write_event) diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index 0269264..65e6de7 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -1,10 +1,10 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Put and Get" do - stream_name = controls::StreamName.example - write_event = controls::EventData::Write.example + stream_name = Controls::StreamName.example + write_event = Controls::EventData::Write.example written_stream_position = Put.(stream_name, write_event) diff --git a/test/automated/put/returns_stream_position.rb b/test/automated/put/returns_stream_position.rb index d826480..3144057 100644 --- a/test/automated/put/returns_stream_position.rb +++ b/test/automated/put/returns_stream_position.rb @@ -1,11 +1,11 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Write Event Data" do - stream_name = controls::StreamName.example + stream_name = Controls::StreamName.example - write_event = controls::EventData::Write.example + write_event = Controls::EventData::Write.example stream_position = Put.(stream_name, write_event) diff --git a/test/automated/put/stream_position_increases.rb b/test/automated/put/stream_position_increases.rb index cb60332..5f4c7d8 100644 --- a/test/automated/put/stream_position_increases.rb +++ b/test/automated/put/stream_position_increases.rb @@ -1,10 +1,10 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Stream Version Increases with Subsequent Writes" do - stream_name = controls::StreamName.example - write_event = controls::EventData::Write.example + stream_name = Controls::StreamName.example + write_event = Controls::EventData::Write.example stream_position_1 = Put.(stream_name, write_event) stream_position_2 = Put.(stream_name, write_event) diff --git a/test/automated/read/missing_block_error.rb b/test/automated/read/missing_block_error.rb index c48d50a..1fd45f1 100644 --- a/test/automated/read/missing_block_error.rb +++ b/test/automated/read/missing_block_error.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Read" do context "No block supplied" do diff --git a/test/automated/read/read.rb b/test/automated/read/read.rb index 3259b61..3f492ff 100644 --- a/test/automated/read/read.rb +++ b/test/automated/read/read.rb @@ -1,9 +1,9 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Read" do - stream_name = controls::Put.(instances: 2) + stream_name = Controls::Put.(instances: 2) batch = [] diff --git a/test/automated/read/synchronous_result.skip.rb b/test/automated/read/synchronous_result.skip.rb index ddac581..af92992 100644 --- a/test/automated/read/synchronous_result.skip.rb +++ b/test/automated/read/synchronous_result.skip.rb @@ -1,6 +1,6 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Read" do context "Synchronous Result" do diff --git a/test/automated/select_statement/category_query.rb b/test/automated/select_statement/category_query.rb index 5599549..554529d 100644 --- a/test/automated/select_statement/category_query.rb +++ b/test/automated/select_statement/category_query.rb @@ -1,10 +1,10 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Select Statement" do context "Category Query" do - stream = controls::Stream::Category.example + stream = Controls::Stream::Category.example select_statement = Get::SelectStatement.build stream diff --git a/test/automated/select_statement/defaults.rb b/test/automated/select_statement/defaults.rb index 4832f09..6aad4d3 100644 --- a/test/automated/select_statement/defaults.rb +++ b/test/automated/select_statement/defaults.rb @@ -1,10 +1,10 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Select Statement" do context "Defaults" do - stream = controls::Stream.example + stream = Controls::Stream.example select_statement = Get::SelectStatement.build stream diff --git a/test/automated/select_statement/stream_name_query.rb b/test/automated/select_statement/stream_name_query.rb index 04a6e1c..6161749 100644 --- a/test/automated/select_statement/stream_name_query.rb +++ b/test/automated/select_statement/stream_name_query.rb @@ -1,10 +1,10 @@ require_relative '../automated_init' -controls = EventSource::Postgres::Controls +Controls = EventSource::Postgres::Controls context "Select Statement" do context "Stream Name Query" do - stream = controls::Stream.example + stream = Controls::Stream.example select_statement = Get::SelectStatement.build stream diff --git a/test/test_init.rb b/test/test_init.rb index 08399e2..c213c58 100644 --- a/test/test_init.rb +++ b/test/test_init.rb @@ -14,7 +14,6 @@ require_relative '../init.rb' require 'event_source/postgres/controls' -controls = EventSource::Postgres::Controls require 'test_bench'; TestBench.activate From c86549550c9a8b370c4a762e7de088335d2a3311 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Oct 2016 20:33:09 -0500 Subject: [PATCH 009/591] Root controls library and namespace is removed --- event_source-postgres.gemspec | 2 -- lib/event_source/postgres/controls.rb | 2 -- test/automated/get/category.rb | 2 -- test/automated/get/get.rb | 2 -- test/automated/get/no_events.rb | 2 -- test/automated/get/precedence.rb | 2 -- .../iterator/cycle/retry_when_no_further_event_data.rb | 2 -- test/automated/iterator/next.rb | 2 -- test/automated/iterator/no_further_event_data.rb | 2 -- test/automated/put/category_as_stream_name.rb | 2 -- test/automated/put/expected_version_error.rb | 2 -- test/automated/put/no_stream/existing_stream.rb | 2 -- test/automated/put/no_stream/no_stream.rb | 2 -- test/automated/put/put.rb | 2 -- test/automated/put/returns_stream_position.rb | 2 -- test/automated/put/stream_position_increases.rb | 2 -- test/automated/read/missing_block_error.rb | 2 -- test/automated/read/read.rb | 2 -- test/automated/read/synchronous_result.skip.rb | 2 -- test/automated/select_statement/category_query.rb | 2 -- test/automated/select_statement/defaults.rb | 2 -- test/automated/select_statement/stream_name_query.rb | 2 -- 22 files changed, 44 deletions(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index a83ac18..e78e2d0 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -25,8 +25,6 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'settings' s.add_runtime_dependency 'async_invocation' - s.add_runtime_dependency 'controls' - s.add_runtime_dependency 'pg' s.add_development_dependency 'test_bench' diff --git a/lib/event_source/postgres/controls.rb b/lib/event_source/postgres/controls.rb index aa30abe..9877ecd 100644 --- a/lib/event_source/postgres/controls.rb +++ b/lib/event_source/postgres/controls.rb @@ -1,5 +1,3 @@ -require 'controls' - require 'event_source/controls' require 'event_source/postgres/controls/category' diff --git a/test/automated/get/category.rb b/test/automated/get/category.rb index e67c5ad..10aeade 100644 --- a/test/automated/get/category.rb +++ b/test/automated/get/category.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Get" do context "Category" do category = Controls::Category.example diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index 65e6de7..cd4ce97 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Put and Get" do stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example diff --git a/test/automated/get/no_events.rb b/test/automated/get/no_events.rb index cef880e..e5c5bf4 100644 --- a/test/automated/get/no_events.rb +++ b/test/automated/get/no_events.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Get" do context "No Events" do stream_name = Controls::StreamName.example diff --git a/test/automated/get/precedence.rb b/test/automated/get/precedence.rb index 854b409..5f4f493 100644 --- a/test/automated/get/precedence.rb +++ b/test/automated/get/precedence.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Get" do context "Precedence" do stream_name = Controls::Put.(instances: 3) diff --git a/test/automated/iterator/cycle/retry_when_no_further_event_data.rb b/test/automated/iterator/cycle/retry_when_no_further_event_data.rb index 25d1614..3334333 100644 --- a/test/automated/iterator/cycle/retry_when_no_further_event_data.rb +++ b/test/automated/iterator/cycle/retry_when_no_further_event_data.rb @@ -1,7 +1,5 @@ require_relative '../../automated_init' -Controls = EventSource::Postgres::Controls - context "Iterator" do context "Cycle" do context "Retry when no further event data" do diff --git a/test/automated/iterator/next.rb b/test/automated/iterator/next.rb index f3a7dcf..b9c8d08 100644 --- a/test/automated/iterator/next.rb +++ b/test/automated/iterator/next.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Iterator" do context "Next" do stream_name = Controls::Put.(instances: 2) diff --git a/test/automated/iterator/no_further_event_data.rb b/test/automated/iterator/no_further_event_data.rb index 6f8513e..53b6661 100644 --- a/test/automated/iterator/no_further_event_data.rb +++ b/test/automated/iterator/no_further_event_data.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Iterator" do context "No further event data" do stream_name = Controls::Put.(instances: 2) diff --git a/test/automated/put/category_as_stream_name.rb b/test/automated/put/category_as_stream_name.rb index 321c30c..a76c027 100644 --- a/test/automated/put/category_as_stream_name.rb +++ b/test/automated/put/category_as_stream_name.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Put" do context "Category as Stream Name" do category = Controls::Category.example diff --git a/test/automated/put/expected_version_error.rb b/test/automated/put/expected_version_error.rb index 1452b1d..7d97ca8 100644 --- a/test/automated/put/expected_version_error.rb +++ b/test/automated/put/expected_version_error.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Expected Version" do context "Expected version does not match the stream version" do stream_name = Controls::StreamName.example diff --git a/test/automated/put/no_stream/existing_stream.rb b/test/automated/put/no_stream/existing_stream.rb index 15e93fc..56de771 100644 --- a/test/automated/put/no_stream/existing_stream.rb +++ b/test/automated/put/no_stream/existing_stream.rb @@ -1,7 +1,5 @@ require_relative '../../automated_init' -Controls = EventSource::Postgres::Controls - context "Put" do context "No Stream" do context "Existing Stream" do diff --git a/test/automated/put/no_stream/no_stream.rb b/test/automated/put/no_stream/no_stream.rb index f668880..4652d5b 100644 --- a/test/automated/put/no_stream/no_stream.rb +++ b/test/automated/put/no_stream/no_stream.rb @@ -1,7 +1,5 @@ require_relative '../../automated_init' -Controls = EventSource::Postgres::Controls - context "Put" do context "No Stream" do context "For a stream that doesn't exist" do diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index 65e6de7..cd4ce97 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Put and Get" do stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example diff --git a/test/automated/put/returns_stream_position.rb b/test/automated/put/returns_stream_position.rb index 3144057..0d5f5f7 100644 --- a/test/automated/put/returns_stream_position.rb +++ b/test/automated/put/returns_stream_position.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Write Event Data" do stream_name = Controls::StreamName.example diff --git a/test/automated/put/stream_position_increases.rb b/test/automated/put/stream_position_increases.rb index 5f4c7d8..2fd9f35 100644 --- a/test/automated/put/stream_position_increases.rb +++ b/test/automated/put/stream_position_increases.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Stream Version Increases with Subsequent Writes" do stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example diff --git a/test/automated/read/missing_block_error.rb b/test/automated/read/missing_block_error.rb index 1fd45f1..f69ffa0 100644 --- a/test/automated/read/missing_block_error.rb +++ b/test/automated/read/missing_block_error.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Read" do context "No block supplied" do test "Is incorrect" do diff --git a/test/automated/read/read.rb b/test/automated/read/read.rb index 3f492ff..bd28fe1 100644 --- a/test/automated/read/read.rb +++ b/test/automated/read/read.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Read" do stream_name = Controls::Put.(instances: 2) diff --git a/test/automated/read/synchronous_result.skip.rb b/test/automated/read/synchronous_result.skip.rb index af92992..39727be 100644 --- a/test/automated/read/synchronous_result.skip.rb +++ b/test/automated/read/synchronous_result.skip.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Read" do context "Synchronous Result" do res = Read.(stream_name: 'some_stream_name') { } diff --git a/test/automated/select_statement/category_query.rb b/test/automated/select_statement/category_query.rb index 554529d..998dc4c 100644 --- a/test/automated/select_statement/category_query.rb +++ b/test/automated/select_statement/category_query.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Select Statement" do context "Category Query" do stream = Controls::Stream::Category.example diff --git a/test/automated/select_statement/defaults.rb b/test/automated/select_statement/defaults.rb index 6aad4d3..acbaa2e 100644 --- a/test/automated/select_statement/defaults.rb +++ b/test/automated/select_statement/defaults.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Select Statement" do context "Defaults" do stream = Controls::Stream.example diff --git a/test/automated/select_statement/stream_name_query.rb b/test/automated/select_statement/stream_name_query.rb index 6161749..07db00e 100644 --- a/test/automated/select_statement/stream_name_query.rb +++ b/test/automated/select_statement/stream_name_query.rb @@ -1,7 +1,5 @@ require_relative '../automated_init' -Controls = EventSource::Postgres::Controls - context "Select Statement" do context "Stream Name Query" do stream = Controls::Stream.example From 65be20cb29393002b8ffa2881b14cfe5020a9cb1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Oct 2016 20:53:59 -0500 Subject: [PATCH 010/591] Automated, rather than bench --- test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.rb b/test.rb index 9efd53e..d92b472 100644 --- a/test.rb +++ b/test.rb @@ -1 +1 @@ -require_relative 'test/bench' +require_relative 'test/automated' From e054d1f64a063bd73a6380351a23f569efbd3b91 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Oct 2016 20:59:25 -0500 Subject: [PATCH 011/591] Project URI is corrected --- MIT-License.txt | 3 +-- event_source-postgres.gemspec | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/MIT-License.txt b/MIT-License.txt index 24cc64e..cd0e498 100644 --- a/MIT-License.txt +++ b/MIT-License.txt @@ -1,5 +1,4 @@ -Original work Copyright (c) 2015-2016 Obsidian Software, Inc. -Modified work Copyright (c) 2016 Scott Bellware +Copyright (c) 2016 Scott Bellware Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index e78e2d0..c8e52ab 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -7,7 +7,7 @@ Gem::Specification.new do |s| s.authors = ['The Eventide Project'] s.email = 'opensource@eventide-project.org' - s.homepage = 'https://github.com/eventide-project/event-stream-postgres' + s.homepage = 'https://github.com/eventide-project/event-source-postgres' s.licenses = ['MIT'] s.require_paths = ['lib'] From 0f5536301518d7b817d557577e8629103d2097a6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Oct 2016 22:34:06 -0500 Subject: [PATCH 012/591] Debug, rather than trace, is logged upon completing --- lib/event_source/postgres/iterator/cycle.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_source/postgres/iterator/cycle.rb b/lib/event_source/postgres/iterator/cycle.rb index 61f86eb..e3bde54 100644 --- a/lib/event_source/postgres/iterator/cycle.rb +++ b/lib/event_source/postgres/iterator/cycle.rb @@ -84,7 +84,7 @@ def call(&action) end end - logger.opt_trace "Cycled (Iterations: #{iteration}, Delay Milliseconds: #{delay_milliseconds}, Timeout Milliseconds: #{timeout_milliseconds.inspect}, Stop Time: #{clock.iso8601(stop_time)})" + logger.opt_debug "Cycled (Iterations: #{iteration}, Delay Milliseconds: #{delay_milliseconds}, Timeout Milliseconds: #{timeout_milliseconds.inspect}, Stop Time: #{clock.iso8601(stop_time)})" return result end From 84bb8816617b2518e870edb7b536dff2687590b7 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Oct 2016 23:41:26 -0500 Subject: [PATCH 013/591] Telemetry data struct is not used --- lib/event_source/postgres/iterator/cycle.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/event_source/postgres/iterator/cycle.rb b/lib/event_source/postgres/iterator/cycle.rb index e3bde54..bc3f10f 100644 --- a/lib/event_source/postgres/iterator/cycle.rb +++ b/lib/event_source/postgres/iterator/cycle.rb @@ -129,8 +129,6 @@ class Sink record :timed_out end - Data = Struct.new :milliseconds - def self.sink Sink.new end From 029c9d780e1385b0dada37b6949318d662c9c3de Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 2 Oct 2016 10:28:42 -0500 Subject: [PATCH 014/591] The cycle library is a package dependency --- event_source-postgres.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index c8e52ab..c725894 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -22,6 +22,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'telemetry' s.add_runtime_dependency 'telemetry-logger' + s.add_runtime_dependency 'cycle' s.add_runtime_dependency 'settings' s.add_runtime_dependency 'async_invocation' From c7746a6b74bf5a269e5763c2e58eebd9f96dd4b8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 2 Oct 2016 14:11:14 -0500 Subject: [PATCH 015/591] Cycle implementation is replaced with the cycle library --- lib/event_source/postgres.rb | 2 +- lib/event_source/postgres/iterator/cycle.rb | 172 ------------------ test/automated.rb | 2 +- .../cycle/retry_when_no_further_event_data.rb | 26 --- test/automated/iterator/cycle/telemetry.rb | 65 ------- 5 files changed, 2 insertions(+), 265 deletions(-) delete mode 100644 lib/event_source/postgres/iterator/cycle.rb delete mode 100644 test/automated/iterator/cycle/retry_when_no_further_event_data.rb delete mode 100644 test/automated/iterator/cycle/telemetry.rb diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index 8518844..c85236a 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -1,6 +1,7 @@ require 'pg' require 'event_source' +require 'cycle' require 'telemetry/logger' require 'telemetry' @@ -15,7 +16,6 @@ require 'event_source/postgres/get/select_statement' require 'event_source/postgres/get' -require 'event_source/postgres/iterator/cycle' require 'event_source/postgres/iterator' require 'event_source/postgres/read' diff --git a/lib/event_source/postgres/iterator/cycle.rb b/lib/event_source/postgres/iterator/cycle.rb deleted file mode 100644 index bc3f10f..0000000 --- a/lib/event_source/postgres/iterator/cycle.rb +++ /dev/null @@ -1,172 +0,0 @@ -module EventSource - module Postgres - class Iterator - class Cycle - dependency :clock, Clock::UTC - dependency :telemetry, Telemetry - dependency :logger, Telemetry::Logger - - def delay_milliseconds - @delay_milliseconds ||= Defaults.delay_milliseconds - end - - def delay_condition - @delay_condition ||= Defaults.delay_condition - end - - initializer na(:delay_milliseconds), :timeout_milliseconds, na(:delay_condition) - - def self.build(delay_milliseconds: nil, timeout_milliseconds: nil, delay_condition: nil) - new(delay_milliseconds, timeout_milliseconds, delay_condition).tap do |instance| - instance.configure - end - end - - def self.configure(receiver, attr_name: nil, delay_milliseconds: nil, timeout_milliseconds: nil, delay_condition: nil, cycle: nil) - attr_name ||= :cycle - - if !cycle.nil? - instance = cycle - else - if delay_milliseconds.nil? && timeout_milliseconds.nil? - instance = None.build - else - instance = build(delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, delay_condition: delay_condition) - end - end - - receiver.public_send "#{attr_name}=", instance - end - - def configure - Clock::UTC.configure self - ::Telemetry.configure self - ::Telemetry::Logger.configure self - end - - def self.call(delay_milliseconds: nil, timeout_milliseconds: nil, delay_condition: nil, &action) - instance = build(delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, delay_condition: delay_condition) - instance.call(&action) - end - - def call(&action) - stop_time = nil - if !timeout_milliseconds.nil? - stop_time = clock.now + (timeout_milliseconds.to_f / 1000.0) - end - - logger.opt_trace "Cycling (Delay Milliseconds: #{delay_milliseconds}, Timeout Milliseconds: #{timeout_milliseconds.inspect}, Stop Time: #{clock.iso8601(stop_time)})" - - iteration = -1 - result = nil - loop do - iteration += 1 - telemetry.record :cycle, iteration - - result = invoke(iteration, &action) - - if delay_condition.(result) - logger.opt_debug "No results (Iteration: #{iteration})" - delay - else - logger.opt_debug "Got results (Iteration: #{iteration})" - telemetry.record :got_result - break - end - - if !timeout_milliseconds.nil? - now = clock.now - if now >= stop_time - logger.opt_debug "Timeout has lapsed (Iteration: #{iteration}, Stop Time: #{clock.iso8601(stop_time)}, Timeout Milliseconds: #{timeout_milliseconds})" - telemetry.record :timed_out, now - break - end - end - end - - logger.opt_debug "Cycled (Iterations: #{iteration}, Delay Milliseconds: #{delay_milliseconds}, Timeout Milliseconds: #{timeout_milliseconds.inspect}, Stop Time: #{clock.iso8601(stop_time)})" - - return result - end - - def invoke(iteration, &action) - logger.opt_trace "Invoking action (Iteration: #{iteration})" - - result = action.call - telemetry.record :invoked_action - - logger.opt_debug "Invoked action (Iteration: #{iteration})" - - result - end - - def delay - logger.opt_trace "Delaying (Milliseconds: #{delay_milliseconds})" - - delay_seconds = (delay_milliseconds.to_f / 1000.0) - - sleep delay_seconds - - telemetry.record :delayed, delay_milliseconds - - logger.opt_debug "Finished delaying (Milliseconds: #{delay_milliseconds})" - end - - def self.register_telemetry_sink(writer) - sink = Telemetry.sink - writer.telemetry.register sink - sink - end - - module Telemetry - class Sink - include ::Telemetry::Sink - - record :cycle - record :invoked_action - record :got_result - record :delayed - record :timed_out - end - - def self.sink - Sink.new - end - end - - module Substitute - def self.build - Cycle::None.build.tap do |instance| - sink = Cycle.register_telemetry_sink(instance) - instance.sink = sink - end - end - end - - class None < Cycle - attr_accessor :sink - - def call(&action) - action.call - end - end - - module Defaults - def self.delay_milliseconds - 200 - end - - def self.delay_condition - lambda do |result| - if result.respond_to? :empty? - result.empty? - else - result.nil? - end - end - end - end - end - end - end -end diff --git a/test/automated.rb b/test/automated.rb index e36d314..de477e0 100644 --- a/test/automated.rb +++ b/test/automated.rb @@ -2,5 +2,5 @@ TestBench::Runner.( 'automated/**/*.rb', - exclude_pattern: %r{/^skip_|(?:_init\.rb|\.sketch\.rb|_sketch\.rb|sketch\.rb|\.skip\.rb|_tests\.rb)\z} + exclude_pattern: %r{\/_|sketch|(_init\.rb|_tests\.rb)\z} ) or exit 1 diff --git a/test/automated/iterator/cycle/retry_when_no_further_event_data.rb b/test/automated/iterator/cycle/retry_when_no_further_event_data.rb deleted file mode 100644 index 3334333..0000000 --- a/test/automated/iterator/cycle/retry_when_no_further_event_data.rb +++ /dev/null @@ -1,26 +0,0 @@ -require_relative '../../automated_init' - -context "Iterator" do - context "Cycle" do - context "Retry when no further event data" do - cycle = Iterator::Cycle.build(delay_milliseconds: 10, timeout_milliseconds: 100) - sink = Iterator::Cycle.register_telemetry_sink(cycle) - - iterator = Iterator.build(stream_name: 'some_stream', cycle: cycle) - - iterator.next - - test "Didn't get result" do - refute(sink.recorded_got_result?) - end - - test "Delayed before retrying" do - assert(sink.recorded_delayed?) - end - - test "Timed out" do - assert(sink.recorded_timed_out?) - end - end - end -end diff --git a/test/automated/iterator/cycle/telemetry.rb b/test/automated/iterator/cycle/telemetry.rb deleted file mode 100644 index d77705c..0000000 --- a/test/automated/iterator/cycle/telemetry.rb +++ /dev/null @@ -1,65 +0,0 @@ -require_relative '../../automated_init' - -context "Iterator" do - context "Cycle" do - context "Telemetry" do - context "Got Result" do - cycle = Iterator::Cycle.build - sink = Iterator::Cycle.register_telemetry_sink(cycle) - - cycle.() do - 'some result' - end - - test "Recorded cycle" do - iteration = 0 - assert(sink.recorded_cycle? { |r| r.data == iteration }) - end - - test "Recorded invoked action" do - assert(sink.recorded_invoked_action?) - end - - test "Recorded got result" do - assert(sink.recorded_got_result?) - end - - test "Didn't record delayed" do - refute(sink.recorded_delayed?) - end - - test "Didn't record timed out" do - refute(sink.recorded_timed_out?) - end - end - - context "Got No Result" do - cycle = Iterator::Cycle.build(delay_milliseconds: 50, timeout_milliseconds: 100) - sink = Iterator::Cycle.register_telemetry_sink(cycle) - cycle.() do - nil - end - - test "Recorded cycle" do - assert(sink.recorded_cycle?) - end - - test "Recorded invoked action" do - assert(sink.recorded_invoked_action?) - end - - test "Recorded delayed" do - assert(sink.recorded_delayed?) - end - - test "Recorded timed out" do - assert(sink.recorded_timed_out?) - end - - test "Didn't record got result" do - refute(sink.recorded_got_result?) - end - end - end - end -end From 5c6d622f54e90bcc377b922007adf4577267d652 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 7 Oct 2016 19:00:45 -0500 Subject: [PATCH 016/591] The log library is added to the gemspec --- event_source-postgres.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index c725894..1dcb8db 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -22,6 +22,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'telemetry' s.add_runtime_dependency 'telemetry-logger' + s.add_runtime_dependency 'log' s.add_runtime_dependency 'cycle' s.add_runtime_dependency 'settings' s.add_runtime_dependency 'async_invocation' From 51459cf383d65d747c9be4b40100f3608e7a1777 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 7 Oct 2016 19:53:13 -0500 Subject: [PATCH 017/591] Log replaces TElemetry::Logger --- event_source-postgres.gemspec | 2 -- lib/event_source/postgres.rb | 2 +- lib/event_source/postgres/get.rb | 16 +++++----- .../postgres/get/select_statement.rb | 18 ++++------- lib/event_source/postgres/iterator.rb | 32 +++++++++---------- lib/event_source/postgres/put.rb | 24 +++++++------- lib/event_source/postgres/read.rb | 8 ++--- lib/event_source/postgres/session.rb | 8 ++--- test/test_init.rb | 12 +------ 9 files changed, 51 insertions(+), 71 deletions(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 1dcb8db..f8e7ee9 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -20,8 +20,6 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'event_source' - s.add_runtime_dependency 'telemetry' - s.add_runtime_dependency 'telemetry-logger' s.add_runtime_dependency 'log' s.add_runtime_dependency 'cycle' s.add_runtime_dependency 'settings' diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index c85236a..2266e9f 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -3,7 +3,7 @@ require 'event_source' require 'cycle' -require 'telemetry/logger' +require 'log' require 'telemetry' require 'settings'; Settings.activate require 'async_invocation' diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 301af6a..22436f6 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -1,10 +1,11 @@ module EventSource module Postgres class Get + include Log::Dependency + initializer :stream_name, :category, :batch_size, :precedence dependency :session, Session - dependency :logger, Telemetry::Logger def self.build(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, session: nil) new(stream_name, category, batch_size, precedence).tap do |instance| @@ -25,36 +26,35 @@ def self.call(stream_name: nil, category: nil, stream_position: nil, batch_size: def configure(session: nil) Session.configure self, session: session - Telemetry::Logger.configure self end def call(stream_position: nil) - logger.opt_trace "Getting event data (Stream Position: #{stream_position.inspect}, Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.trace "Getting event data (Stream Position: #{stream_position.inspect}, Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" stream = Stream.build stream_name: stream_name, category: category records = get_records(stream, stream_position) events = convert(records) - logger.opt_debug "Finished getting event data (Count: #{events.length}, Stream Position: #{stream_position.inspect}, Stream: #{stream.name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.debug "Finished getting event data (Count: #{events.length}, Stream Position: #{stream_position.inspect}, Stream: #{stream.name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" events end def get_records(stream, stream_position) - logger.opt_trace "Getting records (Stream: #{stream.name}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.trace "Getting records (Stream: #{stream.name}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" select_statement = SelectStatement.build(stream, offset: stream_position, batch_size: batch_size, precedence: precedence) records = session.connection.exec(select_statement.sql) - logger.opt_debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" records end def convert(records) - logger.opt_trace "Converting records to event data (Records Count: #{records.ntuples})" + logger.trace "Converting records to event data (Records Count: #{records.ntuples})" events = records.map do |record| record['data'] = Deserialize.data(record['data']) @@ -64,7 +64,7 @@ def convert(records) EventData::Read.build record end - logger.opt_debug "Converted records to event data (Event Data Count: #{events.length})" + logger.debug "Converted records to event data (Event Data Count: #{events.length})" events end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index c1633ae..330ad6b 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -2,9 +2,9 @@ module EventSource module Postgres class Get class SelectStatement - initializer :stream, w(:offset), w(:batch_size), w(:precedence) + include Log::Dependency - dependency :logger, Telemetry::Logger + initializer :stream, w(:offset), w(:batch_size), w(:precedence) def offset @offset ||= Defaults.offset @@ -27,17 +27,11 @@ def stream_type end def self.build(stream, offset: nil, batch_size: nil, precedence: nil) - new(stream, offset, batch_size, precedence).tap do |instance| - instance.configure - end - end - - def configure - Telemetry::Logger.configure self + new(stream, offset, batch_size, precedence) end def sql - logger.opt_trace "Composing select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" + logger.trace "Composing select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" statement = <<-SQL SELECT @@ -61,8 +55,8 @@ def sql ; SQL - logger.opt_debug "Composed select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" - logger.opt_data "Statement: #{statement}" + logger.debug "Composed select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" + logger.data "Statement: #{statement}" statement end diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index 35c5e2d..4097b2f 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -3,6 +3,8 @@ module Postgres class Iterator class Error < RuntimeError; end + include Log::Dependency + attr_accessor :batch attr_writer :batch_position attr_writer :stream_offset @@ -17,15 +19,13 @@ def stream_offset dependency :get, Get dependency :cycle, Cycle - dependency :logger, Telemetry::Logger initializer :stream_name, :category, a(:stream_position, 0), :batch_size, :precedence def self.build(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) new(stream_name, category, stream_position, batch_size, precedence).tap do |instance| Get.configure instance, stream_name: stream_name, category: category, batch_size: batch_size, precedence: precedence, session: session - Telemetry::Logger.configure instance - Iterator::Cycle.configure instance, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle + Cycle.configure instance, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle end end @@ -36,14 +36,14 @@ def self.configure(receiver, attr_name: nil, stream_name: nil, category: nil, st end def next - logger.opt_trace "Getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" + logger.trace "Getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" resupply(batch) event_data = batch[batch_position] - logger.opt_debug "Finished getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" - logger.opt_data "Event Data: #{event_data.inspect}" + logger.debug "Finished getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" + logger.data "Event Data: #{event_data.inspect}" advance_positions @@ -51,7 +51,7 @@ def next end def resupply(batch) - logger.opt_trace "Resupplying batch" + logger.trace "Resupplying batch" if batch.nil? batch_log_text = "Batch: #{batch.inspect}" @@ -68,38 +68,38 @@ def resupply(batch) logger.debug "Current batch not depleted (#{batch_log_text}, Batch Position: #{batch_position})" end - logger.opt_debug "Finished resupplying batch" + logger.debug "Finished resupplying batch" end def reset(batch) - logger.opt_trace "Resetting batch" + logger.trace "Resetting batch" self.batch = batch self.batch_position = 0 - logger.opt_debug "Reset batch" - logger.opt_data ("Batch: #{batch.inspect}") - logger.opt_data ("Batch Position: #{batch_position.inspect}") + logger.debug "Reset batch" + logger.data ("Batch: #{batch.inspect}") + logger.data ("Batch Position: #{batch_position.inspect}") end def get_batch - logger.opt_trace "Getting batch" + logger.trace "Getting batch" batch = nil cycle.() do batch = get.(stream_position: stream_offset) end - logger.opt_debug "Finished getting batch (Count: #{batch.length})" + logger.debug "Finished getting batch (Count: #{batch.length})" batch end def advance_positions - logger.opt_trace "Advancing positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" + logger.trace "Advancing positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" self.batch_position += 1 self.stream_offset += 1 - logger.opt_debug "Advanced positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" + logger.debug "Advanced positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" end end end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 254f935..0a863f9 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -1,10 +1,11 @@ module EventSource module Postgres class Put + include Log::Dependency + attr_reader :stream_name dependency :session, Session - dependency :logger, Telemetry::Logger def initialize(stream_name) @stream_name = stream_name @@ -18,7 +19,6 @@ def self.build(stream_name, session: nil) def configure(session: nil) Session.configure(self, session: session) - Telemetry::Logger.configure(self) end def self.call(stream_name, write_event, expected_version: nil, session: nil) @@ -27,14 +27,14 @@ def self.call(stream_name, write_event, expected_version: nil, session: nil) end def call(write_event, expected_version: nil) - logger.opt_trace "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" + logger.trace "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" type, data, metadata = destructure_event(write_event) expected_version = canonize_expected_version(expected_version) stream_position = insert_event(type, data, metadata, expected_version) - logger.opt_debug "Put event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" + logger.debug "Put event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" stream_position end @@ -44,8 +44,8 @@ def destructure_event(write_event) data = write_event.data metadata = write_event.metadata - logger.opt_data "Data: #{data.inspect}" - logger.opt_data "Metadata: #{metadata.inspect}" + logger.data "Data: #{data.inspect}" + logger.data "Metadata: #{metadata.inspect}" return type, data, metadata end @@ -53,9 +53,9 @@ def destructure_event(write_event) def canonize_expected_version(expected_version) return expected_version unless expected_version == NoStream.name - logger.opt_trace "Canonizing expected version (Expected Version: #{expected_version})" + logger.trace "Canonizing expected version (Expected Version: #{expected_version})" expected_version = NoStream.version - logger.opt_debug "Canonized expected version (Expected Version: #{expected_version})" + logger.debug "Canonized expected version (Expected Version: #{expected_version})" expected_version end @@ -67,7 +67,7 @@ def insert_event(type, data, metadata, expected_version) end def execute_query(type, serialized_data, serialized_metadata, expected_version) - logger.opt_trace "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" + logger.trace "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" sql_args = [ stream_name, @@ -83,7 +83,7 @@ def execute_query(type, serialized_data, serialized_metadata, expected_version) raise_error e end - logger.opt_debug "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" + logger.debug "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" records end @@ -95,7 +95,7 @@ def statement def serialized_data(data) serializable_data = EventData::Hash[data] serialized_data = Serialize::Write.(serializable_data, :json) - logger.opt_data "Serialized Data: #{serialized_data.inspect}" + logger.data "Serialized Data: #{serialized_data.inspect}" serialized_data end @@ -105,7 +105,7 @@ def serialized_metadata(metadata) unless metadata.nil? serialized_metadata = Serialize::Write.(serializable_metadata, :json) end - logger.opt_data "Serialized Metadata: #{serialized_metadata.inspect}" + logger.data "Serialized Metadata: #{serialized_metadata.inspect}" serialized_metadata end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index ab4df61..8a25cb9 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -3,16 +3,16 @@ module Postgres class Read class Error < RuntimeError; end + include Log::Dependency + initializer :stream_name, :category, :stream_position, :batch_size, :precedence dependency :session, Session dependency :iterator, Iterator - dependency :logger, Telemetry::Logger def self.build(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) new(stream_name, category, stream_position, batch_size, precedence).tap do |instance| Iterator.configure instance, stream_name: stream_name, category: category, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session - Telemetry::Logger.configure instance end end @@ -40,7 +40,7 @@ def call(&action) end def enumerate_event_data(&action) - logger.opt_trace "Reading event data (Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.trace "Reading event data (Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" event_data = nil @@ -52,7 +52,7 @@ def enumerate_event_data(&action) action.(event_data) end - logger.opt_debug "Finished reading event data (Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.debug "Finished reading event data (Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" end end end diff --git a/lib/event_source/postgres/session.rb b/lib/event_source/postgres/session.rb index 6a921d4..ecafb72 100644 --- a/lib/event_source/postgres/session.rb +++ b/lib/event_source/postgres/session.rb @@ -1,6 +1,8 @@ module EventSource module Postgres class Session + include Log::Dependency + def self.settings Settings.names end @@ -11,12 +13,8 @@ def self.settings attr_accessor :connection - dependency :logger, Telemetry::Logger - def self.build(connection: nil, settings: nil) new.tap do |instance| - Telemetry::Logger.configure instance - settings ||= Settings.instance settings.set(instance) @@ -90,7 +88,7 @@ def settings end def self.logger - @logger ||= Telemetry::Logger.get self + @logger ||= Log.get self end module LogText diff --git a/test/test_init.rb b/test/test_init.rb index c213c58..ca80aa6 100644 --- a/test/test_init.rb +++ b/test/test_init.rb @@ -1,14 +1,5 @@ ENV['CONSOLE_DEVICE'] ||= 'stdout' -ENV['LOG_COLOR'] ||= 'on' - -if ENV['LOG_LEVEL'] - ENV['LOGGER'] ||= 'on' -else - ENV['LOG_LEVEL'] ||= 'trace' -end - -ENV['LOGGER'] ||= 'off' -ENV['LOG_OPTIONAL'] ||= 'on' +ENV['LOG_LEVEL'] ||= 'fatal' puts RUBY_DESCRIPTION @@ -17,7 +8,6 @@ require 'test_bench'; TestBench.activate -Telemetry::Logger::AdHoc.activate require 'pp' include EventSource From 00698182ed5e8a971e397fee0f890a46a5745824 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 8 Oct 2016 11:51:31 -0500 Subject: [PATCH 018/591] Specialized logger adds this library's tags --- lib/event_source/postgres.rb | 4 ++-- lib/event_source/postgres/log.rb | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 lib/event_source/postgres/log.rb diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index 2266e9f..6041244 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -8,6 +8,8 @@ require 'settings'; Settings.activate require 'async_invocation' +require 'event_source/postgres/log' + require 'event_source/postgres/settings' require 'event_source/postgres/session' @@ -18,5 +20,3 @@ require 'event_source/postgres/get' require 'event_source/postgres/iterator' require 'event_source/postgres/read' - - diff --git a/lib/event_source/postgres/log.rb b/lib/event_source/postgres/log.rb new file mode 100644 index 0000000..714bd42 --- /dev/null +++ b/lib/event_source/postgres/log.rb @@ -0,0 +1,10 @@ +module EventSource + module Postgres + class Log < ::Log + def tag!(tags) + tags << :event_source_postgres + tags << :verbose + end + end + end +end From e134d98899c9d2d4a7b035aacc0e2581476d4705 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 11 Oct 2016 11:22:18 -0500 Subject: [PATCH 019/591] Tag SQL statement log messages with :sql --- .gitignore | 1 + lib/event_source/postgres/get/select_statement.rb | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 5d86689..0bec69b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ Gemfile.lock gems gems_backup *scratch* +*notes* diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 330ad6b..3f97851 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -31,7 +31,7 @@ def self.build(stream, offset: nil, batch_size: nil, precedence: nil) end def sql - logger.trace "Composing select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" + logger.trace "Composing select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})", tag: :sql statement = <<-SQL SELECT @@ -55,8 +55,8 @@ def sql ; SQL - logger.debug "Composed select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" - logger.data "Statement: #{statement}" + logger.debug "Composed select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})", tag: :sql + logger.data "Statement: #{statement}", tag: :sql statement end From 4bc0a0cde02b1420549e34ac311f7808bd647a2f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 15 Oct 2016 22:27:51 -0500 Subject: [PATCH 020/591] Data logging is replaced with a data tag --- lib/event_source/postgres/get/select_statement.rb | 2 +- lib/event_source/postgres/iterator.rb | 6 +++--- lib/event_source/postgres/log.rb | 1 + lib/event_source/postgres/put.rb | 8 ++++---- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 3f97851..f31e1b4 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -56,7 +56,7 @@ def sql SQL logger.debug "Composed select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})", tag: :sql - logger.data "Statement: #{statement}", tag: :sql + logger.debug "Statement: #{statement}", tags: [:data, :sql] statement end diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index 4097b2f..95120b0 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -43,7 +43,7 @@ def next event_data = batch[batch_position] logger.debug "Finished getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" - logger.data "Event Data: #{event_data.inspect}" + logger.debug "Event Data: #{event_data.inspect}", tag: :data advance_positions @@ -78,8 +78,8 @@ def reset(batch) self.batch_position = 0 logger.debug "Reset batch" - logger.data ("Batch: #{batch.inspect}") - logger.data ("Batch Position: #{batch_position.inspect}") + logger.debug "Batch: #{batch.inspect}", tag: :data + logger.debug "Batch Position: #{batch_position.inspect}", tag: :data end def get_batch diff --git a/lib/event_source/postgres/log.rb b/lib/event_source/postgres/log.rb index 714bd42..b6fdf18 100644 --- a/lib/event_source/postgres/log.rb +++ b/lib/event_source/postgres/log.rb @@ -3,6 +3,7 @@ module Postgres class Log < ::Log def tag!(tags) tags << :event_source_postgres + tags << :library tags << :verbose end end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 0a863f9..cf6a0c8 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -44,8 +44,8 @@ def destructure_event(write_event) data = write_event.data metadata = write_event.metadata - logger.data "Data: #{data.inspect}" - logger.data "Metadata: #{metadata.inspect}" + logger.debug "Data: #{data.inspect}", tag: :data + logger.debug "Metadata: #{metadata.inspect}", tag: :data return type, data, metadata end @@ -95,7 +95,7 @@ def statement def serialized_data(data) serializable_data = EventData::Hash[data] serialized_data = Serialize::Write.(serializable_data, :json) - logger.data "Serialized Data: #{serialized_data.inspect}" + logger.debug "Serialized Data: #{serialized_data.inspect}", tag: :data serialized_data end @@ -105,7 +105,7 @@ def serialized_metadata(metadata) unless metadata.nil? serialized_metadata = Serialize::Write.(serializable_metadata, :json) end - logger.data "Serialized Metadata: #{serialized_metadata.inspect}" + logger.debug "Serialized Metadata: #{serialized_metadata.inspect}", tag: :data serialized_metadata end From b5156fbb8f92cbf6f1d6db45e43f6fab4d7bed49 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 15 Oct 2016 23:05:33 -0500 Subject: [PATCH 021/591] Internal cycle implementation is obsolete --- test/automated/read/retry_when_no_further_event_data.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/automated/read/retry_when_no_further_event_data.rb b/test/automated/read/retry_when_no_further_event_data.rb index e8c0203..5655100 100644 --- a/test/automated/read/retry_when_no_further_event_data.rb +++ b/test/automated/read/retry_when_no_further_event_data.rb @@ -1,8 +1,8 @@ require_relative '../automated_init' context "Read" do - cycle = Iterator::Cycle.build(delay_milliseconds: 10, timeout_milliseconds: 100) - sink = Iterator::Cycle.register_telemetry_sink(cycle) + cycle = Cycle.build(delay_milliseconds: 10, timeout_milliseconds: 100) + sink = Cycle.register_telemetry_sink(cycle) Read.(stream_name: 'some_stream', batch_size: 1, cycle: cycle) { |event_data| } From 8458b9a3b0d28f88325f3c8e1dfe583ed50658be Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 15 Oct 2016 23:15:39 -0500 Subject: [PATCH 022/591] Test for incorrect async result use is reinstated --- .../read/{synchronous_result.skip.rb => synchronous_result.rb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/automated/read/{synchronous_result.skip.rb => synchronous_result.rb} (100%) diff --git a/test/automated/read/synchronous_result.skip.rb b/test/automated/read/synchronous_result.rb similarity index 100% rename from test/automated/read/synchronous_result.skip.rb rename to test/automated/read/synchronous_result.rb From 6527e3c2c9b19086bfb2bac088dd3189bceba010 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 19 Oct 2016 16:13:07 -0500 Subject: [PATCH 023/591] Get takes a stream instance --- lib/event_source/postgres/controls/put.rb | 2 +- lib/event_source/postgres/get.rb | 17 ++++++++--------- lib/event_source/postgres/iterator.rb | 12 ++++++------ lib/event_source/postgres/read.rb | 17 +++++++++-------- test/automated/get/batch_size.rb | 4 ++-- test/automated/get/category.rb | 4 ++-- test/automated/get/get.rb | 6 +++--- test/automated/get/no_events.rb | 4 ++-- test/automated/get/precedence.rb | 6 +++--- 9 files changed, 36 insertions(+), 36 deletions(-) diff --git a/lib/event_source/postgres/controls/put.rb b/lib/event_source/postgres/controls/put.rb index 84c20fd..0ee3fe1 100644 --- a/lib/event_source/postgres/controls/put.rb +++ b/lib/event_source/postgres/controls/put.rb @@ -11,7 +11,7 @@ def self.call(instances: nil, stream_name: nil, event: nil, category: nil) EventSource::Postgres::Put.(stream_name, event) end - stream_name + EventSource::Stream.new(stream_name) end end end diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 22436f6..8036653 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -3,24 +3,24 @@ module Postgres class Get include Log::Dependency - initializer :stream_name, :category, :batch_size, :precedence + initializer :stream, :batch_size, :precedence dependency :session, Session - def self.build(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, session: nil) - new(stream_name, category, batch_size, precedence).tap do |instance| + def self.build(stream, batch_size: nil, precedence: nil, session: nil) + new(stream, batch_size, precedence).tap do |instance| instance.configure(session: session) end end - def self.configure(receiver, attr_name: nil, stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, session: nil) + def self.configure(receiver, stream, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, session: nil) attr_name ||= :get - instance = build(stream_name: stream_name, category: category, batch_size: batch_size, precedence: precedence, session: session) + instance = build(stream, batch_size: batch_size, precedence: precedence, session: session) receiver.public_send "#{attr_name}=", instance end - def self.call(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, session: nil) - instance = build(stream_name: stream_name, category: category, batch_size: batch_size, precedence: precedence, session: session) + def self.call(stream, stream_position: nil, batch_size: nil, precedence: nil, session: nil) + instance = build(stream, batch_size: batch_size, precedence: precedence, session: session) instance.(stream_position: stream_position) end @@ -29,9 +29,8 @@ def configure(session: nil) end def call(stream_position: nil) - logger.trace "Getting event data (Stream Position: #{stream_position.inspect}, Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.trace "Getting event data (Stream Position: #{stream_position.inspect}, Stream Name: #{stream.name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" - stream = Stream.build stream_name: stream_name, category: category records = get_records(stream, stream_position) events = convert(records) diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index 95120b0..7992004 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -20,18 +20,18 @@ def stream_offset dependency :get, Get dependency :cycle, Cycle - initializer :stream_name, :category, a(:stream_position, 0), :batch_size, :precedence + initializer :stream, a(:stream_position, 0), :batch_size, :precedence - def self.build(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) - new(stream_name, category, stream_position, batch_size, precedence).tap do |instance| - Get.configure instance, stream_name: stream_name, category: category, batch_size: batch_size, precedence: precedence, session: session + def self.build(stream, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + new(stream, stream_position, batch_size, precedence).tap do |instance| + Get.configure instance, stream, batch_size: batch_size, precedence: precedence, session: session Cycle.configure instance, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle end end - def self.configure(receiver, attr_name: nil, stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + def self.configure(receiver, stream, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) attr_name ||= :iterator - instance = build(stream_name: stream_name, category: category, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) + instance = build(stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) receiver.public_send "#{attr_name}=", instance end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 8a25cb9..b250f1a 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -5,25 +5,26 @@ class Error < RuntimeError; end include Log::Dependency - initializer :stream_name, :category, :stream_position, :batch_size, :precedence + initializer :stream, :stream_position, :batch_size, :precedence dependency :session, Session dependency :iterator, Iterator - def self.build(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) - new(stream_name, category, stream_position, batch_size, precedence).tap do |instance| - Iterator.configure instance, stream_name: stream_name, category: category, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session + def self.build(stream_name, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + stream = Stream.new(stream_name) + new(stream, stream_position, batch_size, precedence).tap do |instance| + Iterator.configure instance, stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session end end - def self.call(stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil, &action) - instance = build(stream_name: stream_name, category: category, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) + def self.call(stream_name, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil, &action) + instance = build(stream_name, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) instance.(&action) end - def self.configure(receiver, attr_name: nil, stream_name: nil, category: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + def self.configure(receiver, stream_name, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) attr_name ||= :reader - instance = build(stream_name: stream_name, category: category, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) + instance = build(stream_name, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) receiver.public_send "#{attr_name}=", instance end diff --git a/test/automated/get/batch_size.rb b/test/automated/get/batch_size.rb index 661f11d..3d3de3c 100644 --- a/test/automated/get/batch_size.rb +++ b/test/automated/get/batch_size.rb @@ -2,9 +2,9 @@ context "Get" do context "Batch Size" do - stream_name = Controls::Put.(instances: 3) + stream = Controls::Put.(instances: 3) - events = Get.(stream_name: stream_name, batch_size: 2) + events = Get.(stream, batch_size: 2) number_of_events = events.length diff --git a/test/automated/get/category.rb b/test/automated/get/category.rb index 10aeade..ec8c9fc 100644 --- a/test/automated/get/category.rb +++ b/test/automated/get/category.rb @@ -4,9 +4,9 @@ context "Category" do category = Controls::Category.example - Controls::Put.(instances: 2, category: category) + stream = Controls::Put.(instances: 2, category: category) - events = Get.(category: category) + events = Get.(stream) test "Number of events retrieved is the number written to the category" do number_of_events = events.length diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index cd4ce97..01bbb71 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -1,12 +1,12 @@ require_relative '../automated_init' context "Put and Get" do - stream_name = Controls::StreamName.example + stream = Controls::Stream.example write_event = Controls::EventData::Write.example - written_stream_position = Put.(stream_name, write_event) + written_stream_position = Put.(stream.name, write_event) - read_event = Get.(stream_name: stream_name, stream_position: written_stream_position)[0] + read_event = Get.(stream, stream_position: written_stream_position)[0] test "Got the event that was written" do assert(read_event.stream_position == written_stream_position) diff --git a/test/automated/get/no_events.rb b/test/automated/get/no_events.rb index e5c5bf4..ba733c2 100644 --- a/test/automated/get/no_events.rb +++ b/test/automated/get/no_events.rb @@ -2,9 +2,9 @@ context "Get" do context "No Events" do - stream_name = Controls::StreamName.example + stream = Controls::Stream.example - batch = Get.(stream_name: stream_name) + batch = Get.(stream) test "Empty array" do assert(batch == []) diff --git a/test/automated/get/precedence.rb b/test/automated/get/precedence.rb index 5f4f493..db49738 100644 --- a/test/automated/get/precedence.rb +++ b/test/automated/get/precedence.rb @@ -2,10 +2,10 @@ context "Get" do context "Precedence" do - stream_name = Controls::Put.(instances: 3) + stream = Controls::Put.(instances: 3) context "Ascending" do - events = Get.(stream_name: stream_name, precedence: :asc) + events = Get.(stream, precedence: :asc) first_event_postition = events.first.stream_position @@ -15,7 +15,7 @@ end context "Descending" do - events = Get.(stream_name: stream_name, precedence: :desc) + events = Get.(stream, precedence: :desc) first_event_postition = events.first.stream_position From 4ef07f3cb30e4ab5aa1bb42bccbd3cdc2649f64d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 19 Oct 2016 16:31:21 -0500 Subject: [PATCH 024/591] Where clause field varies based on stream.category? --- lib/event_source/postgres/get/select_statement.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index f31e1b4..8ea9f1a 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -62,7 +62,7 @@ def sql end def where_clause_field - if stream.type == :stream + unless stream.category? 'stream_name' else 'category(stream_name)' From cde9cb78f43735096d339c640eb2e7254dca2f3e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 19 Oct 2016 16:42:20 -0500 Subject: [PATCH 025/591] Library uses stream instance throughout rather than stream and category primitives --- lib/event_source/postgres/read.rb | 4 ++-- test/automated/iterator/next.rb | 4 ++-- test/automated/iterator/no_further_event_data.rb | 4 ++-- test/automated/put/category_as_stream_name.rb | 4 +++- test/automated/put/put.rb | 6 +++--- test/automated/read/read.rb | 4 ++-- test/automated/read/retry_when_no_further_event_data.rb | 2 +- test/automated/read/synchronous_result.rb | 2 +- 8 files changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index b250f1a..2f81879 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -41,7 +41,7 @@ def call(&action) end def enumerate_event_data(&action) - logger.trace "Reading event data (Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.trace "Reading event data (Stream Name: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" event_data = nil @@ -53,7 +53,7 @@ def enumerate_event_data(&action) action.(event_data) end - logger.debug "Finished reading event data (Stream Name: #{stream_name.inspect}, Category: #{category.inspect}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.debug "Finished reading event data (Stream Name: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" end end end diff --git a/test/automated/iterator/next.rb b/test/automated/iterator/next.rb index b9c8d08..ee75a4a 100644 --- a/test/automated/iterator/next.rb +++ b/test/automated/iterator/next.rb @@ -2,9 +2,9 @@ context "Iterator" do context "Next" do - stream_name = Controls::Put.(instances: 2) + stream = Controls::Put.(instances: 2) - iterator = Iterator.build(stream_name: stream_name, batch_size: 1) + iterator = Iterator.build(stream, batch_size: 1) batch = [] diff --git a/test/automated/iterator/no_further_event_data.rb b/test/automated/iterator/no_further_event_data.rb index 53b6661..60d3f1c 100644 --- a/test/automated/iterator/no_further_event_data.rb +++ b/test/automated/iterator/no_further_event_data.rb @@ -2,9 +2,9 @@ context "Iterator" do context "No further event data" do - stream_name = Controls::Put.(instances: 2) + stream = Controls::Put.(instances: 2) - iterator = Iterator.build(stream_name: stream_name) + iterator = Iterator.build(stream) 2.times { iterator.next } diff --git a/test/automated/put/category_as_stream_name.rb b/test/automated/put/category_as_stream_name.rb index a76c027..ce50d10 100644 --- a/test/automated/put/category_as_stream_name.rb +++ b/test/automated/put/category_as_stream_name.rb @@ -7,7 +7,9 @@ Put.(category, write_event) - read_event = Get.(stream_name: category).first + stream = Stream.new(category) + + read_event = Get.(stream).first test "Got the event that was written" do assert(read_event.stream_name == category) diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index cd4ce97..01bbb71 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -1,12 +1,12 @@ require_relative '../automated_init' context "Put and Get" do - stream_name = Controls::StreamName.example + stream = Controls::Stream.example write_event = Controls::EventData::Write.example - written_stream_position = Put.(stream_name, write_event) + written_stream_position = Put.(stream.name, write_event) - read_event = Get.(stream_name: stream_name, stream_position: written_stream_position)[0] + read_event = Get.(stream, stream_position: written_stream_position)[0] test "Got the event that was written" do assert(read_event.stream_position == written_stream_position) diff --git a/test/automated/read/read.rb b/test/automated/read/read.rb index bd28fe1..ee0c8b6 100644 --- a/test/automated/read/read.rb +++ b/test/automated/read/read.rb @@ -1,11 +1,11 @@ require_relative '../automated_init' context "Read" do - stream_name = Controls::Put.(instances: 2) + stream = Controls::Put.(instances: 2) batch = [] - Read.(stream_name: stream_name, batch_size: 1) do |event_data| + Read.(stream.name, batch_size: 1) do |event_data| batch << event_data end diff --git a/test/automated/read/retry_when_no_further_event_data.rb b/test/automated/read/retry_when_no_further_event_data.rb index 5655100..87a5a2a 100644 --- a/test/automated/read/retry_when_no_further_event_data.rb +++ b/test/automated/read/retry_when_no_further_event_data.rb @@ -4,7 +4,7 @@ cycle = Cycle.build(delay_milliseconds: 10, timeout_milliseconds: 100) sink = Cycle.register_telemetry_sink(cycle) - Read.(stream_name: 'some_stream', batch_size: 1, cycle: cycle) { |event_data| } + Read.('some_stream', batch_size: 1, cycle: cycle) { |event_data| } test "Timed out" do assert(sink.recorded_timed_out?) diff --git a/test/automated/read/synchronous_result.rb b/test/automated/read/synchronous_result.rb index 39727be..8687ea5 100644 --- a/test/automated/read/synchronous_result.rb +++ b/test/automated/read/synchronous_result.rb @@ -2,7 +2,7 @@ context "Read" do context "Synchronous Result" do - res = Read.(stream_name: 'some_stream_name') { } + res = Read.('some_stream_name') { } test "Returns a result that fails if actuated" do assert(res == AsyncInvocation::Incorrect) From 75794c820d9ee56228209ef1430962cc1e172ddb Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 19 Oct 2016 16:47:07 -0500 Subject: [PATCH 026/591] Category disposition is logged in data access --- lib/event_source/postgres/get.rb | 8 ++++---- lib/event_source/postgres/get/select_statement.rb | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 8036653..760211d 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -29,25 +29,25 @@ def configure(session: nil) end def call(stream_position: nil) - logger.trace "Getting event data (Stream Position: #{stream_position.inspect}, Stream Name: #{stream.name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.trace "Getting event data (Stream Position: #{stream_position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" records = get_records(stream, stream_position) events = convert(records) - logger.debug "Finished getting event data (Count: #{events.length}, Stream Position: #{stream_position.inspect}, Stream: #{stream.name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.debug "Finished getting event data (Count: #{events.length}, Stream Position: #{stream_position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" events end def get_records(stream, stream_position) - logger.trace "Getting records (Stream: #{stream.name}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.trace "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" select_statement = SelectStatement.build(stream, offset: stream_position, batch_size: batch_size, precedence: precedence) records = session.connection.exec(select_statement.sql) - logger.debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" records end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 8ea9f1a..a8dcf3a 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -31,7 +31,7 @@ def self.build(stream, offset: nil, batch_size: nil, precedence: nil) end def sql - logger.trace "Composing select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})", tag: :sql + logger.trace "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})", tag: :sql statement = <<-SQL SELECT @@ -55,7 +55,7 @@ def sql ; SQL - logger.debug "Composed select statement (Stream: #{stream_name}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})", tag: :sql + logger.debug "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})", tag: :sql logger.debug "Statement: #{statement}", tags: [:data, :sql] statement From 910f1517b26e46cada8941c085499256480d8e0e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 19 Oct 2016 17:07:42 -0500 Subject: [PATCH 027/591] Expected version error is in the EventSource namespace --- lib/event_source/postgres/iterator.rb | 6 +++--- lib/event_source/postgres/put.rb | 11 ++++++----- lib/event_source/postgres/write.rb | 1 - test/automated/put/expected_version_error.rb | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index 7992004..fd6a2f2 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -43,7 +43,7 @@ def next event_data = batch[batch_position] logger.debug "Finished getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" - logger.debug "Event Data: #{event_data.inspect}", tag: :data + logger.debug "Event Data: #{event_data.inspect}", tags: [:data, :event_data] advance_positions @@ -78,8 +78,8 @@ def reset(batch) self.batch_position = 0 logger.debug "Reset batch" - logger.debug "Batch: #{batch.inspect}", tag: :data - logger.debug "Batch Position: #{batch_position.inspect}", tag: :data + logger.debug "Batch: #{batch.inspect}", tags: [:data, :batch] + logger.debug "Batch Position: #{batch_position.inspect}", tags: [:data, :batch] end def get_batch diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index cf6a0c8..bd906eb 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -28,6 +28,7 @@ def self.call(stream_name, write_event, expected_version: nil, session: nil) def call(write_event, expected_version: nil) logger.trace "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" + logger.trace write_event.inspect, tags: [:data, :event_data] type, data, metadata = destructure_event(write_event) expected_version = canonize_expected_version(expected_version) @@ -44,8 +45,8 @@ def destructure_event(write_event) data = write_event.data metadata = write_event.metadata - logger.debug "Data: #{data.inspect}", tag: :data - logger.debug "Metadata: #{metadata.inspect}", tag: :data + logger.debug "Data: #{data.inspect}", tags: [:data, :event_data] + logger.debug "Metadata: #{metadata.inspect}", tags: [:data, :event_data] return type, data, metadata end @@ -95,7 +96,7 @@ def statement def serialized_data(data) serializable_data = EventData::Hash[data] serialized_data = Serialize::Write.(serializable_data, :json) - logger.debug "Serialized Data: #{serialized_data.inspect}", tag: :data + logger.debug "Serialized Data: #{serialized_data.inspect}", tags: [:data, :serialize] serialized_data end @@ -105,7 +106,7 @@ def serialized_metadata(metadata) unless metadata.nil? serialized_metadata = Serialize::Write.(serializable_metadata, :json) end - logger.debug "Serialized Metadata: #{serialized_metadata.inspect}", tag: :data + logger.debug "Serialized Metadata: #{serialized_metadata.inspect}", tags: [:data, :serialize] serialized_metadata end @@ -121,7 +122,7 @@ def raise_error(pg_error) error_message = pg_error.message if error_message.include? 'Wrong expected version' error_message.gsub!('ERROR:', '').strip! - raise Write::ExpectedVersionError, error_message + raise ExpectedVersionError, error_message end raise pg_error end diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index 3a577e4..d14b52e 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -1,7 +1,6 @@ module EventSource module Postgres class Write - class ExpectedVersionError < RuntimeError; end end end end diff --git a/test/automated/put/expected_version_error.rb b/test/automated/put/expected_version_error.rb index 7d97ca8..0e214a3 100644 --- a/test/automated/put/expected_version_error.rb +++ b/test/automated/put/expected_version_error.rb @@ -11,7 +11,7 @@ test "Is an error" do assert proc { Put.(stream_name, write_event, expected_version: incorrect_stream_version ) } do - raises_error? Write::ExpectedVersionError + raises_error? EventSource::ExpectedVersionError end end end From ddd3e64531823944e54244b3d94f7af8ee8f0397 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 19 Oct 2016 18:14:52 -0500 Subject: [PATCH 028/591] Cycle primitive args are only presented to the reader constructor. Others take a cycle instance. --- lib/event_source/postgres/iterator.rb | 8 ++++---- lib/event_source/postgres/read.rb | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index fd6a2f2..380343b 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -22,16 +22,16 @@ def stream_offset initializer :stream, a(:stream_position, 0), :batch_size, :precedence - def self.build(stream, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + def self.build(stream, stream_position: nil, batch_size: nil, precedence: nil, cycle: nil, session: nil) new(stream, stream_position, batch_size, precedence).tap do |instance| Get.configure instance, stream, batch_size: batch_size, precedence: precedence, session: session - Cycle.configure instance, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle + Cycle.configure instance, cycle: cycle end end - def self.configure(receiver, stream, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + def self.configure(receiver, stream, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, cycle: nil, session: nil) attr_name ||= :iterator - instance = build(stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) + instance = build(stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, cycle: cycle, session: session) receiver.public_send "#{attr_name}=", instance end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 2f81879..0076d9e 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -12,8 +12,13 @@ class Error < RuntimeError; end def self.build(stream_name, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) stream = Stream.new(stream_name) + + if cycle.nil? + cycle = Cycle.build(delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds) + end + new(stream, stream_position, batch_size, precedence).tap do |instance| - Iterator.configure instance, stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session + Iterator.configure instance, stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, cycle: cycle, session: session end end From 74c17c73cdbedb3af1c6a1080c711b19a5e96edb Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 11:42:23 -0500 Subject: [PATCH 029/591] Test purpose is clarified --- test/automated/put/category_as_stream_name.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/automated/put/category_as_stream_name.rb b/test/automated/put/category_as_stream_name.rb index ce50d10..e3c703f 100644 --- a/test/automated/put/category_as_stream_name.rb +++ b/test/automated/put/category_as_stream_name.rb @@ -11,7 +11,7 @@ read_event = Get.(stream).first - test "Got the event that was written" do + test "Writes the category name as the stream name" do assert(read_event.stream_name == category) end end From 815e38d777271b08c7cbc72bf824337c613dd992 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 16:08:16 -0500 Subject: [PATCH 030/591] Other events table used for testing is created and supported with utilities --- bin/create-other-events-table | 3 +++ database/clear-events-table.sh | 16 +++++++++----- database/create-other-events-table.sh | 30 ++++++++++++++++++++++++++ database/install.sh | 1 + database/list-events.sh | 16 +++++++++----- database/tables/other-events-table.sql | 19 ++++++++++++++++ 6 files changed, 75 insertions(+), 10 deletions(-) create mode 100755 bin/create-other-events-table create mode 100755 database/create-other-events-table.sh create mode 100644 database/tables/other-events-table.sql diff --git a/bin/create-other-events-table b/bin/create-other-events-table new file mode 100755 index 0000000..6dbb9d1 --- /dev/null +++ b/bin/create-other-events-table @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +database/create-other-events-table.sh diff --git a/database/clear-events-table.sh b/database/clear-events-table.sh index 0f877eb..75099d3 100755 --- a/database/clear-events-table.sh +++ b/database/clear-events-table.sh @@ -27,10 +27,16 @@ else fi echo "Database name is: $database" -echo +default_table_name=events -function truncate-events-table { - psql $database -c "TRUNCATE events RESTART IDENTITY;" -} +if [ -z ${TABLE_NAME+x} ]; then + echo "(TABLE_NAME is not set)" + table=$default_table_name +else + table=$TABLE_NAME +fi +echo "Table name is: $table" + +echo -truncate-events-table +psql $database -c "TRUNCATE $table RESTART IDENTITY;" diff --git a/database/create-other-events-table.sh b/database/create-other-events-table.sh new file mode 100755 index 0000000..04c9815 --- /dev/null +++ b/database/create-other-events-table.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -e + +clear + +echo +echo "Creating other_events Table" +echo "= = =" +echo + +default_name=eventsource + +if [ -z ${DATABASE_USER+x} ]; then + echo "(DATABASE_USER is not set)" + user=$default_name +else + user=$DATABASE_USER +fi + +if [ -z ${DATABASE_NAME+x} ]; then + echo "(DATABASE_NAME is not set)" + database=$default_name +else + database=$DATABASE_NAME +fi + +echo + +psql $database -f database/tables/other-events-table.sql diff --git a/database/install.sh b/database/install.sh index 60c8f6f..aa66233 100755 --- a/database/install.sh +++ b/database/install.sh @@ -61,6 +61,7 @@ function create-database { function create-tables { psql $database -f database/tables/events-table.sql + psql $database -f database/tables/other-events-table.sql } function create-functions { diff --git a/database/list-events.sh b/database/list-events.sh index 6d43a7a..9715560 100755 --- a/database/list-events.sh +++ b/database/list-events.sh @@ -27,10 +27,16 @@ else fi echo "Database name is: $database" -echo +default_table_name=events -function select-all-events { - psql $database -c "SELECT * FROM events" -} +if [ -z ${TABLE_NAME+x} ]; then + echo "(TABLE_NAME is not set)" + table=$default_table_name +else + table=$TABLE_NAME +fi +echo "Table name is: $table" + +echo -select-all-events +psql $database -c "SELECT * FROM $table" diff --git a/database/tables/other-events-table.sql b/database/tables/other-events-table.sql new file mode 100644 index 0000000..f5c2a5f --- /dev/null +++ b/database/tables/other-events-table.sql @@ -0,0 +1,19 @@ +-- ---------------------------- +-- Table structure for other_events +-- ---------------------------- +DROP TABLE IF EXISTS "public"."other_events"; +CREATE TABLE "public"."other_events" ( + "stream_name" varchar(255) NOT NULL COLLATE "default", + "stream_position" int4 NOT NULL, + "type" varchar(255) NOT NULL COLLATE "default", + "global_position" bigserial NOT NULL , + "data" jsonb NOT NULL, + "metadata" jsonb, + "created_time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL +) +WITH (OIDS=FALSE); + +-- ---------------------------- +-- Primary key structure for table other_events +-- ---------------------------- +ALTER TABLE "public"."other_events" ADD PRIMARY KEY ("global_position") NOT DEFERRABLE INITIALLY IMMEDIATE; From 5ee80d0cc89f3f37ce43ebf7c5635a8a7b4bb46f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 17:31:56 -0500 Subject: [PATCH 031/591] Events table name is variable --- database/functions/write-event.sql | 34 +++++++++++++++++++----------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/database/functions/write-event.sql b/database/functions/write-event.sql index 48af008..6f10c99 100644 --- a/database/functions/write-event.sql +++ b/database/functions/write-event.sql @@ -11,9 +11,14 @@ DECLARE stream_version int; stream_position int; category varchar; + _partition varchar; BEGIN stream_version := stream_version(_stream_name); + if _partition is null then + _partition := 'events'; + end if; + if stream_version is null then stream_version := -1; end if; @@ -26,23 +31,28 @@ BEGIN stream_position := stream_version + 1; - insert into "events" - ( - "stream_name", - "stream_position", - "type", - "data", - "metadata" - ) - values - ( + EXECUTE format('insert into %I (' + '"stream_name", ' + '"stream_position", ' + '"type", ' + '"data", ' + '"metadata"' + ') ' + 'values (' + '$1, ' + '$2, ' + '$3, ' + '$4, ' + '$5 + )', + _partition) + USING _stream_name, stream_position, _type, _data, _metadata - ) - ; + ; return stream_position; END; From b27f9f5ffbc7e9878d52d409bf43291075ca3917 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 17:50:12 -0500 Subject: [PATCH 032/591] Table name is a server function parameter passed from the put --- database/functions/write-event.sql | 3 ++- lib/event_source/postgres/put.rb | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/database/functions/write-event.sql b/database/functions/write-event.sql index 6f10c99..f05e988 100644 --- a/database/functions/write-event.sql +++ b/database/functions/write-event.sql @@ -2,6 +2,7 @@ CREATE OR REPLACE FUNCTION write_event( _stream_name varchar, _type varchar, _data jsonb, + _partition varchar DEFAULT NULL, _metadata jsonb DEFAULT NULL, _expected_version int DEFAULT NULL ) @@ -11,7 +12,7 @@ DECLARE stream_version int; stream_position int; category varchar; - _partition varchar; + -- _partition varchar; BEGIN stream_version := stream_version(_stream_name); diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index bd906eb..4d64fda 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -74,6 +74,7 @@ def execute_query(type, serialized_data, serialized_metadata, expected_version) stream_name, type, serialized_data, + 'events', serialized_metadata, expected_version ] @@ -90,7 +91,7 @@ def execute_query(type, serialized_data, serialized_metadata, expected_version) end def statement - "SELECT write_event($1::varchar, $2::varchar, $3::jsonb, $4::jsonb, $5::int);" + "SELECT write_event($1::varchar, $2::varchar, $3::jsonb, $4::varchar, $5::jsonb, $6::int);" end def serialized_data(data) From e1742b541529995f753f40a66442929edbcb6053 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 17:58:19 -0500 Subject: [PATCH 033/591] Explicit initializer definition is replaced with use of initializer macro --- lib/event_source/postgres/put.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 4d64fda..4a2d481 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -3,13 +3,9 @@ module Postgres class Put include Log::Dependency - attr_reader :stream_name - dependency :session, Session - def initialize(stream_name) - @stream_name = stream_name - end + initializer :stream_name def self.build(stream_name, session: nil) new(stream_name).tap do |instance| From a595c1fc7588fa7dcd84343052b4af429b16bcc6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 18:06:27 -0500 Subject: [PATCH 034/591] Partition is an optional argument to Put --- lib/event_source/postgres/put.rb | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 4a2d481..65d7877 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -5,10 +5,16 @@ class Put dependency :session, Session + def partition + @partition ||= Defaults::Partition.name + end + attr_writer :partition + initializer :stream_name - def self.build(stream_name, session: nil) + def self.build(stream_name, partition: nil, session: nil) new(stream_name).tap do |instance| + instance.partition = partition instance.configure(session: session) end end @@ -17,8 +23,11 @@ def configure(session: nil) Session.configure(self, session: session) end - def self.call(stream_name, write_event, expected_version: nil, session: nil) - instance = build(stream_name, session: session) + ## TODO + # def self.configure + + def self.call(stream_name, write_event, expected_version: nil, partition: nil, session: nil) + instance = build(stream_name, partition: partition, session: session) instance.(write_event, expected_version: expected_version) end @@ -70,7 +79,7 @@ def execute_query(type, serialized_data, serialized_metadata, expected_version) stream_name, type, serialized_data, - 'events', + partition, serialized_metadata, expected_version ] @@ -123,6 +132,14 @@ def raise_error(pg_error) end raise pg_error end + + module Defaults + module Partition + def self.name + 'events' + end + end + end end end end From 96dafeceb48755c01a5679b2efc1e55df60ac87e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 18:14:53 -0500 Subject: [PATCH 035/591] Vestigial code is removed --- database/functions/write-event.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/database/functions/write-event.sql b/database/functions/write-event.sql index f05e988..a3e9bd2 100644 --- a/database/functions/write-event.sql +++ b/database/functions/write-event.sql @@ -12,7 +12,6 @@ DECLARE stream_version int; stream_position int; category varchar; - -- _partition varchar; BEGIN stream_version := stream_version(_stream_name); From 0e9d45a3a7c090d6b4de729854f626114b20b3ae Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 18:30:17 -0500 Subject: [PATCH 036/591] Read from partition --- lib/event_source/postgres/controls.rb | 2 ++ lib/event_source/postgres/controls/partition.rb | 11 +++++++++++ test/automated/put/partition.rb | 14 ++++++++++++++ test/automated/put/put.rb | 2 +- 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 lib/event_source/postgres/controls/partition.rb create mode 100644 test/automated/put/partition.rb diff --git a/lib/event_source/postgres/controls.rb b/lib/event_source/postgres/controls.rb index 9877ecd..0ff66e2 100644 --- a/lib/event_source/postgres/controls.rb +++ b/lib/event_source/postgres/controls.rb @@ -9,4 +9,6 @@ require 'event_source/postgres/controls/event_data/metadata' require 'event_source/postgres/controls/event_data/write' +require 'event_source/postgres/controls/partition' + require 'event_source/postgres/controls/put' diff --git a/lib/event_source/postgres/controls/partition.rb b/lib/event_source/postgres/controls/partition.rb new file mode 100644 index 0000000..55440e9 --- /dev/null +++ b/lib/event_source/postgres/controls/partition.rb @@ -0,0 +1,11 @@ +module EventSource + module Postgres + module Controls + module Partition + def self.example + 'other_events' + end + end + end + end +end diff --git a/test/automated/put/partition.rb b/test/automated/put/partition.rb new file mode 100644 index 0000000..6c5d749 --- /dev/null +++ b/test/automated/put/partition.rb @@ -0,0 +1,14 @@ +require_relative '../automated_init' + +context "Put and Get to another table name" do + stream = Controls::Stream.example + write_event = Controls::EventData::Write.example + + written_stream_position = Put.(stream.name, write_event, partition: Controls::Partition.example) + + read_event = Get.(stream, partition: Controls::Partition.example, stream_position: written_stream_position).first + + test "Got the event that was written" do + assert(read_event.stream_position == written_stream_position) + end +end diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index 01bbb71..0224da5 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -6,7 +6,7 @@ written_stream_position = Put.(stream.name, write_event) - read_event = Get.(stream, stream_position: written_stream_position)[0] + read_event = Get.(stream, stream_position: written_stream_position).first test "Got the event that was written" do assert(read_event.stream_position == written_stream_position) From 6f11193c37056173075d5663ad4b5b224f5e2fbc Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 18:57:29 -0500 Subject: [PATCH 037/591] Get and select statement are constructed with the partition --- lib/event_source/postgres.rb | 2 ++ lib/event_source/postgres/get.rb | 24 +++++++++---------- .../postgres/get/select_statement.rb | 18 ++++++++++---- lib/event_source/postgres/partition.rb | 11 +++++++++ lib/event_source/postgres/put.rb | 8 +++---- test/automated/select_statement/partition.rb | 0 .../select_statement/stream_name_query.rb | 2 +- 7 files changed, 42 insertions(+), 23 deletions(-) create mode 100644 lib/event_source/postgres/partition.rb create mode 100644 test/automated/select_statement/partition.rb diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index 6041244..4a8a46a 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -13,6 +13,8 @@ require 'event_source/postgres/settings' require 'event_source/postgres/session' +require 'event_source/postgres/partition' + require 'event_source/postgres/put' require 'event_source/postgres/write' diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 760211d..c8ca4c0 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -3,24 +3,24 @@ module Postgres class Get include Log::Dependency - initializer :stream, :batch_size, :precedence + initializer :stream, :batch_size, :precedence, :partition dependency :session, Session - def self.build(stream, batch_size: nil, precedence: nil, session: nil) - new(stream, batch_size, precedence).tap do |instance| + def self.build(stream, batch_size: nil, precedence: nil, partition: nil, session: nil) + new(stream, batch_size, precedence, partition).tap do |instance| instance.configure(session: session) end end - def self.configure(receiver, stream, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, session: nil) + def self.configure(receiver, stream, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) attr_name ||= :get - instance = build(stream, batch_size: batch_size, precedence: precedence, session: session) + instance = build(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) receiver.public_send "#{attr_name}=", instance end - def self.call(stream, stream_position: nil, batch_size: nil, precedence: nil, session: nil) - instance = build(stream, batch_size: batch_size, precedence: precedence, session: session) + def self.call(stream, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) + instance = build(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) instance.(stream_position: stream_position) end @@ -29,25 +29,25 @@ def configure(session: nil) end def call(stream_position: nil) - logger.trace "Getting event data (Stream Position: #{stream_position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.trace "Getting event data (Stream Position: #{stream_position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" records = get_records(stream, stream_position) events = convert(records) - logger.debug "Finished getting event data (Count: #{events.length}, Stream Position: #{stream_position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.debug "Finished getting event data (Count: #{events.length}, Stream Position: #{stream_position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" events end def get_records(stream, stream_position) - logger.trace "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.trace "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" - select_statement = SelectStatement.build(stream, offset: stream_position, batch_size: batch_size, precedence: precedence) + select_statement = SelectStatement.build(stream, offset: stream_position, batch_size: batch_size, precedence: precedence, partition: partition) records = session.connection.exec(select_statement.sql) - logger.debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" records end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index a8dcf3a..ca825c1 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -4,7 +4,7 @@ class Get class SelectStatement include Log::Dependency - initializer :stream, w(:offset), w(:batch_size), w(:precedence) + initializer :stream, w(:offset), w(:batch_size), w(:precedence), w(:partition) def offset @offset ||= Defaults.offset @@ -18,6 +18,10 @@ def precedence @precedence ||= Defaults.precedence end + def partition + @partition ||= Defaults.partition + end + def stream_name stream.name end @@ -26,12 +30,12 @@ def stream_type stream.type end - def self.build(stream, offset: nil, batch_size: nil, precedence: nil) - new(stream, offset, batch_size, precedence) + def self.build(stream, offset: nil, batch_size: nil, partition: nil, precedence: nil) + new(stream, offset, batch_size, precedence, partition) end def sql - logger.trace "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})", tag: :sql + logger.trace "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition})", tag: :sql statement = <<-SQL SELECT @@ -55,7 +59,7 @@ def sql ; SQL - logger.debug "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})", tag: :sql + logger.debug "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition.inspect})", tag: :sql logger.debug "Statement: #{statement}", tags: [:data, :sql] statement @@ -81,6 +85,10 @@ def self.batch_size def self.precedence :asc end + + def self.partition + Partition::Defaults.name + end end end end diff --git a/lib/event_source/postgres/partition.rb b/lib/event_source/postgres/partition.rb new file mode 100644 index 0000000..d40af05 --- /dev/null +++ b/lib/event_source/postgres/partition.rb @@ -0,0 +1,11 @@ +module EventSource + module Postgres + module Partition + module Defaults + def self.name + 'events' + end + end + end + end +end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 65d7877..6023647 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -6,7 +6,7 @@ class Put dependency :session, Session def partition - @partition ||= Defaults::Partition.name + @partition ||= Defaults.partition end attr_writer :partition @@ -134,10 +134,8 @@ def raise_error(pg_error) end module Defaults - module Partition - def self.name - 'events' - end + def self.partition + Partition::Defaults.name end end end diff --git a/test/automated/select_statement/partition.rb b/test/automated/select_statement/partition.rb new file mode 100644 index 0000000..e69de29 diff --git a/test/automated/select_statement/stream_name_query.rb b/test/automated/select_statement/stream_name_query.rb index 07db00e..6373530 100644 --- a/test/automated/select_statement/stream_name_query.rb +++ b/test/automated/select_statement/stream_name_query.rb @@ -7,7 +7,7 @@ select_statement = Get::SelectStatement.build stream sql = select_statement.sql - sql.gsub!(/\s+/, " ") + sql.gsub!(/\s+/, ' ') context "Where Clause" do test "Filters on stream name" do From 5889924cd1b169ca1cf17d50879d22be64488c8e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 19:46:15 -0500 Subject: [PATCH 038/591] Select statement switches the from clause based on partition --- .../postgres/get/select_statement.rb | 2 +- test/automated/select_statement/partition.rb | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index ca825c1..6aab1ec 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -47,7 +47,7 @@ def sql metadata::varchar, created_time::timestamp FROM - events + #{partition} WHERE #{where_clause_field} = '#{stream_name}' ORDER BY diff --git a/test/automated/select_statement/partition.rb b/test/automated/select_statement/partition.rb index e69de29..9b87e68 100644 --- a/test/automated/select_statement/partition.rb +++ b/test/automated/select_statement/partition.rb @@ -0,0 +1,18 @@ +require_relative '../automated_init' + +context "Select Statement" do + context "Partition" do + stream = Controls::Stream.example + + select_statement = Get::SelectStatement.build stream, partition: Controls::Partition.example + + sql = select_statement.sql + sql.gsub!(/\s+/, ' ') + + context "From Clause" do + test "Partition table name" do + assert(sql.include? "FROM #{Controls::Partition.example}") + end + end + end +end From be5510b9ee0e234971baa25cdda4f76748e4dd83 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 20:29:08 -0500 Subject: [PATCH 039/591] Read from partition --- lib/event_source/postgres/controls/put.rb | 5 +++-- lib/event_source/postgres/iterator.rb | 13 +++++++------ lib/event_source/postgres/read.rb | 20 ++++++++++---------- test/automated/read/partition.rb | 17 +++++++++++++++++ 4 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 test/automated/read/partition.rb diff --git a/lib/event_source/postgres/controls/put.rb b/lib/event_source/postgres/controls/put.rb index 0ee3fe1..a0e7aad 100644 --- a/lib/event_source/postgres/controls/put.rb +++ b/lib/event_source/postgres/controls/put.rb @@ -2,13 +2,14 @@ module EventSource module Postgres module Controls module Put - def self.call(instances: nil, stream_name: nil, event: nil, category: nil) + def self.call(instances: nil, stream_name: nil, event: nil, category: nil, partition: nil) instances ||= 1 stream_name ||= StreamName.example(category: category) event ||= EventData::Write.example + partition ||= Postgres::Partition::Defaults.name instances.times do - EventSource::Postgres::Put.(stream_name, event) + EventSource::Postgres::Put.(stream_name, event, partition: partition) end EventSource::Stream.new(stream_name) diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index 380343b..2c4cd66 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -20,18 +20,19 @@ def stream_offset dependency :get, Get dependency :cycle, Cycle - initializer :stream, a(:stream_position, 0), :batch_size, :precedence + ## TODO remove a() with default + initializer :stream, a(:stream_position, 0), :batch_size, :precedence, :partition - def self.build(stream, stream_position: nil, batch_size: nil, precedence: nil, cycle: nil, session: nil) - new(stream, stream_position, batch_size, precedence).tap do |instance| - Get.configure instance, stream, batch_size: batch_size, precedence: precedence, session: session + def self.build(stream, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, cycle: nil, session: nil) + new(stream, stream_position, batch_size, precedence, partition).tap do |instance| + Get.configure instance, stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session Cycle.configure instance, cycle: cycle end end - def self.configure(receiver, stream, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, cycle: nil, session: nil) + def self.configure(receiver, stream, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, cycle: nil, session: nil) attr_name ||= :iterator - instance = build(stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, cycle: cycle, session: session) + instance = build(stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, partition: partition, cycle: cycle, session: session) receiver.public_send "#{attr_name}=", instance end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 0076d9e..2c76ce6 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -5,31 +5,31 @@ class Error < RuntimeError; end include Log::Dependency - initializer :stream, :stream_position, :batch_size, :precedence + initializer :stream, :stream_position, :batch_size, :precedence, :partition dependency :session, Session dependency :iterator, Iterator - def self.build(stream_name, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + def self.build(stream_name, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) stream = Stream.new(stream_name) if cycle.nil? cycle = Cycle.build(delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds) end - new(stream, stream_position, batch_size, precedence).tap do |instance| - Iterator.configure instance, stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, cycle: cycle, session: session + new(stream, stream_position, batch_size, precedence, partition).tap do |instance| + Iterator.configure instance, stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, partition: partition, cycle: cycle, session: session end end - def self.call(stream_name, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil, &action) - instance = build(stream_name, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) + def self.call(stream_name, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil, &action) + instance = build(stream_name, stream_position: stream_position, batch_size: batch_size, precedence: precedence, partition: partition, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) instance.(&action) end - def self.configure(receiver, stream_name, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + def self.configure(receiver, stream_name, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, partition: partition, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) attr_name ||= :reader - instance = build(stream_name, stream_position: stream_position, batch_size: batch_size, precedence: precedence, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) + instance = build(stream_name, stream_position: stream_position, batch_size: batch_size, precedence: precedence, partition: partition, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) receiver.public_send "#{attr_name}=", instance end @@ -46,7 +46,7 @@ def call(&action) end def enumerate_event_data(&action) - logger.trace "Reading event data (Stream Name: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.trace "Reading event data (Stream Name: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" event_data = nil @@ -58,7 +58,7 @@ def enumerate_event_data(&action) action.(event_data) end - logger.debug "Finished reading event data (Stream Name: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" + logger.debug "Finished reading event data (Stream Name: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" end end end diff --git a/test/automated/read/partition.rb b/test/automated/read/partition.rb new file mode 100644 index 0000000..8f32341 --- /dev/null +++ b/test/automated/read/partition.rb @@ -0,0 +1,17 @@ +require_relative '../automated_init' + +context "Read" do + partition = Controls::Partition.example + + stream = Controls::Put.(instances: 2, partition: partition) + + batch = [] + + Read.(stream.name, partition: partition, batch_size: 1) do |event_data| + batch << event_data + end + + test "Reads batches of events" do + assert(batch.length == 2) + end +end From 14210ba24efd3e2127d0072878924122ef40dddc Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 20:34:48 -0500 Subject: [PATCH 040/591] Superfluous accessor is removed --- lib/event_source/postgres/iterator.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index 2c4cd66..4e2ab2f 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -9,6 +9,10 @@ class Error < RuntimeError; end attr_writer :batch_position attr_writer :stream_offset + def stream_position + @stream_position ||= 0 + end + def batch_position @batch_position ||= 0 end @@ -20,8 +24,7 @@ def stream_offset dependency :get, Get dependency :cycle, Cycle - ## TODO remove a() with default - initializer :stream, a(:stream_position, 0), :batch_size, :precedence, :partition + initializer :stream, :stream_position, :batch_size, :precedence, :partition def self.build(stream, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, cycle: nil, session: nil) new(stream, stream_position, batch_size, precedence, partition).tap do |instance| From 7b39b1a77d4fbd799c40c9d2ea3d87109afb0000 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 20:44:33 -0500 Subject: [PATCH 041/591] Superfluous attributes are removed --- lib/event_source/postgres/iterator.rb | 14 +++++--------- lib/event_source/postgres/read.rb | 8 ++++---- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index 4e2ab2f..00664a5 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -5,14 +5,13 @@ class Error < RuntimeError; end include Log::Dependency + dependency :get, Get + dependency :cycle, Cycle + attr_accessor :batch attr_writer :batch_position attr_writer :stream_offset - def stream_position - @stream_position ||= 0 - end - def batch_position @batch_position ||= 0 end @@ -21,13 +20,10 @@ def stream_offset @stream_offset ||= (stream_position || 0) end - dependency :get, Get - dependency :cycle, Cycle - - initializer :stream, :stream_position, :batch_size, :precedence, :partition + initializer :stream_position def self.build(stream, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, cycle: nil, session: nil) - new(stream, stream_position, batch_size, precedence, partition).tap do |instance| + new(stream_position).tap do |instance| Get.configure instance, stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session Cycle.configure instance, cycle: cycle end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 2c76ce6..43ef63c 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -5,7 +5,7 @@ class Error < RuntimeError; end include Log::Dependency - initializer :stream, :stream_position, :batch_size, :precedence, :partition + initializer :stream dependency :session, Session dependency :iterator, Iterator @@ -17,7 +17,7 @@ def self.build(stream_name, stream_position: nil, batch_size: nil, precedence: n cycle = Cycle.build(delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds) end - new(stream, stream_position, batch_size, precedence, partition).tap do |instance| + new(stream).tap do |instance| Iterator.configure instance, stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, partition: partition, cycle: cycle, session: session end end @@ -46,7 +46,7 @@ def call(&action) end def enumerate_event_data(&action) - logger.trace "Reading event data (Stream Name: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.trace "Reading (Stream Name: #{stream.name}, Category: #{stream.category?})" event_data = nil @@ -58,7 +58,7 @@ def enumerate_event_data(&action) action.(event_data) end - logger.debug "Finished reading event data (Stream Name: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.debug "Finished reading (Stream Name: #{stream.name}, Category: #{stream.category?})" end end end From fce33cdfa5b8d8acfe84c792769a01bacb5f1a88 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 21:05:51 -0500 Subject: [PATCH 042/591] Test coverage of starting position --- test/automated/read/starting_position.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 test/automated/read/starting_position.rb diff --git a/test/automated/read/starting_position.rb b/test/automated/read/starting_position.rb new file mode 100644 index 0000000..1964cc7 --- /dev/null +++ b/test/automated/read/starting_position.rb @@ -0,0 +1,17 @@ +require_relative '../automated_init' + +context "Read" do + context "Starting Position" do + stream = Controls::Put.(instances: 2) + + batch = [] + + Read.(stream.name, stream_position: 1, batch_size: 1) do |event_data| + batch << event_data + end + + test "Reads from the starting position" do + assert(batch.length == 1) + end + end +end From e304ee1b7c2434739269098a685989d8bc8af9b4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 21:33:23 -0500 Subject: [PATCH 043/591] Position, rather than stream position --- database/functions/stream-version.sql | 2 +- database/functions/write-event.sql | 10 ++++----- database/indexes/events-indexes.sql | 2 +- database/tables/events-table.sql | 2 +- database/tables/other-events-table.sql | 2 +- lib/event_source/postgres/get.rb | 22 +++++++++---------- .../postgres/get/select_statement.rb | 2 +- lib/event_source/postgres/iterator.rb | 14 ++++++------ lib/event_source/postgres/put.rb | 14 ++++++------ lib/event_source/postgres/read.rb | 12 +++++----- test/automated/get/get.rb | 6 ++--- test/automated/get/precedence.rb | 4 ++-- test/automated/put/expected_version_error.rb | 4 ++-- test/automated/put/no_stream/no_stream.rb | 4 ++-- test/automated/put/partition.rb | 6 ++--- test/automated/put/put.rb | 6 ++--- test/automated/put/returns_stream_position.rb | 4 ++-- .../put/stream_position_increases.rb | 6 ++--- test/automated/read/starting_position.rb | 2 +- 19 files changed, 62 insertions(+), 62 deletions(-) diff --git a/database/functions/stream-version.sql b/database/functions/stream-version.sql index 1b4c47b..6b88722 100644 --- a/database/functions/stream-version.sql +++ b/database/functions/stream-version.sql @@ -6,7 +6,7 @@ AS $$ DECLARE stream_version int; BEGIN - select max(stream_position) into stream_version from events where stream_name = _stream_name; + select max(position) into stream_version from events where stream_name = _stream_name; return stream_version; END; diff --git a/database/functions/write-event.sql b/database/functions/write-event.sql index a3e9bd2..305e74b 100644 --- a/database/functions/write-event.sql +++ b/database/functions/write-event.sql @@ -10,7 +10,7 @@ RETURNS int AS $$ DECLARE stream_version int; - stream_position int; + position int; category varchar; BEGIN stream_version := stream_version(_stream_name); @@ -29,11 +29,11 @@ BEGIN end if; end if; - stream_position := stream_version + 1; + position := stream_version + 1; EXECUTE format('insert into %I (' '"stream_name", ' - '"stream_position", ' + '"position", ' '"type", ' '"data", ' '"metadata"' @@ -48,12 +48,12 @@ BEGIN _partition) USING _stream_name, - stream_position, + position, _type, _data, _metadata ; - return stream_position; + return position; END; $$ LANGUAGE plpgsql; diff --git a/database/indexes/events-indexes.sql b/database/indexes/events-indexes.sql index 5ffbda6..f531b9b 100644 --- a/database/indexes/events-indexes.sql +++ b/database/indexes/events-indexes.sql @@ -4,4 +4,4 @@ CREATE INDEX CONCURRENTLY "events_category_global_position_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "global_position" "pg_catalog"."int8_ops" ASC NULLS LAST); CREATE INDEX CONCURRENTLY "events_category_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); CREATE INDEX CONCURRENTLY "events_stream_name_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); -CREATE UNIQUE INDEX CONCURRENTLY "events_stream_name_stream_position_uniq_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "stream_position" "pg_catalog"."int4_ops" ASC NULLS LAST); +CREATE UNIQUE INDEX CONCURRENTLY "events_stream_name_position_uniq_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "position" "pg_catalog"."int4_ops" ASC NULLS LAST); diff --git a/database/tables/events-table.sql b/database/tables/events-table.sql index f4d838b..cbe3327 100644 --- a/database/tables/events-table.sql +++ b/database/tables/events-table.sql @@ -4,7 +4,7 @@ DROP TABLE IF EXISTS "public"."events"; CREATE TABLE "public"."events" ( "stream_name" varchar(255) NOT NULL COLLATE "default", - "stream_position" int4 NOT NULL, + "position" int4 NOT NULL, "type" varchar(255) NOT NULL COLLATE "default", "global_position" bigserial NOT NULL , "data" jsonb NOT NULL, diff --git a/database/tables/other-events-table.sql b/database/tables/other-events-table.sql index f5c2a5f..00a48cf 100644 --- a/database/tables/other-events-table.sql +++ b/database/tables/other-events-table.sql @@ -4,7 +4,7 @@ DROP TABLE IF EXISTS "public"."other_events"; CREATE TABLE "public"."other_events" ( "stream_name" varchar(255) NOT NULL COLLATE "default", - "stream_position" int4 NOT NULL, + "position" int4 NOT NULL, "type" varchar(255) NOT NULL COLLATE "default", "global_position" bigserial NOT NULL , "data" jsonb NOT NULL, diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index c8ca4c0..711751b 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -13,41 +13,41 @@ def self.build(stream, batch_size: nil, precedence: nil, partition: nil, session end end - def self.configure(receiver, stream, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) + def self.configure(receiver, stream, attr_name: nil, position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) attr_name ||= :get instance = build(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) receiver.public_send "#{attr_name}=", instance end - def self.call(stream, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) + def self.call(stream, position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) instance = build(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) - instance.(stream_position: stream_position) + instance.(position: position) end def configure(session: nil) Session.configure self, session: session end - def call(stream_position: nil) - logger.trace "Getting event data (Stream Position: #{stream_position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + def call(position: nil) + logger.trace "Getting event data (Stream Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" - records = get_records(stream, stream_position) + records = get_records(stream, position) events = convert(records) - logger.debug "Finished getting event data (Count: #{events.length}, Stream Position: #{stream_position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.debug "Finished getting event data (Count: #{events.length}, Stream Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" events end - def get_records(stream, stream_position) - logger.trace "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + def get_records(stream, position) + logger.trace "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" - select_statement = SelectStatement.build(stream, offset: stream_position, batch_size: batch_size, precedence: precedence, partition: partition) + select_statement = SelectStatement.build(stream, offset: position, batch_size: batch_size, precedence: precedence, partition: partition) records = session.connection.exec(select_statement.sql) - logger.debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{stream_position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" records end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 6aab1ec..7cacdf5 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -40,7 +40,7 @@ def sql statement = <<-SQL SELECT stream_name::varchar, - stream_position::int, + position::int, type::varchar, global_position::bigint, data::varchar, diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index 00664a5..7ed5864 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -17,21 +17,21 @@ def batch_position end def stream_offset - @stream_offset ||= (stream_position || 0) + @stream_offset ||= (position || 0) end - initializer :stream_position + initializer :position - def self.build(stream, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, cycle: nil, session: nil) - new(stream_position).tap do |instance| + def self.build(stream, position: nil, batch_size: nil, precedence: nil, partition: nil, cycle: nil, session: nil) + new(position).tap do |instance| Get.configure instance, stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session Cycle.configure instance, cycle: cycle end end - def self.configure(receiver, stream, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, cycle: nil, session: nil) + def self.configure(receiver, stream, attr_name: nil, position: nil, batch_size: nil, precedence: nil, partition: nil, cycle: nil, session: nil) attr_name ||= :iterator - instance = build(stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, partition: partition, cycle: cycle, session: session) + instance = build(stream, position: position, batch_size: batch_size, precedence: precedence, partition: partition, cycle: cycle, session: session) receiver.public_send "#{attr_name}=", instance end @@ -87,7 +87,7 @@ def get_batch batch = nil cycle.() do - batch = get.(stream_position: stream_offset) + batch = get.(position: stream_offset) end logger.debug "Finished getting batch (Count: #{batch.length})" diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 6023647..6c94603 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -38,11 +38,11 @@ def call(write_event, expected_version: nil) type, data, metadata = destructure_event(write_event) expected_version = canonize_expected_version(expected_version) - stream_position = insert_event(type, data, metadata, expected_version) + position = insert_event(type, data, metadata, expected_version) logger.debug "Put event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" - stream_position + position end def destructure_event(write_event) @@ -69,7 +69,7 @@ def insert_event(type, data, metadata, expected_version) serialized_data = serialized_data(data) serialized_metadata = serialized_metadata(metadata) records = execute_query(type, serialized_data, serialized_metadata, expected_version) - stream_position(records) + position(records) end def execute_query(type, serialized_data, serialized_metadata, expected_version) @@ -116,12 +116,12 @@ def serialized_metadata(metadata) serialized_metadata end - def stream_position(records) - stream_position = nil + def position(records) + position = nil unless records[0].nil? - stream_position = records[0].values[0] + position = records[0].values[0] end - stream_position + position end def raise_error(pg_error) diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 43ef63c..bd0e1a9 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -10,7 +10,7 @@ class Error < RuntimeError; end dependency :session, Session dependency :iterator, Iterator - def self.build(stream_name, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + def self.build(stream_name, position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) stream = Stream.new(stream_name) if cycle.nil? @@ -18,18 +18,18 @@ def self.build(stream_name, stream_position: nil, batch_size: nil, precedence: n end new(stream).tap do |instance| - Iterator.configure instance, stream, stream_position: stream_position, batch_size: batch_size, precedence: precedence, partition: partition, cycle: cycle, session: session + Iterator.configure instance, stream, position: position, batch_size: batch_size, precedence: precedence, partition: partition, cycle: cycle, session: session end end - def self.call(stream_name, stream_position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil, &action) - instance = build(stream_name, stream_position: stream_position, batch_size: batch_size, precedence: precedence, partition: partition, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) + def self.call(stream_name, position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil, &action) + instance = build(stream_name, position: position, batch_size: batch_size, precedence: precedence, partition: partition, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) instance.(&action) end - def self.configure(receiver, stream_name, attr_name: nil, stream_position: nil, batch_size: nil, precedence: nil, partition: partition, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + def self.configure(receiver, stream_name, attr_name: nil, position: nil, batch_size: nil, precedence: nil, partition: partition, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) attr_name ||= :reader - instance = build(stream_name, stream_position: stream_position, batch_size: batch_size, precedence: precedence, partition: partition, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) + instance = build(stream_name, position: position, batch_size: batch_size, precedence: precedence, partition: partition, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) receiver.public_send "#{attr_name}=", instance end diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index 01bbb71..bd70871 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -4,11 +4,11 @@ stream = Controls::Stream.example write_event = Controls::EventData::Write.example - written_stream_position = Put.(stream.name, write_event) + written_position = Put.(stream.name, write_event) - read_event = Get.(stream, stream_position: written_stream_position)[0] + read_event = Get.(stream, position: written_position)[0] test "Got the event that was written" do - assert(read_event.stream_position == written_stream_position) + assert(read_event.position == written_position) end end diff --git a/test/automated/get/precedence.rb b/test/automated/get/precedence.rb index db49738..2ba047e 100644 --- a/test/automated/get/precedence.rb +++ b/test/automated/get/precedence.rb @@ -7,7 +7,7 @@ context "Ascending" do events = Get.(stream, precedence: :asc) - first_event_postition = events.first.stream_position + first_event_postition = events.first.position test "First event written is first in the list of results" do assert(first_event_postition == 0) @@ -17,7 +17,7 @@ context "Descending" do events = Get.(stream, precedence: :desc) - first_event_postition = events.first.stream_position + first_event_postition = events.first.position test "Last event written is first in the list of results" do assert(first_event_postition == 2) diff --git a/test/automated/put/expected_version_error.rb b/test/automated/put/expected_version_error.rb index 0e214a3..6bda45e 100644 --- a/test/automated/put/expected_version_error.rb +++ b/test/automated/put/expected_version_error.rb @@ -5,9 +5,9 @@ stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example - stream_position = Put.(stream_name, write_event) + position = Put.(stream_name, write_event) - incorrect_stream_version = stream_position + 1 + incorrect_stream_version = position + 1 test "Is an error" do assert proc { Put.(stream_name, write_event, expected_version: incorrect_stream_version ) } do diff --git a/test/automated/put/no_stream/no_stream.rb b/test/automated/put/no_stream/no_stream.rb index 4652d5b..88b0770 100644 --- a/test/automated/put/no_stream/no_stream.rb +++ b/test/automated/put/no_stream/no_stream.rb @@ -6,10 +6,10 @@ stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example - stream_position = Put.(stream_name, write_event) + position = Put.(stream_name, write_event) test "Ensures that the event written is the first event in the stream" do - assert(stream_position == 0) + assert(position == 0) end end end diff --git a/test/automated/put/partition.rb b/test/automated/put/partition.rb index 6c5d749..3b2a984 100644 --- a/test/automated/put/partition.rb +++ b/test/automated/put/partition.rb @@ -4,11 +4,11 @@ stream = Controls::Stream.example write_event = Controls::EventData::Write.example - written_stream_position = Put.(stream.name, write_event, partition: Controls::Partition.example) + written_position = Put.(stream.name, write_event, partition: Controls::Partition.example) - read_event = Get.(stream, partition: Controls::Partition.example, stream_position: written_stream_position).first + read_event = Get.(stream, partition: Controls::Partition.example, position: written_position).first test "Got the event that was written" do - assert(read_event.stream_position == written_stream_position) + assert(read_event.position == written_position) end end diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index 0224da5..50d423b 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -4,11 +4,11 @@ stream = Controls::Stream.example write_event = Controls::EventData::Write.example - written_stream_position = Put.(stream.name, write_event) + written_position = Put.(stream.name, write_event) - read_event = Get.(stream, stream_position: written_stream_position).first + read_event = Get.(stream, position: written_position).first test "Got the event that was written" do - assert(read_event.stream_position == written_stream_position) + assert(read_event.position == written_position) end end diff --git a/test/automated/put/returns_stream_position.rb b/test/automated/put/returns_stream_position.rb index 0d5f5f7..fa93f0a 100644 --- a/test/automated/put/returns_stream_position.rb +++ b/test/automated/put/returns_stream_position.rb @@ -5,9 +5,9 @@ write_event = Controls::EventData::Write.example - stream_position = Put.(stream_name, write_event) + position = Put.(stream_name, write_event) test "Result is stream version" do - refute(stream_position.nil?) + refute(position.nil?) end end diff --git a/test/automated/put/stream_position_increases.rb b/test/automated/put/stream_position_increases.rb index 2fd9f35..aab585c 100644 --- a/test/automated/put/stream_position_increases.rb +++ b/test/automated/put/stream_position_increases.rb @@ -4,10 +4,10 @@ stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example - stream_position_1 = Put.(stream_name, write_event) - stream_position_2 = Put.(stream_name, write_event) + position_1 = Put.(stream_name, write_event) + position_2 = Put.(stream_name, write_event) test "First version is one less than the second version" do - assert(stream_position_1 + 1 == stream_position_2) + assert(position_1 + 1 == position_2) end end diff --git a/test/automated/read/starting_position.rb b/test/automated/read/starting_position.rb index 1964cc7..a0fd88b 100644 --- a/test/automated/read/starting_position.rb +++ b/test/automated/read/starting_position.rb @@ -6,7 +6,7 @@ batch = [] - Read.(stream.name, stream_position: 1, batch_size: 1) do |event_data| + Read.(stream.name, position: 1, batch_size: 1) do |event_data| batch << event_data end From c8278d4984d0856c7f34cc37c2da1dde6a3b0fcc Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 23 Oct 2016 21:43:59 -0500 Subject: [PATCH 044/591] Position, rather than stream position in log messages --- lib/event_source/postgres/get.rb | 8 ++++---- lib/event_source/postgres/get/select_statement.rb | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 711751b..836eaed 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -29,25 +29,25 @@ def configure(session: nil) end def call(position: nil) - logger.trace "Getting event data (Stream Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.trace "Getting event data (Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" records = get_records(stream, position) events = convert(records) - logger.debug "Finished getting event data (Count: #{events.length}, Stream Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.debug "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" events end def get_records(stream, position) - logger.trace "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.trace "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" select_statement = SelectStatement.build(stream, offset: position, batch_size: batch_size, precedence: precedence, partition: partition) records = session.connection.exec(select_statement.sql) - logger.debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Stream Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" records end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 7cacdf5..618bddf 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -35,7 +35,7 @@ def self.build(stream, offset: nil, batch_size: nil, partition: nil, precedence: end def sql - logger.trace "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition})", tag: :sql + logger.trace "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition})", tag: :sql statement = <<-SQL SELECT @@ -59,7 +59,7 @@ def sql ; SQL - logger.debug "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Stream Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition.inspect})", tag: :sql + logger.debug "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition.inspect})", tag: :sql logger.debug "Statement: #{statement}", tags: [:data, :sql] statement From 8c9117c4d83f4cb2588a9a1341ca990f25333520 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 24 Oct 2016 11:27:47 -0500 Subject: [PATCH 045/591] Interactive, rather than script --- test/{script => interactive}/reader_does_not_terminate.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{script => interactive}/reader_does_not_terminate.rb (100%) diff --git a/test/script/reader_does_not_terminate.rb b/test/interactive/reader_does_not_terminate.rb similarity index 100% rename from test/script/reader_does_not_terminate.rb rename to test/interactive/reader_does_not_terminate.rb From bb2dd928f528617c82c722195f813e49e293d1c8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 24 Oct 2016 12:06:32 -0500 Subject: [PATCH 046/591] Suspending work on write while looking into parameter order in Put --- test/automated/write/write.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 test/automated/write/write.rb diff --git a/test/automated/write/write.rb b/test/automated/write/write.rb new file mode 100644 index 0000000..b279529 --- /dev/null +++ b/test/automated/write/write.rb @@ -0,0 +1,14 @@ +require_relative '../automated_init' + +context "Write" do + stream_name = Controls::Stream.example + write_event = Controls::EventData::Write.example + + written_position = Write.(write_event, ) + + read_event = Get.(stream, position: written_position).first + + test "Got the event that was written" do + assert(read_event.position == written_position) + end +end From c47435da65d584aef7440a7a9a57a2a2a66b757b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 24 Oct 2016 12:41:38 -0500 Subject: [PATCH 047/591] Put sql statement is a singleton --- lib/event_source/postgres/put.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 6c94603..f07e3e4 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -85,7 +85,7 @@ def execute_query(type, serialized_data, serialized_metadata, expected_version) ] begin - records = session.connection.exec_params(statement, sql_args) + records = session.connection.exec_params(self.class.statement, sql_args) rescue PG::RaiseException => e raise_error e end @@ -95,8 +95,8 @@ def execute_query(type, serialized_data, serialized_metadata, expected_version) records end - def statement - "SELECT write_event($1::varchar, $2::varchar, $3::jsonb, $4::varchar, $5::jsonb, $6::int);" + def self.statement + @statement ||= "SELECT write_event($1::varchar, $2::varchar, $3::jsonb, $4::varchar, $5::jsonb, $6::int);" end def serialized_data(data) From 7826f64192ca7c7c64daf9d45d7a2a584d16bf7c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 24 Oct 2016 12:48:49 -0500 Subject: [PATCH 048/591] Stream name is no longer held in Put's instance state --- lib/event_source/postgres/put.rb | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index f07e3e4..a68c369 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -10,10 +10,8 @@ def partition end attr_writer :partition - initializer :stream_name - - def self.build(stream_name, partition: nil, session: nil) - new(stream_name).tap do |instance| + def self.build(partition: nil, session: nil) + new.tap do |instance| instance.partition = partition instance.configure(session: session) end @@ -27,18 +25,18 @@ def configure(session: nil) # def self.configure def self.call(stream_name, write_event, expected_version: nil, partition: nil, session: nil) - instance = build(stream_name, partition: partition, session: session) - instance.(write_event, expected_version: expected_version) + instance = build(partition: partition, session: session) + instance.(write_event, stream_name, expected_version: expected_version) end - def call(write_event, expected_version: nil) + def call(write_event, stream_name, expected_version: nil) logger.trace "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" logger.trace write_event.inspect, tags: [:data, :event_data] type, data, metadata = destructure_event(write_event) expected_version = canonize_expected_version(expected_version) - position = insert_event(type, data, metadata, expected_version) + position = insert_event(stream_name, type, data, metadata, expected_version) logger.debug "Put event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" @@ -65,14 +63,14 @@ def canonize_expected_version(expected_version) expected_version end - def insert_event(type, data, metadata, expected_version) + def insert_event(stream_name, type, data, metadata, expected_version) serialized_data = serialized_data(data) serialized_metadata = serialized_metadata(metadata) - records = execute_query(type, serialized_data, serialized_metadata, expected_version) + records = execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version) position(records) end - def execute_query(type, serialized_data, serialized_metadata, expected_version) + def execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version) logger.trace "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" sql_args = [ From c924e41868520f6a6a8faf6a56539f83bcc30e2c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 24 Oct 2016 13:24:07 -0500 Subject: [PATCH 049/591] Event is the first argument to Put --- lib/event_source/postgres/controls/put.rb | 2 +- lib/event_source/postgres/put.rb | 2 +- test/automated/get/get.rb | 2 +- test/automated/put/category_as_stream_name.rb | 2 +- test/automated/put/expected_version_error.rb | 4 ++-- test/automated/put/no_stream/existing_stream.rb | 4 ++-- test/automated/put/no_stream/no_stream.rb | 2 +- test/automated/put/partition.rb | 2 +- test/automated/put/put.rb | 2 +- test/automated/put/returns_stream_position.rb | 2 +- test/automated/put/stream_position_increases.rb | 4 ++-- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/event_source/postgres/controls/put.rb b/lib/event_source/postgres/controls/put.rb index a0e7aad..56bc1a4 100644 --- a/lib/event_source/postgres/controls/put.rb +++ b/lib/event_source/postgres/controls/put.rb @@ -9,7 +9,7 @@ def self.call(instances: nil, stream_name: nil, event: nil, category: nil, parti partition ||= Postgres::Partition::Defaults.name instances.times do - EventSource::Postgres::Put.(stream_name, event, partition: partition) + EventSource::Postgres::Put.(event, stream_name, partition: partition) end EventSource::Stream.new(stream_name) diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index a68c369..a554c20 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -24,7 +24,7 @@ def configure(session: nil) ## TODO # def self.configure - def self.call(stream_name, write_event, expected_version: nil, partition: nil, session: nil) + def self.call(write_event, stream_name, expected_version: nil, partition: nil, session: nil) instance = build(partition: partition, session: session) instance.(write_event, stream_name, expected_version: expected_version) end diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index bd70871..b59386f 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -4,7 +4,7 @@ stream = Controls::Stream.example write_event = Controls::EventData::Write.example - written_position = Put.(stream.name, write_event) + written_position = Put.(write_event, stream.name) read_event = Get.(stream, position: written_position)[0] diff --git a/test/automated/put/category_as_stream_name.rb b/test/automated/put/category_as_stream_name.rb index e3c703f..0588ee7 100644 --- a/test/automated/put/category_as_stream_name.rb +++ b/test/automated/put/category_as_stream_name.rb @@ -5,7 +5,7 @@ category = Controls::Category.example write_event = Controls::EventData::Write.example - Put.(category, write_event) + Put.(write_event, category) stream = Stream.new(category) diff --git a/test/automated/put/expected_version_error.rb b/test/automated/put/expected_version_error.rb index 6bda45e..55e593f 100644 --- a/test/automated/put/expected_version_error.rb +++ b/test/automated/put/expected_version_error.rb @@ -5,12 +5,12 @@ stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example - position = Put.(stream_name, write_event) + position = Put.(write_event, stream_name) incorrect_stream_version = position + 1 test "Is an error" do - assert proc { Put.(stream_name, write_event, expected_version: incorrect_stream_version ) } do + assert proc { Put.(write_event, stream_name, expected_version: incorrect_stream_version ) } do raises_error? EventSource::ExpectedVersionError end end diff --git a/test/automated/put/no_stream/existing_stream.rb b/test/automated/put/no_stream/existing_stream.rb index 56de771..4b46878 100644 --- a/test/automated/put/no_stream/existing_stream.rb +++ b/test/automated/put/no_stream/existing_stream.rb @@ -8,9 +8,9 @@ write_event_1 = Controls::EventData::Write.example(data: {:some_attribute => 'first'}) write_event_2 = Controls::EventData::Write.example(data: {:some_attribute => 'second'}) - Put.(stream_name, write_event_1) + Put.(write_event_1, stream_name) - erroneous = proc { Put.(stream_name, write_event_2, expected_version: NoStream.name) } + erroneous = proc { Put.(write_event_2, stream_name, expected_version: NoStream.name) } test "Is an error" do assert erroneous do diff --git a/test/automated/put/no_stream/no_stream.rb b/test/automated/put/no_stream/no_stream.rb index 88b0770..e4c8b94 100644 --- a/test/automated/put/no_stream/no_stream.rb +++ b/test/automated/put/no_stream/no_stream.rb @@ -6,7 +6,7 @@ stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example - position = Put.(stream_name, write_event) + position = Put.(write_event, stream_name) test "Ensures that the event written is the first event in the stream" do assert(position == 0) diff --git a/test/automated/put/partition.rb b/test/automated/put/partition.rb index 3b2a984..0879223 100644 --- a/test/automated/put/partition.rb +++ b/test/automated/put/partition.rb @@ -4,7 +4,7 @@ stream = Controls::Stream.example write_event = Controls::EventData::Write.example - written_position = Put.(stream.name, write_event, partition: Controls::Partition.example) + written_position = Put.(write_event, stream.name, partition: Controls::Partition.example) read_event = Get.(stream, partition: Controls::Partition.example, position: written_position).first diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index 50d423b..2384f83 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -4,7 +4,7 @@ stream = Controls::Stream.example write_event = Controls::EventData::Write.example - written_position = Put.(stream.name, write_event) + written_position = Put.(write_event, stream.name) read_event = Get.(stream, position: written_position).first diff --git a/test/automated/put/returns_stream_position.rb b/test/automated/put/returns_stream_position.rb index fa93f0a..af7a158 100644 --- a/test/automated/put/returns_stream_position.rb +++ b/test/automated/put/returns_stream_position.rb @@ -5,7 +5,7 @@ write_event = Controls::EventData::Write.example - position = Put.(stream_name, write_event) + position = Put.(write_event, stream_name) test "Result is stream version" do refute(position.nil?) diff --git a/test/automated/put/stream_position_increases.rb b/test/automated/put/stream_position_increases.rb index aab585c..9645fba 100644 --- a/test/automated/put/stream_position_increases.rb +++ b/test/automated/put/stream_position_increases.rb @@ -4,8 +4,8 @@ stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example - position_1 = Put.(stream_name, write_event) - position_2 = Put.(stream_name, write_event) + position_1 = Put.(write_event, stream_name) + position_2 = Put.(write_event, stream_name) test "First version is one less than the second version" do assert(position_1 + 1 == position_2) From b87d168e2e5ffc71191b6eaa2ddf1488f2a2bd17 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 24 Oct 2016 16:54:24 -0500 Subject: [PATCH 050/591] Write event data --- .gitignore | 1 + lib/event_source/postgres/put.rb | 8 +++-- lib/event_source/postgres/write.rb | 50 ++++++++++++++++++++++++++++++ test/automated/write/write.rb | 4 +-- 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 0bec69b..d28b09c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ gems gems_backup *scratch* *notes* +loader.rb diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index a554c20..538ffcd 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -21,8 +21,11 @@ def configure(session: nil) Session.configure(self, session: session) end - ## TODO - # def self.configure + def self.configure(receiver, partition: nil, session: nil, attr_name: nil) + attr_name ||= :put + instance = build(partition: partition, session: session) + receiver.public_send "#{attr_name}=", instance + end def self.call(write_event, stream_name, expected_version: nil, partition: nil, session: nil) instance = build(partition: partition, session: session) @@ -39,6 +42,7 @@ def call(write_event, stream_name, expected_version: nil) position = insert_event(stream_name, type, data, metadata, expected_version) logger.debug "Put event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" + logger.debug write_event.inspect, tags: [:data, :event_data] position end diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index d14b52e..ebbf353 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -1,6 +1,56 @@ module EventSource module Postgres class Write + include Log::Dependency + + dependency :put, Put + + def self.build(partition: nil, session: nil) + instance = new + instance.configure(partition: partition, session: session) + instance + end + + def configure(partition: nil, session: nil) + Put.configure(self, partition: partition, session: session) + end + + def self.call(subject, stream_name, expected_version: nil, partition: nil, session: nil) + instance = build(partition: partition, session: session) + instance.(subject, stream_name, expected_version: expected_version) + end + + def call(subject, stream_name, expected_version: nil) + logger.trace "Writing #{subject.class.subject} (Stream Name: #{stream_name}, Type: #{subject.type}, Expected Version: #{expected_version.inspect})" + logger.trace subject.inspect, tags: [:data, :event_data] + + write_event_data = transform(subject) + + position = put.(write_event_data, stream_name, expected_version: expected_version) + + logger.debug "Wrote #{subject.class.subject} (Stream Name: #{stream_name}, Type: #{subject.type}, Expected Version: #{expected_version.inspect})" + logger.debug write_event_data.inspect + + return position + end + + def transform(subject) + if subject.is_a? EventData + return subject + end + + logger.trace "Converting #{subject.class.message_type} to event data", tag: :transform + logger.trace subject.inspect, tags: [:transform, :data, :message] + + ## TODO + # run conversion on message + write_event_data = subject + + logger.debug "Converted #{subject.class.message_type} to event data", tag: :transform + logger.debug write_event_data.inspect, tags: [:transform, :data, :event_data] + + subject + end end end end diff --git a/test/automated/write/write.rb b/test/automated/write/write.rb index b279529..7c59e35 100644 --- a/test/automated/write/write.rb +++ b/test/automated/write/write.rb @@ -1,10 +1,10 @@ require_relative '../automated_init' context "Write" do - stream_name = Controls::Stream.example + stream = Controls::Stream.example write_event = Controls::EventData::Write.example - written_position = Write.(write_event, ) + written_position = Write.(write_event, stream.name) read_event = Get.(stream, position: written_position).first From c88bd124be2961f23980af772b29777db4487361 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 24 Oct 2016 19:28:43 -0500 Subject: [PATCH 051/591] Single and batch of event data are written --- lib/event_source/postgres/write.rb | 51 +++--- materials/event_store_write_batch_tests.rb | 37 +++++ materials/messaging_substitute_tests.rb | 116 ++++++++++++++ materials/messaging_telemetry_tests.rb | 55 +++++++ materials/messaging_writer.rb | 177 +++++++++++++++++++++ test/automated/write/write_batch.rb | 26 +++ 6 files changed, 440 insertions(+), 22 deletions(-) create mode 100644 materials/event_store_write_batch_tests.rb create mode 100644 materials/messaging_substitute_tests.rb create mode 100644 materials/messaging_telemetry_tests.rb create mode 100644 materials/messaging_writer.rb create mode 100644 test/automated/write/write_batch.rb diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index ebbf353..c7d2528 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -15,41 +15,48 @@ def configure(partition: nil, session: nil) Put.configure(self, partition: partition, session: session) end - def self.call(subject, stream_name, expected_version: nil, partition: nil, session: nil) + def self.call(event_data, stream_name, expected_version: nil, partition: nil, session: nil) instance = build(partition: partition, session: session) - instance.(subject, stream_name, expected_version: expected_version) + instance.(event_data, stream_name, expected_version: expected_version) end - def call(subject, stream_name, expected_version: nil) - logger.trace "Writing #{subject.class.subject} (Stream Name: #{stream_name}, Type: #{subject.type}, Expected Version: #{expected_version.inspect})" - logger.trace subject.inspect, tags: [:data, :event_data] + def call(event_data, stream_name, expected_version: nil) + logger.trace "Writing event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" + logger.trace event_data.inspect, tags: [:data, :event_data] - write_event_data = transform(subject) - - position = put.(write_event_data, stream_name, expected_version: expected_version) + if event_data.is_a? Array + position = write_batch(event_data, stream_name, expected_version: expected_version) + else + position = write(event_data, stream_name, expected_version: expected_version) + end - logger.debug "Wrote #{subject.class.subject} (Stream Name: #{stream_name}, Type: #{subject.type}, Expected Version: #{expected_version.inspect})" - logger.debug write_event_data.inspect + logger.debug "Wrote event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" + logger.debug event_data.inspect - return position + position end - def transform(subject) - if subject.is_a? EventData - return subject + def write_batch(batch, stream_name, expected_version: nil) + logger.trace "Writing batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" + + last_position = nil + batch.each do |event_data| + last_position = write(event_data, stream_name, expected_version: expected_version) end - logger.trace "Converting #{subject.class.message_type} to event data", tag: :transform - logger.trace subject.inspect, tags: [:transform, :data, :message] + logger.debug "Wrote batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" - ## TODO - # run conversion on message - write_event_data = subject + last_position + end - logger.debug "Converted #{subject.class.message_type} to event data", tag: :transform - logger.debug write_event_data.inspect, tags: [:transform, :data, :event_data] + def write(event_data, stream_name, expected_version: nil) + logger.trace "Writing event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" + logger.trace event_data.inspect, tags: [:data, :event_data] - subject + put.(event_data, stream_name, expected_version: expected_version).tap do + logger.debug "Wrote event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" + logger.debug event_data.inspect, tags: [:data, :event_data] + end end end end diff --git a/materials/event_store_write_batch_tests.rb b/materials/event_store_write_batch_tests.rb new file mode 100644 index 0000000..0321d82 --- /dev/null +++ b/materials/event_store_write_batch_tests.rb @@ -0,0 +1,37 @@ +require_relative '../bench_init' + +context "Write Batch of Events" do + stream_name = EventStore::Client::HTTP::Controls::StreamName.get 'testEventWriter' + path = "/streams/#{stream_name}" + + writer = EventStore::Client::HTTP::EventWriter.build + + id_1 = EventStore::Client::HTTP::Controls::ID.example 1 + id_2 = EventStore::Client::HTTP::Controls::ID.example 2 + + event_data_1 = EventStore::Client::HTTP::Controls::EventData::Write.example(id_1) + event_data_2 = EventStore::Client::HTTP::Controls::EventData::Write.example(id_2) + + event_data_1.data[:some_attribute] = id_1 + event_data_2.data[:some_attribute] = id_2 + + writer.write [event_data_1, event_data_2], stream_name + + get = EventStore::Client::HTTP::Request::Get.build + body_text_1, get_response = get.("#{path}/0") + body_text_2, get_response = get.("#{path}/1") + + read_data_1 = Serialize::Read.(body_text_1, EventStore::Client::HTTP::EventData::Read, :json) + read_data_2 = Serialize::Read.(body_text_2, EventStore::Client::HTTP::EventData::Read, :json) + + 2.times do |i| + i += 1 + event_data = binding.local_variable_get "read_data_#{i}" + + test "Individual events are written" do + control_value = binding.local_variable_get "id_#{i}" + + assert event_data.data[:some_attribute] == control_value + end + end +end diff --git a/materials/messaging_substitute_tests.rb b/materials/messaging_substitute_tests.rb new file mode 100644 index 0000000..da4c5fc --- /dev/null +++ b/materials/messaging_substitute_tests.rb @@ -0,0 +1,116 @@ +require_relative '../bench_init' + +context "Writer Substitute" do + context "Records writes" do + substitute_writer = EventStore::Messaging::Writer::Substitute.build + + message = EventStore::Messaging::Controls::Message.example + + stream_name = 'some stream name' + + substitute_writer.write message, stream_name, expected_version: 11, reply_stream_name: 'some_stream_name' + + context "Records telemetry about the write" do + test "No block arguments" do + assert(substitute_writer.written?) + end + + test "Message block argument only" do + assert(substitute_writer.written? { |msg| msg == message }) + end + + test "Message and stream name block arguments" do + assert(substitute_writer.written? { |msg, stream| stream == stream_name }) + end + + test "Message, stream name, and expected_version block arguments" do + assert(substitute_writer.written? { |msg, stream, expected_version | expected_version == 11 }) + end + + test "Message, stream name, expected_version, and reply_stream_name block arguments" do + assert(substitute_writer.written? { |msg, stream, expected_version, reply_stream_name | reply_stream_name == 'some_stream_name' }) + end + end + + context "Access the data recorded" do + test "No block arguments" do + assert(substitute_writer.writes.length == 1) + end + + test "Message block argument only" do + assert(substitute_writer.writes { |msg| msg == message }.length == 1 ) + end + + test "Message and stream name block arguments" do + assert(substitute_writer.writes { |msg, stream| stream == stream_name }.length == 1) + end + + test "Message, stream name, and expected_version block arguments" do + assert(substitute_writer.writes { |msg, stream, expected_version | expected_version == 11 }.length == 1) + end + + test "Message, stream name, expected_version, and reply_stream_name block arguments" do + assert(substitute_writer.writes { |msg, stream, expected_version, reply_stream_name | reply_stream_name == 'some_stream_name' }.length == 1) + end + end + end + + context "Records batch writes" do + substitute_writer = EventStore::Messaging::Writer::Substitute.build + + message_1 = EventStore::Messaging::Controls::Message.example + message_2 = EventStore::Messaging::Controls::Message.example + + stream_name = 'some stream name' + + substitute_writer.write [message_1, message_2], stream_name + + context "Records telemetry about each message written" do + assert substitute_writer do + written? { |msg| msg == message_1 } + end + + assert substitute_writer do + written? { |msg| msg == message_2 } + end + end + end + + context "Records replies" do + substitute_writer = EventStore::Messaging::Writer::Substitute.build + + message = EventStore::Messaging::Controls::Message.example + + stream_name = message.metadata.reply_stream_name + + substitute_writer.reply message + + context "Records replied telemetry" do + test "No block arguments" do + assert(substitute_writer.written?) + end + + test "Message argument only" do + assert(substitute_writer.replied? { |msg| msg == message }) + end + + test "Message and stream name arguments" do + assert(substitute_writer.replied? { |msg, stream| stream == stream_name }) + end + end + + context "Access the data recorded" do + test "No block arguments" do + assert(substitute_writer.replies.length == 1) + end + + test "Message block argument only" do + assert(substitute_writer.replies { |msg| msg == message }.length == 1 ) + end + + test "Message and stream name block arguments" do + assert(substitute_writer.replies { |msg, stream| stream == stream_name }.length == 1) + end + end + end +end diff --git a/materials/messaging_telemetry_tests.rb b/materials/messaging_telemetry_tests.rb new file mode 100644 index 0000000..918f56e --- /dev/null +++ b/materials/messaging_telemetry_tests.rb @@ -0,0 +1,55 @@ +require_relative '../bench_init' + +context "Writer Telemetry" do + context "Write" do + message = EventStore::Messaging::Controls::Message.example + writer = EventStore::Messaging::Writer.build + + SubstAttr::Substitute.(:writer, writer) + + stream_name = EventStore::Messaging::Controls::StreamName.get 'testWriter' + + sink = EventStore::Messaging::Writer.register_telemetry_sink(writer) + + writer.write message, stream_name, expected_version: 11, reply_stream_name: 'some_stream_name' + + test "Records written telemetry" do + assert(sink.recorded_written?) + end + + context "Recorded Data" do + data = sink.records[0].data + + test "message" do + assert(data.message == message) + end + + test "stream_name" do + assert(data.stream_name == stream_name) + end + + test "expected_version" do + assert(data.expected_version == 11) + end + + test "reply_stream_name" do + assert(data.reply_stream_name == 'some_stream_name') + end + end + end + + context "Reply" do + message = EventStore::Messaging::Controls::Message.example + writer = EventStore::Messaging::Writer.build + + SubstAttr::Substitute.(:writer, writer) + + sink = EventStore::Messaging::Writer.register_telemetry_sink(writer) + + writer.reply message + + test "Records replied telemetry" do + assert(sink.recorded_replied?) + end + end +end diff --git a/materials/messaging_writer.rb b/materials/messaging_writer.rb new file mode 100644 index 0000000..b8922b4 --- /dev/null +++ b/materials/messaging_writer.rb @@ -0,0 +1,177 @@ +module EventStore + module Messaging + class Writer + class Error < StandardError; end + + dependency :writer, EventStore::Client::HTTP::EventWriter + dependency :logger, ::Telemetry::Logger + dependency :telemetry, ::Telemetry + + def self.build(session: nil) + new.tap do |instance| + EventStore::Client::HTTP::EventWriter.configure instance, session: session + ::Telemetry::Logger.configure instance + ::Telemetry.configure instance + end + end + + def self.configure(receiver, session: nil) + instance = build(session: session) + receiver.writer = instance + instance + end + + def write(message, stream_name, expected_version: nil, reply_stream_name: nil) + unless message.is_a? Array + logger.trace "Writing (Message Type: #{message.message_type}, Stream Name: #{stream_name}, Expected Version: #{!!expected_version ? expected_version : '(none)'})" + else + logger.trace "Writing batch (Stream Name: #{stream_name}, Expected Version: #{!!expected_version ? expected_version : '(none)'})" + end + + if reply_stream_name + message.metadata.reply_stream_name = reply_stream_name + end + + event_data = event_data_batch(message) + + writer.write(event_data, stream_name, expected_version: expected_version) + + unless message.is_a? Array + logger.debug "Wrote (Message Type: #{message.message_type}, Stream Name: #{stream_name}, Expected Version: #{!!expected_version ? expected_version : '(none)'})" + else + logger.debug "Wrote batch (Stream Name: #{stream_name}, Expected Version: #{!!expected_version ? expected_version : '(none)'})" + end + + Array(message).each do |written_message| + telemetry.record :written, Telemetry::Data.new(written_message, stream_name, expected_version, reply_stream_name) + end + + event_data + end + + def write_initial(message, stream_name) + write(message, stream_name, expected_version: :no_stream) + end + + def event_data_batch(messages) + messages = [messages] unless messages.is_a? Array + + batch = messages.map do |message| + EventStore::Messaging::Message::Export::EventData.(message) + end + + batch + end + + def reply(message) + metadata = message.metadata + reply_stream_name = metadata.reply_stream_name + + logger.trace "Replying (Message Type: #{message.message_type}, Stream Name: #{reply_stream_name})" + + unless reply_stream_name + error_msg = "Message has no reply stream name. Cannot reply. (Message Type: #{message.message_type})" + logger.error error_msg + raise Error, error_msg + end + + metadata.clear_reply_stream_name + + write message, reply_stream_name + + logger.debug "Replied (Message Type: #{message.message_type}, Stream Name: #{reply_stream_name})" + + telemetry.record :replied, Telemetry::Data.new(message, reply_stream_name) + + message + end + + def self.logger + @logger ||= ::Telemetry::Logger.get self + end + + def self.register_telemetry_sink(writer) + sink = Telemetry.sink + writer.telemetry.register sink + sink + end + + module Telemetry + class Sink + include ::Telemetry::Sink + + record :written + record :replied + end + + Data = Struct.new :message, :stream_name, :expected_version, :reply_stream_name + + def self.sink + Sink.new + end + end + + module Substitute + def self.build + Substitute::Writer.build.tap do |substitute_writer| + sink = Messaging::Writer.register_telemetry_sink(substitute_writer) + substitute_writer.sink = sink + end + end + + class Writer < EventStore::Messaging::Writer + attr_accessor :sink + + def self.build(session: nil) + logger.trace "Building substitute" + new.tap do |instance| + ::Telemetry::Logger.configure instance + ::Telemetry.configure instance + logger.debug "Built substitute" + end + end + + def writes(&blk) + if blk.nil? + return sink.written_records + end + + sink.written_records.select do |record| + blk.call(record.data.message, record.data.stream_name, record.data.expected_version, record.data.reply_stream_name) + end + end + + def written?(&blk) + if blk.nil? + return sink.recorded_written? + end + + sink.recorded_written? do |record| + blk.call(record.data.message, record.data.stream_name, record.data.expected_version, record.data.reply_stream_name) + end + end + + def replies(&blk) + if blk.nil? + return sink.replied_records + end + + sink.replied_records.select do |record| + blk.call(record.data.message, record.data.stream_name) + end + end + + def replied?(&blk) + if blk.nil? + return sink.recorded_replied? + end + + sink.recorded_replied? do |record| + blk.call(record.data.message, record.data.stream_name) + end + end + end + end + end + end +end diff --git a/test/automated/write/write_batch.rb b/test/automated/write/write_batch.rb new file mode 100644 index 0000000..31cff00 --- /dev/null +++ b/test/automated/write/write_batch.rb @@ -0,0 +1,26 @@ +require_relative '../automated_init' + +context "Write" do + stream = Controls::Stream.example + + write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + + batch = [write_event_1, write_event_2] + + last_written_position = Write.(batch, stream.name) + + test "Last written position" do + assert(last_written_position == 1) + end + + context "Individual Events are Written" do + 2.times do |i| + read_event = Get.(stream, position: i, batch_size: 1).first + + test "Event #{i + 1}" do + assert(read_event.data[:attribute] == "value_#{i + 1}") + end + end + end +end From dff85be1ecddaed35d3c1f106cdc02dcb90e121c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 24 Oct 2016 19:30:45 -0500 Subject: [PATCH 052/591] Writer always writes a batch --- lib/event_source/postgres/write.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index c7d2528..033143f 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -24,11 +24,8 @@ def call(event_data, stream_name, expected_version: nil) logger.trace "Writing event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" logger.trace event_data.inspect, tags: [:data, :event_data] - if event_data.is_a? Array - position = write_batch(event_data, stream_name, expected_version: expected_version) - else - position = write(event_data, stream_name, expected_version: expected_version) - end + batch = Array(event_data) + position = write_batch(batch, stream_name, expected_version: expected_version) logger.debug "Wrote event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" logger.debug event_data.inspect From 275fee7783ba17103c2acb5d1a30aae6ebcba975 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 24 Oct 2016 20:15:10 -0500 Subject: [PATCH 053/591] Circular reference in named argument is removed --- lib/event_source/postgres/read.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index bd0e1a9..24ff93d 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -27,7 +27,7 @@ def self.call(stream_name, position: nil, batch_size: nil, precedence: nil, part instance.(&action) end - def self.configure(receiver, stream_name, attr_name: nil, position: nil, batch_size: nil, precedence: nil, partition: partition, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) + def self.configure(receiver, stream_name, attr_name: nil, position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) attr_name ||= :reader instance = build(stream_name, position: position, batch_size: batch_size, precedence: precedence, partition: partition, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) receiver.public_send "#{attr_name}=", instance From c68e00510a472d79f2dc692e4d57442a1c507492 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 24 Oct 2016 21:10:11 -0500 Subject: [PATCH 054/591] Batch is postgres transactional --- lib/event_source/postgres/write.rb | 6 ++++-- test/automated/write/{write_batch.rb => batch.rb} | 0 2 files changed, 4 insertions(+), 2 deletions(-) rename test/automated/write/{write_batch.rb => batch.rb} (100%) diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index 033143f..5eefe02 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -37,8 +37,10 @@ def write_batch(batch, stream_name, expected_version: nil) logger.trace "Writing batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" last_position = nil - batch.each do |event_data| - last_position = write(event_data, stream_name, expected_version: expected_version) + put.session.connection.transaction do + batch.each do |event_data| + last_position = write(event_data, stream_name, expected_version: expected_version) + end end logger.debug "Wrote batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" diff --git a/test/automated/write/write_batch.rb b/test/automated/write/batch.rb similarity index 100% rename from test/automated/write/write_batch.rb rename to test/automated/write/batch.rb From 15ef82e92513ff999ddbd9c00d0a76f9f9ed9171 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 24 Oct 2016 23:33:53 -0500 Subject: [PATCH 055/591] Interactive test is compatible with updated API --- test/interactive/reader_does_not_terminate.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/interactive/reader_does_not_terminate.rb b/test/interactive/reader_does_not_terminate.rb index 951f590..fdc95ea 100644 --- a/test/interactive/reader_does_not_terminate.rb +++ b/test/interactive/reader_does_not_terminate.rb @@ -1,3 +1,5 @@ require_relative '../test_init' -Read.(stream_name: 'some_stream', batch_size: 1, delay_milliseconds: 200) { |event_data| } +stream_name = Controls::StreamName.example + +Read.(stream_name, batch_size: 1, delay_milliseconds: 200) { |event_data| } From ecb3db474c159bd1afe348cb67c3c835104a3475 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 25 Oct 2016 01:17:13 -0500 Subject: [PATCH 056/591] Block form logging --- lib/event_source/postgres/get.rb | 12 ++++---- .../postgres/get/select_statement.rb | 6 ++-- lib/event_source/postgres/iterator.rb | 28 +++++++++---------- lib/event_source/postgres/put.rb | 24 ++++++++-------- lib/event_source/postgres/read.rb | 4 +-- lib/event_source/postgres/session.rb | 10 +++---- lib/event_source/postgres/write.rb | 20 ++++++------- 7 files changed, 52 insertions(+), 52 deletions(-) diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 836eaed..2cc50fe 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -29,31 +29,31 @@ def configure(session: nil) end def call(position: nil) - logger.trace "Getting event data (Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.trace { "Getting event data (Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" } records = get_records(stream, position) events = convert(records) - logger.debug "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.debug { "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" } events end def get_records(stream, position) - logger.trace "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.trace { "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" } select_statement = SelectStatement.build(stream, offset: position, batch_size: batch_size, precedence: precedence, partition: partition) records = session.connection.exec(select_statement.sql) - logger.debug "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" + logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" } records end def convert(records) - logger.trace "Converting records to event data (Records Count: #{records.ntuples})" + logger.trace { "Converting records to event data (Records Count: #{records.ntuples})" } events = records.map do |record| record['data'] = Deserialize.data(record['data']) @@ -63,7 +63,7 @@ def convert(records) EventData::Read.build record end - logger.debug "Converted records to event data (Event Data Count: #{events.length})" + logger.debug { "Converted records to event data (Event Data Count: #{events.length})" } events end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 618bddf..df4746c 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -35,7 +35,7 @@ def self.build(stream, offset: nil, batch_size: nil, partition: nil, precedence: end def sql - logger.trace "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition})", tag: :sql + logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition})" } statement = <<-SQL SELECT @@ -59,8 +59,8 @@ def sql ; SQL - logger.debug "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition.inspect})", tag: :sql - logger.debug "Statement: #{statement}", tags: [:data, :sql] + logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition.inspect})" } + logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } statement end diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index 7ed5864..ac2f51c 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -36,14 +36,14 @@ def self.configure(receiver, stream, attr_name: nil, position: nil, batch_size: end def next - logger.trace "Getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" + logger.trace { "Getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" } resupply(batch) event_data = batch[batch_position] - logger.debug "Finished getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" - logger.debug "Event Data: #{event_data.inspect}", tags: [:data, :event_data] + logger.debug { "Finished getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" } + logger.debug(tags: [:data, :event_data]) { "Event Data: #{event_data.inspect}" } advance_positions @@ -51,7 +51,7 @@ def next end def resupply(batch) - logger.trace "Resupplying batch" + logger.trace { "Resupplying batch" } if batch.nil? batch_log_text = "Batch: #{batch.inspect}" @@ -60,26 +60,26 @@ def resupply(batch) end if batch.nil? || batch_position == batch.length - logger.debug "Current batch is depleted (#{batch_log_text}, Batch Position: #{batch_position})" + logger.debug { "Current batch is depleted (#{batch_log_text}, Batch Position: #{batch_position})" } batch = get_batch reset(batch) else - logger.debug "Current batch not depleted (#{batch_log_text}, Batch Position: #{batch_position})" + logger.debug { "Current batch not depleted (#{batch_log_text}, Batch Position: #{batch_position})" } end - logger.debug "Finished resupplying batch" + logger.debug { "Finished resupplying batch" } end def reset(batch) - logger.trace "Resetting batch" + logger.trace { "Resetting batch" } self.batch = batch self.batch_position = 0 - logger.debug "Reset batch" - logger.debug "Batch: #{batch.inspect}", tags: [:data, :batch] - logger.debug "Batch Position: #{batch_position.inspect}", tags: [:data, :batch] + logger.debug { "Reset batch" } + logger.debug(tags: [:data, :batch]) { "Batch: #{batch.inspect}" } + logger.debug(tags: [:data, :batch]) { "Batch Position: #{batch_position.inspect}" } end def get_batch @@ -90,16 +90,16 @@ def get_batch batch = get.(position: stream_offset) end - logger.debug "Finished getting batch (Count: #{batch.length})" + logger.debug { "Finished getting batch (Count: #{batch.length})" } batch end def advance_positions - logger.trace "Advancing positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" + logger.trace { "Advancing positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" } self.batch_position += 1 self.stream_offset += 1 - logger.debug "Advanced positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" + logger.debug { "Advanced positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" } end end end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 538ffcd..5c7c226 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -33,16 +33,16 @@ def self.call(write_event, stream_name, expected_version: nil, partition: nil, s end def call(write_event, stream_name, expected_version: nil) - logger.trace "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" - logger.trace write_event.inspect, tags: [:data, :event_data] + logger.trace { "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } + logger.trace(tags: [:data, :event_data]) { write_event.inspect } type, data, metadata = destructure_event(write_event) expected_version = canonize_expected_version(expected_version) position = insert_event(stream_name, type, data, metadata, expected_version) - logger.debug "Put event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" - logger.debug write_event.inspect, tags: [:data, :event_data] + logger.debug { "Put event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } + logger.debug(tags: [:data, :event_data]) { write_event.inspect } position end @@ -52,8 +52,8 @@ def destructure_event(write_event) data = write_event.data metadata = write_event.metadata - logger.debug "Data: #{data.inspect}", tags: [:data, :event_data] - logger.debug "Metadata: #{metadata.inspect}", tags: [:data, :event_data] + logger.debug(tags: [:data, :event_data]) { "Data: #{data.inspect}" } + logger.debug(tags: [:data, :event_data]) { "Metadata: #{metadata.inspect}" } return type, data, metadata end @@ -61,9 +61,9 @@ def destructure_event(write_event) def canonize_expected_version(expected_version) return expected_version unless expected_version == NoStream.name - logger.trace "Canonizing expected version (Expected Version: #{expected_version})" + logger.trace { "Canonizing expected version (Expected Version: #{expected_version})" } expected_version = NoStream.version - logger.debug "Canonized expected version (Expected Version: #{expected_version})" + logger.debug { "Canonized expected version (Expected Version: #{expected_version})" } expected_version end @@ -75,7 +75,7 @@ def insert_event(stream_name, type, data, metadata, expected_version) end def execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version) - logger.trace "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" + logger.trace { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" } sql_args = [ stream_name, @@ -92,7 +92,7 @@ def execute_query(stream_name, type, serialized_data, serialized_metadata, expec raise_error e end - logger.debug "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" + logger.debug { "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" } records end @@ -104,7 +104,7 @@ def self.statement def serialized_data(data) serializable_data = EventData::Hash[data] serialized_data = Serialize::Write.(serializable_data, :json) - logger.debug "Serialized Data: #{serialized_data.inspect}", tags: [:data, :serialize] + logger.debug(tags: [:data, :serialize]) { "Serialized Data: #{serialized_data.inspect}" } serialized_data end @@ -114,7 +114,7 @@ def serialized_metadata(metadata) unless metadata.nil? serialized_metadata = Serialize::Write.(serializable_metadata, :json) end - logger.debug "Serialized Metadata: #{serialized_metadata.inspect}", tags: [:data, :serialize] + logger.debug(tags: [:data, :serialize]) { "Serialized Metadata: #{serialized_metadata.inspect}" } serialized_metadata end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 24ff93d..4a91688 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -46,7 +46,7 @@ def call(&action) end def enumerate_event_data(&action) - logger.trace "Reading (Stream Name: #{stream.name}, Category: #{stream.category?})" + logger.trace { "Reading (Stream Name: #{stream.name}, Category: #{stream.category?})" } event_data = nil @@ -58,7 +58,7 @@ def enumerate_event_data(&action) action.(event_data) end - logger.debug "Finished reading (Stream Name: #{stream.name}, Category: #{stream.category?})" + logger.debug { "Finished reading (Stream Name: #{stream.name}, Category: #{stream.category?})" } end end end diff --git a/lib/event_source/postgres/session.rb b/lib/event_source/postgres/session.rb index ecafb72..3b5a6b2 100644 --- a/lib/event_source/postgres/session.rb +++ b/lib/event_source/postgres/session.rb @@ -35,27 +35,27 @@ def self.connect(instance, connection=nil) logger.trace "Connecting to database" if connection.nil? - logger.debug "No connection. A new one will be built." + logger.debug { "No connection. A new one will be built." } connection = build_connection(instance) else - logger.debug "Reusing existing connection" + logger.debug { "Reusing existing connection" } end instance.connection = connection - logger.debug "Connected to database" + logger.debug { "Connected to database" } connection end def self.build_connection(instance) settings = instance.settings - logger.trace "Building new connection to database (Settings: #{LogText.settings(settings).inspect})" + logger.trace { "Building new connection to database (Settings: #{LogText.settings(settings).inspect})" } connection = PG::Connection.open(settings) connection.type_map_for_results = PG::BasicTypeMapForResults.new(connection) - logger.trace "Built new connection to database (Settings: #{LogText.settings(settings).inspect})" + logger.trace { "Built new connection to database (Settings: #{LogText.settings(settings).inspect})" } connection end diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index 5eefe02..b850e25 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -21,20 +21,20 @@ def self.call(event_data, stream_name, expected_version: nil, partition: nil, se end def call(event_data, stream_name, expected_version: nil) - logger.trace "Writing event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" - logger.trace event_data.inspect, tags: [:data, :event_data] + logger.trace { "Writing event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" } + logger.trace(tags: [:data, :event_data]) { event_data.inspect } batch = Array(event_data) position = write_batch(batch, stream_name, expected_version: expected_version) - logger.debug "Wrote event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" - logger.debug event_data.inspect + logger.debug { "Wrote event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" } + logger.debug(tags: [:data, :event_data]) { event_data.inspect } position end def write_batch(batch, stream_name, expected_version: nil) - logger.trace "Writing batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" + logger.trace { "Writing batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } last_position = nil put.session.connection.transaction do @@ -43,18 +43,18 @@ def write_batch(batch, stream_name, expected_version: nil) end end - logger.debug "Wrote batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" + logger.debug { "Wrote batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } last_position end def write(event_data, stream_name, expected_version: nil) - logger.trace "Writing event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" - logger.trace event_data.inspect, tags: [:data, :event_data] + logger.trace { "Writing event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } + logger.trace(tags: [:data, :event_data]) { event_data.inspect } put.(event_data, stream_name, expected_version: expected_version).tap do - logger.debug "Wrote event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" - logger.debug event_data.inspect, tags: [:data, :event_data] + logger.debug { "Wrote event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } + logger.debug(tags: [:data, :event_data]) { event_data.inspect } end end end From 214a1240079b743d881d7e89c8b65511bae5bbbf Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 25 Oct 2016 01:43:31 -0500 Subject: [PATCH 057/591] Pretty inspect event data --- lib/event_source/postgres/iterator.rb | 4 ++-- lib/event_source/postgres/put.rb | 8 ++++---- lib/event_source/postgres/write.rb | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index ac2f51c..69b5401 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -43,7 +43,7 @@ def next event_data = batch[batch_position] logger.debug { "Finished getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" } - logger.debug(tags: [:data, :event_data]) { "Event Data: #{event_data.inspect}" } + logger.debug(tags: [:data, :event_data]) { "Event Data: #{event_data.pretty_inspect}" } advance_positions @@ -78,7 +78,7 @@ def reset(batch) self.batch_position = 0 logger.debug { "Reset batch" } - logger.debug(tags: [:data, :batch]) { "Batch: #{batch.inspect}" } + logger.debug(tags: [:data, :batch]) { "Batch: #{batch.pretty_inspect}" } logger.debug(tags: [:data, :batch]) { "Batch Position: #{batch_position.inspect}" } end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 5c7c226..e402776 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -34,7 +34,7 @@ def self.call(write_event, stream_name, expected_version: nil, partition: nil, s def call(write_event, stream_name, expected_version: nil) logger.trace { "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :event_data]) { write_event.inspect } + logger.trace(tags: [:data, :event_data]) { write_event.pretty_inspect } type, data, metadata = destructure_event(write_event) expected_version = canonize_expected_version(expected_version) @@ -42,7 +42,7 @@ def call(write_event, stream_name, expected_version: nil) position = insert_event(stream_name, type, data, metadata, expected_version) logger.debug { "Put event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } - logger.debug(tags: [:data, :event_data]) { write_event.inspect } + logger.debug(tags: [:data, :event_data]) { write_event.pretty_inspect } position end @@ -52,8 +52,8 @@ def destructure_event(write_event) data = write_event.data metadata = write_event.metadata - logger.debug(tags: [:data, :event_data]) { "Data: #{data.inspect}" } - logger.debug(tags: [:data, :event_data]) { "Metadata: #{metadata.inspect}" } + logger.debug(tags: [:data, :event_data]) { "Data: #{data.pretty_inspect}" } + logger.debug(tags: [:data, :event_data]) { "Metadata: #{metadata.pretty_inspect}" } return type, data, metadata end diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index b850e25..ce4a407 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -22,13 +22,13 @@ def self.call(event_data, stream_name, expected_version: nil, partition: nil, se def call(event_data, stream_name, expected_version: nil) logger.trace { "Writing event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :event_data]) { event_data.inspect } + logger.trace(tags: [:data, :event_data]) { event_data.pretty_inspect } batch = Array(event_data) position = write_batch(batch, stream_name, expected_version: expected_version) logger.debug { "Wrote event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" } - logger.debug(tags: [:data, :event_data]) { event_data.inspect } + logger.debug(tags: [:data, :event_data]) { event_data.pretty_inspect } position end @@ -50,11 +50,11 @@ def write_batch(batch, stream_name, expected_version: nil) def write(event_data, stream_name, expected_version: nil) logger.trace { "Writing event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :event_data]) { event_data.inspect } + logger.trace(tags: [:data, :event_data]) { event_data.pretty_inspect } put.(event_data, stream_name, expected_version: expected_version).tap do logger.debug { "Wrote event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } - logger.debug(tags: [:data, :event_data]) { event_data.inspect } + logger.debug(tags: [:data, :event_data]) { event_data.pretty_inspect } end end end From 6bdb308a0d1eef79dc7cb39c53f95c77f1e2f8e1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 25 Oct 2016 18:18:10 -0500 Subject: [PATCH 058/591] Package version set to 0.1.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index f8e7ee9..d7ac6ef 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.0.0.0' + s.version = '0.1.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 7a575c71efed587096a44e9172bccba30a1c6854 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 25 Oct 2016 19:33:29 -0500 Subject: [PATCH 059/591] Minimum logging by default when tests are executed --- test/test_init.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_init.rb b/test/test_init.rb index ca80aa6..a02b760 100644 --- a/test/test_init.rb +++ b/test/test_init.rb @@ -1,5 +1,5 @@ ENV['CONSOLE_DEVICE'] ||= 'stdout' -ENV['LOG_LEVEL'] ||= 'fatal' +ENV['LOG_LEVEL'] ||= '_min' puts RUBY_DESCRIPTION From 8b9b6e4a57e36aff0e0a45337fd923a7971c935b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 27 Oct 2016 17:26:40 -0500 Subject: [PATCH 060/591] Control constants are not redefined --- lib/event_source/postgres/controls/event_data/hash.rb | 2 +- lib/event_source/postgres/controls/event_data/metadata.rb | 2 +- lib/event_source/postgres/controls/event_data/write.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/event_source/postgres/controls/event_data/hash.rb b/lib/event_source/postgres/controls/event_data/hash.rb index 70b28f7..126a9ec 100644 --- a/lib/event_source/postgres/controls/event_data/hash.rb +++ b/lib/event_source/postgres/controls/event_data/hash.rb @@ -2,7 +2,7 @@ module EventSource module Postgres module Controls module EventData - Hash = EventSource::Controls::EventData::Hash + Hash ||= EventSource::Controls::EventData::Hash end end end diff --git a/lib/event_source/postgres/controls/event_data/metadata.rb b/lib/event_source/postgres/controls/event_data/metadata.rb index 255f5ac..40415b1 100644 --- a/lib/event_source/postgres/controls/event_data/metadata.rb +++ b/lib/event_source/postgres/controls/event_data/metadata.rb @@ -2,7 +2,7 @@ module EventSource module Postgres module Controls module EventData - Metadata = EventSource::Controls::EventData::Metadata + Metadata ||= EventSource::Controls::EventData::Metadata end end end diff --git a/lib/event_source/postgres/controls/event_data/write.rb b/lib/event_source/postgres/controls/event_data/write.rb index ca26574..2ce0bdb 100644 --- a/lib/event_source/postgres/controls/event_data/write.rb +++ b/lib/event_source/postgres/controls/event_data/write.rb @@ -2,7 +2,7 @@ module EventSource module Postgres module Controls module EventData - Write = EventSource::Controls::EventData::Write + Write ||= EventSource::Controls::EventData::Write end end end From 29c89c0193d24327c17d681e1bbb4fcdff87c02c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 27 Oct 2016 21:59:29 -0500 Subject: [PATCH 061/591] Package version is increased from 0.1.0.0 to 0.1.0.1 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index d7ac6ef..d8e0c1a 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.1.0.0' + s.version = '0.1.0.1' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 2d40debc7eeea4783563b4b0f2797e0e9c968bf6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 28 Oct 2016 20:43:38 -0500 Subject: [PATCH 062/591] Transform, rather than serialize --- lib/event_source/postgres/get.rb | 4 ++-- lib/event_source/postgres/put.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 2cc50fe..0d5c91b 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -70,14 +70,14 @@ def convert(records) module Deserialize def self.data(serialized_data) - Serialize::Read.(serialized_data, EventData::Hash, :json) + Transform::Read.(serialized_data, EventData::Hash, :json) end def self.metadata(serialized_metadata) if serialized_metadata.nil? nil else - Serialize::Read.(serialized_metadata, EventData::Hash, :json) + Transform::Read.(serialized_metadata, EventData::Hash, :json) end end end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index e402776..b54a1a2 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -103,7 +103,7 @@ def self.statement def serialized_data(data) serializable_data = EventData::Hash[data] - serialized_data = Serialize::Write.(serializable_data, :json) + serialized_data = Transform::Write.(serializable_data, :json) logger.debug(tags: [:data, :serialize]) { "Serialized Data: #{serialized_data.inspect}" } serialized_data end @@ -112,7 +112,7 @@ def serialized_metadata(metadata) serializable_metadata = EventData::Hash[metadata] serialized_metadata = nil unless metadata.nil? - serialized_metadata = Serialize::Write.(serializable_metadata, :json) + serialized_metadata = Transform::Write.(serializable_metadata, :json) end logger.debug(tags: [:data, :serialize]) { "Serialized Metadata: #{serialized_metadata.inspect}" } serialized_metadata From 61a42a9ffd7f03ce486a762aff42d20abdd20a47 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 28 Oct 2016 20:44:20 -0500 Subject: [PATCH 063/591] Package version creased from 0.1.0.1 to 0.1.0.2 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index d8e0c1a..d10f766 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.1.0.1' + s.version = '0.1.0.2' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 57c17ebfce50d9eb86271e93ea3dac976a2dcd58 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 28 Oct 2016 22:08:05 -0500 Subject: [PATCH 064/591] Consistency in style of accessing first element of array --- test/automated/get/get.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index b59386f..2384f83 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -6,7 +6,7 @@ written_position = Put.(write_event, stream.name) - read_event = Get.(stream, position: written_position)[0] + read_event = Get.(stream, position: written_position).first test "Got the event that was written" do assert(read_event.position == written_position) From 18a092a7b59a3d14ef26311f4de30bff8e3874cc Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 28 Oct 2016 22:18:25 -0500 Subject: [PATCH 065/591] Individual fields are checked after event data is retrieved --- test/automated/get/get.rb | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index 2384f83..4024846 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -8,7 +8,24 @@ read_event = Get.(stream, position: written_position).first - test "Got the event that was written" do - assert(read_event.position == written_position) + context "Got the event that was written" do + test "Data" do + # assert(read_event.data == Controls::EventData::Write.data) + end + + test "Position" do + assert(read_event.position == written_position) + end end end + +__END__ + +@created_time=2016-10-29 03:10:44 UTC, + @data={:some_attribute=>"some value"}, + @global_position=1332, + @metadata={:some_meta_attribute=>"some meta value"}, + @position=0, + @stream_name= + "test3e44aba36b15ba24330689a33e403711XX-8236f048-7c04-46de-92b2-ed0fbf7a8a06", + @type="SomeType"> From 004c29b2d8116f633b3ffca07a7c8dfeeff62e1a Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 28 Oct 2016 22:20:53 -0500 Subject: [PATCH 066/591] Superfluous control namespace aliases are removed --- lib/event_source/postgres/controls.rb | 3 --- lib/event_source/postgres/controls/event_data/hash.rb | 9 --------- .../postgres/controls/event_data/metadata.rb | 9 --------- lib/event_source/postgres/controls/event_data/write.rb | 9 --------- 4 files changed, 30 deletions(-) delete mode 100644 lib/event_source/postgres/controls/event_data/hash.rb delete mode 100644 lib/event_source/postgres/controls/event_data/metadata.rb delete mode 100644 lib/event_source/postgres/controls/event_data/write.rb diff --git a/lib/event_source/postgres/controls.rb b/lib/event_source/postgres/controls.rb index 0ff66e2..1bf6992 100644 --- a/lib/event_source/postgres/controls.rb +++ b/lib/event_source/postgres/controls.rb @@ -5,9 +5,6 @@ require 'event_source/postgres/controls/stream' require 'event_source/postgres/controls/event_data' -require 'event_source/postgres/controls/event_data/hash' -require 'event_source/postgres/controls/event_data/metadata' -require 'event_source/postgres/controls/event_data/write' require 'event_source/postgres/controls/partition' diff --git a/lib/event_source/postgres/controls/event_data/hash.rb b/lib/event_source/postgres/controls/event_data/hash.rb deleted file mode 100644 index 126a9ec..0000000 --- a/lib/event_source/postgres/controls/event_data/hash.rb +++ /dev/null @@ -1,9 +0,0 @@ -module EventSource - module Postgres - module Controls - module EventData - Hash ||= EventSource::Controls::EventData::Hash - end - end - end -end diff --git a/lib/event_source/postgres/controls/event_data/metadata.rb b/lib/event_source/postgres/controls/event_data/metadata.rb deleted file mode 100644 index 40415b1..0000000 --- a/lib/event_source/postgres/controls/event_data/metadata.rb +++ /dev/null @@ -1,9 +0,0 @@ -module EventSource - module Postgres - module Controls - module EventData - Metadata ||= EventSource::Controls::EventData::Metadata - end - end - end -end diff --git a/lib/event_source/postgres/controls/event_data/write.rb b/lib/event_source/postgres/controls/event_data/write.rb deleted file mode 100644 index 2ce0bdb..0000000 --- a/lib/event_source/postgres/controls/event_data/write.rb +++ /dev/null @@ -1,9 +0,0 @@ -module EventSource - module Postgres - module Controls - module EventData - Write ||= EventSource::Controls::EventData::Write - end - end - end -end From 0344f13ccea67d4b7f26334d6d1027e6126f0bff Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 28 Oct 2016 22:26:59 -0500 Subject: [PATCH 067/591] Coverage is improved in round trip event storage test --- test/automated/get/get.rb | 33 +++++++++++++++++++++------------ test/automated/put/put.rb | 30 ++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index 4024846..c33a42d 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -9,23 +9,32 @@ read_event = Get.(stream, position: written_position).first context "Got the event that was written" do + test "Type" do + assert(read_event.type == write_event.type) + end + test "Data" do - # assert(read_event.data == Controls::EventData::Write.data) + assert(read_event.data == write_event.data) + end + + test "Metadata" do + assert(read_event.metadata == write_event.metadata) + end + + test "Stream Name" do + assert(read_event.stream_name == stream.name) end test "Position" do assert(read_event.position == written_position) end - end -end -__END__ + test "Global Position" do + assert(read_event.global_position.is_a? Numeric) + end -@created_time=2016-10-29 03:10:44 UTC, - @data={:some_attribute=>"some value"}, - @global_position=1332, - @metadata={:some_meta_attribute=>"some meta value"}, - @position=0, - @stream_name= - "test3e44aba36b15ba24330689a33e403711XX-8236f048-7c04-46de-92b2-ed0fbf7a8a06", - @type="SomeType"> + test "Created Time" do + assert(read_event.created_time.is_a? Time) + end + end +end diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index 2384f83..c33a42d 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -8,7 +8,33 @@ read_event = Get.(stream, position: written_position).first - test "Got the event that was written" do - assert(read_event.position == written_position) + context "Got the event that was written" do + test "Type" do + assert(read_event.type == write_event.type) + end + + test "Data" do + assert(read_event.data == write_event.data) + end + + test "Metadata" do + assert(read_event.metadata == write_event.metadata) + end + + test "Stream Name" do + assert(read_event.stream_name == stream.name) + end + + test "Position" do + assert(read_event.position == written_position) + end + + test "Global Position" do + assert(read_event.global_position.is_a? Numeric) + end + + test "Created Time" do + assert(read_event.created_time.is_a? Time) + end end end From b44d55d618c006a242b259f2263cc29948bef0da Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 28 Oct 2016 22:33:53 -0500 Subject: [PATCH 068/591] Recorded time, rather than created time --- database/tables/events-table.sql | 2 +- database/tables/other-events-table.sql | 2 +- lib/event_source/postgres/get.rb | 2 +- lib/event_source/postgres/get/select_statement.rb | 2 +- test/automated/get/get.rb | 4 ++-- test/automated/put/put.rb | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/database/tables/events-table.sql b/database/tables/events-table.sql index cbe3327..2c9c488 100644 --- a/database/tables/events-table.sql +++ b/database/tables/events-table.sql @@ -9,7 +9,7 @@ CREATE TABLE "public"."events" ( "global_position" bigserial NOT NULL , "data" jsonb NOT NULL, "metadata" jsonb, - "created_time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL + "recorded_time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL ) WITH (OIDS=FALSE); diff --git a/database/tables/other-events-table.sql b/database/tables/other-events-table.sql index 00a48cf..9350624 100644 --- a/database/tables/other-events-table.sql +++ b/database/tables/other-events-table.sql @@ -9,7 +9,7 @@ CREATE TABLE "public"."other_events" ( "global_position" bigserial NOT NULL , "data" jsonb NOT NULL, "metadata" jsonb, - "created_time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL + "recorded_time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL ) WITH (OIDS=FALSE); diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 0d5c91b..822c44d 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -58,7 +58,7 @@ def convert(records) events = records.map do |record| record['data'] = Deserialize.data(record['data']) record['metadata'] = Deserialize.metadata(record['metadata']) - record['created_time'] = Time.utc_coerced(record['created_time']) + record['recorded_time'] = Time.utc_coerced(record['recorded_time']) EventData::Read.build record end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index df4746c..01b9bd9 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -45,7 +45,7 @@ def sql global_position::bigint, data::varchar, metadata::varchar, - created_time::timestamp + recorded_time::timestamp FROM #{partition} WHERE diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index c33a42d..d9f4c39 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -33,8 +33,8 @@ assert(read_event.global_position.is_a? Numeric) end - test "Created Time" do - assert(read_event.created_time.is_a? Time) + test "Recorded Time" do + assert(read_event.recorded_time.is_a? Time) end end end diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index c33a42d..d9f4c39 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -33,8 +33,8 @@ assert(read_event.global_position.is_a? Numeric) end - test "Created Time" do - assert(read_event.created_time.is_a? Time) + test "Recorded Time" do + assert(read_event.recorded_time.is_a? Time) end end end From a6ca92526a8c9f99e4c10e93b01b4f7f134abf75 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 28 Oct 2016 22:35:53 -0500 Subject: [PATCH 069/591] Package version is increased from 0.1.0.2 to 0.2.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index d10f766..7d10971 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.1.0.2' + s.version = '0.2.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 6d75d96c63970263e59a3bdf113ce223c450c14b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 29 Oct 2016 02:06:21 -0500 Subject: [PATCH 070/591] Time, rather than recorded time --- database/tables/events-table.sql | 2 +- database/tables/other-events-table.sql | 2 +- lib/event_source/postgres/get.rb | 2 +- lib/event_source/postgres/get/select_statement.rb | 2 +- test/automated/get/get.rb | 2 +- test/automated/put/put.rb | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/database/tables/events-table.sql b/database/tables/events-table.sql index 2c9c488..80a2e05 100644 --- a/database/tables/events-table.sql +++ b/database/tables/events-table.sql @@ -9,7 +9,7 @@ CREATE TABLE "public"."events" ( "global_position" bigserial NOT NULL , "data" jsonb NOT NULL, "metadata" jsonb, - "recorded_time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL + "time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL ) WITH (OIDS=FALSE); diff --git a/database/tables/other-events-table.sql b/database/tables/other-events-table.sql index 9350624..9810a60 100644 --- a/database/tables/other-events-table.sql +++ b/database/tables/other-events-table.sql @@ -9,7 +9,7 @@ CREATE TABLE "public"."other_events" ( "global_position" bigserial NOT NULL , "data" jsonb NOT NULL, "metadata" jsonb, - "recorded_time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL + "time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL ) WITH (OIDS=FALSE); diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 822c44d..8b19d03 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -58,7 +58,7 @@ def convert(records) events = records.map do |record| record['data'] = Deserialize.data(record['data']) record['metadata'] = Deserialize.metadata(record['metadata']) - record['recorded_time'] = Time.utc_coerced(record['recorded_time']) + record['time'] = Time.utc_coerced(record['time']) EventData::Read.build record end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 01b9bd9..9e9287e 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -45,7 +45,7 @@ def sql global_position::bigint, data::varchar, metadata::varchar, - recorded_time::timestamp + time::timestamp FROM #{partition} WHERE diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index d9f4c39..cdbfa53 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -34,7 +34,7 @@ end test "Recorded Time" do - assert(read_event.recorded_time.is_a? Time) + assert(read_event.time.is_a? Time) end end end diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index d9f4c39..cdbfa53 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -34,7 +34,7 @@ end test "Recorded Time" do - assert(read_event.recorded_time.is_a? Time) + assert(read_event.time.is_a? Time) end end end From e7c46d776de84738048ab4f4c5bad34ba922705f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 29 Oct 2016 02:07:18 -0500 Subject: [PATCH 071/591] Package version increased from 0.2.0.0 to 0.3.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 7d10971..912f76f 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.2.0.0' + s.version = '0.3.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From fba1895809a7ffdacb47b2ff18da8e1d4267988b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 31 Oct 2016 19:55:35 -0500 Subject: [PATCH 072/591] Read's dependency on session is vestigial. It's removed. --- lib/event_source/postgres/read.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 4a91688..299c18f 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -7,7 +7,6 @@ class Error < RuntimeError; end initializer :stream - dependency :session, Session dependency :iterator, Iterator def self.build(stream_name, position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) From 0ce2e5741d08430d56a182a577239a5b963df00f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 15:19:11 -0500 Subject: [PATCH 073/591] Dependency graph is organized to allow subclasses of read to construct their Get instance --- lib/event_source/postgres/iterator.rb | 14 +++++++------- lib/event_source/postgres/read.rb | 10 +++++++++- test/automated/iterator/next.rb | 4 +++- test/automated/iterator/no_further_event_data.rb | 4 +++- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb index 69b5401..7b4734e 100644 --- a/lib/event_source/postgres/iterator.rb +++ b/lib/event_source/postgres/iterator.rb @@ -5,9 +5,9 @@ class Error < RuntimeError; end include Log::Dependency - dependency :get, Get dependency :cycle, Cycle + attr_accessor :position attr_accessor :batch attr_writer :batch_position attr_writer :stream_offset @@ -20,18 +20,18 @@ def stream_offset @stream_offset ||= (position || 0) end - initializer :position + initializer :get - def self.build(stream, position: nil, batch_size: nil, precedence: nil, partition: nil, cycle: nil, session: nil) - new(position).tap do |instance| - Get.configure instance, stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session + def self.build(get, position: nil, cycle: nil) + new(get).tap do |instance| + instance.position = position Cycle.configure instance, cycle: cycle end end - def self.configure(receiver, stream, attr_name: nil, position: nil, batch_size: nil, precedence: nil, partition: nil, cycle: nil, session: nil) + def self.configure(receiver, get, attr_name: nil, position: nil, cycle: nil) attr_name ||= :iterator - instance = build(stream, position: position, batch_size: batch_size, precedence: precedence, partition: partition, cycle: cycle, session: session) + instance = build(get, position: position, cycle: cycle) receiver.public_send "#{attr_name}=", instance end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 299c18f..51831d9 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -16,11 +16,19 @@ def self.build(stream_name, position: nil, batch_size: nil, precedence: nil, par cycle = Cycle.build(delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds) end + cycle ||= Cycle.build(delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds) + new(stream).tap do |instance| - Iterator.configure instance, stream, position: position, batch_size: batch_size, precedence: precedence, partition: partition, cycle: cycle, session: session + get = instance.build_get(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) + Iterator.configure instance, get, position: position, cycle: cycle end end + ## TODO will need to be abstract + def build_get(stream, batch_size: nil, precedence: nil, partition: nil, session: nil) + Get.build(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) + end + def self.call(stream_name, position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil, &action) instance = build(stream_name, position: position, batch_size: batch_size, precedence: precedence, partition: partition, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) instance.(&action) diff --git a/test/automated/iterator/next.rb b/test/automated/iterator/next.rb index ee75a4a..2bcaefd 100644 --- a/test/automated/iterator/next.rb +++ b/test/automated/iterator/next.rb @@ -4,7 +4,9 @@ context "Next" do stream = Controls::Put.(instances: 2) - iterator = Iterator.build(stream, batch_size: 1) + get = Get.build(stream, batch_size: 1) + + iterator = Iterator.build(get) batch = [] diff --git a/test/automated/iterator/no_further_event_data.rb b/test/automated/iterator/no_further_event_data.rb index 60d3f1c..599e9a5 100644 --- a/test/automated/iterator/no_further_event_data.rb +++ b/test/automated/iterator/no_further_event_data.rb @@ -4,7 +4,9 @@ context "No further event data" do stream = Controls::Put.(instances: 2) - iterator = Iterator.build(stream) + get = Get.build(stream) + + iterator = Iterator.build(get) 2.times { iterator.next } From 0847a6f784c1c261d815306d1ba72b0a557d1ebd Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 15:39:11 -0500 Subject: [PATCH 074/591] Iterator class and read implementation is moved to EventSource --- lib/event_source/postgres.rb | 1 - lib/event_source/postgres/iterator.rb | 106 -------------------------- lib/event_source/postgres/read.rb | 65 +--------------- 3 files changed, 1 insertion(+), 171 deletions(-) delete mode 100644 lib/event_source/postgres/iterator.rb diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index 4a8a46a..a2d1d91 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -20,5 +20,4 @@ require 'event_source/postgres/get/select_statement' require 'event_source/postgres/get' -require 'event_source/postgres/iterator' require 'event_source/postgres/read' diff --git a/lib/event_source/postgres/iterator.rb b/lib/event_source/postgres/iterator.rb deleted file mode 100644 index 7b4734e..0000000 --- a/lib/event_source/postgres/iterator.rb +++ /dev/null @@ -1,106 +0,0 @@ -module EventSource - module Postgres - class Iterator - class Error < RuntimeError; end - - include Log::Dependency - - dependency :cycle, Cycle - - attr_accessor :position - attr_accessor :batch - attr_writer :batch_position - attr_writer :stream_offset - - def batch_position - @batch_position ||= 0 - end - - def stream_offset - @stream_offset ||= (position || 0) - end - - initializer :get - - def self.build(get, position: nil, cycle: nil) - new(get).tap do |instance| - instance.position = position - Cycle.configure instance, cycle: cycle - end - end - - def self.configure(receiver, get, attr_name: nil, position: nil, cycle: nil) - attr_name ||= :iterator - instance = build(get, position: position, cycle: cycle) - receiver.public_send "#{attr_name}=", instance - end - - def next - logger.trace { "Getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" } - - resupply(batch) - - event_data = batch[batch_position] - - logger.debug { "Finished getting next event data (Batch Length: #{batch.nil? ? '' : batch.length}, Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" } - logger.debug(tags: [:data, :event_data]) { "Event Data: #{event_data.pretty_inspect}" } - - advance_positions - - event_data - end - - def resupply(batch) - logger.trace { "Resupplying batch" } - - if batch.nil? - batch_log_text = "Batch: #{batch.inspect}" - else - batch_log_text = "Batch Length: #{batch.length}" - end - - if batch.nil? || batch_position == batch.length - logger.debug { "Current batch is depleted (#{batch_log_text}, Batch Position: #{batch_position})" } - - batch = get_batch - reset(batch) - else - logger.debug { "Current batch not depleted (#{batch_log_text}, Batch Position: #{batch_position})" } - end - - logger.debug { "Finished resupplying batch" } - end - - def reset(batch) - logger.trace { "Resetting batch" } - - self.batch = batch - self.batch_position = 0 - - logger.debug { "Reset batch" } - logger.debug(tags: [:data, :batch]) { "Batch: #{batch.pretty_inspect}" } - logger.debug(tags: [:data, :batch]) { "Batch Position: #{batch_position.inspect}" } - end - - def get_batch - logger.trace "Getting batch" - - batch = nil - cycle.() do - batch = get.(position: stream_offset) - end - - logger.debug { "Finished getting batch (Count: #{batch.length})" } - - batch - end - - def advance_positions - logger.trace { "Advancing positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" } - self.batch_position += 1 - self.stream_offset += 1 - logger.debug { "Advanced positions (Batch Position: #{batch_position}, Stream Offset: #{stream_offset})" } - end - end - end -end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 51831d9..5e902a5 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -1,72 +1,9 @@ module EventSource module Postgres - class Read - class Error < RuntimeError; end - - include Log::Dependency - - initializer :stream - - dependency :iterator, Iterator - - def self.build(stream_name, position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) - stream = Stream.new(stream_name) - - if cycle.nil? - cycle = Cycle.build(delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds) - end - - cycle ||= Cycle.build(delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds) - - new(stream).tap do |instance| - get = instance.build_get(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) - Iterator.configure instance, get, position: position, cycle: cycle - end - end - - ## TODO will need to be abstract + class Read < EventSource::Read def build_get(stream, batch_size: nil, precedence: nil, partition: nil, session: nil) Get.build(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) end - - def self.call(stream_name, position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil, &action) - instance = build(stream_name, position: position, batch_size: batch_size, precedence: precedence, partition: partition, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) - instance.(&action) - end - - def self.configure(receiver, stream_name, attr_name: nil, position: nil, batch_size: nil, precedence: nil, partition: nil, delay_milliseconds: nil, timeout_milliseconds: nil, cycle: nil, session: nil) - attr_name ||= :reader - instance = build(stream_name, position: position, batch_size: batch_size, precedence: precedence, partition: partition, delay_milliseconds: delay_milliseconds, timeout_milliseconds: timeout_milliseconds, cycle: cycle, session: session) - receiver.public_send "#{attr_name}=", instance - end - - def call(&action) - if action.nil? - error_message = "Reader must be actuated with a block" - logger.error error_message - raise Error, error_message - end - - enumerate_event_data(&action) - - return AsyncInvocation::Incorrect - end - - def enumerate_event_data(&action) - logger.trace { "Reading (Stream Name: #{stream.name}, Category: #{stream.category?})" } - - event_data = nil - - loop do - event_data = iterator.next - - break if event_data.nil? - - action.(event_data) - end - - logger.debug { "Finished reading (Stream Name: #{stream.name}, Category: #{stream.category?})" } - end end end end From c73751e41b087cde86cc88ce38a1a83d0a4b5660 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 17:02:32 -0500 Subject: [PATCH 075/591] Read base class in now a module --- lib/event_source/postgres/read.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 5e902a5..bf11d58 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -1,7 +1,9 @@ module EventSource module Postgres - class Read < EventSource::Read - def build_get(stream, batch_size: nil, precedence: nil, partition: nil, session: nil) + class Read + include EventSource::Read + + def self.build_get(stream, batch_size: nil, precedence: nil, partition: nil, session: nil) Get.build(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) end end From 9469674d6456219c962d1a289e16aa2a2fa9af69 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 17:12:59 -0500 Subject: [PATCH 076/591] AsyncInvocation is removed --- event_source-postgres.gemspec | 2 -- lib/event_source/postgres.rb | 1 - 2 files changed, 3 deletions(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 912f76f..b513582 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -19,11 +19,9 @@ Gem::Specification.new do |s| s.bindir = 'bin' s.add_runtime_dependency 'event_source' - s.add_runtime_dependency 'log' s.add_runtime_dependency 'cycle' s.add_runtime_dependency 'settings' - s.add_runtime_dependency 'async_invocation' s.add_runtime_dependency 'pg' diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index a2d1d91..c98af31 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -6,7 +6,6 @@ require 'log' require 'telemetry' require 'settings'; Settings.activate -require 'async_invocation' require 'event_source/postgres/log' From 52b9ea88cd6db318143f29750710d8d7756eb15c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 17:16:49 -0500 Subject: [PATCH 077/591] Package version increased from 0.3.0.0 to 0.4.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index b513582..c536b0e 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.3.0.0' + s.version = '0.4.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 538d58a1a989852c34dbcd3d54367915f9f2e4d2 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 18:15:26 -0500 Subject: [PATCH 078/591] Execution is delegated to the session --- lib/event_source/postgres/get.rb | 2 +- lib/event_source/postgres/put.rb | 4 ++-- lib/event_source/postgres/session.rb | 8 ++++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 8b19d03..195b2b6 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -45,7 +45,7 @@ def get_records(stream, position) select_statement = SelectStatement.build(stream, offset: position, batch_size: batch_size, precedence: precedence, partition: partition) - records = session.connection.exec(select_statement.sql) + records = session.execute(select_statement.sql) logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" } diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index b54a1a2..fe91ce8 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -77,7 +77,7 @@ def insert_event(stream_name, type, data, metadata, expected_version) def execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version) logger.trace { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" } - sql_args = [ + params = [ stream_name, type, serialized_data, @@ -87,7 +87,7 @@ def execute_query(stream_name, type, serialized_data, serialized_metadata, expec ] begin - records = session.connection.exec_params(self.class.statement, sql_args) + records = session.execute(self.class.statement, params) rescue PG::RaiseException => e raise_error e end diff --git a/lib/event_source/postgres/session.rb b/lib/event_source/postgres/session.rb index 3b5a6b2..c3f3b85 100644 --- a/lib/event_source/postgres/session.rb +++ b/lib/event_source/postgres/session.rb @@ -87,6 +87,14 @@ def settings settings end + def execute(statement, params=nil) + if params.nil? + connection.exec(statement) + else + connection.exec_params(statement, params) + end + end + def self.logger @logger ||= Log.get self end From 71f832ae7b5f15e5870c515a8631afe2ae2fe546 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 18:49:13 -0500 Subject: [PATCH 079/591] Missing block error test is moved to EventSource --- test/automated/read/missing_block_error.rb | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 test/automated/read/missing_block_error.rb diff --git a/test/automated/read/missing_block_error.rb b/test/automated/read/missing_block_error.rb deleted file mode 100644 index f69ffa0..0000000 --- a/test/automated/read/missing_block_error.rb +++ /dev/null @@ -1,11 +0,0 @@ -require_relative '../automated_init' - -context "Read" do - context "No block supplied" do - test "Is incorrect" do - assert proc { Read.(stream_name: 'some_stream_name') } do - raises_error? Read::Error - end - end - end -end From 1327bf2f341060ebddca7bb03fa6d3587c8934fe Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 18:51:02 -0500 Subject: [PATCH 080/591] Testing of integration of Get and Iterator are a concern of Get --- test/automated/{ => get}/iterator/next.rb | 2 +- test/automated/{ => get}/iterator/no_further_event_data.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename test/automated/{ => get}/iterator/next.rb (90%) rename test/automated/{ => get}/iterator/no_further_event_data.rb (88%) diff --git a/test/automated/iterator/next.rb b/test/automated/get/iterator/next.rb similarity index 90% rename from test/automated/iterator/next.rb rename to test/automated/get/iterator/next.rb index 2bcaefd..471d759 100644 --- a/test/automated/iterator/next.rb +++ b/test/automated/get/iterator/next.rb @@ -1,4 +1,4 @@ -require_relative '../automated_init' +require_relative '../../automated_init' context "Iterator" do context "Next" do diff --git a/test/automated/iterator/no_further_event_data.rb b/test/automated/get/iterator/no_further_event_data.rb similarity index 88% rename from test/automated/iterator/no_further_event_data.rb rename to test/automated/get/iterator/no_further_event_data.rb index 599e9a5..b0b7325 100644 --- a/test/automated/iterator/no_further_event_data.rb +++ b/test/automated/get/iterator/no_further_event_data.rb @@ -1,4 +1,4 @@ -require_relative '../automated_init' +require_relative '../../automated_init' context "Iterator" do context "No further event data" do From fe22a2e870885d9a9c7210d304258d7f1bdbace4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 19:04:38 -0500 Subject: [PATCH 081/591] Synchronous invocation error test is moved to EventSource --- test/automated/read/synchronous_result.rb | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 test/automated/read/synchronous_result.rb diff --git a/test/automated/read/synchronous_result.rb b/test/automated/read/synchronous_result.rb deleted file mode 100644 index 8687ea5..0000000 --- a/test/automated/read/synchronous_result.rb +++ /dev/null @@ -1,11 +0,0 @@ -require_relative '../automated_init' - -context "Read" do - context "Synchronous Result" do - res = Read.('some_stream_name') { } - - test "Returns a result that fails if actuated" do - assert(res == AsyncInvocation::Incorrect) - end - end -end From c1217338f962662c0ec6c4950a293dbb1eac64f6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 19:43:28 -0500 Subject: [PATCH 082/591] Package version increased from 0.4.0.0 to 0.4.0.1 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index c536b0e..3e659a8 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.4.0.0' + s.version = '0.4.0.1' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From aebcf2f1e93e20daee4622ced5b9b49d8f697300 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 20:23:52 -0500 Subject: [PATCH 083/591] Write can be configured --- lib/event_source/postgres/write.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index ce4a407..df28c16 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -11,6 +11,12 @@ def self.build(partition: nil, session: nil) instance end + def self.configure(receiver, partition: nil, session: nil, attr_name: nil) + attr_name ||= :writer + instance = build(partition: partition, session: session) + receiver.public_send "#{attr_name}=", instance + end + def configure(partition: nil, session: nil) Put.configure(self, partition: partition, session: session) end From fc5cf15b74824cbcc0005e12601a14ea41f0c68b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 1 Nov 2016 20:24:31 -0500 Subject: [PATCH 084/591] Package version is increased from 0.4.0.1 to 0.4.1.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 3e659a8..8e7492a 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.4.0.1' + s.version = '0.4.1.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 9c6c0b70c8a6786762623b5e97d6720fb1e87252 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 2 Nov 2016 00:45:13 -0500 Subject: [PATCH 085/591] Write is generalized --- lib/event_source/postgres/write.rb | 63 ++---------------------------- 1 file changed, 3 insertions(+), 60 deletions(-) diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index df28c16..0775961 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -1,67 +1,10 @@ module EventSource module Postgres class Write - include Log::Dependency + include EventSource::Write - dependency :put, Put - - def self.build(partition: nil, session: nil) - instance = new - instance.configure(partition: partition, session: session) - instance - end - - def self.configure(receiver, partition: nil, session: nil, attr_name: nil) - attr_name ||= :writer - instance = build(partition: partition, session: session) - receiver.public_send "#{attr_name}=", instance - end - - def configure(partition: nil, session: nil) - Put.configure(self, partition: partition, session: session) - end - - def self.call(event_data, stream_name, expected_version: nil, partition: nil, session: nil) - instance = build(partition: partition, session: session) - instance.(event_data, stream_name, expected_version: expected_version) - end - - def call(event_data, stream_name, expected_version: nil) - logger.trace { "Writing event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :event_data]) { event_data.pretty_inspect } - - batch = Array(event_data) - position = write_batch(batch, stream_name, expected_version: expected_version) - - logger.debug { "Wrote event data (Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" } - logger.debug(tags: [:data, :event_data]) { event_data.pretty_inspect } - - position - end - - def write_batch(batch, stream_name, expected_version: nil) - logger.trace { "Writing batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } - - last_position = nil - put.session.connection.transaction do - batch.each do |event_data| - last_position = write(event_data, stream_name, expected_version: expected_version) - end - end - - logger.debug { "Wrote batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } - - last_position - end - - def write(event_data, stream_name, expected_version: nil) - logger.trace { "Writing event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :event_data]) { event_data.pretty_inspect } - - put.(event_data, stream_name, expected_version: expected_version).tap do - logger.debug { "Wrote event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } - logger.debug(tags: [:data, :event_data]) { event_data.pretty_inspect } - end + def self.build_put(partition: nil, session: nil) + Put.build(partition: partition, session: session) end end end From 0314e35c1217474b3f438679c31cf16d19a5183f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 3 Nov 2016 19:32:55 -0500 Subject: [PATCH 086/591] Consistency in use of configuration protocol --- lib/event_source/postgres/read.rb | 4 ++-- lib/event_source/postgres/write.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index bf11d58..434479b 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -3,8 +3,8 @@ module Postgres class Read include EventSource::Read - def self.build_get(stream, batch_size: nil, precedence: nil, partition: nil, session: nil) - Get.build(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) + def configure(stream, batch_size: nil, precedence: nil, partition: nil, session: nil) + Get.configure(self, stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) end end end diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index 0775961..d2b4d19 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -3,8 +3,8 @@ module Postgres class Write include EventSource::Write - def self.build_put(partition: nil, session: nil) - Put.build(partition: partition, session: session) + def configure(partition: nil, session: nil) + Put.configure(self, partition: partition, session: session) end end end From dba95897e3d8196ecc179d03f8d1c9c72a83fc60 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 4 Nov 2016 00:20:27 -0500 Subject: [PATCH 087/591] Convenience interfaces operate in terms of stream name rather than stream --- lib/event_source/postgres/controls/put.rb | 2 +- lib/event_source/postgres/get.rb | 11 ++++++----- test/automated/get/batch_size.rb | 4 ++-- test/automated/get/category.rb | 4 ++-- test/automated/get/get.rb | 8 ++++---- test/automated/get/iterator/next.rb | 4 ++-- test/automated/get/iterator/no_further_event_data.rb | 4 ++-- test/automated/get/no_events.rb | 4 ++-- test/automated/get/precedence.rb | 6 +++--- test/automated/put/category_as_stream_name.rb | 4 +--- test/automated/put/partition.rb | 6 +++--- test/automated/put/put.rb | 8 ++++---- test/automated/read/partition.rb | 4 ++-- test/automated/read/read.rb | 4 ++-- test/automated/read/starting_position.rb | 4 ++-- test/automated/write/batch.rb | 6 +++--- test/automated/write/write.rb | 6 +++--- 17 files changed, 44 insertions(+), 45 deletions(-) diff --git a/lib/event_source/postgres/controls/put.rb b/lib/event_source/postgres/controls/put.rb index 56bc1a4..4df9c11 100644 --- a/lib/event_source/postgres/controls/put.rb +++ b/lib/event_source/postgres/controls/put.rb @@ -12,7 +12,7 @@ def self.call(instances: nil, stream_name: nil, event: nil, category: nil, parti EventSource::Postgres::Put.(event, stream_name, partition: partition) end - EventSource::Stream.new(stream_name) + stream_name end end end diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 195b2b6..9d6cc47 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -7,20 +7,21 @@ class Get dependency :session, Session - def self.build(stream, batch_size: nil, precedence: nil, partition: nil, session: nil) + def self.build(stream_name, batch_size: nil, precedence: nil, partition: nil, session: nil) + stream = Stream.new(stream_name) new(stream, batch_size, precedence, partition).tap do |instance| instance.configure(session: session) end end - def self.configure(receiver, stream, attr_name: nil, position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) + def self.configure(receiver, stream_name, attr_name: nil, position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) attr_name ||= :get - instance = build(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) + instance = build(stream_name, batch_size: batch_size, precedence: precedence, partition: partition, session: session) receiver.public_send "#{attr_name}=", instance end - def self.call(stream, position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) - instance = build(stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) + def self.call(stream_name, position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) + instance = build(stream_name, batch_size: batch_size, precedence: precedence, partition: partition, session: session) instance.(position: position) end diff --git a/test/automated/get/batch_size.rb b/test/automated/get/batch_size.rb index 3d3de3c..3ba5bd5 100644 --- a/test/automated/get/batch_size.rb +++ b/test/automated/get/batch_size.rb @@ -2,9 +2,9 @@ context "Get" do context "Batch Size" do - stream = Controls::Put.(instances: 3) + stream_name = Controls::Put.(instances: 3) - events = Get.(stream, batch_size: 2) + events = Get.(stream_name, batch_size: 2) number_of_events = events.length diff --git a/test/automated/get/category.rb b/test/automated/get/category.rb index ec8c9fc..f5db024 100644 --- a/test/automated/get/category.rb +++ b/test/automated/get/category.rb @@ -4,9 +4,9 @@ context "Category" do category = Controls::Category.example - stream = Controls::Put.(instances: 2, category: category) + stream_name = Controls::Put.(instances: 2, category: category) - events = Get.(stream) + events = Get.(stream_name) test "Number of events retrieved is the number written to the category" do number_of_events = events.length diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index cdbfa53..c67a3e9 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -1,12 +1,12 @@ require_relative '../automated_init' context "Put and Get" do - stream = Controls::Stream.example + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example - written_position = Put.(write_event, stream.name) + written_position = Put.(write_event, stream_name) - read_event = Get.(stream, position: written_position).first + read_event = Get.(stream_name, position: written_position).first context "Got the event that was written" do test "Type" do @@ -22,7 +22,7 @@ end test "Stream Name" do - assert(read_event.stream_name == stream.name) + assert(read_event.stream_name == stream_name) end test "Position" do diff --git a/test/automated/get/iterator/next.rb b/test/automated/get/iterator/next.rb index 471d759..cb35b49 100644 --- a/test/automated/get/iterator/next.rb +++ b/test/automated/get/iterator/next.rb @@ -2,9 +2,9 @@ context "Iterator" do context "Next" do - stream = Controls::Put.(instances: 2) + stream_name = Controls::Put.(instances: 2) - get = Get.build(stream, batch_size: 1) + get = Get.build(stream_name, batch_size: 1) iterator = Iterator.build(get) diff --git a/test/automated/get/iterator/no_further_event_data.rb b/test/automated/get/iterator/no_further_event_data.rb index b0b7325..b9effe6 100644 --- a/test/automated/get/iterator/no_further_event_data.rb +++ b/test/automated/get/iterator/no_further_event_data.rb @@ -2,9 +2,9 @@ context "Iterator" do context "No further event data" do - stream = Controls::Put.(instances: 2) + stream_name = Controls::Put.(instances: 2) - get = Get.build(stream) + get = Get.build(stream_name) iterator = Iterator.build(get) diff --git a/test/automated/get/no_events.rb b/test/automated/get/no_events.rb index ba733c2..308d031 100644 --- a/test/automated/get/no_events.rb +++ b/test/automated/get/no_events.rb @@ -2,9 +2,9 @@ context "Get" do context "No Events" do - stream = Controls::Stream.example + stream_name = Controls::StreamName.example - batch = Get.(stream) + batch = Get.(stream_name) test "Empty array" do assert(batch == []) diff --git a/test/automated/get/precedence.rb b/test/automated/get/precedence.rb index 2ba047e..fa93410 100644 --- a/test/automated/get/precedence.rb +++ b/test/automated/get/precedence.rb @@ -2,10 +2,10 @@ context "Get" do context "Precedence" do - stream = Controls::Put.(instances: 3) + stream_name = Controls::Put.(instances: 3) context "Ascending" do - events = Get.(stream, precedence: :asc) + events = Get.(stream_name, precedence: :asc) first_event_postition = events.first.position @@ -15,7 +15,7 @@ end context "Descending" do - events = Get.(stream, precedence: :desc) + events = Get.(stream_name, precedence: :desc) first_event_postition = events.first.position diff --git a/test/automated/put/category_as_stream_name.rb b/test/automated/put/category_as_stream_name.rb index 0588ee7..9771204 100644 --- a/test/automated/put/category_as_stream_name.rb +++ b/test/automated/put/category_as_stream_name.rb @@ -7,9 +7,7 @@ Put.(write_event, category) - stream = Stream.new(category) - - read_event = Get.(stream).first + read_event = Get.(category).first test "Writes the category name as the stream name" do assert(read_event.stream_name == category) diff --git a/test/automated/put/partition.rb b/test/automated/put/partition.rb index 0879223..3b2d8df 100644 --- a/test/automated/put/partition.rb +++ b/test/automated/put/partition.rb @@ -1,12 +1,12 @@ require_relative '../automated_init' context "Put and Get to another table name" do - stream = Controls::Stream.example + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example - written_position = Put.(write_event, stream.name, partition: Controls::Partition.example) + written_position = Put.(write_event, stream_name, partition: Controls::Partition.example) - read_event = Get.(stream, partition: Controls::Partition.example, position: written_position).first + read_event = Get.(stream_name, partition: Controls::Partition.example, position: written_position).first test "Got the event that was written" do assert(read_event.position == written_position) diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index cdbfa53..c67a3e9 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -1,12 +1,12 @@ require_relative '../automated_init' context "Put and Get" do - stream = Controls::Stream.example + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example - written_position = Put.(write_event, stream.name) + written_position = Put.(write_event, stream_name) - read_event = Get.(stream, position: written_position).first + read_event = Get.(stream_name, position: written_position).first context "Got the event that was written" do test "Type" do @@ -22,7 +22,7 @@ end test "Stream Name" do - assert(read_event.stream_name == stream.name) + assert(read_event.stream_name == stream_name) end test "Position" do diff --git a/test/automated/read/partition.rb b/test/automated/read/partition.rb index 8f32341..f3be80c 100644 --- a/test/automated/read/partition.rb +++ b/test/automated/read/partition.rb @@ -3,11 +3,11 @@ context "Read" do partition = Controls::Partition.example - stream = Controls::Put.(instances: 2, partition: partition) + stream_name = Controls::Put.(instances: 2, partition: partition) batch = [] - Read.(stream.name, partition: partition, batch_size: 1) do |event_data| + Read.(stream_name, partition: partition, batch_size: 1) do |event_data| batch << event_data end diff --git a/test/automated/read/read.rb b/test/automated/read/read.rb index ee0c8b6..5b77155 100644 --- a/test/automated/read/read.rb +++ b/test/automated/read/read.rb @@ -1,11 +1,11 @@ require_relative '../automated_init' context "Read" do - stream = Controls::Put.(instances: 2) + stream_name = Controls::Put.(instances: 2) batch = [] - Read.(stream.name, batch_size: 1) do |event_data| + Read.(stream_name, batch_size: 1) do |event_data| batch << event_data end diff --git a/test/automated/read/starting_position.rb b/test/automated/read/starting_position.rb index a0fd88b..1884c6a 100644 --- a/test/automated/read/starting_position.rb +++ b/test/automated/read/starting_position.rb @@ -2,11 +2,11 @@ context "Read" do context "Starting Position" do - stream = Controls::Put.(instances: 2) + stream_name = Controls::Put.(instances: 2) batch = [] - Read.(stream.name, position: 1, batch_size: 1) do |event_data| + Read.(stream_name, position: 1, batch_size: 1) do |event_data| batch << event_data end diff --git a/test/automated/write/batch.rb b/test/automated/write/batch.rb index 31cff00..68327dc 100644 --- a/test/automated/write/batch.rb +++ b/test/automated/write/batch.rb @@ -1,14 +1,14 @@ require_relative '../automated_init' context "Write" do - stream = Controls::Stream.example + stream_name = Controls::StreamName.example write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) batch = [write_event_1, write_event_2] - last_written_position = Write.(batch, stream.name) + last_written_position = Write.(batch, stream_name) test "Last written position" do assert(last_written_position == 1) @@ -16,7 +16,7 @@ context "Individual Events are Written" do 2.times do |i| - read_event = Get.(stream, position: i, batch_size: 1).first + read_event = Get.(stream_name, position: i, batch_size: 1).first test "Event #{i + 1}" do assert(read_event.data[:attribute] == "value_#{i + 1}") diff --git a/test/automated/write/write.rb b/test/automated/write/write.rb index 7c59e35..1fcbef5 100644 --- a/test/automated/write/write.rb +++ b/test/automated/write/write.rb @@ -1,12 +1,12 @@ require_relative '../automated_init' context "Write" do - stream = Controls::Stream.example + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example - written_position = Write.(write_event, stream.name) + written_position = Write.(write_event, stream_name) - read_event = Get.(stream, position: written_position).first + read_event = Get.(stream_name, position: written_position).first test "Got the event that was written" do assert(read_event.position == written_position) From 65b65028023bc2269042492811407a627b829439 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 4 Nov 2016 01:10:19 -0500 Subject: [PATCH 088/591] Info level logging on conclusive operations --- lib/event_source/postgres/get.rb | 3 ++- lib/event_source/postgres/put.rb | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 9d6cc47..9e4291c 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -36,7 +36,8 @@ def call(position: nil) events = convert(records) - logger.debug { "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" } + logger.info { "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" } + logger.info(tags: [:data, :event_data]) { events.pretty_inspect } events end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index fe91ce8..fe90313 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -39,12 +39,10 @@ def call(write_event, stream_name, expected_version: nil) type, data, metadata = destructure_event(write_event) expected_version = canonize_expected_version(expected_version) - position = insert_event(stream_name, type, data, metadata, expected_version) - - logger.debug { "Put event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } - logger.debug(tags: [:data, :event_data]) { write_event.pretty_inspect } - - position + insert_event(stream_name, type, data, metadata, expected_version).tap do + logger.info { "Put event data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } + logger.info(tags: [:data, :event_data]) { write_event.pretty_inspect } + end end def destructure_event(write_event) From c30d17ee1fe2a920a4f49758e2fd548433647186 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 4 Nov 2016 10:21:57 -0500 Subject: [PATCH 089/591] Value interpolated into log message is available --- lib/event_source/postgres/put.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index fe90313..26357a1 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -39,7 +39,7 @@ def call(write_event, stream_name, expected_version: nil) type, data, metadata = destructure_event(write_event) expected_version = canonize_expected_version(expected_version) - insert_event(stream_name, type, data, metadata, expected_version).tap do + insert_event(stream_name, type, data, metadata, expected_version).tap do |position| logger.info { "Put event data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } logger.info(tags: [:data, :event_data]) { write_event.pretty_inspect } end From 98a7ea0cbe20796931310867ba7f5c52aabf04e2 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 4 Nov 2016 17:28:35 -0500 Subject: [PATCH 090/591] Transaction is delegated to session --- lib/event_source/postgres/session.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/event_source/postgres/session.rb b/lib/event_source/postgres/session.rb index c3f3b85..3d4854b 100644 --- a/lib/event_source/postgres/session.rb +++ b/lib/event_source/postgres/session.rb @@ -95,6 +95,10 @@ def execute(statement, params=nil) end end + def transaction(&blk) + connection.transaction(&blk) + end + def self.logger @logger ||= Log.get self end From f2f182ac05947613828ad661622b3cb07c8e1b8b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 4 Nov 2016 22:57:54 -0500 Subject: [PATCH 091/591] Expected version is supported with writing a batch --- lib/event_source/postgres/put.rb | 8 ++------ test/automated/write/batch.rb | 26 ------------------------- test/automated/write/batch/write.rb | 28 +++++++++++++++++++++++++++ test/automated/write/message/write.rb | 16 +++++++++++++++ test/automated/write/write.rb | 14 -------------- 5 files changed, 46 insertions(+), 46 deletions(-) delete mode 100644 test/automated/write/batch.rb create mode 100644 test/automated/write/batch/write.rb create mode 100644 test/automated/write/message/write.rb delete mode 100644 test/automated/write/write.rb diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 26357a1..fa81238 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -37,7 +37,7 @@ def call(write_event, stream_name, expected_version: nil) logger.trace(tags: [:data, :event_data]) { write_event.pretty_inspect } type, data, metadata = destructure_event(write_event) - expected_version = canonize_expected_version(expected_version) + expected_version = ExpectedVersion.canonize(expected_version) insert_event(stream_name, type, data, metadata, expected_version).tap do |position| logger.info { "Put event data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } @@ -58,11 +58,7 @@ def destructure_event(write_event) def canonize_expected_version(expected_version) return expected_version unless expected_version == NoStream.name - - logger.trace { "Canonizing expected version (Expected Version: #{expected_version})" } - expected_version = NoStream.version - logger.debug { "Canonized expected version (Expected Version: #{expected_version})" } - expected_version + NoStream.version end def insert_event(stream_name, type, data, metadata, expected_version) diff --git a/test/automated/write/batch.rb b/test/automated/write/batch.rb deleted file mode 100644 index 68327dc..0000000 --- a/test/automated/write/batch.rb +++ /dev/null @@ -1,26 +0,0 @@ -require_relative '../automated_init' - -context "Write" do - stream_name = Controls::StreamName.example - - write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) - write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) - - batch = [write_event_1, write_event_2] - - last_written_position = Write.(batch, stream_name) - - test "Last written position" do - assert(last_written_position == 1) - end - - context "Individual Events are Written" do - 2.times do |i| - read_event = Get.(stream_name, position: i, batch_size: 1).first - - test "Event #{i + 1}" do - assert(read_event.data[:attribute] == "value_#{i + 1}") - end - end - end -end diff --git a/test/automated/write/batch/write.rb b/test/automated/write/batch/write.rb new file mode 100644 index 0000000..bfd37b3 --- /dev/null +++ b/test/automated/write/batch/write.rb @@ -0,0 +1,28 @@ +require_relative '../../automated_init' + +context "Write" do + context "Batch" do + stream_name = Controls::StreamName.example + + write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + + batch = [write_event_1, write_event_2] + + last_written_position = Write.(batch, stream_name) + + test "Last written position" do + assert(last_written_position == 1) + end + + context "Individual Events are Written" do + 2.times do |i| + read_event = Get.(stream_name, position: i, batch_size: 1).first + + test "Event #{i + 1}" do + assert(read_event.data[:attribute] == "value_#{i + 1}") + end + end + end + end +end diff --git a/test/automated/write/message/write.rb b/test/automated/write/message/write.rb new file mode 100644 index 0000000..5f938b8 --- /dev/null +++ b/test/automated/write/message/write.rb @@ -0,0 +1,16 @@ +require_relative '../../automated_init' + +context "Write" do + context "Message" do + stream_name = Controls::StreamName.example + write_event = Controls::EventData::Write.example + + written_position = Write.(write_event, stream_name) + + read_event = Get.(stream_name, position: written_position).first + + test "Got the event that was written" do + assert(read_event.position == written_position) + end + end +end diff --git a/test/automated/write/write.rb b/test/automated/write/write.rb deleted file mode 100644 index 1fcbef5..0000000 --- a/test/automated/write/write.rb +++ /dev/null @@ -1,14 +0,0 @@ -require_relative '../automated_init' - -context "Write" do - stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example - - written_position = Write.(write_event, stream_name) - - read_event = Get.(stream_name, position: written_position).first - - test "Got the event that was written" do - assert(read_event.position == written_position) - end -end From ddf722589a588b4890e595491aea700fd51a71ee Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 4 Nov 2016 23:15:59 -0500 Subject: [PATCH 092/591] Expected version namespace, and error --- lib/event_source/postgres/put.rb | 7 +------ test/automated/put/expected_version_error.rb | 2 +- test/automated/put/no_stream/existing_stream.rb | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index fa81238..c891812 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -56,11 +56,6 @@ def destructure_event(write_event) return type, data, metadata end - def canonize_expected_version(expected_version) - return expected_version unless expected_version == NoStream.name - NoStream.version - end - def insert_event(stream_name, type, data, metadata, expected_version) serialized_data = serialized_data(data) serialized_metadata = serialized_metadata(metadata) @@ -124,7 +119,7 @@ def raise_error(pg_error) error_message = pg_error.message if error_message.include? 'Wrong expected version' error_message.gsub!('ERROR:', '').strip! - raise ExpectedVersionError, error_message + raise ExpectedVersion::Error, error_message end raise pg_error end diff --git a/test/automated/put/expected_version_error.rb b/test/automated/put/expected_version_error.rb index 55e593f..07c7be6 100644 --- a/test/automated/put/expected_version_error.rb +++ b/test/automated/put/expected_version_error.rb @@ -11,7 +11,7 @@ test "Is an error" do assert proc { Put.(write_event, stream_name, expected_version: incorrect_stream_version ) } do - raises_error? EventSource::ExpectedVersionError + raises_error? EventSource::ExpectedVersion::Error end end end diff --git a/test/automated/put/no_stream/existing_stream.rb b/test/automated/put/no_stream/existing_stream.rb index 4b46878..b462969 100644 --- a/test/automated/put/no_stream/existing_stream.rb +++ b/test/automated/put/no_stream/existing_stream.rb @@ -14,7 +14,7 @@ test "Is an error" do assert erroneous do - raises_error? Write::ExpectedVersionError + raises_error? ExpectedVersion::Error end end end From 46a5268efe3d906aa2001e9a4a1490180380ed64 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 5 Nov 2016 00:13:03 -0500 Subject: [PATCH 093/591] Expected version for batch --- lib/event_source/postgres/put.rb | 1 + .../automated/write/batch/expected_version.rb | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 test/automated/write/batch/expected_version.rb diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index c891812..a9be909 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -119,6 +119,7 @@ def raise_error(pg_error) error_message = pg_error.message if error_message.include? 'Wrong expected version' error_message.gsub!('ERROR:', '').strip! + logger.error { error_message } raise ExpectedVersion::Error, error_message end raise pg_error diff --git a/test/automated/write/batch/expected_version.rb b/test/automated/write/batch/expected_version.rb new file mode 100644 index 0000000..590363c --- /dev/null +++ b/test/automated/write/batch/expected_version.rb @@ -0,0 +1,27 @@ +require_relative '../../automated_init' + +context "Write" do + context "Batch" do + stream_name = Controls::StreamName.example + + write_event = Controls::EventData::Write.example + written_position = Write.(write_event, stream_name) + + write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + + batch = [write_event_1, write_event_2] + + Write.(batch, stream_name, expected_version: written_position) + + context "Individual Events are Written" do + 2.times do |i| + read_event = Get.(stream_name, position: i + written_position, batch_size: 1).first + + test "Event #{i + 1}" do + refute(read_event.nil?) + end + end + end + end +end From 83c95964cf8dc28b732ca3d627f602c5ab0559ff Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 5 Nov 2016 01:03:34 -0500 Subject: [PATCH 094/591] Test for writing batch with expected version --- .../automated/write/batch/expected_version.rb | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/automated/write/batch/expected_version.rb b/test/automated/write/batch/expected_version.rb index 590363c..f4bfbbb 100644 --- a/test/automated/write/batch/expected_version.rb +++ b/test/automated/write/batch/expected_version.rb @@ -2,24 +2,26 @@ context "Write" do context "Batch" do - stream_name = Controls::StreamName.example + context "Expected Version" do + stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example - written_position = Write.(write_event, stream_name) + write_event = Controls::EventData::Write.example + written_position = Write.(write_event, stream_name) - write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) - write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) - batch = [write_event_1, write_event_2] + batch = [write_event_1, write_event_2] - Write.(batch, stream_name, expected_version: written_position) + Write.(batch, stream_name, expected_version: written_position) - context "Individual Events are Written" do - 2.times do |i| - read_event = Get.(stream_name, position: i + written_position, batch_size: 1).first + context "Individual Events are Written" do + 2.times do |i| + read_event = Get.(stream_name, position: i + written_position, batch_size: 1).first - test "Event #{i + 1}" do - refute(read_event.nil?) + test "Event #{i + 1}" do + refute(read_event.nil?) + end end end end From 428f4c914bf692574e366861b924ec3c5c3ad0af Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 5 Nov 2016 20:47:24 -0500 Subject: [PATCH 095/591] Test for message not written when expected version error --- test/automated/put/expected_version_error.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/automated/put/expected_version_error.rb b/test/automated/put/expected_version_error.rb index 07c7be6..4f06f14 100644 --- a/test/automated/put/expected_version_error.rb +++ b/test/automated/put/expected_version_error.rb @@ -14,5 +14,13 @@ raises_error? EventSource::ExpectedVersion::Error end end + + context "Event" do + read_event = Get.(stream_name, position: incorrect_stream_version, batch_size: 1).first + + test "Is not written" do + assert(read_event.nil?) + end + end end end From 8a32e6f658354fcd36bb29529151875e79f3e052 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 6 Nov 2016 14:59:28 -0600 Subject: [PATCH 096/591] Package version is increased from 0.4.1.0 to 0.5.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 8e7492a..6635aed 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.4.1.0' + s.version = '0.5.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 70945e5ce59745b53ff67d1076bb54ec2ccdd3ec Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 9 Nov 2016 20:09:51 -0600 Subject: [PATCH 097/591] Database tools are not distributable binaries. They're developer tools. --- event_source-postgres.gemspec | 3 --- {bin => tools}/clear-events-table | 0 {bin => tools}/create-other-events-table | 0 {bin => tools}/install-event-source-database | 0 {bin => tools}/list-events | 0 {bin => tools}/uninstall-event-source-database | 0 6 files changed, 3 deletions(-) rename {bin => tools}/clear-events-table (100%) rename {bin => tools}/create-other-events-table (100%) rename {bin => tools}/install-event-source-database (100%) rename {bin => tools}/list-events (100%) rename {bin => tools}/uninstall-event-source-database (100%) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 6635aed..841b1e6 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -15,9 +15,6 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.required_ruby_version = '>= 2.2.3' - s.executables = ['install-event-source-database', 'uninstall-event-source-database'] - s.bindir = 'bin' - s.add_runtime_dependency 'event_source' s.add_runtime_dependency 'log' s.add_runtime_dependency 'cycle' diff --git a/bin/clear-events-table b/tools/clear-events-table similarity index 100% rename from bin/clear-events-table rename to tools/clear-events-table diff --git a/bin/create-other-events-table b/tools/create-other-events-table similarity index 100% rename from bin/create-other-events-table rename to tools/create-other-events-table diff --git a/bin/install-event-source-database b/tools/install-event-source-database similarity index 100% rename from bin/install-event-source-database rename to tools/install-event-source-database diff --git a/bin/list-events b/tools/list-events similarity index 100% rename from bin/list-events rename to tools/list-events diff --git a/bin/uninstall-event-source-database b/tools/uninstall-event-source-database similarity index 100% rename from bin/uninstall-event-source-database rename to tools/uninstall-event-source-database From c72940615fd20e65059a2d72fc7c09e803174b2c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 9 Nov 2016 20:10:57 -0600 Subject: [PATCH 098/591] Package version is increased from 0.5.0.0 to 0.6.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 841b1e6..cc421af 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.5.0.0' + s.version = '0.6.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 8d1620158004c79d70d0b324d35fde1edfa7dde4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 17 Nov 2016 15:08:40 -0600 Subject: [PATCH 099/591] Put receives partition as an instance actuator argument --- lib/event_source/postgres/put.rb | 34 +++++++++++++------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index a9be909..5fc2879 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -5,14 +5,8 @@ class Put dependency :session, Session - def partition - @partition ||= Defaults.partition - end - attr_writer :partition - - def self.build(partition: nil, session: nil) + def self.build(session: nil) new.tap do |instance| - instance.partition = partition instance.configure(session: session) end end @@ -21,26 +15,26 @@ def configure(session: nil) Session.configure(self, session: session) end - def self.configure(receiver, partition: nil, session: nil, attr_name: nil) + def self.configure(receiver, session: nil, attr_name: nil) attr_name ||= :put - instance = build(partition: partition, session: session) + instance = build(session: session) receiver.public_send "#{attr_name}=", instance end def self.call(write_event, stream_name, expected_version: nil, partition: nil, session: nil) - instance = build(partition: partition, session: session) - instance.(write_event, stream_name, expected_version: expected_version) + instance = build(session: session) + instance.(write_event, stream_name, expected_version: expected_version, partition: partition) end - def call(write_event, stream_name, expected_version: nil) - logger.trace { "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } + def call(write_event, stream_name, expected_version: nil, partition: nil) + logger.trace { "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect}, Partition: #{partition.inspect})" } logger.trace(tags: [:data, :event_data]) { write_event.pretty_inspect } type, data, metadata = destructure_event(write_event) expected_version = ExpectedVersion.canonize(expected_version) - insert_event(stream_name, type, data, metadata, expected_version).tap do |position| - logger.info { "Put event data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } + insert_event(stream_name, type, data, metadata, expected_version, partition).tap do |position| + logger.info { "Put event data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect}, Partition: #{partition.inspect})" } logger.info(tags: [:data, :event_data]) { write_event.pretty_inspect } end end @@ -56,15 +50,15 @@ def destructure_event(write_event) return type, data, metadata end - def insert_event(stream_name, type, data, metadata, expected_version) + def insert_event(stream_name, type, data, metadata, expected_version, partition) serialized_data = serialized_data(data) serialized_metadata = serialized_metadata(metadata) - records = execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version) + records = execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version, partition) position(records) end - def execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version) - logger.trace { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" } + def execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version, partition) + logger.trace { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, Partition: #{partition.inspect})" } params = [ stream_name, @@ -81,7 +75,7 @@ def execute_query(stream_name, type, serialized_data, serialized_metadata, expec raise_error e end - logger.debug { "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" } + logger.debug { "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, Partition: #{partition.inspect})" } records end From 3e6f1b47feaf5b4e90780e80aa7d7bd2795d8256 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 17 Nov 2016 15:50:55 -0600 Subject: [PATCH 100/591] Write receives partition as an instance actuator arg --- lib/event_source/postgres/write.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index d2b4d19..e2250b2 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -3,8 +3,8 @@ module Postgres class Write include EventSource::Write - def configure(partition: nil, session: nil) - Put.configure(self, partition: partition, session: session) + def configure(session: nil) + Put.configure(self, session: session) end end end From 242e4d0b8241278ab47eb42e68c2e838c157ac0f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 17 Nov 2016 16:31:41 -0600 Subject: [PATCH 101/591] Write to partition is tested --- test/automated/write/batch/partition.rb | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 test/automated/write/batch/partition.rb diff --git a/test/automated/write/batch/partition.rb b/test/automated/write/batch/partition.rb new file mode 100644 index 0000000..7bedfe0 --- /dev/null +++ b/test/automated/write/batch/partition.rb @@ -0,0 +1,28 @@ +require_relative '../../automated_init' + +context "Write" do + context "Batch" do + context "Partition" do + stream_name = Controls::StreamName.example + + write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + + batch = [write_event_1, write_event_2] + + partition = Controls::Partition.example + + Write.(batch, stream_name, partition: partition) + + context "Individual Events are Written" do + 2.times do |i| + read_event = Get.(stream_name, partition: partition, position: i, batch_size: 1).first + + test "Event #{i + 1}" do + refute(read_event.nil?) + end + end + end + end + end +end From 98247bdc83c693f97d72b2424a2a1197541c3bad Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 17 Nov 2016 16:40:10 -0600 Subject: [PATCH 102/591] Package version increased from 0.6.0.0 to 0.7.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index cc421af..580afbe 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.6.0.0' + s.version = '0.7.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 4cbe183870c71be295f0fb71b892de17c2ef4819 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 18 Nov 2016 17:28:02 -0600 Subject: [PATCH 103/591] Partition control --- lib/event_source/postgres/controls/partition.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/event_source/postgres/controls/partition.rb b/lib/event_source/postgres/controls/partition.rb index 55440e9..e06030d 100644 --- a/lib/event_source/postgres/controls/partition.rb +++ b/lib/event_source/postgres/controls/partition.rb @@ -1,11 +1,7 @@ module EventSource module Postgres module Controls - module Partition - def self.example - 'other_events' - end - end + Partition = EventSource::Controls::Partition end end end From 1a258feff6193a8ece0f306b62aa7c30817e33f5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 18 Nov 2016 17:28:39 -0600 Subject: [PATCH 104/591] Package version is increased from 0.7.0.0 to 0.7.0.1 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 580afbe..3b162f2 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.7.0.0' + s.version = '0.7.0.1' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 01450b838e786999dd4e7106be9d8c4d63fb0dcf Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 18 Nov 2016 20:41:56 -0600 Subject: [PATCH 105/591] Overly specific use of test controls is generalized --- test/automated/write/batch/partition.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/automated/write/batch/partition.rb b/test/automated/write/batch/partition.rb index 7bedfe0..214f614 100644 --- a/test/automated/write/batch/partition.rb +++ b/test/automated/write/batch/partition.rb @@ -5,8 +5,8 @@ context "Partition" do stream_name = Controls::StreamName.example - write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) - write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + write_event_1 = Controls::EventData::Write.example + write_event_2 = Controls::EventData::Write.example batch = [write_event_1, write_event_2] From 21b9cd57027672bcf82dc33e90f992cca94388a4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 18 Nov 2016 21:58:51 -0600 Subject: [PATCH 106/591] Stream version Postgres function accounts for partition name --- database/functions/stream-version.sql | 16 ++++++++++++++-- database/functions/write-event.sql | 4 ++-- test/automated/write/batch/partition.rb | 6 +++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/database/functions/stream-version.sql b/database/functions/stream-version.sql index 6b88722..c6d68b0 100644 --- a/database/functions/stream-version.sql +++ b/database/functions/stream-version.sql @@ -1,12 +1,24 @@ CREATE OR REPLACE FUNCTION stream_version( - _stream_name varchar + _stream_name varchar, + _partition varchar DEFAULT NULL ) RETURNS int AS $$ DECLARE stream_version int; BEGIN - select max(position) into stream_version from events where stream_name = _stream_name; + if _partition is null then + _partition := 'events'; + end if; + + EXECUTE format('select max(position) ' + 'from %I ' + 'where stream_name = $1', + _partition) + INTO + stream_version + USING + _stream_name; return stream_version; END; diff --git a/database/functions/write-event.sql b/database/functions/write-event.sql index 305e74b..493bb8f 100644 --- a/database/functions/write-event.sql +++ b/database/functions/write-event.sql @@ -13,12 +13,12 @@ DECLARE position int; category varchar; BEGIN - stream_version := stream_version(_stream_name); - if _partition is null then _partition := 'events'; end if; + stream_version := stream_version(_stream_name, _partition); + if stream_version is null then stream_version := -1; end if; diff --git a/test/automated/write/batch/partition.rb b/test/automated/write/batch/partition.rb index 214f614..65c7607 100644 --- a/test/automated/write/batch/partition.rb +++ b/test/automated/write/batch/partition.rb @@ -12,7 +12,11 @@ partition = Controls::Partition.example - Write.(batch, stream_name, partition: partition) + last_written_position = Write.(batch, stream_name, partition: partition) + + test "Last written position" do + assert(last_written_position == 1) + end context "Individual Events are Written" do 2.times do |i| From bb3d31390bd39c672158d9e565c07ca12443c1ff Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 18 Nov 2016 22:00:37 -0600 Subject: [PATCH 107/591] Package version is increased from 0.7.0.1 to 0.7.0.2 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 3b162f2..6a94764 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.7.0.1' + s.version = '0.7.0.2' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 172f536a308f122eb1507c507439088e5c2b6a44 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 18 Nov 2016 22:20:56 -0600 Subject: [PATCH 108/591] Batch partition write test --- test/automated/write/batch/partition.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/automated/write/batch/partition.rb b/test/automated/write/batch/partition.rb index 65c7607..224928b 100644 --- a/test/automated/write/batch/partition.rb +++ b/test/automated/write/batch/partition.rb @@ -5,8 +5,8 @@ context "Partition" do stream_name = Controls::StreamName.example - write_event_1 = Controls::EventData::Write.example - write_event_2 = Controls::EventData::Write.example + write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) batch = [write_event_1, write_event_2] @@ -14,6 +14,7 @@ last_written_position = Write.(batch, stream_name, partition: partition) + test "Last written position" do assert(last_written_position == 1) end @@ -23,7 +24,7 @@ read_event = Get.(stream_name, partition: partition, position: i, batch_size: 1).first test "Event #{i + 1}" do - refute(read_event.nil?) + assert(read_event.data[:attribute] == "value_#{i + 1}") end end end From 6c368022a2165ea9bd38114501947dfc7b2effd5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 18 Nov 2016 22:44:30 -0600 Subject: [PATCH 109/591] Expected version tests are improved --- test/automated/put/category_as_stream_name.rb | 2 +- test/automated/put/expected_version_error.rb | 30 ++++++++++--------- .../put/no_stream/existing_stream.rb | 2 +- test/automated/put/no_stream/no_stream.rb | 2 +- test/automated/put/partition.rb | 16 +++++----- test/automated/put/put.rb | 2 +- test/automated/put/returns_stream_position.rb | 14 +++++---- .../put/stream_position_increases.rb | 16 +++++----- .../automated/write/batch/expected_version.rb | 29 ------------------ test/automated/write/batch/partition.rb | 2 +- test/automated/write/batch/write.rb | 2 +- .../write/message/expected_version_error.rb | 28 +++++++++++++++++ test/automated/write/message/partition.rb | 16 ++++++++++ test/automated/write/message/write.rb | 2 +- 14 files changed, 93 insertions(+), 70 deletions(-) delete mode 100644 test/automated/write/batch/expected_version.rb create mode 100644 test/automated/write/message/expected_version_error.rb create mode 100644 test/automated/write/message/partition.rb diff --git a/test/automated/put/category_as_stream_name.rb b/test/automated/put/category_as_stream_name.rb index 9771204..1e66248 100644 --- a/test/automated/put/category_as_stream_name.rb +++ b/test/automated/put/category_as_stream_name.rb @@ -2,7 +2,7 @@ context "Put" do context "Category as Stream Name" do - category = Controls::Category.example + category = Controls::Category.example(category: 'testPutCategoryAsStreamName') write_event = Controls::EventData::Write.example Put.(write_event, category) diff --git a/test/automated/put/expected_version_error.rb b/test/automated/put/expected_version_error.rb index 4f06f14..83b173f 100644 --- a/test/automated/put/expected_version_error.rb +++ b/test/automated/put/expected_version_error.rb @@ -1,25 +1,27 @@ require_relative '../automated_init' -context "Expected Version" do - context "Expected version does not match the stream version" do - stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example +context "Put" do + context "Expected Version" do + context "Does not match the stream version" do + stream_name = Controls::StreamName.example(category: 'testPutExpectedVersion') + write_event = Controls::EventData::Write.example - position = Put.(write_event, stream_name) + position = Put.(write_event, stream_name) - incorrect_stream_version = position + 1 + incorrect_stream_version = position + 1 - test "Is an error" do - assert proc { Put.(write_event, stream_name, expected_version: incorrect_stream_version ) } do - raises_error? EventSource::ExpectedVersion::Error + test "Is an error" do + assert proc { Put.(write_event, stream_name, expected_version: incorrect_stream_version ) } do + raises_error? EventSource::ExpectedVersion::Error + end end - end - context "Event" do - read_event = Get.(stream_name, position: incorrect_stream_version, batch_size: 1).first + context "Event" do + read_event = Get.(stream_name, position: incorrect_stream_version, batch_size: 1).first - test "Is not written" do - assert(read_event.nil?) + test "Is not written" do + assert(read_event.nil?) + end end end end diff --git a/test/automated/put/no_stream/existing_stream.rb b/test/automated/put/no_stream/existing_stream.rb index b462969..78a28ce 100644 --- a/test/automated/put/no_stream/existing_stream.rb +++ b/test/automated/put/no_stream/existing_stream.rb @@ -3,7 +3,7 @@ context "Put" do context "No Stream" do context "Existing Stream" do - stream_name = Controls::StreamName.example + stream_name = Controls::StreamName.example(category: 'testPutNoStreamExistingStream') write_event_1 = Controls::EventData::Write.example(data: {:some_attribute => 'first'}) write_event_2 = Controls::EventData::Write.example(data: {:some_attribute => 'second'}) diff --git a/test/automated/put/no_stream/no_stream.rb b/test/automated/put/no_stream/no_stream.rb index e4c8b94..55143bd 100644 --- a/test/automated/put/no_stream/no_stream.rb +++ b/test/automated/put/no_stream/no_stream.rb @@ -3,7 +3,7 @@ context "Put" do context "No Stream" do context "For a stream that doesn't exist" do - stream_name = Controls::StreamName.example + stream_name = Controls::StreamName.example(category: 'testPutNoStream') write_event = Controls::EventData::Write.example position = Put.(write_event, stream_name) diff --git a/test/automated/put/partition.rb b/test/automated/put/partition.rb index 3b2d8df..d78ed2b 100644 --- a/test/automated/put/partition.rb +++ b/test/automated/put/partition.rb @@ -1,14 +1,16 @@ require_relative '../automated_init' -context "Put and Get to another table name" do - stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example +context "Put" do + context "Partition" do + stream_name = Controls::StreamName.example(category: 'testPutPartition') + write_event = Controls::EventData::Write.example - written_position = Put.(write_event, stream_name, partition: Controls::Partition.example) + written_position = Put.(write_event, stream_name, partition: Controls::Partition.example) - read_event = Get.(stream_name, partition: Controls::Partition.example, position: written_position).first + read_event = Get.(stream_name, partition: Controls::Partition.example, position: written_position).first - test "Got the event that was written" do - assert(read_event.position == written_position) + test "Got the event that was written" do + assert(read_event.position == written_position) + end end end diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index c67a3e9..288d03b 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -1,7 +1,7 @@ require_relative '../automated_init' context "Put and Get" do - stream_name = Controls::StreamName.example + stream_name = Controls::StreamName.example(category: 'testPut') write_event = Controls::EventData::Write.example written_position = Put.(write_event, stream_name) diff --git a/test/automated/put/returns_stream_position.rb b/test/automated/put/returns_stream_position.rb index af7a158..82f966a 100644 --- a/test/automated/put/returns_stream_position.rb +++ b/test/automated/put/returns_stream_position.rb @@ -1,13 +1,15 @@ require_relative '../automated_init' -context "Write Event Data" do - stream_name = Controls::StreamName.example +context "Put" do + context "Returns Stream Position" do + stream_name = Controls::StreamName.example(category: 'testPutReturnsStreamPosition') - write_event = Controls::EventData::Write.example + write_event = Controls::EventData::Write.example - position = Put.(write_event, stream_name) + position = Put.(write_event, stream_name) - test "Result is stream version" do - refute(position.nil?) + test "Result is stream position" do + refute(position.nil?) + end end end diff --git a/test/automated/put/stream_position_increases.rb b/test/automated/put/stream_position_increases.rb index 9645fba..346f620 100644 --- a/test/automated/put/stream_position_increases.rb +++ b/test/automated/put/stream_position_increases.rb @@ -1,13 +1,15 @@ require_relative '../automated_init' -context "Stream Version Increases with Subsequent Writes" do - stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example +context "Put" do + context "Stream Position Increases with Subsequent Writes" do + stream_name = Controls::StreamName.example(category: 'testPutPositionIncreases') + write_event = Controls::EventData::Write.example - position_1 = Put.(write_event, stream_name) - position_2 = Put.(write_event, stream_name) + position_1 = Put.(write_event, stream_name) + position_2 = Put.(write_event, stream_name) - test "First version is one less than the second version" do - assert(position_1 + 1 == position_2) + test "First version is one less than the second version" do + assert(position_1 + 1 == position_2) + end end end diff --git a/test/automated/write/batch/expected_version.rb b/test/automated/write/batch/expected_version.rb deleted file mode 100644 index f4bfbbb..0000000 --- a/test/automated/write/batch/expected_version.rb +++ /dev/null @@ -1,29 +0,0 @@ -require_relative '../../automated_init' - -context "Write" do - context "Batch" do - context "Expected Version" do - stream_name = Controls::StreamName.example - - write_event = Controls::EventData::Write.example - written_position = Write.(write_event, stream_name) - - write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) - write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) - - batch = [write_event_1, write_event_2] - - Write.(batch, stream_name, expected_version: written_position) - - context "Individual Events are Written" do - 2.times do |i| - read_event = Get.(stream_name, position: i + written_position, batch_size: 1).first - - test "Event #{i + 1}" do - refute(read_event.nil?) - end - end - end - end - end -end diff --git a/test/automated/write/batch/partition.rb b/test/automated/write/batch/partition.rb index 224928b..303640f 100644 --- a/test/automated/write/batch/partition.rb +++ b/test/automated/write/batch/partition.rb @@ -3,7 +3,7 @@ context "Write" do context "Batch" do context "Partition" do - stream_name = Controls::StreamName.example + stream_name = Controls::StreamName.example(category: 'testWriteBatchPartition') write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) diff --git a/test/automated/write/batch/write.rb b/test/automated/write/batch/write.rb index bfd37b3..96e78a3 100644 --- a/test/automated/write/batch/write.rb +++ b/test/automated/write/batch/write.rb @@ -2,7 +2,7 @@ context "Write" do context "Batch" do - stream_name = Controls::StreamName.example + stream_name = Controls::StreamName.example(category: 'testWriteBatch') write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) diff --git a/test/automated/write/message/expected_version_error.rb b/test/automated/write/message/expected_version_error.rb new file mode 100644 index 0000000..a88c998 --- /dev/null +++ b/test/automated/write/message/expected_version_error.rb @@ -0,0 +1,28 @@ +require_relative '../../automated_init' + +context "Write" do + context "Message" do + context "Expected Version Error" do + stream_name = Controls::StreamName.example(category: 'testWriteMessageExpectedVersionError') + write_event = Controls::EventData::Write.example + + position = Write.(write_event, stream_name) + + incorrect_stream_version = position + 1 + + test "Is an error" do + assert proc { Write.(write_event, stream_name, expected_version: incorrect_stream_version ) } do + raises_error? EventSource::ExpectedVersion::Error + end + end + + context "Event" do + read_event = Get.(stream_name, position: incorrect_stream_version, batch_size: 1).first + + test "Is not written" do + assert(read_event.nil?) + end + end + end + end +end diff --git a/test/automated/write/message/partition.rb b/test/automated/write/message/partition.rb new file mode 100644 index 0000000..23b5ff3 --- /dev/null +++ b/test/automated/write/message/partition.rb @@ -0,0 +1,16 @@ +require_relative '../../automated_init' + +context "Write" do + context "Message" do + stream_name = Controls::StreamName.example(category: 'testWriteMessagePartition') + write_event = Controls::EventData::Write.example + + written_position = Write.(write_event, stream_name) + + read_event = Get.(stream_name, position: written_position).first + + test "Got the event that was written" do + assert(read_event.position == written_position) + end + end +end diff --git a/test/automated/write/message/write.rb b/test/automated/write/message/write.rb index 5f938b8..2acd14b 100644 --- a/test/automated/write/message/write.rb +++ b/test/automated/write/message/write.rb @@ -2,7 +2,7 @@ context "Write" do context "Message" do - stream_name = Controls::StreamName.example + stream_name = Controls::StreamName.example(category: 'testWriteMessage') write_event = Controls::EventData::Write.example written_position = Write.(write_event, stream_name) From 1dbd0210b38575aec2215828d1c97af6c61b9054 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 18 Nov 2016 23:14:38 -0600 Subject: [PATCH 110/591] Test coverage is improved --- .../error.rb} | 2 +- .../put/expected_version/expected_version.rb | 23 ++++++++++++ test/automated/put/partition.rb | 6 ++- test/automated/put/put.rb | 6 +-- .../write/batch/expected_version/error.rb | 37 +++++++++++++++++++ .../expected_version/expected_version.rb | 29 +++++++++++++++ test/automated/write/batch/partition.rb | 1 - .../write/event/expected_version/error.rb | 30 +++++++++++++++ .../expected_version/expected_version.rb | 23 ++++++++++++ test/automated/write/event/partition.rb | 20 ++++++++++ test/automated/write/event/write.rb | 17 +++++++++ .../write/message/expected_version_error.rb | 28 -------------- test/automated/write/message/partition.rb | 16 -------- test/automated/write/message/write.rb | 16 -------- 14 files changed, 187 insertions(+), 67 deletions(-) rename test/automated/put/{expected_version_error.rb => expected_version/error.rb} (95%) create mode 100644 test/automated/put/expected_version/expected_version.rb create mode 100644 test/automated/write/batch/expected_version/error.rb create mode 100644 test/automated/write/batch/expected_version/expected_version.rb create mode 100644 test/automated/write/event/expected_version/error.rb create mode 100644 test/automated/write/event/expected_version/expected_version.rb create mode 100644 test/automated/write/event/partition.rb create mode 100644 test/automated/write/event/write.rb delete mode 100644 test/automated/write/message/expected_version_error.rb delete mode 100644 test/automated/write/message/partition.rb delete mode 100644 test/automated/write/message/write.rb diff --git a/test/automated/put/expected_version_error.rb b/test/automated/put/expected_version/error.rb similarity index 95% rename from test/automated/put/expected_version_error.rb rename to test/automated/put/expected_version/error.rb index 83b173f..5c38a64 100644 --- a/test/automated/put/expected_version_error.rb +++ b/test/automated/put/expected_version/error.rb @@ -1,4 +1,4 @@ -require_relative '../automated_init' +require_relative '../../automated_init' context "Put" do context "Expected Version" do diff --git a/test/automated/put/expected_version/expected_version.rb b/test/automated/put/expected_version/expected_version.rb new file mode 100644 index 0000000..4712ad6 --- /dev/null +++ b/test/automated/put/expected_version/expected_version.rb @@ -0,0 +1,23 @@ +require_relative '../../automated_init' + +context "Put" do + context "Event" do + context "Expected Version" do + stream_name = Controls::StreamName.example(category: 'testPutEventExpectedVersion') + + write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + + position = Put.(write_event_1, stream_name) + + write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + + Put.(write_event_2, stream_name, expected_version: position) + + read_event = Get.(stream_name, position: position + 1).first + + test "Got the event that was written" do + assert(read_event.data[:attribute] == 'value_2') + end + end + end +end diff --git a/test/automated/put/partition.rb b/test/automated/put/partition.rb index d78ed2b..2d0fb44 100644 --- a/test/automated/put/partition.rb +++ b/test/automated/put/partition.rb @@ -3,11 +3,13 @@ context "Put" do context "Partition" do stream_name = Controls::StreamName.example(category: 'testPutPartition') + partition = Controls::Partition.example + write_event = Controls::EventData::Write.example - written_position = Put.(write_event, stream_name, partition: Controls::Partition.example) + written_position = Put.(write_event, stream_name, partition: partition) - read_event = Get.(stream_name, partition: Controls::Partition.example, position: written_position).first + read_event = Get.(stream_name, partition: partition, position: written_position).first test "Got the event that was written" do assert(read_event.position == written_position) diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index 288d03b..7b3215b 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -4,9 +4,9 @@ stream_name = Controls::StreamName.example(category: 'testPut') write_event = Controls::EventData::Write.example - written_position = Put.(write_event, stream_name) + position = Put.(write_event, stream_name) - read_event = Get.(stream_name, position: written_position).first + read_event = Get.(stream_name, position: position).first context "Got the event that was written" do test "Type" do @@ -26,7 +26,7 @@ end test "Position" do - assert(read_event.position == written_position) + assert(read_event.position == position) end test "Global Position" do diff --git a/test/automated/write/batch/expected_version/error.rb b/test/automated/write/batch/expected_version/error.rb new file mode 100644 index 0000000..10fffdc --- /dev/null +++ b/test/automated/write/batch/expected_version/error.rb @@ -0,0 +1,37 @@ +require_relative '../../../automated_init' + +context "Write" do + context "Batch" do + context "Expected Version" do + context "Does not match the stream version" do + stream_name = Controls::StreamName.example(category: 'testWriteBatchExpectedVersionError') + write_event = Controls::EventData::Write.example + + position = Write.(write_event, stream_name) + + incorrect_stream_version = position + 1 + + write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + + batch = [write_event_1, write_event_2] + + test "Is an error" do + assert proc { Write.(batch, stream_name, expected_version: incorrect_stream_version ) } do + raises_error? EventSource::ExpectedVersion::Error + end + end + + context "Events" do + 2.times do |i| + read_event = Get.(stream_name, position: incorrect_stream_version + i, batch_size: 1).first + + test "Event #{i + 1} not written" do + assert(read_event.nil?) + end + end + end + end + end + end +end diff --git a/test/automated/write/batch/expected_version/expected_version.rb b/test/automated/write/batch/expected_version/expected_version.rb new file mode 100644 index 0000000..6b4e59f --- /dev/null +++ b/test/automated/write/batch/expected_version/expected_version.rb @@ -0,0 +1,29 @@ +require_relative '../../../automated_init' + +context "Write" do + context "Batch" do + context "Expected Version" do + stream_name = Controls::StreamName.example(category: 'testWriteBatchExpectedVersion') + + write_event = Controls::EventData::Write.example + position = Write.(write_event, stream_name) + + write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + + batch = [write_event_1, write_event_2] + + Write.(batch, stream_name, expected_version: position) + + context "Individual Events are Written" do + 2.times do |i| + read_event = Get.(stream_name, position: i + position, batch_size: 1).first + + test "Event #{i + 1}" do + refute(read_event.nil?) + end + end + end + end + end +end diff --git a/test/automated/write/batch/partition.rb b/test/automated/write/batch/partition.rb index 303640f..4725392 100644 --- a/test/automated/write/batch/partition.rb +++ b/test/automated/write/batch/partition.rb @@ -14,7 +14,6 @@ last_written_position = Write.(batch, stream_name, partition: partition) - test "Last written position" do assert(last_written_position == 1) end diff --git a/test/automated/write/event/expected_version/error.rb b/test/automated/write/event/expected_version/error.rb new file mode 100644 index 0000000..fc01b14 --- /dev/null +++ b/test/automated/write/event/expected_version/error.rb @@ -0,0 +1,30 @@ +require_relative '../../../automated_init' + +context "Write" do + context "Event" do + context "Expected Version" do + context "Does not match the stream version" do + stream_name = Controls::StreamName.example(category: 'testWriteEventExpectedVersionError') + write_event = Controls::EventData::Write.example + + position = Write.(write_event, stream_name) + + incorrect_stream_version = position + 1 + + test "Is an error" do + assert proc { Write.(write_event, stream_name, expected_version: incorrect_stream_version ) } do + raises_error? EventSource::ExpectedVersion::Error + end + end + + context "Event" do + read_event = Get.(stream_name, position: incorrect_stream_version, batch_size: 1).first + + test "Is not written" do + assert(read_event.nil?) + end + end + end + end + end +end diff --git a/test/automated/write/event/expected_version/expected_version.rb b/test/automated/write/event/expected_version/expected_version.rb new file mode 100644 index 0000000..d4f5d78 --- /dev/null +++ b/test/automated/write/event/expected_version/expected_version.rb @@ -0,0 +1,23 @@ +require_relative '../../../automated_init' + +context "Write" do + context "Event" do + context "Expected Version" do + stream_name = Controls::StreamName.example(category: 'testWriteEventExpectedVersion') + + write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + + position = Write.(write_event_1, stream_name) + + write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + + Write.(write_event_2, stream_name, expected_version: position) + + read_event = Get.(stream_name, position: position + 1).first + + test "Got the event that was written" do + assert(read_event.data[:attribute] == 'value_2') + end + end + end +end diff --git a/test/automated/write/event/partition.rb b/test/automated/write/event/partition.rb new file mode 100644 index 0000000..7aad3c9 --- /dev/null +++ b/test/automated/write/event/partition.rb @@ -0,0 +1,20 @@ +require_relative '../../automated_init' + +context "Write" do + context "Event" do + context "Partition" do + stream_name = Controls::StreamName.example(category: 'testWriteEventPartition') + partition = Controls::Partition.example + + write_event = Controls::EventData::Write.example(data: { attribute: 'some_value' }) + + position = Write.(write_event, stream_name, partition: partition) + + read_event = Get.(stream_name, position: position, partition: partition).first + + test "Got the event that was written" do + assert(read_event.data[:attribute] == "some_value") + end + end + end +end diff --git a/test/automated/write/event/write.rb b/test/automated/write/event/write.rb new file mode 100644 index 0000000..8e0d579 --- /dev/null +++ b/test/automated/write/event/write.rb @@ -0,0 +1,17 @@ +require_relative '../../automated_init' + +context "Write" do + context "Event" do + stream_name = Controls::StreamName.example(category: 'testWriteEvent') + + write_event = Controls::EventData::Write.example(data: { attribute: 'some_value' }) + + position = Write.(write_event, stream_name) + + read_event = Get.(stream_name, position: position).first + + test "Got the event that was written" do + assert(read_event.data[:attribute] == "some_value") + end + end +end diff --git a/test/automated/write/message/expected_version_error.rb b/test/automated/write/message/expected_version_error.rb deleted file mode 100644 index a88c998..0000000 --- a/test/automated/write/message/expected_version_error.rb +++ /dev/null @@ -1,28 +0,0 @@ -require_relative '../../automated_init' - -context "Write" do - context "Message" do - context "Expected Version Error" do - stream_name = Controls::StreamName.example(category: 'testWriteMessageExpectedVersionError') - write_event = Controls::EventData::Write.example - - position = Write.(write_event, stream_name) - - incorrect_stream_version = position + 1 - - test "Is an error" do - assert proc { Write.(write_event, stream_name, expected_version: incorrect_stream_version ) } do - raises_error? EventSource::ExpectedVersion::Error - end - end - - context "Event" do - read_event = Get.(stream_name, position: incorrect_stream_version, batch_size: 1).first - - test "Is not written" do - assert(read_event.nil?) - end - end - end - end -end diff --git a/test/automated/write/message/partition.rb b/test/automated/write/message/partition.rb deleted file mode 100644 index 23b5ff3..0000000 --- a/test/automated/write/message/partition.rb +++ /dev/null @@ -1,16 +0,0 @@ -require_relative '../../automated_init' - -context "Write" do - context "Message" do - stream_name = Controls::StreamName.example(category: 'testWriteMessagePartition') - write_event = Controls::EventData::Write.example - - written_position = Write.(write_event, stream_name) - - read_event = Get.(stream_name, position: written_position).first - - test "Got the event that was written" do - assert(read_event.position == written_position) - end - end -end diff --git a/test/automated/write/message/write.rb b/test/automated/write/message/write.rb deleted file mode 100644 index 2acd14b..0000000 --- a/test/automated/write/message/write.rb +++ /dev/null @@ -1,16 +0,0 @@ -require_relative '../../automated_init' - -context "Write" do - context "Message" do - stream_name = Controls::StreamName.example(category: 'testWriteMessage') - write_event = Controls::EventData::Write.example - - written_position = Write.(write_event, stream_name) - - read_event = Get.(stream_name, position: written_position).first - - test "Got the event that was written" do - assert(read_event.position == written_position) - end - end -end From 71c2eb896cb34c938c91dae5688fa02546080b19 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 18 Nov 2016 23:59:54 -0600 Subject: [PATCH 111/591] Event controls use random data values --- test/automated/get/category.rb | 3 ++- test/automated/get/get.rb | 6 +++--- test/automated/put/category_as_stream_name.rb | 2 +- test/automated/put/expected_version/error.rb | 2 +- .../put/expected_version/expected_version.rb | 8 ++++---- test/automated/put/no_stream/existing_stream.rb | 6 +++--- test/automated/put/no_stream/no_stream.rb | 2 +- test/automated/put/partition.rb | 4 ++-- test/automated/put/put.rb | 2 +- test/automated/put/returns_stream_position.rb | 2 +- test/automated/put/stream_position_increases.rb | 2 +- test/automated/write/batch/expected_version/error.rb | 8 ++++---- .../write/batch/expected_version/expected_version.rb | 11 ++++++----- test/automated/write/batch/partition.rb | 9 +++++---- test/automated/write/batch/write.rb | 9 +++++---- test/automated/write/event/expected_version/error.rb | 2 +- .../write/event/expected_version/expected_version.rb | 8 ++++---- test/automated/write/event/partition.rb | 4 ++-- test/automated/write/event/write.rb | 6 +++--- 19 files changed, 50 insertions(+), 46 deletions(-) diff --git a/test/automated/get/category.rb b/test/automated/get/category.rb index f5db024..e72cc34 100644 --- a/test/automated/get/category.rb +++ b/test/automated/get/category.rb @@ -8,8 +8,9 @@ events = Get.(stream_name) + number_of_events = events.length + test "Number of events retrieved is the number written to the category" do - number_of_events = events.length assert(number_of_events == 2) end end diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index c67a3e9..c137a54 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -4,9 +4,9 @@ stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example - written_position = Put.(write_event, stream_name) + position = Put.(write_event, stream_name) - read_event = Get.(stream_name, position: written_position).first + read_event = Get.(stream_name, position: position).first context "Got the event that was written" do test "Type" do @@ -26,7 +26,7 @@ end test "Position" do - assert(read_event.position == written_position) + assert(read_event.position == position) end test "Global Position" do diff --git a/test/automated/put/category_as_stream_name.rb b/test/automated/put/category_as_stream_name.rb index 1e66248..9771204 100644 --- a/test/automated/put/category_as_stream_name.rb +++ b/test/automated/put/category_as_stream_name.rb @@ -2,7 +2,7 @@ context "Put" do context "Category as Stream Name" do - category = Controls::Category.example(category: 'testPutCategoryAsStreamName') + category = Controls::Category.example write_event = Controls::EventData::Write.example Put.(write_event, category) diff --git a/test/automated/put/expected_version/error.rb b/test/automated/put/expected_version/error.rb index 5c38a64..1b81293 100644 --- a/test/automated/put/expected_version/error.rb +++ b/test/automated/put/expected_version/error.rb @@ -3,7 +3,7 @@ context "Put" do context "Expected Version" do context "Does not match the stream version" do - stream_name = Controls::StreamName.example(category: 'testPutExpectedVersion') + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example position = Put.(write_event, stream_name) diff --git a/test/automated/put/expected_version/expected_version.rb b/test/automated/put/expected_version/expected_version.rb index 4712ad6..6a831b0 100644 --- a/test/automated/put/expected_version/expected_version.rb +++ b/test/automated/put/expected_version/expected_version.rb @@ -3,20 +3,20 @@ context "Put" do context "Event" do context "Expected Version" do - stream_name = Controls::StreamName.example(category: 'testPutEventExpectedVersion') + stream_name = Controls::StreamName.example - write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + write_event_1 = Controls::EventData::Write.example position = Put.(write_event_1, stream_name) - write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + write_event_2 = Controls::EventData::Write.example Put.(write_event_2, stream_name, expected_version: position) read_event = Get.(stream_name, position: position + 1).first test "Got the event that was written" do - assert(read_event.data[:attribute] == 'value_2') + assert(read_event.data == write_event_2.data) end end end diff --git a/test/automated/put/no_stream/existing_stream.rb b/test/automated/put/no_stream/existing_stream.rb index 78a28ce..235b537 100644 --- a/test/automated/put/no_stream/existing_stream.rb +++ b/test/automated/put/no_stream/existing_stream.rb @@ -3,10 +3,10 @@ context "Put" do context "No Stream" do context "Existing Stream" do - stream_name = Controls::StreamName.example(category: 'testPutNoStreamExistingStream') + stream_name = Controls::StreamName.example - write_event_1 = Controls::EventData::Write.example(data: {:some_attribute => 'first'}) - write_event_2 = Controls::EventData::Write.example(data: {:some_attribute => 'second'}) + write_event_1 = Controls::EventData::Write.example + write_event_2 = Controls::EventData::Write.example Put.(write_event_1, stream_name) diff --git a/test/automated/put/no_stream/no_stream.rb b/test/automated/put/no_stream/no_stream.rb index 55143bd..e4c8b94 100644 --- a/test/automated/put/no_stream/no_stream.rb +++ b/test/automated/put/no_stream/no_stream.rb @@ -3,7 +3,7 @@ context "Put" do context "No Stream" do context "For a stream that doesn't exist" do - stream_name = Controls::StreamName.example(category: 'testPutNoStream') + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example position = Put.(write_event, stream_name) diff --git a/test/automated/put/partition.rb b/test/automated/put/partition.rb index 2d0fb44..570ef83 100644 --- a/test/automated/put/partition.rb +++ b/test/automated/put/partition.rb @@ -2,7 +2,7 @@ context "Put" do context "Partition" do - stream_name = Controls::StreamName.example(category: 'testPutPartition') + stream_name = Controls::StreamName.example partition = Controls::Partition.example write_event = Controls::EventData::Write.example @@ -12,7 +12,7 @@ read_event = Get.(stream_name, partition: partition, position: written_position).first test "Got the event that was written" do - assert(read_event.position == written_position) + assert(read_event.data == write_event.data) end end end diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index 7b3215b..c137a54 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -1,7 +1,7 @@ require_relative '../automated_init' context "Put and Get" do - stream_name = Controls::StreamName.example(category: 'testPut') + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example position = Put.(write_event, stream_name) diff --git a/test/automated/put/returns_stream_position.rb b/test/automated/put/returns_stream_position.rb index 82f966a..e23f970 100644 --- a/test/automated/put/returns_stream_position.rb +++ b/test/automated/put/returns_stream_position.rb @@ -2,7 +2,7 @@ context "Put" do context "Returns Stream Position" do - stream_name = Controls::StreamName.example(category: 'testPutReturnsStreamPosition') + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example diff --git a/test/automated/put/stream_position_increases.rb b/test/automated/put/stream_position_increases.rb index 346f620..2d2bbbf 100644 --- a/test/automated/put/stream_position_increases.rb +++ b/test/automated/put/stream_position_increases.rb @@ -2,7 +2,7 @@ context "Put" do context "Stream Position Increases with Subsequent Writes" do - stream_name = Controls::StreamName.example(category: 'testPutPositionIncreases') + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example position_1 = Put.(write_event, stream_name) diff --git a/test/automated/write/batch/expected_version/error.rb b/test/automated/write/batch/expected_version/error.rb index 10fffdc..204e695 100644 --- a/test/automated/write/batch/expected_version/error.rb +++ b/test/automated/write/batch/expected_version/error.rb @@ -4,15 +4,15 @@ context "Batch" do context "Expected Version" do context "Does not match the stream version" do - stream_name = Controls::StreamName.example(category: 'testWriteBatchExpectedVersionError') + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example position = Write.(write_event, stream_name) incorrect_stream_version = position + 1 - write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) - write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + write_event_1 = Controls::EventData::Write.example + write_event_2 = Controls::EventData::Write.example batch = [write_event_1, write_event_2] @@ -24,7 +24,7 @@ context "Events" do 2.times do |i| - read_event = Get.(stream_name, position: incorrect_stream_version + i, batch_size: 1).first + read_event = Get.(stream_name, position: i + 1, batch_size: 1).first test "Event #{i + 1} not written" do assert(read_event.nil?) diff --git a/test/automated/write/batch/expected_version/expected_version.rb b/test/automated/write/batch/expected_version/expected_version.rb index 6b4e59f..b514d89 100644 --- a/test/automated/write/batch/expected_version/expected_version.rb +++ b/test/automated/write/batch/expected_version/expected_version.rb @@ -3,13 +3,13 @@ context "Write" do context "Batch" do context "Expected Version" do - stream_name = Controls::StreamName.example(category: 'testWriteBatchExpectedVersion') + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example position = Write.(write_event, stream_name) - write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) - write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + write_event_1 = Controls::EventData::Write.example + write_event_2 = Controls::EventData::Write.example batch = [write_event_1, write_event_2] @@ -17,10 +17,11 @@ context "Individual Events are Written" do 2.times do |i| - read_event = Get.(stream_name, position: i + position, batch_size: 1).first + read_event = Get.(stream_name, position: i + 1, batch_size: 1).first + write_event = batch[i] test "Event #{i + 1}" do - refute(read_event.nil?) + assert(read_event.data == write_event.data) end end end diff --git a/test/automated/write/batch/partition.rb b/test/automated/write/batch/partition.rb index 4725392..d9500bc 100644 --- a/test/automated/write/batch/partition.rb +++ b/test/automated/write/batch/partition.rb @@ -3,10 +3,10 @@ context "Write" do context "Batch" do context "Partition" do - stream_name = Controls::StreamName.example(category: 'testWriteBatchPartition') + stream_name = Controls::StreamName.example - write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) - write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + write_event_1 = Controls::EventData::Write.example + write_event_2 = Controls::EventData::Write.example batch = [write_event_1, write_event_2] @@ -21,9 +21,10 @@ context "Individual Events are Written" do 2.times do |i| read_event = Get.(stream_name, partition: partition, position: i, batch_size: 1).first + write_event = batch[i] test "Event #{i + 1}" do - assert(read_event.data[:attribute] == "value_#{i + 1}") + assert(read_event.data == write_event.data) end end end diff --git a/test/automated/write/batch/write.rb b/test/automated/write/batch/write.rb index 96e78a3..87aaba8 100644 --- a/test/automated/write/batch/write.rb +++ b/test/automated/write/batch/write.rb @@ -2,10 +2,10 @@ context "Write" do context "Batch" do - stream_name = Controls::StreamName.example(category: 'testWriteBatch') + stream_name = Controls::StreamName.example - write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) - write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + write_event_1 = Controls::EventData::Write.example + write_event_2 = Controls::EventData::Write.example batch = [write_event_1, write_event_2] @@ -18,9 +18,10 @@ context "Individual Events are Written" do 2.times do |i| read_event = Get.(stream_name, position: i, batch_size: 1).first + write_event = batch[i] test "Event #{i + 1}" do - assert(read_event.data[:attribute] == "value_#{i + 1}") + assert(read_event.data == write_event.data) end end end diff --git a/test/automated/write/event/expected_version/error.rb b/test/automated/write/event/expected_version/error.rb index fc01b14..24179c1 100644 --- a/test/automated/write/event/expected_version/error.rb +++ b/test/automated/write/event/expected_version/error.rb @@ -4,7 +4,7 @@ context "Event" do context "Expected Version" do context "Does not match the stream version" do - stream_name = Controls::StreamName.example(category: 'testWriteEventExpectedVersionError') + stream_name = Controls::StreamName.example write_event = Controls::EventData::Write.example position = Write.(write_event, stream_name) diff --git a/test/automated/write/event/expected_version/expected_version.rb b/test/automated/write/event/expected_version/expected_version.rb index d4f5d78..1d8ee1d 100644 --- a/test/automated/write/event/expected_version/expected_version.rb +++ b/test/automated/write/event/expected_version/expected_version.rb @@ -3,20 +3,20 @@ context "Write" do context "Event" do context "Expected Version" do - stream_name = Controls::StreamName.example(category: 'testWriteEventExpectedVersion') + stream_name = Controls::StreamName.example - write_event_1 = Controls::EventData::Write.example(data: { attribute: 'value_1' }) + write_event_1 = Controls::EventData::Write.example position = Write.(write_event_1, stream_name) - write_event_2 = Controls::EventData::Write.example(data: { attribute: 'value_2' }) + write_event_2 = Controls::EventData::Write.example Write.(write_event_2, stream_name, expected_version: position) read_event = Get.(stream_name, position: position + 1).first test "Got the event that was written" do - assert(read_event.data[:attribute] == 'value_2') + assert(read_event.data == write_event_2.data) end end end diff --git a/test/automated/write/event/partition.rb b/test/automated/write/event/partition.rb index 7aad3c9..0858016 100644 --- a/test/automated/write/event/partition.rb +++ b/test/automated/write/event/partition.rb @@ -3,7 +3,7 @@ context "Write" do context "Event" do context "Partition" do - stream_name = Controls::StreamName.example(category: 'testWriteEventPartition') + stream_name = Controls::StreamName.example partition = Controls::Partition.example write_event = Controls::EventData::Write.example(data: { attribute: 'some_value' }) @@ -13,7 +13,7 @@ read_event = Get.(stream_name, position: position, partition: partition).first test "Got the event that was written" do - assert(read_event.data[:attribute] == "some_value") + assert(read_event.data == write_event.data) end end end diff --git a/test/automated/write/event/write.rb b/test/automated/write/event/write.rb index 8e0d579..235a2b2 100644 --- a/test/automated/write/event/write.rb +++ b/test/automated/write/event/write.rb @@ -2,16 +2,16 @@ context "Write" do context "Event" do - stream_name = Controls::StreamName.example(category: 'testWriteEvent') + stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example(data: { attribute: 'some_value' }) + write_event = Controls::EventData::Write.example position = Write.(write_event, stream_name) read_event = Get.(stream_name, position: position).first test "Got the event that was written" do - assert(read_event.data[:attribute] == "some_value") + assert(read_event.data == write_event.data) end end end From a8b72e69d8316db5a19cdc7d4a080aacb06264e4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 19 Nov 2016 12:34:41 -0600 Subject: [PATCH 112/591] Consistency in tests --- test/automated/put/expected_version/error.rb | 2 +- test/automated/write/batch/expected_version/error.rb | 2 +- test/automated/write/event/expected_version/error.rb | 2 +- test/automated/write/event/partition.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/automated/put/expected_version/error.rb b/test/automated/put/expected_version/error.rb index 1b81293..2acff69 100644 --- a/test/automated/put/expected_version/error.rb +++ b/test/automated/put/expected_version/error.rb @@ -4,8 +4,8 @@ context "Expected Version" do context "Does not match the stream version" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example + write_event = Controls::EventData::Write.example position = Put.(write_event, stream_name) incorrect_stream_version = position + 1 diff --git a/test/automated/write/batch/expected_version/error.rb b/test/automated/write/batch/expected_version/error.rb index 204e695..df5bd95 100644 --- a/test/automated/write/batch/expected_version/error.rb +++ b/test/automated/write/batch/expected_version/error.rb @@ -5,8 +5,8 @@ context "Expected Version" do context "Does not match the stream version" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example + write_event = Controls::EventData::Write.example position = Write.(write_event, stream_name) incorrect_stream_version = position + 1 diff --git a/test/automated/write/event/expected_version/error.rb b/test/automated/write/event/expected_version/error.rb index 24179c1..38a12ed 100644 --- a/test/automated/write/event/expected_version/error.rb +++ b/test/automated/write/event/expected_version/error.rb @@ -5,8 +5,8 @@ context "Expected Version" do context "Does not match the stream version" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example + write_event = Controls::EventData::Write.example position = Write.(write_event, stream_name) incorrect_stream_version = position + 1 diff --git a/test/automated/write/event/partition.rb b/test/automated/write/event/partition.rb index 0858016..3af46ba 100644 --- a/test/automated/write/event/partition.rb +++ b/test/automated/write/event/partition.rb @@ -6,7 +6,7 @@ stream_name = Controls::StreamName.example partition = Controls::Partition.example - write_event = Controls::EventData::Write.example(data: { attribute: 'some_value' }) + write_event = Controls::EventData::Write.example position = Write.(write_event, stream_name, partition: partition) From 259f39bd89d24d963929b5f3a9cc92e03a1167c4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 19 Nov 2016 20:15:08 -0600 Subject: [PATCH 113/591] Partition is removed --- database/create-other-events-table.sh | 30 ---------------- database/functions/stream-version.sql | 16 ++------- database/functions/write-event.sql | 36 +++++++------------ database/install.sh | 7 ++-- database/{tables => table}/events-table.sql | 0 database/tables/other-events-table.sql | 19 ---------- lib/event_source/postgres.rb | 2 -- lib/event_source/postgres/controls.rb | 4 --- .../postgres/controls/partition.rb | 7 ---- lib/event_source/postgres/controls/put.rb | 5 ++- lib/event_source/postgres/get.rb | 24 ++++++------- .../postgres/get/select_statement.rb | 20 ++++------- lib/event_source/postgres/partition.rb | 11 ------ lib/event_source/postgres/put.rb | 31 +++++++--------- lib/event_source/postgres/read.rb | 4 +-- test/automated/put/partition.rb | 18 ---------- test/automated/read/partition.rb | 17 --------- test/automated/select_statement/partition.rb | 18 ---------- test/automated/write/batch/partition.rb | 33 ----------------- test/automated/write/event/partition.rb | 20 ----------- tools/create-other-events-table | 3 -- 21 files changed, 52 insertions(+), 273 deletions(-) delete mode 100755 database/create-other-events-table.sh rename database/{tables => table}/events-table.sql (100%) delete mode 100644 database/tables/other-events-table.sql delete mode 100644 lib/event_source/postgres/controls/partition.rb delete mode 100644 lib/event_source/postgres/partition.rb delete mode 100644 test/automated/put/partition.rb delete mode 100644 test/automated/read/partition.rb delete mode 100644 test/automated/select_statement/partition.rb delete mode 100644 test/automated/write/batch/partition.rb delete mode 100644 test/automated/write/event/partition.rb delete mode 100755 tools/create-other-events-table diff --git a/database/create-other-events-table.sh b/database/create-other-events-table.sh deleted file mode 100755 index 04c9815..0000000 --- a/database/create-other-events-table.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -e - -clear - -echo -echo "Creating other_events Table" -echo "= = =" -echo - -default_name=eventsource - -if [ -z ${DATABASE_USER+x} ]; then - echo "(DATABASE_USER is not set)" - user=$default_name -else - user=$DATABASE_USER -fi - -if [ -z ${DATABASE_NAME+x} ]; then - echo "(DATABASE_NAME is not set)" - database=$default_name -else - database=$DATABASE_NAME -fi - -echo - -psql $database -f database/tables/other-events-table.sql diff --git a/database/functions/stream-version.sql b/database/functions/stream-version.sql index c6d68b0..6b88722 100644 --- a/database/functions/stream-version.sql +++ b/database/functions/stream-version.sql @@ -1,24 +1,12 @@ CREATE OR REPLACE FUNCTION stream_version( - _stream_name varchar, - _partition varchar DEFAULT NULL + _stream_name varchar ) RETURNS int AS $$ DECLARE stream_version int; BEGIN - if _partition is null then - _partition := 'events'; - end if; - - EXECUTE format('select max(position) ' - 'from %I ' - 'where stream_name = $1', - _partition) - INTO - stream_version - USING - _stream_name; + select max(position) into stream_version from events where stream_name = _stream_name; return stream_version; END; diff --git a/database/functions/write-event.sql b/database/functions/write-event.sql index 493bb8f..c7a74f9 100644 --- a/database/functions/write-event.sql +++ b/database/functions/write-event.sql @@ -2,7 +2,6 @@ CREATE OR REPLACE FUNCTION write_event( _stream_name varchar, _type varchar, _data jsonb, - _partition varchar DEFAULT NULL, _metadata jsonb DEFAULT NULL, _expected_version int DEFAULT NULL ) @@ -13,11 +12,7 @@ DECLARE position int; category varchar; BEGIN - if _partition is null then - _partition := 'events'; - end if; - - stream_version := stream_version(_stream_name, _partition); + stream_version := stream_version(_stream_name); if stream_version is null then stream_version := -1; @@ -31,28 +26,23 @@ BEGIN position := stream_version + 1; - EXECUTE format('insert into %I (' - '"stream_name", ' - '"position", ' - '"type", ' - '"data", ' - '"metadata"' - ') ' - 'values (' - '$1, ' - '$2, ' - '$3, ' - '$4, ' - '$5 - )', - _partition) - USING + insert into "events" + ( + "stream_name", + "position", + "type", + "data", + "metadata" + ) + values + ( _stream_name, position, _type, _data, _metadata - ; + ) + ; return position; END; diff --git a/database/install.sh b/database/install.sh index aa66233..181641b 100755 --- a/database/install.sh +++ b/database/install.sh @@ -59,9 +59,8 @@ function create-database { echo } -function create-tables { - psql $database -f database/tables/events-table.sql - psql $database -f database/tables/other-events-table.sql +function create-table { + psql $database -f database/table/events-table.sql } function create-functions { @@ -76,6 +75,6 @@ function create-indexes { create-user create-database -create-tables +create-table create-functions create-indexes diff --git a/database/tables/events-table.sql b/database/table/events-table.sql similarity index 100% rename from database/tables/events-table.sql rename to database/table/events-table.sql diff --git a/database/tables/other-events-table.sql b/database/tables/other-events-table.sql deleted file mode 100644 index 9810a60..0000000 --- a/database/tables/other-events-table.sql +++ /dev/null @@ -1,19 +0,0 @@ --- ---------------------------- --- Table structure for other_events --- ---------------------------- -DROP TABLE IF EXISTS "public"."other_events"; -CREATE TABLE "public"."other_events" ( - "stream_name" varchar(255) NOT NULL COLLATE "default", - "position" int4 NOT NULL, - "type" varchar(255) NOT NULL COLLATE "default", - "global_position" bigserial NOT NULL , - "data" jsonb NOT NULL, - "metadata" jsonb, - "time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL -) -WITH (OIDS=FALSE); - --- ---------------------------- --- Primary key structure for table other_events --- ---------------------------- -ALTER TABLE "public"."other_events" ADD PRIMARY KEY ("global_position") NOT DEFERRABLE INITIALLY IMMEDIATE; diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index c98af31..fd83c6c 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -12,8 +12,6 @@ require 'event_source/postgres/settings' require 'event_source/postgres/session' -require 'event_source/postgres/partition' - require 'event_source/postgres/put' require 'event_source/postgres/write' diff --git a/lib/event_source/postgres/controls.rb b/lib/event_source/postgres/controls.rb index 1bf6992..7284844 100644 --- a/lib/event_source/postgres/controls.rb +++ b/lib/event_source/postgres/controls.rb @@ -3,9 +3,5 @@ require 'event_source/postgres/controls/category' require 'event_source/postgres/controls/stream_name' require 'event_source/postgres/controls/stream' - require 'event_source/postgres/controls/event_data' - -require 'event_source/postgres/controls/partition' - require 'event_source/postgres/controls/put' diff --git a/lib/event_source/postgres/controls/partition.rb b/lib/event_source/postgres/controls/partition.rb deleted file mode 100644 index e06030d..0000000 --- a/lib/event_source/postgres/controls/partition.rb +++ /dev/null @@ -1,7 +0,0 @@ -module EventSource - module Postgres - module Controls - Partition = EventSource::Controls::Partition - end - end -end diff --git a/lib/event_source/postgres/controls/put.rb b/lib/event_source/postgres/controls/put.rb index 4df9c11..28cdc6e 100644 --- a/lib/event_source/postgres/controls/put.rb +++ b/lib/event_source/postgres/controls/put.rb @@ -2,14 +2,13 @@ module EventSource module Postgres module Controls module Put - def self.call(instances: nil, stream_name: nil, event: nil, category: nil, partition: nil) + def self.call(instances: nil, stream_name: nil, event: nil, category: nil) instances ||= 1 stream_name ||= StreamName.example(category: category) event ||= EventData::Write.example - partition ||= Postgres::Partition::Defaults.name instances.times do - EventSource::Postgres::Put.(event, stream_name, partition: partition) + EventSource::Postgres::Put.(event, stream_name) end stream_name diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 9e4291c..e65769d 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -3,25 +3,25 @@ module Postgres class Get include Log::Dependency - initializer :stream, :batch_size, :precedence, :partition + initializer :stream, :batch_size, :precedence dependency :session, Session - def self.build(stream_name, batch_size: nil, precedence: nil, partition: nil, session: nil) + def self.build(stream_name, batch_size: nil, precedence: nil, session: nil) stream = Stream.new(stream_name) - new(stream, batch_size, precedence, partition).tap do |instance| + new(stream, batch_size, precedence).tap do |instance| instance.configure(session: session) end end - def self.configure(receiver, stream_name, attr_name: nil, position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) + def self.configure(receiver, stream_name, attr_name: nil, position: nil, batch_size: nil, precedence: nil, session: nil) attr_name ||= :get - instance = build(stream_name, batch_size: batch_size, precedence: precedence, partition: partition, session: session) + instance = build(stream_name, batch_size: batch_size, precedence: precedence, session: session) receiver.public_send "#{attr_name}=", instance end - def self.call(stream_name, position: nil, batch_size: nil, precedence: nil, partition: nil, session: nil) - instance = build(stream_name, batch_size: batch_size, precedence: precedence, partition: partition, session: session) + def self.call(stream_name, position: nil, batch_size: nil, precedence: nil, session: nil) + instance = build(stream_name, batch_size: batch_size, precedence: precedence, session: session) instance.(position: position) end @@ -30,26 +30,26 @@ def configure(session: nil) end def call(position: nil) - logger.trace { "Getting event data (Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" } + logger.trace { "Getting event data (Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } records = get_records(stream, position) events = convert(records) - logger.info { "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" } + logger.info { "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } logger.info(tags: [:data, :event_data]) { events.pretty_inspect } events end def get_records(stream, position) - logger.trace { "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" } + logger.trace { "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } - select_statement = SelectStatement.build(stream, offset: position, batch_size: batch_size, precedence: precedence, partition: partition) + select_statement = SelectStatement.build(stream, offset: position, batch_size: batch_size, precedence: precedence) records = session.execute(select_statement.sql) - logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect}, Partition: #{partition.inspect})" } + logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } records end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 9e9287e..c58268d 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -4,7 +4,7 @@ class Get class SelectStatement include Log::Dependency - initializer :stream, w(:offset), w(:batch_size), w(:precedence), w(:partition) + initializer :stream, w(:offset), w(:batch_size), w(:precedence) def offset @offset ||= Defaults.offset @@ -18,10 +18,6 @@ def precedence @precedence ||= Defaults.precedence end - def partition - @partition ||= Defaults.partition - end - def stream_name stream.name end @@ -30,12 +26,12 @@ def stream_type stream.type end - def self.build(stream, offset: nil, batch_size: nil, partition: nil, precedence: nil) - new(stream, offset, batch_size, precedence, partition) + def self.build(stream, offset: nil, batch_size: nil, precedence: nil) + new(stream, offset, batch_size, precedence) end def sql - logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition})" } + logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" } statement = <<-SQL SELECT @@ -47,7 +43,7 @@ def sql metadata::varchar, time::timestamp FROM - #{partition} + events WHERE #{where_clause_field} = '#{stream_name}' ORDER BY @@ -59,7 +55,7 @@ def sql ; SQL - logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence}, Partition: #{partition.inspect})" } + logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" } logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } statement @@ -85,10 +81,6 @@ def self.batch_size def self.precedence :asc end - - def self.partition - Partition::Defaults.name - end end end end diff --git a/lib/event_source/postgres/partition.rb b/lib/event_source/postgres/partition.rb deleted file mode 100644 index d40af05..0000000 --- a/lib/event_source/postgres/partition.rb +++ /dev/null @@ -1,11 +0,0 @@ -module EventSource - module Postgres - module Partition - module Defaults - def self.name - 'events' - end - end - end - end -end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 5fc2879..768abaf 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -21,20 +21,20 @@ def self.configure(receiver, session: nil, attr_name: nil) receiver.public_send "#{attr_name}=", instance end - def self.call(write_event, stream_name, expected_version: nil, partition: nil, session: nil) + def self.call(write_event, stream_name, expected_version: nil, session: nil) instance = build(session: session) - instance.(write_event, stream_name, expected_version: expected_version, partition: partition) + instance.(write_event, stream_name, expected_version: expected_version) end - def call(write_event, stream_name, expected_version: nil, partition: nil) - logger.trace { "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect}, Partition: #{partition.inspect})" } + def call(write_event, stream_name, expected_version: nil) + logger.trace { "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } logger.trace(tags: [:data, :event_data]) { write_event.pretty_inspect } type, data, metadata = destructure_event(write_event) expected_version = ExpectedVersion.canonize(expected_version) - insert_event(stream_name, type, data, metadata, expected_version, partition).tap do |position| - logger.info { "Put event data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect}, Partition: #{partition.inspect})" } + insert_event(stream_name, type, data, metadata, expected_version).tap do |position| + logger.info { "Put event data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } logger.info(tags: [:data, :event_data]) { write_event.pretty_inspect } end end @@ -50,21 +50,20 @@ def destructure_event(write_event) return type, data, metadata end - def insert_event(stream_name, type, data, metadata, expected_version, partition) + def insert_event(stream_name, type, data, metadata, expected_version) serialized_data = serialized_data(data) serialized_metadata = serialized_metadata(metadata) - records = execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version, partition) + records = execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version) position(records) end - def execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version, partition) - logger.trace { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, Partition: #{partition.inspect})" } + def execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version) + logger.trace { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" } params = [ stream_name, type, serialized_data, - partition, serialized_metadata, expected_version ] @@ -75,13 +74,13 @@ def execute_query(stream_name, type, serialized_data, serialized_metadata, expec raise_error e end - logger.debug { "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, Partition: #{partition.inspect})" } + logger.debug { "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" } records end def self.statement - @statement ||= "SELECT write_event($1::varchar, $2::varchar, $3::jsonb, $4::varchar, $5::jsonb, $6::int);" + @statement ||= "SELECT write_event($1::varchar, $2::varchar, $3::jsonb, $4::jsonb, $5::int);" end def serialized_data(data) @@ -118,12 +117,6 @@ def raise_error(pg_error) end raise pg_error end - - module Defaults - def self.partition - Partition::Defaults.name - end - end end end end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 434479b..b80cd8d 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -3,8 +3,8 @@ module Postgres class Read include EventSource::Read - def configure(stream, batch_size: nil, precedence: nil, partition: nil, session: nil) - Get.configure(self, stream, batch_size: batch_size, precedence: precedence, partition: partition, session: session) + def configure(stream, batch_size: nil, precedence: nil, session: nil) + Get.configure(self, stream, batch_size: batch_size, precedence: precedence, session: session) end end end diff --git a/test/automated/put/partition.rb b/test/automated/put/partition.rb deleted file mode 100644 index 570ef83..0000000 --- a/test/automated/put/partition.rb +++ /dev/null @@ -1,18 +0,0 @@ -require_relative '../automated_init' - -context "Put" do - context "Partition" do - stream_name = Controls::StreamName.example - partition = Controls::Partition.example - - write_event = Controls::EventData::Write.example - - written_position = Put.(write_event, stream_name, partition: partition) - - read_event = Get.(stream_name, partition: partition, position: written_position).first - - test "Got the event that was written" do - assert(read_event.data == write_event.data) - end - end -end diff --git a/test/automated/read/partition.rb b/test/automated/read/partition.rb deleted file mode 100644 index f3be80c..0000000 --- a/test/automated/read/partition.rb +++ /dev/null @@ -1,17 +0,0 @@ -require_relative '../automated_init' - -context "Read" do - partition = Controls::Partition.example - - stream_name = Controls::Put.(instances: 2, partition: partition) - - batch = [] - - Read.(stream_name, partition: partition, batch_size: 1) do |event_data| - batch << event_data - end - - test "Reads batches of events" do - assert(batch.length == 2) - end -end diff --git a/test/automated/select_statement/partition.rb b/test/automated/select_statement/partition.rb deleted file mode 100644 index 9b87e68..0000000 --- a/test/automated/select_statement/partition.rb +++ /dev/null @@ -1,18 +0,0 @@ -require_relative '../automated_init' - -context "Select Statement" do - context "Partition" do - stream = Controls::Stream.example - - select_statement = Get::SelectStatement.build stream, partition: Controls::Partition.example - - sql = select_statement.sql - sql.gsub!(/\s+/, ' ') - - context "From Clause" do - test "Partition table name" do - assert(sql.include? "FROM #{Controls::Partition.example}") - end - end - end -end diff --git a/test/automated/write/batch/partition.rb b/test/automated/write/batch/partition.rb deleted file mode 100644 index d9500bc..0000000 --- a/test/automated/write/batch/partition.rb +++ /dev/null @@ -1,33 +0,0 @@ -require_relative '../../automated_init' - -context "Write" do - context "Batch" do - context "Partition" do - stream_name = Controls::StreamName.example - - write_event_1 = Controls::EventData::Write.example - write_event_2 = Controls::EventData::Write.example - - batch = [write_event_1, write_event_2] - - partition = Controls::Partition.example - - last_written_position = Write.(batch, stream_name, partition: partition) - - test "Last written position" do - assert(last_written_position == 1) - end - - context "Individual Events are Written" do - 2.times do |i| - read_event = Get.(stream_name, partition: partition, position: i, batch_size: 1).first - write_event = batch[i] - - test "Event #{i + 1}" do - assert(read_event.data == write_event.data) - end - end - end - end - end -end diff --git a/test/automated/write/event/partition.rb b/test/automated/write/event/partition.rb deleted file mode 100644 index 3af46ba..0000000 --- a/test/automated/write/event/partition.rb +++ /dev/null @@ -1,20 +0,0 @@ -require_relative '../../automated_init' - -context "Write" do - context "Event" do - context "Partition" do - stream_name = Controls::StreamName.example - partition = Controls::Partition.example - - write_event = Controls::EventData::Write.example - - position = Write.(write_event, stream_name, partition: partition) - - read_event = Get.(stream_name, position: position, partition: partition).first - - test "Got the event that was written" do - assert(read_event.data == write_event.data) - end - end - end -end diff --git a/tools/create-other-events-table b/tools/create-other-events-table deleted file mode 100755 index 6dbb9d1..0000000 --- a/tools/create-other-events-table +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -database/create-other-events-table.sh From 39c070be5451ed4cb7c32187390bea00c3f83196 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 19 Nov 2016 20:31:17 -0600 Subject: [PATCH 114/591] Package version is increased from 0.7.0.2 to 0.8.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 6a94764..e0d497a 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.7.0.2' + s.version = '0.8.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 34a539069d8032e1e63cb6f1a935da87c2980539 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 20 Nov 2016 17:17:18 -0600 Subject: [PATCH 115/591] Test reverse read --- test/automated/read/reverse.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 test/automated/read/reverse.rb diff --git a/test/automated/read/reverse.rb b/test/automated/read/reverse.rb new file mode 100644 index 0000000..1794e72 --- /dev/null +++ b/test/automated/read/reverse.rb @@ -0,0 +1,18 @@ +require_relative '../automated_init' + +context "Read" do + context "Reverse" do + stream_name = Controls::Put.(instances: 3) + + events = [] + Read.(stream_name, precedence: :desc) do |event| + events << event + end + + first_event_postition = events.first.position + + test "Last event written is first in the list of results" do + assert(first_event_postition == 2) + end + end +end From 45682cacfdec924426105233aafa218dacd0a5a6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 23 Nov 2016 19:35:45 -0600 Subject: [PATCH 116/591] Metadata is optional --- lib/event_source/postgres/put.rb | 2 +- test/automated/put/nil_metadata.rb | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 test/automated/put/nil_metadata.rb diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 768abaf..15ac989 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -91,9 +91,9 @@ def serialized_data(data) end def serialized_metadata(metadata) - serializable_metadata = EventData::Hash[metadata] serialized_metadata = nil unless metadata.nil? + serializable_metadata = EventData::Hash[metadata] serialized_metadata = Transform::Write.(serializable_metadata, :json) end logger.debug(tags: [:data, :serialize]) { "Serialized Metadata: #{serialized_metadata.inspect}" } diff --git a/test/automated/put/nil_metadata.rb b/test/automated/put/nil_metadata.rb new file mode 100644 index 0000000..13fe7a4 --- /dev/null +++ b/test/automated/put/nil_metadata.rb @@ -0,0 +1,20 @@ +require_relative '../automated_init' + +context "Put" do + context "Metadata is nil" do + stream_name = Controls::StreamName.example + write_event = Controls::EventData::Write.example + + write_event.metadata = nil + + position = Put.(write_event, stream_name) + + read_event = Get.(stream_name, position: position).first + + context "Read metadata" do + test "Is nil" do + assert(read_event.metadata.nil?) + end + end + end +end From a5d42630bc60c4a59f7bdabe1b358eb4b9bf3dd7 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 23 Nov 2016 19:36:24 -0600 Subject: [PATCH 117/591] Package version is increased from 0.8.0.0 to 0.8.1.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index e0d497a..572c8b8 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.8.0.0' + s.version = '0.8.1.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 1502c137a6f401002a51356ab796b17d6ee52391 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 24 Nov 2016 01:24:40 -0600 Subject: [PATCH 118/591] Select statement takes stream name --- lib/event_source/postgres/get.rb | 2 +- lib/event_source/postgres/get/select_statement.rb | 3 ++- test/automated/select_statement/category_query.rb | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index e65769d..ef04cad 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -3,7 +3,7 @@ module Postgres class Get include Log::Dependency - initializer :stream, :batch_size, :precedence + initializer :batch_size, :precedence dependency :session, Session diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index c58268d..6a1db23 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -26,7 +26,8 @@ def stream_type stream.type end - def self.build(stream, offset: nil, batch_size: nil, precedence: nil) + def self.build(stream_name, offset: nil, batch_size: nil, precedence: nil) + stream = Stream.new(stream_name) new(stream, offset, batch_size, precedence) end diff --git a/test/automated/select_statement/category_query.rb b/test/automated/select_statement/category_query.rb index 998dc4c..9797729 100644 --- a/test/automated/select_statement/category_query.rb +++ b/test/automated/select_statement/category_query.rb @@ -2,9 +2,9 @@ context "Select Statement" do context "Category Query" do - stream = Controls::Stream::Category.example + category = Controls::Category.example - select_statement = Get::SelectStatement.build stream + select_statement = Get::SelectStatement.build(category) sql = select_statement.sql sql.gsub!(/\s+/, ' ') From 4608448289e7ca96c1ca7afa14290923ec181a03 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 24 Nov 2016 02:47:20 -0600 Subject: [PATCH 119/591] Stream name is an actuator argument to get --- lib/event_source/postgres/get.rb | 35 +++++++++---------- lib/event_source/postgres/read.rb | 4 +-- test/automated/get/iterator/next.rb | 4 +-- .../get/iterator/no_further_event_data.rb | 4 +-- .../select_statement/stream_name_query.rb | 4 +-- 5 files changed, 25 insertions(+), 26 deletions(-) diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index ef04cad..4a67db3 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -7,49 +7,48 @@ class Get dependency :session, Session - def self.build(stream_name, batch_size: nil, precedence: nil, session: nil) - stream = Stream.new(stream_name) - new(stream, batch_size, precedence).tap do |instance| + def self.build(batch_size: nil, precedence: nil, session: nil) + new(batch_size, precedence).tap do |instance| instance.configure(session: session) end end - def self.configure(receiver, stream_name, attr_name: nil, position: nil, batch_size: nil, precedence: nil, session: nil) + def self.configure(receiver, attr_name: nil, position: nil, batch_size: nil, precedence: nil, session: nil) attr_name ||= :get - instance = build(stream_name, batch_size: batch_size, precedence: precedence, session: session) + instance = build(batch_size: batch_size, precedence: precedence, session: session) receiver.public_send "#{attr_name}=", instance end - def self.call(stream_name, position: nil, batch_size: nil, precedence: nil, session: nil) - instance = build(stream_name, batch_size: batch_size, precedence: precedence, session: session) - instance.(position: position) - end - def configure(session: nil) Session.configure self, session: session end - def call(position: nil) - logger.trace { "Getting event data (Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } + def self.call(stream_name, position: nil, batch_size: nil, precedence: nil, session: nil) + instance = build(batch_size: batch_size, precedence: precedence, session: session) + instance.(stream_name, position: position) + end + + def call(stream_name, position: nil) + logger.trace { "Getting event data (Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } - records = get_records(stream, position) + records = get_records(stream_name, position) events = convert(records) - logger.info { "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream.name}, Category: #{stream.category?}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } + logger.info { "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } logger.info(tags: [:data, :event_data]) { events.pretty_inspect } events end - def get_records(stream, position) - logger.trace { "Getting records (Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } + def get_records(stream_name, position) + logger.trace { "Getting records (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } - select_statement = SelectStatement.build(stream, offset: position, batch_size: batch_size, precedence: precedence) + select_statement = SelectStatement.build(stream_name, offset: position, batch_size: batch_size, precedence: precedence) records = session.execute(select_statement.sql) - logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream.name}, Category: #{stream.category?}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } + logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } records end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index b80cd8d..19fdea8 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -3,8 +3,8 @@ module Postgres class Read include EventSource::Read - def configure(stream, batch_size: nil, precedence: nil, session: nil) - Get.configure(self, stream, batch_size: batch_size, precedence: precedence, session: session) + def configure(batch_size: nil, precedence: nil, session: nil) + Get.configure(self, batch_size: batch_size, precedence: precedence, session: session) end end end diff --git a/test/automated/get/iterator/next.rb b/test/automated/get/iterator/next.rb index cb35b49..3fec1ed 100644 --- a/test/automated/get/iterator/next.rb +++ b/test/automated/get/iterator/next.rb @@ -4,9 +4,9 @@ context "Next" do stream_name = Controls::Put.(instances: 2) - get = Get.build(stream_name, batch_size: 1) + get = Get.build(batch_size: 1) - iterator = Iterator.build(get) + iterator = Iterator.build(get, stream_name) batch = [] diff --git a/test/automated/get/iterator/no_further_event_data.rb b/test/automated/get/iterator/no_further_event_data.rb index b9effe6..a89bbfe 100644 --- a/test/automated/get/iterator/no_further_event_data.rb +++ b/test/automated/get/iterator/no_further_event_data.rb @@ -4,9 +4,9 @@ context "No further event data" do stream_name = Controls::Put.(instances: 2) - get = Get.build(stream_name) + get = Get.build - iterator = Iterator.build(get) + iterator = Iterator.build(get, stream_name) 2.times { iterator.next } diff --git a/test/automated/select_statement/stream_name_query.rb b/test/automated/select_statement/stream_name_query.rb index 6373530..f879a2a 100644 --- a/test/automated/select_statement/stream_name_query.rb +++ b/test/automated/select_statement/stream_name_query.rb @@ -2,9 +2,9 @@ context "Select Statement" do context "Stream Name Query" do - stream = Controls::Stream.example + stream_name = Controls::StreamName.example - select_statement = Get::SelectStatement.build stream + select_statement = Get::SelectStatement.build(stream_name) sql = select_statement.sql sql.gsub!(/\s+/, ' ') From 2ac317c30a2f2ae0278df98ca54712c4bce1390e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 24 Nov 2016 02:50:10 -0600 Subject: [PATCH 120/591] Package version is increased from 0.8.1.0 to 0.9.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 572c8b8..0165f54 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.8.1.0' + s.version = '0.9.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From dd3e98eb31443a3ed36d42a64b7ba0077aacb9e5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 25 Nov 2016 13:08:47 -0600 Subject: [PATCH 121/591] Empty metadata is stored as nil, rather than a serialized empty hash --- lib/event_source/postgres/put.rb | 5 +++++ test/automated/put/metadata/empty.rb | 22 ++++++++++++++++++++++ test/automated/put/metadata/nil.rb | 22 ++++++++++++++++++++++ test/automated/put/nil_metadata.rb | 20 -------------------- 4 files changed, 49 insertions(+), 20 deletions(-) create mode 100644 test/automated/put/metadata/empty.rb create mode 100644 test/automated/put/metadata/nil.rb delete mode 100644 test/automated/put/nil_metadata.rb diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 15ac989..4d5279b 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -92,6 +92,11 @@ def serialized_data(data) def serialized_metadata(metadata) serialized_metadata = nil + + if metadata.is_a?(Hash) && metadata.empty? + metadata = nil + end + unless metadata.nil? serializable_metadata = EventData::Hash[metadata] serialized_metadata = Transform::Write.(serializable_metadata, :json) diff --git a/test/automated/put/metadata/empty.rb b/test/automated/put/metadata/empty.rb new file mode 100644 index 0000000..ce2c617 --- /dev/null +++ b/test/automated/put/metadata/empty.rb @@ -0,0 +1,22 @@ +require_relative '../../automated_init' + +context "Put" do + context "Metadata" do + context "Nil" do + stream_name = Controls::StreamName.example + write_event = Controls::EventData::Write.example + + write_event.metadata = {} + + position = Put.(write_event, stream_name) + + read_event = Get.(stream_name, position: position).first + + context "Read metadata" do + test "Is nil" do + assert(read_event.metadata.nil?) + end + end + end + end +end diff --git a/test/automated/put/metadata/nil.rb b/test/automated/put/metadata/nil.rb new file mode 100644 index 0000000..fc50526 --- /dev/null +++ b/test/automated/put/metadata/nil.rb @@ -0,0 +1,22 @@ +require_relative '../../automated_init' + +context "Put" do + context "Metadata" do + context "Nil" do + stream_name = Controls::StreamName.example + write_event = Controls::EventData::Write.example + + write_event.metadata = nil + + position = Put.(write_event, stream_name) + + read_event = Get.(stream_name, position: position).first + + context "Read metadata" do + test "Is nil" do + assert(read_event.metadata.nil?) + end + end + end + end +end diff --git a/test/automated/put/nil_metadata.rb b/test/automated/put/nil_metadata.rb deleted file mode 100644 index 13fe7a4..0000000 --- a/test/automated/put/nil_metadata.rb +++ /dev/null @@ -1,20 +0,0 @@ -require_relative '../automated_init' - -context "Put" do - context "Metadata is nil" do - stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example - - write_event.metadata = nil - - position = Put.(write_event, stream_name) - - read_event = Get.(stream_name, position: position).first - - context "Read metadata" do - test "Is nil" do - assert(read_event.metadata.nil?) - end - end - end -end From b9f36a13920e815048c3e893ef8e0479d8852aec Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 25 Nov 2016 13:43:45 -0600 Subject: [PATCH 122/591] Data and metadata are nilable, and stored as nil --- database/table/events-table.sql | 2 +- lib/event_source/postgres/get.rb | 8 +++----- lib/event_source/postgres/put.rb | 14 ++++++++++++-- test/automated/put/data/empty.rb | 21 +++++++++++++++++++++ test/automated/put/data/nil.rb | 20 ++++++++++++++++++++ test/automated/put/metadata/empty.rb | 5 +++-- test/automated/put/metadata/nil.rb | 3 +-- 7 files changed, 61 insertions(+), 12 deletions(-) create mode 100644 test/automated/put/data/empty.rb create mode 100644 test/automated/put/data/nil.rb diff --git a/database/table/events-table.sql b/database/table/events-table.sql index 80a2e05..3d8805a 100644 --- a/database/table/events-table.sql +++ b/database/table/events-table.sql @@ -7,7 +7,7 @@ CREATE TABLE "public"."events" ( "position" int4 NOT NULL, "type" varchar(255) NOT NULL COLLATE "default", "global_position" bigserial NOT NULL , - "data" jsonb NOT NULL, + "data" jsonb, "metadata" jsonb, "time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL ) diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 4a67db3..705a336 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -71,15 +71,13 @@ def convert(records) module Deserialize def self.data(serialized_data) + return nil if serialized_data.nil? Transform::Read.(serialized_data, EventData::Hash, :json) end def self.metadata(serialized_metadata) - if serialized_metadata.nil? - nil - else - Transform::Read.(serialized_metadata, EventData::Hash, :json) - end + return nil if serialized_metadata.nil? + Transform::Read.(serialized_metadata, EventData::Hash, :json) end end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 4d5279b..7dc4e5a 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -84,8 +84,17 @@ def self.statement end def serialized_data(data) - serializable_data = EventData::Hash[data] - serialized_data = Transform::Write.(serializable_data, :json) + serialized_data = nil + + if data.is_a?(Hash) && data.empty? + data = nil + end + + unless data.nil? + serializable_data = EventData::Hash[data] + serialized_data = Transform::Write.(serializable_data, :json) + end + logger.debug(tags: [:data, :serialize]) { "Serialized Data: #{serialized_data.inspect}" } serialized_data end @@ -101,6 +110,7 @@ def serialized_metadata(metadata) serializable_metadata = EventData::Hash[metadata] serialized_metadata = Transform::Write.(serializable_metadata, :json) end + logger.debug(tags: [:data, :serialize]) { "Serialized Metadata: #{serialized_metadata.inspect}" } serialized_metadata end diff --git a/test/automated/put/data/empty.rb b/test/automated/put/data/empty.rb new file mode 100644 index 0000000..c34e3c2 --- /dev/null +++ b/test/automated/put/data/empty.rb @@ -0,0 +1,21 @@ +require_relative '../../automated_init' + +context "Put" do + context "Metadata" do + context "Nil" do + stream_name = Controls::StreamName.example + + write_event = Controls::EventData::Write.example(data: {}) + + position = Put.(write_event, stream_name) + + read_event = Get.(stream_name, position: position).first + + context "Read metadata" do + test "Is nil" do + assert(read_event.data.nil?) + end + end + end + end +end diff --git a/test/automated/put/data/nil.rb b/test/automated/put/data/nil.rb new file mode 100644 index 0000000..2ae1bb7 --- /dev/null +++ b/test/automated/put/data/nil.rb @@ -0,0 +1,20 @@ +require_relative '../../automated_init' + +context "Put" do + context "Data" do + context "Nil" do + stream_name = Controls::StreamName.example + write_event = Controls::EventData::Write.example(data: :none) + + position = Put.(write_event, stream_name) + + read_event = Get.(stream_name, position: position).first + + context "Read metadata" do + test "Is nil" do + assert(read_event.data.nil?) + end + end + end + end +end diff --git a/test/automated/put/metadata/empty.rb b/test/automated/put/metadata/empty.rb index ce2c617..8bdff4f 100644 --- a/test/automated/put/metadata/empty.rb +++ b/test/automated/put/metadata/empty.rb @@ -4,9 +4,10 @@ context "Metadata" do context "Nil" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example - write_event.metadata = {} + write_event = Controls::EventData::Write.example(metadata: {}) + + # write_event.metadata = {} position = Put.(write_event, stream_name) diff --git a/test/automated/put/metadata/nil.rb b/test/automated/put/metadata/nil.rb index fc50526..3ecba32 100644 --- a/test/automated/put/metadata/nil.rb +++ b/test/automated/put/metadata/nil.rb @@ -4,9 +4,8 @@ context "Metadata" do context "Nil" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example - write_event.metadata = nil + write_event = Controls::EventData::Write.example(metadata: :none) position = Put.(write_event, stream_name) From 7e093ac62fe1abf4388aa2da12c18a827157567f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 25 Nov 2016 14:04:07 -0600 Subject: [PATCH 123/591] Package version is increased from 0.9.0.0 to 0.9.1.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 0165f54..dfc18eb 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.9.0.0' + s.version = '0.9.1.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 57093b23ee893a10a982b63d9a185ceb33c67dd3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 26 Nov 2016 16:17:14 -0600 Subject: [PATCH 124/591] Read's cycle parameters are prefixed with cycle --- test/interactive/reader_does_not_terminate.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/interactive/reader_does_not_terminate.rb b/test/interactive/reader_does_not_terminate.rb index fdc95ea..5cebd9f 100644 --- a/test/interactive/reader_does_not_terminate.rb +++ b/test/interactive/reader_does_not_terminate.rb @@ -2,4 +2,4 @@ stream_name = Controls::StreamName.example -Read.(stream_name, batch_size: 1, delay_milliseconds: 200) { |event_data| } +Read.(stream_name, batch_size: 1, cycle_delay_milliseconds: 200) { |event_data| } From bba2031da4307bed33e3c90abc370853945f3dac Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 30 Nov 2016 09:26:44 -0600 Subject: [PATCH 125/591] Pretty printing is already activated --- test/test_init.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_init.rb b/test/test_init.rb index a02b760..ccb7400 100644 --- a/test/test_init.rb +++ b/test/test_init.rb @@ -8,7 +8,5 @@ require 'test_bench'; TestBench.activate -require 'pp' - include EventSource include EventSource::Postgres From f5697d93167b4ad43ffae9ee93813a98b7bbcad6 Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Wed, 7 Dec 2016 20:47:45 -0600 Subject: [PATCH 126/591] Project name is corrected in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b23d82c..8c4571f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ ## License -The `event_stream-postgres` library is released under the [MIT License](https://github.com/eventide-project/event-stream-postgres/blob/master/MIT-License.txt). +The `event_source-postgres` library is released under the [MIT License](https://github.com/eventide-project/event-source-postgres/blob/master/MIT-License.txt). From 46e480c54f5b5fae7611e375918f245adcc8626c Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Wed, 7 Dec 2016 20:50:23 -0600 Subject: [PATCH 127/591] Project name is corrected in remove-lib-symlinks.sh --- remove-lib-symlinks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remove-lib-symlinks.sh b/remove-lib-symlinks.sh index 2227b7c..6146bbd 100755 --- a/remove-lib-symlinks.sh +++ b/remove-lib-symlinks.sh @@ -1,3 +1,3 @@ source ./library-symlinks.sh -remove_lib_symlinks 'event_stream' 'postgres' +remove_lib_symlinks 'event_source' 'postgres' From c4e49f721c9e2d1d93d2ddd8dbb84042bf95e4f3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 9 Dec 2016 18:03:02 -0600 Subject: [PATCH 128/591] Session test folder --- test/automated/session/session.rb | 10 ++++++++++ test/automated/{session.rb => session/settings.rb} | 9 +-------- 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 test/automated/session/session.rb rename test/automated/{session.rb => session/settings.rb} (70%) diff --git a/test/automated/session/session.rb b/test/automated/session/session.rb new file mode 100644 index 0000000..a24cbe9 --- /dev/null +++ b/test/automated/session/session.rb @@ -0,0 +1,10 @@ +require_relative '../automated_init' + +context "Session" do + test "Connected" do + connected_session = Session.build + connected = connected_session.connected? + + assert(connected) + end +end diff --git a/test/automated/session.rb b/test/automated/session/settings.rb similarity index 70% rename from test/automated/session.rb rename to test/automated/session/settings.rb index 9a97756..6676102 100644 --- a/test/automated/session.rb +++ b/test/automated/session/settings.rb @@ -1,13 +1,6 @@ -require_relative 'automated_init' +require_relative '../automated_init' context "Session" do - test "Connected" do - connected_session = Session.build - connected = connected_session.connected? - - assert(connected) - end - context "Settings" do session = Session.build From 4e51870f07263974f44ef2e3181cc12af74b94ec Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 9 Dec 2016 20:46:53 -0600 Subject: [PATCH 129/591] Connection is made only at the point of use of the connection, either for executing a statement or starting a transaction. --- lib/event_source/postgres/session.rb | 31 ++++++++++++------------ test/automated/session/_initial_state.rb | 12 +++++++++ test/automated/session/session.rb | 7 ++---- 3 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 test/automated/session/_initial_state.rb diff --git a/lib/event_source/postgres/session.rb b/lib/event_source/postgres/session.rb index 3d4854b..386db64 100644 --- a/lib/event_source/postgres/session.rb +++ b/lib/event_source/postgres/session.rb @@ -13,13 +13,10 @@ def self.settings attr_accessor :connection - def self.build(connection: nil, settings: nil) + def self.build(settings: nil) new.tap do |instance| settings ||= Settings.instance - settings.set(instance) - - connect(instance, connection) end end @@ -31,17 +28,17 @@ def self.configure(receiver, session: nil, attr_name: nil) instance end - def self.connect(instance, connection=nil) + def connect logger.trace "Connecting to database" - if connection.nil? - logger.debug { "No connection. A new one will be built." } - connection = build_connection(instance) - else - logger.debug { "Reusing existing connection" } + if connected? + logger.debug { "Already connected. A new connection will not be built." } + return end - instance.connection = connection + logger.debug { "Not connected. A new connection will be built." } + connection = self.class.build_connection(self) + self.connection = connection logger.debug { "Connected to database" } @@ -60,10 +57,6 @@ def self.build_connection(instance) connection end - def connect - self.class.connect(self) - end - def connected? !connection.nil? && connection.status == PG::CONNECTION_OK end @@ -88,6 +81,10 @@ def settings end def execute(statement, params=nil) + unless connected? + connect + end + if params.nil? connection.exec(statement) else @@ -96,6 +93,10 @@ def execute(statement, params=nil) end def transaction(&blk) + unless connected? + connect + end + connection.transaction(&blk) end diff --git a/test/automated/session/_initial_state.rb b/test/automated/session/_initial_state.rb new file mode 100644 index 0000000..01cd50d --- /dev/null +++ b/test/automated/session/_initial_state.rb @@ -0,0 +1,12 @@ +require_relative '../automated_init' + +context "Session" do + test "Initial State" do + session = Session.build + connected = session.connected? + + test "Not connected" do + refute(connected) + end + end +end diff --git a/test/automated/session/session.rb b/test/automated/session/session.rb index a24cbe9..6d67e2c 100644 --- a/test/automated/session/session.rb +++ b/test/automated/session/session.rb @@ -1,10 +1,7 @@ require_relative '../automated_init' context "Session" do - test "Connected" do - connected_session = Session.build - connected = connected_session.connected? - - assert(connected) + test "On First Use" do + test "Connects" end end From bcfcb062bdd65ad3922c63ecc03518b07a34b130 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 9 Dec 2016 21:11:11 -0600 Subject: [PATCH 130/591] Package version increased from 0.9.1.0 to 0.10.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index dfc18eb..9ff5885 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'event_source-postgres' - s.version = '0.9.1.0' + s.version = '0.10.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 0e33010996d29ba470e5e5ac2a63b9ac723ed679 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 9 Dec 2016 22:32:45 -0600 Subject: [PATCH 131/591] Materials folder is removed --- materials/event_store_write_batch_tests.rb | 37 ----- materials/messaging_substitute_tests.rb | 116 -------------- materials/messaging_telemetry_tests.rb | 55 ------- materials/messaging_writer.rb | 177 --------------------- materials/nathans_sketch.rb | 33 ---- materials/new_school_sketch.rb | 82 ---------- materials/poll.rb | 21 --- materials/retry_batch_message_sketch.rb | 64 -------- materials/subscription_sketch.rb | 13 -- 9 files changed, 598 deletions(-) delete mode 100644 materials/event_store_write_batch_tests.rb delete mode 100644 materials/messaging_substitute_tests.rb delete mode 100644 materials/messaging_telemetry_tests.rb delete mode 100644 materials/messaging_writer.rb delete mode 100644 materials/nathans_sketch.rb delete mode 100644 materials/new_school_sketch.rb delete mode 100644 materials/poll.rb delete mode 100644 materials/retry_batch_message_sketch.rb delete mode 100644 materials/subscription_sketch.rb diff --git a/materials/event_store_write_batch_tests.rb b/materials/event_store_write_batch_tests.rb deleted file mode 100644 index 0321d82..0000000 --- a/materials/event_store_write_batch_tests.rb +++ /dev/null @@ -1,37 +0,0 @@ -require_relative '../bench_init' - -context "Write Batch of Events" do - stream_name = EventStore::Client::HTTP::Controls::StreamName.get 'testEventWriter' - path = "/streams/#{stream_name}" - - writer = EventStore::Client::HTTP::EventWriter.build - - id_1 = EventStore::Client::HTTP::Controls::ID.example 1 - id_2 = EventStore::Client::HTTP::Controls::ID.example 2 - - event_data_1 = EventStore::Client::HTTP::Controls::EventData::Write.example(id_1) - event_data_2 = EventStore::Client::HTTP::Controls::EventData::Write.example(id_2) - - event_data_1.data[:some_attribute] = id_1 - event_data_2.data[:some_attribute] = id_2 - - writer.write [event_data_1, event_data_2], stream_name - - get = EventStore::Client::HTTP::Request::Get.build - body_text_1, get_response = get.("#{path}/0") - body_text_2, get_response = get.("#{path}/1") - - read_data_1 = Serialize::Read.(body_text_1, EventStore::Client::HTTP::EventData::Read, :json) - read_data_2 = Serialize::Read.(body_text_2, EventStore::Client::HTTP::EventData::Read, :json) - - 2.times do |i| - i += 1 - event_data = binding.local_variable_get "read_data_#{i}" - - test "Individual events are written" do - control_value = binding.local_variable_get "id_#{i}" - - assert event_data.data[:some_attribute] == control_value - end - end -end diff --git a/materials/messaging_substitute_tests.rb b/materials/messaging_substitute_tests.rb deleted file mode 100644 index da4c5fc..0000000 --- a/materials/messaging_substitute_tests.rb +++ /dev/null @@ -1,116 +0,0 @@ -require_relative '../bench_init' - -context "Writer Substitute" do - context "Records writes" do - substitute_writer = EventStore::Messaging::Writer::Substitute.build - - message = EventStore::Messaging::Controls::Message.example - - stream_name = 'some stream name' - - substitute_writer.write message, stream_name, expected_version: 11, reply_stream_name: 'some_stream_name' - - context "Records telemetry about the write" do - test "No block arguments" do - assert(substitute_writer.written?) - end - - test "Message block argument only" do - assert(substitute_writer.written? { |msg| msg == message }) - end - - test "Message and stream name block arguments" do - assert(substitute_writer.written? { |msg, stream| stream == stream_name }) - end - - test "Message, stream name, and expected_version block arguments" do - assert(substitute_writer.written? { |msg, stream, expected_version | expected_version == 11 }) - end - - test "Message, stream name, expected_version, and reply_stream_name block arguments" do - assert(substitute_writer.written? { |msg, stream, expected_version, reply_stream_name | reply_stream_name == 'some_stream_name' }) - end - end - - context "Access the data recorded" do - test "No block arguments" do - assert(substitute_writer.writes.length == 1) - end - - test "Message block argument only" do - assert(substitute_writer.writes { |msg| msg == message }.length == 1 ) - end - - test "Message and stream name block arguments" do - assert(substitute_writer.writes { |msg, stream| stream == stream_name }.length == 1) - end - - test "Message, stream name, and expected_version block arguments" do - assert(substitute_writer.writes { |msg, stream, expected_version | expected_version == 11 }.length == 1) - end - - test "Message, stream name, expected_version, and reply_stream_name block arguments" do - assert(substitute_writer.writes { |msg, stream, expected_version, reply_stream_name | reply_stream_name == 'some_stream_name' }.length == 1) - end - end - end - - context "Records batch writes" do - substitute_writer = EventStore::Messaging::Writer::Substitute.build - - message_1 = EventStore::Messaging::Controls::Message.example - message_2 = EventStore::Messaging::Controls::Message.example - - stream_name = 'some stream name' - - substitute_writer.write [message_1, message_2], stream_name - - context "Records telemetry about each message written" do - assert substitute_writer do - written? { |msg| msg == message_1 } - end - - assert substitute_writer do - written? { |msg| msg == message_2 } - end - end - end - - context "Records replies" do - substitute_writer = EventStore::Messaging::Writer::Substitute.build - - message = EventStore::Messaging::Controls::Message.example - - stream_name = message.metadata.reply_stream_name - - substitute_writer.reply message - - context "Records replied telemetry" do - test "No block arguments" do - assert(substitute_writer.written?) - end - - test "Message argument only" do - assert(substitute_writer.replied? { |msg| msg == message }) - end - - test "Message and stream name arguments" do - assert(substitute_writer.replied? { |msg, stream| stream == stream_name }) - end - end - - context "Access the data recorded" do - test "No block arguments" do - assert(substitute_writer.replies.length == 1) - end - - test "Message block argument only" do - assert(substitute_writer.replies { |msg| msg == message }.length == 1 ) - end - - test "Message and stream name block arguments" do - assert(substitute_writer.replies { |msg, stream| stream == stream_name }.length == 1) - end - end - end -end diff --git a/materials/messaging_telemetry_tests.rb b/materials/messaging_telemetry_tests.rb deleted file mode 100644 index 918f56e..0000000 --- a/materials/messaging_telemetry_tests.rb +++ /dev/null @@ -1,55 +0,0 @@ -require_relative '../bench_init' - -context "Writer Telemetry" do - context "Write" do - message = EventStore::Messaging::Controls::Message.example - writer = EventStore::Messaging::Writer.build - - SubstAttr::Substitute.(:writer, writer) - - stream_name = EventStore::Messaging::Controls::StreamName.get 'testWriter' - - sink = EventStore::Messaging::Writer.register_telemetry_sink(writer) - - writer.write message, stream_name, expected_version: 11, reply_stream_name: 'some_stream_name' - - test "Records written telemetry" do - assert(sink.recorded_written?) - end - - context "Recorded Data" do - data = sink.records[0].data - - test "message" do - assert(data.message == message) - end - - test "stream_name" do - assert(data.stream_name == stream_name) - end - - test "expected_version" do - assert(data.expected_version == 11) - end - - test "reply_stream_name" do - assert(data.reply_stream_name == 'some_stream_name') - end - end - end - - context "Reply" do - message = EventStore::Messaging::Controls::Message.example - writer = EventStore::Messaging::Writer.build - - SubstAttr::Substitute.(:writer, writer) - - sink = EventStore::Messaging::Writer.register_telemetry_sink(writer) - - writer.reply message - - test "Records replied telemetry" do - assert(sink.recorded_replied?) - end - end -end diff --git a/materials/messaging_writer.rb b/materials/messaging_writer.rb deleted file mode 100644 index b8922b4..0000000 --- a/materials/messaging_writer.rb +++ /dev/null @@ -1,177 +0,0 @@ -module EventStore - module Messaging - class Writer - class Error < StandardError; end - - dependency :writer, EventStore::Client::HTTP::EventWriter - dependency :logger, ::Telemetry::Logger - dependency :telemetry, ::Telemetry - - def self.build(session: nil) - new.tap do |instance| - EventStore::Client::HTTP::EventWriter.configure instance, session: session - ::Telemetry::Logger.configure instance - ::Telemetry.configure instance - end - end - - def self.configure(receiver, session: nil) - instance = build(session: session) - receiver.writer = instance - instance - end - - def write(message, stream_name, expected_version: nil, reply_stream_name: nil) - unless message.is_a? Array - logger.trace "Writing (Message Type: #{message.message_type}, Stream Name: #{stream_name}, Expected Version: #{!!expected_version ? expected_version : '(none)'})" - else - logger.trace "Writing batch (Stream Name: #{stream_name}, Expected Version: #{!!expected_version ? expected_version : '(none)'})" - end - - if reply_stream_name - message.metadata.reply_stream_name = reply_stream_name - end - - event_data = event_data_batch(message) - - writer.write(event_data, stream_name, expected_version: expected_version) - - unless message.is_a? Array - logger.debug "Wrote (Message Type: #{message.message_type}, Stream Name: #{stream_name}, Expected Version: #{!!expected_version ? expected_version : '(none)'})" - else - logger.debug "Wrote batch (Stream Name: #{stream_name}, Expected Version: #{!!expected_version ? expected_version : '(none)'})" - end - - Array(message).each do |written_message| - telemetry.record :written, Telemetry::Data.new(written_message, stream_name, expected_version, reply_stream_name) - end - - event_data - end - - def write_initial(message, stream_name) - write(message, stream_name, expected_version: :no_stream) - end - - def event_data_batch(messages) - messages = [messages] unless messages.is_a? Array - - batch = messages.map do |message| - EventStore::Messaging::Message::Export::EventData.(message) - end - - batch - end - - def reply(message) - metadata = message.metadata - reply_stream_name = metadata.reply_stream_name - - logger.trace "Replying (Message Type: #{message.message_type}, Stream Name: #{reply_stream_name})" - - unless reply_stream_name - error_msg = "Message has no reply stream name. Cannot reply. (Message Type: #{message.message_type})" - logger.error error_msg - raise Error, error_msg - end - - metadata.clear_reply_stream_name - - write message, reply_stream_name - - logger.debug "Replied (Message Type: #{message.message_type}, Stream Name: #{reply_stream_name})" - - telemetry.record :replied, Telemetry::Data.new(message, reply_stream_name) - - message - end - - def self.logger - @logger ||= ::Telemetry::Logger.get self - end - - def self.register_telemetry_sink(writer) - sink = Telemetry.sink - writer.telemetry.register sink - sink - end - - module Telemetry - class Sink - include ::Telemetry::Sink - - record :written - record :replied - end - - Data = Struct.new :message, :stream_name, :expected_version, :reply_stream_name - - def self.sink - Sink.new - end - end - - module Substitute - def self.build - Substitute::Writer.build.tap do |substitute_writer| - sink = Messaging::Writer.register_telemetry_sink(substitute_writer) - substitute_writer.sink = sink - end - end - - class Writer < EventStore::Messaging::Writer - attr_accessor :sink - - def self.build(session: nil) - logger.trace "Building substitute" - new.tap do |instance| - ::Telemetry::Logger.configure instance - ::Telemetry.configure instance - logger.debug "Built substitute" - end - end - - def writes(&blk) - if blk.nil? - return sink.written_records - end - - sink.written_records.select do |record| - blk.call(record.data.message, record.data.stream_name, record.data.expected_version, record.data.reply_stream_name) - end - end - - def written?(&blk) - if blk.nil? - return sink.recorded_written? - end - - sink.recorded_written? do |record| - blk.call(record.data.message, record.data.stream_name, record.data.expected_version, record.data.reply_stream_name) - end - end - - def replies(&blk) - if blk.nil? - return sink.replied_records - end - - sink.replied_records.select do |record| - blk.call(record.data.message, record.data.stream_name) - end - end - - def replied?(&blk) - if blk.nil? - return sink.recorded_replied? - end - - sink.recorded_replied? do |record| - blk.call(record.data.message, record.data.stream_name) - end - end - end - end - end - end -end diff --git a/materials/nathans_sketch.rb b/materials/nathans_sketch.rb deleted file mode 100644 index 972b012..0000000 --- a/materials/nathans_sketch.rb +++ /dev/null @@ -1,33 +0,0 @@ -class PostgresqlReader - attr_writer :batch - attr_writer :position - - def initialize(category) - @category = category - end - - # Implementing the reader as a pull system - def next - row = pg.execute "SELECT * FROM events WHERE position = ? AND category = ?", position, category - - position += 1 - - event_data = Serialize::Read.(row, :postgresql, EventData) - - event_data - end - - # the pull system (actuated via #next) easily can become a push system. - # The inverse, going from push to pull, requires synchronization via - # eg fibers, Enumerable::Lazy. etc. - def start(&block) - loop do - event_data = self.next - block.(event_data) - end - end - - def position - @position ||= 0 - end -end diff --git a/materials/new_school_sketch.rb b/materials/new_school_sketch.rb deleted file mode 100644 index bfc2a43..0000000 --- a/materials/new_school_sketch.rb +++ /dev/null @@ -1,82 +0,0 @@ -# calls the handle method even if it doesn't exist -handler.send(message) - -# only calls the handle method if it exists -handler.handle(message) - -# Proposal: Just the reader and writer in the client library -# No messaging reader, subscription, or writer in the -# messaging library - -Read.(stream: stream_name) do |event_data| - dispatcher.(event_data) -end - -# or the instance - -reader.(stream: stream_name) do |event_data| - dispatcher.(event_data) -end - -# Note: no need for messaging reader above. The dispatcher -# is the place where deserialization is done right now. -# So there's no need for a Messaging::MessageReader. - -# the handler is still in the dispatcher - -class SomeDispatcher - handler SomeHandler -end - -# This is a subscription - -Read.(category: category_stream_name, poll_milliseconds: 500) do |event_data| - # ... -end - -# Projection - -# (from within the projection's instance code) -reader.(stream_name: stream_name) do |event_data| - apply(event_data) -end - -# In order to convert to/from event_data and message -# types, the reader needs to get a list of types -# So, message types can be had from anything that -# implements a message registry method, returning -# the existing implementation of a Message::Registry - -message_classes = [ - SomeMessage, - SomeOtherMessage -] - -Read.(stream_name: stream_name) do |event_data| - handler.(event_data) - # handler has enough info about message classes that it can - # xvert them -end - -# The writer -# This converts to event_data -writer.(message) - -# This doesn't need to convert to event_data -writer.(event_data) - -# Or with the optional block -writer.(message) do |event_data| - # do something with the event_data if wanted - # this block is for transparency to the opaque event_data before it's saved -end - -# Writer returns event_data, either way -event_data = writer.(message) -event_data = writer.(event_data) - - -# The writer's class actuator -Write.(message, message_classes) - # why does this take message classes? - # it does not de-serialize diff --git a/materials/poll.rb b/materials/poll.rb deleted file mode 100644 index 1e44be4..0000000 --- a/materials/poll.rb +++ /dev/null @@ -1,21 +0,0 @@ -Read.(category: category_stream_name, poll_milliseconds: 500) do |event_data| - # ... -end - - -Poll.(delay_milliseconds: 500, timeout_milliseconds: 5000) do |poll_controller| - Read.(category: category_stream_name) - # has to decide how and whether to poll again - # if result is a nil, then it has to poll - # internally, poll tracks iterations, timeout. does reset of timeout, etc - -end - - -# Maybe an internal poll control? - - res = get_events ... - - if res.nil? && poll? # <- poll milliseconds are set - end - diff --git a/materials/retry_batch_message_sketch.rb b/materials/retry_batch_message_sketch.rb deleted file mode 100644 index e309ef6..0000000 --- a/materials/retry_batch_message_sketch.rb +++ /dev/null @@ -1,64 +0,0 @@ -require 'securerandom' -require 'http_eventstore' -require 'active_support/core_ext/array' - -# Process message -class MessageProcessor - - def initialize - @client = HttpEventstore::Connection.new - end - - def process_message(message) - puts "Receieved message #{message.id}" - product_ids = get_product_ids(message) - - product_ids.in_groups_of(5, false).each_with_index do |group, index| - if index < 2 - stream_name = "inventory" - event_data = { event_type: "InventoryItemUpdated", - data: { product_ids: group }, - metadata: { "$correlationId": message.id, - "$causationId": message.id }, - event_id: SecureRandom.uuid } - puts "Processed ids #{group.join(',')}" - puts "Publishing event #{event_data}" - client.append_to_stream(stream_name, event_data) - else - puts "Could not process ids #{group.join(',')}" - raise "Some Error" - end - end - - end - - private - - attr_reader :client - - def get_product_ids(message) - # Here we would query the event store by the message id - # to find all events that may have completed. - # the completed event product_ids would be removed - # from the original message, to resume the updates - message.product_ids - end -end - -PRODUCT_IDS = (0..20).to_a -MESSAGE_ID = SecureRandom.uuid - -message_processor = MessageProcessor.new -Message = Struct.new(:id, :product_ids) -retry_count = 0 - -begin - message_processor.process_message(message = Message.new(MESSAGE_ID, PRODUCT_IDS)) -rescue RuntimeError => e - if retry_count > 3 - puts "Dead letter message, couldnt process message #{message.id}" - else - retry_count += 1 - retry - end -end diff --git a/materials/subscription_sketch.rb b/materials/subscription_sketch.rb deleted file mode 100644 index 8257eab..0000000 --- a/materials/subscription_sketch.rb +++ /dev/null @@ -1,13 +0,0 @@ -# Timeout and retry - -retry.() do - loop do - get_batch - - if no_batch - break unless timeout.nil? # maybe raise Timeout::Error - - sleep timeout - end - end -end From 6d4b8bb1e9560635bab9004b3c4667168084ba30 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 14 Dec 2016 13:21:09 -0600 Subject: [PATCH 132/591] Table name is event_source --- database/clear-events-table.sh | 2 +- database/install.sh | 2 +- database/list-events.sh | 2 +- database/uninstall.sh | 2 +- settings/event_source_postgres.json | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/database/clear-events-table.sh b/database/clear-events-table.sh index 75099d3..de683ac 100755 --- a/database/clear-events-table.sh +++ b/database/clear-events-table.sh @@ -9,7 +9,7 @@ echo "Clearing Events Table" echo "= = =" echo -default_name=eventsource +default_name=event_source if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" diff --git a/database/install.sh b/database/install.sh index 181641b..8cbc143 100755 --- a/database/install.sh +++ b/database/install.sh @@ -9,7 +9,7 @@ echo "Installing Database" echo "= = =" echo -default_name=eventsource +default_name=event_source if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" diff --git a/database/list-events.sh b/database/list-events.sh index 9715560..74d7920 100755 --- a/database/list-events.sh +++ b/database/list-events.sh @@ -9,7 +9,7 @@ echo "Listing Events" echo "= = =" echo -default_name=eventsource +default_name=event_source if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" diff --git a/database/uninstall.sh b/database/uninstall.sh index b2f5a9c..f099b55 100755 --- a/database/uninstall.sh +++ b/database/uninstall.sh @@ -9,7 +9,7 @@ echo "Uninstalling Database" echo "= = =" echo -default_name=eventsource +default_name=event_source if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" diff --git a/settings/event_source_postgres.json b/settings/event_source_postgres.json index 4f17e59..392938b 100644 --- a/settings/event_source_postgres.json +++ b/settings/event_source_postgres.json @@ -1,9 +1,9 @@ { - "dbname": "eventsource", + "dbname": "event_source", "host": "localhost", "hostaddr": "127.0.0.1", "port": 5432, - "user": "eventsource", + "user": "event_source", "password": null, "connect_timeout": null, "options": null, From e64eedaf377e0850ddf25610666daea85a37190c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 14 Dec 2016 14:55:28 -0600 Subject: [PATCH 133/591] Test for connect on first use --- .../session/{_initial_state.rb => initial_state.rb} | 0 test/automated/session/session.rb | 10 +++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) rename test/automated/session/{_initial_state.rb => initial_state.rb} (100%) diff --git a/test/automated/session/_initial_state.rb b/test/automated/session/initial_state.rb similarity index 100% rename from test/automated/session/_initial_state.rb rename to test/automated/session/initial_state.rb diff --git a/test/automated/session/session.rb b/test/automated/session/session.rb index 6d67e2c..9f4301d 100644 --- a/test/automated/session/session.rb +++ b/test/automated/session/session.rb @@ -2,6 +2,14 @@ context "Session" do test "On First Use" do - test "Connects" + session = Session.build + + refute(session.connected?) + + test "Connects" do + refute proc { session.execute('SELECT 1;') } do + raises_error? + end + end end end From d9058af98579a9be35b454a04e03f3464f40976d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 19 Dec 2016 17:47:46 -0600 Subject: [PATCH 134/591] 'Required Ruby version is 2.3.3' --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 9ff5885..e78266a 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -13,7 +13,7 @@ Gem::Specification.new do |s| s.require_paths = ['lib'] s.files = Dir.glob('{lib}/**/*') s.platform = Gem::Platform::RUBY - s.required_ruby_version = '>= 2.2.3' + s.required_ruby_version = '>= 2.3.3' s.add_runtime_dependency 'event_source' s.add_runtime_dependency 'log' From bcd432b17edc5504691e66210c2a1ae11f8dbeec Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 19 Dec 2016 21:37:27 -0600 Subject: [PATCH 135/591] Gem renamed to use the evt prefix --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index e78266a..14eaba5 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,6 +1,6 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| - s.name = 'event_source-postgres' + s.name = 'evt-event_source-postgres' s.version = '0.10.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 8b70c4ceb52f8828142ff3a741c9e42e2159565c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 19 Dec 2016 22:45:19 -0600 Subject: [PATCH 136/591] Dependencies on rubygems.org --- event_source-postgres.gemspec | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 14eaba5..cc1ef47 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -15,12 +15,12 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.required_ruby_version = '>= 2.3.3' - s.add_runtime_dependency 'event_source' - s.add_runtime_dependency 'log' - s.add_runtime_dependency 'cycle' - s.add_runtime_dependency 'settings' + s.add_runtime_dependency 'evt-event_source' + s.add_runtime_dependency 'evt-log' + s.add_runtime_dependency 'evt-cycle' + s.add_runtime_dependency 'evt-settings' s.add_runtime_dependency 'pg' - s.add_development_dependency 'test_bench' + s.add_development_dependency 'ntl-test_bench' end From 9902aaf294a6313b504ca2fd978c0aaf8fe9672b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 20 Dec 2016 00:54:10 -0600 Subject: [PATCH 137/591] Package version is increased from 0.10.0.0 to 0.10.0.1 --- event_source-postgres.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index cc1ef47..e54a2c8 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.10.0.0' + s.version = '0.10.0.1' s.summary = 'Event source client for PostgreSQL' s.description = ' ' @@ -22,5 +22,5 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'pg' - s.add_development_dependency 'ntl-test_bench' + s.add_development_dependency 'test_bench' end From 4ad5d5ca2c7c5a208edb0de2a1cdf9fb152c2f49 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 20 Dec 2016 18:02:44 -0600 Subject: [PATCH 138/591] Package version is increased from 0.10.0.1 to 0.10.0.2 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index e54a2c8..52e1504 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.10.0.1' + s.version = '0.10.0.2' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From b534ceae8db799a2f6f1ef3b171e9d59ad9cd29e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 10 Jan 2017 17:35:31 -0600 Subject: [PATCH 139/591] Specialized write code is moved from the generalized library --- lib/event_source/postgres/write.rb | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index e2250b2..657da7a 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -3,9 +3,44 @@ module Postgres class Write include EventSource::Write + dependency :put + def configure(session: nil) Put.configure(self, session: session) end + + def write(batch, stream_name, expected_version: nil) + logger.trace { "Writing batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } + + unless expected_version.nil? + expected_version = ExpectedVersion.canonize(expected_version) + end + + last_position = nil + put.session.transaction do + batch.each do |event_data| + last_position = write_event(event_data, stream_name, expected_version: expected_version) + + unless expected_version.nil? + expected_version += 1 + end + end + end + + logger.debug { "Wrote batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } + + last_position + end + + def write_event(event_data, stream_name, expected_version: nil) + logger.trace { "Writing event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } + logger.trace(tags: [:data, :event_data]) { event_data.pretty_inspect } + + put.(event_data, stream_name, expected_version: expected_version).tap do + logger.debug { "Wrote event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } + logger.debug(tags: [:data, :event_data]) { event_data.pretty_inspect } + end + end end end end From 8418f16eee39c2ba6ce783f4afde0e8ca3e36537 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 10 Jan 2017 17:42:50 -0600 Subject: [PATCH 140/591] Package version is increased from 0.10.0.2 to 0.11.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 52e1504..914accc 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.10.0.2' + s.version = '0.11.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 74dcec81c1f7fdf4de4e1bbe10180d2cdbc22b09 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 10 Jan 2017 17:46:46 -0600 Subject: [PATCH 141/591] write_event_data, rather than write_event --- lib/event_source/postgres/write.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index 657da7a..5e50720 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -32,7 +32,7 @@ def write(batch, stream_name, expected_version: nil) last_position end - def write_event(event_data, stream_name, expected_version: nil) + def write_event_data(event_data, stream_name, expected_version: nil) logger.trace { "Writing event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } logger.trace(tags: [:data, :event_data]) { event_data.pretty_inspect } From 49c489b8fa2495710a709ee14f2915d70335f309 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 10 Jan 2017 17:49:44 -0600 Subject: [PATCH 142/591] Package version is increased from 0.11.0.0 to 0.12.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 914accc..5d50124 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.11.0.0' + s.version = '0.12.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 1d38eefd5399d83434f436a8684a551e51b387c3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 10 Jan 2017 18:38:03 -0600 Subject: [PATCH 143/591] Callsite was not updated --- lib/event_source/postgres/write.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index 5e50720..6a8b10b 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -19,7 +19,7 @@ def write(batch, stream_name, expected_version: nil) last_position = nil put.session.transaction do batch.each do |event_data| - last_position = write_event(event_data, stream_name, expected_version: expected_version) + last_position = write_event_data(event_data, stream_name, expected_version: expected_version) unless expected_version.nil? expected_version += 1 From 2d63e0f344c28cff3fe94c100e41f37df35b8105 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 10 Jan 2017 18:38:39 -0600 Subject: [PATCH 144/591] Package version is increased from 0.12.0.0 to 0.12.0.1 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 5d50124..0727b54 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.12.0.0' + s.version = '0.12.0.1' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From ab9bf2370cab2b6d49f83ad546f25b0ded3a8938 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 11 Jan 2017 16:28:34 -0600 Subject: [PATCH 145/591] The events table has an ID field of UUID type that is defaulted to a pgcrypto random UUID --- database/extensions.sql | 1 + database/indexes/events-indexes.sql | 1 + database/install.sh | 5 +++++ database/table/events-table.sql | 1 + 4 files changed, 8 insertions(+) create mode 100644 database/extensions.sql diff --git a/database/extensions.sql b/database/extensions.sql new file mode 100644 index 0000000..7ca3626 --- /dev/null +++ b/database/extensions.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS "pgcrypto"; diff --git a/database/indexes/events-indexes.sql b/database/indexes/events-indexes.sql index f531b9b..727c8ee 100644 --- a/database/indexes/events-indexes.sql +++ b/database/indexes/events-indexes.sql @@ -1,6 +1,7 @@ -- ---------------------------- -- Indexes structure for table events -- ---------------------------- +CREATE INDEX CONCURRENTLY "events_id_idx" ON "public"."events" USING btree(id ASC NULLS LAST); CREATE INDEX CONCURRENTLY "events_category_global_position_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "global_position" "pg_catalog"."int8_ops" ASC NULLS LAST); CREATE INDEX CONCURRENTLY "events_category_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); CREATE INDEX CONCURRENTLY "events_stream_name_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); diff --git a/database/install.sh b/database/install.sh index 8cbc143..f1826b0 100755 --- a/database/install.sh +++ b/database/install.sh @@ -59,6 +59,10 @@ function create-database { echo } +function create-extensions { + psql $database -f database/extensions.sql +} + function create-table { psql $database -f database/table/events-table.sql } @@ -75,6 +79,7 @@ function create-indexes { create-user create-database +create-extensions create-table create-functions create-indexes diff --git a/database/table/events-table.sql b/database/table/events-table.sql index 3d8805a..6978d47 100644 --- a/database/table/events-table.sql +++ b/database/table/events-table.sql @@ -3,6 +3,7 @@ -- ---------------------------- DROP TABLE IF EXISTS "public"."events"; CREATE TABLE "public"."events" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), "stream_name" varchar(255) NOT NULL COLLATE "default", "position" int4 NOT NULL, "type" varchar(255) NOT NULL COLLATE "default", From 14e1344c0d834a8a4f669d35966eb884288b072e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 11 Jan 2017 19:17:08 -0600 Subject: [PATCH 146/591] Database default value for the ID UUID is disabled --- database/install.sh | 8 ++++---- database/table/events-table.sql | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/database/install.sh b/database/install.sh index f1826b0..f422b83 100755 --- a/database/install.sh +++ b/database/install.sh @@ -59,9 +59,9 @@ function create-database { echo } -function create-extensions { - psql $database -f database/extensions.sql -} +# function create-extensions { +# psql $database -f database/extensions.sql +# } function create-table { psql $database -f database/table/events-table.sql @@ -79,7 +79,7 @@ function create-indexes { create-user create-database -create-extensions +# create-extensions create-table create-functions create-indexes diff --git a/database/table/events-table.sql b/database/table/events-table.sql index 6978d47..30d7a44 100644 --- a/database/table/events-table.sql +++ b/database/table/events-table.sql @@ -3,7 +3,8 @@ -- ---------------------------- DROP TABLE IF EXISTS "public"."events"; CREATE TABLE "public"."events" ( - "id" UUID NOT NULL DEFAULT gen_random_uuid(), + -- "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "id" UUID NOT NULL, "stream_name" varchar(255) NOT NULL COLLATE "default", "position" int4 NOT NULL, "type" varchar(255) NOT NULL COLLATE "default", From b2442ae1422ccae92e01d9af69e8d70a6dbbee39 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 12 Jan 2017 14:52:41 -0600 Subject: [PATCH 147/591] Event ID is sent to the write_event Postgres function --- database/functions/write-event.sql | 6 ++++++ lib/event_source/postgres/put.rb | 24 ++++++++++++++---------- lib/event_source/postgres/session.rb | 14 +++++++++++--- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/database/functions/write-event.sql b/database/functions/write-event.sql index c7a74f9..1f67e27 100644 --- a/database/functions/write-event.sql +++ b/database/functions/write-event.sql @@ -1,4 +1,5 @@ CREATE OR REPLACE FUNCTION write_event( + _id varchar, _stream_name varchar, _type varchar, _data jsonb, @@ -8,10 +9,13 @@ CREATE OR REPLACE FUNCTION write_event( RETURNS int AS $$ DECLARE + event_id uuid; stream_version int; position int; category varchar; BEGIN + event_id = uuid(_id); + stream_version := stream_version(_stream_name); if stream_version is null then @@ -28,6 +32,7 @@ BEGIN insert into "events" ( + "id", "stream_name", "position", "type", @@ -36,6 +41,7 @@ BEGIN ) values ( + event_id, _stream_name, position, _type, diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 7dc4e5a..b009f1d 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -30,37 +30,41 @@ def call(write_event, stream_name, expected_version: nil) logger.trace { "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } logger.trace(tags: [:data, :event_data]) { write_event.pretty_inspect } - type, data, metadata = destructure_event(write_event) + id, type, data, metadata = destructure_event(write_event) expected_version = ExpectedVersion.canonize(expected_version) - insert_event(stream_name, type, data, metadata, expected_version).tap do |position| - logger.info { "Put event data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } + insert_event(id, stream_name, type, data, metadata, expected_version).tap do |position| + logger.info { "Put event data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } logger.info(tags: [:data, :event_data]) { write_event.pretty_inspect } end end def destructure_event(write_event) + id = write_event.id type = write_event.type data = write_event.data metadata = write_event.metadata + logger.debug(tags: [:data, :event_data]) { "ID: #{id.pretty_inspect}" } + logger.debug(tags: [:data, :event_data]) { "Type: #{type.pretty_inspect}" } logger.debug(tags: [:data, :event_data]) { "Data: #{data.pretty_inspect}" } logger.debug(tags: [:data, :event_data]) { "Metadata: #{metadata.pretty_inspect}" } - return type, data, metadata + return id, type, data, metadata end - def insert_event(stream_name, type, data, metadata, expected_version) + def insert_event(id, stream_name, type, data, metadata, expected_version) serialized_data = serialized_data(data) serialized_metadata = serialized_metadata(metadata) - records = execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version) + records = execute_query(id, stream_name, type, serialized_data, serialized_metadata, expected_version) position(records) end - def execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version) - logger.trace { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" } + def execute_query(id, stream_name, type, serialized_data, serialized_metadata, expected_version) + logger.trace { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } params = [ + id, stream_name, type, serialized_data, @@ -74,13 +78,13 @@ def execute_query(stream_name, type, serialized_data, serialized_metadata, expec raise_error e end - logger.debug { "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" } + logger.debug { "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } records end def self.statement - @statement ||= "SELECT write_event($1::varchar, $2::varchar, $3::jsonb, $4::jsonb, $5::int);" + @statement ||= "SELECT write_event($1::varchar, $2::varchar, $3::varchar, $4::jsonb, $5::jsonb, $6::int);" end def serialized_data(data) diff --git a/lib/event_source/postgres/session.rb b/lib/event_source/postgres/session.rb index 386db64..f88b5ae 100644 --- a/lib/event_source/postgres/session.rb +++ b/lib/event_source/postgres/session.rb @@ -29,7 +29,7 @@ def self.configure(receiver, session: nil, attr_name: nil) end def connect - logger.trace "Connecting to database" + logger.trace { "Connecting to database" } if connected? logger.debug { "Already connected. A new connection will not be built." } @@ -81,14 +81,22 @@ def settings end def execute(statement, params=nil) + logger.trace { "Executing statement" } + logger.trace(tag: :data) { statement } + logger.trace(tag: :data) { params.pretty_inspect } + unless connected? connect end if params.nil? - connection.exec(statement) + connection.exec(statement).tap do + logger.debug { "Executed statement" } + end else - connection.exec_params(statement, params) + connection.exec_params(statement, params).tap do + logger.debug { "Executed statement with params" } + end end end From a93cc10fc58e99c786b0ff678dbd937d5d77fbdf Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 12 Jan 2017 15:12:53 -0600 Subject: [PATCH 148/591] Control assigns new ID for each event written --- lib/event_source/postgres/controls/put.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/event_source/postgres/controls/put.rb b/lib/event_source/postgres/controls/put.rb index 28cdc6e..dcc15e9 100644 --- a/lib/event_source/postgres/controls/put.rb +++ b/lib/event_source/postgres/controls/put.rb @@ -5,10 +5,17 @@ module Put def self.call(instances: nil, stream_name: nil, event: nil, category: nil) instances ||= 1 stream_name ||= StreamName.example(category: category) + + event_specified = !event.nil? + event ||= EventData::Write.example instances.times do EventSource::Postgres::Put.(event, stream_name) + + unless event_specified + event.id = EventData::Write.id + end end stream_name From 0da2ebaf4f3d43830c83afaa5930ab298aa0b0d8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 12 Jan 2017 16:03:37 -0600 Subject: [PATCH 149/591] Event ID defaults to a random UUID --- database/install.sh | 8 ++++---- database/table/events-table.sql | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/database/install.sh b/database/install.sh index f422b83..f1826b0 100755 --- a/database/install.sh +++ b/database/install.sh @@ -59,9 +59,9 @@ function create-database { echo } -# function create-extensions { -# psql $database -f database/extensions.sql -# } +function create-extensions { + psql $database -f database/extensions.sql +} function create-table { psql $database -f database/table/events-table.sql @@ -79,7 +79,7 @@ function create-indexes { create-user create-database -# create-extensions +create-extensions create-table create-functions create-indexes diff --git a/database/table/events-table.sql b/database/table/events-table.sql index 30d7a44..6978d47 100644 --- a/database/table/events-table.sql +++ b/database/table/events-table.sql @@ -3,8 +3,7 @@ -- ---------------------------- DROP TABLE IF EXISTS "public"."events"; CREATE TABLE "public"."events" ( - -- "id" UUID NOT NULL DEFAULT gen_random_uuid(), - "id" UUID NOT NULL, + "id" UUID NOT NULL DEFAULT gen_random_uuid(), "stream_name" varchar(255) NOT NULL COLLATE "default", "position" int4 NOT NULL, "type" varchar(255) NOT NULL COLLATE "default", From 801b13ca4e88746c0fb70f64353d25e5e4dcfc47 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 12 Jan 2017 17:09:03 -0600 Subject: [PATCH 150/591] Read event has the event ID attribute --- test/automated/get/get.rb | 4 ++++ test/automated/put/put.rb | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index c137a54..5829573 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -9,6 +9,10 @@ read_event = Get.(stream_name, position: position).first context "Got the event that was written" do + test "ID" do + assert(read_event.id == write_event.id) + end + test "Type" do assert(read_event.type == write_event.type) end diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index c137a54..5829573 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -9,6 +9,10 @@ read_event = Get.(stream_name, position: position).first context "Got the event that was written" do + test "ID" do + assert(read_event.id == write_event.id) + end + test "Type" do assert(read_event.type == write_event.type) end From 75244f438eb696a250976b3f195602c135d98965 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 12 Jan 2017 17:11:41 -0600 Subject: [PATCH 151/591] Event ID is retrieved --- lib/event_source/postgres/get/select_statement.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 6a1db23..42931bc 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -36,6 +36,7 @@ def sql statement = <<-SQL SELECT + id::varchar, stream_name::varchar, position::int, type::varchar, From 8fa16be9ddf4971c0877ee60f887311d930a9d14 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 12 Jan 2017 17:13:15 -0600 Subject: [PATCH 152/591] Package version is increased from 0.12.0.1 to 0.12.1.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 0727b54..abfd3fb 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.12.0.1' + s.version = '0.12.1.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From e200b833050a52255d9532a80052eba00c8f80f9 Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Thu, 19 Jan 2017 12:18:36 -0600 Subject: [PATCH 153/591] Maximum milliseconds, not delay milliseconds --- test/automated/read/retry_when_no_further_event_data.rb | 2 +- test/interactive/reader_does_not_terminate.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/automated/read/retry_when_no_further_event_data.rb b/test/automated/read/retry_when_no_further_event_data.rb index 87a5a2a..b5e0b4e 100644 --- a/test/automated/read/retry_when_no_further_event_data.rb +++ b/test/automated/read/retry_when_no_further_event_data.rb @@ -1,7 +1,7 @@ require_relative '../automated_init' context "Read" do - cycle = Cycle.build(delay_milliseconds: 10, timeout_milliseconds: 100) + cycle = Cycle.build(maximum_milliseconds: 10, timeout_milliseconds: 100) sink = Cycle.register_telemetry_sink(cycle) Read.('some_stream', batch_size: 1, cycle: cycle) { |event_data| } diff --git a/test/interactive/reader_does_not_terminate.rb b/test/interactive/reader_does_not_terminate.rb index 5cebd9f..81bab98 100644 --- a/test/interactive/reader_does_not_terminate.rb +++ b/test/interactive/reader_does_not_terminate.rb @@ -2,4 +2,4 @@ stream_name = Controls::StreamName.example -Read.(stream_name, batch_size: 1, cycle_delay_milliseconds: 200) { |event_data| } +Read.(stream_name, batch_size: 1, cycle_maximum_milliseconds: 200) { |event_data| } From 332a548fb71a1259540b6260187f142f7632cdd6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 24 Jan 2017 22:52:58 -0600 Subject: [PATCH 154/591] An ID is assigned to a written event if one is not assigned already --- lib/event_source/postgres/put.rb | 4 ++++ test/automated/put/missing_id.rb | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 test/automated/put/missing_id.rb diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index b009f1d..c6510cd 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -4,6 +4,7 @@ class Put include Log::Dependency dependency :session, Session + dependency :identifier, Session def self.build(session: nil) new.tap do |instance| @@ -13,6 +14,7 @@ def self.build(session: nil) def configure(session: nil) Session.configure(self, session: session) + Identifier::UUID::Random.configure(self) end def self.configure(receiver, session: nil, attr_name: nil) @@ -30,6 +32,8 @@ def call(write_event, stream_name, expected_version: nil) logger.trace { "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } logger.trace(tags: [:data, :event_data]) { write_event.pretty_inspect } + write_event.id ||= identifier.get + id, type, data, metadata = destructure_event(write_event) expected_version = ExpectedVersion.canonize(expected_version) diff --git a/test/automated/put/missing_id.rb b/test/automated/put/missing_id.rb new file mode 100644 index 0000000..93c08c9 --- /dev/null +++ b/test/automated/put/missing_id.rb @@ -0,0 +1,22 @@ +require_relative '../automated_init' + +context "Put" do + context "Missing ID" do + stream_name = Controls::StreamName.example + write_event = Controls::EventData::Write.example(id: :none) + + position = Put.(write_event, stream_name) + + read_event = Get.(stream_name, position: position).first + + context "An ID is assigned to the event" do + test "Write event" do + refute(write_event.id.nil?) + end + + test "Read event" do + refute(read_event.id.nil?) + end + end + end +end From 8749bfadebfaae50f64f34305c78e1f873e4be42 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 24 Jan 2017 22:53:56 -0600 Subject: [PATCH 155/591] Package version is increased from 0.12.1.0 to 0.12.1.1 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index abfd3fb..a1313a1 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.12.1.0' + s.version = '0.12.1.1' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 79d951263bf35f0d642cae82717bcc8984ae4e93 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 25 Jan 2017 15:19:34 -0600 Subject: [PATCH 156/591] Precedence is removed --- lib/event_source/postgres/get.rb | 24 ++++++++--------- .../postgres/get/select_statement.rb | 20 +++++--------- lib/event_source/postgres/read.rb | 4 +-- test/automated/get/precedence.rb | 27 ------------------- test/automated/read/reverse.rb | 18 ------------- test/automated/select_statement/defaults.rb | 7 ----- 6 files changed, 20 insertions(+), 80 deletions(-) delete mode 100644 test/automated/get/precedence.rb delete mode 100644 test/automated/read/reverse.rb diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 705a336..03cb027 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -3,19 +3,19 @@ module Postgres class Get include Log::Dependency - initializer :batch_size, :precedence + initializer :batch_size dependency :session, Session - def self.build(batch_size: nil, precedence: nil, session: nil) - new(batch_size, precedence).tap do |instance| + def self.build(batch_size: nil, session: nil) + new(batch_size).tap do |instance| instance.configure(session: session) end end - def self.configure(receiver, attr_name: nil, position: nil, batch_size: nil, precedence: nil, session: nil) + def self.configure(receiver, attr_name: nil, position: nil, batch_size: nil, session: nil) attr_name ||= :get - instance = build(batch_size: batch_size, precedence: precedence, session: session) + instance = build(batch_size: batch_size, session: session) receiver.public_send "#{attr_name}=", instance end @@ -23,32 +23,32 @@ def configure(session: nil) Session.configure self, session: session end - def self.call(stream_name, position: nil, batch_size: nil, precedence: nil, session: nil) - instance = build(batch_size: batch_size, precedence: precedence, session: session) + def self.call(stream_name, position: nil, batch_size: nil, session: nil) + instance = build(batch_size: batch_size, session: session) instance.(stream_name, position: position) end def call(stream_name, position: nil) - logger.trace { "Getting event data (Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } + logger.trace { "Getting event data (Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } records = get_records(stream_name, position) events = convert(records) - logger.info { "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } + logger.info { "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } logger.info(tags: [:data, :event_data]) { events.pretty_inspect } events end def get_records(stream_name, position) - logger.trace { "Getting records (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } + logger.trace { "Getting records (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } - select_statement = SelectStatement.build(stream_name, offset: position, batch_size: batch_size, precedence: precedence) + select_statement = SelectStatement.build(stream_name, offset: position, batch_size: batch_size) records = session.execute(select_statement.sql) - logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" } + logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } records end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 42931bc..f40a3b5 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -4,7 +4,7 @@ class Get class SelectStatement include Log::Dependency - initializer :stream, w(:offset), w(:batch_size), w(:precedence) + initializer :stream, w(:offset), w(:batch_size) def offset @offset ||= Defaults.offset @@ -14,10 +14,6 @@ def batch_size @batch_size ||= Defaults.batch_size end - def precedence - @precedence ||= Defaults.precedence - end - def stream_name stream.name end @@ -26,13 +22,13 @@ def stream_type stream.type end - def self.build(stream_name, offset: nil, batch_size: nil, precedence: nil) + def self.build(stream_name, offset: nil, batch_size: nil) stream = Stream.new(stream_name) - new(stream, offset, batch_size, precedence) + new(stream, offset, batch_size) end def sql - logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" } + logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size})" } statement = <<-SQL SELECT @@ -49,7 +45,7 @@ def sql WHERE #{where_clause_field} = '#{stream_name}' ORDER BY - global_position #{precedence.to_s.upcase} + global_position asc LIMIT #{batch_size} OFFSET @@ -57,7 +53,7 @@ def sql ; SQL - logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" } + logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size})" } logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } statement @@ -79,10 +75,6 @@ def self.offset def self.batch_size 1000 end - - def self.precedence - :asc - end end end end diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 19fdea8..e401221 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -3,8 +3,8 @@ module Postgres class Read include EventSource::Read - def configure(batch_size: nil, precedence: nil, session: nil) - Get.configure(self, batch_size: batch_size, precedence: precedence, session: session) + def configure(batch_size: nil, session: nil) + Get.configure(self, batch_size: batch_size, session: session) end end end diff --git a/test/automated/get/precedence.rb b/test/automated/get/precedence.rb deleted file mode 100644 index fa93410..0000000 --- a/test/automated/get/precedence.rb +++ /dev/null @@ -1,27 +0,0 @@ -require_relative '../automated_init' - -context "Get" do - context "Precedence" do - stream_name = Controls::Put.(instances: 3) - - context "Ascending" do - events = Get.(stream_name, precedence: :asc) - - first_event_postition = events.first.position - - test "First event written is first in the list of results" do - assert(first_event_postition == 0) - end - end - - context "Descending" do - events = Get.(stream_name, precedence: :desc) - - first_event_postition = events.first.position - - test "Last event written is first in the list of results" do - assert(first_event_postition == 2) - end - end - end -end diff --git a/test/automated/read/reverse.rb b/test/automated/read/reverse.rb deleted file mode 100644 index 1794e72..0000000 --- a/test/automated/read/reverse.rb +++ /dev/null @@ -1,18 +0,0 @@ -require_relative '../automated_init' - -context "Read" do - context "Reverse" do - stream_name = Controls::Put.(instances: 3) - - events = [] - Read.(stream_name, precedence: :desc) do |event| - events << event - end - - first_event_postition = events.first.position - - test "Last event written is first in the list of results" do - assert(first_event_postition == 2) - end - end -end diff --git a/test/automated/select_statement/defaults.rb b/test/automated/select_statement/defaults.rb index acbaa2e..2b3c451 100644 --- a/test/automated/select_statement/defaults.rb +++ b/test/automated/select_statement/defaults.rb @@ -19,12 +19,5 @@ assert(select_statement.batch_size == default_batch_size) end end - - context "Precedence" do - default_precedence = Get::SelectStatement::Defaults.precedence - test "#{default_precedence}" do - assert(select_statement.precedence == default_precedence) - end - end end end From 7eddd84fb0da74e1113c2ed678cc1f0a8a9de0c9 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 25 Jan 2017 17:37:35 -0600 Subject: [PATCH 157/591] Cycle is removed --- .../read/retry_when_no_further_event_data.rb | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 test/automated/read/retry_when_no_further_event_data.rb diff --git a/test/automated/read/retry_when_no_further_event_data.rb b/test/automated/read/retry_when_no_further_event_data.rb deleted file mode 100644 index b5e0b4e..0000000 --- a/test/automated/read/retry_when_no_further_event_data.rb +++ /dev/null @@ -1,12 +0,0 @@ -require_relative '../automated_init' - -context "Read" do - cycle = Cycle.build(maximum_milliseconds: 10, timeout_milliseconds: 100) - sink = Cycle.register_telemetry_sink(cycle) - - Read.('some_stream', batch_size: 1, cycle: cycle) { |event_data| } - - test "Timed out" do - assert(sink.recorded_timed_out?) - end -end From e48bcb10c5168910341b65180085be12e456076f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 25 Jan 2017 18:10:08 -0600 Subject: [PATCH 158/591] Position, rather than offset (offset is no longe the SQL mechanism used) --- lib/event_source/postgres/get.rb | 2 +- .../postgres/get/select_statement.rb | 31 ++++++++++++------- test/automated/select_statement/defaults.rb | 8 ++--- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 03cb027..dc6c9c7 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -44,7 +44,7 @@ def call(stream_name, position: nil) def get_records(stream_name, position) logger.trace { "Getting records (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } - select_statement = SelectStatement.build(stream_name, offset: position, batch_size: batch_size) + select_statement = SelectStatement.build(stream_name, position: position, batch_size: batch_size) records = session.execute(select_statement.sql) diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index f40a3b5..93e6f35 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -4,10 +4,10 @@ class Get class SelectStatement include Log::Dependency - initializer :stream, w(:offset), w(:batch_size) + initializer :stream, w(:position), w(:batch_size) - def offset - @offset ||= Defaults.offset + def position + @position ||= Defaults.position end def batch_size @@ -22,13 +22,13 @@ def stream_type stream.type end - def self.build(stream_name, offset: nil, batch_size: nil) + def self.build(stream_name, position: nil, batch_size: nil) stream = Stream.new(stream_name) - new(stream, offset, batch_size) + new(stream, position, batch_size) end def sql - logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size})" } + logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{position}, Batch Size: #{batch_size})" } statement = <<-SQL SELECT @@ -43,17 +43,16 @@ def sql FROM events WHERE - #{where_clause_field} = '#{stream_name}' + #{where_clause_field} = '#{stream_name}' AND + #{position_field} >= #{position} ORDER BY - global_position asc + global_position ASC LIMIT #{batch_size} - OFFSET - #{offset} ; SQL - logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size})" } + logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{position}, Batch Size: #{batch_size})" } logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } statement @@ -67,8 +66,16 @@ def where_clause_field end end + def position_field + unless stream.category? + 'position' + else + 'global_position' + end + end + module Defaults - def self.offset + def self.position 0 end diff --git a/test/automated/select_statement/defaults.rb b/test/automated/select_statement/defaults.rb index 2b3c451..ef6d965 100644 --- a/test/automated/select_statement/defaults.rb +++ b/test/automated/select_statement/defaults.rb @@ -6,10 +6,10 @@ select_statement = Get::SelectStatement.build stream - context "Offset" do - default_offset = Get::SelectStatement::Defaults.offset - test "#{default_offset}" do - assert(select_statement.offset == default_offset) + context "Position" do + default_position = Get::SelectStatement::Defaults.position + test "#{default_position}" do + assert(select_statement.position == default_position) end end From 9b9619054bd26fde8f876665d2869abbff20d0ff Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 25 Jan 2017 18:22:26 -0600 Subject: [PATCH 159/591] Ordering and filtering on position or global position --- lib/event_source/postgres/get/select_statement.rb | 2 +- test/automated/select_statement/category_query.rb | 12 +++++++++++- test/automated/select_statement/stream_name_query.rb | 10 ++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 93e6f35..8f51c79 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -46,7 +46,7 @@ def sql #{where_clause_field} = '#{stream_name}' AND #{position_field} >= #{position} ORDER BY - global_position ASC + #{position_field} ASC LIMIT #{batch_size} ; diff --git a/test/automated/select_statement/category_query.rb b/test/automated/select_statement/category_query.rb index 9797729..f59fdcb 100644 --- a/test/automated/select_statement/category_query.rb +++ b/test/automated/select_statement/category_query.rb @@ -10,9 +10,19 @@ sql.gsub!(/\s+/, ' ') context "Where Clause" do - test "Filters on stream name" do + test "Filters on category of stream name" do assert(sql.include? 'WHERE category(stream_name) =') end + + test "Filters on global position" do + assert(sql.include? 'global_position >=') + end + end + + context "Order Clause" do + test "Orders by the global position" do + assert(sql.include? 'ORDER BY global_position') + end end end end diff --git a/test/automated/select_statement/stream_name_query.rb b/test/automated/select_statement/stream_name_query.rb index f879a2a..d407748 100644 --- a/test/automated/select_statement/stream_name_query.rb +++ b/test/automated/select_statement/stream_name_query.rb @@ -13,6 +13,16 @@ test "Filters on stream name" do assert(sql.include? 'WHERE stream_name =') end + + test "Filters on position" do + assert(sql.include? 'position >=') + end + end + + context "Order Clause" do + test "Orders by the global position" do + assert(sql.include? 'ORDER BY position') + end end end end From edaf4739f668fb5920520206938da8ad916dddf3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 26 Jan 2017 01:03:52 -0600 Subject: [PATCH 160/591] Implement abstract Get --- lib/event_source/postgres/get.rb | 2 +- test/automated/get/position.rb | 13 +++++++++++++ .../read/{starting_position.rb => position.rb} | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 test/automated/get/position.rb rename test/automated/read/{starting_position.rb => position.rb} (90%) diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index dc6c9c7..006ef6d 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -1,7 +1,7 @@ module EventSource module Postgres class Get - include Log::Dependency + include EventSource::Get initializer :batch_size diff --git a/test/automated/get/position.rb b/test/automated/get/position.rb new file mode 100644 index 0000000..fea072f --- /dev/null +++ b/test/automated/get/position.rb @@ -0,0 +1,13 @@ +require_relative '../automated_init' + +context "Get" do + context "Position" do + stream_name = Controls::Put.(instances: 2) + + batch = Get.(stream_name, position: 1, batch_size: 1) + + test "Gets from the starting position" do + assert(batch.length == 1) + end + end +end diff --git a/test/automated/read/starting_position.rb b/test/automated/read/position.rb similarity index 90% rename from test/automated/read/starting_position.rb rename to test/automated/read/position.rb index 1884c6a..ecfba8a 100644 --- a/test/automated/read/starting_position.rb +++ b/test/automated/read/position.rb @@ -1,7 +1,7 @@ require_relative '../automated_init' context "Read" do - context "Starting Position" do + context "Position" do stream_name = Controls::Put.(instances: 2) batch = [] From d7a9b4d435e1bffa5b3d13d39ed040d949828007 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 26 Jan 2017 01:07:12 -0600 Subject: [PATCH 161/591] Package version is increased from 0.12.1.1 to 0.13.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index a1313a1..d543e70 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.12.1.1' + s.version = '0.13.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From c4b9f9a5d086d3d497d1e0ec9b4f4f8f823681e4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 26 Jan 2017 01:49:53 -0600 Subject: [PATCH 162/591] Get last event in stream query --- lib/event_source/postgres.rb | 2 + lib/event_source/postgres/get/last.rb | 111 ++++++++++++++++++ .../postgres/get/last/select_statement.rb | 52 ++++++++ .../postgres/get/select_statement.rb | 4 +- test/automated/get_last/get_last.rb | 15 +++ test/automated/get_last/no_events.rb | 13 ++ 6 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 lib/event_source/postgres/get/last.rb create mode 100644 lib/event_source/postgres/get/last/select_statement.rb create mode 100644 test/automated/get_last/get_last.rb create mode 100644 test/automated/get_last/no_events.rb diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index fd83c6c..a98d558 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -17,4 +17,6 @@ require 'event_source/postgres/get/select_statement' require 'event_source/postgres/get' +require 'event_source/postgres/get/last/select_statement' +require 'event_source/postgres/get/last' require 'event_source/postgres/read' diff --git a/lib/event_source/postgres/get/last.rb b/lib/event_source/postgres/get/last.rb new file mode 100644 index 0000000..8b43da8 --- /dev/null +++ b/lib/event_source/postgres/get/last.rb @@ -0,0 +1,111 @@ +module EventSource + module Postgres + class Get + class Last + include EventSource::Get + + dependency :session, Session + + def self.build(session: nil) + new.tap do |instance| + instance.configure(session: session) + end + end + + def self.configure(receiver, attr_name: nil, session: nil) + attr_name ||= :get + instance = build(session: session) + receiver.public_send "#{attr_name}=", instance + end + + def configure(session: nil) + Session.configure self, session: session + end + + def self.call(stream_name, session: nil) + instance = build(session: session) + instance.(stream_name) + end + + def call(stream_name) + logger.trace { "Getting last event data (Stream Name: #{stream_name})" } + + record = get_record(stream_name) + + return nil if record.nil? + + event = convert(record) + + logger.info { "Finished getting event data (Stream Name: #{stream_name})" } + logger.info(tags: [:data, :event_data]) { event.pretty_inspect } + + event + end + + def get_record(stream_name) + logger.trace { "Getting last record (Stream: #{stream_name})" } + + select_statement = SelectStatement.build(stream_name) + + records = session.execute(select_statement.sql) + + logger.debug { "Finished getting record (Stream: #{stream_name})" } + + return nil if records.ntuples == 0 + + records[0] + end + + def convert(record) + logger.trace { "Converting record to event data" } + + record['data'] = Deserialize.data(record['data']) + record['metadata'] = Deserialize.metadata(record['metadata']) + record['time'] = Time.utc_coerced(record['time']) + + event = EventData::Read.build(record) + + logger.debug { "Converted record to event data" } + + event + end + + def __convert(records) + logger.trace { "Converting records to event data (Records Count: #{records.ntuples})" } + + events = records.map do |record| + record['data'] = Deserialize.data(record['data']) + record['metadata'] = Deserialize.metadata(record['metadata']) + record['time'] = Time.utc_coerced(record['time']) + + EventData::Read.build record + + break + end + + logger.debug { "Converted records to event data (Event Data Count: #{events.length})" } + + events + end + + module Deserialize + def self.data(serialized_data) + return nil if serialized_data.nil? + Transform::Read.(serialized_data, EventData::Hash, :json) + end + + def self.metadata(serialized_metadata) + return nil if serialized_metadata.nil? + Transform::Read.(serialized_metadata, EventData::Hash, :json) + end + end + + module Time + def self.utc_coerced(local_time) + Clock::UTC.coerce(local_time) + end + end + end + end + end +end diff --git a/lib/event_source/postgres/get/last/select_statement.rb b/lib/event_source/postgres/get/last/select_statement.rb new file mode 100644 index 0000000..cf31c93 --- /dev/null +++ b/lib/event_source/postgres/get/last/select_statement.rb @@ -0,0 +1,52 @@ +module EventSource + module Postgres + class Get + class Last + class SelectStatement + include Log::Dependency + + initializer :stream + + def stream_name + stream.name + end + + def self.build(stream_name) + stream = Stream.new(stream_name) + new(stream) + end + + def sql + logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name})" } + + statement = <<-SQL + SELECT + id::varchar, + stream_name::varchar, + position::int, + type::varchar, + global_position::bigint, + data::varchar, + metadata::varchar, + time::timestamp + FROM + events + WHERE + stream_name = '#{stream_name}' + ORDER BY + position DESC + LIMIT + 1 + ; + SQL + + logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name})" } + logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } + + statement + end + end + end + end + end +end diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 8f51c79..e26e472 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -28,7 +28,7 @@ def self.build(stream_name, position: nil, batch_size: nil) end def sql - logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{position}, Batch Size: #{batch_size})" } + logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type.inspect}, Position: #{position}, Batch Size: #{batch_size})" } statement = <<-SQL SELECT @@ -52,7 +52,7 @@ def sql ; SQL - logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{position}, Batch Size: #{batch_size})" } + logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type.inspect}, Position: #{position}, Batch Size: #{batch_size})" } logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } statement diff --git a/test/automated/get_last/get_last.rb b/test/automated/get_last/get_last.rb new file mode 100644 index 0000000..77e131e --- /dev/null +++ b/test/automated/get_last/get_last.rb @@ -0,0 +1,15 @@ +require_relative '../automated_init' + +context "Get Last" do + stream_name = Controls::Put.(instances: 2) + + write_event = Controls::EventData::Write.example + + position = Put.(write_event, stream_name) + + last_event = Get::Last.(stream_name) + + test "Gets the last event in the stream" do + assert(last_event.data == write_event.data) + end +end diff --git a/test/automated/get_last/no_events.rb b/test/automated/get_last/no_events.rb new file mode 100644 index 0000000..dcabf8e --- /dev/null +++ b/test/automated/get_last/no_events.rb @@ -0,0 +1,13 @@ +require_relative '../automated_init' + +context "Get Last" do + context "No Events" do + stream_name = Controls::StreamName.example + + last_event = Get::Last.(stream_name) + + test "Nil event" do + assert(last_event.nil?) + end + end +end From c07c4b57c095a14211b7832f4445c43220e934ef Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 26 Jan 2017 01:50:51 -0600 Subject: [PATCH 163/591] Package version is increased from 0.13.0.0 to 0.13.1.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index d543e70..6749b56 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.13.0.0' + s.version = '0.13.1.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 3f10e7e55fce02e9a870632717d10ef484969198 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 31 Jan 2017 21:56:51 -0600 Subject: [PATCH 164/591] Select statement generators don't use the Stream class --- .../postgres/get/last/select_statement.rb | 9 ++------ .../postgres/get/select_statement.rb | 21 +++++++++---------- test/automated/select_statement/defaults.rb | 4 ++-- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/lib/event_source/postgres/get/last/select_statement.rb b/lib/event_source/postgres/get/last/select_statement.rb index cf31c93..7c02cf6 100644 --- a/lib/event_source/postgres/get/last/select_statement.rb +++ b/lib/event_source/postgres/get/last/select_statement.rb @@ -5,15 +5,10 @@ class Last class SelectStatement include Log::Dependency - initializer :stream - - def stream_name - stream.name - end + initializer :stream_name def self.build(stream_name) - stream = Stream.new(stream_name) - new(stream) + new(stream_name) end def sql diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index e26e472..50cfbc3 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -4,7 +4,7 @@ class Get class SelectStatement include Log::Dependency - initializer :stream, w(:position), w(:batch_size) + initializer :stream_name, w(:position), w(:batch_size) def position @position ||= Defaults.position @@ -14,21 +14,20 @@ def batch_size @batch_size ||= Defaults.batch_size end - def stream_name - stream.name + def stream_type + @stream_type ||= StreamName.get_type(stream_name) end - def stream_type - stream.type + def category_stream? + is_category_stream ||= StreamName.category?(stream_name) end def self.build(stream_name, position: nil, batch_size: nil) - stream = Stream.new(stream_name) - new(stream, position, batch_size) + new(stream_name, position, batch_size) end def sql - logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type.inspect}, Position: #{position}, Batch Size: #{batch_size})" } + logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{category_stream?}, Type: #{stream_type.inspect}, Position: #{position}, Batch Size: #{batch_size})" } statement = <<-SQL SELECT @@ -52,14 +51,14 @@ def sql ; SQL - logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type.inspect}, Position: #{position}, Batch Size: #{batch_size})" } + logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{category_stream?}, Type: #{stream_type.inspect}, Position: #{position}, Batch Size: #{batch_size})" } logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } statement end def where_clause_field - unless stream.category? + unless category_stream? 'stream_name' else 'category(stream_name)' @@ -67,7 +66,7 @@ def where_clause_field end def position_field - unless stream.category? + unless category_stream? 'position' else 'global_position' diff --git a/test/automated/select_statement/defaults.rb b/test/automated/select_statement/defaults.rb index ef6d965..13096c3 100644 --- a/test/automated/select_statement/defaults.rb +++ b/test/automated/select_statement/defaults.rb @@ -2,9 +2,9 @@ context "Select Statement" do context "Defaults" do - stream = Controls::Stream.example + stream_name = Controls::StreamName.example - select_statement = Get::SelectStatement.build stream + select_statement = Get::SelectStatement.build(stream_name) context "Position" do default_position = Get::SelectStatement::Defaults.position From a666807d67136f3062534ae8b7d12cdbbb1bdd64 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 31 Jan 2017 22:43:34 -0600 Subject: [PATCH 165/591] Stream class is no longer used --- lib/event_source/postgres/controls.rb | 1 - lib/event_source/postgres/controls/stream.rb | 7 ------- 2 files changed, 8 deletions(-) delete mode 100644 lib/event_source/postgres/controls/stream.rb diff --git a/lib/event_source/postgres/controls.rb b/lib/event_source/postgres/controls.rb index 7284844..e1142d8 100644 --- a/lib/event_source/postgres/controls.rb +++ b/lib/event_source/postgres/controls.rb @@ -2,6 +2,5 @@ require 'event_source/postgres/controls/category' require 'event_source/postgres/controls/stream_name' -require 'event_source/postgres/controls/stream' require 'event_source/postgres/controls/event_data' require 'event_source/postgres/controls/put' diff --git a/lib/event_source/postgres/controls/stream.rb b/lib/event_source/postgres/controls/stream.rb deleted file mode 100644 index 9ba4d2f..0000000 --- a/lib/event_source/postgres/controls/stream.rb +++ /dev/null @@ -1,7 +0,0 @@ -module EventSource - module Postgres - module Controls - Stream = EventSource::Controls::Stream - end - end -end From 79d1f5f3f1ddb741460fd079f8ed6ba677751a3c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 31 Jan 2017 22:44:08 -0600 Subject: [PATCH 166/591] Package version is increased from 0.13.1.0 to 0.14.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 6749b56..f96f630 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.13.1.0' + s.version = '0.14.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 0f39772446799425afc41f1a8fb1be4757320609 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 3 Feb 2017 15:42:01 -0600 Subject: [PATCH 167/591] Iterator is specialized --- lib/event_source/postgres.rb | 1 + lib/event_source/postgres/read.rb | 4 + lib/event_source/postgres/read/iterator.rb | 128 +++++++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100644 lib/event_source/postgres/read/iterator.rb diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index a98d558..21ec079 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -19,4 +19,5 @@ require 'event_source/postgres/get' require 'event_source/postgres/get/last/select_statement' require 'event_source/postgres/get/last' +require 'event_source/postgres/read/iterator' require 'event_source/postgres/read' diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index e401221..7feb083 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -3,8 +3,12 @@ module Postgres class Read include EventSource::Read + ## TODO examine args def configure(batch_size: nil, session: nil) Get.configure(self, batch_size: batch_size, session: session) + + ## TODO examine args + Iterator.configure self, self.get, self.stream_name, position: self.position end end end diff --git a/lib/event_source/postgres/read/iterator.rb b/lib/event_source/postgres/read/iterator.rb new file mode 100644 index 0000000..39a1428 --- /dev/null +++ b/lib/event_source/postgres/read/iterator.rb @@ -0,0 +1,128 @@ +module EventSource + module Postgres + class Read + class Iterator + include Log::Dependency + + attr_accessor :starting_position + attr_accessor :batch + + def batch_index + @batch_index ||= 0 + end + attr_writer :batch_index + + initializer :get, :stream_name + + def self.build(get, stream_name, position: nil) + new(get, stream_name).tap do |instance| + instance.starting_position = position + Log.get(self).debug { "Built Iterator (Stream Name: #{stream_name}, Starting Position: #{position.inspect})" } + end + end + + def self.configure(receiver, get, stream_name, attr_name: nil, position: nil) + attr_name ||= :iterator + instance = build(get, stream_name, position: position) + receiver.public_send "#{attr_name}=", instance + end + + def next + logger.trace { "Getting next event data (Batch Length: #{(batch &.length).inspect}, Batch Index: #{batch_index})" } + + resupply if batch_depleted? + + event_data = batch[batch_index] + + logger.debug(tags: [:data, :event_data]) { "Next event data: #{event_data.pretty_inspect}" } + logger.debug { "Done getting next event data (Batch Length: #{(batch &.length).inspect}, Batch Index: #{batch_index})" } + + advance_batch_index + + event_data + end + + def advance_batch_index + logger.trace { "Advancing batch index (Batch Index: #{batch_index})" } + self.batch_index += 1 + logger.debug { "Advanced batch index (Batch Index: #{batch_index})" } + end + + def batch_depleted? + if batch.nil? + logger.debug { "Batch is depleted (Batch is nil)" } + return true + end + + if batch.empty? + logger.debug { "Batch is depleted (Batch is empty)" } + return true + end + + if batch_index == batch.length + logger.debug { "Batch is depleted (Batch Index: #{batch_index}, Batch Length: #{batch.length})" } + return true + end + + false + end + + def resupply + logger.trace { "Resupplying batch (Current Batch Length: #{(batch &.length).inspect})" } + + batch = get_batch + reset(batch) + + logger.debug { "Batch resupplied (Next Batch Length: #{(batch &.length).inspect})" } + end + + def get_batch + position = next_batch_starting_position + + logger.trace "Getting batch (Position: #{position.inspect})" + + batch = [] + if position.nil? || position >= 0 + batch = get.(stream_name, position: position) + end + + logger.debug { "Finished getting batch (Count: #{batch.length}, Position: #{position.inspect})" } + + batch + end + + def next_batch_starting_position + if batch.nil? + logger.debug { "Batch is nil (Next batch starting position: #{starting_position.inspect})" } + return starting_position + end + + previous_position = last_position + next_position = previous_position + 1 + logger.debug { "End of batch (Next starting position: #{next_position}, Previous Position: #{previous_position})" } + + next_position + end + + def last_position + unless EventSource::StreamName.category?(stream_name) + batch.last.position + else + batch.last.global_position + end + end + + def reset(batch) + logger.trace { "Resetting batch" } + + self.batch = batch + self.batch_index = 0 + + logger.debug(tags: [:data, :batch]) { "Batch set to: \n#{batch.pretty_inspect}" } + logger.debug(tags: [:data, :batch]) { "Batch position set to: #{batch_index.inspect}" } + logger.debug { "Done resetting batch" } + end + end + end + end +end From c51cd4f728920bf278cf2529efd815d23b8f082d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 3 Feb 2017 23:34:51 -0600 Subject: [PATCH 168/591] Dependencies are adapted to new read, iterator and get design --- lib/event_source/postgres/read.rb | 7 ++----- lib/event_source/postgres/read/iterator.rb | 12 +++++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 7feb083..479e660 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -3,12 +3,9 @@ module Postgres class Read include EventSource::Read - ## TODO examine args def configure(batch_size: nil, session: nil) - Get.configure(self, batch_size: batch_size, session: session) - - ## TODO examine args - Iterator.configure self, self.get, self.stream_name, position: self.position + Iterator.configure(self, self.stream_name, position: self.position) + Get.configure(self.iterator, batch_size: batch_size, session: session) end end end diff --git a/lib/event_source/postgres/read/iterator.rb b/lib/event_source/postgres/read/iterator.rb index 39a1428..52ed2a4 100644 --- a/lib/event_source/postgres/read/iterator.rb +++ b/lib/event_source/postgres/read/iterator.rb @@ -4,6 +4,8 @@ class Read class Iterator include Log::Dependency + dependency :get, Get + attr_accessor :starting_position attr_accessor :batch @@ -12,18 +14,18 @@ def batch_index end attr_writer :batch_index - initializer :get, :stream_name + initializer :stream_name - def self.build(get, stream_name, position: nil) - new(get, stream_name).tap do |instance| + def self.build(stream_name, position: nil) + new(stream_name).tap do |instance| instance.starting_position = position Log.get(self).debug { "Built Iterator (Stream Name: #{stream_name}, Starting Position: #{position.inspect})" } end end - def self.configure(receiver, get, stream_name, attr_name: nil, position: nil) + def self.configure(receiver, stream_name, attr_name: nil, position: nil) attr_name ||= :iterator - instance = build(get, stream_name, position: position) + instance = build(stream_name, position: position) receiver.public_send "#{attr_name}=", instance end From 3f33da8dd19e5e66eeb756f83e3f1f2c4170bcca Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 4 Feb 2017 00:21:16 -0600 Subject: [PATCH 169/591] Iterator is abstract --- lib/event_source/postgres/read/iterator.rb | 122 +----------------- test/automated/get/iterator/next.rb | 5 +- .../get/iterator/no_further_event_data.rb | 5 +- 3 files changed, 10 insertions(+), 122 deletions(-) diff --git a/lib/event_source/postgres/read/iterator.rb b/lib/event_source/postgres/read/iterator.rb index 52ed2a4..849b024 100644 --- a/lib/event_source/postgres/read/iterator.rb +++ b/lib/event_source/postgres/read/iterator.rb @@ -2,127 +2,9 @@ module EventSource module Postgres class Read class Iterator - include Log::Dependency + include EventSource::Iterator - dependency :get, Get - - attr_accessor :starting_position - attr_accessor :batch - - def batch_index - @batch_index ||= 0 - end - attr_writer :batch_index - - initializer :stream_name - - def self.build(stream_name, position: nil) - new(stream_name).tap do |instance| - instance.starting_position = position - Log.get(self).debug { "Built Iterator (Stream Name: #{stream_name}, Starting Position: #{position.inspect})" } - end - end - - def self.configure(receiver, stream_name, attr_name: nil, position: nil) - attr_name ||= :iterator - instance = build(stream_name, position: position) - receiver.public_send "#{attr_name}=", instance - end - - def next - logger.trace { "Getting next event data (Batch Length: #{(batch &.length).inspect}, Batch Index: #{batch_index})" } - - resupply if batch_depleted? - - event_data = batch[batch_index] - - logger.debug(tags: [:data, :event_data]) { "Next event data: #{event_data.pretty_inspect}" } - logger.debug { "Done getting next event data (Batch Length: #{(batch &.length).inspect}, Batch Index: #{batch_index})" } - - advance_batch_index - - event_data - end - - def advance_batch_index - logger.trace { "Advancing batch index (Batch Index: #{batch_index})" } - self.batch_index += 1 - logger.debug { "Advanced batch index (Batch Index: #{batch_index})" } - end - - def batch_depleted? - if batch.nil? - logger.debug { "Batch is depleted (Batch is nil)" } - return true - end - - if batch.empty? - logger.debug { "Batch is depleted (Batch is empty)" } - return true - end - - if batch_index == batch.length - logger.debug { "Batch is depleted (Batch Index: #{batch_index}, Batch Length: #{batch.length})" } - return true - end - - false - end - - def resupply - logger.trace { "Resupplying batch (Current Batch Length: #{(batch &.length).inspect})" } - - batch = get_batch - reset(batch) - - logger.debug { "Batch resupplied (Next Batch Length: #{(batch &.length).inspect})" } - end - - def get_batch - position = next_batch_starting_position - - logger.trace "Getting batch (Position: #{position.inspect})" - - batch = [] - if position.nil? || position >= 0 - batch = get.(stream_name, position: position) - end - - logger.debug { "Finished getting batch (Count: #{batch.length}, Position: #{position.inspect})" } - - batch - end - - def next_batch_starting_position - if batch.nil? - logger.debug { "Batch is nil (Next batch starting position: #{starting_position.inspect})" } - return starting_position - end - - previous_position = last_position - next_position = previous_position + 1 - logger.debug { "End of batch (Next starting position: #{next_position}, Previous Position: #{previous_position})" } - - next_position - end - - def last_position - unless EventSource::StreamName.category?(stream_name) - batch.last.position - else - batch.last.global_position - end - end - - def reset(batch) - logger.trace { "Resetting batch" } - - self.batch = batch - self.batch_index = 0 - - logger.debug(tags: [:data, :batch]) { "Batch set to: \n#{batch.pretty_inspect}" } - logger.debug(tags: [:data, :batch]) { "Batch position set to: #{batch_index.inspect}" } - logger.debug { "Done resetting batch" } + def configure end end end diff --git a/test/automated/get/iterator/next.rb b/test/automated/get/iterator/next.rb index 3fec1ed..9829b4f 100644 --- a/test/automated/get/iterator/next.rb +++ b/test/automated/get/iterator/next.rb @@ -6,7 +6,10 @@ get = Get.build(batch_size: 1) - iterator = Iterator.build(get, stream_name) + iterator = Read::Iterator.build(stream_name) + + ## TODO + iterator.get = get batch = [] diff --git a/test/automated/get/iterator/no_further_event_data.rb b/test/automated/get/iterator/no_further_event_data.rb index a89bbfe..cceb6d4 100644 --- a/test/automated/get/iterator/no_further_event_data.rb +++ b/test/automated/get/iterator/no_further_event_data.rb @@ -6,7 +6,10 @@ get = Get.build - iterator = Iterator.build(get, stream_name) + iterator = Read::Iterator.build(stream_name) + + ## TODO + iterator.get = get 2.times { iterator.next } From ba7f1c696b1a8ae832ec2f0900052a6df204e55f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 4 Feb 2017 00:34:33 -0600 Subject: [PATCH 170/591] Instance properties don't require self prefix --- lib/event_source/postgres/read.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 479e660..9b41ed0 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -4,7 +4,7 @@ class Read include EventSource::Read def configure(batch_size: nil, session: nil) - Iterator.configure(self, self.stream_name, position: self.position) + Iterator.configure(self, stream_name, position: position) Get.configure(self.iterator, batch_size: batch_size, session: session) end end From 7641b960621b3b8e0c7f9453c293895a8640d200 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 4 Feb 2017 00:38:36 -0600 Subject: [PATCH 171/591] Read has batch size property --- lib/event_source/postgres/read.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 9b41ed0..684c035 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -3,7 +3,7 @@ module Postgres class Read include EventSource::Read - def configure(batch_size: nil, session: nil) + def configure(session: nil) Iterator.configure(self, stream_name, position: position) Get.configure(self.iterator, batch_size: batch_size, session: session) end From cc572e78079268db0ded8eaffac7f80e6c68cdad Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 4 Feb 2017 00:45:43 -0600 Subject: [PATCH 172/591] Get is explicitly configured to iterator --- test/automated/get/iterator/next.rb | 6 +----- test/automated/get/iterator/no_further_event_data.rb | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/test/automated/get/iterator/next.rb b/test/automated/get/iterator/next.rb index 9829b4f..763c297 100644 --- a/test/automated/get/iterator/next.rb +++ b/test/automated/get/iterator/next.rb @@ -4,12 +4,8 @@ context "Next" do stream_name = Controls::Put.(instances: 2) - get = Get.build(batch_size: 1) - iterator = Read::Iterator.build(stream_name) - - ## TODO - iterator.get = get + Get.configure(iterator, batch_size: 1) batch = [] diff --git a/test/automated/get/iterator/no_further_event_data.rb b/test/automated/get/iterator/no_further_event_data.rb index cceb6d4..bf15b6a 100644 --- a/test/automated/get/iterator/no_further_event_data.rb +++ b/test/automated/get/iterator/no_further_event_data.rb @@ -4,12 +4,8 @@ context "No further event data" do stream_name = Controls::Put.(instances: 2) - get = Get.build - iterator = Read::Iterator.build(stream_name) - - ## TODO - iterator.get = get + Get.configure(iterator, batch_size: 1) 2.times { iterator.next } From 00f886b30a4f8b597202edb25d8080213459153b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 4 Feb 2017 01:12:07 -0600 Subject: [PATCH 173/591] Last position determination is part of the specialized iterator --- lib/event_source/postgres/read/iterator.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/event_source/postgres/read/iterator.rb b/lib/event_source/postgres/read/iterator.rb index 849b024..37f7da3 100644 --- a/lib/event_source/postgres/read/iterator.rb +++ b/lib/event_source/postgres/read/iterator.rb @@ -4,7 +4,12 @@ class Read class Iterator include EventSource::Iterator - def configure + def last_position + unless EventSource::StreamName.category?(stream_name) + batch.last.position + else + batch.last.global_position + end end end end From 6736a9ad4eeb7ebc3214d387ff9c064207649e39 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 4 Feb 2017 01:13:49 -0600 Subject: [PATCH 174/591] Iterator tests are not part of the get namespace --- test/automated/{get => }/iterator/next.rb | 2 +- test/automated/{get => }/iterator/no_further_event_data.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename test/automated/{get => }/iterator/next.rb (90%) rename test/automated/{get => }/iterator/no_further_event_data.rb (89%) diff --git a/test/automated/get/iterator/next.rb b/test/automated/iterator/next.rb similarity index 90% rename from test/automated/get/iterator/next.rb rename to test/automated/iterator/next.rb index 763c297..f93fbef 100644 --- a/test/automated/get/iterator/next.rb +++ b/test/automated/iterator/next.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../automated_init' context "Iterator" do context "Next" do diff --git a/test/automated/get/iterator/no_further_event_data.rb b/test/automated/iterator/no_further_event_data.rb similarity index 89% rename from test/automated/get/iterator/no_further_event_data.rb rename to test/automated/iterator/no_further_event_data.rb index bf15b6a..ca17372 100644 --- a/test/automated/get/iterator/no_further_event_data.rb +++ b/test/automated/iterator/no_further_event_data.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../automated_init' context "Iterator" do context "No further event data" do From 285c9297ad3e6735e841ec2095e316468b4a15df Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 4 Feb 2017 01:16:03 -0600 Subject: [PATCH 175/591] Generalized iterator is in the read namespace --- lib/event_source/postgres/read/iterator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_source/postgres/read/iterator.rb b/lib/event_source/postgres/read/iterator.rb index 37f7da3..aae482b 100644 --- a/lib/event_source/postgres/read/iterator.rb +++ b/lib/event_source/postgres/read/iterator.rb @@ -2,7 +2,7 @@ module EventSource module Postgres class Read class Iterator - include EventSource::Iterator + include EventSource::Read::Iterator def last_position unless EventSource::StreamName.category?(stream_name) From 80a67894ed5c28000062f636b27a2cd7d9e10b62 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 4 Feb 2017 02:01:12 -0600 Subject: [PATCH 176/591] Package version is increased from 0.14.0.0 to 0.15.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index f96f630..f043e3c 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.14.0.0' + s.version = '0.15.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From c9c4ccf46d3a8971f91ab7fe0429f63d4a0e83f1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 4 Feb 2017 18:06:42 -0600 Subject: [PATCH 177/591] StreamName is specialized to postgres --- lib/event_source/postgres.rb | 2 + .../postgres/controls/category.rb | 29 ++++++++++- .../postgres/controls/stream_name.rb | 17 +++++- lib/event_source/postgres/read/iterator.rb | 2 +- lib/event_source/postgres/stream_name.rb | 52 +++++++++++++++++++ 5 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 lib/event_source/postgres/stream_name.rb diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index 21ec079..ea5b77c 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -12,6 +12,8 @@ require 'event_source/postgres/settings' require 'event_source/postgres/session' +require 'event_source/postgres/stream_name' + require 'event_source/postgres/put' require 'event_source/postgres/write' diff --git a/lib/event_source/postgres/controls/category.rb b/lib/event_source/postgres/controls/category.rb index f5c2f34..33dcf3d 100644 --- a/lib/event_source/postgres/controls/category.rb +++ b/lib/event_source/postgres/controls/category.rb @@ -1,7 +1,34 @@ +# module EventSource +# module Postgres +# module Controls +# Category = EventSource::Controls::Category +# end +# end +# end + + module EventSource module Postgres module Controls - Category = EventSource::Controls::Category + module Category + def self.example(category: nil, randomize_category: nil) + if randomize_category.nil? + if !category.nil? + randomize_category = false + end + end + + randomize_category = true if randomize_category.nil? + + category ||= 'test' + + if randomize_category + category = "#{category}#{SecureRandom.hex(16)}XX" + end + + category + end + end end end end diff --git a/lib/event_source/postgres/controls/stream_name.rb b/lib/event_source/postgres/controls/stream_name.rb index 4f70ad6..0467c0c 100644 --- a/lib/event_source/postgres/controls/stream_name.rb +++ b/lib/event_source/postgres/controls/stream_name.rb @@ -1,7 +1,22 @@ +# module EventSource +# module Postgres +# module Controls +# StreamName = EventSource::Controls::StreamName +# end +# end +# end + module EventSource module Postgres module Controls - StreamName = EventSource::Controls::StreamName + module StreamName + def self.example(category: nil, id: nil, type: nil, types: nil, randomize_category: nil) + category ||= Category.example(category: category, randomize_category: randomize_category) + id ||= Identifier::UUID.random + + EventSource::Postgres::StreamName.stream_name(category, id, type: type, types: types) + end + end end end end diff --git a/lib/event_source/postgres/read/iterator.rb b/lib/event_source/postgres/read/iterator.rb index aae482b..d4d2ebd 100644 --- a/lib/event_source/postgres/read/iterator.rb +++ b/lib/event_source/postgres/read/iterator.rb @@ -5,7 +5,7 @@ class Iterator include EventSource::Read::Iterator def last_position - unless EventSource::StreamName.category?(stream_name) + unless EventSource::Postgres::StreamName.category?(stream_name) batch.last.position else batch.last.global_position diff --git a/lib/event_source/postgres/stream_name.rb b/lib/event_source/postgres/stream_name.rb new file mode 100644 index 0000000..d7f81fe --- /dev/null +++ b/lib/event_source/postgres/stream_name.rb @@ -0,0 +1,52 @@ +module EventSource + module Postgres + module StreamName + def self.stream_name(category_name, id=nil, type: nil, types: nil) + types = Array(types) + types.unshift(type) unless type.nil? + + type_list = nil + type_list = types.join('+') unless types.empty? + + stream_name = category_name + stream_name = "#{stream_name}:#{type_list}" unless type_list.nil? + stream_name = "#{stream_name}-#{id}" unless id.nil? + + stream_name + end + + def self.get_id(stream_name) + id = stream_name.partition('-').last + id.empty? ? nil : id + end + + def self.get_category(stream_name) + stream_name.split('-').first + end + + def self.category?(stream_name) + !stream_name.include?('-') + end + + def self.get_type_list(stream_name) + type = stream_name.split(':').last.split('-').first + + return nil if stream_name.start_with?(type) + + type + end + + def self.get_types(stream_name) + type_list = get_type_list(stream_name) + + return [] if type_list.nil? + + type_list.split('+') + end + + def self.get_entity_name(stream_name) + get_category(stream_name).split(':').first + end + end + end +end From ee3456b8cdff70b5585de6fc9d51ec313969bfd5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 4 Feb 2017 20:35:26 -0600 Subject: [PATCH 178/591] Package version is increased from 0.15.0.0 to 0.15.1.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index f043e3c..99f7a39 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.15.0.0' + s.version = '0.15.1.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 38a623b5bcb046863166b79307d58c4fe73ef0ee Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 4 Feb 2017 22:01:07 -0600 Subject: [PATCH 179/591] Stream name test coverage is increased --- test/automated/stream_name/get_category.rb | 57 +++++++++++++++++++ test/automated/stream_name/get_entity_name.rb | 55 ++++++++++++++++++ test/automated/stream_name/get_id.rb | 19 +++++++ test/automated/stream_name/get_types.rb | 30 ++++++++++ test/automated/stream_name/is_category.rb | 26 +++++++++ test/automated/stream_name/stream_name.rb | 51 +++++++++++++++++ 6 files changed, 238 insertions(+) create mode 100644 test/automated/stream_name/get_category.rb create mode 100644 test/automated/stream_name/get_entity_name.rb create mode 100644 test/automated/stream_name/get_id.rb create mode 100644 test/automated/stream_name/get_types.rb create mode 100644 test/automated/stream_name/is_category.rb create mode 100644 test/automated/stream_name/stream_name.rb diff --git a/test/automated/stream_name/get_category.rb b/test/automated/stream_name/get_category.rb new file mode 100644 index 0000000..e13742c --- /dev/null +++ b/test/automated/stream_name/get_category.rb @@ -0,0 +1,57 @@ +require_relative '../automated_init' + +context "Stream Name" do + context "Get Category" do + category = 'someStream' + + context "Stream Name Contains an ID" do + id = Identifier::UUID.random + stream_name = "#{category}-#{id}" + + stream_category = StreamName.get_category(stream_name) + + test "Category name is the part of the stream name before the first dash" do + assert(stream_category == category) + end + end + + context "Stream Name Contains no ID" do + stream_name = category + stream_category = StreamName.get_category(stream_name) + + test "Category name is the stream name" do + assert(stream_category == category) + end + end + + context "Stream Name Contains Type" do + stream_name = "#{category}:someType" + stream_category = StreamName.get_category(stream_name) + + test "Category name is the stream name" do + assert(stream_category == stream_name) + end + end + + context "Stream Name Contains Types" do + stream_name = "#{category}:someType+someOtherType" + stream_category = StreamName.get_category(stream_name) + + test "Category name is the stream name" do + assert(stream_category == stream_name) + end + end + + context "Stream Name Contains ID and Types" do + id = Identifier::UUID.random + category_and_types = "#{category}:someType+someOtherType" + stream_name = "#{category_and_types}-#{id}" + + stream_category = StreamName.get_category(stream_name) + + test "Category name is the stream name" do + assert(stream_category == category_and_types) + end + end + end +end diff --git a/test/automated/stream_name/get_entity_name.rb b/test/automated/stream_name/get_entity_name.rb new file mode 100644 index 0000000..67292eb --- /dev/null +++ b/test/automated/stream_name/get_entity_name.rb @@ -0,0 +1,55 @@ +require_relative '../automated_init' + +context "Stream Name" do + context "Get Entity Name" do + category = 'someStream' + + context "Stream Name Contains an ID" do + id = Identifier::UUID.random + stream_name = "#{category}-#{id}" + + entity_name = StreamName.get_entity_name(stream_name) + + test "Entity name is the part of the stream name before the first dash" do + assert(entity_name == category) + end + end + + context "Stream Name Contains no ID" do + stream_name = category + entity_name = StreamName.get_entity_name(stream_name) + + test "Entity name is the stream name" do + assert(entity_name == category) + end + end + + context "Stream Name Contains Type" do + stream_name = "#{category}:someType" + entity_name = StreamName.get_entity_name(stream_name) + + test "Entity name is the category without any types" do + assert(entity_name == category) + end + end + + context "Stream Name Contains Types" do + stream_name = "#{category}:someType+someOtherType" + entity_name = StreamName.get_entity_name(stream_name) + + test "Entity name is the category without any types" do + assert(entity_name == category) + end + end + + context "Stream Name Contains ID and Types" do + id = Identifier::UUID.random + stream_name = "#{category}:someType+someOtherType-#{id}" + entity_name = StreamName.get_entity_name(stream_name) + + test "Entity name is the category without any types" do + assert(entity_name == category) + end + end + end +end diff --git a/test/automated/stream_name/get_id.rb b/test/automated/stream_name/get_id.rb new file mode 100644 index 0000000..e8fa9d8 --- /dev/null +++ b/test/automated/stream_name/get_id.rb @@ -0,0 +1,19 @@ +require_relative '../automated_init' + +context "Stream Name" do + context "Get ID" do + test "Is the part of a stream name after the first dash" do + id = Identifier::UUID.random + stream_name = "someStream-#{id}" + + stream_id = StreamName.get_id(stream_name) + + assert(stream_id == id) + end + + test "Is nil if there is no ID part in the stream name" do + stream_id = StreamName.get_id('someStream') + assert(stream_id.nil?) + end + end +end diff --git a/test/automated/stream_name/get_types.rb b/test/automated/stream_name/get_types.rb new file mode 100644 index 0000000..2d00211 --- /dev/null +++ b/test/automated/stream_name/get_types.rb @@ -0,0 +1,30 @@ +require_relative '../automated_init' + +context "Stream Name" do + context "Get Types" do + context "Many Types" do + test "Types are the list of elements following a colon separator and preceding the ID" do + stream_name = "someStream:someType+someOtherType" + + types = StreamName.get_types(stream_name) + + assert(types == ['someType', 'someOtherType']) + end + end + + context "Single Type" do + test "Types are the list of elements following a colon separator and preceding the ID" do + stream_name = "someStream:someType" + + types = StreamName.get_types(stream_name) + + assert(types == ['someType']) + end + end + + test "Is empty if there is no type list in the stream name" do + types = StreamName.get_types('someStream') + assert(types.empty?) + end + end +end diff --git a/test/automated/stream_name/is_category.rb b/test/automated/stream_name/is_category.rb new file mode 100644 index 0000000..f4032cd --- /dev/null +++ b/test/automated/stream_name/is_category.rb @@ -0,0 +1,26 @@ +require_relative '../automated_init' + +context "Stream Name" do + context "Is Category" do + category = 'someStream' + + context "Stream Name Contains a Dash (-)" do + id = Identifier::UUID.random + stream_name = "#{category}-#{id}" + + is_category = StreamName.category?(stream_name) + + test "Not a category" do + refute(is_category) + end + end + + context "Stream Name Contains no Dash (-)" do + is_category = StreamName.category?(category) + + test "Is a category" do + assert(is_category) + end + end + end +end diff --git a/test/automated/stream_name/stream_name.rb b/test/automated/stream_name/stream_name.rb new file mode 100644 index 0000000..82475db --- /dev/null +++ b/test/automated/stream_name/stream_name.rb @@ -0,0 +1,51 @@ +require_relative '../automated_init' + +context "Stream Name" do + context "Category" do + stream_name = StreamName.stream_name('someCategory') + + test "Stream name is the category" do + assert(stream_name == 'someCategory') + end + end + + context "Category and ID" do + stream_name = StreamName.stream_name('someCategory', 'some_id') + + test "Stream name is the category and the ID" do + assert(stream_name == 'someCategory-some_id') + end + end + + context "Category and Type" do + stream_name = StreamName.stream_name('someCategory', type: 'someType') + + test "Stream name is the category and the Type" do + assert(stream_name == 'someCategory:someType') + end + end + + context "Category and Types" do + stream_name = StreamName.stream_name('someCategory', types: ['someType', 'someOtherType']) + + test "Stream name is the category and the types delimited by the plus (+) sign" do + assert(stream_name == 'someCategory:someType+someOtherType') + end + end + + context "Category, Type, and Types" do + stream_name = StreamName.stream_name('someCategory', type: 'someType', types: ['someOtherType', 'yetAnotherYet']) + + test "Stream name is the category and the types delimited by the plus (+) sign" do + assert(stream_name == 'someCategory:someType+someOtherType+yetAnotherYet') + end + end + + context "Category, Type and ID" do + stream_name = StreamName.stream_name('someCategory', 'some_id', type: 'someType') + + test "Stream name is the category, type and ID" do + assert(stream_name == 'someCategory:someType-some_id') + end + end +end From 33e94815f4db7ef95b3374ac1b229e5da91bf1ab Mon Sep 17 00:00:00 2001 From: ntl Date: Mon, 27 Feb 2017 11:39:18 -0600 Subject: [PATCH 180/591] Postgres root database is specified when creating database --- database/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/install.sh b/database/install.sh index f1826b0..9419e96 100755 --- a/database/install.sh +++ b/database/install.sh @@ -46,7 +46,7 @@ function create-user { function create-database { echo "Database name is: $database" - database_exists=`psql -tAc "SELECT 1 FROM pg_database WHERE datname='$database'"` + database_exists=`psql postgres -tAc "SELECT 1 FROM pg_database WHERE datname='$database'"` if [ "$database_exists" = "1" ]; then echo "Database \"$database\" was previously created. Not creating again." From 8cf62f6a8d0e541019d93576649ce6ffd6aff0a3 Mon Sep 17 00:00:00 2001 From: ntl Date: Mon, 27 Feb 2017 11:45:45 -0600 Subject: [PATCH 181/591] Get::Last does not include EventSource::Get --- lib/event_source/postgres/get/last.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/event_source/postgres/get/last.rb b/lib/event_source/postgres/get/last.rb index 8b43da8..90dd30b 100644 --- a/lib/event_source/postgres/get/last.rb +++ b/lib/event_source/postgres/get/last.rb @@ -2,7 +2,6 @@ module EventSource module Postgres class Get class Last - include EventSource::Get dependency :session, Session From 2b5e14053a5f8e6b7dd2b913225d2593417321ae Mon Sep 17 00:00:00 2001 From: ntl Date: Mon, 27 Feb 2017 11:46:06 -0600 Subject: [PATCH 182/591] Get::Last includes Log::Dependency --- lib/event_source/postgres/get/last.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/event_source/postgres/get/last.rb b/lib/event_source/postgres/get/last.rb index 90dd30b..f903fae 100644 --- a/lib/event_source/postgres/get/last.rb +++ b/lib/event_source/postgres/get/last.rb @@ -2,6 +2,7 @@ module EventSource module Postgres class Get class Last + include Log::Dependency dependency :session, Session From adfca1d1a7b56f2155dcbab9cb86f45e44f0ed0b Mon Sep 17 00:00:00 2001 From: ntl Date: Mon, 27 Feb 2017 11:46:39 -0600 Subject: [PATCH 183/591] Default attribute name of Get::Last is get_last, not get --- lib/event_source/postgres/get/last.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_source/postgres/get/last.rb b/lib/event_source/postgres/get/last.rb index f903fae..dbd9a82 100644 --- a/lib/event_source/postgres/get/last.rb +++ b/lib/event_source/postgres/get/last.rb @@ -13,7 +13,7 @@ def self.build(session: nil) end def self.configure(receiver, attr_name: nil, session: nil) - attr_name ||= :get + attr_name ||= :get_last instance = build(session: session) receiver.public_send "#{attr_name}=", instance end From a353d090895a45034588014412de450065c142a3 Mon Sep 17 00:00:00 2001 From: ntl Date: Tue, 14 Mar 2017 10:10:48 -0500 Subject: [PATCH 184/591] Package version is increased from 0.15.1.0 to 0.16.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 99f7a39..64f342a 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.15.1.0' + s.version = '0.16.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 40e1d7f0dd2e9e8584e9bfcfac47396a4fec1ad5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 14:12:28 -0500 Subject: [PATCH 185/591] Test shim is installed with the package --- .gitignore | 5 +---- event_source-postgres.gemspec | 3 +++ scripts/evt-shim-test | 6 ++++++ test/package/package.sh | 22 ++++++++++++++++++++++ 4 files changed, 32 insertions(+), 4 deletions(-) create mode 100755 scripts/evt-shim-test create mode 100755 test/package/package.sh diff --git a/.gitignore b/.gitignore index d28b09c..3bb1e20 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,10 @@ .DS_store .bundle/ -.ruby-version -.ruby-gemset -.rvmrc Gemfile.lock *.log *.gem gems -gems_backup *scratch* *notes* loader.rb +/test/package/installed diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 64f342a..b630350 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -15,6 +15,9 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.required_ruby_version = '>= 2.3.3' + s.executables = Dir.glob('scripts/*').map(&File.method(:basename)) + s.bindir = 'scripts' + s.add_runtime_dependency 'evt-event_source' s.add_runtime_dependency 'evt-log' s.add_runtime_dependency 'evt-cycle' diff --git a/scripts/evt-shim-test b/scripts/evt-shim-test new file mode 100755 index 0000000..8c17222 --- /dev/null +++ b/scripts/evt-shim-test @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +puts +puts RUBY_DESCRIPTION +puts "Executing: #{__FILE__}" +puts diff --git a/test/package/package.sh b/test/package/package.sh new file mode 100755 index 0000000..ec2cafc --- /dev/null +++ b/test/package/package.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -ue + +rm -f *.gem +rm -rf test/package/installed + +for gemspec in *.gemspec; do + echo "Building $gemspec" + gem build $gemspec --norc +done + +for gem in *.gem; do + echo "Installing $gem" + gem install $gem \ + --install-dir ./test/package/installed \ + --norc \ + --no-document \ + --no-ri +done + +GEM_PATH=test/package/installed test/package/installed/bin/evt-shim-test From a7ccfc705c083c1cc9a6bdd6cd5965d6f438646b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 14:22:08 -0500 Subject: [PATCH 186/591] Shim test runs script from the database folder --- database/shim_test.rb | 6 ++++++ event_source-postgres.gemspec | 2 +- scripts/evt-shim-test | 7 +------ 3 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 database/shim_test.rb diff --git a/database/shim_test.rb b/database/shim_test.rb new file mode 100644 index 0000000..8c17222 --- /dev/null +++ b/database/shim_test.rb @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +puts +puts RUBY_DESCRIPTION +puts "Executing: #{__FILE__}" +puts diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index b630350..be4a45f 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.licenses = ['MIT'] s.require_paths = ['lib'] - s.files = Dir.glob('{lib}/**/*') + s.files = Dir.glob('{lib,database}/**/*') s.platform = Gem::Platform::RUBY s.required_ruby_version = '>= 2.3.3' diff --git a/scripts/evt-shim-test b/scripts/evt-shim-test index 8c17222..16483c5 100755 --- a/scripts/evt-shim-test +++ b/scripts/evt-shim-test @@ -1,6 +1 @@ -#!/usr/bin/env ruby - -puts -puts RUBY_DESCRIPTION -puts "Executing: #{__FILE__}" -puts +require_relative '../database/shim_test' From 960c5187a15a5cec72d89f863a2eab03d2c77fa8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 15:27:06 -0500 Subject: [PATCH 187/591] Skeleton of executable scripts --- database/shim_test.rb | 6 ------ event_source-postgres.gemspec | 4 ++-- lib/event_source/postgres/database.rb | 0 scripts/evt-shim-test | 1 - scripts/scripts_init.rb | 14 ++++++++++++++ 5 files changed, 16 insertions(+), 9 deletions(-) delete mode 100644 database/shim_test.rb create mode 100644 lib/event_source/postgres/database.rb delete mode 100755 scripts/evt-shim-test create mode 100644 scripts/scripts_init.rb diff --git a/database/shim_test.rb b/database/shim_test.rb deleted file mode 100644 index 8c17222..0000000 --- a/database/shim_test.rb +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env ruby - -puts -puts RUBY_DESCRIPTION -puts "Executing: #{__FILE__}" -puts diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index be4a45f..656565e 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -11,11 +11,11 @@ Gem::Specification.new do |s| s.licenses = ['MIT'] s.require_paths = ['lib'] - s.files = Dir.glob('{lib,database}/**/*') + s.files = Dir.glob('{lib,database}/**/*') + ['scripts/scripts_init.rb'] s.platform = Gem::Platform::RUBY s.required_ruby_version = '>= 2.3.3' - s.executables = Dir.glob('scripts/*').map(&File.method(:basename)) + s.executables = Dir.glob('scripts/evt-*').map(&File.method(:basename)) s.bindir = 'scripts' s.add_runtime_dependency 'evt-event_source' diff --git a/lib/event_source/postgres/database.rb b/lib/event_source/postgres/database.rb new file mode 100644 index 0000000..e69de29 diff --git a/scripts/evt-shim-test b/scripts/evt-shim-test deleted file mode 100755 index 16483c5..0000000 --- a/scripts/evt-shim-test +++ /dev/null @@ -1 +0,0 @@ -require_relative '../database/shim_test' diff --git a/scripts/scripts_init.rb b/scripts/scripts_init.rb new file mode 100644 index 0000000..012f628 --- /dev/null +++ b/scripts/scripts_init.rb @@ -0,0 +1,14 @@ +ENV['CONSOLE_DEVICE'] ||= 'stdout' +ENV['LOG_LEVEL'] ||= '_min' + +puts RUBY_DESCRIPTION + +init_file = File.expand_path('../init.rb', __dir__) +# if File.exist?('init.rb') +if File.exist?(init_file) + require_relative '../init' +end + +require 'event_source/postgres/database' + +puts "SCRIPTS INIT" From 244f8e8259fb36749cf4ec32fe6269cc4130554e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 15:29:06 -0500 Subject: [PATCH 188/591] Inactive code is removed --- scripts/scripts_init.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/scripts_init.rb b/scripts/scripts_init.rb index 012f628..78ab0a2 100644 --- a/scripts/scripts_init.rb +++ b/scripts/scripts_init.rb @@ -4,7 +4,6 @@ puts RUBY_DESCRIPTION init_file = File.expand_path('../init.rb', __dir__) -# if File.exist?('init.rb') if File.exist?(init_file) require_relative '../init' end From 52e94f6e528b48ba38b55f233e03ba17ec4af53d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 16:50:27 -0500 Subject: [PATCH 189/591] Sketch of delete user and database --- lib/event_source/postgres/database.rb | 1 + lib/event_source/postgres/database/delete.rb | 18 ++++++++++++++++++ test/database/database_init.rb | 3 +++ test/database/delete.rb | 3 +++ 4 files changed, 25 insertions(+) create mode 100644 lib/event_source/postgres/database/delete.rb create mode 100644 test/database/database_init.rb create mode 100644 test/database/delete.rb diff --git a/lib/event_source/postgres/database.rb b/lib/event_source/postgres/database.rb index e69de29..9bb3f6c 100644 --- a/lib/event_source/postgres/database.rb +++ b/lib/event_source/postgres/database.rb @@ -0,0 +1 @@ +require 'event_source/postgres/database/delete' diff --git a/lib/event_source/postgres/database/delete.rb b/lib/event_source/postgres/database/delete.rb new file mode 100644 index 0000000..ee2fe17 --- /dev/null +++ b/lib/event_source/postgres/database/delete.rb @@ -0,0 +1,18 @@ +module EventSource + module Postgres + module Database + module Delete + def self.call + settings = Settings.build(Hash.new) + session = Session.build(settings: settings) + + user = ENV['DATABASE_USER'] || 'event_source' + database = ENV['DATABASE_NAME'] || 'event_source' + + session.execute("DROP USER IF EXISTS #{user}") + session.execute("DROP DATABASE IF EXISTS #{database}") + end + end + end + end +end diff --git a/test/database/database_init.rb b/test/database/database_init.rb new file mode 100644 index 0000000..e2a18f8 --- /dev/null +++ b/test/database/database_init.rb @@ -0,0 +1,3 @@ +require_relative '../test_init' + +require 'event_source/postgres/database' diff --git a/test/database/delete.rb b/test/database/delete.rb new file mode 100644 index 0000000..838b9a6 --- /dev/null +++ b/test/database/delete.rb @@ -0,0 +1,3 @@ +require_relative 'database_init' + +Database::Delete.() From 128c037a7aaac60c3c6228622d69e76d3b8fc6a6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 17:12:47 -0500 Subject: [PATCH 190/591] Create and delete database and user --- lib/event_source/postgres/database.rb | 2 ++ lib/event_source/postgres/database/create.rb | 30 +++++++++++++++++++ lib/event_source/postgres/database/delete.rb | 3 +- lib/event_source/postgres/database/session.rb | 14 +++++++++ test/database/create.rb | 3 ++ 5 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 lib/event_source/postgres/database/create.rb create mode 100644 lib/event_source/postgres/database/session.rb create mode 100644 test/database/create.rb diff --git a/lib/event_source/postgres/database.rb b/lib/event_source/postgres/database.rb index 9bb3f6c..05fba99 100644 --- a/lib/event_source/postgres/database.rb +++ b/lib/event_source/postgres/database.rb @@ -1 +1,3 @@ +require 'event_source/postgres/database/session' require 'event_source/postgres/database/delete' +require 'event_source/postgres/database/create' diff --git a/lib/event_source/postgres/database/create.rb b/lib/event_source/postgres/database/create.rb new file mode 100644 index 0000000..d81d61e --- /dev/null +++ b/lib/event_source/postgres/database/create.rb @@ -0,0 +1,30 @@ +module EventSource + module Postgres + module Database + module Create + def self.call + session = Session.build + + user = ENV['DATABASE_USER'] || 'event_source' + password = ENV['DATABASE_PASSWORD'] + database = ENV['DATABASE_NAME'] || 'event_source' + + create_user_statement = "CREATE USER #{user}" + create_user_statement << " PASSWORD #{password}" unless password.nil? + + begin + session.execute(create_user_statement) + rescue PG::DuplicateObject + warn "NOTICE: role \"#{user}\" already exists, skipping" + end + + begin + session.execute("CREATE DATABASE #{database}") + rescue PG::DuplicateDatabase + warn "NOTICE: database \"#{database}\" already exists, skipping" + end + end + end + end + end +end diff --git a/lib/event_source/postgres/database/delete.rb b/lib/event_source/postgres/database/delete.rb index ee2fe17..4bb5106 100644 --- a/lib/event_source/postgres/database/delete.rb +++ b/lib/event_source/postgres/database/delete.rb @@ -3,8 +3,7 @@ module Postgres module Database module Delete def self.call - settings = Settings.build(Hash.new) - session = Session.build(settings: settings) + session = Session.build user = ENV['DATABASE_USER'] || 'event_source' database = ENV['DATABASE_NAME'] || 'event_source' diff --git a/lib/event_source/postgres/database/session.rb b/lib/event_source/postgres/database/session.rb new file mode 100644 index 0000000..ebe897a --- /dev/null +++ b/lib/event_source/postgres/database/session.rb @@ -0,0 +1,14 @@ +module EventSource + module Postgres + module Database + class Session < Postgres::Session + def self.build + instance = new + settings = Settings.build({}) + settings.set(instance) + instance + end + end + end + end +end diff --git a/test/database/create.rb b/test/database/create.rb new file mode 100644 index 0000000..685fdc7 --- /dev/null +++ b/test/database/create.rb @@ -0,0 +1,3 @@ +require_relative 'database_init' + +Database::Create.() From 0e11885c3d6a268ea295fc640af5eebbbef65da7 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 17:44:10 -0500 Subject: [PATCH 191/591] Code is organized per subject --- lib/event_source/postgres/database.rb | 3 ++ lib/event_source/postgres/database/create.rb | 21 ++----------- .../postgres/database/database.rb | 26 ++++++++++++++++ lib/event_source/postgres/database/delete.rb | 7 ++--- lib/event_source/postgres/database/session.rb | 2 +- .../postgres/database/settings.rb | 19 ++++++++++++ lib/event_source/postgres/database/table.rb | 16 ++++++++++ lib/event_source/postgres/database/user.rb | 30 +++++++++++++++++++ 8 files changed, 100 insertions(+), 24 deletions(-) create mode 100644 lib/event_source/postgres/database/database.rb create mode 100644 lib/event_source/postgres/database/settings.rb create mode 100644 lib/event_source/postgres/database/table.rb create mode 100644 lib/event_source/postgres/database/user.rb diff --git a/lib/event_source/postgres/database.rb b/lib/event_source/postgres/database.rb index 05fba99..4ae0e99 100644 --- a/lib/event_source/postgres/database.rb +++ b/lib/event_source/postgres/database.rb @@ -1,3 +1,6 @@ +require 'event_source/postgres/database/settings' require 'event_source/postgres/database/session' require 'event_source/postgres/database/delete' require 'event_source/postgres/database/create' +require 'event_source/postgres/database/user' +require 'event_source/postgres/database/database' diff --git a/lib/event_source/postgres/database/create.rb b/lib/event_source/postgres/database/create.rb index d81d61e..0f09853 100644 --- a/lib/event_source/postgres/database/create.rb +++ b/lib/event_source/postgres/database/create.rb @@ -5,24 +5,9 @@ module Create def self.call session = Session.build - user = ENV['DATABASE_USER'] || 'event_source' - password = ENV['DATABASE_PASSWORD'] - database = ENV['DATABASE_NAME'] || 'event_source' - - create_user_statement = "CREATE USER #{user}" - create_user_statement << " PASSWORD #{password}" unless password.nil? - - begin - session.execute(create_user_statement) - rescue PG::DuplicateObject - warn "NOTICE: role \"#{user}\" already exists, skipping" - end - - begin - session.execute("CREATE DATABASE #{database}") - rescue PG::DuplicateDatabase - warn "NOTICE: database \"#{database}\" already exists, skipping" - end + User::Create.(session) + Database::Create.(session) + Table::Create.(session) end end end diff --git a/lib/event_source/postgres/database/database.rb b/lib/event_source/postgres/database/database.rb new file mode 100644 index 0000000..b3b3bff --- /dev/null +++ b/lib/event_source/postgres/database/database.rb @@ -0,0 +1,26 @@ +module EventSource + module Postgres + module Database + module Database + module Create + def self.call(session) + database = Settings.database + + begin + session.execute("CREATE DATABASE #{database}") + rescue PG::DuplicateDatabase + warn "NOTICE: database \"#{database}\" already exists, skipping" + end + end + end + + module Delete + def self.call(session) + database = Settings.database + session.execute("DROP DATABASE IF EXISTS #{database}") + end + end + end + end + end +end diff --git a/lib/event_source/postgres/database/delete.rb b/lib/event_source/postgres/database/delete.rb index 4bb5106..34b6296 100644 --- a/lib/event_source/postgres/database/delete.rb +++ b/lib/event_source/postgres/database/delete.rb @@ -5,11 +5,8 @@ module Delete def self.call session = Session.build - user = ENV['DATABASE_USER'] || 'event_source' - database = ENV['DATABASE_NAME'] || 'event_source' - - session.execute("DROP USER IF EXISTS #{user}") - session.execute("DROP DATABASE IF EXISTS #{database}") + User::Delete.(session) + Database::Delete.(session) end end end diff --git a/lib/event_source/postgres/database/session.rb b/lib/event_source/postgres/database/session.rb index ebe897a..b65197a 100644 --- a/lib/event_source/postgres/database/session.rb +++ b/lib/event_source/postgres/database/session.rb @@ -4,7 +4,7 @@ module Database class Session < Postgres::Session def self.build instance = new - settings = Settings.build({}) + settings = Postgres::Settings.build({}) settings.set(instance) instance end diff --git a/lib/event_source/postgres/database/settings.rb b/lib/event_source/postgres/database/settings.rb new file mode 100644 index 0000000..bc04625 --- /dev/null +++ b/lib/event_source/postgres/database/settings.rb @@ -0,0 +1,19 @@ +module EventSource + module Postgres + module Database + module Settings + def self.user + ENV['DATABASE_USER'] || 'event_source' + end + + def self.password + ENV['DATABASE_PASSWORD'] + end + + def self.database + ENV['DATABASE_NAME'] || 'event_source' + end + end + end + end +end diff --git a/lib/event_source/postgres/database/table.rb b/lib/event_source/postgres/database/table.rb new file mode 100644 index 0000000..3918990 --- /dev/null +++ b/lib/event_source/postgres/database/table.rb @@ -0,0 +1,16 @@ +module EventSource + module Postgres + module Database + module Table + module Create + def self.sql_filename + 'table/events-table.sql' + end + + def self.call(session) + end + end + end + end + end +end diff --git a/lib/event_source/postgres/database/user.rb b/lib/event_source/postgres/database/user.rb new file mode 100644 index 0000000..55cc63e --- /dev/null +++ b/lib/event_source/postgres/database/user.rb @@ -0,0 +1,30 @@ +module EventSource + module Postgres + module Database + module User + module Create + def self.call(session) + user = Settings.user + password = Settings.password + + create_user_statement = "CREATE USER #{user}" + create_user_statement << " PASSWORD #{password}" unless password.nil? + + begin + session.execute(create_user_statement) + rescue PG::DuplicateObject + warn "NOTICE: role \"#{user}\" already exists, skipping" + end + end + end + + module Delete + def self.call(session) + user = Settings.user + session.execute("DROP USER IF EXISTS #{user}") + end + end + end + end + end +end From 7ce7ad4629303716b4e735eae84164990160beff Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 17:48:08 -0500 Subject: [PATCH 192/591] Skeleton of extensions --- lib/event_source/postgres/database.rb | 2 ++ .../postgres/database/extensions.rb | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 lib/event_source/postgres/database/extensions.rb diff --git a/lib/event_source/postgres/database.rb b/lib/event_source/postgres/database.rb index 4ae0e99..06fb522 100644 --- a/lib/event_source/postgres/database.rb +++ b/lib/event_source/postgres/database.rb @@ -4,3 +4,5 @@ require 'event_source/postgres/database/create' require 'event_source/postgres/database/user' require 'event_source/postgres/database/database' +require 'event_source/postgres/database/extensions' +require 'event_source/postgres/database/table' diff --git a/lib/event_source/postgres/database/extensions.rb b/lib/event_source/postgres/database/extensions.rb new file mode 100644 index 0000000..8c51704 --- /dev/null +++ b/lib/event_source/postgres/database/extensions.rb @@ -0,0 +1,17 @@ +module EventSource + module Postgres + module Database + module Extensions + module Create + def self.sql_filename + 'database/extensions.sql' + end + + def self.call(session) + session.execute(sql_code) + end + end + end + end + end +end From 0b5224b6b0533474671516229d6366107ca4e6fe Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 31 Mar 2017 17:59:56 -0500 Subject: [PATCH 193/591] Extensions and table can be created --- lib/event_source/postgres/database.rb | 1 + lib/event_source/postgres/database/create.rb | 1 + .../postgres/database/extensions.rb | 4 +++- lib/event_source/postgres/database/sql_code.rb | 17 +++++++++++++++++ lib/event_source/postgres/database/table.rb | 3 +++ 5 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 lib/event_source/postgres/database/sql_code.rb diff --git a/lib/event_source/postgres/database.rb b/lib/event_source/postgres/database.rb index 06fb522..17799f0 100644 --- a/lib/event_source/postgres/database.rb +++ b/lib/event_source/postgres/database.rb @@ -6,3 +6,4 @@ require 'event_source/postgres/database/database' require 'event_source/postgres/database/extensions' require 'event_source/postgres/database/table' +require 'event_source/postgres/database/sql_code' diff --git a/lib/event_source/postgres/database/create.rb b/lib/event_source/postgres/database/create.rb index 0f09853..b4f46f0 100644 --- a/lib/event_source/postgres/database/create.rb +++ b/lib/event_source/postgres/database/create.rb @@ -7,6 +7,7 @@ def self.call User::Create.(session) Database::Create.(session) + Extensions::Create.(session) Table::Create.(session) end end diff --git a/lib/event_source/postgres/database/extensions.rb b/lib/event_source/postgres/database/extensions.rb index 8c51704..ecbfeeb 100644 --- a/lib/event_source/postgres/database/extensions.rb +++ b/lib/event_source/postgres/database/extensions.rb @@ -4,10 +4,12 @@ module Database module Extensions module Create def self.sql_filename - 'database/extensions.sql' + 'extensions.sql' end def self.call(session) + sql_code = SQLCode.read sql_filename + session.execute(sql_code) end end diff --git a/lib/event_source/postgres/database/sql_code.rb b/lib/event_source/postgres/database/sql_code.rb new file mode 100644 index 0000000..3173ee7 --- /dev/null +++ b/lib/event_source/postgres/database/sql_code.rb @@ -0,0 +1,17 @@ +module EventSource + module Postgres + module Database + module SQLCode + def self.read(filename) + path = File.join root, filename + + File.read path + end + + def self.root + File.expand_path '../../../../database', __dir__ + end + end + end + end +end diff --git a/lib/event_source/postgres/database/table.rb b/lib/event_source/postgres/database/table.rb index 3918990..be4b62f 100644 --- a/lib/event_source/postgres/database/table.rb +++ b/lib/event_source/postgres/database/table.rb @@ -8,6 +8,9 @@ def self.sql_filename end def self.call(session) + sql_code = SQLCode.read sql_filename + + session.execute(sql_code) end end end From d5dd6bcd401654aa2dfcc37fb6c5516c09088b21 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 31 Mar 2017 18:05:43 -0500 Subject: [PATCH 194/591] Functions and indexes are added to database create procedure --- lib/event_source/postgres/database.rb | 2 + lib/event_source/postgres/database/create.rb | 2 + .../postgres/database/functions.rb | 57 +++++++++++++++++++ lib/event_source/postgres/database/indexes.rb | 19 +++++++ 4 files changed, 80 insertions(+) create mode 100644 lib/event_source/postgres/database/functions.rb create mode 100644 lib/event_source/postgres/database/indexes.rb diff --git a/lib/event_source/postgres/database.rb b/lib/event_source/postgres/database.rb index 17799f0..dd742c7 100644 --- a/lib/event_source/postgres/database.rb +++ b/lib/event_source/postgres/database.rb @@ -6,4 +6,6 @@ require 'event_source/postgres/database/database' require 'event_source/postgres/database/extensions' require 'event_source/postgres/database/table' +require 'event_source/postgres/database/indexes' +require 'event_source/postgres/database/functions' require 'event_source/postgres/database/sql_code' diff --git a/lib/event_source/postgres/database/create.rb b/lib/event_source/postgres/database/create.rb index b4f46f0..58519a8 100644 --- a/lib/event_source/postgres/database/create.rb +++ b/lib/event_source/postgres/database/create.rb @@ -9,6 +9,8 @@ def self.call Database::Create.(session) Extensions::Create.(session) Table::Create.(session) + Functions::Create.(session) + Indexes::Create.(session) end end end diff --git a/lib/event_source/postgres/database/functions.rb b/lib/event_source/postgres/database/functions.rb new file mode 100644 index 0000000..8c45d8d --- /dev/null +++ b/lib/event_source/postgres/database/functions.rb @@ -0,0 +1,57 @@ +module EventSource + module Postgres + module Database + module Functions + module Create + def self.call(session) + Category::Create.(session) + StreamVersion::Create.(session) + WriteEvent::Create.(session) + end + + module Category + module Create + def self.sql_filename + 'functions/category.sql' + end + + def self.call(session) + sql_code = SQLCode.read sql_filename + + session.execute(sql_code) + end + end + end + + module StreamVersion + module Create + def self.sql_filename + 'functions/stream-version.sql' + end + + def self.call(session) + sql_code = SQLCode.read sql_filename + + session.execute(sql_code) + end + end + end + + module WriteEvent + module Create + def self.sql_filename + 'functions/write-event.sql' + end + + def self.call(session) + sql_code = SQLCode.read sql_filename + + session.execute(sql_code) + end + end + end + end + end + end + end +end diff --git a/lib/event_source/postgres/database/indexes.rb b/lib/event_source/postgres/database/indexes.rb new file mode 100644 index 0000000..444854d --- /dev/null +++ b/lib/event_source/postgres/database/indexes.rb @@ -0,0 +1,19 @@ +module EventSource + module Postgres + module Database + module Indexes + module Create + def self.sql_filename + 'indexes/events-indexes.sql' + end + + def self.call(session) + sql_code = SQLCode.read sql_filename + + session.execute(sql_code) + end + end + end + end + end +end From c1a003afd7d4a819cd5ec7fe4a6a80ae19f8884b Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 31 Mar 2017 18:14:39 -0500 Subject: [PATCH 195/591] Indexes are added individually --- lib/event_source/postgres/database/indexes.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/event_source/postgres/database/indexes.rb b/lib/event_source/postgres/database/indexes.rb index 444854d..42851e7 100644 --- a/lib/event_source/postgres/database/indexes.rb +++ b/lib/event_source/postgres/database/indexes.rb @@ -10,7 +10,11 @@ def self.sql_filename def self.call(session) sql_code = SQLCode.read sql_filename - session.execute(sql_code) + sql_statements = sql_code.split ';' + + sql_statements.each do |sql_statement| + session.execute(sql_statement) + end end end end From b50f85fe271a786dc758d00f3ab0c260a2bcc7e9 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 18:27:29 -0500 Subject: [PATCH 196/591] Each index is in its own SQL file --- database/indexes/events-category-global-position.sql | 1 + database/indexes/events-category.sql | 1 + database/indexes/events-id.sql | 1 + database/indexes/events-indexes.sql | 8 -------- database/indexes/events-stream-name-position-uniq.sql | 1 + database/indexes/events-stream-name.sql | 1 + database/install.sh | 6 +++++- 7 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 database/indexes/events-category-global-position.sql create mode 100644 database/indexes/events-category.sql create mode 100644 database/indexes/events-id.sql delete mode 100644 database/indexes/events-indexes.sql create mode 100644 database/indexes/events-stream-name-position-uniq.sql create mode 100644 database/indexes/events-stream-name.sql diff --git a/database/indexes/events-category-global-position.sql b/database/indexes/events-category-global-position.sql new file mode 100644 index 0000000..798ba04 --- /dev/null +++ b/database/indexes/events-category-global-position.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY "events_category_global_position_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "global_position" "pg_catalog"."int8_ops" ASC NULLS LAST); diff --git a/database/indexes/events-category.sql b/database/indexes/events-category.sql new file mode 100644 index 0000000..c4f6aae --- /dev/null +++ b/database/indexes/events-category.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY "events_category_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); diff --git a/database/indexes/events-id.sql b/database/indexes/events-id.sql new file mode 100644 index 0000000..4eb428c --- /dev/null +++ b/database/indexes/events-id.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY "events_id_idx" ON "public"."events" USING btree(id ASC NULLS LAST); diff --git a/database/indexes/events-indexes.sql b/database/indexes/events-indexes.sql deleted file mode 100644 index 727c8ee..0000000 --- a/database/indexes/events-indexes.sql +++ /dev/null @@ -1,8 +0,0 @@ --- ---------------------------- --- Indexes structure for table events --- ---------------------------- -CREATE INDEX CONCURRENTLY "events_id_idx" ON "public"."events" USING btree(id ASC NULLS LAST); -CREATE INDEX CONCURRENTLY "events_category_global_position_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "global_position" "pg_catalog"."int8_ops" ASC NULLS LAST); -CREATE INDEX CONCURRENTLY "events_category_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); -CREATE INDEX CONCURRENTLY "events_stream_name_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); -CREATE UNIQUE INDEX CONCURRENTLY "events_stream_name_position_uniq_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "position" "pg_catalog"."int4_ops" ASC NULLS LAST); diff --git a/database/indexes/events-stream-name-position-uniq.sql b/database/indexes/events-stream-name-position-uniq.sql new file mode 100644 index 0000000..1864048 --- /dev/null +++ b/database/indexes/events-stream-name-position-uniq.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX CONCURRENTLY "events_stream_name_position_uniq_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "position" "pg_catalog"."int4_ops" ASC NULLS LAST); diff --git a/database/indexes/events-stream-name.sql b/database/indexes/events-stream-name.sql new file mode 100644 index 0000000..7f097ba --- /dev/null +++ b/database/indexes/events-stream-name.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY "events_stream_name_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); diff --git a/database/install.sh b/database/install.sh index 9419e96..55c087a 100755 --- a/database/install.sh +++ b/database/install.sh @@ -74,7 +74,11 @@ function create-functions { } function create-indexes { - psql $database -f database/indexes/events-indexes.sql + psql $database -f database/indexes/events-id.sql + psql $database -f database/indexes/events-category-global-position.sql + psql $database -f database/indexes/events-category.sql + psql $database -f database/indexes/events-stream-name.sql + psql $database -f database/indexes/events-stream-name-position-uniq.sql } create-user From 3c3dad42196c7bc4a29971d8354f66250716c0ed Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 31 Mar 2017 18:32:54 -0500 Subject: [PATCH 197/591] Indexes are updated to use individual SQL files --- lib/event_source/postgres/database/indexes.rb | 78 +++++++++++++++++-- 1 file changed, 71 insertions(+), 7 deletions(-) diff --git a/lib/event_source/postgres/database/indexes.rb b/lib/event_source/postgres/database/indexes.rb index 42851e7..9374806 100644 --- a/lib/event_source/postgres/database/indexes.rb +++ b/lib/event_source/postgres/database/indexes.rb @@ -3,17 +3,81 @@ module Postgres module Database module Indexes module Create - def self.sql_filename - 'indexes/events-indexes.sql' + def self.call(session) + Category::GlobalPosition::Create.(session) + Category::Create.(session) + ID::Create.(session) + StreamName::PositionUniq::Create.(session) + StreamName::Create.(session) end - def self.call(session) - sql_code = SQLCode.read sql_filename + module Category + module Create + def self.sql_filename + 'indexes/events-category.sql' + end + + def self.call(session) + sql_code = SQLCode.read sql_filename + + session.execute(sql_code) + end + end + + module GlobalPosition + module Create + def self.sql_filename + 'indexes/events-category-global-position.sql' + end + + def self.call(session) + sql_code = SQLCode.read sql_filename + + session.execute(sql_code) + end + end + end + end + + module ID + module Create + def self.sql_filename + 'indexes/events-id.sql' + end + + def self.call(session) + sql_code = SQLCode.read sql_filename + + session.execute(sql_code) + end + end + end + + module StreamName + module Create + def self.sql_filename + 'indexes/events-stream-name.sql' + end + + def self.call(session) + sql_code = SQLCode.read sql_filename + + session.execute(sql_code) + end + end + + module PositionUniq + module Create + def self.sql_filename + 'indexes/events-stream-name-position-uniq.sql' + end - sql_statements = sql_code.split ';' + def self.call(session) + sql_code = SQLCode.read sql_filename - sql_statements.each do |sql_statement| - session.execute(sql_statement) + session.execute(sql_code) + end + end end end end From eeb564061410e054875fa8b8572ae7a971465e44 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 31 Mar 2017 18:34:47 -0500 Subject: [PATCH 198/591] Tools directory is removed --- tools/clear-events-table | 3 --- tools/install-event-source-database | 3 --- tools/list-events | 3 --- tools/uninstall-event-source-database | 3 --- 4 files changed, 12 deletions(-) delete mode 100755 tools/clear-events-table delete mode 100755 tools/install-event-source-database delete mode 100755 tools/list-events delete mode 100755 tools/uninstall-event-source-database diff --git a/tools/clear-events-table b/tools/clear-events-table deleted file mode 100755 index 0108fa8..0000000 --- a/tools/clear-events-table +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -database/clear-events-table.sh diff --git a/tools/install-event-source-database b/tools/install-event-source-database deleted file mode 100755 index 7e57268..0000000 --- a/tools/install-event-source-database +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -database/install.sh diff --git a/tools/list-events b/tools/list-events deleted file mode 100755 index c75c2d9..0000000 --- a/tools/list-events +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -database/list-events.sh diff --git a/tools/uninstall-event-source-database b/tools/uninstall-event-source-database deleted file mode 100755 index 3c238f9..0000000 --- a/tools/uninstall-event-source-database +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -database/uninstall.sh From 267da291afa0825ad4c63b04d0670cdc4125a913 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 31 Mar 2017 18:36:17 -0500 Subject: [PATCH 199/591] Reset is added --- lib/event_source/postgres/database.rb | 1 + lib/event_source/postgres/database/reset.rb | 12 ++++++++++++ test/database/reset.rb | 3 +++ 3 files changed, 16 insertions(+) create mode 100644 lib/event_source/postgres/database/reset.rb create mode 100644 test/database/reset.rb diff --git a/lib/event_source/postgres/database.rb b/lib/event_source/postgres/database.rb index dd742c7..4095890 100644 --- a/lib/event_source/postgres/database.rb +++ b/lib/event_source/postgres/database.rb @@ -9,3 +9,4 @@ require 'event_source/postgres/database/indexes' require 'event_source/postgres/database/functions' require 'event_source/postgres/database/sql_code' +require 'event_source/postgres/database/reset' diff --git a/lib/event_source/postgres/database/reset.rb b/lib/event_source/postgres/database/reset.rb new file mode 100644 index 0000000..16c754c --- /dev/null +++ b/lib/event_source/postgres/database/reset.rb @@ -0,0 +1,12 @@ +module EventSource + module Postgres + module Database + module Reset + def self.call + Delete.() + Create.() + end + end + end + end +end diff --git a/test/database/reset.rb b/test/database/reset.rb new file mode 100644 index 0000000..d59182d --- /dev/null +++ b/test/database/reset.rb @@ -0,0 +1,3 @@ +require_relative 'database_init' + +Database::Reset.() From bc695f18b170b2bcc404b832fbdccbe82437d5cd Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 31 Mar 2017 18:38:08 -0500 Subject: [PATCH 200/591] Output is added to database scripts --- lib/event_source/postgres/database/create.rb | 4 ++++ lib/event_source/postgres/database/delete.rb | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/lib/event_source/postgres/database/create.rb b/lib/event_source/postgres/database/create.rb index 58519a8..b2af27e 100644 --- a/lib/event_source/postgres/database/create.rb +++ b/lib/event_source/postgres/database/create.rb @@ -3,6 +3,8 @@ module Postgres module Database module Create def self.call + warn "Creating event source database" + session = Session.build User::Create.(session) @@ -11,6 +13,8 @@ def self.call Table::Create.(session) Functions::Create.(session) Indexes::Create.(session) + + warn "Done creating event source database" end end end diff --git a/lib/event_source/postgres/database/delete.rb b/lib/event_source/postgres/database/delete.rb index 34b6296..b0bb16d 100644 --- a/lib/event_source/postgres/database/delete.rb +++ b/lib/event_source/postgres/database/delete.rb @@ -3,10 +3,14 @@ module Postgres module Database module Delete def self.call + warn "Deleting event source database" + session = Session.build User::Delete.(session) Database::Delete.(session) + + warn "Done deleting event source database" end end end From 25fe8b55b11a8d63b713d2641024233f9d9f73ee Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 31 Mar 2017 18:42:59 -0500 Subject: [PATCH 201/591] Recreate, not reset --- lib/event_source/postgres/database.rb | 2 +- lib/event_source/postgres/database/{reset.rb => recreate.rb} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename lib/event_source/postgres/database/{reset.rb => recreate.rb} (87%) diff --git a/lib/event_source/postgres/database.rb b/lib/event_source/postgres/database.rb index 4095890..29de5fe 100644 --- a/lib/event_source/postgres/database.rb +++ b/lib/event_source/postgres/database.rb @@ -9,4 +9,4 @@ require 'event_source/postgres/database/indexes' require 'event_source/postgres/database/functions' require 'event_source/postgres/database/sql_code' -require 'event_source/postgres/database/reset' +require 'event_source/postgres/database/recreate' diff --git a/lib/event_source/postgres/database/reset.rb b/lib/event_source/postgres/database/recreate.rb similarity index 87% rename from lib/event_source/postgres/database/reset.rb rename to lib/event_source/postgres/database/recreate.rb index 16c754c..d9c0b86 100644 --- a/lib/event_source/postgres/database/reset.rb +++ b/lib/event_source/postgres/database/recreate.rb @@ -1,7 +1,7 @@ module EventSource module Postgres module Database - module Reset + module Recreate def self.call Delete.() Create.() From e1eca9da76e6c3be53fc5b8ea7ab167cf87c93ec Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 31 Mar 2017 18:43:08 -0500 Subject: [PATCH 202/591] Executables are added --- scripts/evt-pg-create | 5 +++++ scripts/evt-pg-delete | 5 +++++ scripts/evt-pg-recreate | 5 +++++ 3 files changed, 15 insertions(+) create mode 100755 scripts/evt-pg-create create mode 100755 scripts/evt-pg-delete create mode 100755 scripts/evt-pg-recreate diff --git a/scripts/evt-pg-create b/scripts/evt-pg-create new file mode 100755 index 0000000..065197c --- /dev/null +++ b/scripts/evt-pg-create @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby + +require_relative './scripts_init' + +EventSource::Postgres::Database::Create.() diff --git a/scripts/evt-pg-delete b/scripts/evt-pg-delete new file mode 100755 index 0000000..595edd4 --- /dev/null +++ b/scripts/evt-pg-delete @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby + +require_relative './scripts_init' + +EventSource::Postgres::Database::Delete.() diff --git a/scripts/evt-pg-recreate b/scripts/evt-pg-recreate new file mode 100755 index 0000000..b2ab797 --- /dev/null +++ b/scripts/evt-pg-recreate @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby + +require_relative './scripts_init' + +EventSource::Postgres::Database::Recreate.() From 51fd2530af8a76f9db5b8f08cbed4f45f479e00b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 18:43:45 -0500 Subject: [PATCH 203/591] Potential ambiguity in console messages is removed --- lib/event_source/postgres/database/create.rb | 4 ++-- lib/event_source/postgres/database/delete.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/event_source/postgres/database/create.rb b/lib/event_source/postgres/database/create.rb index b2af27e..d4445e6 100644 --- a/lib/event_source/postgres/database/create.rb +++ b/lib/event_source/postgres/database/create.rb @@ -3,7 +3,7 @@ module Postgres module Database module Create def self.call - warn "Creating event source database" + warn "Creating database" session = Session.build @@ -14,7 +14,7 @@ def self.call Functions::Create.(session) Indexes::Create.(session) - warn "Done creating event source database" + warn "Done creating database" end end end diff --git a/lib/event_source/postgres/database/delete.rb b/lib/event_source/postgres/database/delete.rb index b0bb16d..8883547 100644 --- a/lib/event_source/postgres/database/delete.rb +++ b/lib/event_source/postgres/database/delete.rb @@ -3,14 +3,14 @@ module Postgres module Database module Delete def self.call - warn "Deleting event source database" + warn "Deleting database" session = Session.build User::Delete.(session) Database::Delete.(session) - warn "Done deleting event source database" + warn "Done deleting database" end end end From f29ef045b8c06cfa39909af04aaa73e0082f64b1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 18:45:33 -0500 Subject: [PATCH 204/591] Vestigial console message is removed --- scripts/scripts_init.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/scripts_init.rb b/scripts/scripts_init.rb index 78ab0a2..de06c0c 100644 --- a/scripts/scripts_init.rb +++ b/scripts/scripts_init.rb @@ -9,5 +9,3 @@ end require 'event_source/postgres/database' - -puts "SCRIPTS INIT" From 815e423bb27416b0dd6508c9d596b7d1f408c5f2 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 31 Mar 2017 18:48:35 -0500 Subject: [PATCH 205/591] scripts_init loads library --- scripts/scripts_init.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/scripts_init.rb b/scripts/scripts_init.rb index de06c0c..a6ecbc0 100644 --- a/scripts/scripts_init.rb +++ b/scripts/scripts_init.rb @@ -8,4 +8,5 @@ require_relative '../init' end +require 'event_source/postgres' require 'event_source/postgres/database' From afe3db0a09b53b59271d5d3caafb96d268220378 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 18:47:55 -0500 Subject: [PATCH 206/591] Package test runs all commands --- test/package/package.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/package/package.sh b/test/package/package.sh index ec2cafc..d37dd29 100755 --- a/test/package/package.sh +++ b/test/package/package.sh @@ -19,4 +19,6 @@ for gem in *.gem; do --no-ri done -GEM_PATH=test/package/installed test/package/installed/bin/evt-shim-test +GEM_PATH=test/package/installed test/package/installed/bin/evt-pg-delete +GEM_PATH=test/package/installed test/package/installed/bin/evt-pg-create +GEM_PATH=test/package/installed test/package/installed/bin/evt-pg-recreate From 0eaed5d38029e52cd46f5768198e3fe38944e78e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 31 Mar 2017 18:50:40 -0500 Subject: [PATCH 207/591] Package version increased from 0.16.0.0 to 0.16.1.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 656565e..cdc5998 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.16.0.0' + s.version = '0.16.1.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 13251c18dc8fa48b12bde703f6692cd5ed4325ba Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Apr 2017 09:51:17 -0500 Subject: [PATCH 208/591] Database commands are suffixed with -db --- scripts/{evt-pg-create => evt-pg-create-db} | 0 scripts/{evt-pg-delete => evt-pg-delete-db} | 0 scripts/{evt-pg-recreate => evt-pg-recreate-db} | 0 test/package/package.sh | 6 +++--- 4 files changed, 3 insertions(+), 3 deletions(-) rename scripts/{evt-pg-create => evt-pg-create-db} (100%) rename scripts/{evt-pg-delete => evt-pg-delete-db} (100%) rename scripts/{evt-pg-recreate => evt-pg-recreate-db} (100%) diff --git a/scripts/evt-pg-create b/scripts/evt-pg-create-db similarity index 100% rename from scripts/evt-pg-create rename to scripts/evt-pg-create-db diff --git a/scripts/evt-pg-delete b/scripts/evt-pg-delete-db similarity index 100% rename from scripts/evt-pg-delete rename to scripts/evt-pg-delete-db diff --git a/scripts/evt-pg-recreate b/scripts/evt-pg-recreate-db similarity index 100% rename from scripts/evt-pg-recreate rename to scripts/evt-pg-recreate-db diff --git a/test/package/package.sh b/test/package/package.sh index d37dd29..156475e 100755 --- a/test/package/package.sh +++ b/test/package/package.sh @@ -19,6 +19,6 @@ for gem in *.gem; do --no-ri done -GEM_PATH=test/package/installed test/package/installed/bin/evt-pg-delete -GEM_PATH=test/package/installed test/package/installed/bin/evt-pg-create -GEM_PATH=test/package/installed test/package/installed/bin/evt-pg-recreate +GEM_PATH=test/package/installed test/package/installed/bin/evt-pg-delete-db +GEM_PATH=test/package/installed test/package/installed/bin/evt-pg-create-db +GEM_PATH=test/package/installed test/package/installed/bin/evt-pg-recreate-db From 3eb137e0b95795b49c39173f2d76716f40767158 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Apr 2017 15:42:27 -0500 Subject: [PATCH 209/591] Database utilities are executed via shell --- database/install.sh | 2 - database/uninstall.sh | 2 - lib/event_source/postgres/database.rb | 12 --- lib/event_source/postgres/database/create.rb | 22 ----- .../postgres/database/database.rb | 26 ------ lib/event_source/postgres/database/delete.rb | 18 ---- .../postgres/database/extensions.rb | 19 ---- .../postgres/database/functions.rb | 57 ------------ lib/event_source/postgres/database/indexes.rb | 87 ------------------- .../postgres/database/recreate.rb | 12 --- lib/event_source/postgres/database/session.rb | 14 --- .../postgres/database/settings.rb | 19 ---- .../postgres/database/sql_code.rb | 17 ---- lib/event_source/postgres/database/table.rb | 19 ---- lib/event_source/postgres/database/user.rb | 30 ------- scripts/evt-pg-create-db | 6 +- scripts/evt-pg-delete-db | 6 +- scripts/evt-pg-recreate-db | 10 ++- test/database/create.rb | 3 - test/database/database_init.rb | 3 - test/database/delete.rb | 3 - test/database/reset.rb | 3 - 22 files changed, 16 insertions(+), 374 deletions(-) delete mode 100644 lib/event_source/postgres/database.rb delete mode 100644 lib/event_source/postgres/database/create.rb delete mode 100644 lib/event_source/postgres/database/database.rb delete mode 100644 lib/event_source/postgres/database/delete.rb delete mode 100644 lib/event_source/postgres/database/extensions.rb delete mode 100644 lib/event_source/postgres/database/functions.rb delete mode 100644 lib/event_source/postgres/database/indexes.rb delete mode 100644 lib/event_source/postgres/database/recreate.rb delete mode 100644 lib/event_source/postgres/database/session.rb delete mode 100644 lib/event_source/postgres/database/settings.rb delete mode 100644 lib/event_source/postgres/database/sql_code.rb delete mode 100644 lib/event_source/postgres/database/table.rb delete mode 100644 lib/event_source/postgres/database/user.rb delete mode 100644 test/database/create.rb delete mode 100644 test/database/database_init.rb delete mode 100644 test/database/delete.rb delete mode 100644 test/database/reset.rb diff --git a/database/install.sh b/database/install.sh index 55c087a..48e230a 100755 --- a/database/install.sh +++ b/database/install.sh @@ -2,8 +2,6 @@ set -e -clear - echo echo "Installing Database" echo "= = =" diff --git a/database/uninstall.sh b/database/uninstall.sh index f099b55..ec1f389 100755 --- a/database/uninstall.sh +++ b/database/uninstall.sh @@ -2,8 +2,6 @@ set -e -clear - echo echo "Uninstalling Database" echo "= = =" diff --git a/lib/event_source/postgres/database.rb b/lib/event_source/postgres/database.rb deleted file mode 100644 index 29de5fe..0000000 --- a/lib/event_source/postgres/database.rb +++ /dev/null @@ -1,12 +0,0 @@ -require 'event_source/postgres/database/settings' -require 'event_source/postgres/database/session' -require 'event_source/postgres/database/delete' -require 'event_source/postgres/database/create' -require 'event_source/postgres/database/user' -require 'event_source/postgres/database/database' -require 'event_source/postgres/database/extensions' -require 'event_source/postgres/database/table' -require 'event_source/postgres/database/indexes' -require 'event_source/postgres/database/functions' -require 'event_source/postgres/database/sql_code' -require 'event_source/postgres/database/recreate' diff --git a/lib/event_source/postgres/database/create.rb b/lib/event_source/postgres/database/create.rb deleted file mode 100644 index d4445e6..0000000 --- a/lib/event_source/postgres/database/create.rb +++ /dev/null @@ -1,22 +0,0 @@ -module EventSource - module Postgres - module Database - module Create - def self.call - warn "Creating database" - - session = Session.build - - User::Create.(session) - Database::Create.(session) - Extensions::Create.(session) - Table::Create.(session) - Functions::Create.(session) - Indexes::Create.(session) - - warn "Done creating database" - end - end - end - end -end diff --git a/lib/event_source/postgres/database/database.rb b/lib/event_source/postgres/database/database.rb deleted file mode 100644 index b3b3bff..0000000 --- a/lib/event_source/postgres/database/database.rb +++ /dev/null @@ -1,26 +0,0 @@ -module EventSource - module Postgres - module Database - module Database - module Create - def self.call(session) - database = Settings.database - - begin - session.execute("CREATE DATABASE #{database}") - rescue PG::DuplicateDatabase - warn "NOTICE: database \"#{database}\" already exists, skipping" - end - end - end - - module Delete - def self.call(session) - database = Settings.database - session.execute("DROP DATABASE IF EXISTS #{database}") - end - end - end - end - end -end diff --git a/lib/event_source/postgres/database/delete.rb b/lib/event_source/postgres/database/delete.rb deleted file mode 100644 index 8883547..0000000 --- a/lib/event_source/postgres/database/delete.rb +++ /dev/null @@ -1,18 +0,0 @@ -module EventSource - module Postgres - module Database - module Delete - def self.call - warn "Deleting database" - - session = Session.build - - User::Delete.(session) - Database::Delete.(session) - - warn "Done deleting database" - end - end - end - end -end diff --git a/lib/event_source/postgres/database/extensions.rb b/lib/event_source/postgres/database/extensions.rb deleted file mode 100644 index ecbfeeb..0000000 --- a/lib/event_source/postgres/database/extensions.rb +++ /dev/null @@ -1,19 +0,0 @@ -module EventSource - module Postgres - module Database - module Extensions - module Create - def self.sql_filename - 'extensions.sql' - end - - def self.call(session) - sql_code = SQLCode.read sql_filename - - session.execute(sql_code) - end - end - end - end - end -end diff --git a/lib/event_source/postgres/database/functions.rb b/lib/event_source/postgres/database/functions.rb deleted file mode 100644 index 8c45d8d..0000000 --- a/lib/event_source/postgres/database/functions.rb +++ /dev/null @@ -1,57 +0,0 @@ -module EventSource - module Postgres - module Database - module Functions - module Create - def self.call(session) - Category::Create.(session) - StreamVersion::Create.(session) - WriteEvent::Create.(session) - end - - module Category - module Create - def self.sql_filename - 'functions/category.sql' - end - - def self.call(session) - sql_code = SQLCode.read sql_filename - - session.execute(sql_code) - end - end - end - - module StreamVersion - module Create - def self.sql_filename - 'functions/stream-version.sql' - end - - def self.call(session) - sql_code = SQLCode.read sql_filename - - session.execute(sql_code) - end - end - end - - module WriteEvent - module Create - def self.sql_filename - 'functions/write-event.sql' - end - - def self.call(session) - sql_code = SQLCode.read sql_filename - - session.execute(sql_code) - end - end - end - end - end - end - end -end diff --git a/lib/event_source/postgres/database/indexes.rb b/lib/event_source/postgres/database/indexes.rb deleted file mode 100644 index 9374806..0000000 --- a/lib/event_source/postgres/database/indexes.rb +++ /dev/null @@ -1,87 +0,0 @@ -module EventSource - module Postgres - module Database - module Indexes - module Create - def self.call(session) - Category::GlobalPosition::Create.(session) - Category::Create.(session) - ID::Create.(session) - StreamName::PositionUniq::Create.(session) - StreamName::Create.(session) - end - - module Category - module Create - def self.sql_filename - 'indexes/events-category.sql' - end - - def self.call(session) - sql_code = SQLCode.read sql_filename - - session.execute(sql_code) - end - end - - module GlobalPosition - module Create - def self.sql_filename - 'indexes/events-category-global-position.sql' - end - - def self.call(session) - sql_code = SQLCode.read sql_filename - - session.execute(sql_code) - end - end - end - end - - module ID - module Create - def self.sql_filename - 'indexes/events-id.sql' - end - - def self.call(session) - sql_code = SQLCode.read sql_filename - - session.execute(sql_code) - end - end - end - - module StreamName - module Create - def self.sql_filename - 'indexes/events-stream-name.sql' - end - - def self.call(session) - sql_code = SQLCode.read sql_filename - - session.execute(sql_code) - end - end - - module PositionUniq - module Create - def self.sql_filename - 'indexes/events-stream-name-position-uniq.sql' - end - - def self.call(session) - sql_code = SQLCode.read sql_filename - - session.execute(sql_code) - end - end - end - end - end - end - end - end -end diff --git a/lib/event_source/postgres/database/recreate.rb b/lib/event_source/postgres/database/recreate.rb deleted file mode 100644 index d9c0b86..0000000 --- a/lib/event_source/postgres/database/recreate.rb +++ /dev/null @@ -1,12 +0,0 @@ -module EventSource - module Postgres - module Database - module Recreate - def self.call - Delete.() - Create.() - end - end - end - end -end diff --git a/lib/event_source/postgres/database/session.rb b/lib/event_source/postgres/database/session.rb deleted file mode 100644 index b65197a..0000000 --- a/lib/event_source/postgres/database/session.rb +++ /dev/null @@ -1,14 +0,0 @@ -module EventSource - module Postgres - module Database - class Session < Postgres::Session - def self.build - instance = new - settings = Postgres::Settings.build({}) - settings.set(instance) - instance - end - end - end - end -end diff --git a/lib/event_source/postgres/database/settings.rb b/lib/event_source/postgres/database/settings.rb deleted file mode 100644 index bc04625..0000000 --- a/lib/event_source/postgres/database/settings.rb +++ /dev/null @@ -1,19 +0,0 @@ -module EventSource - module Postgres - module Database - module Settings - def self.user - ENV['DATABASE_USER'] || 'event_source' - end - - def self.password - ENV['DATABASE_PASSWORD'] - end - - def self.database - ENV['DATABASE_NAME'] || 'event_source' - end - end - end - end -end diff --git a/lib/event_source/postgres/database/sql_code.rb b/lib/event_source/postgres/database/sql_code.rb deleted file mode 100644 index 3173ee7..0000000 --- a/lib/event_source/postgres/database/sql_code.rb +++ /dev/null @@ -1,17 +0,0 @@ -module EventSource - module Postgres - module Database - module SQLCode - def self.read(filename) - path = File.join root, filename - - File.read path - end - - def self.root - File.expand_path '../../../../database', __dir__ - end - end - end - end -end diff --git a/lib/event_source/postgres/database/table.rb b/lib/event_source/postgres/database/table.rb deleted file mode 100644 index be4b62f..0000000 --- a/lib/event_source/postgres/database/table.rb +++ /dev/null @@ -1,19 +0,0 @@ -module EventSource - module Postgres - module Database - module Table - module Create - def self.sql_filename - 'table/events-table.sql' - end - - def self.call(session) - sql_code = SQLCode.read sql_filename - - session.execute(sql_code) - end - end - end - end - end -end diff --git a/lib/event_source/postgres/database/user.rb b/lib/event_source/postgres/database/user.rb deleted file mode 100644 index 55cc63e..0000000 --- a/lib/event_source/postgres/database/user.rb +++ /dev/null @@ -1,30 +0,0 @@ -module EventSource - module Postgres - module Database - module User - module Create - def self.call(session) - user = Settings.user - password = Settings.password - - create_user_statement = "CREATE USER #{user}" - create_user_statement << " PASSWORD #{password}" unless password.nil? - - begin - session.execute(create_user_statement) - rescue PG::DuplicateObject - warn "NOTICE: role \"#{user}\" already exists, skipping" - end - end - end - - module Delete - def self.call(session) - user = Settings.user - session.execute("DROP USER IF EXISTS #{user}") - end - end - end - end - end -end diff --git a/scripts/evt-pg-create-db b/scripts/evt-pg-create-db index 065197c..970a497 100755 --- a/scripts/evt-pg-create-db +++ b/scripts/evt-pg-create-db @@ -1,5 +1,7 @@ #!/usr/bin/env ruby -require_relative './scripts_init' +root = File.expand_path '../database', __dir__ +script_filename = 'install.sh' +script_filepath = File.join root, script_filename -EventSource::Postgres::Database::Create.() +system script_filepath diff --git a/scripts/evt-pg-delete-db b/scripts/evt-pg-delete-db index 595edd4..1177be8 100755 --- a/scripts/evt-pg-delete-db +++ b/scripts/evt-pg-delete-db @@ -1,5 +1,7 @@ #!/usr/bin/env ruby -require_relative './scripts_init' +root = File.expand_path '../database', __dir__ +script_filename = 'uninstall.sh' +script_filepath = File.join root, script_filename -EventSource::Postgres::Database::Delete.() +system script_filepath diff --git a/scripts/evt-pg-recreate-db b/scripts/evt-pg-recreate-db index b2ab797..e276fa7 100755 --- a/scripts/evt-pg-recreate-db +++ b/scripts/evt-pg-recreate-db @@ -1,5 +1,11 @@ #!/usr/bin/env ruby -require_relative './scripts_init' +root = File.expand_path '../database', __dir__ -EventSource::Postgres::Database::Recreate.() +script_filename = 'uninstall.sh' +script_filepath = File.join root, script_filename +system script_filepath + +script_filename = 'install.sh' +script_filepath = File.join root, script_filename +system script_filepath diff --git a/test/database/create.rb b/test/database/create.rb deleted file mode 100644 index 685fdc7..0000000 --- a/test/database/create.rb +++ /dev/null @@ -1,3 +0,0 @@ -require_relative 'database_init' - -Database::Create.() diff --git a/test/database/database_init.rb b/test/database/database_init.rb deleted file mode 100644 index e2a18f8..0000000 --- a/test/database/database_init.rb +++ /dev/null @@ -1,3 +0,0 @@ -require_relative '../test_init' - -require 'event_source/postgres/database' diff --git a/test/database/delete.rb b/test/database/delete.rb deleted file mode 100644 index 838b9a6..0000000 --- a/test/database/delete.rb +++ /dev/null @@ -1,3 +0,0 @@ -require_relative 'database_init' - -Database::Delete.() diff --git a/test/database/reset.rb b/test/database/reset.rb deleted file mode 100644 index d59182d..0000000 --- a/test/database/reset.rb +++ /dev/null @@ -1,3 +0,0 @@ -require_relative 'database_init' - -Database::Reset.() From 6e8b34d2de524781d2ad7528bff9f84ac76f2f6d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Apr 2017 15:48:20 -0500 Subject: [PATCH 210/591] Package version is increased from 0.16.1.0 to 0.17.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index cdc5998..093e93d 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.16.1.0' + s.version = '0.17.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 383d49a0a3e2979df6d2c4757553f8571a506f55 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Apr 2017 17:24:41 -0500 Subject: [PATCH 211/591] Find location of scripts to make sure they are executed in the proper context --- database/install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/database/install.sh b/database/install.sh index 48e230a..2beeb11 100755 --- a/database/install.sh +++ b/database/install.sh @@ -58,7 +58,9 @@ function create-database { } function create-extensions { + script_dir=dirname $0 psql $database -f database/extensions.sql + # psql $database -f $script_dir/extensions.sql } function create-table { From ef818520fd7a8260a59a245c61bb87460fffc1dc Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Apr 2017 17:35:58 -0500 Subject: [PATCH 212/591] Database scripts execute relative to the install script --- database/install.sh | 31 +++++++++++++++++++------------ event_source-postgres.gemspec | 2 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/database/install.sh b/database/install.sh index 2beeb11..4838d1e 100755 --- a/database/install.sh +++ b/database/install.sh @@ -57,28 +57,35 @@ function create-database { echo } +function script_dir { + val="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + echo "$val" +} + function create-extensions { - script_dir=dirname $0 - psql $database -f database/extensions.sql - # psql $database -f $script_dir/extensions.sql + base=$(script_dir) + psql $database -f $base/extensions.sql } function create-table { - psql $database -f database/table/events-table.sql + base=$(script_dir) + psql $database -f $base/table/events-table.sql } function create-functions { - psql $database -f database/functions/category.sql - psql $database -f database/functions/stream-version.sql - psql $database -f database/functions/write-event.sql + base=$(script_dir) + psql $database -f $base/functions/category.sql + psql $database -f $base/functions/stream-version.sql + psql $database -f $base/functions/write-event.sql } function create-indexes { - psql $database -f database/indexes/events-id.sql - psql $database -f database/indexes/events-category-global-position.sql - psql $database -f database/indexes/events-category.sql - psql $database -f database/indexes/events-stream-name.sql - psql $database -f database/indexes/events-stream-name-position-uniq.sql + base=$(script_dir) + psql $database -f $base/indexes/events-id.sql + psql $database -f $base/indexes/events-category-global-position.sql + psql $database -f $base/indexes/events-category.sql + psql $database -f $base/indexes/events-stream-name.sql + psql $database -f $base/indexes/events-stream-name-position-uniq.sql } create-user diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 093e93d..3f21107 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.17.0.0' + s.version = '0.17.0.1' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 548f366f97b904931fe60e8ce457b983089613c8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 1 Apr 2017 17:36:42 -0500 Subject: [PATCH 213/591] Package version is increased from 0.17.0.1 to 0.17.0.2 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 3f21107..6be9c3f 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.17.0.1' + s.version = '0.17.0.2' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From e7f3a2b682725bd20f2bd2ccfd25364d7e2cd848 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 3 Apr 2017 14:31:28 -0500 Subject: [PATCH 214/591] Database operator scripts don't clear the terminal screen --- database/clear-events-table.sh | 2 -- database/list-events.sh | 2 -- 2 files changed, 4 deletions(-) diff --git a/database/clear-events-table.sh b/database/clear-events-table.sh index de683ac..03a6bfa 100755 --- a/database/clear-events-table.sh +++ b/database/clear-events-table.sh @@ -2,8 +2,6 @@ set -e -clear - echo echo "Clearing Events Table" echo "= = =" diff --git a/database/list-events.sh b/database/list-events.sh index 74d7920..80cbdaf 100755 --- a/database/list-events.sh +++ b/database/list-events.sh @@ -2,8 +2,6 @@ set -e -clear - echo echo "Listing Events" echo "= = =" From ae7cdc5ee5b6a2899bcfc383cc748b77182912dd Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 3 Apr 2017 14:47:07 -0500 Subject: [PATCH 215/591] The list events commands is one of the package's installed executables --- scripts/evt-pg-list-events | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 scripts/evt-pg-list-events diff --git a/scripts/evt-pg-list-events b/scripts/evt-pg-list-events new file mode 100755 index 0000000..21e3f07 --- /dev/null +++ b/scripts/evt-pg-list-events @@ -0,0 +1,7 @@ +#!/usr/bin/env ruby + +root = File.expand_path '../database', __dir__ +script_filename = 'list-events.sh' +script_filepath = File.join root, script_filename + +system script_filepath From 253fa9212e44910bfb8682d75321240c67eae5c9 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 3 Apr 2017 14:48:36 -0500 Subject: [PATCH 216/591] Package version is increased from 0.17.0.2 to 0.17.1.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 6be9c3f..6b1148e 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.17.0.2' + s.version = '0.17.1.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From b2654f3098e37bf04ebab1f0e4872fe006da23d4 Mon Sep 17 00:00:00 2001 From: ntl Date: Wed, 5 Apr 2017 16:29:34 -0500 Subject: [PATCH 217/591] list-events.sh accepts STREAM_NAME environment variable --- database/list-events.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/database/list-events.sh b/database/list-events.sh index 80cbdaf..08f2475 100755 --- a/database/list-events.sh +++ b/database/list-events.sh @@ -35,6 +35,18 @@ else fi echo "Table name is: $table" +if [ -z ${STREAM_NAME+x} ]; then + echo "(STREAM_NAME is not set)" + stream_name='' +else + stream_name=$STREAM_NAME + echo "Stream name is: $STREAM_NAME" +fi + echo -psql $database -c "SELECT * FROM $table" +if [ -z $stream_name ]; then + psql $database -c "SELECT * FROM $table" +else + psql $database -c "SELECT * FROM $table WHERE stream_name = '$stream_name'" +fi From ad7ec1b754dfe77e1b13235a82d4c30b5c01706d Mon Sep 17 00:00:00 2001 From: ntl Date: Wed, 5 Apr 2017 16:29:45 -0500 Subject: [PATCH 218/591] Package version is increased from 0.17.1.0 to 0.17.2.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 6b1148e..7f6c035 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.17.1.0' + s.version = '0.17.2.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 3c9b0c19b3f2787ee991feac935f7d87d8036389 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 10 Apr 2017 10:07:12 -0500 Subject: [PATCH 219/591] Log messages recording the actuation of the writer are tagged with 'write' --- lib/event_source/postgres/write.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index 6a8b10b..8294ec3 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -10,7 +10,7 @@ def configure(session: nil) end def write(batch, stream_name, expected_version: nil) - logger.trace { "Writing batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } + logger.trace(tag: :write) { "Writing batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } unless expected_version.nil? expected_version = ExpectedVersion.canonize(expected_version) @@ -27,18 +27,18 @@ def write(batch, stream_name, expected_version: nil) end end - logger.debug { "Wrote batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } + logger.debug(tag: :write) { "Wrote batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } last_position end def write_event_data(event_data, stream_name, expected_version: nil) - logger.trace { "Writing event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :event_data]) { event_data.pretty_inspect } + logger.trace(tag: :write) { "Writing event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } + logger.trace(tags: [:data, :event_data, :write]) { event_data.pretty_inspect } put.(event_data, stream_name, expected_version: expected_version).tap do - logger.debug { "Wrote event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } - logger.debug(tags: [:data, :event_data]) { event_data.pretty_inspect } + logger.debug(tag: :write) { "Wrote event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } + logger.debug(tags: [:data, :event_data, :write]) { event_data.pretty_inspect } end end end From ddc2893099a94998cae81b491216af6be0184dab Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 10 Apr 2017 10:08:02 -0500 Subject: [PATCH 220/591] Package version is increased from 0.17.2.0 to 0.17.2.1 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 7f6c035..70fb370 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.17.2.0' + s.version = '0.17.2.1' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 17f3c64ff7f5ae34f0b95f95fffff7b1c2a74e14 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 10 Apr 2017 10:13:06 -0500 Subject: [PATCH 221/591] Vestiges of the integration of cycle and reader are removed --- event_source-postgres.gemspec | 1 - lib/event_source/postgres.rb | 1 - test/interactive/reader_does_not_terminate.rb | 5 ----- 3 files changed, 7 deletions(-) delete mode 100644 test/interactive/reader_does_not_terminate.rb diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 70fb370..3181211 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -20,7 +20,6 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'evt-event_source' s.add_runtime_dependency 'evt-log' - s.add_runtime_dependency 'evt-cycle' s.add_runtime_dependency 'evt-settings' s.add_runtime_dependency 'pg' diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index ea5b77c..430b56f 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -1,7 +1,6 @@ require 'pg' require 'event_source' -require 'cycle' require 'log' require 'telemetry' diff --git a/test/interactive/reader_does_not_terminate.rb b/test/interactive/reader_does_not_terminate.rb deleted file mode 100644 index 81bab98..0000000 --- a/test/interactive/reader_does_not_terminate.rb +++ /dev/null @@ -1,5 +0,0 @@ -require_relative '../test_init' - -stream_name = Controls::StreamName.example - -Read.(stream_name, batch_size: 1, cycle_maximum_milliseconds: 200) { |event_data| } From 7a60c437ea3e112cc2aeac5c83adadd606b5b19f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 10 Apr 2017 10:15:20 -0500 Subject: [PATCH 222/591] Package version is increased from 0.17.2.1 to 0.18.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 3181211..d063c40 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.17.2.1' + s.version = '0.18.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 07a850aa0719800d8245dc275b474a69c9d99e1a Mon Sep 17 00:00:00 2001 From: ntl Date: Wed, 19 Apr 2017 15:33:56 -0500 Subject: [PATCH 223/591] System database is used when deleting database --- database/uninstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/uninstall.sh b/database/uninstall.sh index ec1f389..63e7ea9 100755 --- a/database/uninstall.sh +++ b/database/uninstall.sh @@ -43,7 +43,7 @@ function delete-user { function delete-database { echo "Database name is: $database" - database_exists=`psql -tAc "SELECT 1 FROM pg_database WHERE datname='$database'"` + database_exists=`psql postgres -tAc "SELECT 1 FROM pg_database WHERE datname='$database'"` if [ "$database_exists" = "1" ]; then echo "Deleting database \"$database\"" From bdd4967f6692ce76cd76cedc2092a5238d64002b Mon Sep 17 00:00:00 2001 From: ntl Date: Wed, 19 Apr 2017 16:09:40 -0500 Subject: [PATCH 224/591] Package version is increased from 0.18.0.0 to 0.18.0.1 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index d063c40..be23ef7 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.18.0.0' + s.version = '0.18.0.1' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 6c6ecdf3ae7d76b1a627ec0ad0132c0e1eb090db Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 20 Apr 2017 15:25:54 -0500 Subject: [PATCH 225/591] Database install doesn't check for pre-existence of database before creating it --- database/install.sh | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/database/install.sh b/database/install.sh index 4838d1e..a0f0a69 100755 --- a/database/install.sh +++ b/database/install.sh @@ -2,6 +2,12 @@ set -e +# function foo { +# createdb event_source +# } + +# foo + echo echo "Installing Database" echo "= = =" @@ -43,17 +49,7 @@ function create-user { function create-database { echo "Database name is: $database" - - database_exists=`psql postgres -tAc "SELECT 1 FROM pg_database WHERE datname='$database'"` - - if [ "$database_exists" = "1" ]; then - echo "Database \"$database\" was previously created. Not creating again." - else - echo "Database \"$database\" has not yet been created" - echo "Creating database \"$database\"" - createdb $database || true - fi - + createdb $database echo } From 935ebb6cac54a9dbd97d37629e960d11fb4d7adc Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 20 Apr 2017 15:38:41 -0500 Subject: [PATCH 226/591] Database scripts offer a bit more visibility --- database/install.sh | 29 ++++++++++++++++++++--------- database/table/events-table.sql | 1 - database/uninstall.sh | 8 ++++---- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/database/install.sh b/database/install.sh index a0f0a69..080486f 100755 --- a/database/install.sh +++ b/database/install.sh @@ -2,12 +2,6 @@ set -e -# function foo { -# createdb event_source -# } - -# foo - echo echo "Installing Database" echo "= = =" @@ -16,14 +10,14 @@ echo default_name=event_source if [ -z ${DATABASE_USER+x} ]; then - echo "(DATABASE_USER is not set)" + echo "(DATABASE_USER is not set. Default will be used.)" user=$default_name else user=$DATABASE_USER fi if [ -z ${DATABASE_NAME+x} ]; then - echo "(DATABASE_NAME is not set)" + echo "(DATABASE_NAME is not set. Default will be used.)" database=$default_name else database=$DATABASE_NAME @@ -40,7 +34,7 @@ function create-user { echo "Database user \"$user\" was previously created. Not creating again." else echo "Database user \"$user\" has not yet been created" - echo "Creating database user \"$user\"" + echo "Creating database user \"$user\"..." createuser -s $user fi @@ -49,6 +43,7 @@ function create-user { function create-database { echo "Database name is: $database" + echo "Creating database \"$database\"..." createdb $database echo } @@ -59,29 +54,45 @@ function script_dir { } function create-extensions { + echo "Creating extensions..." base=$(script_dir) psql $database -f $base/extensions.sql + echo } function create-table { + echo "Creating events table..." base=$(script_dir) psql $database -f $base/table/events-table.sql + echo } function create-functions { base=$(script_dir) + echo "Creating functions..." + echo "category function" psql $database -f $base/functions/category.sql + echo "stream_version function" psql $database -f $base/functions/stream-version.sql + echo "write_sql function" psql $database -f $base/functions/write-event.sql + echo } function create-indexes { base=$(script_dir) + echo "Creating indexes..." + echo "events_id_idx" psql $database -f $base/indexes/events-id.sql + echo "events_category_global_position_idx" psql $database -f $base/indexes/events-category-global-position.sql + echo "events_category_idx" psql $database -f $base/indexes/events-category.sql + echo "events_stream_name_idx" psql $database -f $base/indexes/events-stream-name.sql + echo "events_stream_name_position_uniq_idx" psql $database -f $base/indexes/events-stream-name-position-uniq.sql + echo } create-user diff --git a/database/table/events-table.sql b/database/table/events-table.sql index 6978d47..93a323c 100644 --- a/database/table/events-table.sql +++ b/database/table/events-table.sql @@ -1,7 +1,6 @@ -- ---------------------------- -- Table structure for events -- ---------------------------- -DROP TABLE IF EXISTS "public"."events"; CREATE TABLE "public"."events" ( "id" UUID NOT NULL DEFAULT gen_random_uuid(), "stream_name" varchar(255) NOT NULL COLLATE "default", diff --git a/database/uninstall.sh b/database/uninstall.sh index ec1f389..dc6d94b 100755 --- a/database/uninstall.sh +++ b/database/uninstall.sh @@ -10,14 +10,14 @@ echo default_name=event_source if [ -z ${DATABASE_USER+x} ]; then - echo "(DATABASE_USER is not set)" + echo "(DATABASE_USER is not set. Default will be used.)" user=$default_name else user=$DATABASE_USER fi if [ -z ${DATABASE_NAME+x} ]; then - echo "(DATABASE_NAME is not set)" + echo "(DATABASE_NAME is not set. Default will be used.)" database=$default_name else database=$DATABASE_NAME @@ -31,7 +31,7 @@ function delete-user { user_exists=`psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` if [ "$user_exists" = "1" ]; then - echo "Deleting database user \"$user\"" + echo "Deleting database user \"$user\"..." dropuser $user else echo "Database user \"$user\" does not exist. Not deleting." @@ -46,7 +46,7 @@ function delete-database { database_exists=`psql -tAc "SELECT 1 FROM pg_database WHERE datname='$database'"` if [ "$database_exists" = "1" ]; then - echo "Deleting database \"$database\"" + echo "Deleting database \"$database\"..." dropdb $database else echo "Database \"$database\" does not exist. Not deleting." From 0749c97333dcaf95109cb6c66f426e5c838d1bd4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 20 Apr 2017 15:39:36 -0500 Subject: [PATCH 227/591] Package version is increased from 0.18.0.0 to 0.19.0.0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index d063c40..74d6a57 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.18.0.0' + s.version = '0.19.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 982501c5bdff66e7cf2909ac574f8357c93b8c18 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 20 Apr 2017 18:23:47 -0500 Subject: [PATCH 228/591] Type column is listed before position columns --- database/table/events-table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/table/events-table.sql b/database/table/events-table.sql index 93a323c..5192f33 100644 --- a/database/table/events-table.sql +++ b/database/table/events-table.sql @@ -4,8 +4,8 @@ CREATE TABLE "public"."events" ( "id" UUID NOT NULL DEFAULT gen_random_uuid(), "stream_name" varchar(255) NOT NULL COLLATE "default", - "position" int4 NOT NULL, "type" varchar(255) NOT NULL COLLATE "default", + "position" int4 NOT NULL, "global_position" bigserial NOT NULL , "data" jsonb, "metadata" jsonb, From 36ebb29f8dece0f02b9725de5e7e41a584d04848 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 20 Apr 2017 18:24:25 -0500 Subject: [PATCH 229/591] Package version is increased from 0.19.0.0 to 0.19.0.1 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 74d6a57..25210fa 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.19.0.0' + s.version = '0.19.0.1' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From c7917d38e483c0452cf299976b0fbc8c8b29e69d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 20 Apr 2017 19:16:59 -0500 Subject: [PATCH 230/591] Package version is increased from 0.19.0.1 to 0.19.0.2 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 9d17630..2cdaf7e 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,6 +1,6 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| - s.version = '0.19.0.1' + s.version = '0.19.0.2' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From c4b03a33c8743fbfba24241bd76308982734c795 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 20 Apr 2017 19:17:44 -0500 Subject: [PATCH 231/591] Gem name is restored --- event_source-postgres.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 2cdaf7e..ca8c467 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,5 +1,6 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| + s.name = 'evt-event_source-postgres' s.version = '0.19.0.2' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 5ef8786516b0e3a0f8ad034d43951f046ef65efc Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 5 May 2017 18:11:44 -0500 Subject: [PATCH 232/591] Log message for select statement no longer raises undefined method error --- lib/event_source/postgres/get/select_statement.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index 50cfbc3..ba82318 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -14,8 +14,8 @@ def batch_size @batch_size ||= Defaults.batch_size end - def stream_type - @stream_type ||= StreamName.get_type(stream_name) + def stream_type_list + @stream_type ||= StreamName.get_type_list(stream_name) end def category_stream? @@ -27,7 +27,7 @@ def self.build(stream_name, position: nil, batch_size: nil) end def sql - logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{category_stream?}, Type: #{stream_type.inspect}, Position: #{position}, Batch Size: #{batch_size})" } + logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{category_stream?}, Types: #{stream_type_list.inspect}, Position: #{position}, Batch Size: #{batch_size})" } statement = <<-SQL SELECT @@ -51,7 +51,7 @@ def sql ; SQL - logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{category_stream?}, Type: #{stream_type.inspect}, Position: #{position}, Batch Size: #{batch_size})" } + logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{category_stream?}, Type: #{stream_type_list.inspect}, Position: #{position}, Batch Size: #{batch_size})" } logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } statement From 6bcb5729bd09a061fc5671b40ba02f5810bc54f4 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 5 May 2017 18:32:23 -0500 Subject: [PATCH 233/591] Variable label typo is fixed --- lib/event_source/postgres/get/select_statement.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index ba82318..ea46e39 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -51,7 +51,7 @@ def sql ; SQL - logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{category_stream?}, Type: #{stream_type_list.inspect}, Position: #{position}, Batch Size: #{batch_size})" } + logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{category_stream?}, Types: #{stream_type_list.inspect}, Position: #{position}, Batch Size: #{batch_size})" } logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } statement From 5dfca5c712edf646ff233fbb24c65ea647ee8a90 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 5 May 2017 18:54:07 -0500 Subject: [PATCH 234/591] Package version is increased from 0.19.0.2 to 0.19.0.3 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index ca8c467..94551ae 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.19.0.2' + s.version = '0.19.0.3' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 1bed549ba5b148487b115ac5ed695f0844e3a695 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 23 May 2017 22:06:00 -0500 Subject: [PATCH 235/591] Gem installation is up to date --- event_source-postgres.gemspec | 1 + install-gems.sh => install_gems.sh | 28 ++----------------- lib/event_source/postgres.rb | 2 +- ...local-gem-path.sh => set_local_gem_path.sh | 2 ++ 4 files changed, 6 insertions(+), 27 deletions(-) rename install-gems.sh => install_gems.sh (52%) rename set-local-gem-path.sh => set_local_gem_path.sh (93%) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 94551ae..394f189 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -19,6 +19,7 @@ Gem::Specification.new do |s| s.bindir = 'scripts' s.add_runtime_dependency 'evt-event_source' + # s.add_runtime_dependency 'evt-message_store' s.add_runtime_dependency 'evt-log' s.add_runtime_dependency 'evt-settings' diff --git a/install-gems.sh b/install_gems.sh similarity index 52% rename from install-gems.sh rename to install_gems.sh index 2ea4b74..8cd76dc 100755 --- a/install-gems.sh +++ b/install_gems.sh @@ -2,16 +2,11 @@ set -e -if [ -z ${GEM_AUTHORITY_PATH+x} ]; then - echo "GEM_AUTHORITY_PATH is not set" - exit -fi - echo echo 'Installing local gems' echo '= = =' -source ./set-local-gem-path.sh +source ./set_local_gem_path.sh echo echo 'Removing gem files' @@ -40,31 +35,12 @@ else posture=$POSTURE fi -scheme="https:" -gem_repo_authority_path=$GEM_AUTHORITY_PATH -public_gem_repo_uri="$scheme//$gem_repo_authority_path" - -gemfury_token="" -if [ ! -z ${GEMFURY_TOKEN+x} ]; then - gemfury_token=$GEMFURY_TOKEN -fi - -private_source="" -if [ ! $gemfury_token = "" ]; then - private_gem_repo_uri="$scheme//$gemfury_token@$gem_repo_authority_path" - private_source="--source $private_gem_repo_uri" -fi - -public_source="--source $public_gem_repo_uri" - -ruby_gems_source="--source https://rubygems.org" - echo echo "Installing gems locally (posture: $posture)" echo '- - -' for gem in *.gem; do echo "($gem)" - cmd="gem install $gem --clear-sources $private_source $public_source $ruby_gems_source --install-dir ./gems" + cmd="gem install $gem --install-dir ./gems" if [ operational != "$posture" ]; then cmd="$cmd --development" diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb index 430b56f..07d12c3 100644 --- a/lib/event_source/postgres.rb +++ b/lib/event_source/postgres.rb @@ -1,6 +1,6 @@ require 'pg' -require 'event_source' +require 'message_store' require 'log' require 'telemetry' diff --git a/set-local-gem-path.sh b/set_local_gem_path.sh similarity index 93% rename from set-local-gem-path.sh rename to set_local_gem_path.sh index f19a99e..c0b633e 100755 --- a/set-local-gem-path.sh +++ b/set_local_gem_path.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + unchanged_gem_path=$GEM_PATH if [[ ! $GEM_PATH == *"./gems"* ]]; then From 7bbf192de3a21d636a215ccd4c2f6ceed04121c5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 23 May 2017 22:11:19 -0500 Subject: [PATCH 236/591] Package version set to 0 --- event_source-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_source-postgres.gemspec b/event_source-postgres.gemspec index 394f189..90cb7c2 100644 --- a/event_source-postgres.gemspec +++ b/event_source-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-event_source-postgres' - s.version = '0.19.0.3' + s.version = '0.0.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 1e4740cc577a2a3657510764af0242193448b708 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 24 May 2017 14:31:03 -0500 Subject: [PATCH 237/591] Superfluous test is removed from gem installation script --- install_gems.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/install_gems.sh b/install_gems.sh index 8cd76dc..6bceba1 100755 --- a/install_gems.sh +++ b/install_gems.sh @@ -11,14 +11,7 @@ source ./set_local_gem_path.sh echo echo 'Removing gem files' echo '- - -' -if test -n "$(find . -maxdepth 1 -name '*.gem' -print -quit)"; then - for gem in *.gem; do - echo "- $gem" - rm $gem - done -else - echo "(No gem files found)" -fi +! rm -v *.gem echo echo 'Building gems' From 2017e2544fcfa6d20dbf73ee6aefe2e82ced4227 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 24 May 2017 14:40:52 -0500 Subject: [PATCH 238/591] Gem installation compiles with updated standards --- install_gems.sh => install-gems.sh | 2 +- set_local_gem_path.sh => set-local-gem-path.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename install_gems.sh => install-gems.sh (95%) rename set_local_gem_path.sh => set-local-gem-path.sh (100%) diff --git a/install_gems.sh b/install-gems.sh similarity index 95% rename from install_gems.sh rename to install-gems.sh index 6bceba1..53973ad 100755 --- a/install_gems.sh +++ b/install-gems.sh @@ -6,7 +6,7 @@ echo echo 'Installing local gems' echo '= = =' -source ./set_local_gem_path.sh +source ./set-local-gem-path.sh echo echo 'Removing gem files' diff --git a/set_local_gem_path.sh b/set-local-gem-path.sh similarity index 100% rename from set_local_gem_path.sh rename to set-local-gem-path.sh From 2ef7436c2363a276c372962160056d19ecc8fc85 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 24 May 2017 23:09:31 -0500 Subject: [PATCH 239/591] Gem installation is brought up to current standards --- Gemfile | 3 ++ MIT-License.txt | 2 +- init.rb | 9 +---- install-gems.sh | 39 +++++-------------- ...source-postgres.gemspec => library.gemspec | 0 load_path.rb | 18 +++++++++ set-local-gem-path.sh | 16 -------- 7 files changed, 32 insertions(+), 55 deletions(-) create mode 100644 Gemfile rename event_source-postgres.gemspec => library.gemspec (100%) create mode 100644 load_path.rb delete mode 100755 set-local-gem-path.sh diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..fa75df1 --- /dev/null +++ b/Gemfile @@ -0,0 +1,3 @@ +source 'https://rubygems.org' + +gemspec diff --git a/MIT-License.txt b/MIT-License.txt index cd0e498..b75881d 100644 --- a/MIT-License.txt +++ b/MIT-License.txt @@ -1,4 +1,4 @@ -Copyright (c) 2016 Scott Bellware +Modified work Copyright (c) 2016-2017 Scott Bellware Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/init.rb b/init.rb index 5c945fe..4a13dbb 100644 --- a/init.rb +++ b/init.rb @@ -1,10 +1,3 @@ -lib_dir = File.expand_path('../lib', __FILE__) -$LOAD_PATH.unshift lib_dir unless $LOAD_PATH.include?(lib_dir) - -libraries_dir = ENV['LIBRARIES_HOME'] -unless libraries_dir.nil? - libraries_dir = File.expand_path(libraries_dir) - $LOAD_PATH.unshift libraries_dir unless $LOAD_PATH.include?(libraries_dir) -end +require_relative 'load_path' require 'event_source/postgres' diff --git a/install-gems.sh b/install-gems.sh index 53973ad..40cac53 100755 --- a/install-gems.sh +++ b/install-gems.sh @@ -2,25 +2,6 @@ set -e -echo -echo 'Installing local gems' -echo '= = =' - -source ./set-local-gem-path.sh - -echo -echo 'Removing gem files' -echo '- - -' -! rm -v *.gem - -echo -echo 'Building gems' -echo '- - -' -for gemspec in *.gemspec; do - echo "- $gemspec" - gem build $gemspec -done - if [ -z ${POSTURE+x} ]; then echo "(POSTURE is not set. Using \"operational\" by default.)" posture="operational" @@ -30,19 +11,17 @@ fi echo echo "Installing gems locally (posture: $posture)" -echo '- - -' -for gem in *.gem; do - echo "($gem)" - cmd="gem install $gem --install-dir ./gems" +echo '= = =' - if [ operational != "$posture" ]; then - cmd="$cmd --development" - fi +cmd="bundle install --standalone --path=./gems" - echo $cmd - ($cmd) || exit 1 -done +if [ operational == "$posture" ]; then + cmd="$cmd --without=development" +fi -echo '= = =' +echo $cmd +($cmd) + +echo '- - -' echo '(done)' echo diff --git a/event_source-postgres.gemspec b/library.gemspec similarity index 100% rename from event_source-postgres.gemspec rename to library.gemspec diff --git a/load_path.rb b/load_path.rb new file mode 100644 index 0000000..2257423 --- /dev/null +++ b/load_path.rb @@ -0,0 +1,18 @@ +bundler_standalone_loader = 'gems/bundler/setup' + +begin + require_relative bundler_standalone_loader +rescue LoadError + puts "WARNING: Standalone bundle loader is not at #{bundler_standalone_loader}. Using Bundler to load gems." + require "bundler/setup" + Bundler.require +end + +lib_dir = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift lib_dir unless $LOAD_PATH.include?(lib_dir) + +libraries_dir = ENV['LIBRARIES_HOME'] +unless libraries_dir.nil? + libraries_dir = File.expand_path(libraries_dir) + $LOAD_PATH.unshift libraries_dir unless $LOAD_PATH.include?(libraries_dir) +end diff --git a/set-local-gem-path.sh b/set-local-gem-path.sh deleted file mode 100755 index c0b633e..0000000 --- a/set-local-gem-path.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -unchanged_gem_path=$GEM_PATH - -if [[ ! $GEM_PATH == *"./gems"* ]]; then - export GEM_PATH=./gems:$GEM_PATH - - echo "Gem path was changed" - echo " from: $unchanged_gem_path" - echo " to: $GEM_PATH" -else - echo "Gem path was unchanged" - echo " from: $unchanged_gem_path" -fi - -echo From 126eab983c2fda49afed84003f068c3c20b7dee1 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 26 May 2017 15:32:43 -0500 Subject: [PATCH 240/591] Event, EventData are renamed to Message, MessageData --- lib/event_source/postgres/controls.rb | 4 +-- .../{event_data.rb => message_data.rb} | 2 +- lib/event_source/postgres/controls/put.rb | 4 +-- lib/event_source/postgres/get.rb | 24 ++++++++--------- lib/event_source/postgres/get/last.rb | 26 +++++++++---------- lib/event_source/postgres/put.rb | 20 +++++++------- lib/event_source/postgres/write.rb | 20 +++++++------- test/automated/get/batch_size.rb | 8 +++--- test/automated/get/category.rb | 8 +++--- test/automated/get/get.rb | 24 ++++++++--------- .../get/{no_events.rb => no_messages.rb} | 2 +- test/automated/get_last/get_last.rb | 10 +++---- test/automated/get_last/no_events.rb | 13 ---------- test/automated/get_last/no_messages.rb | 13 ++++++++++ test/automated/iterator/next.rb | 6 ++--- .../iterator/no_further_event_data.rb | 2 +- test/automated/put/category_as_stream_name.rb | 8 +++--- test/automated/put/data/empty.rb | 8 +++--- test/automated/put/data/nil.rb | 8 +++--- test/automated/put/expected_version/error.rb | 12 ++++----- .../put/expected_version/expected_version.rb | 16 ++++++------ test/automated/put/metadata/empty.rb | 10 +++---- test/automated/put/metadata/nil.rb | 8 +++--- test/automated/put/missing_id.rb | 16 ++++++------ .../put/no_stream/existing_stream.rb | 8 +++--- test/automated/put/no_stream/no_stream.rb | 6 ++--- test/automated/put/put.rb | 24 ++++++++--------- test/automated/put/returns_stream_position.rb | 4 +-- .../put/stream_position_increases.rb | 6 ++--- test/automated/read/position.rb | 4 +-- test/automated/read/read.rb | 6 ++--- .../write/batch/expected_version/error.rb | 18 ++++++------- .../expected_version/expected_version.rb | 20 +++++++------- test/automated/write/batch/write.rb | 16 ++++++------ .../expected_version/expected_version.rb | 23 ---------------- test/automated/write/event/write.rb | 17 ------------ .../expected_version/error.rb | 14 +++++----- .../expected_version/expected_version.rb | 23 ++++++++++++++++ test/automated/write/message/write.rb | 17 ++++++++++++ 39 files changed, 239 insertions(+), 239 deletions(-) rename lib/event_source/postgres/controls/{event_data.rb => message_data.rb} (57%) rename test/automated/get/{no_events.rb => no_messages.rb} (88%) delete mode 100644 test/automated/get_last/no_events.rb create mode 100644 test/automated/get_last/no_messages.rb delete mode 100644 test/automated/write/event/expected_version/expected_version.rb delete mode 100644 test/automated/write/event/write.rb rename test/automated/write/{event => message}/expected_version/error.rb (52%) create mode 100644 test/automated/write/message/expected_version/expected_version.rb create mode 100644 test/automated/write/message/write.rb diff --git a/lib/event_source/postgres/controls.rb b/lib/event_source/postgres/controls.rb index e1142d8..30fe10d 100644 --- a/lib/event_source/postgres/controls.rb +++ b/lib/event_source/postgres/controls.rb @@ -1,6 +1,6 @@ -require 'event_source/controls' +require 'message_store/controls' require 'event_source/postgres/controls/category' require 'event_source/postgres/controls/stream_name' -require 'event_source/postgres/controls/event_data' +require 'event_source/postgres/controls/message_data' require 'event_source/postgres/controls/put' diff --git a/lib/event_source/postgres/controls/event_data.rb b/lib/event_source/postgres/controls/message_data.rb similarity index 57% rename from lib/event_source/postgres/controls/event_data.rb rename to lib/event_source/postgres/controls/message_data.rb index de77de3..54c9b50 100644 --- a/lib/event_source/postgres/controls/event_data.rb +++ b/lib/event_source/postgres/controls/message_data.rb @@ -1,7 +1,7 @@ module EventSource module Postgres module Controls - EventData = EventSource::Controls::EventData + MessageData = MessageStore::Controls::MessageData end end end diff --git a/lib/event_source/postgres/controls/put.rb b/lib/event_source/postgres/controls/put.rb index dcc15e9..a63da2f 100644 --- a/lib/event_source/postgres/controls/put.rb +++ b/lib/event_source/postgres/controls/put.rb @@ -8,13 +8,13 @@ def self.call(instances: nil, stream_name: nil, event: nil, category: nil) event_specified = !event.nil? - event ||= EventData::Write.example + event ||= MessageData::Write.example instances.times do EventSource::Postgres::Put.(event, stream_name) unless event_specified - event.id = EventData::Write.id + event.id = MessageData::Write.id end end diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index 006ef6d..a878fda 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -29,16 +29,16 @@ def self.call(stream_name, position: nil, batch_size: nil, session: nil) end def call(stream_name, position: nil) - logger.trace { "Getting event data (Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } + logger.trace { "Getting message data (Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } records = get_records(stream_name, position) - events = convert(records) + messages = convert(records) - logger.info { "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } - logger.info(tags: [:data, :event_data]) { events.pretty_inspect } + logger.info { "Finished getting message data (Count: #{messages.length}, Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } + logger.info(tags: [:data, :message_data]) { messages.pretty_inspect } - events + messages end def get_records(stream_name, position) @@ -54,30 +54,30 @@ def get_records(stream_name, position) end def convert(records) - logger.trace { "Converting records to event data (Records Count: #{records.ntuples})" } + logger.trace { "Converting records to message data (Records Count: #{records.ntuples})" } - events = records.map do |record| + messages = records.map do |record| record['data'] = Deserialize.data(record['data']) record['metadata'] = Deserialize.metadata(record['metadata']) record['time'] = Time.utc_coerced(record['time']) - EventData::Read.build record + MessageData::Read.build record end - logger.debug { "Converted records to event data (Event Data Count: #{events.length})" } + logger.debug { "Converted records to message data (Message Data Count: #{messages.length})" } - events + messages end module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? - Transform::Read.(serialized_data, EventData::Hash, :json) + Transform::Read.(serialized_data, MessageData::Hash, :json) end def self.metadata(serialized_metadata) return nil if serialized_metadata.nil? - Transform::Read.(serialized_metadata, EventData::Hash, :json) + Transform::Read.(serialized_metadata, MessageData::Hash, :json) end end diff --git a/lib/event_source/postgres/get/last.rb b/lib/event_source/postgres/get/last.rb index dbd9a82..65c7560 100644 --- a/lib/event_source/postgres/get/last.rb +++ b/lib/event_source/postgres/get/last.rb @@ -28,7 +28,7 @@ def self.call(stream_name, session: nil) end def call(stream_name) - logger.trace { "Getting last event data (Stream Name: #{stream_name})" } + logger.trace { "Getting last message data (Stream Name: #{stream_name})" } record = get_record(stream_name) @@ -36,8 +36,8 @@ def call(stream_name) event = convert(record) - logger.info { "Finished getting event data (Stream Name: #{stream_name})" } - logger.info(tags: [:data, :event_data]) { event.pretty_inspect } + logger.info { "Finished getting message data (Stream Name: #{stream_name})" } + logger.info(tags: [:data, :message_data]) { event.pretty_inspect } event end @@ -57,46 +57,46 @@ def get_record(stream_name) end def convert(record) - logger.trace { "Converting record to event data" } + logger.trace { "Converting record to message data" } record['data'] = Deserialize.data(record['data']) record['metadata'] = Deserialize.metadata(record['metadata']) record['time'] = Time.utc_coerced(record['time']) - event = EventData::Read.build(record) + event = MessageData::Read.build(record) - logger.debug { "Converted record to event data" } + logger.debug { "Converted record to message data" } event end def __convert(records) - logger.trace { "Converting records to event data (Records Count: #{records.ntuples})" } + logger.trace { "Converting records to message data (Records Count: #{records.ntuples})" } - events = records.map do |record| + messages = records.map do |record| record['data'] = Deserialize.data(record['data']) record['metadata'] = Deserialize.metadata(record['metadata']) record['time'] = Time.utc_coerced(record['time']) - EventData::Read.build record + MessageData::Read.build record break end - logger.debug { "Converted records to event data (Event Data Count: #{events.length})" } + logger.debug { "Converted records to message data (Message Data Count: #{messages.length})" } - events + messages end module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? - Transform::Read.(serialized_data, EventData::Hash, :json) + Transform::Read.(serialized_data, MessageData::Hash, :json) end def self.metadata(serialized_metadata) return nil if serialized_metadata.nil? - Transform::Read.(serialized_metadata, EventData::Hash, :json) + Transform::Read.(serialized_metadata, MessageData::Hash, :json) end end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index c6510cd..1f0a88a 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -29,8 +29,8 @@ def self.call(write_event, stream_name, expected_version: nil, session: nil) end def call(write_event, stream_name, expected_version: nil) - logger.trace { "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :event_data]) { write_event.pretty_inspect } + logger.trace { "Putting message data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } + logger.trace(tags: [:data, :message_data]) { write_event.pretty_inspect } write_event.id ||= identifier.get @@ -38,8 +38,8 @@ def call(write_event, stream_name, expected_version: nil) expected_version = ExpectedVersion.canonize(expected_version) insert_event(id, stream_name, type, data, metadata, expected_version).tap do |position| - logger.info { "Put event data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } - logger.info(tags: [:data, :event_data]) { write_event.pretty_inspect } + logger.info { "Put message data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } + logger.info(tags: [:data, :message_data]) { write_event.pretty_inspect } end end @@ -49,10 +49,10 @@ def destructure_event(write_event) data = write_event.data metadata = write_event.metadata - logger.debug(tags: [:data, :event_data]) { "ID: #{id.pretty_inspect}" } - logger.debug(tags: [:data, :event_data]) { "Type: #{type.pretty_inspect}" } - logger.debug(tags: [:data, :event_data]) { "Data: #{data.pretty_inspect}" } - logger.debug(tags: [:data, :event_data]) { "Metadata: #{metadata.pretty_inspect}" } + logger.debug(tags: [:data, :message_data]) { "ID: #{id.pretty_inspect}" } + logger.debug(tags: [:data, :message_data]) { "Type: #{type.pretty_inspect}" } + logger.debug(tags: [:data, :message_data]) { "Data: #{data.pretty_inspect}" } + logger.debug(tags: [:data, :message_data]) { "Metadata: #{metadata.pretty_inspect}" } return id, type, data, metadata end @@ -99,7 +99,7 @@ def serialized_data(data) end unless data.nil? - serializable_data = EventData::Hash[data] + serializable_data = MessageData::Hash[data] serialized_data = Transform::Write.(serializable_data, :json) end @@ -115,7 +115,7 @@ def serialized_metadata(metadata) end unless metadata.nil? - serializable_metadata = EventData::Hash[metadata] + serializable_metadata = MessageData::Hash[metadata] serialized_metadata = Transform::Write.(serializable_metadata, :json) end diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index 8294ec3..bc4981f 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -10,7 +10,7 @@ def configure(session: nil) end def write(batch, stream_name, expected_version: nil) - logger.trace(tag: :write) { "Writing batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } + logger.trace(tag: :write) { "Writing batch (Stream Name: #{stream_name}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})" } unless expected_version.nil? expected_version = ExpectedVersion.canonize(expected_version) @@ -18,8 +18,8 @@ def write(batch, stream_name, expected_version: nil) last_position = nil put.session.transaction do - batch.each do |event_data| - last_position = write_event_data(event_data, stream_name, expected_version: expected_version) + batch.each do |message_data| + last_position = write_message_data(message_data, stream_name, expected_version: expected_version) unless expected_version.nil? expected_version += 1 @@ -27,18 +27,18 @@ def write(batch, stream_name, expected_version: nil) end end - logger.debug(tag: :write) { "Wrote batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } + logger.debug(tag: :write) { "Wrote batch (Stream Name: #{stream_name}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})" } last_position end - def write_event_data(event_data, stream_name, expected_version: nil) - logger.trace(tag: :write) { "Writing event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :event_data, :write]) { event_data.pretty_inspect } + def write_message_data(message_data, stream_name, expected_version: nil) + logger.trace(tag: :write) { "Writing message data (Stream Name: #{stream_name}, Type: #{message_data.type}, Expected Version: #{expected_version.inspect})" } + logger.trace(tags: [:data, :message_data, :write]) { message_data.pretty_inspect } - put.(event_data, stream_name, expected_version: expected_version).tap do - logger.debug(tag: :write) { "Wrote event data (Stream Name: #{stream_name}, Type: #{event_data.type}, Expected Version: #{expected_version.inspect})" } - logger.debug(tags: [:data, :event_data, :write]) { event_data.pretty_inspect } + put.(message_data, stream_name, expected_version: expected_version).tap do + logger.debug(tag: :write) { "Wrote message data (Stream Name: #{stream_name}, Type: #{message_data.type}, Expected Version: #{expected_version.inspect})" } + logger.debug(tags: [:data, :message_data, :write]) { message_data.pretty_inspect } end end end diff --git a/test/automated/get/batch_size.rb b/test/automated/get/batch_size.rb index 3ba5bd5..b44a6f7 100644 --- a/test/automated/get/batch_size.rb +++ b/test/automated/get/batch_size.rb @@ -4,12 +4,12 @@ context "Batch Size" do stream_name = Controls::Put.(instances: 3) - events = Get.(stream_name, batch_size: 2) + messages = Get.(stream_name, batch_size: 2) - number_of_events = events.length + number_of_messages = messages.length - test "Number of events retrieved is the specified batch size" do - assert(number_of_events == 2) + test "Number of messages retrieved is the specified batch size" do + assert(number_of_messages == 2) end end end diff --git a/test/automated/get/category.rb b/test/automated/get/category.rb index e72cc34..bb751e4 100644 --- a/test/automated/get/category.rb +++ b/test/automated/get/category.rb @@ -6,12 +6,12 @@ stream_name = Controls::Put.(instances: 2, category: category) - events = Get.(stream_name) + messages = Get.(stream_name) - number_of_events = events.length + number_of_messages = messages.length - test "Number of events retrieved is the number written to the category" do - assert(number_of_events == 2) + test "Number of messages retrieved is the number written to the category" do + assert(number_of_messages == 2) end end end diff --git a/test/automated/get/get.rb b/test/automated/get/get.rb index 5829573..4bdcbbf 100644 --- a/test/automated/get/get.rb +++ b/test/automated/get/get.rb @@ -2,43 +2,43 @@ context "Put and Get" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example + write_message = Controls::MessageData::Write.example - position = Put.(write_event, stream_name) + position = Put.(write_message, stream_name) - read_event = Get.(stream_name, position: position).first + read_message = Get.(stream_name, position: position).first - context "Got the event that was written" do + context "Got the message that was written" do test "ID" do - assert(read_event.id == write_event.id) + assert(read_message.id == write_message.id) end test "Type" do - assert(read_event.type == write_event.type) + assert(read_message.type == write_message.type) end test "Data" do - assert(read_event.data == write_event.data) + assert(read_message.data == write_message.data) end test "Metadata" do - assert(read_event.metadata == write_event.metadata) + assert(read_message.metadata == write_message.metadata) end test "Stream Name" do - assert(read_event.stream_name == stream_name) + assert(read_message.stream_name == stream_name) end test "Position" do - assert(read_event.position == position) + assert(read_message.position == position) end test "Global Position" do - assert(read_event.global_position.is_a? Numeric) + assert(read_message.global_position.is_a? Numeric) end test "Recorded Time" do - assert(read_event.time.is_a? Time) + assert(read_message.time.is_a? Time) end end end diff --git a/test/automated/get/no_events.rb b/test/automated/get/no_messages.rb similarity index 88% rename from test/automated/get/no_events.rb rename to test/automated/get/no_messages.rb index 308d031..79cefa8 100644 --- a/test/automated/get/no_events.rb +++ b/test/automated/get/no_messages.rb @@ -1,7 +1,7 @@ require_relative '../automated_init' context "Get" do - context "No Events" do + context "No Messages" do stream_name = Controls::StreamName.example batch = Get.(stream_name) diff --git a/test/automated/get_last/get_last.rb b/test/automated/get_last/get_last.rb index 77e131e..3ca7f02 100644 --- a/test/automated/get_last/get_last.rb +++ b/test/automated/get_last/get_last.rb @@ -3,13 +3,13 @@ context "Get Last" do stream_name = Controls::Put.(instances: 2) - write_event = Controls::EventData::Write.example + write_message = Controls::MessageData::Write.example - position = Put.(write_event, stream_name) + position = Put.(write_message, stream_name) - last_event = Get::Last.(stream_name) + last_message = Get::Last.(stream_name) - test "Gets the last event in the stream" do - assert(last_event.data == write_event.data) + test "Gets the last message in the stream" do + assert(last_message.data == write_message.data) end end diff --git a/test/automated/get_last/no_events.rb b/test/automated/get_last/no_events.rb deleted file mode 100644 index dcabf8e..0000000 --- a/test/automated/get_last/no_events.rb +++ /dev/null @@ -1,13 +0,0 @@ -require_relative '../automated_init' - -context "Get Last" do - context "No Events" do - stream_name = Controls::StreamName.example - - last_event = Get::Last.(stream_name) - - test "Nil event" do - assert(last_event.nil?) - end - end -end diff --git a/test/automated/get_last/no_messages.rb b/test/automated/get_last/no_messages.rb new file mode 100644 index 0000000..68bd0af --- /dev/null +++ b/test/automated/get_last/no_messages.rb @@ -0,0 +1,13 @@ +require_relative '../automated_init' + +context "Get Last" do + context "No Messages" do + stream_name = Controls::StreamName.example + + last_message = Get::Last.(stream_name) + + test "Nil message" do + assert(last_message.nil?) + end + end +end diff --git a/test/automated/iterator/next.rb b/test/automated/iterator/next.rb index f93fbef..3fbb0a3 100644 --- a/test/automated/iterator/next.rb +++ b/test/automated/iterator/next.rb @@ -10,11 +10,11 @@ batch = [] 2.times do - event_data = iterator.next - batch << event_data unless event_data.nil? + message_data = iterator.next + batch << message_data unless message_data.nil? end - test "Gets each event" do + test "Gets each message" do assert(batch.length == 2) end end diff --git a/test/automated/iterator/no_further_event_data.rb b/test/automated/iterator/no_further_event_data.rb index ca17372..c47bf07 100644 --- a/test/automated/iterator/no_further_event_data.rb +++ b/test/automated/iterator/no_further_event_data.rb @@ -1,7 +1,7 @@ require_relative '../automated_init' context "Iterator" do - context "No further event data" do + context "No further message data" do stream_name = Controls::Put.(instances: 2) iterator = Read::Iterator.build(stream_name) diff --git a/test/automated/put/category_as_stream_name.rb b/test/automated/put/category_as_stream_name.rb index 9771204..8144dba 100644 --- a/test/automated/put/category_as_stream_name.rb +++ b/test/automated/put/category_as_stream_name.rb @@ -3,14 +3,14 @@ context "Put" do context "Category as Stream Name" do category = Controls::Category.example - write_event = Controls::EventData::Write.example + write_message = Controls::MessageData::Write.example - Put.(write_event, category) + Put.(write_message, category) - read_event = Get.(category).first + read_message = Get.(category).first test "Writes the category name as the stream name" do - assert(read_event.stream_name == category) + assert(read_message.stream_name == category) end end end diff --git a/test/automated/put/data/empty.rb b/test/automated/put/data/empty.rb index c34e3c2..717c4e5 100644 --- a/test/automated/put/data/empty.rb +++ b/test/automated/put/data/empty.rb @@ -5,15 +5,15 @@ context "Nil" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example(data: {}) + write_message = Controls::MessageData::Write.example(data: {}) - position = Put.(write_event, stream_name) + position = Put.(write_message, stream_name) - read_event = Get.(stream_name, position: position).first + read_message = Get.(stream_name, position: position).first context "Read metadata" do test "Is nil" do - assert(read_event.data.nil?) + assert(read_message.data.nil?) end end end diff --git a/test/automated/put/data/nil.rb b/test/automated/put/data/nil.rb index 2ae1bb7..b9c89e6 100644 --- a/test/automated/put/data/nil.rb +++ b/test/automated/put/data/nil.rb @@ -4,15 +4,15 @@ context "Data" do context "Nil" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example(data: :none) + write_message = Controls::MessageData::Write.example(data: :none) - position = Put.(write_event, stream_name) + position = Put.(write_message, stream_name) - read_event = Get.(stream_name, position: position).first + read_message = Get.(stream_name, position: position).first context "Read metadata" do test "Is nil" do - assert(read_event.data.nil?) + assert(read_message.data.nil?) end end end diff --git a/test/automated/put/expected_version/error.rb b/test/automated/put/expected_version/error.rb index 2acff69..18e4bb4 100644 --- a/test/automated/put/expected_version/error.rb +++ b/test/automated/put/expected_version/error.rb @@ -5,22 +5,22 @@ context "Does not match the stream version" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example - position = Put.(write_event, stream_name) + write_message = Controls::MessageData::Write.example + position = Put.(write_message, stream_name) incorrect_stream_version = position + 1 test "Is an error" do - assert proc { Put.(write_event, stream_name, expected_version: incorrect_stream_version ) } do + assert proc { Put.(write_message, stream_name, expected_version: incorrect_stream_version ) } do raises_error? EventSource::ExpectedVersion::Error end end - context "Event" do - read_event = Get.(stream_name, position: incorrect_stream_version, batch_size: 1).first + context "Message" do + read_message = Get.(stream_name, position: incorrect_stream_version, batch_size: 1).first test "Is not written" do - assert(read_event.nil?) + assert(read_message.nil?) end end end diff --git a/test/automated/put/expected_version/expected_version.rb b/test/automated/put/expected_version/expected_version.rb index 6a831b0..9d96c6d 100644 --- a/test/automated/put/expected_version/expected_version.rb +++ b/test/automated/put/expected_version/expected_version.rb @@ -1,22 +1,22 @@ require_relative '../../automated_init' context "Put" do - context "Event" do + context "Message" do context "Expected Version" do stream_name = Controls::StreamName.example - write_event_1 = Controls::EventData::Write.example + write_message_1 = Controls::MessageData::Write.example - position = Put.(write_event_1, stream_name) + position = Put.(write_message_1, stream_name) - write_event_2 = Controls::EventData::Write.example + write_message_2 = Controls::MessageData::Write.example - Put.(write_event_2, stream_name, expected_version: position) + Put.(write_message_2, stream_name, expected_version: position) - read_event = Get.(stream_name, position: position + 1).first + read_message = Get.(stream_name, position: position + 1).first - test "Got the event that was written" do - assert(read_event.data == write_event_2.data) + test "Got the message that was written" do + assert(read_message.data == write_message_2.data) end end end diff --git a/test/automated/put/metadata/empty.rb b/test/automated/put/metadata/empty.rb index 8bdff4f..d83e903 100644 --- a/test/automated/put/metadata/empty.rb +++ b/test/automated/put/metadata/empty.rb @@ -5,17 +5,17 @@ context "Nil" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example(metadata: {}) + write_message = Controls::MessageData::Write.example(metadata: {}) - # write_event.metadata = {} + # write_message.metadata = {} - position = Put.(write_event, stream_name) + position = Put.(write_message, stream_name) - read_event = Get.(stream_name, position: position).first + read_message = Get.(stream_name, position: position).first context "Read metadata" do test "Is nil" do - assert(read_event.metadata.nil?) + assert(read_message.metadata.nil?) end end end diff --git a/test/automated/put/metadata/nil.rb b/test/automated/put/metadata/nil.rb index 3ecba32..9069e1d 100644 --- a/test/automated/put/metadata/nil.rb +++ b/test/automated/put/metadata/nil.rb @@ -5,15 +5,15 @@ context "Nil" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example(metadata: :none) + write_message = Controls::MessageData::Write.example(metadata: :none) - position = Put.(write_event, stream_name) + position = Put.(write_message, stream_name) - read_event = Get.(stream_name, position: position).first + read_message = Get.(stream_name, position: position).first context "Read metadata" do test "Is nil" do - assert(read_event.metadata.nil?) + assert(read_message.metadata.nil?) end end end diff --git a/test/automated/put/missing_id.rb b/test/automated/put/missing_id.rb index 93c08c9..d99e41c 100644 --- a/test/automated/put/missing_id.rb +++ b/test/automated/put/missing_id.rb @@ -3,19 +3,19 @@ context "Put" do context "Missing ID" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example(id: :none) + write_message = Controls::MessageData::Write.example(id: :none) - position = Put.(write_event, stream_name) + position = Put.(write_message, stream_name) - read_event = Get.(stream_name, position: position).first + read_message = Get.(stream_name, position: position).first - context "An ID is assigned to the event" do - test "Write event" do - refute(write_event.id.nil?) + context "An ID is assigned to the message" do + test "Write message" do + refute(write_message.id.nil?) end - test "Read event" do - refute(read_event.id.nil?) + test "Read message" do + refute(read_message.id.nil?) end end end diff --git a/test/automated/put/no_stream/existing_stream.rb b/test/automated/put/no_stream/existing_stream.rb index 235b537..1f654db 100644 --- a/test/automated/put/no_stream/existing_stream.rb +++ b/test/automated/put/no_stream/existing_stream.rb @@ -5,12 +5,12 @@ context "Existing Stream" do stream_name = Controls::StreamName.example - write_event_1 = Controls::EventData::Write.example - write_event_2 = Controls::EventData::Write.example + write_message_1 = Controls::MessageData::Write.example + write_message_2 = Controls::MessageData::Write.example - Put.(write_event_1, stream_name) + Put.(write_message_1, stream_name) - erroneous = proc { Put.(write_event_2, stream_name, expected_version: NoStream.name) } + erroneous = proc { Put.(write_message_2, stream_name, expected_version: NoStream.name) } test "Is an error" do assert erroneous do diff --git a/test/automated/put/no_stream/no_stream.rb b/test/automated/put/no_stream/no_stream.rb index e4c8b94..6b47a07 100644 --- a/test/automated/put/no_stream/no_stream.rb +++ b/test/automated/put/no_stream/no_stream.rb @@ -4,11 +4,11 @@ context "No Stream" do context "For a stream that doesn't exist" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example + write_message = Controls::MessageData::Write.example - position = Put.(write_event, stream_name) + position = Put.(write_message, stream_name) - test "Ensures that the event written is the first event in the stream" do + test "Ensures that the message written is the first message in the stream" do assert(position == 0) end end diff --git a/test/automated/put/put.rb b/test/automated/put/put.rb index 5829573..4bdcbbf 100644 --- a/test/automated/put/put.rb +++ b/test/automated/put/put.rb @@ -2,43 +2,43 @@ context "Put and Get" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example + write_message = Controls::MessageData::Write.example - position = Put.(write_event, stream_name) + position = Put.(write_message, stream_name) - read_event = Get.(stream_name, position: position).first + read_message = Get.(stream_name, position: position).first - context "Got the event that was written" do + context "Got the message that was written" do test "ID" do - assert(read_event.id == write_event.id) + assert(read_message.id == write_message.id) end test "Type" do - assert(read_event.type == write_event.type) + assert(read_message.type == write_message.type) end test "Data" do - assert(read_event.data == write_event.data) + assert(read_message.data == write_message.data) end test "Metadata" do - assert(read_event.metadata == write_event.metadata) + assert(read_message.metadata == write_message.metadata) end test "Stream Name" do - assert(read_event.stream_name == stream_name) + assert(read_message.stream_name == stream_name) end test "Position" do - assert(read_event.position == position) + assert(read_message.position == position) end test "Global Position" do - assert(read_event.global_position.is_a? Numeric) + assert(read_message.global_position.is_a? Numeric) end test "Recorded Time" do - assert(read_event.time.is_a? Time) + assert(read_message.time.is_a? Time) end end end diff --git a/test/automated/put/returns_stream_position.rb b/test/automated/put/returns_stream_position.rb index e23f970..1cff268 100644 --- a/test/automated/put/returns_stream_position.rb +++ b/test/automated/put/returns_stream_position.rb @@ -4,9 +4,9 @@ context "Returns Stream Position" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example + write_message = Controls::MessageData::Write.example - position = Put.(write_event, stream_name) + position = Put.(write_message, stream_name) test "Result is stream position" do refute(position.nil?) diff --git a/test/automated/put/stream_position_increases.rb b/test/automated/put/stream_position_increases.rb index 2d2bbbf..3b5828f 100644 --- a/test/automated/put/stream_position_increases.rb +++ b/test/automated/put/stream_position_increases.rb @@ -3,10 +3,10 @@ context "Put" do context "Stream Position Increases with Subsequent Writes" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example + write_message = Controls::MessageData::Write.example - position_1 = Put.(write_event, stream_name) - position_2 = Put.(write_event, stream_name) + position_1 = Put.(write_message, stream_name) + position_2 = Put.(write_message, stream_name) test "First version is one less than the second version" do assert(position_1 + 1 == position_2) diff --git a/test/automated/read/position.rb b/test/automated/read/position.rb index ecfba8a..c96adca 100644 --- a/test/automated/read/position.rb +++ b/test/automated/read/position.rb @@ -6,8 +6,8 @@ batch = [] - Read.(stream_name, position: 1, batch_size: 1) do |event_data| - batch << event_data + Read.(stream_name, position: 1, batch_size: 1) do |message_data| + batch << message_data end test "Reads from the starting position" do diff --git a/test/automated/read/read.rb b/test/automated/read/read.rb index 5b77155..ce7170c 100644 --- a/test/automated/read/read.rb +++ b/test/automated/read/read.rb @@ -5,11 +5,11 @@ batch = [] - Read.(stream_name, batch_size: 1) do |event_data| - batch << event_data + Read.(stream_name, batch_size: 1) do |message_data| + batch << message_data end - test "Reads batches of events" do + test "Reads batches of messages" do assert(batch.length == 2) end end diff --git a/test/automated/write/batch/expected_version/error.rb b/test/automated/write/batch/expected_version/error.rb index df5bd95..5f824ae 100644 --- a/test/automated/write/batch/expected_version/error.rb +++ b/test/automated/write/batch/expected_version/error.rb @@ -6,15 +6,15 @@ context "Does not match the stream version" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example - position = Write.(write_event, stream_name) + write_message = Controls::MessageData::Write.example + position = Write.(write_message, stream_name) incorrect_stream_version = position + 1 - write_event_1 = Controls::EventData::Write.example - write_event_2 = Controls::EventData::Write.example + write_message_1 = Controls::MessageData::Write.example + write_message_2 = Controls::MessageData::Write.example - batch = [write_event_1, write_event_2] + batch = [write_message_1, write_message_2] test "Is an error" do assert proc { Write.(batch, stream_name, expected_version: incorrect_stream_version ) } do @@ -22,12 +22,12 @@ end end - context "Events" do + context "Messages" do 2.times do |i| - read_event = Get.(stream_name, position: i + 1, batch_size: 1).first + read_message = Get.(stream_name, position: i + 1, batch_size: 1).first - test "Event #{i + 1} not written" do - assert(read_event.nil?) + test "Message #{i + 1} not written" do + assert(read_message.nil?) end end end diff --git a/test/automated/write/batch/expected_version/expected_version.rb b/test/automated/write/batch/expected_version/expected_version.rb index b514d89..de922b2 100644 --- a/test/automated/write/batch/expected_version/expected_version.rb +++ b/test/automated/write/batch/expected_version/expected_version.rb @@ -5,23 +5,23 @@ context "Expected Version" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example - position = Write.(write_event, stream_name) + write_message = Controls::MessageData::Write.example + position = Write.(write_message, stream_name) - write_event_1 = Controls::EventData::Write.example - write_event_2 = Controls::EventData::Write.example + write_message_1 = Controls::MessageData::Write.example + write_message_2 = Controls::MessageData::Write.example - batch = [write_event_1, write_event_2] + batch = [write_message_1, write_message_2] Write.(batch, stream_name, expected_version: position) - context "Individual Events are Written" do + context "Individual Messages are Written" do 2.times do |i| - read_event = Get.(stream_name, position: i + 1, batch_size: 1).first - write_event = batch[i] + read_message = Get.(stream_name, position: i + 1, batch_size: 1).first + write_message = batch[i] - test "Event #{i + 1}" do - assert(read_event.data == write_event.data) + test "Message #{i + 1}" do + assert(read_message.data == write_message.data) end end end diff --git a/test/automated/write/batch/write.rb b/test/automated/write/batch/write.rb index 87aaba8..d053b5f 100644 --- a/test/automated/write/batch/write.rb +++ b/test/automated/write/batch/write.rb @@ -4,10 +4,10 @@ context "Batch" do stream_name = Controls::StreamName.example - write_event_1 = Controls::EventData::Write.example - write_event_2 = Controls::EventData::Write.example + write_message_1 = Controls::MessageData::Write.example + write_message_2 = Controls::MessageData::Write.example - batch = [write_event_1, write_event_2] + batch = [write_message_1, write_message_2] last_written_position = Write.(batch, stream_name) @@ -15,13 +15,13 @@ assert(last_written_position == 1) end - context "Individual Events are Written" do + context "Individual Messages are Written" do 2.times do |i| - read_event = Get.(stream_name, position: i, batch_size: 1).first - write_event = batch[i] + read_message = Get.(stream_name, position: i, batch_size: 1).first + write_message = batch[i] - test "Event #{i + 1}" do - assert(read_event.data == write_event.data) + test "Message #{i + 1}" do + assert(read_message.data == write_message.data) end end end diff --git a/test/automated/write/event/expected_version/expected_version.rb b/test/automated/write/event/expected_version/expected_version.rb deleted file mode 100644 index 1d8ee1d..0000000 --- a/test/automated/write/event/expected_version/expected_version.rb +++ /dev/null @@ -1,23 +0,0 @@ -require_relative '../../../automated_init' - -context "Write" do - context "Event" do - context "Expected Version" do - stream_name = Controls::StreamName.example - - write_event_1 = Controls::EventData::Write.example - - position = Write.(write_event_1, stream_name) - - write_event_2 = Controls::EventData::Write.example - - Write.(write_event_2, stream_name, expected_version: position) - - read_event = Get.(stream_name, position: position + 1).first - - test "Got the event that was written" do - assert(read_event.data == write_event_2.data) - end - end - end -end diff --git a/test/automated/write/event/write.rb b/test/automated/write/event/write.rb deleted file mode 100644 index 235a2b2..0000000 --- a/test/automated/write/event/write.rb +++ /dev/null @@ -1,17 +0,0 @@ -require_relative '../../automated_init' - -context "Write" do - context "Event" do - stream_name = Controls::StreamName.example - - write_event = Controls::EventData::Write.example - - position = Write.(write_event, stream_name) - - read_event = Get.(stream_name, position: position).first - - test "Got the event that was written" do - assert(read_event.data == write_event.data) - end - end -end diff --git a/test/automated/write/event/expected_version/error.rb b/test/automated/write/message/expected_version/error.rb similarity index 52% rename from test/automated/write/event/expected_version/error.rb rename to test/automated/write/message/expected_version/error.rb index 38a12ed..1b3e72b 100644 --- a/test/automated/write/event/expected_version/error.rb +++ b/test/automated/write/message/expected_version/error.rb @@ -1,27 +1,27 @@ require_relative '../../../automated_init' context "Write" do - context "Event" do + context "Message" do context "Expected Version" do context "Does not match the stream version" do stream_name = Controls::StreamName.example - write_event = Controls::EventData::Write.example - position = Write.(write_event, stream_name) + write_message = Controls::MessageData::Write.example + position = Write.(write_message, stream_name) incorrect_stream_version = position + 1 test "Is an error" do - assert proc { Write.(write_event, stream_name, expected_version: incorrect_stream_version ) } do + assert proc { Write.(write_message, stream_name, expected_version: incorrect_stream_version ) } do raises_error? EventSource::ExpectedVersion::Error end end - context "Event" do - read_event = Get.(stream_name, position: incorrect_stream_version, batch_size: 1).first + context "Message" do + read_message = Get.(stream_name, position: incorrect_stream_version, batch_size: 1).first test "Is not written" do - assert(read_event.nil?) + assert(read_message.nil?) end end end diff --git a/test/automated/write/message/expected_version/expected_version.rb b/test/automated/write/message/expected_version/expected_version.rb new file mode 100644 index 0000000..a353879 --- /dev/null +++ b/test/automated/write/message/expected_version/expected_version.rb @@ -0,0 +1,23 @@ +require_relative '../../../automated_init' + +context "Write" do + context "Message" do + context "Expected Version" do + stream_name = Controls::StreamName.example + + write_message_1 = Controls::MessageData::Write.example + + position = Write.(write_message_1, stream_name) + + write_message_2 = Controls::MessageData::Write.example + + Write.(write_message_2, stream_name, expected_version: position) + + read_message = Get.(stream_name, position: position + 1).first + + test "Got the message that was written" do + assert(read_message.data == write_message_2.data) + end + end + end +end diff --git a/test/automated/write/message/write.rb b/test/automated/write/message/write.rb new file mode 100644 index 0000000..ea43ff3 --- /dev/null +++ b/test/automated/write/message/write.rb @@ -0,0 +1,17 @@ +require_relative '../../automated_init' + +context "Write" do + context "Message" do + stream_name = Controls::StreamName.example + + write_message = Controls::MessageData::Write.example + + position = Write.(write_message, stream_name) + + read_message = Get.(stream_name, position: position).first + + test "Got the message that was written" do + assert(read_message.data == write_message.data) + end + end +end From 9e26f74d303647d767df40085c0e940f49a18d8b Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 26 May 2017 15:40:25 -0500 Subject: [PATCH 241/591] Log tag is updated per new library name --- lib/event_source/postgres/controls/put.rb | 12 ++++----- lib/event_source/postgres/get/last.rb | 10 +++---- lib/event_source/postgres/log.rb | 2 +- lib/event_source/postgres/put.rb | 32 +++++++++++------------ 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/lib/event_source/postgres/controls/put.rb b/lib/event_source/postgres/controls/put.rb index a63da2f..8da4252 100644 --- a/lib/event_source/postgres/controls/put.rb +++ b/lib/event_source/postgres/controls/put.rb @@ -2,19 +2,19 @@ module EventSource module Postgres module Controls module Put - def self.call(instances: nil, stream_name: nil, event: nil, category: nil) + def self.call(instances: nil, stream_name: nil, message: nil, category: nil) instances ||= 1 stream_name ||= StreamName.example(category: category) - event_specified = !event.nil? + message_specified = !message.nil? - event ||= MessageData::Write.example + message ||= MessageData::Write.example instances.times do - EventSource::Postgres::Put.(event, stream_name) + EventSource::Postgres::Put.(message, stream_name) - unless event_specified - event.id = MessageData::Write.id + unless message_specified + message.id = MessageData::Write.id end end diff --git a/lib/event_source/postgres/get/last.rb b/lib/event_source/postgres/get/last.rb index 65c7560..a7441d8 100644 --- a/lib/event_source/postgres/get/last.rb +++ b/lib/event_source/postgres/get/last.rb @@ -34,12 +34,12 @@ def call(stream_name) return nil if record.nil? - event = convert(record) + message_data = convert(record) logger.info { "Finished getting message data (Stream Name: #{stream_name})" } - logger.info(tags: [:data, :message_data]) { event.pretty_inspect } + logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } - event + message_data end def get_record(stream_name) @@ -63,11 +63,11 @@ def convert(record) record['metadata'] = Deserialize.metadata(record['metadata']) record['time'] = Time.utc_coerced(record['time']) - event = MessageData::Read.build(record) + message_data = MessageData::Read.build(record) logger.debug { "Converted record to message data" } - event + message_data end def __convert(records) diff --git a/lib/event_source/postgres/log.rb b/lib/event_source/postgres/log.rb index b6fdf18..cedb2c6 100644 --- a/lib/event_source/postgres/log.rb +++ b/lib/event_source/postgres/log.rb @@ -2,7 +2,7 @@ module EventSource module Postgres class Log < ::Log def tag!(tags) - tags << :event_source_postgres + tags << :message_store_postgres tags << :library tags << :verbose end diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index 1f0a88a..ff64754 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -23,31 +23,31 @@ def self.configure(receiver, session: nil, attr_name: nil) receiver.public_send "#{attr_name}=", instance end - def self.call(write_event, stream_name, expected_version: nil, session: nil) + def self.call(write_message, stream_name, expected_version: nil, session: nil) instance = build(session: session) - instance.(write_event, stream_name, expected_version: expected_version) + instance.(write_message, stream_name, expected_version: expected_version) end - def call(write_event, stream_name, expected_version: nil) - logger.trace { "Putting message data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :message_data]) { write_event.pretty_inspect } + def call(write_message, stream_name, expected_version: nil) + logger.trace { "Putting message data (Stream Name: #{stream_name}, Type: #{write_message.type}, Expected Version: #{expected_version.inspect})" } + logger.trace(tags: [:data, :message_data]) { write_message.pretty_inspect } - write_event.id ||= identifier.get + write_message.id ||= identifier.get - id, type, data, metadata = destructure_event(write_event) + id, type, data, metadata = destructure_message(write_message) expected_version = ExpectedVersion.canonize(expected_version) - insert_event(id, stream_name, type, data, metadata, expected_version).tap do |position| - logger.info { "Put message data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } - logger.info(tags: [:data, :message_data]) { write_event.pretty_inspect } + insert_message(id, stream_name, type, data, metadata, expected_version).tap do |position| + logger.info { "Put message data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_message.type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } + logger.info(tags: [:data, :message_data]) { write_message.pretty_inspect } end end - def destructure_event(write_event) - id = write_event.id - type = write_event.type - data = write_event.data - metadata = write_event.metadata + def destructure_message(write_message) + id = write_message.id + type = write_message.type + data = write_message.data + metadata = write_message.metadata logger.debug(tags: [:data, :message_data]) { "ID: #{id.pretty_inspect}" } logger.debug(tags: [:data, :message_data]) { "Type: #{type.pretty_inspect}" } @@ -57,7 +57,7 @@ def destructure_event(write_event) return id, type, data, metadata end - def insert_event(id, stream_name, type, data, metadata, expected_version) + def insert_message(id, stream_name, type, data, metadata, expected_version) serialized_data = serialized_data(data) serialized_metadata = serialized_metadata(metadata) records = execute_query(id, stream_name, type, serialized_data, serialized_metadata, expected_version) From 9b3ef3504c49a80fb8d4e5490285b2f988b28acc Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 26 May 2017 15:40:54 -0500 Subject: [PATCH 242/591] Settings filename is updated per new library name --- lib/event_source/postgres/settings.rb | 2 +- .../{event_source_postgres.json => message_store_postgres.json} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename settings/{event_source_postgres.json => message_store_postgres.json} (100%) diff --git a/lib/event_source/postgres/settings.rb b/lib/event_source/postgres/settings.rb index b52705a..26d3ec0 100644 --- a/lib/event_source/postgres/settings.rb +++ b/lib/event_source/postgres/settings.rb @@ -6,7 +6,7 @@ def self.instance end def self.data_source - 'settings/event_source_postgres.json' + 'settings/message_store_postgres.json' end def self.names diff --git a/settings/event_source_postgres.json b/settings/message_store_postgres.json similarity index 100% rename from settings/event_source_postgres.json rename to settings/message_store_postgres.json From 5b278c8ec8ba4c732e919602a28e793698f2e317 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 26 May 2017 15:46:42 -0500 Subject: [PATCH 243/591] EventSource namespace is removed from error classes in tests --- test/automated/put/expected_version/error.rb | 2 +- test/automated/write/batch/expected_version/error.rb | 2 +- test/automated/write/message/expected_version/error.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/automated/put/expected_version/error.rb b/test/automated/put/expected_version/error.rb index 18e4bb4..ecfcbf8 100644 --- a/test/automated/put/expected_version/error.rb +++ b/test/automated/put/expected_version/error.rb @@ -12,7 +12,7 @@ test "Is an error" do assert proc { Put.(write_message, stream_name, expected_version: incorrect_stream_version ) } do - raises_error? EventSource::ExpectedVersion::Error + raises_error? ExpectedVersion::Error end end diff --git a/test/automated/write/batch/expected_version/error.rb b/test/automated/write/batch/expected_version/error.rb index 5f824ae..4c4e6c8 100644 --- a/test/automated/write/batch/expected_version/error.rb +++ b/test/automated/write/batch/expected_version/error.rb @@ -18,7 +18,7 @@ test "Is an error" do assert proc { Write.(batch, stream_name, expected_version: incorrect_stream_version ) } do - raises_error? EventSource::ExpectedVersion::Error + raises_error? ExpectedVersion::Error end end diff --git a/test/automated/write/message/expected_version/error.rb b/test/automated/write/message/expected_version/error.rb index 1b3e72b..4651e44 100644 --- a/test/automated/write/message/expected_version/error.rb +++ b/test/automated/write/message/expected_version/error.rb @@ -13,7 +13,7 @@ test "Is an error" do assert proc { Write.(write_message, stream_name, expected_version: incorrect_stream_version ) } do - raises_error? EventSource::ExpectedVersion::Error + raises_error? ExpectedVersion::Error end end From 412793d544807c8096143710ed21c5af040947ba Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 26 May 2017 15:48:37 -0500 Subject: [PATCH 244/591] EventSource namespace is renamed to MessageStore --- lib/event_source/postgres/controls/category.rb | 6 +++--- lib/event_source/postgres/controls/message_data.rb | 2 +- lib/event_source/postgres/controls/put.rb | 4 ++-- lib/event_source/postgres/controls/stream_name.rb | 8 ++++---- lib/event_source/postgres/get.rb | 4 ++-- lib/event_source/postgres/get/last.rb | 2 +- lib/event_source/postgres/get/last/select_statement.rb | 2 +- lib/event_source/postgres/get/select_statement.rb | 2 +- lib/event_source/postgres/log.rb | 2 +- lib/event_source/postgres/put.rb | 2 +- lib/event_source/postgres/read.rb | 4 ++-- lib/event_source/postgres/read/iterator.rb | 6 +++--- lib/event_source/postgres/session.rb | 2 +- lib/event_source/postgres/settings.rb | 2 +- lib/event_source/postgres/stream_name.rb | 2 +- lib/event_source/postgres/write.rb | 4 ++-- test/test_init.rb | 4 ++-- 17 files changed, 29 insertions(+), 29 deletions(-) diff --git a/lib/event_source/postgres/controls/category.rb b/lib/event_source/postgres/controls/category.rb index 33dcf3d..0d717eb 100644 --- a/lib/event_source/postgres/controls/category.rb +++ b/lib/event_source/postgres/controls/category.rb @@ -1,13 +1,13 @@ -# module EventSource +# module MessageStore # module Postgres # module Controls -# Category = EventSource::Controls::Category +# Category = MessageStore::Controls::Category # end # end # end -module EventSource +module MessageStore module Postgres module Controls module Category diff --git a/lib/event_source/postgres/controls/message_data.rb b/lib/event_source/postgres/controls/message_data.rb index 54c9b50..0febad2 100644 --- a/lib/event_source/postgres/controls/message_data.rb +++ b/lib/event_source/postgres/controls/message_data.rb @@ -1,4 +1,4 @@ -module EventSource +module MessageStore module Postgres module Controls MessageData = MessageStore::Controls::MessageData diff --git a/lib/event_source/postgres/controls/put.rb b/lib/event_source/postgres/controls/put.rb index 8da4252..660b20a 100644 --- a/lib/event_source/postgres/controls/put.rb +++ b/lib/event_source/postgres/controls/put.rb @@ -1,4 +1,4 @@ -module EventSource +module MessageStore module Postgres module Controls module Put @@ -11,7 +11,7 @@ def self.call(instances: nil, stream_name: nil, message: nil, category: nil) message ||= MessageData::Write.example instances.times do - EventSource::Postgres::Put.(message, stream_name) + MessageStore::Postgres::Put.(message, stream_name) unless message_specified message.id = MessageData::Write.id diff --git a/lib/event_source/postgres/controls/stream_name.rb b/lib/event_source/postgres/controls/stream_name.rb index 0467c0c..f928339 100644 --- a/lib/event_source/postgres/controls/stream_name.rb +++ b/lib/event_source/postgres/controls/stream_name.rb @@ -1,12 +1,12 @@ -# module EventSource +# module MessageStore # module Postgres # module Controls -# StreamName = EventSource::Controls::StreamName +# StreamName = MessageStore::Controls::StreamName # end # end # end -module EventSource +module MessageStore module Postgres module Controls module StreamName @@ -14,7 +14,7 @@ def self.example(category: nil, id: nil, type: nil, types: nil, randomize_catego category ||= Category.example(category: category, randomize_category: randomize_category) id ||= Identifier::UUID.random - EventSource::Postgres::StreamName.stream_name(category, id, type: type, types: types) + MessageStore::Postgres::StreamName.stream_name(category, id, type: type, types: types) end end end diff --git a/lib/event_source/postgres/get.rb b/lib/event_source/postgres/get.rb index a878fda..26ac05f 100644 --- a/lib/event_source/postgres/get.rb +++ b/lib/event_source/postgres/get.rb @@ -1,7 +1,7 @@ -module EventSource +module MessageStore module Postgres class Get - include EventSource::Get + include MessageStore::Get initializer :batch_size diff --git a/lib/event_source/postgres/get/last.rb b/lib/event_source/postgres/get/last.rb index a7441d8..6033c8a 100644 --- a/lib/event_source/postgres/get/last.rb +++ b/lib/event_source/postgres/get/last.rb @@ -1,4 +1,4 @@ -module EventSource +module MessageStore module Postgres class Get class Last diff --git a/lib/event_source/postgres/get/last/select_statement.rb b/lib/event_source/postgres/get/last/select_statement.rb index 7c02cf6..80c80ab 100644 --- a/lib/event_source/postgres/get/last/select_statement.rb +++ b/lib/event_source/postgres/get/last/select_statement.rb @@ -1,4 +1,4 @@ -module EventSource +module MessageStore module Postgres class Get class Last diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/event_source/postgres/get/select_statement.rb index ea46e39..7f5f259 100644 --- a/lib/event_source/postgres/get/select_statement.rb +++ b/lib/event_source/postgres/get/select_statement.rb @@ -1,4 +1,4 @@ -module EventSource +module MessageStore module Postgres class Get class SelectStatement diff --git a/lib/event_source/postgres/log.rb b/lib/event_source/postgres/log.rb index cedb2c6..1c55997 100644 --- a/lib/event_source/postgres/log.rb +++ b/lib/event_source/postgres/log.rb @@ -1,4 +1,4 @@ -module EventSource +module MessageStore module Postgres class Log < ::Log def tag!(tags) diff --git a/lib/event_source/postgres/put.rb b/lib/event_source/postgres/put.rb index ff64754..3040fcf 100644 --- a/lib/event_source/postgres/put.rb +++ b/lib/event_source/postgres/put.rb @@ -1,4 +1,4 @@ -module EventSource +module MessageStore module Postgres class Put include Log::Dependency diff --git a/lib/event_source/postgres/read.rb b/lib/event_source/postgres/read.rb index 684c035..b45ca17 100644 --- a/lib/event_source/postgres/read.rb +++ b/lib/event_source/postgres/read.rb @@ -1,7 +1,7 @@ -module EventSource +module MessageStore module Postgres class Read - include EventSource::Read + include MessageStore::Read def configure(session: nil) Iterator.configure(self, stream_name, position: position) diff --git a/lib/event_source/postgres/read/iterator.rb b/lib/event_source/postgres/read/iterator.rb index d4d2ebd..5b197c5 100644 --- a/lib/event_source/postgres/read/iterator.rb +++ b/lib/event_source/postgres/read/iterator.rb @@ -1,11 +1,11 @@ -module EventSource +module MessageStore module Postgres class Read class Iterator - include EventSource::Read::Iterator + include MessageStore::Read::Iterator def last_position - unless EventSource::Postgres::StreamName.category?(stream_name) + unless MessageStore::Postgres::StreamName.category?(stream_name) batch.last.position else batch.last.global_position diff --git a/lib/event_source/postgres/session.rb b/lib/event_source/postgres/session.rb index f88b5ae..90a9365 100644 --- a/lib/event_source/postgres/session.rb +++ b/lib/event_source/postgres/session.rb @@ -1,4 +1,4 @@ -module EventSource +module MessageStore module Postgres class Session include Log::Dependency diff --git a/lib/event_source/postgres/settings.rb b/lib/event_source/postgres/settings.rb index 26d3ec0..f7d775f 100644 --- a/lib/event_source/postgres/settings.rb +++ b/lib/event_source/postgres/settings.rb @@ -1,4 +1,4 @@ -module EventSource +module MessageStore module Postgres class Settings < ::Settings def self.instance diff --git a/lib/event_source/postgres/stream_name.rb b/lib/event_source/postgres/stream_name.rb index d7f81fe..29e6cd6 100644 --- a/lib/event_source/postgres/stream_name.rb +++ b/lib/event_source/postgres/stream_name.rb @@ -1,4 +1,4 @@ -module EventSource +module MessageStore module Postgres module StreamName def self.stream_name(category_name, id=nil, type: nil, types: nil) diff --git a/lib/event_source/postgres/write.rb b/lib/event_source/postgres/write.rb index bc4981f..b1789e6 100644 --- a/lib/event_source/postgres/write.rb +++ b/lib/event_source/postgres/write.rb @@ -1,7 +1,7 @@ -module EventSource +module MessageStore module Postgres class Write - include EventSource::Write + include MessageStore::Write dependency :put diff --git a/test/test_init.rb b/test/test_init.rb index ccb7400..b7777bc 100644 --- a/test/test_init.rb +++ b/test/test_init.rb @@ -8,5 +8,5 @@ require 'test_bench'; TestBench.activate -include EventSource -include EventSource::Postgres +include MessageStore +include MessageStore::Postgres From 4d373b240d180c22692da8bdd765d46b62b3ec00 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 26 May 2017 15:49:40 -0500 Subject: [PATCH 245/591] Files are moved from lib/event_source to lib/message_store --- init.rb | 2 +- lib/event_source/postgres.rb | 24 ------------------- lib/event_source/postgres/controls.rb | 6 ----- lib/message_store/postgres.rb | 24 +++++++++++++++++++ lib/message_store/postgres/controls.rb | 6 +++++ .../postgres/controls/category.rb | 0 .../postgres/controls/message_data.rb | 0 .../postgres/controls/put.rb | 0 .../postgres/controls/stream_name.rb | 0 .../postgres/get.rb | 0 .../postgres/get/last.rb | 0 .../postgres/get/last/select_statement.rb | 0 .../postgres/get/select_statement.rb | 0 .../postgres/log.rb | 0 .../postgres/put.rb | 0 .../postgres/read.rb | 0 .../postgres/read/iterator.rb | 0 .../postgres/session.rb | 0 .../postgres/settings.rb | 0 .../postgres/stream_name.rb | 0 .../postgres/write.rb | 0 test/test_init.rb | 2 +- 22 files changed, 32 insertions(+), 32 deletions(-) delete mode 100644 lib/event_source/postgres.rb delete mode 100644 lib/event_source/postgres/controls.rb create mode 100644 lib/message_store/postgres.rb create mode 100644 lib/message_store/postgres/controls.rb rename lib/{event_source => message_store}/postgres/controls/category.rb (100%) rename lib/{event_source => message_store}/postgres/controls/message_data.rb (100%) rename lib/{event_source => message_store}/postgres/controls/put.rb (100%) rename lib/{event_source => message_store}/postgres/controls/stream_name.rb (100%) rename lib/{event_source => message_store}/postgres/get.rb (100%) rename lib/{event_source => message_store}/postgres/get/last.rb (100%) rename lib/{event_source => message_store}/postgres/get/last/select_statement.rb (100%) rename lib/{event_source => message_store}/postgres/get/select_statement.rb (100%) rename lib/{event_source => message_store}/postgres/log.rb (100%) rename lib/{event_source => message_store}/postgres/put.rb (100%) rename lib/{event_source => message_store}/postgres/read.rb (100%) rename lib/{event_source => message_store}/postgres/read/iterator.rb (100%) rename lib/{event_source => message_store}/postgres/session.rb (100%) rename lib/{event_source => message_store}/postgres/settings.rb (100%) rename lib/{event_source => message_store}/postgres/stream_name.rb (100%) rename lib/{event_source => message_store}/postgres/write.rb (100%) diff --git a/init.rb b/init.rb index 5c945fe..1c93086 100644 --- a/init.rb +++ b/init.rb @@ -7,4 +7,4 @@ $LOAD_PATH.unshift libraries_dir unless $LOAD_PATH.include?(libraries_dir) end -require 'event_source/postgres' +require 'message_store/postgres' diff --git a/lib/event_source/postgres.rb b/lib/event_source/postgres.rb deleted file mode 100644 index 430b56f..0000000 --- a/lib/event_source/postgres.rb +++ /dev/null @@ -1,24 +0,0 @@ -require 'pg' - -require 'event_source' - -require 'log' -require 'telemetry' -require 'settings'; Settings.activate - -require 'event_source/postgres/log' - -require 'event_source/postgres/settings' -require 'event_source/postgres/session' - -require 'event_source/postgres/stream_name' - -require 'event_source/postgres/put' -require 'event_source/postgres/write' - -require 'event_source/postgres/get/select_statement' -require 'event_source/postgres/get' -require 'event_source/postgres/get/last/select_statement' -require 'event_source/postgres/get/last' -require 'event_source/postgres/read/iterator' -require 'event_source/postgres/read' diff --git a/lib/event_source/postgres/controls.rb b/lib/event_source/postgres/controls.rb deleted file mode 100644 index 30fe10d..0000000 --- a/lib/event_source/postgres/controls.rb +++ /dev/null @@ -1,6 +0,0 @@ -require 'message_store/controls' - -require 'event_source/postgres/controls/category' -require 'event_source/postgres/controls/stream_name' -require 'event_source/postgres/controls/message_data' -require 'event_source/postgres/controls/put' diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb new file mode 100644 index 0000000..63c15ca --- /dev/null +++ b/lib/message_store/postgres.rb @@ -0,0 +1,24 @@ +require 'pg' + +require 'message_store' + +require 'log' +require 'telemetry' +require 'settings'; Settings.activate + +require 'message_store/postgres/log' + +require 'message_store/postgres/settings' +require 'message_store/postgres/session' + +require 'message_store/postgres/stream_name' + +require 'message_store/postgres/put' +require 'message_store/postgres/write' + +require 'message_store/postgres/get/select_statement' +require 'message_store/postgres/get' +require 'message_store/postgres/get/last/select_statement' +require 'message_store/postgres/get/last' +require 'message_store/postgres/read/iterator' +require 'message_store/postgres/read' diff --git a/lib/message_store/postgres/controls.rb b/lib/message_store/postgres/controls.rb new file mode 100644 index 0000000..3b51255 --- /dev/null +++ b/lib/message_store/postgres/controls.rb @@ -0,0 +1,6 @@ +require 'message_store/controls' + +require 'message_store/postgres/controls/category' +require 'message_store/postgres/controls/stream_name' +require 'message_store/postgres/controls/message_data' +require 'message_store/postgres/controls/put' diff --git a/lib/event_source/postgres/controls/category.rb b/lib/message_store/postgres/controls/category.rb similarity index 100% rename from lib/event_source/postgres/controls/category.rb rename to lib/message_store/postgres/controls/category.rb diff --git a/lib/event_source/postgres/controls/message_data.rb b/lib/message_store/postgres/controls/message_data.rb similarity index 100% rename from lib/event_source/postgres/controls/message_data.rb rename to lib/message_store/postgres/controls/message_data.rb diff --git a/lib/event_source/postgres/controls/put.rb b/lib/message_store/postgres/controls/put.rb similarity index 100% rename from lib/event_source/postgres/controls/put.rb rename to lib/message_store/postgres/controls/put.rb diff --git a/lib/event_source/postgres/controls/stream_name.rb b/lib/message_store/postgres/controls/stream_name.rb similarity index 100% rename from lib/event_source/postgres/controls/stream_name.rb rename to lib/message_store/postgres/controls/stream_name.rb diff --git a/lib/event_source/postgres/get.rb b/lib/message_store/postgres/get.rb similarity index 100% rename from lib/event_source/postgres/get.rb rename to lib/message_store/postgres/get.rb diff --git a/lib/event_source/postgres/get/last.rb b/lib/message_store/postgres/get/last.rb similarity index 100% rename from lib/event_source/postgres/get/last.rb rename to lib/message_store/postgres/get/last.rb diff --git a/lib/event_source/postgres/get/last/select_statement.rb b/lib/message_store/postgres/get/last/select_statement.rb similarity index 100% rename from lib/event_source/postgres/get/last/select_statement.rb rename to lib/message_store/postgres/get/last/select_statement.rb diff --git a/lib/event_source/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb similarity index 100% rename from lib/event_source/postgres/get/select_statement.rb rename to lib/message_store/postgres/get/select_statement.rb diff --git a/lib/event_source/postgres/log.rb b/lib/message_store/postgres/log.rb similarity index 100% rename from lib/event_source/postgres/log.rb rename to lib/message_store/postgres/log.rb diff --git a/lib/event_source/postgres/put.rb b/lib/message_store/postgres/put.rb similarity index 100% rename from lib/event_source/postgres/put.rb rename to lib/message_store/postgres/put.rb diff --git a/lib/event_source/postgres/read.rb b/lib/message_store/postgres/read.rb similarity index 100% rename from lib/event_source/postgres/read.rb rename to lib/message_store/postgres/read.rb diff --git a/lib/event_source/postgres/read/iterator.rb b/lib/message_store/postgres/read/iterator.rb similarity index 100% rename from lib/event_source/postgres/read/iterator.rb rename to lib/message_store/postgres/read/iterator.rb diff --git a/lib/event_source/postgres/session.rb b/lib/message_store/postgres/session.rb similarity index 100% rename from lib/event_source/postgres/session.rb rename to lib/message_store/postgres/session.rb diff --git a/lib/event_source/postgres/settings.rb b/lib/message_store/postgres/settings.rb similarity index 100% rename from lib/event_source/postgres/settings.rb rename to lib/message_store/postgres/settings.rb diff --git a/lib/event_source/postgres/stream_name.rb b/lib/message_store/postgres/stream_name.rb similarity index 100% rename from lib/event_source/postgres/stream_name.rb rename to lib/message_store/postgres/stream_name.rb diff --git a/lib/event_source/postgres/write.rb b/lib/message_store/postgres/write.rb similarity index 100% rename from lib/event_source/postgres/write.rb rename to lib/message_store/postgres/write.rb diff --git a/test/test_init.rb b/test/test_init.rb index b7777bc..1bbc2ae 100644 --- a/test/test_init.rb +++ b/test/test_init.rb @@ -4,7 +4,7 @@ puts RUBY_DESCRIPTION require_relative '../init.rb' -require 'event_source/postgres/controls' +require 'message_store/postgres/controls' require 'test_bench'; TestBench.activate From 6d04ebc5e049c5bd1a1ea2706e4bf143538ca788 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 26 May 2017 15:51:53 -0500 Subject: [PATCH 246/591] Symlinks are updated to new library name --- remove-lib-symlinks.sh | 2 +- symlink-lib.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/remove-lib-symlinks.sh b/remove-lib-symlinks.sh index 6146bbd..01f2203 100755 --- a/remove-lib-symlinks.sh +++ b/remove-lib-symlinks.sh @@ -1,3 +1,3 @@ source ./library-symlinks.sh -remove_lib_symlinks 'event_source' 'postgres' +remove_lib_symlinks 'postgres' 'message_store' diff --git a/symlink-lib.sh b/symlink-lib.sh index 0be5c40..5a5f9bb 100755 --- a/symlink-lib.sh +++ b/symlink-lib.sh @@ -1,3 +1,3 @@ source ./library-symlinks.sh -symlink_lib 'event_source' 'postgres' +symlink_lib 'postgres' 'message_store' From 3405d0251c2563cbb86c80e89d51136ab8b785ca Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 26 May 2017 16:02:15 -0500 Subject: [PATCH 247/591] Gemspec is updated to reflect new project name --- ...postgres.gemspec => message_store-postgres.gemspec | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) rename event_source-postgres.gemspec => message_store-postgres.gemspec (73%) diff --git a/event_source-postgres.gemspec b/message_store-postgres.gemspec similarity index 73% rename from event_source-postgres.gemspec rename to message_store-postgres.gemspec index 94551ae..b9b738e 100644 --- a/event_source-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,24 +1,25 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| - s.name = 'evt-event_source-postgres' - s.version = '0.19.0.3' + s.name = 'evt-message_store-postgres' + s.version = '0.0.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' s.authors = ['The Eventide Project'] s.email = 'opensource@eventide-project.org' - s.homepage = 'https://github.com/eventide-project/event-source-postgres' + s.homepage = 'https://github.com/eventide-project/message-store-postgres' s.licenses = ['MIT'] s.require_paths = ['lib'] s.files = Dir.glob('{lib,database}/**/*') + ['scripts/scripts_init.rb'] s.platform = Gem::Platform::RUBY - s.required_ruby_version = '>= 2.3.3' + s.required_ruby_version = '>= 2.4.0' s.executables = Dir.glob('scripts/evt-*').map(&File.method(:basename)) s.bindir = 'scripts' - s.add_runtime_dependency 'evt-event_source' + s.add_runtime_dependency 'evt-message_store' + s.add_runtime_dependency 'evt-log' s.add_runtime_dependency 'evt-settings' From a1ebdccbe5c973f552aa56b250b3978be82d910a Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 26 May 2017 16:03:51 -0500 Subject: [PATCH 248/591] Package version is set to 0.1.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index b9b738e..cb25591 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.0.0.0' + s.version = '0.1.0.0' s.summary = 'Event source client for PostgreSQL' s.description = ' ' From 6a1d02ecbf90ce9028bbc1ebfdb0646d1333e554 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 26 May 2017 16:06:47 -0500 Subject: [PATCH 249/591] Gem name and summary --- library.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library.gemspec b/library.gemspec index 90cb7c2..f636be5 100644 --- a/library.gemspec +++ b/library.gemspec @@ -1,8 +1,8 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| - s.name = 'evt-event_source-postgres' + s.name = 'evt-message_store-postgres' s.version = '0.0.0.0' - s.summary = 'Event source client for PostgreSQL' + s.summary = 'Message store for PostgreSQL' s.description = ' ' s.authors = ['The Eventide Project'] From 5a33aeab2fea7404ee5f421db0ec645c2ac2cf05 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 26 May 2017 16:13:44 -0500 Subject: [PATCH 250/591] Obsolete gemspec is removed --- message_store-postgres.__gemspec | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 message_store-postgres.__gemspec diff --git a/message_store-postgres.__gemspec b/message_store-postgres.__gemspec deleted file mode 100644 index cb25591..0000000 --- a/message_store-postgres.__gemspec +++ /dev/null @@ -1,29 +0,0 @@ -# -*- encoding: utf-8 -*- -Gem::Specification.new do |s| - s.name = 'evt-message_store-postgres' - s.version = '0.1.0.0' - s.summary = 'Event source client for PostgreSQL' - s.description = ' ' - - s.authors = ['The Eventide Project'] - s.email = 'opensource@eventide-project.org' - s.homepage = 'https://github.com/eventide-project/message-store-postgres' - s.licenses = ['MIT'] - - s.require_paths = ['lib'] - s.files = Dir.glob('{lib,database}/**/*') + ['scripts/scripts_init.rb'] - s.platform = Gem::Platform::RUBY - s.required_ruby_version = '>= 2.4.0' - - s.executables = Dir.glob('scripts/evt-*').map(&File.method(:basename)) - s.bindir = 'scripts' - - s.add_runtime_dependency 'evt-message_store' - - s.add_runtime_dependency 'evt-log' - s.add_runtime_dependency 'evt-settings' - - s.add_runtime_dependency 'pg' - - s.add_development_dependency 'test_bench' -end From 6141d1cc3dcc31c25679db050b6c103ef7563be5 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 26 May 2017 16:25:15 -0500 Subject: [PATCH 251/591] Test is renamed to reflect removal of event concept --- .../{no_further_event_data.rb => no_further_message_data.rb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/automated/iterator/{no_further_event_data.rb => no_further_message_data.rb} (100%) diff --git a/test/automated/iterator/no_further_event_data.rb b/test/automated/iterator/no_further_message_data.rb similarity index 100% rename from test/automated/iterator/no_further_event_data.rb rename to test/automated/iterator/no_further_message_data.rb From a3f3d25ad50f8895b3fcd21cc25e96284baac098 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 20:02:01 -0500 Subject: [PATCH 252/591] Table installation script reflects messages, rather than events --- database/install.sh | 6 +++--- database/table/{events-table.sql => messages-table.sql} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename database/table/{events-table.sql => messages-table.sql} (100%) diff --git a/database/install.sh b/database/install.sh index 080486f..1b900a6 100755 --- a/database/install.sh +++ b/database/install.sh @@ -7,7 +7,7 @@ echo "Installing Database" echo "= = =" echo -default_name=event_source +default_name=message_store if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set. Default will be used.)" @@ -61,9 +61,9 @@ function create-extensions { } function create-table { - echo "Creating events table..." + echo "Creating messages table..." base=$(script_dir) - psql $database -f $base/table/events-table.sql + psql $database -f $base/table/messages-table.sql echo } diff --git a/database/table/events-table.sql b/database/table/messages-table.sql similarity index 100% rename from database/table/events-table.sql rename to database/table/messages-table.sql From 74b2dbd90eaab45dc0780b8b583b40a09bb30f24 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 21:10:48 -0500 Subject: [PATCH 253/591] Events table is renamed to messages --- database/functions/stream-version.sql | 2 +- database/functions/write-event.sql | 2 +- database/indexes/events-category-global-position.sql | 2 +- database/indexes/events-category.sql | 2 +- database/indexes/events-id.sql | 2 +- database/indexes/events-stream-name-position-uniq.sql | 2 +- database/indexes/events-stream-name.sql | 2 +- database/table/messages-table.sql | 8 ++++---- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/database/functions/stream-version.sql b/database/functions/stream-version.sql index 6b88722..b9f7e47 100644 --- a/database/functions/stream-version.sql +++ b/database/functions/stream-version.sql @@ -6,7 +6,7 @@ AS $$ DECLARE stream_version int; BEGIN - select max(position) into stream_version from events where stream_name = _stream_name; + select max(position) into stream_version from messages where stream_name = _stream_name; return stream_version; END; diff --git a/database/functions/write-event.sql b/database/functions/write-event.sql index 1f67e27..1981c02 100644 --- a/database/functions/write-event.sql +++ b/database/functions/write-event.sql @@ -30,7 +30,7 @@ BEGIN position := stream_version + 1; - insert into "events" + insert into "messages" ( "id", "stream_name", diff --git a/database/indexes/events-category-global-position.sql b/database/indexes/events-category-global-position.sql index 798ba04..cd60c3c 100644 --- a/database/indexes/events-category-global-position.sql +++ b/database/indexes/events-category-global-position.sql @@ -1 +1 @@ -CREATE INDEX CONCURRENTLY "events_category_global_position_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "global_position" "pg_catalog"."int8_ops" ASC NULLS LAST); +CREATE INDEX CONCURRENTLY "messages_category_global_position_idx" ON "public"."messages" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "global_position" "pg_catalog"."int8_ops" ASC NULLS LAST); diff --git a/database/indexes/events-category.sql b/database/indexes/events-category.sql index c4f6aae..0ee9a57 100644 --- a/database/indexes/events-category.sql +++ b/database/indexes/events-category.sql @@ -1 +1 @@ -CREATE INDEX CONCURRENTLY "events_category_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); +CREATE INDEX CONCURRENTLY "messages_category_idx" ON "public"."messages" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); diff --git a/database/indexes/events-id.sql b/database/indexes/events-id.sql index 4eb428c..7ca81f3 100644 --- a/database/indexes/events-id.sql +++ b/database/indexes/events-id.sql @@ -1 +1 @@ -CREATE INDEX CONCURRENTLY "events_id_idx" ON "public"."events" USING btree(id ASC NULLS LAST); +CREATE INDEX CONCURRENTLY "messages_id_idx" ON "public"."messages" USING btree(id ASC NULLS LAST); diff --git a/database/indexes/events-stream-name-position-uniq.sql b/database/indexes/events-stream-name-position-uniq.sql index 1864048..cf79bd8 100644 --- a/database/indexes/events-stream-name-position-uniq.sql +++ b/database/indexes/events-stream-name-position-uniq.sql @@ -1 +1 @@ -CREATE UNIQUE INDEX CONCURRENTLY "events_stream_name_position_uniq_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "position" "pg_catalog"."int4_ops" ASC NULLS LAST); +CREATE UNIQUE INDEX CONCURRENTLY "messages_stream_name_position_uniq_idx" ON "public"."messages" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "position" "pg_catalog"."int4_ops" ASC NULLS LAST); diff --git a/database/indexes/events-stream-name.sql b/database/indexes/events-stream-name.sql index 7f097ba..d9d3518 100644 --- a/database/indexes/events-stream-name.sql +++ b/database/indexes/events-stream-name.sql @@ -1 +1 @@ -CREATE INDEX CONCURRENTLY "events_stream_name_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); +CREATE INDEX CONCURRENTLY "messages_stream_name_idx" ON "public"."messages" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); diff --git a/database/table/messages-table.sql b/database/table/messages-table.sql index 5192f33..94dd358 100644 --- a/database/table/messages-table.sql +++ b/database/table/messages-table.sql @@ -1,7 +1,7 @@ -- ---------------------------- --- Table structure for events +-- Table structure for messages -- ---------------------------- -CREATE TABLE "public"."events" ( +CREATE TABLE "public"."messages" ( "id" UUID NOT NULL DEFAULT gen_random_uuid(), "stream_name" varchar(255) NOT NULL COLLATE "default", "type" varchar(255) NOT NULL COLLATE "default", @@ -14,6 +14,6 @@ CREATE TABLE "public"."events" ( WITH (OIDS=FALSE); -- ---------------------------- --- Primary key structure for table events +-- Primary key structure for table messages -- ---------------------------- -ALTER TABLE "public"."events" ADD PRIMARY KEY ("global_position") NOT DEFERRABLE INITIALLY IMMEDIATE; +ALTER TABLE "public"."messages" ADD PRIMARY KEY ("global_position") NOT DEFERRABLE INITIALLY IMMEDIATE; From 6238d05a1f1ea48dfbdbab701f5854ed4bd56baa Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 21:13:24 -0500 Subject: [PATCH 254/591] Message, rather than event --- database/functions/{write-event.sql => write-message.sql} | 0 ...-global-position.sql => messages-category-global-position.sql} | 0 database/indexes/{events-category.sql => messages-category.sql} | 0 database/indexes/{events-id.sql => messages-id.sql} | 0 ...e-position-uniq.sql => messages-stream-name-position-uniq.sql} | 0 .../indexes/{events-stream-name.sql => messages-stream-name.sql} | 0 ...nt-expected-version.sql => write-message-expected-version.sql} | 0 database/test/{write-event.sql => write-message.sql} | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename database/functions/{write-event.sql => write-message.sql} (100%) rename database/indexes/{events-category-global-position.sql => messages-category-global-position.sql} (100%) rename database/indexes/{events-category.sql => messages-category.sql} (100%) rename database/indexes/{events-id.sql => messages-id.sql} (100%) rename database/indexes/{events-stream-name-position-uniq.sql => messages-stream-name-position-uniq.sql} (100%) rename database/indexes/{events-stream-name.sql => messages-stream-name.sql} (100%) rename database/test/{write-event-expected-version.sql => write-message-expected-version.sql} (100%) rename database/test/{write-event.sql => write-message.sql} (100%) diff --git a/database/functions/write-event.sql b/database/functions/write-message.sql similarity index 100% rename from database/functions/write-event.sql rename to database/functions/write-message.sql diff --git a/database/indexes/events-category-global-position.sql b/database/indexes/messages-category-global-position.sql similarity index 100% rename from database/indexes/events-category-global-position.sql rename to database/indexes/messages-category-global-position.sql diff --git a/database/indexes/events-category.sql b/database/indexes/messages-category.sql similarity index 100% rename from database/indexes/events-category.sql rename to database/indexes/messages-category.sql diff --git a/database/indexes/events-id.sql b/database/indexes/messages-id.sql similarity index 100% rename from database/indexes/events-id.sql rename to database/indexes/messages-id.sql diff --git a/database/indexes/events-stream-name-position-uniq.sql b/database/indexes/messages-stream-name-position-uniq.sql similarity index 100% rename from database/indexes/events-stream-name-position-uniq.sql rename to database/indexes/messages-stream-name-position-uniq.sql diff --git a/database/indexes/events-stream-name.sql b/database/indexes/messages-stream-name.sql similarity index 100% rename from database/indexes/events-stream-name.sql rename to database/indexes/messages-stream-name.sql diff --git a/database/test/write-event-expected-version.sql b/database/test/write-message-expected-version.sql similarity index 100% rename from database/test/write-event-expected-version.sql rename to database/test/write-message-expected-version.sql diff --git a/database/test/write-event.sql b/database/test/write-message.sql similarity index 100% rename from database/test/write-event.sql rename to database/test/write-message.sql From 318c279d80485ff75c4e39b4e3d8504c04100a22 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 21:17:06 -0500 Subject: [PATCH 255/591] Message ID, rather than event ID --- database/functions/write-message.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/database/functions/write-message.sql b/database/functions/write-message.sql index 1981c02..f9fac69 100644 --- a/database/functions/write-message.sql +++ b/database/functions/write-message.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION write_event( +CREATE OR REPLACE FUNCTION write_message( _id varchar, _stream_name varchar, _type varchar, @@ -9,12 +9,12 @@ CREATE OR REPLACE FUNCTION write_event( RETURNS int AS $$ DECLARE - event_id uuid; + message_id uuid; stream_version int; position int; category varchar; BEGIN - event_id = uuid(_id); + message_id = uuid(_id); stream_version := stream_version(_stream_name); @@ -41,7 +41,7 @@ BEGIN ) values ( - event_id, + message_id, _stream_name, position, _type, From dc752eeca795614e01da7347814983321ea9db2c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 21:21:54 -0500 Subject: [PATCH 256/591] Message, rather than event --- README.md | 2 +- database/install.sh | 22 +++++++++---------- .../test/write-message-expected-version.sql | 2 +- database/test/write-message.sql | 2 +- database/uninstall.sh | 2 +- .../postgres/get/last/select_statement.rb | 2 +- .../postgres/get/select_statement.rb | 2 +- lib/message_store/postgres/put.rb | 2 +- scripts/scripts_init.rb | 12 ---------- settings/message_store_postgres.json | 4 ++-- 10 files changed, 20 insertions(+), 32 deletions(-) delete mode 100644 scripts/scripts_init.rb diff --git a/README.md b/README.md index 8c4571f..a6bcc1f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ ## License -The `event_source-postgres` library is released under the [MIT License](https://github.com/eventide-project/event-source-postgres/blob/master/MIT-License.txt). +The `message_store-postgres` library is released under the [MIT License](https://github.com/eventide-project/event-source-postgres/blob/master/MIT-License.txt). diff --git a/database/install.sh b/database/install.sh index 1b900a6..07fa5ee 100755 --- a/database/install.sh +++ b/database/install.sh @@ -75,23 +75,23 @@ function create-functions { echo "stream_version function" psql $database -f $base/functions/stream-version.sql echo "write_sql function" - psql $database -f $base/functions/write-event.sql + psql $database -f $base/functions/write-message.sql echo } function create-indexes { base=$(script_dir) echo "Creating indexes..." - echo "events_id_idx" - psql $database -f $base/indexes/events-id.sql - echo "events_category_global_position_idx" - psql $database -f $base/indexes/events-category-global-position.sql - echo "events_category_idx" - psql $database -f $base/indexes/events-category.sql - echo "events_stream_name_idx" - psql $database -f $base/indexes/events-stream-name.sql - echo "events_stream_name_position_uniq_idx" - psql $database -f $base/indexes/events-stream-name-position-uniq.sql + echo "messages_id_idx" + psql $database -f $base/indexes/messages-id.sql + echo "messages_category_global_position_idx" + psql $database -f $base/indexes/messages-category-global-position.sql + echo "messages_category_idx" + psql $database -f $base/indexes/messages-category.sql + echo "messages_stream_name_idx" + psql $database -f $base/indexes/messages-stream-name.sql + echo "messages_stream_name_position_uniq_idx" + psql $database -f $base/indexes/messages-stream-name-position-uniq.sql echo } diff --git a/database/test/write-message-expected-version.sql b/database/test/write-message-expected-version.sql index 4ac2c20..330c5b6 100644 --- a/database/test/write-message-expected-version.sql +++ b/database/test/write-message-expected-version.sql @@ -1 +1 @@ -SELECT write_event('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}', 1); +SELECT write_message('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}', 1); diff --git a/database/test/write-message.sql b/database/test/write-message.sql index 46109b9..6f17139 100644 --- a/database/test/write-message.sql +++ b/database/test/write-message.sql @@ -1 +1 @@ -SELECT write_event('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}'); +SELECT write_message('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}'); diff --git a/database/uninstall.sh b/database/uninstall.sh index 7d9c7af..cf3117a 100755 --- a/database/uninstall.sh +++ b/database/uninstall.sh @@ -7,7 +7,7 @@ echo "Uninstalling Database" echo "= = =" echo -default_name=event_source +default_name=message_store if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set. Default will be used.)" diff --git a/lib/message_store/postgres/get/last/select_statement.rb b/lib/message_store/postgres/get/last/select_statement.rb index 80c80ab..ca0647a 100644 --- a/lib/message_store/postgres/get/last/select_statement.rb +++ b/lib/message_store/postgres/get/last/select_statement.rb @@ -25,7 +25,7 @@ def sql metadata::varchar, time::timestamp FROM - events + messages WHERE stream_name = '#{stream_name}' ORDER BY diff --git a/lib/message_store/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb index 7f5f259..7595d92 100644 --- a/lib/message_store/postgres/get/select_statement.rb +++ b/lib/message_store/postgres/get/select_statement.rb @@ -40,7 +40,7 @@ def sql metadata::varchar, time::timestamp FROM - events + messages WHERE #{where_clause_field} = '#{stream_name}' AND #{position_field} >= #{position} diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index 3040fcf..c420e93 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -88,7 +88,7 @@ def execute_query(id, stream_name, type, serialized_data, serialized_metadata, e end def self.statement - @statement ||= "SELECT write_event($1::varchar, $2::varchar, $3::varchar, $4::jsonb, $5::jsonb, $6::int);" + @statement ||= "SELECT write_message($1::varchar, $2::varchar, $3::varchar, $4::jsonb, $5::jsonb, $6::int);" end def serialized_data(data) diff --git a/scripts/scripts_init.rb b/scripts/scripts_init.rb deleted file mode 100644 index a6ecbc0..0000000 --- a/scripts/scripts_init.rb +++ /dev/null @@ -1,12 +0,0 @@ -ENV['CONSOLE_DEVICE'] ||= 'stdout' -ENV['LOG_LEVEL'] ||= '_min' - -puts RUBY_DESCRIPTION - -init_file = File.expand_path('../init.rb', __dir__) -if File.exist?(init_file) - require_relative '../init' -end - -require 'event_source/postgres' -require 'event_source/postgres/database' diff --git a/settings/message_store_postgres.json b/settings/message_store_postgres.json index 392938b..96139f8 100644 --- a/settings/message_store_postgres.json +++ b/settings/message_store_postgres.json @@ -1,9 +1,9 @@ { - "dbname": "event_source", + "dbname": "message_store", "host": "localhost", "hostaddr": "127.0.0.1", "port": 5432, - "user": "event_source", + "user": "message_store", "password": null, "connect_timeout": null, "options": null, From 3f260545da6f13ad503b3cde811333406e594410 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 21:50:37 -0500 Subject: [PATCH 257/591] Vestigial code is removed --- README.md | 4 +++- database/clear-events-table.sh | 4 ++-- database/list-events.sh | 4 ++-- library.gemspec | 33 +-------------------------------- 4 files changed, 8 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index a6bcc1f..6d24cb8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# message_store-postgres + ## License -The `message_store-postgres` library is released under the [MIT License](https://github.com/eventide-project/event-source-postgres/blob/master/MIT-License.txt). +The `message_store-postgres` library is released under the [MIT License](https://github.com/eventide-project/message-store-postgres/blob/master/MIT-License.txt). diff --git a/database/clear-events-table.sh b/database/clear-events-table.sh index 03a6bfa..4977358 100755 --- a/database/clear-events-table.sh +++ b/database/clear-events-table.sh @@ -7,7 +7,7 @@ echo "Clearing Events Table" echo "= = =" echo -default_name=event_source +default_name=message_store if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" @@ -25,7 +25,7 @@ else fi echo "Database name is: $database" -default_table_name=events +default_table_name=messages if [ -z ${TABLE_NAME+x} ]; then echo "(TABLE_NAME is not set)" diff --git a/database/list-events.sh b/database/list-events.sh index 08f2475..9043f5b 100755 --- a/database/list-events.sh +++ b/database/list-events.sh @@ -7,7 +7,7 @@ echo "Listing Events" echo "= = =" echo -default_name=event_source +default_name=message_store if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" @@ -25,7 +25,7 @@ else fi echo "Database name is: $database" -default_table_name=events +default_table_name=messages if [ -z ${TABLE_NAME+x} ]; then echo "(TABLE_NAME is not set)" diff --git a/library.gemspec b/library.gemspec index 4316278..5e8d858 100644 --- a/library.gemspec +++ b/library.gemspec @@ -1,39 +1,8 @@ -# # -*- encoding: utf-8 -*- -# Gem::Specification.new do |s| -# s.name = 'evt-message_store-postgres' -# s.version = '0.0.0.0' -# s.summary = 'Message store for PostgreSQL' -# s.description = ' ' - -# s.authors = ['The Eventide Project'] -# s.email = 'opensource@eventide-project.org' -# s.homepage = 'https://github.com/eventide-project/event-source-postgres' -# s.licenses = ['MIT'] - -# s.require_paths = ['lib'] -# s.files = Dir.glob('{lib,database}/**/*') + ['scripts/scripts_init.rb'] -# s.platform = Gem::Platform::RUBY -# s.required_ruby_version = '>= 2.3.3' - -# s.executables = Dir.glob('scripts/evt-*').map(&File.method(:basename)) -# s.bindir = 'scripts' - -# s.add_runtime_dependency 'evt-event_source' -# # s.add_runtime_dependency 'evt-message_store' -# s.add_runtime_dependency 'evt-log' -# s.add_runtime_dependency 'evt-settings' - -# s.add_runtime_dependency 'pg' - -# s.add_development_dependency 'test_bench' -# end - - # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' s.version = '0.1.0.0' - s.summary = 'Event source client for PostgreSQL' + s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' s.authors = ['The Eventide Project'] From bb811f7e9d12af6888c0c5a0f38a8c0d9e022f80 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 22:06:32 -0500 Subject: [PATCH 258/591] Package version is increased from 0.1.0.0 to 0.2.0.0 --- library.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library.gemspec b/library.gemspec index 5e8d858..bcaf8d3 100644 --- a/library.gemspec +++ b/library.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.1.0.0' + s.version = '0.2.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.licenses = ['MIT'] s.require_paths = ['lib'] - s.files = Dir.glob('{lib,database}/**/*') + ['scripts/scripts_init.rb'] + s.files = Dir.glob('{lib,database}/**/*') s.platform = Gem::Platform::RUBY s.required_ruby_version = '>= 2.4.0' From f087eafa722b68a39af75483a7c6824f21388b38 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 20:02:01 -0500 Subject: [PATCH 259/591] Table installation script reflects messages, rather than events --- database/install.sh | 6 +++--- database/table/{events-table.sql => messages-table.sql} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename database/table/{events-table.sql => messages-table.sql} (100%) diff --git a/database/install.sh b/database/install.sh index 080486f..1b900a6 100755 --- a/database/install.sh +++ b/database/install.sh @@ -7,7 +7,7 @@ echo "Installing Database" echo "= = =" echo -default_name=event_source +default_name=message_store if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set. Default will be used.)" @@ -61,9 +61,9 @@ function create-extensions { } function create-table { - echo "Creating events table..." + echo "Creating messages table..." base=$(script_dir) - psql $database -f $base/table/events-table.sql + psql $database -f $base/table/messages-table.sql echo } diff --git a/database/table/events-table.sql b/database/table/messages-table.sql similarity index 100% rename from database/table/events-table.sql rename to database/table/messages-table.sql From 3fea19e09ef06561436a2f31f674d71d57fb27e3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 21:10:48 -0500 Subject: [PATCH 260/591] Events table is renamed to messages --- database/functions/stream-version.sql | 2 +- database/functions/write-event.sql | 2 +- database/indexes/events-category-global-position.sql | 2 +- database/indexes/events-category.sql | 2 +- database/indexes/events-id.sql | 2 +- database/indexes/events-stream-name-position-uniq.sql | 2 +- database/indexes/events-stream-name.sql | 2 +- database/table/messages-table.sql | 8 ++++---- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/database/functions/stream-version.sql b/database/functions/stream-version.sql index 6b88722..b9f7e47 100644 --- a/database/functions/stream-version.sql +++ b/database/functions/stream-version.sql @@ -6,7 +6,7 @@ AS $$ DECLARE stream_version int; BEGIN - select max(position) into stream_version from events where stream_name = _stream_name; + select max(position) into stream_version from messages where stream_name = _stream_name; return stream_version; END; diff --git a/database/functions/write-event.sql b/database/functions/write-event.sql index 1f67e27..1981c02 100644 --- a/database/functions/write-event.sql +++ b/database/functions/write-event.sql @@ -30,7 +30,7 @@ BEGIN position := stream_version + 1; - insert into "events" + insert into "messages" ( "id", "stream_name", diff --git a/database/indexes/events-category-global-position.sql b/database/indexes/events-category-global-position.sql index 798ba04..cd60c3c 100644 --- a/database/indexes/events-category-global-position.sql +++ b/database/indexes/events-category-global-position.sql @@ -1 +1 @@ -CREATE INDEX CONCURRENTLY "events_category_global_position_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "global_position" "pg_catalog"."int8_ops" ASC NULLS LAST); +CREATE INDEX CONCURRENTLY "messages_category_global_position_idx" ON "public"."messages" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "global_position" "pg_catalog"."int8_ops" ASC NULLS LAST); diff --git a/database/indexes/events-category.sql b/database/indexes/events-category.sql index c4f6aae..0ee9a57 100644 --- a/database/indexes/events-category.sql +++ b/database/indexes/events-category.sql @@ -1 +1 @@ -CREATE INDEX CONCURRENTLY "events_category_idx" ON "public"."events" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); +CREATE INDEX CONCURRENTLY "messages_category_idx" ON "public"."messages" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); diff --git a/database/indexes/events-id.sql b/database/indexes/events-id.sql index 4eb428c..7ca81f3 100644 --- a/database/indexes/events-id.sql +++ b/database/indexes/events-id.sql @@ -1 +1 @@ -CREATE INDEX CONCURRENTLY "events_id_idx" ON "public"."events" USING btree(id ASC NULLS LAST); +CREATE INDEX CONCURRENTLY "messages_id_idx" ON "public"."messages" USING btree(id ASC NULLS LAST); diff --git a/database/indexes/events-stream-name-position-uniq.sql b/database/indexes/events-stream-name-position-uniq.sql index 1864048..cf79bd8 100644 --- a/database/indexes/events-stream-name-position-uniq.sql +++ b/database/indexes/events-stream-name-position-uniq.sql @@ -1 +1 @@ -CREATE UNIQUE INDEX CONCURRENTLY "events_stream_name_position_uniq_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "position" "pg_catalog"."int4_ops" ASC NULLS LAST); +CREATE UNIQUE INDEX CONCURRENTLY "messages_stream_name_position_uniq_idx" ON "public"."messages" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "position" "pg_catalog"."int4_ops" ASC NULLS LAST); diff --git a/database/indexes/events-stream-name.sql b/database/indexes/events-stream-name.sql index 7f097ba..d9d3518 100644 --- a/database/indexes/events-stream-name.sql +++ b/database/indexes/events-stream-name.sql @@ -1 +1 @@ -CREATE INDEX CONCURRENTLY "events_stream_name_idx" ON "public"."events" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); +CREATE INDEX CONCURRENTLY "messages_stream_name_idx" ON "public"."messages" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); diff --git a/database/table/messages-table.sql b/database/table/messages-table.sql index 5192f33..94dd358 100644 --- a/database/table/messages-table.sql +++ b/database/table/messages-table.sql @@ -1,7 +1,7 @@ -- ---------------------------- --- Table structure for events +-- Table structure for messages -- ---------------------------- -CREATE TABLE "public"."events" ( +CREATE TABLE "public"."messages" ( "id" UUID NOT NULL DEFAULT gen_random_uuid(), "stream_name" varchar(255) NOT NULL COLLATE "default", "type" varchar(255) NOT NULL COLLATE "default", @@ -14,6 +14,6 @@ CREATE TABLE "public"."events" ( WITH (OIDS=FALSE); -- ---------------------------- --- Primary key structure for table events +-- Primary key structure for table messages -- ---------------------------- -ALTER TABLE "public"."events" ADD PRIMARY KEY ("global_position") NOT DEFERRABLE INITIALLY IMMEDIATE; +ALTER TABLE "public"."messages" ADD PRIMARY KEY ("global_position") NOT DEFERRABLE INITIALLY IMMEDIATE; From d265449983728134e5ceb52c44a19a9a3032f2eb Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 21:13:24 -0500 Subject: [PATCH 261/591] Message, rather than event --- database/functions/{write-event.sql => write-message.sql} | 0 ...-global-position.sql => messages-category-global-position.sql} | 0 database/indexes/{events-category.sql => messages-category.sql} | 0 database/indexes/{events-id.sql => messages-id.sql} | 0 ...e-position-uniq.sql => messages-stream-name-position-uniq.sql} | 0 .../indexes/{events-stream-name.sql => messages-stream-name.sql} | 0 ...nt-expected-version.sql => write-message-expected-version.sql} | 0 database/test/{write-event.sql => write-message.sql} | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename database/functions/{write-event.sql => write-message.sql} (100%) rename database/indexes/{events-category-global-position.sql => messages-category-global-position.sql} (100%) rename database/indexes/{events-category.sql => messages-category.sql} (100%) rename database/indexes/{events-id.sql => messages-id.sql} (100%) rename database/indexes/{events-stream-name-position-uniq.sql => messages-stream-name-position-uniq.sql} (100%) rename database/indexes/{events-stream-name.sql => messages-stream-name.sql} (100%) rename database/test/{write-event-expected-version.sql => write-message-expected-version.sql} (100%) rename database/test/{write-event.sql => write-message.sql} (100%) diff --git a/database/functions/write-event.sql b/database/functions/write-message.sql similarity index 100% rename from database/functions/write-event.sql rename to database/functions/write-message.sql diff --git a/database/indexes/events-category-global-position.sql b/database/indexes/messages-category-global-position.sql similarity index 100% rename from database/indexes/events-category-global-position.sql rename to database/indexes/messages-category-global-position.sql diff --git a/database/indexes/events-category.sql b/database/indexes/messages-category.sql similarity index 100% rename from database/indexes/events-category.sql rename to database/indexes/messages-category.sql diff --git a/database/indexes/events-id.sql b/database/indexes/messages-id.sql similarity index 100% rename from database/indexes/events-id.sql rename to database/indexes/messages-id.sql diff --git a/database/indexes/events-stream-name-position-uniq.sql b/database/indexes/messages-stream-name-position-uniq.sql similarity index 100% rename from database/indexes/events-stream-name-position-uniq.sql rename to database/indexes/messages-stream-name-position-uniq.sql diff --git a/database/indexes/events-stream-name.sql b/database/indexes/messages-stream-name.sql similarity index 100% rename from database/indexes/events-stream-name.sql rename to database/indexes/messages-stream-name.sql diff --git a/database/test/write-event-expected-version.sql b/database/test/write-message-expected-version.sql similarity index 100% rename from database/test/write-event-expected-version.sql rename to database/test/write-message-expected-version.sql diff --git a/database/test/write-event.sql b/database/test/write-message.sql similarity index 100% rename from database/test/write-event.sql rename to database/test/write-message.sql From cb640badaf7e4dae0edde7664178880c5142d9cc Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 21:17:06 -0500 Subject: [PATCH 262/591] Message ID, rather than event ID --- database/functions/write-message.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/database/functions/write-message.sql b/database/functions/write-message.sql index 1981c02..f9fac69 100644 --- a/database/functions/write-message.sql +++ b/database/functions/write-message.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION write_event( +CREATE OR REPLACE FUNCTION write_message( _id varchar, _stream_name varchar, _type varchar, @@ -9,12 +9,12 @@ CREATE OR REPLACE FUNCTION write_event( RETURNS int AS $$ DECLARE - event_id uuid; + message_id uuid; stream_version int; position int; category varchar; BEGIN - event_id = uuid(_id); + message_id = uuid(_id); stream_version := stream_version(_stream_name); @@ -41,7 +41,7 @@ BEGIN ) values ( - event_id, + message_id, _stream_name, position, _type, From 2db9f80f60212d8ff8202ad863bbf3ac91187bb5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 21:21:54 -0500 Subject: [PATCH 263/591] Message, rather than event --- README.md | 2 +- database/install.sh | 22 +++++++++---------- .../test/write-message-expected-version.sql | 2 +- database/test/write-message.sql | 2 +- database/uninstall.sh | 2 +- .../postgres/get/last/select_statement.rb | 2 +- .../postgres/get/select_statement.rb | 2 +- lib/message_store/postgres/put.rb | 2 +- scripts/scripts_init.rb | 12 ---------- settings/message_store_postgres.json | 4 ++-- 10 files changed, 20 insertions(+), 32 deletions(-) delete mode 100644 scripts/scripts_init.rb diff --git a/README.md b/README.md index 8c4571f..a6bcc1f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ ## License -The `event_source-postgres` library is released under the [MIT License](https://github.com/eventide-project/event-source-postgres/blob/master/MIT-License.txt). +The `message_store-postgres` library is released under the [MIT License](https://github.com/eventide-project/event-source-postgres/blob/master/MIT-License.txt). diff --git a/database/install.sh b/database/install.sh index 1b900a6..07fa5ee 100755 --- a/database/install.sh +++ b/database/install.sh @@ -75,23 +75,23 @@ function create-functions { echo "stream_version function" psql $database -f $base/functions/stream-version.sql echo "write_sql function" - psql $database -f $base/functions/write-event.sql + psql $database -f $base/functions/write-message.sql echo } function create-indexes { base=$(script_dir) echo "Creating indexes..." - echo "events_id_idx" - psql $database -f $base/indexes/events-id.sql - echo "events_category_global_position_idx" - psql $database -f $base/indexes/events-category-global-position.sql - echo "events_category_idx" - psql $database -f $base/indexes/events-category.sql - echo "events_stream_name_idx" - psql $database -f $base/indexes/events-stream-name.sql - echo "events_stream_name_position_uniq_idx" - psql $database -f $base/indexes/events-stream-name-position-uniq.sql + echo "messages_id_idx" + psql $database -f $base/indexes/messages-id.sql + echo "messages_category_global_position_idx" + psql $database -f $base/indexes/messages-category-global-position.sql + echo "messages_category_idx" + psql $database -f $base/indexes/messages-category.sql + echo "messages_stream_name_idx" + psql $database -f $base/indexes/messages-stream-name.sql + echo "messages_stream_name_position_uniq_idx" + psql $database -f $base/indexes/messages-stream-name-position-uniq.sql echo } diff --git a/database/test/write-message-expected-version.sql b/database/test/write-message-expected-version.sql index 4ac2c20..330c5b6 100644 --- a/database/test/write-message-expected-version.sql +++ b/database/test/write-message-expected-version.sql @@ -1 +1 @@ -SELECT write_event('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}', 1); +SELECT write_message('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}', 1); diff --git a/database/test/write-message.sql b/database/test/write-message.sql index 46109b9..6f17139 100644 --- a/database/test/write-message.sql +++ b/database/test/write-message.sql @@ -1 +1 @@ -SELECT write_event('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}'); +SELECT write_message('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}'); diff --git a/database/uninstall.sh b/database/uninstall.sh index 7d9c7af..cf3117a 100755 --- a/database/uninstall.sh +++ b/database/uninstall.sh @@ -7,7 +7,7 @@ echo "Uninstalling Database" echo "= = =" echo -default_name=event_source +default_name=message_store if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set. Default will be used.)" diff --git a/lib/message_store/postgres/get/last/select_statement.rb b/lib/message_store/postgres/get/last/select_statement.rb index 80c80ab..ca0647a 100644 --- a/lib/message_store/postgres/get/last/select_statement.rb +++ b/lib/message_store/postgres/get/last/select_statement.rb @@ -25,7 +25,7 @@ def sql metadata::varchar, time::timestamp FROM - events + messages WHERE stream_name = '#{stream_name}' ORDER BY diff --git a/lib/message_store/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb index 7f5f259..7595d92 100644 --- a/lib/message_store/postgres/get/select_statement.rb +++ b/lib/message_store/postgres/get/select_statement.rb @@ -40,7 +40,7 @@ def sql metadata::varchar, time::timestamp FROM - events + messages WHERE #{where_clause_field} = '#{stream_name}' AND #{position_field} >= #{position} diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index 3040fcf..c420e93 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -88,7 +88,7 @@ def execute_query(id, stream_name, type, serialized_data, serialized_metadata, e end def self.statement - @statement ||= "SELECT write_event($1::varchar, $2::varchar, $3::varchar, $4::jsonb, $5::jsonb, $6::int);" + @statement ||= "SELECT write_message($1::varchar, $2::varchar, $3::varchar, $4::jsonb, $5::jsonb, $6::int);" end def serialized_data(data) diff --git a/scripts/scripts_init.rb b/scripts/scripts_init.rb deleted file mode 100644 index a6ecbc0..0000000 --- a/scripts/scripts_init.rb +++ /dev/null @@ -1,12 +0,0 @@ -ENV['CONSOLE_DEVICE'] ||= 'stdout' -ENV['LOG_LEVEL'] ||= '_min' - -puts RUBY_DESCRIPTION - -init_file = File.expand_path('../init.rb', __dir__) -if File.exist?(init_file) - require_relative '../init' -end - -require 'event_source/postgres' -require 'event_source/postgres/database' diff --git a/settings/message_store_postgres.json b/settings/message_store_postgres.json index 392938b..96139f8 100644 --- a/settings/message_store_postgres.json +++ b/settings/message_store_postgres.json @@ -1,9 +1,9 @@ { - "dbname": "event_source", + "dbname": "message_store", "host": "localhost", "hostaddr": "127.0.0.1", "port": 5432, - "user": "event_source", + "user": "message_store", "password": null, "connect_timeout": null, "options": null, From c35ecbeb13e7d563a1f3db2ab1da3821e4afb92d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 21:50:37 -0500 Subject: [PATCH 264/591] Vestigial code is removed --- README.md | 4 +++- database/clear-events-table.sh | 4 ++-- database/list-events.sh | 4 ++-- library.gemspec | 33 +-------------------------------- 4 files changed, 8 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index a6bcc1f..6d24cb8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# message_store-postgres + ## License -The `message_store-postgres` library is released under the [MIT License](https://github.com/eventide-project/event-source-postgres/blob/master/MIT-License.txt). +The `message_store-postgres` library is released under the [MIT License](https://github.com/eventide-project/message-store-postgres/blob/master/MIT-License.txt). diff --git a/database/clear-events-table.sh b/database/clear-events-table.sh index 03a6bfa..4977358 100755 --- a/database/clear-events-table.sh +++ b/database/clear-events-table.sh @@ -7,7 +7,7 @@ echo "Clearing Events Table" echo "= = =" echo -default_name=event_source +default_name=message_store if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" @@ -25,7 +25,7 @@ else fi echo "Database name is: $database" -default_table_name=events +default_table_name=messages if [ -z ${TABLE_NAME+x} ]; then echo "(TABLE_NAME is not set)" diff --git a/database/list-events.sh b/database/list-events.sh index 08f2475..9043f5b 100755 --- a/database/list-events.sh +++ b/database/list-events.sh @@ -7,7 +7,7 @@ echo "Listing Events" echo "= = =" echo -default_name=event_source +default_name=message_store if [ -z ${DATABASE_USER+x} ]; then echo "(DATABASE_USER is not set)" @@ -25,7 +25,7 @@ else fi echo "Database name is: $database" -default_table_name=events +default_table_name=messages if [ -z ${TABLE_NAME+x} ]; then echo "(TABLE_NAME is not set)" diff --git a/library.gemspec b/library.gemspec index 4316278..5e8d858 100644 --- a/library.gemspec +++ b/library.gemspec @@ -1,39 +1,8 @@ -# # -*- encoding: utf-8 -*- -# Gem::Specification.new do |s| -# s.name = 'evt-message_store-postgres' -# s.version = '0.0.0.0' -# s.summary = 'Message store for PostgreSQL' -# s.description = ' ' - -# s.authors = ['The Eventide Project'] -# s.email = 'opensource@eventide-project.org' -# s.homepage = 'https://github.com/eventide-project/event-source-postgres' -# s.licenses = ['MIT'] - -# s.require_paths = ['lib'] -# s.files = Dir.glob('{lib,database}/**/*') + ['scripts/scripts_init.rb'] -# s.platform = Gem::Platform::RUBY -# s.required_ruby_version = '>= 2.3.3' - -# s.executables = Dir.glob('scripts/evt-*').map(&File.method(:basename)) -# s.bindir = 'scripts' - -# s.add_runtime_dependency 'evt-event_source' -# # s.add_runtime_dependency 'evt-message_store' -# s.add_runtime_dependency 'evt-log' -# s.add_runtime_dependency 'evt-settings' - -# s.add_runtime_dependency 'pg' - -# s.add_development_dependency 'test_bench' -# end - - # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' s.version = '0.1.0.0' - s.summary = 'Event source client for PostgreSQL' + s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' s.authors = ['The Eventide Project'] From 448015d2ec510ec14188d3a5a599d2e1c8c5c450 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 22:06:32 -0500 Subject: [PATCH 265/591] Package version is increased from 0.1.0.0 to 0.2.0.0 --- library.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library.gemspec b/library.gemspec index 5e8d858..bcaf8d3 100644 --- a/library.gemspec +++ b/library.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.1.0.0' + s.version = '0.2.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.licenses = ['MIT'] s.require_paths = ['lib'] - s.files = Dir.glob('{lib,database}/**/*') + ['scripts/scripts_init.rb'] + s.files = Dir.glob('{lib,database}/**/*') s.platform = Gem::Platform::RUBY s.required_ruby_version = '>= 2.4.0' From 9e537c9f7dced29e3f53fac231ba428b27fd2a6f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 23:11:31 -0500 Subject: [PATCH 266/591] List messages, rather than list events --- database/{list-events.sh => list-messages.sh} | 2 +- scripts/{evt-pg-list-events => evt-pg-list-messages} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename database/{list-events.sh => list-messages.sh} (97%) rename scripts/{evt-pg-list-events => evt-pg-list-messages} (79%) diff --git a/database/list-events.sh b/database/list-messages.sh similarity index 97% rename from database/list-events.sh rename to database/list-messages.sh index 9043f5b..8f637a9 100755 --- a/database/list-events.sh +++ b/database/list-messages.sh @@ -3,7 +3,7 @@ set -e echo -echo "Listing Events" +echo "Listing Messages" echo "= = =" echo diff --git a/scripts/evt-pg-list-events b/scripts/evt-pg-list-messages similarity index 79% rename from scripts/evt-pg-list-events rename to scripts/evt-pg-list-messages index 21e3f07..b7ab7fd 100755 --- a/scripts/evt-pg-list-events +++ b/scripts/evt-pg-list-messages @@ -1,7 +1,7 @@ #!/usr/bin/env ruby root = File.expand_path '../database', __dir__ -script_filename = 'list-events.sh' +script_filename = 'list-messages.sh' script_filepath = File.join root, script_filename system script_filepath From d231fdeadc1ed231a0835042980001f6b73c594b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 May 2017 23:13:24 -0500 Subject: [PATCH 267/591] Package version is increased from 0.2.0.0 to 0.3.0.0 --- library.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library.gemspec b/library.gemspec index bcaf8d3..3b32965 100644 --- a/library.gemspec +++ b/library.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.2.0.0' + s.version = '0.3.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 764b7642568b42f48c024f5eddf53168d5ba2446 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 31 May 2017 16:45:12 -0500 Subject: [PATCH 268/591] Gemspec file is named for the gem --- library.gemspec => message_store-postgres.gemspec | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename library.gemspec => message_store-postgres.gemspec (100%) diff --git a/library.gemspec b/message_store-postgres.gemspec similarity index 100% rename from library.gemspec rename to message_store-postgres.gemspec From 4d5cf26399f9cb363676e556d0bad61a02017786 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 31 May 2017 16:45:53 -0500 Subject: [PATCH 269/591] Package version is increased from 0.3.0.0 to 0.3.0.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 3b32965..f637d87 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.3.0.0' + s.version = '0.3.0.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 223017a4f409aba6da2cf604d13f96d407aa4de2 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 31 May 2017 21:04:04 -0500 Subject: [PATCH 270/591] Package version is increased from 0.3.0.1 to 0.3.0.2 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index f637d87..8c6cb59 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.3.0.1' + s.version = '0.3.0.2' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 17fd87d100116f8dfa7456e6e4dd387f5ba347b9 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Sep 2017 16:30:11 -0500 Subject: [PATCH 271/591] Position is a 64-bit integer --- database/functions/stream-version.sql | 4 ++-- database/functions/write-message.sql | 8 ++++---- .../messages-stream-name-position-uniq.sql | 2 +- database/table/messages-table.sql | 2 +- lib/message_store/postgres/controls.rb | 1 + lib/message_store/postgres/controls/position.rb | 15 +++++++++++++++ .../postgres/controls/stream_name.rb | 8 -------- 7 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 lib/message_store/postgres/controls/position.rb diff --git a/database/functions/stream-version.sql b/database/functions/stream-version.sql index b9f7e47..4f9abf3 100644 --- a/database/functions/stream-version.sql +++ b/database/functions/stream-version.sql @@ -1,10 +1,10 @@ CREATE OR REPLACE FUNCTION stream_version( _stream_name varchar ) -RETURNS int +RETURNS bigint AS $$ DECLARE - stream_version int; + stream_version bigint; BEGIN select max(position) into stream_version from messages where stream_name = _stream_name; diff --git a/database/functions/write-message.sql b/database/functions/write-message.sql index f9fac69..a16f45e 100644 --- a/database/functions/write-message.sql +++ b/database/functions/write-message.sql @@ -4,14 +4,14 @@ CREATE OR REPLACE FUNCTION write_message( _type varchar, _data jsonb, _metadata jsonb DEFAULT NULL, - _expected_version int DEFAULT NULL + _expected_version bigint DEFAULT NULL ) -RETURNS int +RETURNS bigint AS $$ DECLARE message_id uuid; - stream_version int; - position int; + stream_version bigint; + position bigint; category varchar; BEGIN message_id = uuid(_id); diff --git a/database/indexes/messages-stream-name-position-uniq.sql b/database/indexes/messages-stream-name-position-uniq.sql index cf79bd8..7cd8d47 100644 --- a/database/indexes/messages-stream-name-position-uniq.sql +++ b/database/indexes/messages-stream-name-position-uniq.sql @@ -1 +1 @@ -CREATE UNIQUE INDEX CONCURRENTLY "messages_stream_name_position_uniq_idx" ON "public"."messages" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "position" "pg_catalog"."int4_ops" ASC NULLS LAST); +CREATE UNIQUE INDEX CONCURRENTLY "messages_stream_name_position_uniq_idx" ON "public"."messages" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "position" "pg_catalog"."int8_ops" ASC NULLS LAST); diff --git a/database/table/messages-table.sql b/database/table/messages-table.sql index 94dd358..f5d5ae1 100644 --- a/database/table/messages-table.sql +++ b/database/table/messages-table.sql @@ -5,7 +5,7 @@ CREATE TABLE "public"."messages" ( "id" UUID NOT NULL DEFAULT gen_random_uuid(), "stream_name" varchar(255) NOT NULL COLLATE "default", "type" varchar(255) NOT NULL COLLATE "default", - "position" int4 NOT NULL, + "position" bigint NOT NULL, "global_position" bigserial NOT NULL , "data" jsonb, "metadata" jsonb, diff --git a/lib/message_store/postgres/controls.rb b/lib/message_store/postgres/controls.rb index 3b51255..847c5d0 100644 --- a/lib/message_store/postgres/controls.rb +++ b/lib/message_store/postgres/controls.rb @@ -1,5 +1,6 @@ require 'message_store/controls' +require 'message_store/postgres/controls/position' require 'message_store/postgres/controls/category' require 'message_store/postgres/controls/stream_name' require 'message_store/postgres/controls/message_data' diff --git a/lib/message_store/postgres/controls/position.rb b/lib/message_store/postgres/controls/position.rb new file mode 100644 index 0000000..5b5c5d0 --- /dev/null +++ b/lib/message_store/postgres/controls/position.rb @@ -0,0 +1,15 @@ +module MessageStore + module Postgres + module Controls + module Position + def self.example + 1 + end + + def self.max + (2 ** 63) - 1 + end + end + end + end +end diff --git a/lib/message_store/postgres/controls/stream_name.rb b/lib/message_store/postgres/controls/stream_name.rb index f928339..70e854a 100644 --- a/lib/message_store/postgres/controls/stream_name.rb +++ b/lib/message_store/postgres/controls/stream_name.rb @@ -1,11 +1,3 @@ -# module MessageStore -# module Postgres -# module Controls -# StreamName = MessageStore::Controls::StreamName -# end -# end -# end - module MessageStore module Postgres module Controls From 29d1d346a521fab20d28a7979aed80aa73e9677c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Sep 2017 16:31:29 -0500 Subject: [PATCH 272/591] Package version is increased from 0.3.0.2 to 0.4.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 8c6cb59..1c77fce 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.3.0.2' + s.version = '0.4.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 90e615d4d9cf7b0f69833e4e65fbf4014c63351d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Sep 2017 17:00:00 -0500 Subject: [PATCH 273/591] Expected version parameter binding is a bigint --- lib/message_store/postgres/put.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index c420e93..de4a548 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -88,7 +88,7 @@ def execute_query(id, stream_name, type, serialized_data, serialized_metadata, e end def self.statement - @statement ||= "SELECT write_message($1::varchar, $2::varchar, $3::varchar, $4::jsonb, $5::jsonb, $6::int);" + @statement ||= "SELECT write_message($1::varchar, $2::varchar, $3::varchar, $4::jsonb, $5::jsonb, $6::bigint);" end def serialized_data(data) From ac85ddbd1a35d9834445b13f29b6df183be65ce8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Sep 2017 17:00:45 -0500 Subject: [PATCH 274/591] Package version is increased from 0.4.0.0 to 0.4.0.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 1c77fce..e92f062 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.4.0.0' + s.version = '0.4.0.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From fcb2c9368f452e269354c9849b4cf12405395bc5 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 29 Sep 2017 14:37:08 -0500 Subject: [PATCH 275/591] Get::Last includes generic MessageStore::Get::Last module --- lib/message_store/postgres/get/last.rb | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/message_store/postgres/get/last.rb b/lib/message_store/postgres/get/last.rb index 6033c8a..900c6ef 100644 --- a/lib/message_store/postgres/get/last.rb +++ b/lib/message_store/postgres/get/last.rb @@ -2,16 +2,10 @@ module MessageStore module Postgres class Get class Last - include Log::Dependency + include MessageStore::Get::Last dependency :session, Session - def self.build(session: nil) - new.tap do |instance| - instance.configure(session: session) - end - end - def self.configure(receiver, attr_name: nil, session: nil) attr_name ||= :get_last instance = build(session: session) @@ -19,7 +13,7 @@ def self.configure(receiver, attr_name: nil, session: nil) end def configure(session: nil) - Session.configure self, session: session + Session.configure(self, session: session) end def self.call(stream_name, session: nil) From 7946eb6cec502841144950ae7b950bb0a5300205 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 29 Sep 2017 14:39:28 -0500 Subject: [PATCH 276/591] Class call and configure methods are removed from Get::Last --- lib/message_store/postgres/get/last.rb | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/message_store/postgres/get/last.rb b/lib/message_store/postgres/get/last.rb index 900c6ef..bd47a55 100644 --- a/lib/message_store/postgres/get/last.rb +++ b/lib/message_store/postgres/get/last.rb @@ -6,21 +6,10 @@ class Last dependency :session, Session - def self.configure(receiver, attr_name: nil, session: nil) - attr_name ||= :get_last - instance = build(session: session) - receiver.public_send "#{attr_name}=", instance - end - def configure(session: nil) Session.configure(self, session: session) end - def self.call(stream_name, session: nil) - instance = build(session: session) - instance.(stream_name) - end - def call(stream_name) logger.trace { "Getting last message data (Stream Name: #{stream_name})" } From 8a60bc9ac4e6846fe218d7686bb5862bae626358 Mon Sep 17 00:00:00 2001 From: ntl Date: Fri, 29 Sep 2017 14:39:54 -0500 Subject: [PATCH 277/591] Package version is increased from 0.4.0.1 to 0.4.1.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index e92f062..bb57c80 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.4.0.1' + s.version = '0.4.1.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 35ebafb258d66b454b627ca12c42b4c17af371b3 Mon Sep 17 00:00:00 2001 From: ntl Date: Wed, 18 Oct 2017 11:26:17 -0500 Subject: [PATCH 278/591] SelectStatement interpolates optional where fragment --- .../postgres/get/select_statement.rb | 26 +++++++++++++---- .../select_statement/where_fragment.rb | 28 +++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 test/automated/select_statement/where_fragment.rb diff --git a/lib/message_store/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb index 7595d92..d147740 100644 --- a/lib/message_store/postgres/get/select_statement.rb +++ b/lib/message_store/postgres/get/select_statement.rb @@ -4,7 +4,7 @@ class Get class SelectStatement include Log::Dependency - initializer :stream_name, w(:position), w(:batch_size) + initializer :stream_name, w(:position), w(:batch_size), :where_fragment def position @position ||= Defaults.position @@ -22,14 +22,16 @@ def category_stream? is_category_stream ||= StreamName.category?(stream_name) end - def self.build(stream_name, position: nil, batch_size: nil) - new(stream_name, position, batch_size) + def self.build(stream_name, position: nil, batch_size: nil, where_fragment: nil) + new(stream_name, position, batch_size, where_fragment) end def sql logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{category_stream?}, Types: #{stream_type_list.inspect}, Position: #{position}, Batch Size: #{batch_size})" } - statement = <<-SQL + formatted_where_clause = where_clause.each_line.to_a.join(" ") + + statement = <<~SQL SELECT id::varchar, stream_name::varchar, @@ -42,8 +44,7 @@ def sql FROM messages WHERE - #{where_clause_field} = '#{stream_name}' AND - #{position_field} >= #{position} + #{formatted_where_clause} ORDER BY #{position_field} ASC LIMIT @@ -57,6 +58,19 @@ def sql statement end + def where_clause + clause = <<~SQL.chomp + #{where_clause_field} = '#{stream_name}' AND + #{position_field} >= #{position} + SQL + + unless where_fragment.nil? + clause << " AND\n(#{where_fragment})" + end + + clause + end + def where_clause_field unless category_stream? 'stream_name' diff --git a/test/automated/select_statement/where_fragment.rb b/test/automated/select_statement/where_fragment.rb new file mode 100644 index 0000000..b5d1ac0 --- /dev/null +++ b/test/automated/select_statement/where_fragment.rb @@ -0,0 +1,28 @@ +require_relative '../automated_init' + +context "Select Statement" do + context "Where Fragment" do + stream_name = Controls::StreamName.example + + where_fragment = '1 = 1' + + select_statement = Get::SelectStatement.build(stream_name, where_fragment: where_fragment) + + sql = select_statement.sql + sql.gsub!(/\s+/, ' ') + + context "Where Clause" do + test "Filters on stream name" do + assert(sql.include? 'WHERE stream_name =') + end + + test "Filters on position" do + assert(sql.include? 'position >=') + end + + test "Includes where fragment encased in parentheses" do + assert(sql.include? "AND (#{where_fragment})") + end + end + end +end From e83d5a8fad69442251c53f76810714b5bd11909c Mon Sep 17 00:00:00 2001 From: ntl Date: Wed, 18 Oct 2017 14:00:52 -0500 Subject: [PATCH 279/591] Get accepts condition argument --- lib/message_store/postgres/get.rb | 18 ++++++++++-------- test/automated/get/condition.rb | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 test/automated/get/condition.rb diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 26ac05f..bac0774 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -3,19 +3,19 @@ module Postgres class Get include MessageStore::Get - initializer :batch_size + initializer :batch_size, :condition dependency :session, Session - def self.build(batch_size: nil, session: nil) - new(batch_size).tap do |instance| + def self.build(batch_size: nil, session: nil, condition: nil) + new(batch_size, condition).tap do |instance| instance.configure(session: session) end end - def self.configure(receiver, attr_name: nil, position: nil, batch_size: nil, session: nil) + def self.configure(receiver, attr_name: nil, position: nil, batch_size: nil, condition: nil, session: nil) attr_name ||= :get - instance = build(batch_size: batch_size, session: session) + instance = build(batch_size: batch_size, condition: condition, session: session) receiver.public_send "#{attr_name}=", instance end @@ -23,8 +23,8 @@ def configure(session: nil) Session.configure self, session: session end - def self.call(stream_name, position: nil, batch_size: nil, session: nil) - instance = build(batch_size: batch_size, session: session) + def self.call(stream_name, position: nil, batch_size: nil, condition: nil, session: nil) + instance = build(batch_size: batch_size, condition: condition, session: session) instance.(stream_name, position: position) end @@ -44,7 +44,9 @@ def call(stream_name, position: nil) def get_records(stream_name, position) logger.trace { "Getting records (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } - select_statement = SelectStatement.build(stream_name, position: position, batch_size: batch_size) + where_fragment = self.condition + + select_statement = SelectStatement.build(stream_name, position: position, batch_size: batch_size, where_fragment: where_fragment) records = session.execute(select_statement.sql) diff --git a/test/automated/get/condition.rb b/test/automated/get/condition.rb new file mode 100644 index 0000000..0423189 --- /dev/null +++ b/test/automated/get/condition.rb @@ -0,0 +1,19 @@ +require_relative '../automated_init' + +context "Get" do + context "Condition" do + stream_name = Controls::Put.(instances: 3) + + condition = 'position = 0 OR position = 2' + + messages = Get.(stream_name, batch_size: 3, condition: condition) + + message_positions = messages.map do |message| + message.position + end + + test "Returns messages that meet the condition" do + assert(message_positions == [0, 2]) + end + end +end From 3b0a02927f836e40e022ff2e8c00f228b225d545 Mon Sep 17 00:00:00 2001 From: ntl Date: Wed, 18 Oct 2017 14:15:56 -0500 Subject: [PATCH 280/591] Read configures Get with optional condition --- lib/message_store/postgres/read.rb | 4 ++-- test/automated/read/condition.rb | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 test/automated/read/condition.rb diff --git a/lib/message_store/postgres/read.rb b/lib/message_store/postgres/read.rb index b45ca17..e6d499b 100644 --- a/lib/message_store/postgres/read.rb +++ b/lib/message_store/postgres/read.rb @@ -3,9 +3,9 @@ module Postgres class Read include MessageStore::Read - def configure(session: nil) + def configure(session: nil, condition: nil) Iterator.configure(self, stream_name, position: position) - Get.configure(self.iterator, batch_size: batch_size, session: session) + Get.configure(self.iterator, batch_size: batch_size, condition: condition, session: session) end end end diff --git a/test/automated/read/condition.rb b/test/automated/read/condition.rb new file mode 100644 index 0000000..132a022 --- /dev/null +++ b/test/automated/read/condition.rb @@ -0,0 +1,19 @@ +require_relative '../automated_init' + +context "Read" do + context "Condition" do + stream_name = Controls::Put.(instances: 3) + + condition = 'position = 0' + + message_count = 0 + + Read.(stream_name, condition: condition) do + message_count += 1 + end + + test "Reads messages that meet condition" do + assert(message_count == 1) + end + end +end From ebe5173d909a1f644e97ba4713880b76d0cb9020 Mon Sep 17 00:00:00 2001 From: ntl Date: Wed, 18 Oct 2017 17:48:19 -0500 Subject: [PATCH 281/591] SelectStatement argument is condition, not where_fragment --- lib/message_store/postgres/get.rb | 2 +- lib/message_store/postgres/get/select_statement.rb | 10 +++++----- .../{where_fragment.rb => condition.rb} | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) rename test/automated/select_statement/{where_fragment.rb => condition.rb} (60%) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index bac0774..c475d07 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -46,7 +46,7 @@ def get_records(stream_name, position) where_fragment = self.condition - select_statement = SelectStatement.build(stream_name, position: position, batch_size: batch_size, where_fragment: where_fragment) + select_statement = SelectStatement.build(stream_name, position: position, batch_size: batch_size, condition: condition) records = session.execute(select_statement.sql) diff --git a/lib/message_store/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb index d147740..b8c140d 100644 --- a/lib/message_store/postgres/get/select_statement.rb +++ b/lib/message_store/postgres/get/select_statement.rb @@ -4,7 +4,7 @@ class Get class SelectStatement include Log::Dependency - initializer :stream_name, w(:position), w(:batch_size), :where_fragment + initializer :stream_name, w(:position), w(:batch_size), :condition def position @position ||= Defaults.position @@ -22,8 +22,8 @@ def category_stream? is_category_stream ||= StreamName.category?(stream_name) end - def self.build(stream_name, position: nil, batch_size: nil, where_fragment: nil) - new(stream_name, position, batch_size, where_fragment) + def self.build(stream_name, position: nil, batch_size: nil, condition: nil) + new(stream_name, position, batch_size, condition) end def sql @@ -64,8 +64,8 @@ def where_clause #{position_field} >= #{position} SQL - unless where_fragment.nil? - clause << " AND\n(#{where_fragment})" + unless condition.nil? + clause << " AND\n(#{condition})" end clause diff --git a/test/automated/select_statement/where_fragment.rb b/test/automated/select_statement/condition.rb similarity index 60% rename from test/automated/select_statement/where_fragment.rb rename to test/automated/select_statement/condition.rb index b5d1ac0..a1d5ac4 100644 --- a/test/automated/select_statement/where_fragment.rb +++ b/test/automated/select_statement/condition.rb @@ -1,12 +1,12 @@ require_relative '../automated_init' context "Select Statement" do - context "Where Fragment" do + context "Condition" do stream_name = Controls::StreamName.example - where_fragment = '1 = 1' + condition = '1 = 1' - select_statement = Get::SelectStatement.build(stream_name, where_fragment: where_fragment) + select_statement = Get::SelectStatement.build(stream_name, condition: condition) sql = select_statement.sql sql.gsub!(/\s+/, ' ') @@ -20,8 +20,8 @@ assert(sql.include? 'position >=') end - test "Includes where fragment encased in parentheses" do - assert(sql.include? "AND (#{where_fragment})") + test "Includes condition encased in parentheses" do + assert(sql.include? "AND (#{condition})") end end end From 107e5c7d43f0aaac0b81ae62fe97ed87631d9514 Mon Sep 17 00:00:00 2001 From: ntl Date: Wed, 18 Oct 2017 17:59:50 -0500 Subject: [PATCH 282/591] discriminator_field, not where_clause_field --- lib/message_store/postgres/get/select_statement.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/message_store/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb index b8c140d..2be59df 100644 --- a/lib/message_store/postgres/get/select_statement.rb +++ b/lib/message_store/postgres/get/select_statement.rb @@ -60,7 +60,7 @@ def sql def where_clause clause = <<~SQL.chomp - #{where_clause_field} = '#{stream_name}' AND + #{discriminator_field} = '#{stream_name}' AND #{position_field} >= #{position} SQL @@ -71,7 +71,7 @@ def where_clause clause end - def where_clause_field + def discriminator_field unless category_stream? 'stream_name' else From 0862db599c69a1ba2ccd7c1021ac4160de29de93 Mon Sep 17 00:00:00 2001 From: ntl Date: Wed, 18 Oct 2017 18:06:09 -0500 Subject: [PATCH 283/591] Condition is written to log messages --- lib/message_store/postgres/get.rb | 4 ++-- lib/message_store/postgres/get/select_statement.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index c475d07..2d0452f 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -42,7 +42,7 @@ def call(stream_name, position: nil) end def get_records(stream_name, position) - logger.trace { "Getting records (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } + logger.trace { "Getting records (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } where_fragment = self.condition @@ -50,7 +50,7 @@ def get_records(stream_name, position) records = session.execute(select_statement.sql) - logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } + logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } records end diff --git a/lib/message_store/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb index 2be59df..53cccda 100644 --- a/lib/message_store/postgres/get/select_statement.rb +++ b/lib/message_store/postgres/get/select_statement.rb @@ -27,7 +27,7 @@ def self.build(stream_name, position: nil, batch_size: nil, condition: nil) end def sql - logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{category_stream?}, Types: #{stream_type_list.inspect}, Position: #{position}, Batch Size: #{batch_size})" } + logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{category_stream?}, Types: #{stream_type_list.inspect}, Position: #{position}, Batch Size: #{batch_size}, Condition: #{condition || '(none)'})" } formatted_where_clause = where_clause.each_line.to_a.join(" ") @@ -52,7 +52,7 @@ def sql ; SQL - logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{category_stream?}, Types: #{stream_type_list.inspect}, Position: #{position}, Batch Size: #{batch_size})" } + logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{category_stream?}, Types: #{stream_type_list.inspect}, Position: #{position}, Batch Size: #{batch_size}, Condition: #{condition || '(none)'})" } logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } statement From 497ba76956fb6c1a839ded12998b1d19be7943b9 Mon Sep 17 00:00:00 2001 From: ntl Date: Wed, 18 Oct 2017 20:55:38 -0500 Subject: [PATCH 284/591] Package version is increased from 0.4.1.0 to 0.4.2.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index bb57c80..5c4eccd 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.4.1.0' + s.version = '0.4.2.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 22cba3f8e81f9831daa1a3933a4fe4dcab684b43 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 18 Oct 2017 21:41:33 -0500 Subject: [PATCH 285/591] Select statement's position and batch size attributes don't need to be writable --- lib/message_store/postgres/get/select_statement.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb index 53cccda..279d67f 100644 --- a/lib/message_store/postgres/get/select_statement.rb +++ b/lib/message_store/postgres/get/select_statement.rb @@ -4,7 +4,7 @@ class Get class SelectStatement include Log::Dependency - initializer :stream_name, w(:position), w(:batch_size), :condition + initializer :stream_name, na(:position), na(:batch_size), :condition def position @position ||= Defaults.position From f5390fc5b284a01eeded9d4e9cb721fd2f5e5a0c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 18 Oct 2017 21:42:46 -0500 Subject: [PATCH 286/591] Package version is increased from 0.4.2.0 to 0.5.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 5c4eccd..16556a9 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.4.2.0' + s.version = '0.5.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From d566de47254b0069f61a6a517ca2757ddf21f3cb Mon Sep 17 00:00:00 2001 From: ntl Date: Thu, 16 Nov 2017 09:18:19 -0600 Subject: [PATCH 287/591] Concurrent writers test is implemented --- message_store-postgres.gemspec | 1 + ...ncurrent_threads_write_to_single_stream.rb | 36 +++++++++++++++++++ test/interactive/interactive_init.rb | 4 +++ 3 files changed, 41 insertions(+) create mode 100644 test/interactive/concurrent_threads_write_to_single_stream.rb create mode 100644 test/interactive/interactive_init.rb diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 16556a9..a28a176 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -26,4 +26,5 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'pg' s.add_development_dependency 'test_bench' + s.add_development_dependency 'ntl-actor' end diff --git a/test/interactive/concurrent_threads_write_to_single_stream.rb b/test/interactive/concurrent_threads_write_to_single_stream.rb new file mode 100644 index 0000000..237b05f --- /dev/null +++ b/test/interactive/concurrent_threads_write_to_single_stream.rb @@ -0,0 +1,36 @@ +require_relative './interactive_init' + +require 'actor' + +class ExampleProcess + include Actor + include Log::Dependency + + attr_reader :stream_name + + def initialize(stream_name) + @stream_name = stream_name + end + + handle :start do + :write_message + end + + handle :write_message do + message_data = ::Controls::MessageData::Write.example + + MessageStore::Postgres::Write.(message_data, stream_name) + + logger.info { "Wrote message (Type: #{message_data.type.inspect}, Stream Name: #{stream_name.inspect})" } + + :write_message + end +end + +stream_name = Controls::StreamName.example + +Actor::Supervisor.start do + 2.times do + ExampleProcess.start(stream_name) + end +end diff --git a/test/interactive/interactive_init.rb b/test/interactive/interactive_init.rb new file mode 100644 index 0000000..0893a61 --- /dev/null +++ b/test/interactive/interactive_init.rb @@ -0,0 +1,4 @@ +ENV['LOG_LEVEL'] ||= 'info' +ENV['LOG_TAGS'] ||= '_untagged,message_store_postgres,-data' + +require_relative '../test_init' From f3feaaca0e03b91278b7a78b3d019a9c9876a6c0 Mon Sep 17 00:00:00 2001 From: Joshua Flanagan Date: Wed, 6 Dec 2017 20:15:29 -0600 Subject: [PATCH 288/591] Database setup scripts are more robust The scripts rely on comparing the exact output of `psql` commands to check for the existence of objects in the database. The psql output can be impacted by user preferences in their `.psqlrc` file, breaking the scripts. To reproduce, add the following line to `~/.psqlrc`: ``` \pset format wrapped ``` The install/uninstall scripts will now fail to find existing objects. The -q flag silences all output other than the query result. The -X flag ignores the user's .psqlrc so that output is more deterministic. --- database/install.sh | 2 +- database/uninstall.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/database/install.sh b/database/install.sh index 07fa5ee..5085b61 100755 --- a/database/install.sh +++ b/database/install.sh @@ -28,7 +28,7 @@ echo function create-user { echo "Database user is: $user" - user_exists=`psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` + user_exists=`psql postgres -qtAXc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` if [ "$user_exists" = "1" ]; then echo "Database user \"$user\" was previously created. Not creating again." diff --git a/database/uninstall.sh b/database/uninstall.sh index cf3117a..d635193 100755 --- a/database/uninstall.sh +++ b/database/uninstall.sh @@ -28,7 +28,7 @@ echo function delete-user { echo "Database user is: $user" - user_exists=`psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` + user_exists=`psql postgres -qtAXc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` if [ "$user_exists" = "1" ]; then echo "Deleting database user \"$user\"..." @@ -43,7 +43,7 @@ function delete-user { function delete-database { echo "Database name is: $database" - database_exists=`psql postgres -tAc "SELECT 1 FROM pg_database WHERE datname='$database'"` + database_exists=`psql postgres -qtAXc "SELECT 1 FROM pg_database WHERE datname='$database'"` if [ "$database_exists" = "1" ]; then echo "Deleting database \"$database\"..." From da4fe0200f7e1ff434bab87dd51136630ce51eec Mon Sep 17 00:00:00 2001 From: Joshua Flanagan Date: Wed, 6 Dec 2017 21:18:19 -0600 Subject: [PATCH 289/591] Remove redundant indexes There are already multicolumn indexes on stream,position and category/global_position. Those cover the two most common queries. PostgreSQL can still use those indexes for queries on stream or category alone, since they are the leftmost column: > A multicolumn B-tree index can be used with query conditions that involve any subset of the index's columns, but the index is most efficient when there are constraints on the leading (leftmost) columns. The exact rule is that equality constraints on leading columns, plus any inequality constraints on the first column that does not have an equality constraint, will be used to limit the portion of the index that is scanned https://www.postgresql.org/docs/9.6/static/indexes-multicolumn.html The redundant single column indexes may have a slight performance advantage over the multi-column indexes for queries that only use the first column (since the indexes are smaller), but those queries are likely rare, and probably not worth the cost of the redundant indexes (disk space, and slower INSERTs). --- database/indexes/messages-category.sql | 1 - database/indexes/messages-stream-name.sql | 1 - database/install.sh | 4 ---- 3 files changed, 6 deletions(-) delete mode 100644 database/indexes/messages-category.sql delete mode 100644 database/indexes/messages-stream-name.sql diff --git a/database/indexes/messages-category.sql b/database/indexes/messages-category.sql deleted file mode 100644 index 0ee9a57..0000000 --- a/database/indexes/messages-category.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE INDEX CONCURRENTLY "messages_category_idx" ON "public"."messages" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); diff --git a/database/indexes/messages-stream-name.sql b/database/indexes/messages-stream-name.sql deleted file mode 100644 index d9d3518..0000000 --- a/database/indexes/messages-stream-name.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE INDEX CONCURRENTLY "messages_stream_name_idx" ON "public"."messages" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); diff --git a/database/install.sh b/database/install.sh index 07fa5ee..d349c54 100755 --- a/database/install.sh +++ b/database/install.sh @@ -86,10 +86,6 @@ function create-indexes { psql $database -f $base/indexes/messages-id.sql echo "messages_category_global_position_idx" psql $database -f $base/indexes/messages-category-global-position.sql - echo "messages_category_idx" - psql $database -f $base/indexes/messages-category.sql - echo "messages_stream_name_idx" - psql $database -f $base/indexes/messages-stream-name.sql echo "messages_stream_name_position_uniq_idx" psql $database -f $base/indexes/messages-stream-name-position-uniq.sql echo From 69f51db81a3c5df2c504a843bf1d17cae2d1b351 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 1 Jan 2018 19:28:15 -0600 Subject: [PATCH 290/591] Package version increased from 0.5.0.0 to 0.6.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 16556a9..092b81c 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.5.0.0' + s.version = '0.6.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 7941e19f5be5ba06a2f4bb914f6aab321c843a24 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 3 Apr 2018 17:45:11 -0500 Subject: [PATCH 291/591] Transform::Read interface use is updated --- lib/message_store/postgres/get.rb | 4 ++-- lib/message_store/postgres/get/last.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 2d0452f..65c001a 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -74,12 +74,12 @@ def convert(records) module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? - Transform::Read.(serialized_data, MessageData::Hash, :json) + Transform::Read.(serialized_data, :json, MessageData::Hash) end def self.metadata(serialized_metadata) return nil if serialized_metadata.nil? - Transform::Read.(serialized_metadata, MessageData::Hash, :json) + Transform::Read.(serialized_metadata, :json, MessageData::Hash) end end diff --git a/lib/message_store/postgres/get/last.rb b/lib/message_store/postgres/get/last.rb index bd47a55..7013d97 100644 --- a/lib/message_store/postgres/get/last.rb +++ b/lib/message_store/postgres/get/last.rb @@ -74,12 +74,12 @@ def __convert(records) module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? - Transform::Read.(serialized_data, MessageData::Hash, :json) + Transform::Read.(serialized_data, :json, MessageData::Hash) end def self.metadata(serialized_metadata) return nil if serialized_metadata.nil? - Transform::Read.(serialized_metadata, MessageData::Hash, :json) + Transform::Read.(serialized_metadata, :json, MessageData::Hash) end end From 557f8d5a3103a7d58769dcdf9cf1cff3ee785534 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 3 Apr 2018 17:45:54 -0500 Subject: [PATCH 292/591] Obsolete code is removed --- lib/message_store/postgres/get/last.rb | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/lib/message_store/postgres/get/last.rb b/lib/message_store/postgres/get/last.rb index 7013d97..d9911a5 100644 --- a/lib/message_store/postgres/get/last.rb +++ b/lib/message_store/postgres/get/last.rb @@ -53,24 +53,6 @@ def convert(record) message_data end - def __convert(records) - logger.trace { "Converting records to message data (Records Count: #{records.ntuples})" } - - messages = records.map do |record| - record['data'] = Deserialize.data(record['data']) - record['metadata'] = Deserialize.metadata(record['metadata']) - record['time'] = Time.utc_coerced(record['time']) - - MessageData::Read.build record - - break - end - - logger.debug { "Converted records to message data (Message Data Count: #{messages.length})" } - - messages - end - module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? From e8fc19ed52b0c347bfb8e429e5c43d9a68ba459a Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 3 Apr 2018 17:46:29 -0500 Subject: [PATCH 293/591] Package version is increased from 0.6.0.0 to 0.6.0.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 092b81c..bd03452 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.6.0.0' + s.version = '0.6.0.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 9f1d71ae893ff486c2ce37cf16e5d82fdd6f9264 Mon Sep 17 00:00:00 2001 From: Will Howard Date: Tue, 24 Apr 2018 21:34:59 -0400 Subject: [PATCH 294/591] use expanded table output when listing messages from postgres --- database/list-messages.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/list-messages.sh b/database/list-messages.sh index 8f637a9..9d6f968 100755 --- a/database/list-messages.sh +++ b/database/list-messages.sh @@ -46,7 +46,7 @@ fi echo if [ -z $stream_name ]; then - psql $database -c "SELECT * FROM $table" + psql $database -x -c "SELECT * FROM $table" else - psql $database -c "SELECT * FROM $table WHERE stream_name = '$stream_name'" + psql $database -x -c "SELECT * FROM $table WHERE stream_name = '$stream_name'" fi From 027300a7b549f7480833393e4e5465677875fded Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 24 Apr 2018 21:04:52 -0500 Subject: [PATCH 295/591] Package version is increased from 0.6.0.1 to 0.6.0.2 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index bd03452..32e22d1 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.6.0.1' + s.version = '0.6.0.2' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 20f024fc9048a285cfd948cfa812744ec4142d0d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 24 Apr 2018 22:13:56 -0500 Subject: [PATCH 296/591] Print messages, rather than list messages --- database/{list-messages.sh => print-messages.sh} | 2 +- scripts/{evt-pg-list-messages => evt-pg-print-messages} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename database/{list-messages.sh => print-messages.sh} (97%) rename scripts/{evt-pg-list-messages => evt-pg-print-messages} (78%) diff --git a/database/list-messages.sh b/database/print-messages.sh similarity index 97% rename from database/list-messages.sh rename to database/print-messages.sh index 9d6f968..111b394 100755 --- a/database/list-messages.sh +++ b/database/print-messages.sh @@ -3,7 +3,7 @@ set -e echo -echo "Listing Messages" +echo "Printing Messages" echo "= = =" echo diff --git a/scripts/evt-pg-list-messages b/scripts/evt-pg-print-messages similarity index 78% rename from scripts/evt-pg-list-messages rename to scripts/evt-pg-print-messages index b7ab7fd..17e5924 100755 --- a/scripts/evt-pg-list-messages +++ b/scripts/evt-pg-print-messages @@ -1,7 +1,7 @@ #!/usr/bin/env ruby root = File.expand_path '../database', __dir__ -script_filename = 'list-messages.sh' +script_filename = 'print-messages.sh' script_filepath = File.join root, script_filename system script_filepath From 649a7c7430d1387deb2bc28335382e0743a9b31c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 24 Apr 2018 22:15:00 -0500 Subject: [PATCH 297/591] Package version is increased from 0.6.0.2 to 0.7.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 32e22d1..0c12710 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.6.0.2' + s.version = '0.7.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 10ceb40afa05a4488ec66c63eb9f0b628642af7a Mon Sep 17 00:00:00 2001 From: Fire-Dragon-DoL Date: Tue, 24 Apr 2018 22:05:44 -0700 Subject: [PATCH 298/591] Fix Get select_statement position type --- lib/message_store/postgres/get/last/select_statement.rb | 2 +- lib/message_store/postgres/get/select_statement.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/message_store/postgres/get/last/select_statement.rb b/lib/message_store/postgres/get/last/select_statement.rb index ca0647a..96cd4d1 100644 --- a/lib/message_store/postgres/get/last/select_statement.rb +++ b/lib/message_store/postgres/get/last/select_statement.rb @@ -18,7 +18,7 @@ def sql SELECT id::varchar, stream_name::varchar, - position::int, + position::bigint, type::varchar, global_position::bigint, data::varchar, diff --git a/lib/message_store/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb index 279d67f..0fb4c1e 100644 --- a/lib/message_store/postgres/get/select_statement.rb +++ b/lib/message_store/postgres/get/select_statement.rb @@ -35,7 +35,7 @@ def sql SELECT id::varchar, stream_name::varchar, - position::int, + position::bigint, type::varchar, global_position::bigint, data::varchar, From 74de70aeaa32d1501294311db26c50ee369c190a Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 25 Apr 2018 00:10:16 -0500 Subject: [PATCH 299/591] Vestigial code is removed --- lib/message_store/postgres/get.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 65c001a..5d13bf5 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -44,8 +44,6 @@ def call(stream_name, position: nil) def get_records(stream_name, position) logger.trace { "Getting records (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } - where_fragment = self.condition - select_statement = SelectStatement.build(stream_name, position: position, batch_size: batch_size, condition: condition) records = session.execute(select_statement.sql) From 857adaa8347254ac3a2dfb9849b1d6309242cd96 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 25 Apr 2018 00:13:39 -0500 Subject: [PATCH 300/591] Package version is increased from 0.7.0.0 to 0.7.0.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 0c12710..ffecfe2 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.7.0.0' + s.version = '0.7.0.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 464bae00430e8dfb557521cdc67e785550e20158 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 30 May 2018 18:07:39 -0500 Subject: [PATCH 301/591] Rudimentary write benchmark --- message_store-postgres.gemspec | 1 + test/benchmark/benchmark_init.rb | 3 +++ test/benchmark/write.rb | 26 ++++++++++++++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 test/benchmark/benchmark_init.rb create mode 100644 test/benchmark/write.rb diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index ffecfe2..fdc6884 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -26,4 +26,5 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'pg' s.add_development_dependency 'test_bench' + s.add_development_dependency 'evt-diagnostics-sample' end diff --git a/test/benchmark/benchmark_init.rb b/test/benchmark/benchmark_init.rb new file mode 100644 index 0000000..76c1854 --- /dev/null +++ b/test/benchmark/benchmark_init.rb @@ -0,0 +1,3 @@ +require_relative '../test_init' + +require 'diagnostics/sample' diff --git a/test/benchmark/write.rb b/test/benchmark/write.rb new file mode 100644 index 0000000..3e6ea4a --- /dev/null +++ b/test/benchmark/write.rb @@ -0,0 +1,26 @@ +require_relative 'benchmark_init' +require 'benchmark' + +warmup_cycles = 10 +test_cycles = 10_000 +cycles = warmup_cycles + test_cycles + +Entry = Struct.new(:stream_name, :message_data) + +entries = [] +cycles.times do + stream_name = Controls::StreamName.example + write_message = Controls::MessageData::Write.example + + entries << Entry.new(stream_name, write_message) +end + +session = Session.build +write = Write.build(session: session) + +result = Diagnostics::Sample.(test_cycles, warmup_cycles: warmup_cycles, gc: true) do |i| + entry = entries[i] + write.(entry.message_data, entry.stream_name) +end + +puts result From 247e7be45bb60df2249bf28bf90e48dc58765e08 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 30 May 2018 22:04:58 -0500 Subject: [PATCH 302/591] Write benchmark structure is stable. Next: moves writes to controls --- .../postgres/controls/message_data.rb | 3 ++ test/benchmark/benchmark_init.rb | 1 + test/benchmark/defaults.rb | 33 +++++++++++++++++++ test/benchmark/{ => write}/write.rb | 19 ++++++----- 4 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 test/benchmark/defaults.rb rename test/benchmark/{ => write}/write.rb (50%) diff --git a/lib/message_store/postgres/controls/message_data.rb b/lib/message_store/postgres/controls/message_data.rb index 0febad2..9fede3a 100644 --- a/lib/message_store/postgres/controls/message_data.rb +++ b/lib/message_store/postgres/controls/message_data.rb @@ -2,6 +2,9 @@ module MessageStore module Postgres module Controls MessageData = MessageStore::Controls::MessageData + + module MessageData + end end end end diff --git a/test/benchmark/benchmark_init.rb b/test/benchmark/benchmark_init.rb index 76c1854..49f1f2f 100644 --- a/test/benchmark/benchmark_init.rb +++ b/test/benchmark/benchmark_init.rb @@ -1,3 +1,4 @@ require_relative '../test_init' require 'diagnostics/sample' +require_relative 'defaults' diff --git a/test/benchmark/defaults.rb b/test/benchmark/defaults.rb new file mode 100644 index 0000000..95eb0e5 --- /dev/null +++ b/test/benchmark/defaults.rb @@ -0,0 +1,33 @@ +module Benchmark + class Defaults + initializer :cycles, :warmup_cycles, :gc + + def total_cycles + cycles + warmup_cycles + end + + def to_s + <<~TEXT + Cycles: #{cycles} + Warmup Cycles: #{warmup_cycles} + GC: #{gc} + TEXT + end + + def self.build + new(cycles, warmup_cycles, gc) + end + + def self.cycles + Integer(ENV['CYCLES'] || 2500) + end + + def self.warmup_cycles + Integer(ENV['WARMUP_CYCLES'] || 10) + end + + def self.gc + ['on', 'true'].include?(ENV['GC']) ? true : false + end + end +end diff --git a/test/benchmark/write.rb b/test/benchmark/write/write.rb similarity index 50% rename from test/benchmark/write.rb rename to test/benchmark/write/write.rb index 3e6ea4a..9ee71f6 100644 --- a/test/benchmark/write.rb +++ b/test/benchmark/write/write.rb @@ -1,26 +1,27 @@ -require_relative 'benchmark_init' -require 'benchmark' +require_relative '../benchmark_init' -warmup_cycles = 10 -test_cycles = 10_000 -cycles = warmup_cycles + test_cycles +defaults = Benchmark::Defaults.build + +puts +puts 'Write Benchmark' +puts '- - -' Entry = Struct.new(:stream_name, :message_data) entries = [] -cycles.times do +defaults.total_cycles.times do stream_name = Controls::StreamName.example write_message = Controls::MessageData::Write.example entries << Entry.new(stream_name, write_message) end -session = Session.build -write = Write.build(session: session) +write = Write.build -result = Diagnostics::Sample.(test_cycles, warmup_cycles: warmup_cycles, gc: true) do |i| +result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = entries[i] write.(entry.message_data, entry.stream_name) end puts result +puts From 0114989a587fe817aced00d7b2a8ab018f7fea7d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 30 May 2018 23:06:15 -0500 Subject: [PATCH 303/591] Message data used in write benchmark is included in the controls --- .../postgres/controls/message_data.rb | 21 +++++++++++++++++++ test/benchmark/write/write.rb | 12 ++--------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/lib/message_store/postgres/controls/message_data.rb b/lib/message_store/postgres/controls/message_data.rb index 9fede3a..b819973 100644 --- a/lib/message_store/postgres/controls/message_data.rb +++ b/lib/message_store/postgres/controls/message_data.rb @@ -4,6 +4,27 @@ module Controls MessageData = MessageStore::Controls::MessageData module MessageData + module Write + module List + Entry = Struct.new(:stream_name, :message_data) + + def self.get(instances: nil) + instances ||= 1 + + stream_name = StreamName.example + + list = [] + instances.times do + stream_name = Controls::StreamName.example + write_message = Controls::MessageData::Write.example + + list << Entry.new(stream_name, write_message) + end + + list + end + end + end end end end diff --git a/test/benchmark/write/write.rb b/test/benchmark/write/write.rb index 9ee71f6..87d4487 100644 --- a/test/benchmark/write/write.rb +++ b/test/benchmark/write/write.rb @@ -6,20 +6,12 @@ puts 'Write Benchmark' puts '- - -' -Entry = Struct.new(:stream_name, :message_data) - -entries = [] -defaults.total_cycles.times do - stream_name = Controls::StreamName.example - write_message = Controls::MessageData::Write.example - - entries << Entry.new(stream_name, write_message) -end +list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) write = Write.build result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| - entry = entries[i] + entry = list[i] write.(entry.message_data, entry.stream_name) end From 7d9bd32bd5e6db93a12e35e4ee05355b10fed8a4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 30 May 2018 23:11:26 -0500 Subject: [PATCH 304/591] Write benchmark is replaced with put benchmark --- test/benchmark/{write/write.rb => put.rb} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename test/benchmark/{write/write.rb => put.rb} (69%) diff --git a/test/benchmark/write/write.rb b/test/benchmark/put.rb similarity index 69% rename from test/benchmark/write/write.rb rename to test/benchmark/put.rb index 87d4487..a41a19c 100644 --- a/test/benchmark/write/write.rb +++ b/test/benchmark/put.rb @@ -1,18 +1,18 @@ -require_relative '../benchmark_init' +require_relative 'benchmark_init' defaults = Benchmark::Defaults.build puts -puts 'Write Benchmark' +puts 'Put Benchmark' puts '- - -' list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) -write = Write.build +put = Put.build result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] - write.(entry.message_data, entry.stream_name) + put.(entry.message_data, entry.stream_name) end puts result From cb453ff851269130f4d12935d0eb557f31e6f594 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 31 May 2018 02:11:00 -0500 Subject: [PATCH 305/591] Get benchmark --- test/benchmark/get.rb | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 test/benchmark/get.rb diff --git a/test/benchmark/get.rb b/test/benchmark/get.rb new file mode 100644 index 0000000..ce44889 --- /dev/null +++ b/test/benchmark/get.rb @@ -0,0 +1,25 @@ +require_relative 'benchmark_init' + +defaults = Benchmark::Defaults.build + +puts +puts 'Get Benchmark' +puts '- - -' + +list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) + +put = Put.build + +list.each do |entry| + put.(entry.message_data, entry.stream_name) +end + +get = Get.build + +result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| + entry = list[i] + get.(entry.stream_name) +end + +puts result +puts From 7e32f9e1de2681892b247d8d751ab2f0e776b975 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 31 May 2018 02:17:27 -0500 Subject: [PATCH 306/591] Superfluous control implementation is replaced with an alias --- lib/message_store/postgres/controls/category.rb | 9 --------- lib/message_store/postgres/controls/stream_name.rb | 9 +-------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/lib/message_store/postgres/controls/category.rb b/lib/message_store/postgres/controls/category.rb index 0d717eb..9678269 100644 --- a/lib/message_store/postgres/controls/category.rb +++ b/lib/message_store/postgres/controls/category.rb @@ -1,12 +1,3 @@ -# module MessageStore -# module Postgres -# module Controls -# Category = MessageStore::Controls::Category -# end -# end -# end - - module MessageStore module Postgres module Controls diff --git a/lib/message_store/postgres/controls/stream_name.rb b/lib/message_store/postgres/controls/stream_name.rb index 70e854a..b35f009 100644 --- a/lib/message_store/postgres/controls/stream_name.rb +++ b/lib/message_store/postgres/controls/stream_name.rb @@ -1,14 +1,7 @@ module MessageStore module Postgres module Controls - module StreamName - def self.example(category: nil, id: nil, type: nil, types: nil, randomize_category: nil) - category ||= Category.example(category: category, randomize_category: randomize_category) - id ||= Identifier::UUID.random - - MessageStore::Postgres::StreamName.stream_name(category, id, type: type, types: types) - end - end + StreamName = MessageStore::Controls::StreamName end end end From f44a550f5966d09c5f695b0deeb263331fa70e47 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 31 May 2018 02:20:32 -0500 Subject: [PATCH 307/591] Package version is increased from 0.7.0.1 to 0.7.0.2 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index fdc6884..3748b58 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.7.0.1' + s.version = '0.7.0.2' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 9d2a0a1b935da9688a34bec271e77bb263c1e921 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 16:01:42 -0500 Subject: [PATCH 308/591] Record benchmark results to a file --- .gitignore | 1 + test/benchmark/benchmark_init.rb | 4 +++ test/benchmark/put.rb | 7 +--- test/benchmark/record_result.rb | 57 ++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 test/benchmark/record_result.rb diff --git a/.gitignore b/.gitignore index 3bb1e20..740a161 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ gems *notes* loader.rb /test/package/installed +/test/benchmark/tmp/**/* diff --git a/test/benchmark/benchmark_init.rb b/test/benchmark/benchmark_init.rb index 49f1f2f..9226419 100644 --- a/test/benchmark/benchmark_init.rb +++ b/test/benchmark/benchmark_init.rb @@ -1,4 +1,8 @@ require_relative '../test_init' require 'diagnostics/sample' +require 'fileutils' +require 'pathname' + require_relative 'defaults' +require_relative 'record_result' diff --git a/test/benchmark/put.rb b/test/benchmark/put.rb index a41a19c..a1d362b 100644 --- a/test/benchmark/put.rb +++ b/test/benchmark/put.rb @@ -2,10 +2,6 @@ defaults = Benchmark::Defaults.build -puts -puts 'Put Benchmark' -puts '- - -' - list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) put = Put.build @@ -15,5 +11,4 @@ put.(entry.message_data, entry.stream_name) end -puts result -puts +Benchmark::RecordResult.('Put Benchmark', result) diff --git a/test/benchmark/record_result.rb b/test/benchmark/record_result.rb new file mode 100644 index 0000000..eb154dd --- /dev/null +++ b/test/benchmark/record_result.rb @@ -0,0 +1,57 @@ +module Benchmark + class RecordResult + dependency :clock + + initializer :name, :content + + def self.build(name, content) + instance = new(name, content) + instance.configure + instance + end + + def self.call(name, content) + instance = build(name, content) + instance.() + end + + def call + digest = write_result_file + + puts + puts digest + puts + end + + def write_result_file + fn = filename + + FileUtils.mkdir_p(fn.dirname) + + d = digest + fn.write(d) + + d + end + + def filename + directory = Pathname.new('test/benchmark/tmp') + filename = directory.join("#{name} - #{clock.iso8601}.txt") + + filename + end + + def digest + <<~TEXT + #{name} + - - - + #{content} + (#{filename}) + TEXT + end + + def configure + Clock::UTC.configure(self) + end + end +end From 54ab56e4cb0123a458bc12469cc4418e5b2cc7ef Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 17:02:56 -0500 Subject: [PATCH 309/591] Benchmark output is formatted --- test/benchmark/put.rb | 6 +++++- test/benchmark/record_result.rb | 10 +++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/benchmark/put.rb b/test/benchmark/put.rb index a1d362b..d753f39 100644 --- a/test/benchmark/put.rb +++ b/test/benchmark/put.rb @@ -11,4 +11,8 @@ put.(entry.message_data, entry.stream_name) end -Benchmark::RecordResult.('Put Benchmark', result) +puts +filename = Benchmark::RecordResult.('Put Benchmark', result) +puts +puts filename +puts diff --git a/test/benchmark/record_result.rb b/test/benchmark/record_result.rb index eb154dd..729fc24 100644 --- a/test/benchmark/record_result.rb +++ b/test/benchmark/record_result.rb @@ -16,11 +16,11 @@ def self.call(name, content) end def call - digest = write_result_file + digest, filename = write_result_file - puts puts digest - puts + + filename.to_s end def write_result_file @@ -31,7 +31,7 @@ def write_result_file d = digest fn.write(d) - d + [d, fn] end def filename @@ -46,8 +46,8 @@ def digest #{name} - - - #{content} - (#{filename}) TEXT + .chomp end def configure From 6c281cfa49706616998e46ad6a48818f045c9b4e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 17:40:54 -0500 Subject: [PATCH 310/591] Get benchmark results are recorded --- test/benchmark/get.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/benchmark/get.rb b/test/benchmark/get.rb index ce44889..a6854e6 100644 --- a/test/benchmark/get.rb +++ b/test/benchmark/get.rb @@ -2,10 +2,6 @@ defaults = Benchmark::Defaults.build -puts -puts 'Get Benchmark' -puts '- - -' - list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) put = Put.build @@ -21,5 +17,8 @@ get.(entry.stream_name) end -puts result +puts +filename = Benchmark::RecordResult.('Get Benchmark', result) +puts +puts filename puts From 618b6c92dbd0a5b69aba4c98286f99e9f595105e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 17:42:35 -0500 Subject: [PATCH 311/591] Baseline benchmarks for get and put --- .../results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt | 7 +++++++ .../results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt | 7 +++++++ 2 files changed, 14 insertions(+) create mode 100644 test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt create mode 100644 test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt new file mode 100644 index 0000000..1d9bcff --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt @@ -0,0 +1,7 @@ +Get Benchmark +- - - +Cycles: 20000 +Time: 34606.411000ms +Mean Cycle Time: 1.730321ms (± 9.642108ms) +Cycles Per Second: 577.927598 +GC: off diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt new file mode 100644 index 0000000..7758f2e --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt @@ -0,0 +1,7 @@ +Put Benchmark +- - - +Cycles: 20000 +Time: 62569.619000ms +Mean Cycle Time: 3.128481ms (± 16.107108ms) +Cycles Per Second: 319.643947 +GC: off From aef9fc6f2f972834aba105a16736e6866e0e73bc Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 18:17:20 -0500 Subject: [PATCH 312/591] Baseline benchmarks for get and put for 100,000 cycles --- .../results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt | 7 ------- .../results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt | 7 +++++++ .../results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt | 7 ------- .../results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt | 7 +++++++ 4 files changed, 14 insertions(+), 14 deletions(-) delete mode 100644 test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt create mode 100644 test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt delete mode 100644 test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt create mode 100644 test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt deleted file mode 100644 index 1d9bcff..0000000 --- a/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt +++ /dev/null @@ -1,7 +0,0 @@ -Get Benchmark -- - - -Cycles: 20000 -Time: 34606.411000ms -Mean Cycle Time: 1.730321ms (± 9.642108ms) -Cycles Per Second: 577.927598 -GC: off diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt new file mode 100644 index 0000000..424c4e2 --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt @@ -0,0 +1,7 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 207252.004000ms +Mean Cycle Time: 2.072520ms (± 8.178097ms) +Cycles Per Second: 482.504381 +GC: off diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt deleted file mode 100644 index 7758f2e..0000000 --- a/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt +++ /dev/null @@ -1,7 +0,0 @@ -Put Benchmark -- - - -Cycles: 20000 -Time: 62569.619000ms -Mean Cycle Time: 3.128481ms (± 16.107108ms) -Cycles Per Second: 319.643947 -GC: off diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt new file mode 100644 index 0000000..34cde09 --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt @@ -0,0 +1,7 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 164384.657000ms +Mean Cycle Time: 1.643847ms (± 10.087307ms) +Cycles Per Second: 608.329280 +GC: off From e6b0842c8aee9dd405fbd909f5392af3a4a507a3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 18:39:53 -0500 Subject: [PATCH 313/591] Baseline benchmarks for get and put for 100,000 cycles on a new database --- .../results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt | 7 ------- .../results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt | 7 +++++++ .../results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt | 7 ------- .../results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt | 7 +++++++ 4 files changed, 14 insertions(+), 14 deletions(-) delete mode 100644 test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt create mode 100644 test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt delete mode 100644 test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt create mode 100644 test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt deleted file mode 100644 index 424c4e2..0000000 --- a/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt +++ /dev/null @@ -1,7 +0,0 @@ -Get Benchmark -- - - -Cycles: 100000 -Time: 207252.004000ms -Mean Cycle Time: 2.072520ms (± 8.178097ms) -Cycles Per Second: 482.504381 -GC: off diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt new file mode 100644 index 0000000..a546f8f --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt @@ -0,0 +1,7 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 155466.029000ms +Mean Cycle Time: 1.554660ms (± 6.764445ms) +Cycles Per Second: 643.227338 +GC: off diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt deleted file mode 100644 index 34cde09..0000000 --- a/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt +++ /dev/null @@ -1,7 +0,0 @@ -Put Benchmark -- - - -Cycles: 100000 -Time: 164384.657000ms -Mean Cycle Time: 1.643847ms (± 10.087307ms) -Cycles Per Second: 608.329280 -GC: off diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt new file mode 100644 index 0000000..6f5da4c --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt @@ -0,0 +1,7 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 278854.650000ms +Mean Cycle Time: 2.788546ms (± 11.991595ms) +Cycles Per Second: 358.609763 +GC: off From 03dca6502b36f3778792ffb91551969598eb3b20 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 18:42:23 -0500 Subject: [PATCH 314/591] System information is included in benchmarks --- .../Get Benchmark - 2018-06-01T23:25:15.505Z.txt | 12 ++++++++++++ .../Put Benchmark - 2018-06-01T23:37:23.568Z.txt | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt index a546f8f..c200a52 100644 --- a/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt +++ b/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt @@ -5,3 +5,15 @@ Time: 155466.029000ms Mean Cycle Time: 1.554660ms (± 6.764445ms) Cycles Per Second: 643.227338 GC: off + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt index 6f5da4c..c1e4476 100644 --- a/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt +++ b/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt @@ -5,3 +5,15 @@ Time: 278854.650000ms Mean Cycle Time: 2.788546ms (± 11.991595ms) Cycles Per Second: 358.609763 GC: off + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 From 14fde117f66d9c6ac2d9effc9e9edc5a65ec1ba9 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 2 Jun 2018 21:55:20 -0500 Subject: [PATCH 315/591] Hash server function converts a string (typically a stream name) to a 64-bit integer --- database/functions/hash-64.sql | 12 ++++++++++++ database/install.sh | 7 +++++++ 2 files changed, 19 insertions(+) create mode 100644 database/functions/hash-64.sql diff --git a/database/functions/hash-64.sql b/database/functions/hash-64.sql new file mode 100644 index 0000000..a1aace9 --- /dev/null +++ b/database/functions/hash-64.sql @@ -0,0 +1,12 @@ +CREATE OR REPLACE FUNCTION hash_64( + _stream_name varchar +) +RETURNS bigint +AS $$ +DECLARE + hash bigint; +BEGIN + select @left('x' || md5(_stream_name), 16)::bit(64)::bigint into hash; + return hash; +END; +$$ LANGUAGE plpgsql diff --git a/database/install.sh b/database/install.sh index 74ce803..6741458 100755 --- a/database/install.sh +++ b/database/install.sh @@ -70,12 +70,19 @@ function create-table { function create-functions { base=$(script_dir) echo "Creating functions..." + + echo "hash_64 function" + psql $database -f $base/functions/hash-64.sql + echo "category function" psql $database -f $base/functions/category.sql + echo "stream_version function" psql $database -f $base/functions/stream-version.sql + echo "write_sql function" psql $database -f $base/functions/write-message.sql + echo } From 20bdfce06686551d5e31c5f1c9bcdcdb135ad326 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 3 Jun 2018 18:36:06 -0500 Subject: [PATCH 316/591] Test script for the 64-bit hash of strings --- database/test/hash-64.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 database/test/hash-64.sql diff --git a/database/test/hash-64.sql b/database/test/hash-64.sql new file mode 100644 index 0000000..3d958c7 --- /dev/null +++ b/database/test/hash-64.sql @@ -0,0 +1 @@ +SELECT hash_64('someStream-123'); From 5e81fff4111155798a22f206528c20a4bfc4b6c3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 3 Jun 2018 22:57:09 -0500 Subject: [PATCH 317/591] Stream write concurrency is serialized with advisory locks --- database/functions/write-message.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/database/functions/write-message.sql b/database/functions/write-message.sql index a16f45e..939d90d 100644 --- a/database/functions/write-message.sql +++ b/database/functions/write-message.sql @@ -13,9 +13,13 @@ DECLARE stream_version bigint; position bigint; category varchar; + stream_name_hash bigint; BEGIN message_id = uuid(_id); + stream_name_hash = hash_64(_stream_name); + PERFORM pg_advisory_lock(stream_name_hash); + stream_version := stream_version(_stream_name); if stream_version is null then @@ -24,6 +28,7 @@ BEGIN if _expected_version is not null then if _expected_version != stream_version then + PERFORM pg_advisory_unlock(stream_name_hash); raise exception 'Wrong expected version: % (Stream: %, Stream Version: %)', _expected_version, _stream_name, stream_version; end if; end if; @@ -50,6 +55,8 @@ BEGIN ) ; + PERFORM pg_advisory_unlock(stream_name_hash); + return position; END; $$ LANGUAGE plpgsql; From 14f5306fc1f518f30c566a9643ce0beb77e2634d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 3 Jun 2018 22:59:36 -0500 Subject: [PATCH 318/591] Interactive test script for concurrent writes --- test/interactive/interactive_init.rb | 1 + test/interactive/write.rb | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 test/interactive/interactive_init.rb create mode 100644 test/interactive/write.rb diff --git a/test/interactive/interactive_init.rb b/test/interactive/interactive_init.rb new file mode 100644 index 0000000..faa8577 --- /dev/null +++ b/test/interactive/interactive_init.rb @@ -0,0 +1 @@ +require_relative '../test_init' diff --git a/test/interactive/write.rb b/test/interactive/write.rb new file mode 100644 index 0000000..7af8899 --- /dev/null +++ b/test/interactive/write.rb @@ -0,0 +1,14 @@ +require_relative 'interactive_init' + +message_data = Controls::MessageData::Write.example +stream_name = 'testWriteIsolation' + +write = Write.build + +i = 0 +loop do + position = write.(message_data, stream_name) + puts "i: #{i}, p: #{position}" + i += 1 +end + From 68867eae6abceacb41f81397688463bcd3ced1fe Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 18:11:02 -0500 Subject: [PATCH 319/591] Advisory locks baseline implementation --- database/functions/write-message.sql | 5 +---- database/test/write-message-expected-version.sql | 2 +- database/test/write-message.sql | 2 +- test/interactive/write.rb | 7 +++++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/database/functions/write-message.sql b/database/functions/write-message.sql index 939d90d..1f95698 100644 --- a/database/functions/write-message.sql +++ b/database/functions/write-message.sql @@ -18,7 +18,7 @@ BEGIN message_id = uuid(_id); stream_name_hash = hash_64(_stream_name); - PERFORM pg_advisory_lock(stream_name_hash); + PERFORM pg_advisory_xact_lock(stream_name_hash); stream_version := stream_version(_stream_name); @@ -28,7 +28,6 @@ BEGIN if _expected_version is not null then if _expected_version != stream_version then - PERFORM pg_advisory_unlock(stream_name_hash); raise exception 'Wrong expected version: % (Stream: %, Stream Version: %)', _expected_version, _stream_name, stream_version; end if; end if; @@ -55,8 +54,6 @@ BEGIN ) ; - PERFORM pg_advisory_unlock(stream_name_hash); - return position; END; $$ LANGUAGE plpgsql; diff --git a/database/test/write-message-expected-version.sql b/database/test/write-message-expected-version.sql index 330c5b6..be4ab20 100644 --- a/database/test/write-message-expected-version.sql +++ b/database/test/write-message-expected-version.sql @@ -1 +1 @@ -SELECT write_message('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}', 1); +SELECT write_message(gen_random_uuid()::varchar, 'testWriteIsolation'::varchar, 'SomeType'::varchar, '{"attribute": "79a59e513f9182d5754abdf9b90e2179"}'::jsonb, '{"metaAttribute": "6b3ae03d713a51a445290f3fc4b5223a"}'::jsonb, 1::bigint); diff --git a/database/test/write-message.sql b/database/test/write-message.sql index 6f17139..9ae0c5b 100644 --- a/database/test/write-message.sql +++ b/database/test/write-message.sql @@ -1 +1 @@ -SELECT write_message('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}'); +SELECT write_message(gen_random_uuid()::varchar, 'testWriteIsolation'::varchar, 'SomeType'::varchar, '{"attribute": "79a59e513f9182d5754abdf9b90e2179"}'::jsonb, '{"metaAttribute": "6b3ae03d713a51a445290f3fc4b5223a"}'::jsonb); diff --git a/test/interactive/write.rb b/test/interactive/write.rb index 7af8899..57c9409 100644 --- a/test/interactive/write.rb +++ b/test/interactive/write.rb @@ -1,13 +1,16 @@ require_relative 'interactive_init' -message_data = Controls::MessageData::Write.example stream_name = 'testWriteIsolation' +message_data_1 = Controls::MessageData::Write.example(data: { attribute: 1 }) +message_data_2 = Controls::MessageData::Write.example(data: { attribute: 2 }) +batch = [message_data_1, message_data_2] + write = Write.build i = 0 loop do - position = write.(message_data, stream_name) + position = write.(batch, stream_name) puts "i: #{i}, p: #{position}" i += 1 end From d39ea3a6e80035b5fbd8940f6df1cb5d8c22c279 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 18:13:59 -0500 Subject: [PATCH 320/591] Hash value does not have to be unsigned (positive) --- database/functions/hash-64.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/functions/hash-64.sql b/database/functions/hash-64.sql index a1aace9..ab6fa4d 100644 --- a/database/functions/hash-64.sql +++ b/database/functions/hash-64.sql @@ -6,7 +6,7 @@ AS $$ DECLARE hash bigint; BEGIN - select @left('x' || md5(_stream_name), 16)::bit(64)::bigint into hash; + select left('x' || md5(_stream_name), 16)::bit(64)::bigint into hash; return hash; END; $$ LANGUAGE plpgsql From 1c428185644076653a2fa70e6492a212edc70780 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 18:33:52 -0500 Subject: [PATCH 321/591] Benchmarks for advisory locks --- ...t Benchmark - 2018-06-04T23:30:23.531Z.txt | 21 +++++++++++++++++++ ...t Benchmark - 2018-06-04T23:22:20.699Z.txt | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt create mode 100644 test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt new file mode 100644 index 0000000..11610e4 --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt @@ -0,0 +1,21 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 74936.710000ms +Mean Cycle Time: 0.749367ms (± 0.818795ms) +Cycles Per Second: 1334.459439 +GC: off + +Note: Post advisory lock implementation + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt new file mode 100644 index 0000000..6a92162 --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt @@ -0,0 +1,21 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 133606.236000ms +Mean Cycle Time: 1.336062ms (± 0.960286ms) +Cycles Per Second: 748.468058 +GC: off + +Note: Post advisory lock implementation + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 From f2f6ae13d6bbfb4ef3c4c510dde4711896ab6caf Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 20:54:32 -0500 Subject: [PATCH 322/591] Today's benchmarks are removed. Will recreate them in their entirety, including the benchmarks prior to the advisory locks. --- ...t Benchmark - 2018-06-04T23:30:23.531Z.txt | 21 ------------------- ...t Benchmark - 2018-06-04T23:22:20.699Z.txt | 21 ------------------- 2 files changed, 42 deletions(-) delete mode 100644 test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt delete mode 100644 test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt deleted file mode 100644 index 11610e4..0000000 --- a/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt +++ /dev/null @@ -1,21 +0,0 @@ -Get Benchmark -- - - -Cycles: 100000 -Time: 74936.710000ms -Mean Cycle Time: 0.749367ms (± 0.818795ms) -Cycles Per Second: 1334.459439 -GC: off - -Note: Post advisory lock implementation - -Model Name: MacBook -Model Identifier: MacBook10,1 -Processor Name: Intel Core i7 -Processor Speed: 1.4 GHz -Number of Processors: 1 -Total Number of Cores: 2 -L2 Cache (per Core): 256 KB -L3 Cache: 4 MB -Memory: 16 GB -Boot ROM Version: MB101.0156.B00 -SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt deleted file mode 100644 index 6a92162..0000000 --- a/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt +++ /dev/null @@ -1,21 +0,0 @@ -Put Benchmark -- - - -Cycles: 100000 -Time: 133606.236000ms -Mean Cycle Time: 1.336062ms (± 0.960286ms) -Cycles Per Second: 748.468058 -GC: off - -Note: Post advisory lock implementation - -Model Name: MacBook -Model Identifier: MacBook10,1 -Processor Name: Intel Core i7 -Processor Speed: 1.4 GHz -Number of Processors: 1 -Total Number of Cores: 2 -L2 Cache (per Core): 256 KB -L3 Cache: 4 MB -Memory: 16 GB -Boot ROM Version: MB101.0156.B00 -SMC Version (system): 2.42f10 From 0abe9d8f5cdcc7b86872415a966450191c330ee6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 21:29:26 -0500 Subject: [PATCH 323/591] Benchmarks are updated --- ... Benchmark - 2018-06-05T02:03:42.080Z.txt} | 8 ++++--- ...t Benchmark - 2018-06-05T02:27:02.253Z.txt | 21 ++++++++++++++++++ ... Benchmark - 2018-06-05T01:59:19.666Z.txt} | 8 ++++--- ...t Benchmark - 2018-06-05T02:14:58.644Z.txt | 22 +++++++++++++++++++ 4 files changed, 53 insertions(+), 6 deletions(-) rename test/benchmark/results/{Get Benchmark - 2018-06-01T23:25:15.505Z.txt => Get Benchmark - 2018-06-05T02:03:42.080Z.txt} (70%) create mode 100644 test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt rename test/benchmark/results/{Put Benchmark - 2018-06-01T23:37:23.568Z.txt => Put Benchmark - 2018-06-05T01:59:19.666Z.txt} (70%) create mode 100644 test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02:03:42.080Z.txt similarity index 70% rename from test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt rename to test/benchmark/results/Get Benchmark - 2018-06-05T02:03:42.080Z.txt index c200a52..5d8edf9 100644 --- a/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt +++ b/test/benchmark/results/Get Benchmark - 2018-06-05T02:03:42.080Z.txt @@ -1,11 +1,13 @@ Get Benchmark - - - Cycles: 100000 -Time: 155466.029000ms -Mean Cycle Time: 1.554660ms (± 6.764445ms) -Cycles Per Second: 643.227338 +Time: 77864.587000ms +Mean Cycle Time: 0.778646ms (± 1.020442ms) +Cycles Per Second: 1284.280876 GC: off +Note: Pre-advisory lock implementation + Model Name: MacBook Model Identifier: MacBook10,1 Processor Name: Intel Core i7 diff --git a/test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt new file mode 100644 index 0000000..70d5376 --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt @@ -0,0 +1,21 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 89734.621000ms +Mean Cycle Time: 0.897346ms (± 2.752222ms) +Cycles Per Second: 1114.397084 +GC: off + +Note: Post-advisory lock implementation + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-05T01:59:19.666Z.txt similarity index 70% rename from test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt rename to test/benchmark/results/Put Benchmark - 2018-06-05T01:59:19.666Z.txt index c1e4476..4f89f45 100644 --- a/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt +++ b/test/benchmark/results/Put Benchmark - 2018-06-05T01:59:19.666Z.txt @@ -1,11 +1,13 @@ Put Benchmark - - - Cycles: 100000 -Time: 278854.650000ms -Mean Cycle Time: 2.788546ms (± 11.991595ms) -Cycles Per Second: 358.609763 +Time: 137845.491000ms +Mean Cycle Time: 1.378455ms (± 1.311459ms) +Cycles Per Second: 725.449917 GC: off +Note: Pre-advisory lock implementation + Model Name: MacBook Model Identifier: MacBook10,1 Processor Name: Intel Core i7 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt new file mode 100644 index 0000000..d8297eb --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt @@ -0,0 +1,22 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 146689.646000ms +Mean Cycle Time: 1.466896ms (± 1.990456ms) +Cycles Per Second: 681.711373 +GC: off + +Note: Post-advisory lock implementation + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 + From 05ff30fd19e07553c0a26d888713ad93181fb3ab Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 21:42:18 -0500 Subject: [PATCH 324/591] Benchmark result file timestamps are punctuated with dashes Use of colons conflicts with OS filename rules --- test/benchmark/record_result.rb | 3 ++- ...2.080Z.txt => Get Benchmark - 2018-06-05T02-03-42.080Z.txt} | 0 ...2.253Z.txt => Get Benchmark - 2018-06-05T02-27-02.253Z.txt} | 0 ...9.666Z.txt => Put Benchmark - 2018-06-05T01-59-19.666Z.txt} | 0 ...8.644Z.txt => Put Benchmark - 2018-06-05T02-14-58.644Z.txt} | 0 5 files changed, 2 insertions(+), 1 deletion(-) rename test/benchmark/results/{Get Benchmark - 2018-06-05T02:03:42.080Z.txt => Get Benchmark - 2018-06-05T02-03-42.080Z.txt} (100%) rename test/benchmark/results/{Get Benchmark - 2018-06-05T02:27:02.253Z.txt => Get Benchmark - 2018-06-05T02-27-02.253Z.txt} (100%) rename test/benchmark/results/{Put Benchmark - 2018-06-05T01:59:19.666Z.txt => Put Benchmark - 2018-06-05T01-59-19.666Z.txt} (100%) rename test/benchmark/results/{Put Benchmark - 2018-06-05T02:14:58.644Z.txt => Put Benchmark - 2018-06-05T02-14-58.644Z.txt} (100%) diff --git a/test/benchmark/record_result.rb b/test/benchmark/record_result.rb index 729fc24..c51e817 100644 --- a/test/benchmark/record_result.rb +++ b/test/benchmark/record_result.rb @@ -35,8 +35,9 @@ def write_result_file end def filename + time = clock.iso8601.gsub(':','-') directory = Pathname.new('test/benchmark/tmp') - filename = directory.join("#{name} - #{clock.iso8601}.txt") + filename = directory.join("#{name} - #{time}.txt") filename end diff --git a/test/benchmark/results/Get Benchmark - 2018-06-05T02:03:42.080Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z.txt similarity index 100% rename from test/benchmark/results/Get Benchmark - 2018-06-05T02:03:42.080Z.txt rename to test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z.txt similarity index 100% rename from test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt rename to test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z.txt diff --git a/test/benchmark/results/Put Benchmark - 2018-06-05T01:59:19.666Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z.txt similarity index 100% rename from test/benchmark/results/Put Benchmark - 2018-06-05T01:59:19.666Z.txt rename to test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z.txt diff --git a/test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z.txt similarity index 100% rename from test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt rename to test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z.txt From 817e3c37b753c089e3ed20cdf9aaec16a81e0043 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 22:15:37 -0500 Subject: [PATCH 325/591] Package version is increased from 0.7.0.2 to 0.8.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 3748b58..7d8b1b9 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.7.0.2' + s.version = '0.8.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From fe734b7308f01ad6abc1ea5ee12b8f2b1178a996 Mon Sep 17 00:00:00 2001 From: Joshua Flanagan Date: Wed, 6 Dec 2017 20:15:29 -0600 Subject: [PATCH 326/591] Database setup scripts are more robust The scripts rely on comparing the exact output of `psql` commands to check for the existence of objects in the database. The psql output can be impacted by user preferences in their `.psqlrc` file, breaking the scripts. To reproduce, add the following line to `~/.psqlrc`: ``` \pset format wrapped ``` The install/uninstall scripts will now fail to find existing objects. The -q flag silences all output other than the query result. The -X flag ignores the user's .psqlrc so that output is more deterministic. --- database/install.sh | 2 +- database/uninstall.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/database/install.sh b/database/install.sh index 07fa5ee..5085b61 100755 --- a/database/install.sh +++ b/database/install.sh @@ -28,7 +28,7 @@ echo function create-user { echo "Database user is: $user" - user_exists=`psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` + user_exists=`psql postgres -qtAXc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` if [ "$user_exists" = "1" ]; then echo "Database user \"$user\" was previously created. Not creating again." diff --git a/database/uninstall.sh b/database/uninstall.sh index cf3117a..d635193 100755 --- a/database/uninstall.sh +++ b/database/uninstall.sh @@ -28,7 +28,7 @@ echo function delete-user { echo "Database user is: $user" - user_exists=`psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` + user_exists=`psql postgres -qtAXc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` if [ "$user_exists" = "1" ]; then echo "Deleting database user \"$user\"..." @@ -43,7 +43,7 @@ function delete-user { function delete-database { echo "Database name is: $database" - database_exists=`psql postgres -tAc "SELECT 1 FROM pg_database WHERE datname='$database'"` + database_exists=`psql postgres -qtAXc "SELECT 1 FROM pg_database WHERE datname='$database'"` if [ "$database_exists" = "1" ]; then echo "Deleting database \"$database\"..." From afb576af7994804afc2c1839279f2a54d908067b Mon Sep 17 00:00:00 2001 From: Joshua Flanagan Date: Wed, 6 Dec 2017 21:18:19 -0600 Subject: [PATCH 327/591] Remove redundant indexes There are already multicolumn indexes on stream,position and category/global_position. Those cover the two most common queries. PostgreSQL can still use those indexes for queries on stream or category alone, since they are the leftmost column: > A multicolumn B-tree index can be used with query conditions that involve any subset of the index's columns, but the index is most efficient when there are constraints on the leading (leftmost) columns. The exact rule is that equality constraints on leading columns, plus any inequality constraints on the first column that does not have an equality constraint, will be used to limit the portion of the index that is scanned https://www.postgresql.org/docs/9.6/static/indexes-multicolumn.html The redundant single column indexes may have a slight performance advantage over the multi-column indexes for queries that only use the first column (since the indexes are smaller), but those queries are likely rare, and probably not worth the cost of the redundant indexes (disk space, and slower INSERTs). --- database/indexes/messages-category.sql | 1 - database/indexes/messages-stream-name.sql | 1 - database/install.sh | 4 ---- 3 files changed, 6 deletions(-) delete mode 100644 database/indexes/messages-category.sql delete mode 100644 database/indexes/messages-stream-name.sql diff --git a/database/indexes/messages-category.sql b/database/indexes/messages-category.sql deleted file mode 100644 index 0ee9a57..0000000 --- a/database/indexes/messages-category.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE INDEX CONCURRENTLY "messages_category_idx" ON "public"."messages" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); diff --git a/database/indexes/messages-stream-name.sql b/database/indexes/messages-stream-name.sql deleted file mode 100644 index d9d3518..0000000 --- a/database/indexes/messages-stream-name.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE INDEX CONCURRENTLY "messages_stream_name_idx" ON "public"."messages" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST); diff --git a/database/install.sh b/database/install.sh index 5085b61..74ce803 100755 --- a/database/install.sh +++ b/database/install.sh @@ -86,10 +86,6 @@ function create-indexes { psql $database -f $base/indexes/messages-id.sql echo "messages_category_global_position_idx" psql $database -f $base/indexes/messages-category-global-position.sql - echo "messages_category_idx" - psql $database -f $base/indexes/messages-category.sql - echo "messages_stream_name_idx" - psql $database -f $base/indexes/messages-stream-name.sql echo "messages_stream_name_position_uniq_idx" psql $database -f $base/indexes/messages-stream-name-position-uniq.sql echo From 0e619a7593bb5b35dd65872a83ee2038e8533126 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 1 Jan 2018 19:28:15 -0600 Subject: [PATCH 328/591] Package version increased from 0.5.0.0 to 0.6.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index a28a176..7e31519 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.5.0.0' + s.version = '0.6.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From aa5f34ce547dbd4d98709a3ef0198feb4b29ec94 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 3 Apr 2018 17:45:11 -0500 Subject: [PATCH 329/591] Transform::Read interface use is updated --- lib/message_store/postgres/get.rb | 4 ++-- lib/message_store/postgres/get/last.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 2d0452f..65c001a 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -74,12 +74,12 @@ def convert(records) module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? - Transform::Read.(serialized_data, MessageData::Hash, :json) + Transform::Read.(serialized_data, :json, MessageData::Hash) end def self.metadata(serialized_metadata) return nil if serialized_metadata.nil? - Transform::Read.(serialized_metadata, MessageData::Hash, :json) + Transform::Read.(serialized_metadata, :json, MessageData::Hash) end end diff --git a/lib/message_store/postgres/get/last.rb b/lib/message_store/postgres/get/last.rb index bd47a55..7013d97 100644 --- a/lib/message_store/postgres/get/last.rb +++ b/lib/message_store/postgres/get/last.rb @@ -74,12 +74,12 @@ def __convert(records) module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? - Transform::Read.(serialized_data, MessageData::Hash, :json) + Transform::Read.(serialized_data, :json, MessageData::Hash) end def self.metadata(serialized_metadata) return nil if serialized_metadata.nil? - Transform::Read.(serialized_metadata, MessageData::Hash, :json) + Transform::Read.(serialized_metadata, :json, MessageData::Hash) end end From 47a9ce1f08483ab356f7c966f3bb6f192ade92d7 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 3 Apr 2018 17:45:54 -0500 Subject: [PATCH 330/591] Obsolete code is removed --- lib/message_store/postgres/get/last.rb | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/lib/message_store/postgres/get/last.rb b/lib/message_store/postgres/get/last.rb index 7013d97..d9911a5 100644 --- a/lib/message_store/postgres/get/last.rb +++ b/lib/message_store/postgres/get/last.rb @@ -53,24 +53,6 @@ def convert(record) message_data end - def __convert(records) - logger.trace { "Converting records to message data (Records Count: #{records.ntuples})" } - - messages = records.map do |record| - record['data'] = Deserialize.data(record['data']) - record['metadata'] = Deserialize.metadata(record['metadata']) - record['time'] = Time.utc_coerced(record['time']) - - MessageData::Read.build record - - break - end - - logger.debug { "Converted records to message data (Message Data Count: #{messages.length})" } - - messages - end - module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? From 6d2a38555118a58fb4b5181a9a611674a46dcf89 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 3 Apr 2018 17:46:29 -0500 Subject: [PATCH 331/591] Package version is increased from 0.6.0.0 to 0.6.0.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 7e31519..eba1030 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.6.0.0' + s.version = '0.6.0.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 132605864bbae526ec6f57909d1db8971133c3ec Mon Sep 17 00:00:00 2001 From: Will Howard Date: Tue, 24 Apr 2018 21:34:59 -0400 Subject: [PATCH 332/591] use expanded table output when listing messages from postgres --- database/list-messages.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/list-messages.sh b/database/list-messages.sh index 8f637a9..9d6f968 100755 --- a/database/list-messages.sh +++ b/database/list-messages.sh @@ -46,7 +46,7 @@ fi echo if [ -z $stream_name ]; then - psql $database -c "SELECT * FROM $table" + psql $database -x -c "SELECT * FROM $table" else - psql $database -c "SELECT * FROM $table WHERE stream_name = '$stream_name'" + psql $database -x -c "SELECT * FROM $table WHERE stream_name = '$stream_name'" fi From a94b16ad5207418467983317e09073667a47b166 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 24 Apr 2018 21:04:52 -0500 Subject: [PATCH 333/591] Package version is increased from 0.6.0.1 to 0.6.0.2 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index eba1030..0a952ea 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.6.0.1' + s.version = '0.6.0.2' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From f86f18061b1c4a7016aa67725794667a840002d8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 24 Apr 2018 22:13:56 -0500 Subject: [PATCH 334/591] Print messages, rather than list messages --- database/{list-messages.sh => print-messages.sh} | 2 +- scripts/{evt-pg-list-messages => evt-pg-print-messages} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename database/{list-messages.sh => print-messages.sh} (97%) rename scripts/{evt-pg-list-messages => evt-pg-print-messages} (78%) diff --git a/database/list-messages.sh b/database/print-messages.sh similarity index 97% rename from database/list-messages.sh rename to database/print-messages.sh index 9d6f968..111b394 100755 --- a/database/list-messages.sh +++ b/database/print-messages.sh @@ -3,7 +3,7 @@ set -e echo -echo "Listing Messages" +echo "Printing Messages" echo "= = =" echo diff --git a/scripts/evt-pg-list-messages b/scripts/evt-pg-print-messages similarity index 78% rename from scripts/evt-pg-list-messages rename to scripts/evt-pg-print-messages index b7ab7fd..17e5924 100755 --- a/scripts/evt-pg-list-messages +++ b/scripts/evt-pg-print-messages @@ -1,7 +1,7 @@ #!/usr/bin/env ruby root = File.expand_path '../database', __dir__ -script_filename = 'list-messages.sh' +script_filename = 'print-messages.sh' script_filepath = File.join root, script_filename system script_filepath From cfc9716ebf48c56e08a63efa9f1b25768dd9e56a Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 24 Apr 2018 22:15:00 -0500 Subject: [PATCH 335/591] Package version is increased from 0.6.0.2 to 0.7.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 0a952ea..bc8be31 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.6.0.2' + s.version = '0.7.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 026e1265587c7c9117c36fc7fb72579dd2791c23 Mon Sep 17 00:00:00 2001 From: Fire-Dragon-DoL Date: Tue, 24 Apr 2018 22:05:44 -0700 Subject: [PATCH 336/591] Fix Get select_statement position type --- lib/message_store/postgres/get/last/select_statement.rb | 2 +- lib/message_store/postgres/get/select_statement.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/message_store/postgres/get/last/select_statement.rb b/lib/message_store/postgres/get/last/select_statement.rb index ca0647a..96cd4d1 100644 --- a/lib/message_store/postgres/get/last/select_statement.rb +++ b/lib/message_store/postgres/get/last/select_statement.rb @@ -18,7 +18,7 @@ def sql SELECT id::varchar, stream_name::varchar, - position::int, + position::bigint, type::varchar, global_position::bigint, data::varchar, diff --git a/lib/message_store/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb index 279d67f..0fb4c1e 100644 --- a/lib/message_store/postgres/get/select_statement.rb +++ b/lib/message_store/postgres/get/select_statement.rb @@ -35,7 +35,7 @@ def sql SELECT id::varchar, stream_name::varchar, - position::int, + position::bigint, type::varchar, global_position::bigint, data::varchar, From 78bcd3408c73d48b4e4e8f0037f9bed5238cdb38 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 25 Apr 2018 00:10:16 -0500 Subject: [PATCH 337/591] Vestigial code is removed --- lib/message_store/postgres/get.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 65c001a..5d13bf5 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -44,8 +44,6 @@ def call(stream_name, position: nil) def get_records(stream_name, position) logger.trace { "Getting records (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } - where_fragment = self.condition - select_statement = SelectStatement.build(stream_name, position: position, batch_size: batch_size, condition: condition) records = session.execute(select_statement.sql) From d6280f324134221a3c07e0bbd5bd2a755accfb33 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 25 Apr 2018 00:13:39 -0500 Subject: [PATCH 338/591] Package version is increased from 0.7.0.0 to 0.7.0.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index bc8be31..9b96fe2 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.7.0.0' + s.version = '0.7.0.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 017cf23acfafceabbaa07e51669d7a47b06a4b44 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 30 May 2018 18:07:39 -0500 Subject: [PATCH 339/591] Rudimentary write benchmark --- message_store-postgres.gemspec | 1 + test/benchmark/benchmark_init.rb | 3 +++ test/benchmark/write.rb | 26 ++++++++++++++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 test/benchmark/benchmark_init.rb create mode 100644 test/benchmark/write.rb diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 9b96fe2..9b9ee15 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -26,5 +26,6 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'pg' s.add_development_dependency 'test_bench' + s.add_development_dependency 'evt-diagnostics-sample' s.add_development_dependency 'ntl-actor' end diff --git a/test/benchmark/benchmark_init.rb b/test/benchmark/benchmark_init.rb new file mode 100644 index 0000000..76c1854 --- /dev/null +++ b/test/benchmark/benchmark_init.rb @@ -0,0 +1,3 @@ +require_relative '../test_init' + +require 'diagnostics/sample' diff --git a/test/benchmark/write.rb b/test/benchmark/write.rb new file mode 100644 index 0000000..3e6ea4a --- /dev/null +++ b/test/benchmark/write.rb @@ -0,0 +1,26 @@ +require_relative 'benchmark_init' +require 'benchmark' + +warmup_cycles = 10 +test_cycles = 10_000 +cycles = warmup_cycles + test_cycles + +Entry = Struct.new(:stream_name, :message_data) + +entries = [] +cycles.times do + stream_name = Controls::StreamName.example + write_message = Controls::MessageData::Write.example + + entries << Entry.new(stream_name, write_message) +end + +session = Session.build +write = Write.build(session: session) + +result = Diagnostics::Sample.(test_cycles, warmup_cycles: warmup_cycles, gc: true) do |i| + entry = entries[i] + write.(entry.message_data, entry.stream_name) +end + +puts result From 621d2ee7a254bf25d85cc7b31b718aba12979efa Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 30 May 2018 22:04:58 -0500 Subject: [PATCH 340/591] Write benchmark structure is stable. Next: moves writes to controls --- .../postgres/controls/message_data.rb | 3 ++ test/benchmark/benchmark_init.rb | 1 + test/benchmark/defaults.rb | 33 +++++++++++++++++++ test/benchmark/{ => write}/write.rb | 19 ++++++----- 4 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 test/benchmark/defaults.rb rename test/benchmark/{ => write}/write.rb (50%) diff --git a/lib/message_store/postgres/controls/message_data.rb b/lib/message_store/postgres/controls/message_data.rb index 0febad2..9fede3a 100644 --- a/lib/message_store/postgres/controls/message_data.rb +++ b/lib/message_store/postgres/controls/message_data.rb @@ -2,6 +2,9 @@ module MessageStore module Postgres module Controls MessageData = MessageStore::Controls::MessageData + + module MessageData + end end end end diff --git a/test/benchmark/benchmark_init.rb b/test/benchmark/benchmark_init.rb index 76c1854..49f1f2f 100644 --- a/test/benchmark/benchmark_init.rb +++ b/test/benchmark/benchmark_init.rb @@ -1,3 +1,4 @@ require_relative '../test_init' require 'diagnostics/sample' +require_relative 'defaults' diff --git a/test/benchmark/defaults.rb b/test/benchmark/defaults.rb new file mode 100644 index 0000000..95eb0e5 --- /dev/null +++ b/test/benchmark/defaults.rb @@ -0,0 +1,33 @@ +module Benchmark + class Defaults + initializer :cycles, :warmup_cycles, :gc + + def total_cycles + cycles + warmup_cycles + end + + def to_s + <<~TEXT + Cycles: #{cycles} + Warmup Cycles: #{warmup_cycles} + GC: #{gc} + TEXT + end + + def self.build + new(cycles, warmup_cycles, gc) + end + + def self.cycles + Integer(ENV['CYCLES'] || 2500) + end + + def self.warmup_cycles + Integer(ENV['WARMUP_CYCLES'] || 10) + end + + def self.gc + ['on', 'true'].include?(ENV['GC']) ? true : false + end + end +end diff --git a/test/benchmark/write.rb b/test/benchmark/write/write.rb similarity index 50% rename from test/benchmark/write.rb rename to test/benchmark/write/write.rb index 3e6ea4a..9ee71f6 100644 --- a/test/benchmark/write.rb +++ b/test/benchmark/write/write.rb @@ -1,26 +1,27 @@ -require_relative 'benchmark_init' -require 'benchmark' +require_relative '../benchmark_init' -warmup_cycles = 10 -test_cycles = 10_000 -cycles = warmup_cycles + test_cycles +defaults = Benchmark::Defaults.build + +puts +puts 'Write Benchmark' +puts '- - -' Entry = Struct.new(:stream_name, :message_data) entries = [] -cycles.times do +defaults.total_cycles.times do stream_name = Controls::StreamName.example write_message = Controls::MessageData::Write.example entries << Entry.new(stream_name, write_message) end -session = Session.build -write = Write.build(session: session) +write = Write.build -result = Diagnostics::Sample.(test_cycles, warmup_cycles: warmup_cycles, gc: true) do |i| +result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = entries[i] write.(entry.message_data, entry.stream_name) end puts result +puts From 72ccd0b91ca9662ea27246f0ab5148d16aa08199 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 30 May 2018 23:06:15 -0500 Subject: [PATCH 341/591] Message data used in write benchmark is included in the controls --- .../postgres/controls/message_data.rb | 21 +++++++++++++++++++ test/benchmark/write/write.rb | 12 ++--------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/lib/message_store/postgres/controls/message_data.rb b/lib/message_store/postgres/controls/message_data.rb index 9fede3a..b819973 100644 --- a/lib/message_store/postgres/controls/message_data.rb +++ b/lib/message_store/postgres/controls/message_data.rb @@ -4,6 +4,27 @@ module Controls MessageData = MessageStore::Controls::MessageData module MessageData + module Write + module List + Entry = Struct.new(:stream_name, :message_data) + + def self.get(instances: nil) + instances ||= 1 + + stream_name = StreamName.example + + list = [] + instances.times do + stream_name = Controls::StreamName.example + write_message = Controls::MessageData::Write.example + + list << Entry.new(stream_name, write_message) + end + + list + end + end + end end end end diff --git a/test/benchmark/write/write.rb b/test/benchmark/write/write.rb index 9ee71f6..87d4487 100644 --- a/test/benchmark/write/write.rb +++ b/test/benchmark/write/write.rb @@ -6,20 +6,12 @@ puts 'Write Benchmark' puts '- - -' -Entry = Struct.new(:stream_name, :message_data) - -entries = [] -defaults.total_cycles.times do - stream_name = Controls::StreamName.example - write_message = Controls::MessageData::Write.example - - entries << Entry.new(stream_name, write_message) -end +list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) write = Write.build result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| - entry = entries[i] + entry = list[i] write.(entry.message_data, entry.stream_name) end From 655b3f780d89082b5a66d75f067d0360da9b0ca7 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 30 May 2018 23:11:26 -0500 Subject: [PATCH 342/591] Write benchmark is replaced with put benchmark --- test/benchmark/{write/write.rb => put.rb} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename test/benchmark/{write/write.rb => put.rb} (69%) diff --git a/test/benchmark/write/write.rb b/test/benchmark/put.rb similarity index 69% rename from test/benchmark/write/write.rb rename to test/benchmark/put.rb index 87d4487..a41a19c 100644 --- a/test/benchmark/write/write.rb +++ b/test/benchmark/put.rb @@ -1,18 +1,18 @@ -require_relative '../benchmark_init' +require_relative 'benchmark_init' defaults = Benchmark::Defaults.build puts -puts 'Write Benchmark' +puts 'Put Benchmark' puts '- - -' list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) -write = Write.build +put = Put.build result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] - write.(entry.message_data, entry.stream_name) + put.(entry.message_data, entry.stream_name) end puts result From a8b3fe04d38d81c1bc64bbe015573d4758a2d97b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 31 May 2018 02:11:00 -0500 Subject: [PATCH 343/591] Get benchmark --- test/benchmark/get.rb | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 test/benchmark/get.rb diff --git a/test/benchmark/get.rb b/test/benchmark/get.rb new file mode 100644 index 0000000..ce44889 --- /dev/null +++ b/test/benchmark/get.rb @@ -0,0 +1,25 @@ +require_relative 'benchmark_init' + +defaults = Benchmark::Defaults.build + +puts +puts 'Get Benchmark' +puts '- - -' + +list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) + +put = Put.build + +list.each do |entry| + put.(entry.message_data, entry.stream_name) +end + +get = Get.build + +result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| + entry = list[i] + get.(entry.stream_name) +end + +puts result +puts From 2e8fad4504c9a4509d380770c53a086ae7f178ac Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 31 May 2018 02:17:27 -0500 Subject: [PATCH 344/591] Superfluous control implementation is replaced with an alias --- lib/message_store/postgres/controls/category.rb | 9 --------- lib/message_store/postgres/controls/stream_name.rb | 9 +-------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/lib/message_store/postgres/controls/category.rb b/lib/message_store/postgres/controls/category.rb index 0d717eb..9678269 100644 --- a/lib/message_store/postgres/controls/category.rb +++ b/lib/message_store/postgres/controls/category.rb @@ -1,12 +1,3 @@ -# module MessageStore -# module Postgres -# module Controls -# Category = MessageStore::Controls::Category -# end -# end -# end - - module MessageStore module Postgres module Controls diff --git a/lib/message_store/postgres/controls/stream_name.rb b/lib/message_store/postgres/controls/stream_name.rb index 70e854a..b35f009 100644 --- a/lib/message_store/postgres/controls/stream_name.rb +++ b/lib/message_store/postgres/controls/stream_name.rb @@ -1,14 +1,7 @@ module MessageStore module Postgres module Controls - module StreamName - def self.example(category: nil, id: nil, type: nil, types: nil, randomize_category: nil) - category ||= Category.example(category: category, randomize_category: randomize_category) - id ||= Identifier::UUID.random - - MessageStore::Postgres::StreamName.stream_name(category, id, type: type, types: types) - end - end + StreamName = MessageStore::Controls::StreamName end end end From b11a7f8edb5de1641a62748d64c9c3dec10922ea Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 31 May 2018 02:20:32 -0500 Subject: [PATCH 345/591] Package version is increased from 0.7.0.1 to 0.7.0.2 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 9b9ee15..942a670 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.7.0.1' + s.version = '0.7.0.2' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 7f54aba35114ce3720c9e300436adb81c0f70bf1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 16:01:42 -0500 Subject: [PATCH 346/591] Record benchmark results to a file --- .gitignore | 1 + test/benchmark/benchmark_init.rb | 4 +++ test/benchmark/put.rb | 7 +--- test/benchmark/record_result.rb | 57 ++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 test/benchmark/record_result.rb diff --git a/.gitignore b/.gitignore index 3bb1e20..740a161 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ gems *notes* loader.rb /test/package/installed +/test/benchmark/tmp/**/* diff --git a/test/benchmark/benchmark_init.rb b/test/benchmark/benchmark_init.rb index 49f1f2f..9226419 100644 --- a/test/benchmark/benchmark_init.rb +++ b/test/benchmark/benchmark_init.rb @@ -1,4 +1,8 @@ require_relative '../test_init' require 'diagnostics/sample' +require 'fileutils' +require 'pathname' + require_relative 'defaults' +require_relative 'record_result' diff --git a/test/benchmark/put.rb b/test/benchmark/put.rb index a41a19c..a1d362b 100644 --- a/test/benchmark/put.rb +++ b/test/benchmark/put.rb @@ -2,10 +2,6 @@ defaults = Benchmark::Defaults.build -puts -puts 'Put Benchmark' -puts '- - -' - list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) put = Put.build @@ -15,5 +11,4 @@ put.(entry.message_data, entry.stream_name) end -puts result -puts +Benchmark::RecordResult.('Put Benchmark', result) diff --git a/test/benchmark/record_result.rb b/test/benchmark/record_result.rb new file mode 100644 index 0000000..eb154dd --- /dev/null +++ b/test/benchmark/record_result.rb @@ -0,0 +1,57 @@ +module Benchmark + class RecordResult + dependency :clock + + initializer :name, :content + + def self.build(name, content) + instance = new(name, content) + instance.configure + instance + end + + def self.call(name, content) + instance = build(name, content) + instance.() + end + + def call + digest = write_result_file + + puts + puts digest + puts + end + + def write_result_file + fn = filename + + FileUtils.mkdir_p(fn.dirname) + + d = digest + fn.write(d) + + d + end + + def filename + directory = Pathname.new('test/benchmark/tmp') + filename = directory.join("#{name} - #{clock.iso8601}.txt") + + filename + end + + def digest + <<~TEXT + #{name} + - - - + #{content} + (#{filename}) + TEXT + end + + def configure + Clock::UTC.configure(self) + end + end +end From 5c7a25a370c419e6359f0fd21b8b2a3db50ba54a Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 17:02:56 -0500 Subject: [PATCH 347/591] Benchmark output is formatted --- test/benchmark/put.rb | 6 +++++- test/benchmark/record_result.rb | 10 +++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/benchmark/put.rb b/test/benchmark/put.rb index a1d362b..d753f39 100644 --- a/test/benchmark/put.rb +++ b/test/benchmark/put.rb @@ -11,4 +11,8 @@ put.(entry.message_data, entry.stream_name) end -Benchmark::RecordResult.('Put Benchmark', result) +puts +filename = Benchmark::RecordResult.('Put Benchmark', result) +puts +puts filename +puts diff --git a/test/benchmark/record_result.rb b/test/benchmark/record_result.rb index eb154dd..729fc24 100644 --- a/test/benchmark/record_result.rb +++ b/test/benchmark/record_result.rb @@ -16,11 +16,11 @@ def self.call(name, content) end def call - digest = write_result_file + digest, filename = write_result_file - puts puts digest - puts + + filename.to_s end def write_result_file @@ -31,7 +31,7 @@ def write_result_file d = digest fn.write(d) - d + [d, fn] end def filename @@ -46,8 +46,8 @@ def digest #{name} - - - #{content} - (#{filename}) TEXT + .chomp end def configure From 0bc0b3711c443d0cde4dea8309320ba10b9980b0 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 17:40:54 -0500 Subject: [PATCH 348/591] Get benchmark results are recorded --- test/benchmark/get.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/benchmark/get.rb b/test/benchmark/get.rb index ce44889..a6854e6 100644 --- a/test/benchmark/get.rb +++ b/test/benchmark/get.rb @@ -2,10 +2,6 @@ defaults = Benchmark::Defaults.build -puts -puts 'Get Benchmark' -puts '- - -' - list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) put = Put.build @@ -21,5 +17,8 @@ get.(entry.stream_name) end -puts result +puts +filename = Benchmark::RecordResult.('Get Benchmark', result) +puts +puts filename puts From db52cdd07e7bf1b42e9d2bd1e27b01d2cbefdc3e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 17:42:35 -0500 Subject: [PATCH 349/591] Baseline benchmarks for get and put --- .../results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt | 7 +++++++ .../results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt | 7 +++++++ 2 files changed, 14 insertions(+) create mode 100644 test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt create mode 100644 test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt new file mode 100644 index 0000000..1d9bcff --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt @@ -0,0 +1,7 @@ +Get Benchmark +- - - +Cycles: 20000 +Time: 34606.411000ms +Mean Cycle Time: 1.730321ms (± 9.642108ms) +Cycles Per Second: 577.927598 +GC: off diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt new file mode 100644 index 0000000..7758f2e --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt @@ -0,0 +1,7 @@ +Put Benchmark +- - - +Cycles: 20000 +Time: 62569.619000ms +Mean Cycle Time: 3.128481ms (± 16.107108ms) +Cycles Per Second: 319.643947 +GC: off From 9461629a410340bfc14f87e8087681f5e2ad67cf Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 18:17:20 -0500 Subject: [PATCH 350/591] Baseline benchmarks for get and put for 100,000 cycles --- .../results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt | 7 ------- .../results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt | 7 +++++++ .../results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt | 7 ------- .../results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt | 7 +++++++ 4 files changed, 14 insertions(+), 14 deletions(-) delete mode 100644 test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt create mode 100644 test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt delete mode 100644 test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt create mode 100644 test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt deleted file mode 100644 index 1d9bcff..0000000 --- a/test/benchmark/results/Get Benchmark - 2018-06-01T22:29:04.178Z.txt +++ /dev/null @@ -1,7 +0,0 @@ -Get Benchmark -- - - -Cycles: 20000 -Time: 34606.411000ms -Mean Cycle Time: 1.730321ms (± 9.642108ms) -Cycles Per Second: 577.927598 -GC: off diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt new file mode 100644 index 0000000..424c4e2 --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt @@ -0,0 +1,7 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 207252.004000ms +Mean Cycle Time: 2.072520ms (± 8.178097ms) +Cycles Per Second: 482.504381 +GC: off diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt deleted file mode 100644 index 7758f2e..0000000 --- a/test/benchmark/results/Put Benchmark - 2018-06-01T22:06:17.972Z.txt +++ /dev/null @@ -1,7 +0,0 @@ -Put Benchmark -- - - -Cycles: 20000 -Time: 62569.619000ms -Mean Cycle Time: 3.128481ms (± 16.107108ms) -Cycles Per Second: 319.643947 -GC: off diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt new file mode 100644 index 0000000..34cde09 --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt @@ -0,0 +1,7 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 164384.657000ms +Mean Cycle Time: 1.643847ms (± 10.087307ms) +Cycles Per Second: 608.329280 +GC: off From 1743356b9fd3a8d20215a84dad4f8ace69378a72 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 18:39:53 -0500 Subject: [PATCH 351/591] Baseline benchmarks for get and put for 100,000 cycles on a new database --- .../results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt | 7 ------- .../results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt | 7 +++++++ .../results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt | 7 ------- .../results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt | 7 +++++++ 4 files changed, 14 insertions(+), 14 deletions(-) delete mode 100644 test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt create mode 100644 test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt delete mode 100644 test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt create mode 100644 test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt deleted file mode 100644 index 424c4e2..0000000 --- a/test/benchmark/results/Get Benchmark - 2018-06-01T23:13:18.203Z.txt +++ /dev/null @@ -1,7 +0,0 @@ -Get Benchmark -- - - -Cycles: 100000 -Time: 207252.004000ms -Mean Cycle Time: 2.072520ms (± 8.178097ms) -Cycles Per Second: 482.504381 -GC: off diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt new file mode 100644 index 0000000..a546f8f --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt @@ -0,0 +1,7 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 155466.029000ms +Mean Cycle Time: 1.554660ms (± 6.764445ms) +Cycles Per Second: 643.227338 +GC: off diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt deleted file mode 100644 index 34cde09..0000000 --- a/test/benchmark/results/Put Benchmark - 2018-06-01T22:46:57.539Z.txt +++ /dev/null @@ -1,7 +0,0 @@ -Put Benchmark -- - - -Cycles: 100000 -Time: 164384.657000ms -Mean Cycle Time: 1.643847ms (± 10.087307ms) -Cycles Per Second: 608.329280 -GC: off diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt new file mode 100644 index 0000000..6f5da4c --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt @@ -0,0 +1,7 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 278854.650000ms +Mean Cycle Time: 2.788546ms (± 11.991595ms) +Cycles Per Second: 358.609763 +GC: off From e03148b473836599bc19ba04a4ef90fecf2dfbd3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Jun 2018 18:42:23 -0500 Subject: [PATCH 352/591] System information is included in benchmarks --- .../Get Benchmark - 2018-06-01T23:25:15.505Z.txt | 12 ++++++++++++ .../Put Benchmark - 2018-06-01T23:37:23.568Z.txt | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt index a546f8f..c200a52 100644 --- a/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt +++ b/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt @@ -5,3 +5,15 @@ Time: 155466.029000ms Mean Cycle Time: 1.554660ms (± 6.764445ms) Cycles Per Second: 643.227338 GC: off + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt index 6f5da4c..c1e4476 100644 --- a/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt +++ b/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt @@ -5,3 +5,15 @@ Time: 278854.650000ms Mean Cycle Time: 2.788546ms (± 11.991595ms) Cycles Per Second: 358.609763 GC: off + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 From d60b527ec685108f32b980cc033dd9e5b297ffb1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 2 Jun 2018 21:55:20 -0500 Subject: [PATCH 353/591] Hash server function converts a string (typically a stream name) to a 64-bit integer --- database/functions/hash-64.sql | 12 ++++++++++++ database/install.sh | 7 +++++++ 2 files changed, 19 insertions(+) create mode 100644 database/functions/hash-64.sql diff --git a/database/functions/hash-64.sql b/database/functions/hash-64.sql new file mode 100644 index 0000000..a1aace9 --- /dev/null +++ b/database/functions/hash-64.sql @@ -0,0 +1,12 @@ +CREATE OR REPLACE FUNCTION hash_64( + _stream_name varchar +) +RETURNS bigint +AS $$ +DECLARE + hash bigint; +BEGIN + select @left('x' || md5(_stream_name), 16)::bit(64)::bigint into hash; + return hash; +END; +$$ LANGUAGE plpgsql diff --git a/database/install.sh b/database/install.sh index 74ce803..6741458 100755 --- a/database/install.sh +++ b/database/install.sh @@ -70,12 +70,19 @@ function create-table { function create-functions { base=$(script_dir) echo "Creating functions..." + + echo "hash_64 function" + psql $database -f $base/functions/hash-64.sql + echo "category function" psql $database -f $base/functions/category.sql + echo "stream_version function" psql $database -f $base/functions/stream-version.sql + echo "write_sql function" psql $database -f $base/functions/write-message.sql + echo } From 3973a5979f2454e3f5f489109ce170a5063ea424 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 3 Jun 2018 18:36:06 -0500 Subject: [PATCH 354/591] Test script for the 64-bit hash of strings --- database/test/hash-64.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 database/test/hash-64.sql diff --git a/database/test/hash-64.sql b/database/test/hash-64.sql new file mode 100644 index 0000000..3d958c7 --- /dev/null +++ b/database/test/hash-64.sql @@ -0,0 +1 @@ +SELECT hash_64('someStream-123'); From 3e1d2c12dc6cf1ce0efdcee84484ca026dec3ded Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 3 Jun 2018 22:57:09 -0500 Subject: [PATCH 355/591] Stream write concurrency is serialized with advisory locks --- database/functions/write-message.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/database/functions/write-message.sql b/database/functions/write-message.sql index a16f45e..939d90d 100644 --- a/database/functions/write-message.sql +++ b/database/functions/write-message.sql @@ -13,9 +13,13 @@ DECLARE stream_version bigint; position bigint; category varchar; + stream_name_hash bigint; BEGIN message_id = uuid(_id); + stream_name_hash = hash_64(_stream_name); + PERFORM pg_advisory_lock(stream_name_hash); + stream_version := stream_version(_stream_name); if stream_version is null then @@ -24,6 +28,7 @@ BEGIN if _expected_version is not null then if _expected_version != stream_version then + PERFORM pg_advisory_unlock(stream_name_hash); raise exception 'Wrong expected version: % (Stream: %, Stream Version: %)', _expected_version, _stream_name, stream_version; end if; end if; @@ -50,6 +55,8 @@ BEGIN ) ; + PERFORM pg_advisory_unlock(stream_name_hash); + return position; END; $$ LANGUAGE plpgsql; From 7efac7f1227ca418a443b4f6b7c28eeaa54ec6e0 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 09:27:15 -0500 Subject: [PATCH 356/591] Merge conflict is resolved --- test/interactive/interactive_init.rb | 3 +++ test/interactive/write.rb | 14 ++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 test/interactive/write.rb diff --git a/test/interactive/interactive_init.rb b/test/interactive/interactive_init.rb index 0893a61..2eb9cf5 100644 --- a/test/interactive/interactive_init.rb +++ b/test/interactive/interactive_init.rb @@ -1,4 +1,7 @@ +<<<<<<< HEAD ENV['LOG_LEVEL'] ||= 'info' ENV['LOG_TAGS'] ||= '_untagged,message_store_postgres,-data' +======= +>>>>>>> Interactive test script for concurrent writes require_relative '../test_init' diff --git a/test/interactive/write.rb b/test/interactive/write.rb new file mode 100644 index 0000000..7af8899 --- /dev/null +++ b/test/interactive/write.rb @@ -0,0 +1,14 @@ +require_relative 'interactive_init' + +message_data = Controls::MessageData::Write.example +stream_name = 'testWriteIsolation' + +write = Write.build + +i = 0 +loop do + position = write.(message_data, stream_name) + puts "i: #{i}, p: #{position}" + i += 1 +end + From 79fe7c2dd2e6122ae11b0b571d564ae20ea421f0 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 3 Jun 2018 22:59:36 -0500 Subject: [PATCH 357/591] Interactive test script for concurrent writes --- test/interactive/interactive_init.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/interactive/interactive_init.rb b/test/interactive/interactive_init.rb index 2eb9cf5..0893a61 100644 --- a/test/interactive/interactive_init.rb +++ b/test/interactive/interactive_init.rb @@ -1,7 +1,4 @@ -<<<<<<< HEAD ENV['LOG_LEVEL'] ||= 'info' ENV['LOG_TAGS'] ||= '_untagged,message_store_postgres,-data' -======= ->>>>>>> Interactive test script for concurrent writes require_relative '../test_init' From 5303b3a085bea07247c1bc7c0fac7d7f7e9216be Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 18:11:02 -0500 Subject: [PATCH 358/591] Advisory locks baseline implementation --- database/functions/write-message.sql | 5 +---- database/test/write-message-expected-version.sql | 2 +- database/test/write-message.sql | 2 +- test/interactive/write.rb | 7 +++++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/database/functions/write-message.sql b/database/functions/write-message.sql index 939d90d..1f95698 100644 --- a/database/functions/write-message.sql +++ b/database/functions/write-message.sql @@ -18,7 +18,7 @@ BEGIN message_id = uuid(_id); stream_name_hash = hash_64(_stream_name); - PERFORM pg_advisory_lock(stream_name_hash); + PERFORM pg_advisory_xact_lock(stream_name_hash); stream_version := stream_version(_stream_name); @@ -28,7 +28,6 @@ BEGIN if _expected_version is not null then if _expected_version != stream_version then - PERFORM pg_advisory_unlock(stream_name_hash); raise exception 'Wrong expected version: % (Stream: %, Stream Version: %)', _expected_version, _stream_name, stream_version; end if; end if; @@ -55,8 +54,6 @@ BEGIN ) ; - PERFORM pg_advisory_unlock(stream_name_hash); - return position; END; $$ LANGUAGE plpgsql; diff --git a/database/test/write-message-expected-version.sql b/database/test/write-message-expected-version.sql index 330c5b6..be4ab20 100644 --- a/database/test/write-message-expected-version.sql +++ b/database/test/write-message-expected-version.sql @@ -1 +1 @@ -SELECT write_message('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}', 1); +SELECT write_message(gen_random_uuid()::varchar, 'testWriteIsolation'::varchar, 'SomeType'::varchar, '{"attribute": "79a59e513f9182d5754abdf9b90e2179"}'::jsonb, '{"metaAttribute": "6b3ae03d713a51a445290f3fc4b5223a"}'::jsonb, 1::bigint); diff --git a/database/test/write-message.sql b/database/test/write-message.sql index 6f17139..9ae0c5b 100644 --- a/database/test/write-message.sql +++ b/database/test/write-message.sql @@ -1 +1 @@ -SELECT write_message('someStream-123', 'SomeType', '{"someField":"some value"}', '{"someMetadataField":"some metadata value"}'); +SELECT write_message(gen_random_uuid()::varchar, 'testWriteIsolation'::varchar, 'SomeType'::varchar, '{"attribute": "79a59e513f9182d5754abdf9b90e2179"}'::jsonb, '{"metaAttribute": "6b3ae03d713a51a445290f3fc4b5223a"}'::jsonb); diff --git a/test/interactive/write.rb b/test/interactive/write.rb index 7af8899..57c9409 100644 --- a/test/interactive/write.rb +++ b/test/interactive/write.rb @@ -1,13 +1,16 @@ require_relative 'interactive_init' -message_data = Controls::MessageData::Write.example stream_name = 'testWriteIsolation' +message_data_1 = Controls::MessageData::Write.example(data: { attribute: 1 }) +message_data_2 = Controls::MessageData::Write.example(data: { attribute: 2 }) +batch = [message_data_1, message_data_2] + write = Write.build i = 0 loop do - position = write.(message_data, stream_name) + position = write.(batch, stream_name) puts "i: #{i}, p: #{position}" i += 1 end From 9fb73dbdba63d30681fe9a22e3bdb2b4afb43f55 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 18:13:59 -0500 Subject: [PATCH 359/591] Hash value does not have to be unsigned (positive) --- database/functions/hash-64.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/functions/hash-64.sql b/database/functions/hash-64.sql index a1aace9..ab6fa4d 100644 --- a/database/functions/hash-64.sql +++ b/database/functions/hash-64.sql @@ -6,7 +6,7 @@ AS $$ DECLARE hash bigint; BEGIN - select @left('x' || md5(_stream_name), 16)::bit(64)::bigint into hash; + select left('x' || md5(_stream_name), 16)::bit(64)::bigint into hash; return hash; END; $$ LANGUAGE plpgsql From 73ea42e5442a9f1095c21e6c9a3c6986eb60222b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 18:33:52 -0500 Subject: [PATCH 360/591] Benchmarks for advisory locks --- ...t Benchmark - 2018-06-04T23:30:23.531Z.txt | 21 +++++++++++++++++++ ...t Benchmark - 2018-06-04T23:22:20.699Z.txt | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt create mode 100644 test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt new file mode 100644 index 0000000..11610e4 --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt @@ -0,0 +1,21 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 74936.710000ms +Mean Cycle Time: 0.749367ms (± 0.818795ms) +Cycles Per Second: 1334.459439 +GC: off + +Note: Post advisory lock implementation + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt new file mode 100644 index 0000000..6a92162 --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt @@ -0,0 +1,21 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 133606.236000ms +Mean Cycle Time: 1.336062ms (± 0.960286ms) +Cycles Per Second: 748.468058 +GC: off + +Note: Post advisory lock implementation + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 From 59b08b318a77d81e61a5fe4ecbddfc6d848daa0b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 20:54:32 -0500 Subject: [PATCH 361/591] Today's benchmarks are removed. Will recreate them in their entirety, including the benchmarks prior to the advisory locks. --- ...t Benchmark - 2018-06-04T23:30:23.531Z.txt | 21 ------------------- ...t Benchmark - 2018-06-04T23:22:20.699Z.txt | 21 ------------------- 2 files changed, 42 deletions(-) delete mode 100644 test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt delete mode 100644 test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt deleted file mode 100644 index 11610e4..0000000 --- a/test/benchmark/results/Get Benchmark - 2018-06-04T23:30:23.531Z.txt +++ /dev/null @@ -1,21 +0,0 @@ -Get Benchmark -- - - -Cycles: 100000 -Time: 74936.710000ms -Mean Cycle Time: 0.749367ms (± 0.818795ms) -Cycles Per Second: 1334.459439 -GC: off - -Note: Post advisory lock implementation - -Model Name: MacBook -Model Identifier: MacBook10,1 -Processor Name: Intel Core i7 -Processor Speed: 1.4 GHz -Number of Processors: 1 -Total Number of Cores: 2 -L2 Cache (per Core): 256 KB -L3 Cache: 4 MB -Memory: 16 GB -Boot ROM Version: MB101.0156.B00 -SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt deleted file mode 100644 index 6a92162..0000000 --- a/test/benchmark/results/Put Benchmark - 2018-06-04T23:22:20.699Z.txt +++ /dev/null @@ -1,21 +0,0 @@ -Put Benchmark -- - - -Cycles: 100000 -Time: 133606.236000ms -Mean Cycle Time: 1.336062ms (± 0.960286ms) -Cycles Per Second: 748.468058 -GC: off - -Note: Post advisory lock implementation - -Model Name: MacBook -Model Identifier: MacBook10,1 -Processor Name: Intel Core i7 -Processor Speed: 1.4 GHz -Number of Processors: 1 -Total Number of Cores: 2 -L2 Cache (per Core): 256 KB -L3 Cache: 4 MB -Memory: 16 GB -Boot ROM Version: MB101.0156.B00 -SMC Version (system): 2.42f10 From d05c9bddbceb6a521268bbacb7d73d9f4400d1c6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 21:29:26 -0500 Subject: [PATCH 362/591] Benchmarks are updated --- ... Benchmark - 2018-06-05T02:03:42.080Z.txt} | 8 ++++--- ...t Benchmark - 2018-06-05T02:27:02.253Z.txt | 21 ++++++++++++++++++ ... Benchmark - 2018-06-05T01:59:19.666Z.txt} | 8 ++++--- ...t Benchmark - 2018-06-05T02:14:58.644Z.txt | 22 +++++++++++++++++++ 4 files changed, 53 insertions(+), 6 deletions(-) rename test/benchmark/results/{Get Benchmark - 2018-06-01T23:25:15.505Z.txt => Get Benchmark - 2018-06-05T02:03:42.080Z.txt} (70%) create mode 100644 test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt rename test/benchmark/results/{Put Benchmark - 2018-06-01T23:37:23.568Z.txt => Put Benchmark - 2018-06-05T01:59:19.666Z.txt} (70%) create mode 100644 test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02:03:42.080Z.txt similarity index 70% rename from test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt rename to test/benchmark/results/Get Benchmark - 2018-06-05T02:03:42.080Z.txt index c200a52..5d8edf9 100644 --- a/test/benchmark/results/Get Benchmark - 2018-06-01T23:25:15.505Z.txt +++ b/test/benchmark/results/Get Benchmark - 2018-06-05T02:03:42.080Z.txt @@ -1,11 +1,13 @@ Get Benchmark - - - Cycles: 100000 -Time: 155466.029000ms -Mean Cycle Time: 1.554660ms (± 6.764445ms) -Cycles Per Second: 643.227338 +Time: 77864.587000ms +Mean Cycle Time: 0.778646ms (± 1.020442ms) +Cycles Per Second: 1284.280876 GC: off +Note: Pre-advisory lock implementation + Model Name: MacBook Model Identifier: MacBook10,1 Processor Name: Intel Core i7 diff --git a/test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt new file mode 100644 index 0000000..70d5376 --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt @@ -0,0 +1,21 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 89734.621000ms +Mean Cycle Time: 0.897346ms (± 2.752222ms) +Cycles Per Second: 1114.397084 +GC: off + +Note: Post-advisory lock implementation + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-05T01:59:19.666Z.txt similarity index 70% rename from test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt rename to test/benchmark/results/Put Benchmark - 2018-06-05T01:59:19.666Z.txt index c1e4476..4f89f45 100644 --- a/test/benchmark/results/Put Benchmark - 2018-06-01T23:37:23.568Z.txt +++ b/test/benchmark/results/Put Benchmark - 2018-06-05T01:59:19.666Z.txt @@ -1,11 +1,13 @@ Put Benchmark - - - Cycles: 100000 -Time: 278854.650000ms -Mean Cycle Time: 2.788546ms (± 11.991595ms) -Cycles Per Second: 358.609763 +Time: 137845.491000ms +Mean Cycle Time: 1.378455ms (± 1.311459ms) +Cycles Per Second: 725.449917 GC: off +Note: Pre-advisory lock implementation + Model Name: MacBook Model Identifier: MacBook10,1 Processor Name: Intel Core i7 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt new file mode 100644 index 0000000..d8297eb --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt @@ -0,0 +1,22 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 146689.646000ms +Mean Cycle Time: 1.466896ms (± 1.990456ms) +Cycles Per Second: 681.711373 +GC: off + +Note: Post-advisory lock implementation + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 + From 79c3a2908f0cb03b1ad5f8aea85585840fa9c5b5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 21:42:18 -0500 Subject: [PATCH 363/591] Benchmark result file timestamps are punctuated with dashes Use of colons conflicts with OS filename rules --- test/benchmark/record_result.rb | 3 ++- ...2.080Z.txt => Get Benchmark - 2018-06-05T02-03-42.080Z.txt} | 0 ...2.253Z.txt => Get Benchmark - 2018-06-05T02-27-02.253Z.txt} | 0 ...9.666Z.txt => Put Benchmark - 2018-06-05T01-59-19.666Z.txt} | 0 ...8.644Z.txt => Put Benchmark - 2018-06-05T02-14-58.644Z.txt} | 0 5 files changed, 2 insertions(+), 1 deletion(-) rename test/benchmark/results/{Get Benchmark - 2018-06-05T02:03:42.080Z.txt => Get Benchmark - 2018-06-05T02-03-42.080Z.txt} (100%) rename test/benchmark/results/{Get Benchmark - 2018-06-05T02:27:02.253Z.txt => Get Benchmark - 2018-06-05T02-27-02.253Z.txt} (100%) rename test/benchmark/results/{Put Benchmark - 2018-06-05T01:59:19.666Z.txt => Put Benchmark - 2018-06-05T01-59-19.666Z.txt} (100%) rename test/benchmark/results/{Put Benchmark - 2018-06-05T02:14:58.644Z.txt => Put Benchmark - 2018-06-05T02-14-58.644Z.txt} (100%) diff --git a/test/benchmark/record_result.rb b/test/benchmark/record_result.rb index 729fc24..c51e817 100644 --- a/test/benchmark/record_result.rb +++ b/test/benchmark/record_result.rb @@ -35,8 +35,9 @@ def write_result_file end def filename + time = clock.iso8601.gsub(':','-') directory = Pathname.new('test/benchmark/tmp') - filename = directory.join("#{name} - #{clock.iso8601}.txt") + filename = directory.join("#{name} - #{time}.txt") filename end diff --git a/test/benchmark/results/Get Benchmark - 2018-06-05T02:03:42.080Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z.txt similarity index 100% rename from test/benchmark/results/Get Benchmark - 2018-06-05T02:03:42.080Z.txt rename to test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z.txt similarity index 100% rename from test/benchmark/results/Get Benchmark - 2018-06-05T02:27:02.253Z.txt rename to test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z.txt diff --git a/test/benchmark/results/Put Benchmark - 2018-06-05T01:59:19.666Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z.txt similarity index 100% rename from test/benchmark/results/Put Benchmark - 2018-06-05T01:59:19.666Z.txt rename to test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z.txt diff --git a/test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z.txt similarity index 100% rename from test/benchmark/results/Put Benchmark - 2018-06-05T02:14:58.644Z.txt rename to test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z.txt From 6d91dccf501e6cbbafb9aa66f42a555b2e49c49c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 4 Jun 2018 22:15:37 -0500 Subject: [PATCH 364/591] Package version is increased from 0.7.0.2 to 0.8.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 942a670..3a5051c 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.7.0.2' + s.version = '0.8.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From e9122588955c50a2d3774fdcd3f8b0d6b6032d5c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 09:26:06 -0500 Subject: [PATCH 365/591] Actor logs its object ID --- test/interactive/concurrent_threads_write_to_single_stream.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/interactive/concurrent_threads_write_to_single_stream.rb b/test/interactive/concurrent_threads_write_to_single_stream.rb index 237b05f..f8b6709 100644 --- a/test/interactive/concurrent_threads_write_to_single_stream.rb +++ b/test/interactive/concurrent_threads_write_to_single_stream.rb @@ -21,7 +21,7 @@ def initialize(stream_name) MessageStore::Postgres::Write.(message_data, stream_name) - logger.info { "Wrote message (Type: #{message_data.type.inspect}, Stream Name: #{stream_name.inspect})" } + logger.info { "Wrote message (Object ID: #{object_id}, Type: #{message_data.type.inspect}, Stream Name: #{stream_name.inspect})" } :write_message end From 38584ac86cb7b94f721a1dff4f08c32f8403dd63 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 11:57:38 -0500 Subject: [PATCH 366/591] Concurrency test writes a batch, supporting verification of transactional isolation --- .../concurrent_threads_write_to_single_stream.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/interactive/concurrent_threads_write_to_single_stream.rb b/test/interactive/concurrent_threads_write_to_single_stream.rb index f8b6709..446dd49 100644 --- a/test/interactive/concurrent_threads_write_to_single_stream.rb +++ b/test/interactive/concurrent_threads_write_to_single_stream.rb @@ -17,17 +17,20 @@ def initialize(stream_name) end handle :write_message do - message_data = ::Controls::MessageData::Write.example + message_data_1 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { attribute: 1 }) + message_data_2 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { attribute: 2 }) + batch = [message_data_1, message_data_2] - MessageStore::Postgres::Write.(message_data, stream_name) + position = MessageStore::Postgres::Write.(batch, stream_name) - logger.info { "Wrote message (Object ID: #{object_id}, Type: #{message_data.type.inspect}, Stream Name: #{stream_name.inspect})" } + logger.info { "Wrote message data (Object ID: #{object_id}, Position: #{position}, Message Type: #{message_data_1.type.inspect}, Stream Name: #{stream_name.inspect})" } :write_message end end -stream_name = Controls::StreamName.example +random = SecureRandom.hex[0..5] +stream_name = Controls::StreamName.example(category: "testConcurrentWrite-#{random}", randomize_category: false) Actor::Supervisor.start do 2.times do From 94dadf01616112c99153c6a207f225f72a00c958 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 12:12:35 -0500 Subject: [PATCH 367/591] Concurrency tests are in their own file system namespace --- .../write_to_single_stream.rb} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename test/interactive/{concurrent_threads_write_to_single_stream.rb => concurrency/write_to_single_stream.rb} (70%) diff --git a/test/interactive/concurrent_threads_write_to_single_stream.rb b/test/interactive/concurrency/write_to_single_stream.rb similarity index 70% rename from test/interactive/concurrent_threads_write_to_single_stream.rb rename to test/interactive/concurrency/write_to_single_stream.rb index 446dd49..3dc7985 100644 --- a/test/interactive/concurrent_threads_write_to_single_stream.rb +++ b/test/interactive/concurrency/write_to_single_stream.rb @@ -1,4 +1,4 @@ -require_relative './interactive_init' +require_relative '../interactive_init' require 'actor' @@ -17,13 +17,13 @@ def initialize(stream_name) end handle :write_message do - message_data_1 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { attribute: 1 }) - message_data_2 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { attribute: 2 }) + message_data_1 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { actor: object_id }) + message_data_2 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { actor: object_id }) batch = [message_data_1, message_data_2] position = MessageStore::Postgres::Write.(batch, stream_name) - logger.info { "Wrote message data (Object ID: #{object_id}, Position: #{position}, Message Type: #{message_data_1.type.inspect}, Stream Name: #{stream_name.inspect})" } + logger.info(tag: :actor) { "Wrote message data (Object ID: #{object_id}, Position: #{position}, Message Type: #{message_data_1.type.inspect}, Stream Name: #{stream_name.inspect})" } :write_message end From 9fb65cd9fea7af588bdfa13b977094104b3db37c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 12:20:31 -0500 Subject: [PATCH 368/591] Number of actors can be specified as an environment variable --- test/interactive/concurrency/concurrency_init.rb | 5 +++++ test/interactive/concurrency/defaults.rb | 11 +++++++++++ .../interactive/concurrency/write_to_single_stream.rb | 11 +++++++++-- 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 test/interactive/concurrency/concurrency_init.rb create mode 100644 test/interactive/concurrency/defaults.rb diff --git a/test/interactive/concurrency/concurrency_init.rb b/test/interactive/concurrency/concurrency_init.rb new file mode 100644 index 0000000..89303b6 --- /dev/null +++ b/test/interactive/concurrency/concurrency_init.rb @@ -0,0 +1,5 @@ +ENV['LOG_TAGS'] ||= 'actor' + +require_relative '../interactive_init' + +require_relative 'defaults' diff --git a/test/interactive/concurrency/defaults.rb b/test/interactive/concurrency/defaults.rb new file mode 100644 index 0000000..6e6e9dd --- /dev/null +++ b/test/interactive/concurrency/defaults.rb @@ -0,0 +1,11 @@ +module Test + module Interactive + module Concurrency + class Defaults + def self.actors + Integer(ENV['ACTORS'] || 2) + end + end + end + end +end diff --git a/test/interactive/concurrency/write_to_single_stream.rb b/test/interactive/concurrency/write_to_single_stream.rb index 3dc7985..c0d713b 100644 --- a/test/interactive/concurrency/write_to_single_stream.rb +++ b/test/interactive/concurrency/write_to_single_stream.rb @@ -1,4 +1,4 @@ -require_relative '../interactive_init' +require_relative 'concurrency_init' require 'actor' @@ -32,8 +32,15 @@ def initialize(stream_name) random = SecureRandom.hex[0..5] stream_name = Controls::StreamName.example(category: "testConcurrentWrite-#{random}", randomize_category: false) +number_of_actors = Test::Interactive::Concurrency::Defaults.actors + +puts +puts "Concurrent Write (#{number_of_actors} actors)" +puts "- - -" +puts + Actor::Supervisor.start do - 2.times do + number_of_actors.times do ExampleProcess.start(stream_name) end end From 45be726f1871ca6ae68d15270a334d43519539c4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 12:27:38 -0500 Subject: [PATCH 369/591] Obsolete concurrent write script is removed --- test/interactive/write.rb | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 test/interactive/write.rb diff --git a/test/interactive/write.rb b/test/interactive/write.rb deleted file mode 100644 index 57c9409..0000000 --- a/test/interactive/write.rb +++ /dev/null @@ -1,17 +0,0 @@ -require_relative 'interactive_init' - -stream_name = 'testWriteIsolation' - -message_data_1 = Controls::MessageData::Write.example(data: { attribute: 1 }) -message_data_2 = Controls::MessageData::Write.example(data: { attribute: 2 }) -batch = [message_data_1, message_data_2] - -write = Write.build - -i = 0 -loop do - position = write.(batch, stream_name) - puts "i: #{i}, p: #{position}" - i += 1 -end - From 228cde47d0213ed202d8d29f5f1156c5f0ce482d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 12:36:52 -0500 Subject: [PATCH 370/591] Concurrent write actor is namespaced --- .../concurrency/write_to_single_stream.rb | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/test/interactive/concurrency/write_to_single_stream.rb b/test/interactive/concurrency/write_to_single_stream.rb index c0d713b..5f91f08 100644 --- a/test/interactive/concurrency/write_to_single_stream.rb +++ b/test/interactive/concurrency/write_to_single_stream.rb @@ -2,35 +2,41 @@ require 'actor' -class ExampleProcess - include Actor - include Log::Dependency +module Test + module Interactive + module Concurrency + class Write + include Actor + include Log::Dependency - attr_reader :stream_name + attr_reader :stream_name - def initialize(stream_name) - @stream_name = stream_name - end + def initialize(stream_name) + @stream_name = stream_name + end - handle :start do - :write_message - end + handle :start do + :write_message + end - handle :write_message do - message_data_1 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { actor: object_id }) - message_data_2 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { actor: object_id }) - batch = [message_data_1, message_data_2] + handle :write_message do + message_data_1 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { actor: object_id }) + message_data_2 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { actor: object_id }) + batch = [message_data_1, message_data_2] - position = MessageStore::Postgres::Write.(batch, stream_name) + position = MessageStore::Postgres::Write.(batch, stream_name) - logger.info(tag: :actor) { "Wrote message data (Object ID: #{object_id}, Position: #{position}, Message Type: #{message_data_1.type.inspect}, Stream Name: #{stream_name.inspect})" } + logger.info(tag: :actor) { "Wrote message data (Object ID: #{object_id}, Position: #{position}, Message Type: #{message_data_1.type.inspect}, Stream Name: #{stream_name.inspect})" } - :write_message + :write_message + end + end + end end end -random = SecureRandom.hex[0..5] -stream_name = Controls::StreamName.example(category: "testConcurrentWrite-#{random}", randomize_category: false) +random = SecureRandom.hex[0..3] +stream_name = Controls::StreamName.example(category: "testConcurrentWrite_#{random}", randomize_category: false, id: :none) number_of_actors = Test::Interactive::Concurrency::Defaults.actors @@ -41,6 +47,6 @@ def initialize(stream_name) Actor::Supervisor.start do number_of_actors.times do - ExampleProcess.start(stream_name) + Test::Interactive::Concurrency::Write.start(stream_name) end end From 23c638e69b02cb8e1f3c42b652873d1c73d1e5e1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 12:38:56 -0500 Subject: [PATCH 371/591] Obsolete concurrent write script is removed --- test/interactive/write.rb | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 test/interactive/write.rb diff --git a/test/interactive/write.rb b/test/interactive/write.rb deleted file mode 100644 index 57c9409..0000000 --- a/test/interactive/write.rb +++ /dev/null @@ -1,17 +0,0 @@ -require_relative 'interactive_init' - -stream_name = 'testWriteIsolation' - -message_data_1 = Controls::MessageData::Write.example(data: { attribute: 1 }) -message_data_2 = Controls::MessageData::Write.example(data: { attribute: 2 }) -batch = [message_data_1, message_data_2] - -write = Write.build - -i = 0 -loop do - position = write.(batch, stream_name) - puts "i: #{i}, p: #{position}" - i += 1 -end - From 053080c646394ec008285e46c23463645b89af42 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 13:05:15 -0500 Subject: [PATCH 372/591] Server function volatility is explicit --- database/functions/hash-64.sql | 1 + database/functions/stream-version.sql | 3 ++- database/functions/write-message.sql | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/database/functions/hash-64.sql b/database/functions/hash-64.sql index ab6fa4d..fb35e94 100644 --- a/database/functions/hash-64.sql +++ b/database/functions/hash-64.sql @@ -10,3 +10,4 @@ BEGIN return hash; END; $$ LANGUAGE plpgsql +IMMUTABLE; diff --git a/database/functions/stream-version.sql b/database/functions/stream-version.sql index 4f9abf3..220d405 100644 --- a/database/functions/stream-version.sql +++ b/database/functions/stream-version.sql @@ -10,4 +10,5 @@ BEGIN return stream_version; END; -$$ LANGUAGE plpgsql; +$$ LANGUAGE plpgsql +VOLATILE; diff --git a/database/functions/write-message.sql b/database/functions/write-message.sql index 1f95698..38e4613 100644 --- a/database/functions/write-message.sql +++ b/database/functions/write-message.sql @@ -56,4 +56,5 @@ BEGIN return position; END; -$$ LANGUAGE plpgsql; +$$ LANGUAGE plpgsql +VOLATILE; From f935bf815303016087a63b97e6f98092dea0a1ee Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 14:11:05 -0500 Subject: [PATCH 373/591] Instance variable name matches attribute name --- lib/message_store/postgres/get/select_statement.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb index 0fb4c1e..07302be 100644 --- a/lib/message_store/postgres/get/select_statement.rb +++ b/lib/message_store/postgres/get/select_statement.rb @@ -15,7 +15,7 @@ def batch_size end def stream_type_list - @stream_type ||= StreamName.get_type_list(stream_name) + @stream_type_list ||= StreamName.get_type_list(stream_name) end def category_stream? From 1a9b04e1d60f625c5920f45f6e08ff2e81db42f0 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 14:17:00 -0500 Subject: [PATCH 374/591] Obsolete position parameter is removed --- lib/message_store/postgres/get.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 5d13bf5..64e2f74 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -13,7 +13,7 @@ def self.build(batch_size: nil, session: nil, condition: nil) end end - def self.configure(receiver, attr_name: nil, position: nil, batch_size: nil, condition: nil, session: nil) + def self.configure(receiver, attr_name: nil, batch_size: nil, condition: nil, session: nil) attr_name ||= :get instance = build(batch_size: batch_size, condition: condition, session: session) receiver.public_send "#{attr_name}=", instance From b8083b8eadafeaa89d620a5c24a5e3ab823c0c48 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 14:18:36 -0500 Subject: [PATCH 375/591] Package version is increased from 0.8.0.0 to 0.9.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 3a5051c..ac87dbb 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.8.0.0' + s.version = '0.9.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 08a6a9c4333c054d843c8a62db41d1746a8b395f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 16:33:26 -0500 Subject: [PATCH 376/591] Benchmark defaults are in the test namespace --- database/functions/get-messages.sql | 8 +++++ test/benchmark/defaults.rb | 50 +++++++++++++++-------------- test/benchmark/get.rb | 2 +- test/benchmark/put.rb | 2 +- 4 files changed, 36 insertions(+), 26 deletions(-) create mode 100644 database/functions/get-messages.sql diff --git a/database/functions/get-messages.sql b/database/functions/get-messages.sql new file mode 100644 index 0000000..2a0fc5c --- /dev/null +++ b/database/functions/get-messages.sql @@ -0,0 +1,8 @@ +CREATE OR REPLACE FUNCTION get_messages() +RETURNS messages +AS $$ +BEGIN + SELECT * FROM messages WHERE global_position > 0; +END; +$$ LANGUAGE plpgsql +VOLATILE; diff --git a/test/benchmark/defaults.rb b/test/benchmark/defaults.rb index 95eb0e5..dc178e3 100644 --- a/test/benchmark/defaults.rb +++ b/test/benchmark/defaults.rb @@ -1,33 +1,35 @@ -module Benchmark - class Defaults - initializer :cycles, :warmup_cycles, :gc +module Test + module Benchmark + class Defaults + initializer :cycles, :warmup_cycles, :gc - def total_cycles - cycles + warmup_cycles - end + def total_cycles + cycles + warmup_cycles + end - def to_s - <<~TEXT - Cycles: #{cycles} - Warmup Cycles: #{warmup_cycles} - GC: #{gc} - TEXT - end + def to_s + <<~TEXT + Cycles: #{cycles} + Warmup Cycles: #{warmup_cycles} + GC: #{gc} + TEXT + end - def self.build - new(cycles, warmup_cycles, gc) - end + def self.build + new(cycles, warmup_cycles, gc) + end - def self.cycles - Integer(ENV['CYCLES'] || 2500) - end + def self.cycles + Integer(ENV['CYCLES'] || 2500) + end - def self.warmup_cycles - Integer(ENV['WARMUP_CYCLES'] || 10) - end + def self.warmup_cycles + Integer(ENV['WARMUP_CYCLES'] || 10) + end - def self.gc - ['on', 'true'].include?(ENV['GC']) ? true : false + def self.gc + ['on', 'true'].include?(ENV['GC']) ? true : false + end end end end diff --git a/test/benchmark/get.rb b/test/benchmark/get.rb index a6854e6..b650099 100644 --- a/test/benchmark/get.rb +++ b/test/benchmark/get.rb @@ -1,6 +1,6 @@ require_relative 'benchmark_init' -defaults = Benchmark::Defaults.build +defaults = Test::Benchmark::Defaults.build list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) diff --git a/test/benchmark/put.rb b/test/benchmark/put.rb index d753f39..0332464 100644 --- a/test/benchmark/put.rb +++ b/test/benchmark/put.rb @@ -1,6 +1,6 @@ require_relative 'benchmark_init' -defaults = Benchmark::Defaults.build +defaults = Test::Benchmark::Defaults.build list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) From f9e6de4d38bccd20acd507bc8a37764b260e6f34 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Jun 2018 16:30:59 -0500 Subject: [PATCH 377/591] Retrieval function --- database/install.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/database/install.sh b/database/install.sh index 6741458..9fd2c65 100755 --- a/database/install.sh +++ b/database/install.sh @@ -80,9 +80,12 @@ function create-functions { echo "stream_version function" psql $database -f $base/functions/stream-version.sql - echo "write_sql function" + echo "write_message function" psql $database -f $base/functions/write-message.sql + echo "get_messages function" + psql $database -f $base/functions/get-messages.sql + echo } From 84fb9162dd15ce43a4cc1e158b24ba726162894f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 11:22:22 -0500 Subject: [PATCH 378/591] Basic proof implementation of the get_messages Postgres server function --- database/functions/get-messages.sql | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/database/functions/get-messages.sql b/database/functions/get-messages.sql index 2a0fc5c..8208640 100644 --- a/database/functions/get-messages.sql +++ b/database/functions/get-messages.sql @@ -1,8 +1,6 @@ CREATE OR REPLACE FUNCTION get_messages() -RETURNS messages +RETURNS SETOF messages AS $$ -BEGIN - SELECT * FROM messages WHERE global_position > 0; -END; -$$ LANGUAGE plpgsql + SELECT * FROM messages; +$$ LANGUAGE SQL VOLATILE; From 0b7acf436b59a1a571d6c5374b626b32af74cc85 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 13:31:17 -0500 Subject: [PATCH 379/591] Select statement is dynamic --- database/functions/get-messages.sql | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/database/functions/get-messages.sql b/database/functions/get-messages.sql index 8208640..b09eb71 100644 --- a/database/functions/get-messages.sql +++ b/database/functions/get-messages.sql @@ -1,6 +1,11 @@ CREATE OR REPLACE FUNCTION get_messages() RETURNS SETOF messages AS $$ - SELECT * FROM messages; -$$ LANGUAGE SQL +DECLARE + command text; +BEGIN + command := 'SELECT * FROM messages'; + RETURN QUERY EXECUTE command; +END; +$$ LANGUAGE plpgsql VOLATILE; From 2b65234269cfb8f3b28c47c2023aee09762bdf64 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 14:14:44 -0500 Subject: [PATCH 380/591] Where clause is dynamic and optional --- database/functions/get-messages.sql | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/database/functions/get-messages.sql b/database/functions/get-messages.sql index b09eb71..5bc48bc 100644 --- a/database/functions/get-messages.sql +++ b/database/functions/get-messages.sql @@ -1,10 +1,18 @@ -CREATE OR REPLACE FUNCTION get_messages() +CREATE OR REPLACE FUNCTION get_messages( + condition varchar DEFAULT NULL +) RETURNS SETOF messages AS $$ DECLARE command text; BEGIN command := 'SELECT * FROM messages'; + + if condition is not null then + command := command || ' WHERE %s'; + command := format(command, condition); + end if; + RETURN QUERY EXECUTE command; END; $$ LANGUAGE plpgsql From f18536ffdda90733b0ce801f9b70d94a75824ae2 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 16:21:42 -0500 Subject: [PATCH 381/591] Message type --- database/functions/get-messages.sql | 27 +++++++++++++++++++++++++++ database/install.sh | 20 +++++++++++++++++--- database/types/message.sql | 10 ++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 database/types/message.sql diff --git a/database/functions/get-messages.sql b/database/functions/get-messages.sql index 5bc48bc..253d42d 100644 --- a/database/functions/get-messages.sql +++ b/database/functions/get-messages.sql @@ -1,5 +1,9 @@ + CREATE OR REPLACE FUNCTION get_messages( condition varchar DEFAULT NULL + -- stream_name varchar, + -- position bigint DEFAULT 0, + -- batch_size bigint DEFAULT 1000, ) RETURNS SETOF messages AS $$ @@ -8,11 +12,34 @@ DECLARE BEGIN command := 'SELECT * FROM messages'; + -- command := ' + -- SELECT + -- id, + -- stream_name, + -- position, + -- type, + -- global_position, + -- data, + -- metadata, + -- time + -- FROM + -- messages'; + + -- WHERE + -- #{formatted_where_clause} + -- ORDER BY + -- #{position_field} ASC + -- LIMIT + -- #{batch_size} + -- ; + + if condition is not null then command := command || ' WHERE %s'; command := format(command, condition); end if; + RETURN QUERY EXECUTE command; END; $$ LANGUAGE plpgsql diff --git a/database/install.sh b/database/install.sh index 9fd2c65..66a2db6 100755 --- a/database/install.sh +++ b/database/install.sh @@ -62,14 +62,23 @@ function create-extensions { function create-table { echo "Creating messages table..." - base=$(script_dir) psql $database -f $base/table/messages-table.sql echo } -function create-functions { +function create-types { base=$(script_dir) + echo "Creating types..." + + echo "message type" + psql $database -f $base/types/message.sql + + echo +} + +function create-functions { echo "Creating functions..." + base=$(script_dir) echo "hash_64 function" psql $database -f $base/functions/hash-64.sql @@ -90,14 +99,18 @@ function create-functions { } function create-indexes { - base=$(script_dir) echo "Creating indexes..." + base=$(script_dir) + echo "messages_id_idx" psql $database -f $base/indexes/messages-id.sql + echo "messages_category_global_position_idx" psql $database -f $base/indexes/messages-category-global-position.sql + echo "messages_stream_name_position_uniq_idx" psql $database -f $base/indexes/messages-stream-name-position-uniq.sql + echo } @@ -105,5 +118,6 @@ create-user create-database create-extensions create-table +create-types create-functions create-indexes diff --git a/database/types/message.sql b/database/types/message.sql new file mode 100644 index 0000000..c2e7e7f --- /dev/null +++ b/database/types/message.sql @@ -0,0 +1,10 @@ +CREATE TYPE message AS ( + id varchar, + stream_name varchar, + position bigint, + type varchar, + global_position bigint, + data varchar, + metadata varchar, + time timestamp +); From da37f9990ec37cf4d0e1fd5bcf117eb706d80b86 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 17:01:58 -0500 Subject: [PATCH 382/591] Get stream messages --- database/functions/get-messages.sql | 65 ++++++++++++++--------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/database/functions/get-messages.sql b/database/functions/get-messages.sql index 253d42d..afcc27e 100644 --- a/database/functions/get-messages.sql +++ b/database/functions/get-messages.sql @@ -1,46 +1,45 @@ - -CREATE OR REPLACE FUNCTION get_messages( - condition varchar DEFAULT NULL +CREATE OR REPLACE FUNCTION get_stream_messages( -- stream_name varchar, - -- position bigint DEFAULT 0, - -- batch_size bigint DEFAULT 1000, + + _position bigint DEFAULT 0, + _batch_size bigint DEFAULT 1000, + _condition varchar DEFAULT NULL ) -RETURNS SETOF messages +RETURNS SETOF message AS $$ DECLARE command text; BEGIN - command := 'SELECT * FROM messages'; - - -- command := ' - -- SELECT - -- id, - -- stream_name, - -- position, - -- type, - -- global_position, - -- data, - -- metadata, - -- time - -- FROM - -- messages'; + command := ' +SELECT + id::varchar, + stream_name::varchar, + position::bigint, + type::varchar, + global_position::bigint, + data::varchar, + metadata::varchar, + time::timestamp +FROM + messages +WHERE + position >= $1'; - -- WHERE - -- #{formatted_where_clause} - -- ORDER BY - -- #{position_field} ASC - -- LIMIT - -- #{batch_size} - -- ; - - - if condition is not null then - command := command || ' WHERE %s'; - command := format(command, condition); + if _condition is not null then + command := command || ' AND + %s'; + command := format(command, _condition); end if; + command := command || ' +ORDER BY + position ASC +LIMIT + $2'; + + RAISE NOTICE '%', command; - RETURN QUERY EXECUTE command; + RETURN QUERY EXECUTE command USING _position, _batch_size; END; $$ LANGUAGE plpgsql VOLATILE; From 273f579655a16401ec92ab8267e00e8c322493e8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 18:02:09 -0500 Subject: [PATCH 383/591] Generated SQL statement is indented for scannability --- database/functions/get-messages.sql | 36 ++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/database/functions/get-messages.sql b/database/functions/get-messages.sql index afcc27e..b66652a 100644 --- a/database/functions/get-messages.sql +++ b/database/functions/get-messages.sql @@ -11,31 +11,31 @@ DECLARE command text; BEGIN command := ' -SELECT - id::varchar, - stream_name::varchar, - position::bigint, - type::varchar, - global_position::bigint, - data::varchar, - metadata::varchar, - time::timestamp -FROM - messages -WHERE - position >= $1'; + SELECT + id::varchar, + stream_name::varchar, + position::bigint, + type::varchar, + global_position::bigint, + data::varchar, + metadata::varchar, + time::timestamp + FROM + messages + WHERE + position >= $1'; if _condition is not null then command := command || ' AND - %s'; + %s'; command := format(command, _condition); end if; command := command || ' -ORDER BY - position ASC -LIMIT - $2'; + ORDER BY + position ASC + LIMIT + $2'; RAISE NOTICE '%', command; From 9ebeb9ac0181facb3c98a5e38b872082a30690f3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 18:36:12 -0500 Subject: [PATCH 384/591] Tool writes messages to store --- lib/message_store/postgres/controls/put.rb | 5 ++-- test/automated/get/batch_size.rb | 2 +- test/automated/get/category.rb | 2 +- test/automated/get/condition.rb | 2 +- test/automated/get/position.rb | 2 +- test/automated/get_last/get_last.rb | 2 +- test/automated/iterator/next.rb | 2 +- .../iterator/no_further_message_data.rb | 2 +- test/automated/read/condition.rb | 2 +- test/automated/read/position.rb | 2 +- test/automated/read/read.rb | 2 +- test/test_init.rb | 2 +- tools/write_message.rb | 27 +++++++++++++++++++ 13 files changed, 41 insertions(+), 13 deletions(-) create mode 100755 tools/write_message.rb diff --git a/lib/message_store/postgres/controls/put.rb b/lib/message_store/postgres/controls/put.rb index 660b20a..3822625 100644 --- a/lib/message_store/postgres/controls/put.rb +++ b/lib/message_store/postgres/controls/put.rb @@ -10,15 +10,16 @@ def self.call(instances: nil, stream_name: nil, message: nil, category: nil) message ||= MessageData::Write.example + position = nil instances.times do - MessageStore::Postgres::Put.(message, stream_name) + position = MessageStore::Postgres::Put.(message, stream_name) unless message_specified message.id = MessageData::Write.id end end - stream_name + [stream_name, position] end end end diff --git a/test/automated/get/batch_size.rb b/test/automated/get/batch_size.rb index b44a6f7..72e338f 100644 --- a/test/automated/get/batch_size.rb +++ b/test/automated/get/batch_size.rb @@ -2,7 +2,7 @@ context "Get" do context "Batch Size" do - stream_name = Controls::Put.(instances: 3) + stream_name, _ = Controls::Put.(instances: 3) messages = Get.(stream_name, batch_size: 2) diff --git a/test/automated/get/category.rb b/test/automated/get/category.rb index bb751e4..e9edc95 100644 --- a/test/automated/get/category.rb +++ b/test/automated/get/category.rb @@ -4,7 +4,7 @@ context "Category" do category = Controls::Category.example - stream_name = Controls::Put.(instances: 2, category: category) + stream_name, _ = Controls::Put.(instances: 2, category: category) messages = Get.(stream_name) diff --git a/test/automated/get/condition.rb b/test/automated/get/condition.rb index 0423189..1c616b9 100644 --- a/test/automated/get/condition.rb +++ b/test/automated/get/condition.rb @@ -2,7 +2,7 @@ context "Get" do context "Condition" do - stream_name = Controls::Put.(instances: 3) + stream_name, _ = Controls::Put.(instances: 3) condition = 'position = 0 OR position = 2' diff --git a/test/automated/get/position.rb b/test/automated/get/position.rb index fea072f..b3eed96 100644 --- a/test/automated/get/position.rb +++ b/test/automated/get/position.rb @@ -2,7 +2,7 @@ context "Get" do context "Position" do - stream_name = Controls::Put.(instances: 2) + stream_name, _ = Controls::Put.(instances: 2) batch = Get.(stream_name, position: 1, batch_size: 1) diff --git a/test/automated/get_last/get_last.rb b/test/automated/get_last/get_last.rb index 3ca7f02..86c4e8f 100644 --- a/test/automated/get_last/get_last.rb +++ b/test/automated/get_last/get_last.rb @@ -1,7 +1,7 @@ require_relative '../automated_init' context "Get Last" do - stream_name = Controls::Put.(instances: 2) + stream_name, _ = Controls::Put.(instances: 2) write_message = Controls::MessageData::Write.example diff --git a/test/automated/iterator/next.rb b/test/automated/iterator/next.rb index 3fbb0a3..b133bec 100644 --- a/test/automated/iterator/next.rb +++ b/test/automated/iterator/next.rb @@ -2,7 +2,7 @@ context "Iterator" do context "Next" do - stream_name = Controls::Put.(instances: 2) + stream_name, _ = Controls::Put.(instances: 2) iterator = Read::Iterator.build(stream_name) Get.configure(iterator, batch_size: 1) diff --git a/test/automated/iterator/no_further_message_data.rb b/test/automated/iterator/no_further_message_data.rb index c47bf07..9cdf5c4 100644 --- a/test/automated/iterator/no_further_message_data.rb +++ b/test/automated/iterator/no_further_message_data.rb @@ -2,7 +2,7 @@ context "Iterator" do context "No further message data" do - stream_name = Controls::Put.(instances: 2) + stream_name, _ = Controls::Put.(instances: 2) iterator = Read::Iterator.build(stream_name) Get.configure(iterator, batch_size: 1) diff --git a/test/automated/read/condition.rb b/test/automated/read/condition.rb index 132a022..44a8214 100644 --- a/test/automated/read/condition.rb +++ b/test/automated/read/condition.rb @@ -2,7 +2,7 @@ context "Read" do context "Condition" do - stream_name = Controls::Put.(instances: 3) + stream_name, _ = Controls::Put.(instances: 3) condition = 'position = 0' diff --git a/test/automated/read/position.rb b/test/automated/read/position.rb index c96adca..c27ad82 100644 --- a/test/automated/read/position.rb +++ b/test/automated/read/position.rb @@ -2,7 +2,7 @@ context "Read" do context "Position" do - stream_name = Controls::Put.(instances: 2) + stream_name, _ = Controls::Put.(instances: 2) batch = [] diff --git a/test/automated/read/read.rb b/test/automated/read/read.rb index ce7170c..6bf03c9 100644 --- a/test/automated/read/read.rb +++ b/test/automated/read/read.rb @@ -1,7 +1,7 @@ require_relative '../automated_init' context "Read" do - stream_name = Controls::Put.(instances: 2) + stream_name, _ = Controls::Put.(instances: 2) batch = [] diff --git a/test/test_init.rb b/test/test_init.rb index 1bbc2ae..1cefd49 100644 --- a/test/test_init.rb +++ b/test/test_init.rb @@ -3,7 +3,7 @@ puts RUBY_DESCRIPTION -require_relative '../init.rb' +require_relative '../init' require 'message_store/postgres/controls' require 'test_bench'; TestBench.activate diff --git a/tools/write_message.rb b/tools/write_message.rb new file mode 100755 index 0000000..4e3ce48 --- /dev/null +++ b/tools/write_message.rb @@ -0,0 +1,27 @@ +#!/usr/bin/env ruby + +ENV['CONSOLE_DEVICE'] ||= 'stdout' +ENV['LOG_LEVEL'] ||= '_min' + +puts RUBY_DESCRIPTION + +require_relative '../init' + +require 'message_store/postgres/controls' + +include MessageStore +include MessageStore::Postgres + +instances = Integer(ENV['INSTANCES'] || 1) +stream_name = ENV['STREAM_NAME'] + +puts +puts "Writing #{instances} messages" +puts "- - -" + +instances.times do |i| + stream_name, position = Controls::Put.(stream_name: stream_name) + puts "Instance: #{i}, Position: #{position}, Stream Name: #{stream_name}" +end + +puts From 4e8f7d47bb645468ddb12acbef94a159a8c41070 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 18:54:01 -0500 Subject: [PATCH 385/591] Sever function returns stream messages --- database/functions/get-messages.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/database/functions/get-messages.sql b/database/functions/get-messages.sql index b66652a..99a5fdf 100644 --- a/database/functions/get-messages.sql +++ b/database/functions/get-messages.sql @@ -1,6 +1,5 @@ CREATE OR REPLACE FUNCTION get_stream_messages( - -- stream_name varchar, - + _stream_name varchar, _position bigint DEFAULT 0, _batch_size bigint DEFAULT 1000, _condition varchar DEFAULT NULL @@ -23,7 +22,8 @@ BEGIN FROM messages WHERE - position >= $1'; + stream_name = $1 AND + position >= $2'; if _condition is not null then command := command || ' AND @@ -35,11 +35,11 @@ BEGIN ORDER BY position ASC LIMIT - $2'; + $3'; RAISE NOTICE '%', command; - RETURN QUERY EXECUTE command USING _position, _batch_size; + RETURN QUERY EXECUTE command USING _stream_name, _position, _batch_size; END; $$ LANGUAGE plpgsql VOLATILE; From 7b3954903032ecf077d782b3300322446694722e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 18:54:44 -0500 Subject: [PATCH 386/591] Debug message is disabled --- database/functions/get-messages.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/functions/get-messages.sql b/database/functions/get-messages.sql index 99a5fdf..6012286 100644 --- a/database/functions/get-messages.sql +++ b/database/functions/get-messages.sql @@ -37,7 +37,7 @@ BEGIN LIMIT $3'; - RAISE NOTICE '%', command; + -- RAISE NOTICE '%', command; RETURN QUERY EXECUTE command USING _stream_name, _position, _batch_size; END; From 25b3785d544b56231aa25d38a4d0907ab895d565 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 18:55:43 -0500 Subject: [PATCH 387/591] File name matches its contents --- database/functions/{get-messages.sql => get-stream-messages.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename database/functions/{get-messages.sql => get-stream-messages.sql} (100%) diff --git a/database/functions/get-messages.sql b/database/functions/get-stream-messages.sql similarity index 100% rename from database/functions/get-messages.sql rename to database/functions/get-stream-messages.sql From ca05e6f69646e7b362dfc78a0891c049167fe305 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 19:15:06 -0500 Subject: [PATCH 388/591] Server function returns category messages --- database/functions/get-category-messages.sql | 45 ++++++++++++++++++++ database/install.sh | 7 ++- 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 database/functions/get-category-messages.sql diff --git a/database/functions/get-category-messages.sql b/database/functions/get-category-messages.sql new file mode 100644 index 0000000..623a848 --- /dev/null +++ b/database/functions/get-category-messages.sql @@ -0,0 +1,45 @@ +CREATE OR REPLACE FUNCTION get_category_messages( + _stream_name varchar, + _position bigint DEFAULT 0, + _batch_size bigint DEFAULT 1000, + _condition varchar DEFAULT NULL +) +RETURNS SETOF message +AS $$ +DECLARE + command text; +BEGIN + command := ' + SELECT + id::varchar, + stream_name::varchar, + position::bigint, + type::varchar, + global_position::bigint, + data::varchar, + metadata::varchar, + time::timestamp + FROM + messages + WHERE + category(stream_name) = $1 AND + position >= $2'; + + if _condition is not null then + command := command || ' AND + %s'; + command := format(command, _condition); + end if; + + command := command || ' + ORDER BY + global_position ASC + LIMIT + $3'; + + -- RAISE NOTICE '%', command; + + RETURN QUERY EXECUTE command USING _stream_name, _position, _batch_size; +END; +$$ LANGUAGE plpgsql +VOLATILE; diff --git a/database/install.sh b/database/install.sh index 66a2db6..da24e88 100755 --- a/database/install.sh +++ b/database/install.sh @@ -92,8 +92,11 @@ function create-functions { echo "write_message function" psql $database -f $base/functions/write-message.sql - echo "get_messages function" - psql $database -f $base/functions/get-messages.sql + echo "get_stream_messages function" + psql $database -f $base/functions/get-stream-messages.sql + + echo "get_category_messages function" + psql $database -f $base/functions/get-category-messages.sql echo } From dcdcfe7ec39a77478b0843c2f149e3bd649b142e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 20:43:02 -0500 Subject: [PATCH 389/591] Server function get last message in a stream --- database/functions/get-last-message.sql | 33 +++++++++++++++++++++++++ database/install.sh | 3 +++ 2 files changed, 36 insertions(+) create mode 100644 database/functions/get-last-message.sql diff --git a/database/functions/get-last-message.sql b/database/functions/get-last-message.sql new file mode 100644 index 0000000..727f801 --- /dev/null +++ b/database/functions/get-last-message.sql @@ -0,0 +1,33 @@ +CREATE OR REPLACE FUNCTION get_last_message( + _stream_name varchar +) +RETURNS SETOF message +AS $$ +DECLARE + command text; +BEGIN + command := ' + SELECT + id::varchar, + stream_name::varchar, + position::bigint, + type::varchar, + global_position::bigint, + data::varchar, + metadata::varchar, + time::timestamp + FROM + messages + WHERE + stream_name = $1 + ORDER BY + position DESC + LIMIT + 1'; + + RAISE NOTICE '%', command; + + RETURN QUERY EXECUTE command USING _stream_name; +END; +$$ LANGUAGE plpgsql +VOLATILE; diff --git a/database/install.sh b/database/install.sh index da24e88..43b38de 100755 --- a/database/install.sh +++ b/database/install.sh @@ -98,6 +98,9 @@ function create-functions { echo "get_category_messages function" psql $database -f $base/functions/get-category-messages.sql + echo "get_last_message function" + psql $database -f $base/functions/get-last-message.sql + echo } From b4d67cd1b65dd0525e195bdd19e905a64704d1b0 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 21:19:32 -0500 Subject: [PATCH 390/591] Test scripts for retrieval server functions --- database/functions/get-category-messages.sql | 2 +- test/interactive/database/get_category_messages.sh | 5 +++++ test/interactive/database/get_last_message.sh | 5 +++++ test/interactive/database/get_stream_messages.sh | 5 +++++ test/interactive/database/recreate_database.sh | 4 ++++ test/interactive/database/setup.sh | 4 ++++ test/interactive/database/write_messages.sh | 5 +++++ 7 files changed, 29 insertions(+), 1 deletion(-) create mode 100755 test/interactive/database/get_category_messages.sh create mode 100755 test/interactive/database/get_last_message.sh create mode 100755 test/interactive/database/get_stream_messages.sh create mode 100755 test/interactive/database/recreate_database.sh create mode 100755 test/interactive/database/setup.sh create mode 100755 test/interactive/database/write_messages.sh diff --git a/database/functions/get-category-messages.sql b/database/functions/get-category-messages.sql index 623a848..e8eb2c7 100644 --- a/database/functions/get-category-messages.sql +++ b/database/functions/get-category-messages.sql @@ -23,7 +23,7 @@ BEGIN messages WHERE category(stream_name) = $1 AND - position >= $2'; + global_position >= $2'; if _condition is not null then command := command || ' AND diff --git a/test/interactive/database/get_category_messages.sh b/test/interactive/database/get_category_messages.sh new file mode 100755 index 0000000..ff662e3 --- /dev/null +++ b/test/interactive/database/get_category_messages.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +test/interactive/database/setup.sh + +psql message_store -c "select * from get_category_messages('someStream', 4, 1, _condition => 'position = 3');" diff --git a/test/interactive/database/get_last_message.sh b/test/interactive/database/get_last_message.sh new file mode 100755 index 0000000..2f016d8 --- /dev/null +++ b/test/interactive/database/get_last_message.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +test/interactive/database/setup.sh + +psql message_store -c "select * from get_last_message('someStream-123');" diff --git a/test/interactive/database/get_stream_messages.sh b/test/interactive/database/get_stream_messages.sh new file mode 100755 index 0000000..a30a5f2 --- /dev/null +++ b/test/interactive/database/get_stream_messages.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +test/interactive/database/setup.sh + +psql message_store -c "select * from get_stream_messages('someStream-123', 2, 1, _condition => 'global_position = 3');" diff --git a/test/interactive/database/recreate_database.sh b/test/interactive/database/recreate_database.sh new file mode 100755 index 0000000..1b7d981 --- /dev/null +++ b/test/interactive/database/recreate_database.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +database/uninstall.sh +database/install.sh diff --git a/test/interactive/database/setup.sh b/test/interactive/database/setup.sh new file mode 100755 index 0000000..559a955 --- /dev/null +++ b/test/interactive/database/setup.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +test/interactive/database/recreate_database.sh +test/interactive/database/write_messages.sh diff --git a/test/interactive/database/write_messages.sh b/test/interactive/database/write_messages.sh new file mode 100755 index 0000000..4410c48 --- /dev/null +++ b/test/interactive/database/write_messages.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +STREAM_NAME=someStream-123 INSTANCES=4 tools/write_message.rb +STREAM_NAME=someStream-456 INSTANCES=4 tools/write_message.rb +STREAM_NAME=someOtherStream-456 INSTANCES=4 tools/write_message.rb From cdc93edd5aebae80c96536df072e14406957bb46 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 21:35:51 -0500 Subject: [PATCH 391/591] Test scripts for writing message --- database/test/hash-64.sql | 1 - database/test/write-message-expected-version.sql | 1 - database/test/write-message.sql | 1 - test/interactive/database/hash_64.sh | 3 +++ test/interactive/database/write_message.sh | 3 +++ .../interactive/database/write_message_expected_version.sh | 7 +++++++ .../database/write_message_expected_version_error.sh | 7 +++++++ 7 files changed, 20 insertions(+), 3 deletions(-) delete mode 100644 database/test/hash-64.sql delete mode 100644 database/test/write-message-expected-version.sql delete mode 100644 database/test/write-message.sql create mode 100755 test/interactive/database/hash_64.sh create mode 100755 test/interactive/database/write_message.sh create mode 100755 test/interactive/database/write_message_expected_version.sh create mode 100755 test/interactive/database/write_message_expected_version_error.sh diff --git a/database/test/hash-64.sql b/database/test/hash-64.sql deleted file mode 100644 index 3d958c7..0000000 --- a/database/test/hash-64.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT hash_64('someStream-123'); diff --git a/database/test/write-message-expected-version.sql b/database/test/write-message-expected-version.sql deleted file mode 100644 index be4ab20..0000000 --- a/database/test/write-message-expected-version.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT write_message(gen_random_uuid()::varchar, 'testWriteIsolation'::varchar, 'SomeType'::varchar, '{"attribute": "79a59e513f9182d5754abdf9b90e2179"}'::jsonb, '{"metaAttribute": "6b3ae03d713a51a445290f3fc4b5223a"}'::jsonb, 1::bigint); diff --git a/database/test/write-message.sql b/database/test/write-message.sql deleted file mode 100644 index 9ae0c5b..0000000 --- a/database/test/write-message.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT write_message(gen_random_uuid()::varchar, 'testWriteIsolation'::varchar, 'SomeType'::varchar, '{"attribute": "79a59e513f9182d5754abdf9b90e2179"}'::jsonb, '{"metaAttribute": "6b3ae03d713a51a445290f3fc4b5223a"}'::jsonb); diff --git a/test/interactive/database/hash_64.sh b/test/interactive/database/hash_64.sh new file mode 100755 index 0000000..5183683 --- /dev/null +++ b/test/interactive/database/hash_64.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +psql message_store -c "SELECT hash_64('someStream-123');" diff --git a/test/interactive/database/write_message.sh b/test/interactive/database/write_message.sh new file mode 100755 index 0000000..72cc02c --- /dev/null +++ b/test/interactive/database/write_message.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +psql message_store -c "SELECT write_message(gen_random_uuid()::varchar, 'testWriteIsolation'::varchar, 'SomeType'::varchar, '{\"attribute\": \"some value\"}'::jsonb, '{\"metaAttribute\": \"some meta value\"}'::jsonb);" diff --git a/test/interactive/database/write_message_expected_version.sh b/test/interactive/database/write_message_expected_version.sh new file mode 100755 index 0000000..d27afac --- /dev/null +++ b/test/interactive/database/write_message_expected_version.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +test/interactive/database/recreate_database.sh + +STREAM_NAME=someStream-123 INSTANCES=4 tools/write_message.rb + +psql message_store -c "SELECT write_message(gen_random_uuid()::varchar, 'someStream-123'::varchar, 'SomeType'::varchar, '{\"attribute\": \"some value\"}'::jsonb, '{\"metaAttribute\": \"some meta value\"}'::jsonb, 3::bigint);" diff --git a/test/interactive/database/write_message_expected_version_error.sh b/test/interactive/database/write_message_expected_version_error.sh new file mode 100755 index 0000000..4d975a6 --- /dev/null +++ b/test/interactive/database/write_message_expected_version_error.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +test/interactive/database/recreate_database.sh + +STREAM_NAME=someStream-123 INSTANCES=4 tools/write_message.rb + +psql message_store -c "SELECT write_message(gen_random_uuid()::varchar, 'someStream-123'::varchar, 'SomeType'::varchar, '{\"attribute\": \"some value\"}'::jsonb, '{\"metaAttribute\": \"some meta value\"}'::jsonb, 4::bigint);" From 82cdacd1279c3d43b1ae4e97a3d4bfaf581fa5c8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 21:44:58 -0500 Subject: [PATCH 392/591] Database test suite script --- test/interactive/database.sh | 11 +++++++++++ test/interactive/database/setup.sh | 2 +- .../database/{write_messages.sh => setup_messages.sh} | 0 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100755 test/interactive/database.sh rename test/interactive/database/{write_messages.sh => setup_messages.sh} (100%) diff --git a/test/interactive/database.sh b/test/interactive/database.sh new file mode 100755 index 0000000..6fac823 --- /dev/null +++ b/test/interactive/database.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +test/interactive/database/hash_64.sh + +test/interactive/database/write_message.sh +test/interactive/database/write_message_expected_version.sh +test/interactive/database/write_message_expected_version_error.sh + +test/interactive/database/get_stream_messages.sh +test/interactive/database/get_category_messages.sh +test/interactive/database/get_last_message.sh diff --git a/test/interactive/database/setup.sh b/test/interactive/database/setup.sh index 559a955..6442719 100755 --- a/test/interactive/database/setup.sh +++ b/test/interactive/database/setup.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash test/interactive/database/recreate_database.sh -test/interactive/database/write_messages.sh +test/interactive/database/setup_messages.sh diff --git a/test/interactive/database/write_messages.sh b/test/interactive/database/setup_messages.sh similarity index 100% rename from test/interactive/database/write_messages.sh rename to test/interactive/database/setup_messages.sh From f2768b4213674f96cc153fafdcf6f63f8c7d1c29 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Jun 2018 21:53:25 -0500 Subject: [PATCH 393/591] Output is not pages in database test scripts --- database/functions/get-last-message.sql | 2 +- test/interactive/database/get_category_messages.sh | 2 +- test/interactive/database/get_last_message.sh | 2 +- test/interactive/database/get_stream_messages.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/database/functions/get-last-message.sql b/database/functions/get-last-message.sql index 727f801..ba372b0 100644 --- a/database/functions/get-last-message.sql +++ b/database/functions/get-last-message.sql @@ -25,7 +25,7 @@ BEGIN LIMIT 1'; - RAISE NOTICE '%', command; + -- RAISE NOTICE '%', command; RETURN QUERY EXECUTE command USING _stream_name; END; diff --git a/test/interactive/database/get_category_messages.sh b/test/interactive/database/get_category_messages.sh index ff662e3..f043dd3 100755 --- a/test/interactive/database/get_category_messages.sh +++ b/test/interactive/database/get_category_messages.sh @@ -2,4 +2,4 @@ test/interactive/database/setup.sh -psql message_store -c "select * from get_category_messages('someStream', 4, 1, _condition => 'position = 3');" +psql message_store -P pager=off -c "select * from get_category_messages('someStream', 4, 1, _condition => 'position = 3');" diff --git a/test/interactive/database/get_last_message.sh b/test/interactive/database/get_last_message.sh index 2f016d8..f237d4e 100755 --- a/test/interactive/database/get_last_message.sh +++ b/test/interactive/database/get_last_message.sh @@ -2,4 +2,4 @@ test/interactive/database/setup.sh -psql message_store -c "select * from get_last_message('someStream-123');" +psql message_store -P pager=off -c "select * from get_last_message('someStream-123');" diff --git a/test/interactive/database/get_stream_messages.sh b/test/interactive/database/get_stream_messages.sh index a30a5f2..43e104d 100755 --- a/test/interactive/database/get_stream_messages.sh +++ b/test/interactive/database/get_stream_messages.sh @@ -2,4 +2,4 @@ test/interactive/database/setup.sh -psql message_store -c "select * from get_stream_messages('someStream-123', 2, 1, _condition => 'global_position = 3');" +psql message_store -c -P pager=off "select * from get_stream_messages('someStream-123', 2, 1, _condition => 'global_position = 3');" From e60f3a508be392ea1c4ce3a6cbc615fb6c4273d3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 7 Jun 2018 19:29:11 -0500 Subject: [PATCH 394/591] Data is retrieved using the Postgres server functinos --- database/functions/get-category-messages.sql | 2 +- database/functions/get-last-message.sql | 2 +- database/functions/get-stream-messages.sql | 2 +- database/types/message.sql | 2 +- lib/message_store/postgres.rb | 2 - lib/message_store/postgres/get.rb | 81 ++++++++++---- lib/message_store/postgres/get/last.rb | 28 +++-- .../postgres/get/last/select_statement.rb | 47 -------- .../postgres/get/select_statement.rb | 102 ------------------ lib/message_store/postgres/put.rb | 1 + lib/message_store/postgres/session.rb | 18 ++-- materials/raw_response.txt | 22 ++++ .../select_statement/category_query.rb | 28 ----- test/automated/select_statement/condition.rb | 28 ----- test/automated/select_statement/defaults.rb | 23 ---- .../select_statement/stream_name_query.rb | 28 ----- .../database/get_stream_messages.sh | 2 +- test/sketch.rb | 10 ++ 18 files changed, 129 insertions(+), 299 deletions(-) delete mode 100644 lib/message_store/postgres/get/last/select_statement.rb delete mode 100644 lib/message_store/postgres/get/select_statement.rb create mode 100644 materials/raw_response.txt delete mode 100644 test/automated/select_statement/category_query.rb delete mode 100644 test/automated/select_statement/condition.rb delete mode 100644 test/automated/select_statement/defaults.rb delete mode 100644 test/automated/select_statement/stream_name_query.rb create mode 100644 test/sketch.rb diff --git a/database/functions/get-category-messages.sql b/database/functions/get-category-messages.sql index e8eb2c7..0290ec3 100644 --- a/database/functions/get-category-messages.sql +++ b/database/functions/get-category-messages.sql @@ -13,8 +13,8 @@ BEGIN SELECT id::varchar, stream_name::varchar, - position::bigint, type::varchar, + position::bigint, global_position::bigint, data::varchar, metadata::varchar, diff --git a/database/functions/get-last-message.sql b/database/functions/get-last-message.sql index ba372b0..7125820 100644 --- a/database/functions/get-last-message.sql +++ b/database/functions/get-last-message.sql @@ -10,8 +10,8 @@ BEGIN SELECT id::varchar, stream_name::varchar, - position::bigint, type::varchar, + position::bigint, global_position::bigint, data::varchar, metadata::varchar, diff --git a/database/functions/get-stream-messages.sql b/database/functions/get-stream-messages.sql index 6012286..89bd3cf 100644 --- a/database/functions/get-stream-messages.sql +++ b/database/functions/get-stream-messages.sql @@ -13,8 +13,8 @@ BEGIN SELECT id::varchar, stream_name::varchar, - position::bigint, type::varchar, + position::bigint, global_position::bigint, data::varchar, metadata::varchar, diff --git a/database/types/message.sql b/database/types/message.sql index c2e7e7f..28b3f06 100644 --- a/database/types/message.sql +++ b/database/types/message.sql @@ -1,8 +1,8 @@ CREATE TYPE message AS ( id varchar, stream_name varchar, - position bigint, type varchar, + position bigint, global_position bigint, data varchar, metadata varchar, diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index 63c15ca..0ebdf2f 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -16,9 +16,7 @@ require 'message_store/postgres/put' require 'message_store/postgres/write' -require 'message_store/postgres/get/select_statement' require 'message_store/postgres/get' -require 'message_store/postgres/get/last/select_statement' require 'message_store/postgres/get/last' require 'message_store/postgres/read/iterator' require 'message_store/postgres/read' diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 64e2f74..77fee19 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -3,10 +3,14 @@ module Postgres class Get include MessageStore::Get - initializer :batch_size, :condition - dependency :session, Session + initializer na(:batch_size), :condition + + def batch_size + @batch_size ||= Defaults.batch_size + end + def self.build(batch_size: nil, session: nil, condition: nil) new(batch_size, condition).tap do |instance| instance.configure(session: session) @@ -31,32 +35,63 @@ def self.call(stream_name, position: nil, batch_size: nil, condition: nil, sess def call(stream_name, position: nil) logger.trace { "Getting message data (Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } - records = get_records(stream_name, position) + position ||= Defaults.position + + result = get_result(stream_name, position) - messages = convert(records) + message_data = convert(result) - logger.info { "Finished getting message data (Count: #{messages.length}, Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } - logger.info(tags: [:data, :message_data]) { messages.pretty_inspect } + logger.info { "Finished getting message data (Count: #{message_data.length}, Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } + logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } - messages + message_data end - def get_records(stream_name, position) - logger.trace { "Getting records (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } + def get_result(stream_name, position) + logger.trace { "Getting result (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } + + sql_command = self.class.sql_command(stream_name, position, batch_size, condition) - select_statement = SelectStatement.build(stream_name, position: position, batch_size: batch_size, condition: condition) + cond = self.class.constrain_condition(condition) - records = session.execute(select_statement.sql) + params = [ + stream_name, + position, + batch_size, + cond + ] - logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } + result = session.execute(sql_command, params) - records + logger.debug { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } + + result end - def convert(records) - logger.trace { "Converting records to message data (Records Count: #{records.ntuples})" } + def self.constrain_condition(condition) + return nil if condition.nil? + + "(#{condition})" + end - messages = records.map do |record| + def self.sql_command(stream_name, position, batch_size, condition) + parameters = '$1::varchar, $2::bigint, $3::bigint, $4::varchar' + + if category_stream?(stream_name) + return "SELECT * FROM get_category_messages(#{parameters});" + else + return "SELECT * FROM get_stream_messages(#{parameters});" + end + end + + def self.category_stream?(stream_name) + StreamName.category?(stream_name) + end + + def convert(result) + logger.trace { "Converting result to message data (Result Count: #{result.ntuples})" } + + message_data = result.map do |record| record['data'] = Deserialize.data(record['data']) record['metadata'] = Deserialize.metadata(record['metadata']) record['time'] = Time.utc_coerced(record['time']) @@ -64,9 +99,9 @@ def convert(records) MessageData::Read.build record end - logger.debug { "Converted records to message data (Message Data Count: #{messages.length})" } + logger.debug { "Converted result to message data (Message Data Count: #{message_data.length})" } - messages + message_data end module Deserialize @@ -86,6 +121,16 @@ def self.utc_coerced(local_time) Clock::UTC.coerce(local_time) end end + + module Defaults + def self.position + 0 + end + + def self.batch_size + 1000 + end + end end end end diff --git a/lib/message_store/postgres/get/last.rb b/lib/message_store/postgres/get/last.rb index d9911a5..cb04b75 100644 --- a/lib/message_store/postgres/get/last.rb +++ b/lib/message_store/postgres/get/last.rb @@ -13,11 +13,11 @@ def configure(session: nil) def call(stream_name) logger.trace { "Getting last message data (Stream Name: #{stream_name})" } - record = get_record(stream_name) + result = get_result(stream_name) - return nil if record.nil? + return nil if result.nil? - message_data = convert(record) + message_data = convert(result[0]) logger.info { "Finished getting message data (Stream Name: #{stream_name})" } logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } @@ -25,18 +25,28 @@ def call(stream_name) message_data end - def get_record(stream_name) + def get_result(stream_name) logger.trace { "Getting last record (Stream: #{stream_name})" } - select_statement = SelectStatement.build(stream_name) + sql_command = self.class.sql_command(stream_name) - records = session.execute(select_statement.sql) + params = [ + stream_name + ] - logger.debug { "Finished getting record (Stream: #{stream_name})" } + result = session.execute(sql_command, params) - return nil if records.ntuples == 0 + logger.debug { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" } - records[0] + return nil if result.ntuples == 0 + + result + end + + def self.sql_command(stream_name) + parameters = '$1::varchar' + + "SELECT * FROM get_last_message(#{parameters});" end def convert(record) diff --git a/lib/message_store/postgres/get/last/select_statement.rb b/lib/message_store/postgres/get/last/select_statement.rb deleted file mode 100644 index 96cd4d1..0000000 --- a/lib/message_store/postgres/get/last/select_statement.rb +++ /dev/null @@ -1,47 +0,0 @@ -module MessageStore - module Postgres - class Get - class Last - class SelectStatement - include Log::Dependency - - initializer :stream_name - - def self.build(stream_name) - new(stream_name) - end - - def sql - logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name})" } - - statement = <<-SQL - SELECT - id::varchar, - stream_name::varchar, - position::bigint, - type::varchar, - global_position::bigint, - data::varchar, - metadata::varchar, - time::timestamp - FROM - messages - WHERE - stream_name = '#{stream_name}' - ORDER BY - position DESC - LIMIT - 1 - ; - SQL - - logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name})" } - logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } - - statement - end - end - end - end - end -end diff --git a/lib/message_store/postgres/get/select_statement.rb b/lib/message_store/postgres/get/select_statement.rb deleted file mode 100644 index 07302be..0000000 --- a/lib/message_store/postgres/get/select_statement.rb +++ /dev/null @@ -1,102 +0,0 @@ -module MessageStore - module Postgres - class Get - class SelectStatement - include Log::Dependency - - initializer :stream_name, na(:position), na(:batch_size), :condition - - def position - @position ||= Defaults.position - end - - def batch_size - @batch_size ||= Defaults.batch_size - end - - def stream_type_list - @stream_type_list ||= StreamName.get_type_list(stream_name) - end - - def category_stream? - is_category_stream ||= StreamName.category?(stream_name) - end - - def self.build(stream_name, position: nil, batch_size: nil, condition: nil) - new(stream_name, position, batch_size, condition) - end - - def sql - logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{category_stream?}, Types: #{stream_type_list.inspect}, Position: #{position}, Batch Size: #{batch_size}, Condition: #{condition || '(none)'})" } - - formatted_where_clause = where_clause.each_line.to_a.join(" ") - - statement = <<~SQL - SELECT - id::varchar, - stream_name::varchar, - position::bigint, - type::varchar, - global_position::bigint, - data::varchar, - metadata::varchar, - time::timestamp - FROM - messages - WHERE - #{formatted_where_clause} - ORDER BY - #{position_field} ASC - LIMIT - #{batch_size} - ; - SQL - - logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{category_stream?}, Types: #{stream_type_list.inspect}, Position: #{position}, Batch Size: #{batch_size}, Condition: #{condition || '(none)'})" } - logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" } - - statement - end - - def where_clause - clause = <<~SQL.chomp - #{discriminator_field} = '#{stream_name}' AND - #{position_field} >= #{position} - SQL - - unless condition.nil? - clause << " AND\n(#{condition})" - end - - clause - end - - def discriminator_field - unless category_stream? - 'stream_name' - else - 'category(stream_name)' - end - end - - def position_field - unless category_stream? - 'position' - else - 'global_position' - end - end - - module Defaults - def self.position - 0 - end - - def self.batch_size - 1000 - end - end - end - end - end -end diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index de4a548..f57bbf5 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -57,6 +57,7 @@ def destructure_message(write_message) return id, type, data, metadata end +## TODO Rename to put_messages def insert_message(id, stream_name, type, data, metadata, expected_version) serialized_data = serialized_data(data) serialized_metadata = serialized_metadata(metadata) diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index 90a9365..e1a4e87 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -33,7 +33,7 @@ def connect if connected? logger.debug { "Already connected. A new connection will not be built." } - return + return connection end logger.debug { "Not connected. A new connection will be built." } @@ -80,9 +80,9 @@ def settings settings end - def execute(statement, params=nil) - logger.trace { "Executing statement" } - logger.trace(tag: :data) { statement } + def execute(sql_command, params=nil) + logger.trace { "Executing SQL command" } + logger.trace(tag: :data) { sql_command } logger.trace(tag: :data) { params.pretty_inspect } unless connected? @@ -90,12 +90,12 @@ def execute(statement, params=nil) end if params.nil? - connection.exec(statement).tap do - logger.debug { "Executed statement" } + connection.exec(sql_command).tap do + logger.debug { "Executed SQL command (no params)" } end else - connection.exec_params(statement, params).tap do - logger.debug { "Executed statement with params" } + connection.exec_params(sql_command, params).tap do + logger.debug { "Executed SQL command with params" } end end end @@ -117,7 +117,7 @@ def self.settings(settings) s = settings.dup if s.has_key?(:password) - s[:password] = '(hidden)' + s[:password] = '*' * 8 end s diff --git a/materials/raw_response.txt b/materials/raw_response.txt new file mode 100644 index 0000000..a7a84af --- /dev/null +++ b/materials/raw_response.txt @@ -0,0 +1,22 @@ +"(06f65766-6aca-486f-806b-24972b36cff8,testeece91155e0996ed3990babee359e868XX-8371f81b-010c-433b-a53b-9883855c2124,SomeType,0,1,\"{\"\"attribute\"\": \"\"bde32e7f42426b17f665536b31731987\"\"}\",\"{\"\"metaAttribute\"\": \"\"fd334ca5e1d54743160144ab5da0eb09\"\"}\",\"2018-06-08 01:23:05.114566\")" + + +"(06f65766-6aca-486f-806b-24972b36cff8, +testeece91155e0996ed3990babee359e868XX-8371f81b-010c-433b-a53b-9883855c2124, +SomeType, +0, +1, +\"{\"\"attribute\"\": \"\"bde32e7f42426b17f665536b31731987\"\"}\", +\"{\"\"metaAttribute\"\": \"\"fd334ca5e1d54743160144ab5da0eb09\"\"}\", +\"2018-06-08 01:23:05.114566\")" + + +strip parentheses +uuid (string): content up to first comma +stream name (string): content up to next comma +type (string): content up to next comma +position (integer): content up to next comma +global position (integer): content up to next comma +data (string of json): no idea +metadata (string of json): no idea +time (string): last field after the json fields diff --git a/test/automated/select_statement/category_query.rb b/test/automated/select_statement/category_query.rb deleted file mode 100644 index f59fdcb..0000000 --- a/test/automated/select_statement/category_query.rb +++ /dev/null @@ -1,28 +0,0 @@ -require_relative '../automated_init' - -context "Select Statement" do - context "Category Query" do - category = Controls::Category.example - - select_statement = Get::SelectStatement.build(category) - - sql = select_statement.sql - sql.gsub!(/\s+/, ' ') - - context "Where Clause" do - test "Filters on category of stream name" do - assert(sql.include? 'WHERE category(stream_name) =') - end - - test "Filters on global position" do - assert(sql.include? 'global_position >=') - end - end - - context "Order Clause" do - test "Orders by the global position" do - assert(sql.include? 'ORDER BY global_position') - end - end - end -end diff --git a/test/automated/select_statement/condition.rb b/test/automated/select_statement/condition.rb deleted file mode 100644 index a1d5ac4..0000000 --- a/test/automated/select_statement/condition.rb +++ /dev/null @@ -1,28 +0,0 @@ -require_relative '../automated_init' - -context "Select Statement" do - context "Condition" do - stream_name = Controls::StreamName.example - - condition = '1 = 1' - - select_statement = Get::SelectStatement.build(stream_name, condition: condition) - - sql = select_statement.sql - sql.gsub!(/\s+/, ' ') - - context "Where Clause" do - test "Filters on stream name" do - assert(sql.include? 'WHERE stream_name =') - end - - test "Filters on position" do - assert(sql.include? 'position >=') - end - - test "Includes condition encased in parentheses" do - assert(sql.include? "AND (#{condition})") - end - end - end -end diff --git a/test/automated/select_statement/defaults.rb b/test/automated/select_statement/defaults.rb deleted file mode 100644 index 13096c3..0000000 --- a/test/automated/select_statement/defaults.rb +++ /dev/null @@ -1,23 +0,0 @@ -require_relative '../automated_init' - -context "Select Statement" do - context "Defaults" do - stream_name = Controls::StreamName.example - - select_statement = Get::SelectStatement.build(stream_name) - - context "Position" do - default_position = Get::SelectStatement::Defaults.position - test "#{default_position}" do - assert(select_statement.position == default_position) - end - end - - context "Batch Size" do - default_batch_size = Get::SelectStatement::Defaults.batch_size - test "#{default_batch_size}" do - assert(select_statement.batch_size == default_batch_size) - end - end - end -end diff --git a/test/automated/select_statement/stream_name_query.rb b/test/automated/select_statement/stream_name_query.rb deleted file mode 100644 index d407748..0000000 --- a/test/automated/select_statement/stream_name_query.rb +++ /dev/null @@ -1,28 +0,0 @@ -require_relative '../automated_init' - -context "Select Statement" do - context "Stream Name Query" do - stream_name = Controls::StreamName.example - - select_statement = Get::SelectStatement.build(stream_name) - - sql = select_statement.sql - sql.gsub!(/\s+/, ' ') - - context "Where Clause" do - test "Filters on stream name" do - assert(sql.include? 'WHERE stream_name =') - end - - test "Filters on position" do - assert(sql.include? 'position >=') - end - end - - context "Order Clause" do - test "Orders by the global position" do - assert(sql.include? 'ORDER BY position') - end - end - end -end diff --git a/test/interactive/database/get_stream_messages.sh b/test/interactive/database/get_stream_messages.sh index 43e104d..e89b8f2 100755 --- a/test/interactive/database/get_stream_messages.sh +++ b/test/interactive/database/get_stream_messages.sh @@ -2,4 +2,4 @@ test/interactive/database/setup.sh -psql message_store -c -P pager=off "select * from get_stream_messages('someStream-123', 2, 1, _condition => 'global_position = 3');" +psql message_store -P pager=off -c "select * from get_stream_messages('someStream-123', 2, 1, _condition => 'global_position = 3');" diff --git a/test/sketch.rb b/test/sketch.rb new file mode 100644 index 0000000..e334fe7 --- /dev/null +++ b/test/sketch.rb @@ -0,0 +1,10 @@ +require_relative 'test_init' + +# stream_name = Controls::StreamName.example + +stream_name = 'someStream-123' +write_message = Controls::MessageData::Write.example + +position = Put.(write_message, stream_name) + +pp Get.(stream_name, position: position).first From 7d3f9d665e3a3214b0b8285d71f1f5a7a4df3db1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 8 Jun 2018 01:28:01 -0500 Subject: [PATCH 395/591] Script installs server functions --- database/install_functions.sh | 65 +++++++++++++++++++++++++++++++++++ database/types/message.sql | 25 ++++++++------ 2 files changed, 80 insertions(+), 10 deletions(-) create mode 100755 database/install_functions.sh diff --git a/database/install_functions.sh b/database/install_functions.sh new file mode 100755 index 0000000..70771a1 --- /dev/null +++ b/database/install_functions.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -e + +echo +echo "Installing Function" +echo "= = =" +echo + +default_name=message_store + +if [ -z ${DATABASE_NAME+x} ]; then + echo "(DATABASE_NAME is not set. Default will be used.)" + database=$default_name +else + database=$DATABASE_NAME +fi + +echo + +function script_dir { + val="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + echo "$val" +} + +function create-types { + base=$(script_dir) + echo "Creating types..." + + echo "message type" + psql $database -f $base/types/message.sql + + echo +} + +function create-functions { + echo "Creating functions..." + base=$(script_dir) + + echo "hash_64 function" + psql $database -f $base/functions/hash-64.sql + + echo "category function" + psql $database -f $base/functions/category.sql + + echo "stream_version function" + psql $database -f $base/functions/stream-version.sql + + echo "write_message function" + psql $database -f $base/functions/write-message.sql + + echo "get_stream_messages function" + psql $database -f $base/functions/get-stream-messages.sql + + echo "get_category_messages function" + psql $database -f $base/functions/get-category-messages.sql + + echo "get_last_message function" + psql $database -f $base/functions/get-last-message.sql + + echo +} + +create-types +create-functions diff --git a/database/types/message.sql b/database/types/message.sql index 28b3f06..a6b791e 100644 --- a/database/types/message.sql +++ b/database/types/message.sql @@ -1,10 +1,15 @@ -CREATE TYPE message AS ( - id varchar, - stream_name varchar, - type varchar, - position bigint, - global_position bigint, - data varchar, - metadata varchar, - time timestamp -); +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'message') THEN + CREATE TYPE message AS ( + id varchar, + stream_name varchar, + type varchar, + position bigint, + global_position bigint, + data varchar, + metadata varchar, + time timestamp + ); + END IF; +END$$; From b4e4138b72a0417a5cc4c71b4936b054b842f243 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 8 Jun 2018 01:29:14 -0500 Subject: [PATCH 396/591] Spelling mistake is corrected --- database/install_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/install_functions.sh b/database/install_functions.sh index 70771a1..e7833b8 100755 --- a/database/install_functions.sh +++ b/database/install_functions.sh @@ -3,7 +3,7 @@ set -e echo -echo "Installing Function" +echo "Installing Functions" echo "= = =" echo From 34be689d0c83b311324f8d33d21c0e54f78c1a49 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 8 Jun 2018 01:33:14 -0500 Subject: [PATCH 397/591] Existence of database user is not checked before creating it The database must not exist, so checking for the user is superfluous --- database/install.sh | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/database/install.sh b/database/install.sh index 43b38de..4c964f0 100755 --- a/database/install.sh +++ b/database/install.sh @@ -27,17 +27,7 @@ echo function create-user { echo "Database user is: $user" - - user_exists=`psql postgres -qtAXc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` - - if [ "$user_exists" = "1" ]; then - echo "Database user \"$user\" was previously created. Not creating again." - else - echo "Database user \"$user\" has not yet been created" - echo "Creating database user \"$user\"..." - createuser -s $user - fi - + createuser -s $user echo } From ea9abd12732645b38d72631abfcaacf801861e10 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 8 Jun 2018 02:06:25 -0500 Subject: [PATCH 398/591] Obsolete materials are removed --- materials/raw_response.txt | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 materials/raw_response.txt diff --git a/materials/raw_response.txt b/materials/raw_response.txt deleted file mode 100644 index a7a84af..0000000 --- a/materials/raw_response.txt +++ /dev/null @@ -1,22 +0,0 @@ -"(06f65766-6aca-486f-806b-24972b36cff8,testeece91155e0996ed3990babee359e868XX-8371f81b-010c-433b-a53b-9883855c2124,SomeType,0,1,\"{\"\"attribute\"\": \"\"bde32e7f42426b17f665536b31731987\"\"}\",\"{\"\"metaAttribute\"\": \"\"fd334ca5e1d54743160144ab5da0eb09\"\"}\",\"2018-06-08 01:23:05.114566\")" - - -"(06f65766-6aca-486f-806b-24972b36cff8, -testeece91155e0996ed3990babee359e868XX-8371f81b-010c-433b-a53b-9883855c2124, -SomeType, -0, -1, -\"{\"\"attribute\"\": \"\"bde32e7f42426b17f665536b31731987\"\"}\", -\"{\"\"metaAttribute\"\": \"\"fd334ca5e1d54743160144ab5da0eb09\"\"}\", -\"2018-06-08 01:23:05.114566\")" - - -strip parentheses -uuid (string): content up to first comma -stream name (string): content up to next comma -type (string): content up to next comma -position (integer): content up to next comma -global position (integer): content up to next comma -data (string of json): no idea -metadata (string of json): no idea -time (string): last field after the json fields From f372a3b68f4cf938bd22d30998ef8f33d73ee7a4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 8 Jun 2018 02:07:19 -0500 Subject: [PATCH 399/591] Get benchmark, post retrieval server functions --- ...t Benchmark - 2018-06-08T06-42-47.108Z.txt | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z.txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z.txt new file mode 100644 index 0000000..22a5fec --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z.txt @@ -0,0 +1,21 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 81638.440000ms +Mean Cycle Time: 0.816384ms (± 0.346983ms) +Cycles Per Second: 1224.913166 +GC: off + +Note: Post-retrieval server function implementation + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 From 1f2eaf832a1d01443b00a00d62cb86b211a55e70 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 8 Jun 2018 10:15:55 -0500 Subject: [PATCH 400/591] Vestigial file is removed --- test/sketch.rb | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 test/sketch.rb diff --git a/test/sketch.rb b/test/sketch.rb deleted file mode 100644 index e334fe7..0000000 --- a/test/sketch.rb +++ /dev/null @@ -1,10 +0,0 @@ -require_relative 'test_init' - -# stream_name = Controls::StreamName.example - -stream_name = 'someStream-123' -write_message = Controls::MessageData::Write.example - -position = Put.(write_message, stream_name) - -pp Get.(stream_name, position: position).first From 1b8a90cba1388779abaaafa46f2de2cc831321b7 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 8 Jun 2018 10:21:26 -0500 Subject: [PATCH 401/591] Date is corrected --- MIT-License.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MIT-License.txt b/MIT-License.txt index b75881d..b6856d0 100644 --- a/MIT-License.txt +++ b/MIT-License.txt @@ -1,4 +1,4 @@ -Modified work Copyright (c) 2016-2017 Scott Bellware +Copyright (c) 2015-present Scott Bellware Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the From d71b20716eb5e80a4db9e0fea9d5bee9714c2ec9 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 8 Jun 2018 12:41:08 -0500 Subject: [PATCH 402/591] Database artifacts are removed to the message_store-postgres-database package --- database/clear-events-table.sh | 40 ------ database/extensions.sql | 1 - database/functions/category.sql | 10 -- database/functions/get-category-messages.sql | 45 ------- database/functions/get-last-message.sql | 33 ----- database/functions/get-stream-messages.sql | 45 ------- database/functions/hash-64.sql | 13 -- database/functions/stream-version.sql | 14 --- database/functions/write-message.sql | 60 --------- .../messages-category-global-position.sql | 1 - database/indexes/messages-id.sql | 1 - .../messages-stream-name-position-uniq.sql | 1 - database/install.sh | 119 ------------------ database/install_functions.sh | 65 ---------- database/print-messages.sh | 52 -------- database/table/messages-table.sql | 19 --- database/types/message.sql | 15 --- database/uninstall.sh | 60 --------- scripts/evt-pg-create-db | 7 -- scripts/evt-pg-delete-db | 7 -- scripts/evt-pg-print-messages | 7 -- scripts/evt-pg-recreate-db | 11 -- test/interactive/database.sh | 11 -- .../database/get_category_messages.sh | 5 - test/interactive/database/get_last_message.sh | 5 - .../database/get_stream_messages.sh | 5 - test/interactive/database/hash_64.sh | 3 - .../interactive/database/recreate_database.sh | 4 - test/interactive/database/setup.sh | 4 - test/interactive/database/setup_messages.sh | 5 - test/interactive/database/write_message.sh | 3 - .../write_message_expected_version.sh | 7 -- .../write_message_expected_version_error.sh | 7 -- 33 files changed, 685 deletions(-) delete mode 100755 database/clear-events-table.sh delete mode 100644 database/extensions.sql delete mode 100644 database/functions/category.sql delete mode 100644 database/functions/get-category-messages.sql delete mode 100644 database/functions/get-last-message.sql delete mode 100644 database/functions/get-stream-messages.sql delete mode 100644 database/functions/hash-64.sql delete mode 100644 database/functions/stream-version.sql delete mode 100644 database/functions/write-message.sql delete mode 100644 database/indexes/messages-category-global-position.sql delete mode 100644 database/indexes/messages-id.sql delete mode 100644 database/indexes/messages-stream-name-position-uniq.sql delete mode 100755 database/install.sh delete mode 100755 database/install_functions.sh delete mode 100755 database/print-messages.sh delete mode 100644 database/table/messages-table.sql delete mode 100644 database/types/message.sql delete mode 100755 database/uninstall.sh delete mode 100755 scripts/evt-pg-create-db delete mode 100755 scripts/evt-pg-delete-db delete mode 100755 scripts/evt-pg-print-messages delete mode 100755 scripts/evt-pg-recreate-db delete mode 100755 test/interactive/database.sh delete mode 100755 test/interactive/database/get_category_messages.sh delete mode 100755 test/interactive/database/get_last_message.sh delete mode 100755 test/interactive/database/get_stream_messages.sh delete mode 100755 test/interactive/database/hash_64.sh delete mode 100755 test/interactive/database/recreate_database.sh delete mode 100755 test/interactive/database/setup.sh delete mode 100755 test/interactive/database/setup_messages.sh delete mode 100755 test/interactive/database/write_message.sh delete mode 100755 test/interactive/database/write_message_expected_version.sh delete mode 100755 test/interactive/database/write_message_expected_version_error.sh diff --git a/database/clear-events-table.sh b/database/clear-events-table.sh deleted file mode 100755 index 4977358..0000000 --- a/database/clear-events-table.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo -echo "Clearing Events Table" -echo "= = =" -echo - -default_name=message_store - -if [ -z ${DATABASE_USER+x} ]; then - echo "(DATABASE_USER is not set)" - user=$default_name -else - user=$DATABASE_USER -fi -echo "Database user is: $user" - -if [ -z ${DATABASE_NAME+x} ]; then - echo "(DATABASE_NAME is not set)" - database=$default_name -else - database=$DATABASE_NAME -fi -echo "Database name is: $database" - -default_table_name=messages - -if [ -z ${TABLE_NAME+x} ]; then - echo "(TABLE_NAME is not set)" - table=$default_table_name -else - table=$TABLE_NAME -fi -echo "Table name is: $table" - -echo - -psql $database -c "TRUNCATE $table RESTART IDENTITY;" diff --git a/database/extensions.sql b/database/extensions.sql deleted file mode 100644 index 7ca3626..0000000 --- a/database/extensions.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS "pgcrypto"; diff --git a/database/functions/category.sql b/database/functions/category.sql deleted file mode 100644 index 014fc74..0000000 --- a/database/functions/category.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE OR REPLACE FUNCTION category( - _stream_name varchar -) -RETURNS varchar -AS $$ -BEGIN - return split_part(_stream_name, '-', 1); -END; -$$ LANGUAGE plpgsql -IMMUTABLE; diff --git a/database/functions/get-category-messages.sql b/database/functions/get-category-messages.sql deleted file mode 100644 index 0290ec3..0000000 --- a/database/functions/get-category-messages.sql +++ /dev/null @@ -1,45 +0,0 @@ -CREATE OR REPLACE FUNCTION get_category_messages( - _stream_name varchar, - _position bigint DEFAULT 0, - _batch_size bigint DEFAULT 1000, - _condition varchar DEFAULT NULL -) -RETURNS SETOF message -AS $$ -DECLARE - command text; -BEGIN - command := ' - SELECT - id::varchar, - stream_name::varchar, - type::varchar, - position::bigint, - global_position::bigint, - data::varchar, - metadata::varchar, - time::timestamp - FROM - messages - WHERE - category(stream_name) = $1 AND - global_position >= $2'; - - if _condition is not null then - command := command || ' AND - %s'; - command := format(command, _condition); - end if; - - command := command || ' - ORDER BY - global_position ASC - LIMIT - $3'; - - -- RAISE NOTICE '%', command; - - RETURN QUERY EXECUTE command USING _stream_name, _position, _batch_size; -END; -$$ LANGUAGE plpgsql -VOLATILE; diff --git a/database/functions/get-last-message.sql b/database/functions/get-last-message.sql deleted file mode 100644 index 7125820..0000000 --- a/database/functions/get-last-message.sql +++ /dev/null @@ -1,33 +0,0 @@ -CREATE OR REPLACE FUNCTION get_last_message( - _stream_name varchar -) -RETURNS SETOF message -AS $$ -DECLARE - command text; -BEGIN - command := ' - SELECT - id::varchar, - stream_name::varchar, - type::varchar, - position::bigint, - global_position::bigint, - data::varchar, - metadata::varchar, - time::timestamp - FROM - messages - WHERE - stream_name = $1 - ORDER BY - position DESC - LIMIT - 1'; - - -- RAISE NOTICE '%', command; - - RETURN QUERY EXECUTE command USING _stream_name; -END; -$$ LANGUAGE plpgsql -VOLATILE; diff --git a/database/functions/get-stream-messages.sql b/database/functions/get-stream-messages.sql deleted file mode 100644 index 89bd3cf..0000000 --- a/database/functions/get-stream-messages.sql +++ /dev/null @@ -1,45 +0,0 @@ -CREATE OR REPLACE FUNCTION get_stream_messages( - _stream_name varchar, - _position bigint DEFAULT 0, - _batch_size bigint DEFAULT 1000, - _condition varchar DEFAULT NULL -) -RETURNS SETOF message -AS $$ -DECLARE - command text; -BEGIN - command := ' - SELECT - id::varchar, - stream_name::varchar, - type::varchar, - position::bigint, - global_position::bigint, - data::varchar, - metadata::varchar, - time::timestamp - FROM - messages - WHERE - stream_name = $1 AND - position >= $2'; - - if _condition is not null then - command := command || ' AND - %s'; - command := format(command, _condition); - end if; - - command := command || ' - ORDER BY - position ASC - LIMIT - $3'; - - -- RAISE NOTICE '%', command; - - RETURN QUERY EXECUTE command USING _stream_name, _position, _batch_size; -END; -$$ LANGUAGE plpgsql -VOLATILE; diff --git a/database/functions/hash-64.sql b/database/functions/hash-64.sql deleted file mode 100644 index fb35e94..0000000 --- a/database/functions/hash-64.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE OR REPLACE FUNCTION hash_64( - _stream_name varchar -) -RETURNS bigint -AS $$ -DECLARE - hash bigint; -BEGIN - select left('x' || md5(_stream_name), 16)::bit(64)::bigint into hash; - return hash; -END; -$$ LANGUAGE plpgsql -IMMUTABLE; diff --git a/database/functions/stream-version.sql b/database/functions/stream-version.sql deleted file mode 100644 index 220d405..0000000 --- a/database/functions/stream-version.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE OR REPLACE FUNCTION stream_version( - _stream_name varchar -) -RETURNS bigint -AS $$ -DECLARE - stream_version bigint; -BEGIN - select max(position) into stream_version from messages where stream_name = _stream_name; - - return stream_version; -END; -$$ LANGUAGE plpgsql -VOLATILE; diff --git a/database/functions/write-message.sql b/database/functions/write-message.sql deleted file mode 100644 index 38e4613..0000000 --- a/database/functions/write-message.sql +++ /dev/null @@ -1,60 +0,0 @@ -CREATE OR REPLACE FUNCTION write_message( - _id varchar, - _stream_name varchar, - _type varchar, - _data jsonb, - _metadata jsonb DEFAULT NULL, - _expected_version bigint DEFAULT NULL -) -RETURNS bigint -AS $$ -DECLARE - message_id uuid; - stream_version bigint; - position bigint; - category varchar; - stream_name_hash bigint; -BEGIN - message_id = uuid(_id); - - stream_name_hash = hash_64(_stream_name); - PERFORM pg_advisory_xact_lock(stream_name_hash); - - stream_version := stream_version(_stream_name); - - if stream_version is null then - stream_version := -1; - end if; - - if _expected_version is not null then - if _expected_version != stream_version then - raise exception 'Wrong expected version: % (Stream: %, Stream Version: %)', _expected_version, _stream_name, stream_version; - end if; - end if; - - position := stream_version + 1; - - insert into "messages" - ( - "id", - "stream_name", - "position", - "type", - "data", - "metadata" - ) - values - ( - message_id, - _stream_name, - position, - _type, - _data, - _metadata - ) - ; - - return position; -END; -$$ LANGUAGE plpgsql -VOLATILE; diff --git a/database/indexes/messages-category-global-position.sql b/database/indexes/messages-category-global-position.sql deleted file mode 100644 index cd60c3c..0000000 --- a/database/indexes/messages-category-global-position.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE INDEX CONCURRENTLY "messages_category_global_position_idx" ON "public"."messages" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "global_position" "pg_catalog"."int8_ops" ASC NULLS LAST); diff --git a/database/indexes/messages-id.sql b/database/indexes/messages-id.sql deleted file mode 100644 index 7ca81f3..0000000 --- a/database/indexes/messages-id.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE INDEX CONCURRENTLY "messages_id_idx" ON "public"."messages" USING btree(id ASC NULLS LAST); diff --git a/database/indexes/messages-stream-name-position-uniq.sql b/database/indexes/messages-stream-name-position-uniq.sql deleted file mode 100644 index 7cd8d47..0000000 --- a/database/indexes/messages-stream-name-position-uniq.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE UNIQUE INDEX CONCURRENTLY "messages_stream_name_position_uniq_idx" ON "public"."messages" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "position" "pg_catalog"."int8_ops" ASC NULLS LAST); diff --git a/database/install.sh b/database/install.sh deleted file mode 100755 index 4c964f0..0000000 --- a/database/install.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo -echo "Installing Database" -echo "= = =" -echo - -default_name=message_store - -if [ -z ${DATABASE_USER+x} ]; then - echo "(DATABASE_USER is not set. Default will be used.)" - user=$default_name -else - user=$DATABASE_USER -fi - -if [ -z ${DATABASE_NAME+x} ]; then - echo "(DATABASE_NAME is not set. Default will be used.)" - database=$default_name -else - database=$DATABASE_NAME -fi - -echo - -function create-user { - echo "Database user is: $user" - createuser -s $user - echo -} - -function create-database { - echo "Database name is: $database" - echo "Creating database \"$database\"..." - createdb $database - echo -} - -function script_dir { - val="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - echo "$val" -} - -function create-extensions { - echo "Creating extensions..." - base=$(script_dir) - psql $database -f $base/extensions.sql - echo -} - -function create-table { - echo "Creating messages table..." - psql $database -f $base/table/messages-table.sql - echo -} - -function create-types { - base=$(script_dir) - echo "Creating types..." - - echo "message type" - psql $database -f $base/types/message.sql - - echo -} - -function create-functions { - echo "Creating functions..." - base=$(script_dir) - - echo "hash_64 function" - psql $database -f $base/functions/hash-64.sql - - echo "category function" - psql $database -f $base/functions/category.sql - - echo "stream_version function" - psql $database -f $base/functions/stream-version.sql - - echo "write_message function" - psql $database -f $base/functions/write-message.sql - - echo "get_stream_messages function" - psql $database -f $base/functions/get-stream-messages.sql - - echo "get_category_messages function" - psql $database -f $base/functions/get-category-messages.sql - - echo "get_last_message function" - psql $database -f $base/functions/get-last-message.sql - - echo -} - -function create-indexes { - echo "Creating indexes..." - base=$(script_dir) - - echo "messages_id_idx" - psql $database -f $base/indexes/messages-id.sql - - echo "messages_category_global_position_idx" - psql $database -f $base/indexes/messages-category-global-position.sql - - echo "messages_stream_name_position_uniq_idx" - psql $database -f $base/indexes/messages-stream-name-position-uniq.sql - - echo -} - -create-user -create-database -create-extensions -create-table -create-types -create-functions -create-indexes diff --git a/database/install_functions.sh b/database/install_functions.sh deleted file mode 100755 index e7833b8..0000000 --- a/database/install_functions.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo -echo "Installing Functions" -echo "= = =" -echo - -default_name=message_store - -if [ -z ${DATABASE_NAME+x} ]; then - echo "(DATABASE_NAME is not set. Default will be used.)" - database=$default_name -else - database=$DATABASE_NAME -fi - -echo - -function script_dir { - val="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - echo "$val" -} - -function create-types { - base=$(script_dir) - echo "Creating types..." - - echo "message type" - psql $database -f $base/types/message.sql - - echo -} - -function create-functions { - echo "Creating functions..." - base=$(script_dir) - - echo "hash_64 function" - psql $database -f $base/functions/hash-64.sql - - echo "category function" - psql $database -f $base/functions/category.sql - - echo "stream_version function" - psql $database -f $base/functions/stream-version.sql - - echo "write_message function" - psql $database -f $base/functions/write-message.sql - - echo "get_stream_messages function" - psql $database -f $base/functions/get-stream-messages.sql - - echo "get_category_messages function" - psql $database -f $base/functions/get-category-messages.sql - - echo "get_last_message function" - psql $database -f $base/functions/get-last-message.sql - - echo -} - -create-types -create-functions diff --git a/database/print-messages.sh b/database/print-messages.sh deleted file mode 100755 index 111b394..0000000 --- a/database/print-messages.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo -echo "Printing Messages" -echo "= = =" -echo - -default_name=message_store - -if [ -z ${DATABASE_USER+x} ]; then - echo "(DATABASE_USER is not set)" - user=$default_name -else - user=$DATABASE_USER -fi -echo "Database user is: $user" - -if [ -z ${DATABASE_NAME+x} ]; then - echo "(DATABASE_NAME is not set)" - database=$default_name -else - database=$DATABASE_NAME -fi -echo "Database name is: $database" - -default_table_name=messages - -if [ -z ${TABLE_NAME+x} ]; then - echo "(TABLE_NAME is not set)" - table=$default_table_name -else - table=$TABLE_NAME -fi -echo "Table name is: $table" - -if [ -z ${STREAM_NAME+x} ]; then - echo "(STREAM_NAME is not set)" - stream_name='' -else - stream_name=$STREAM_NAME - echo "Stream name is: $STREAM_NAME" -fi - -echo - -if [ -z $stream_name ]; then - psql $database -x -c "SELECT * FROM $table" -else - psql $database -x -c "SELECT * FROM $table WHERE stream_name = '$stream_name'" -fi diff --git a/database/table/messages-table.sql b/database/table/messages-table.sql deleted file mode 100644 index f5d5ae1..0000000 --- a/database/table/messages-table.sql +++ /dev/null @@ -1,19 +0,0 @@ --- ---------------------------- --- Table structure for messages --- ---------------------------- -CREATE TABLE "public"."messages" ( - "id" UUID NOT NULL DEFAULT gen_random_uuid(), - "stream_name" varchar(255) NOT NULL COLLATE "default", - "type" varchar(255) NOT NULL COLLATE "default", - "position" bigint NOT NULL, - "global_position" bigserial NOT NULL , - "data" jsonb, - "metadata" jsonb, - "time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL -) -WITH (OIDS=FALSE); - --- ---------------------------- --- Primary key structure for table messages --- ---------------------------- -ALTER TABLE "public"."messages" ADD PRIMARY KEY ("global_position") NOT DEFERRABLE INITIALLY IMMEDIATE; diff --git a/database/types/message.sql b/database/types/message.sql deleted file mode 100644 index a6b791e..0000000 --- a/database/types/message.sql +++ /dev/null @@ -1,15 +0,0 @@ -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'message') THEN - CREATE TYPE message AS ( - id varchar, - stream_name varchar, - type varchar, - position bigint, - global_position bigint, - data varchar, - metadata varchar, - time timestamp - ); - END IF; -END$$; diff --git a/database/uninstall.sh b/database/uninstall.sh deleted file mode 100755 index d635193..0000000 --- a/database/uninstall.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo -echo "Uninstalling Database" -echo "= = =" -echo - -default_name=message_store - -if [ -z ${DATABASE_USER+x} ]; then - echo "(DATABASE_USER is not set. Default will be used.)" - user=$default_name -else - user=$DATABASE_USER -fi - -if [ -z ${DATABASE_NAME+x} ]; then - echo "(DATABASE_NAME is not set. Default will be used.)" - database=$default_name -else - database=$DATABASE_NAME -fi - -echo - -function delete-user { - echo "Database user is: $user" - - user_exists=`psql postgres -qtAXc "SELECT 1 FROM pg_roles WHERE rolname='$user'"` - - if [ "$user_exists" = "1" ]; then - echo "Deleting database user \"$user\"..." - dropuser $user - else - echo "Database user \"$user\" does not exist. Not deleting." - fi - - echo -} - -function delete-database { - echo "Database name is: $database" - - database_exists=`psql postgres -qtAXc "SELECT 1 FROM pg_database WHERE datname='$database'"` - - if [ "$database_exists" = "1" ]; then - echo "Deleting database \"$database\"..." - dropdb $database - else - echo "Database \"$database\" does not exist. Not deleting." - fi - - echo -} - - -delete-database -delete-user diff --git a/scripts/evt-pg-create-db b/scripts/evt-pg-create-db deleted file mode 100755 index 970a497..0000000 --- a/scripts/evt-pg-create-db +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env ruby - -root = File.expand_path '../database', __dir__ -script_filename = 'install.sh' -script_filepath = File.join root, script_filename - -system script_filepath diff --git a/scripts/evt-pg-delete-db b/scripts/evt-pg-delete-db deleted file mode 100755 index 1177be8..0000000 --- a/scripts/evt-pg-delete-db +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env ruby - -root = File.expand_path '../database', __dir__ -script_filename = 'uninstall.sh' -script_filepath = File.join root, script_filename - -system script_filepath diff --git a/scripts/evt-pg-print-messages b/scripts/evt-pg-print-messages deleted file mode 100755 index 17e5924..0000000 --- a/scripts/evt-pg-print-messages +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env ruby - -root = File.expand_path '../database', __dir__ -script_filename = 'print-messages.sh' -script_filepath = File.join root, script_filename - -system script_filepath diff --git a/scripts/evt-pg-recreate-db b/scripts/evt-pg-recreate-db deleted file mode 100755 index e276fa7..0000000 --- a/scripts/evt-pg-recreate-db +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env ruby - -root = File.expand_path '../database', __dir__ - -script_filename = 'uninstall.sh' -script_filepath = File.join root, script_filename -system script_filepath - -script_filename = 'install.sh' -script_filepath = File.join root, script_filename -system script_filepath diff --git a/test/interactive/database.sh b/test/interactive/database.sh deleted file mode 100755 index 6fac823..0000000 --- a/test/interactive/database.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -test/interactive/database/hash_64.sh - -test/interactive/database/write_message.sh -test/interactive/database/write_message_expected_version.sh -test/interactive/database/write_message_expected_version_error.sh - -test/interactive/database/get_stream_messages.sh -test/interactive/database/get_category_messages.sh -test/interactive/database/get_last_message.sh diff --git a/test/interactive/database/get_category_messages.sh b/test/interactive/database/get_category_messages.sh deleted file mode 100755 index f043dd3..0000000 --- a/test/interactive/database/get_category_messages.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -test/interactive/database/setup.sh - -psql message_store -P pager=off -c "select * from get_category_messages('someStream', 4, 1, _condition => 'position = 3');" diff --git a/test/interactive/database/get_last_message.sh b/test/interactive/database/get_last_message.sh deleted file mode 100755 index f237d4e..0000000 --- a/test/interactive/database/get_last_message.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -test/interactive/database/setup.sh - -psql message_store -P pager=off -c "select * from get_last_message('someStream-123');" diff --git a/test/interactive/database/get_stream_messages.sh b/test/interactive/database/get_stream_messages.sh deleted file mode 100755 index e89b8f2..0000000 --- a/test/interactive/database/get_stream_messages.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -test/interactive/database/setup.sh - -psql message_store -P pager=off -c "select * from get_stream_messages('someStream-123', 2, 1, _condition => 'global_position = 3');" diff --git a/test/interactive/database/hash_64.sh b/test/interactive/database/hash_64.sh deleted file mode 100755 index 5183683..0000000 --- a/test/interactive/database/hash_64.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -psql message_store -c "SELECT hash_64('someStream-123');" diff --git a/test/interactive/database/recreate_database.sh b/test/interactive/database/recreate_database.sh deleted file mode 100755 index 1b7d981..0000000 --- a/test/interactive/database/recreate_database.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -database/uninstall.sh -database/install.sh diff --git a/test/interactive/database/setup.sh b/test/interactive/database/setup.sh deleted file mode 100755 index 6442719..0000000 --- a/test/interactive/database/setup.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -test/interactive/database/recreate_database.sh -test/interactive/database/setup_messages.sh diff --git a/test/interactive/database/setup_messages.sh b/test/interactive/database/setup_messages.sh deleted file mode 100755 index 4410c48..0000000 --- a/test/interactive/database/setup_messages.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -STREAM_NAME=someStream-123 INSTANCES=4 tools/write_message.rb -STREAM_NAME=someStream-456 INSTANCES=4 tools/write_message.rb -STREAM_NAME=someOtherStream-456 INSTANCES=4 tools/write_message.rb diff --git a/test/interactive/database/write_message.sh b/test/interactive/database/write_message.sh deleted file mode 100755 index 72cc02c..0000000 --- a/test/interactive/database/write_message.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -psql message_store -c "SELECT write_message(gen_random_uuid()::varchar, 'testWriteIsolation'::varchar, 'SomeType'::varchar, '{\"attribute\": \"some value\"}'::jsonb, '{\"metaAttribute\": \"some meta value\"}'::jsonb);" diff --git a/test/interactive/database/write_message_expected_version.sh b/test/interactive/database/write_message_expected_version.sh deleted file mode 100755 index d27afac..0000000 --- a/test/interactive/database/write_message_expected_version.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -test/interactive/database/recreate_database.sh - -STREAM_NAME=someStream-123 INSTANCES=4 tools/write_message.rb - -psql message_store -c "SELECT write_message(gen_random_uuid()::varchar, 'someStream-123'::varchar, 'SomeType'::varchar, '{\"attribute\": \"some value\"}'::jsonb, '{\"metaAttribute\": \"some meta value\"}'::jsonb, 3::bigint);" diff --git a/test/interactive/database/write_message_expected_version_error.sh b/test/interactive/database/write_message_expected_version_error.sh deleted file mode 100755 index 4d975a6..0000000 --- a/test/interactive/database/write_message_expected_version_error.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -test/interactive/database/recreate_database.sh - -STREAM_NAME=someStream-123 INSTANCES=4 tools/write_message.rb - -psql message_store -c "SELECT write_message(gen_random_uuid()::varchar, 'someStream-123'::varchar, 'SomeType'::varchar, '{\"attribute\": \"some value\"}'::jsonb, '{\"metaAttribute\": \"some meta value\"}'::jsonb, 4::bigint);" From 11b9bae072b792518e87b27546d1bb1be6d48d34 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 8 Jun 2018 13:34:08 -0500 Subject: [PATCH 403/591] Package version is increased from 0.9.0.0 to 0.10.0.0 --- message_store-postgres.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index ac87dbb..985caa1 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.9.0.0' + s.version = '0.10.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' @@ -19,7 +19,7 @@ Gem::Specification.new do |s| s.bindir = 'scripts' s.add_runtime_dependency 'evt-message_store' - + s.add_runtime_dependency 'evt-message_store-postgres-database' s.add_runtime_dependency 'evt-log' s.add_runtime_dependency 'evt-settings' From 0f413c183bd59d724ba993b48209386f4dc6a4b3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 25 Jun 2018 10:29:06 -0500 Subject: [PATCH 404/591] Test does not accidentally cause uniqueness violation by writing the same message with the same ID more than once --- test/automated/put/stream_position_increases.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/automated/put/stream_position_increases.rb b/test/automated/put/stream_position_increases.rb index 3b5828f..0c45bcf 100644 --- a/test/automated/put/stream_position_increases.rb +++ b/test/automated/put/stream_position_increases.rb @@ -3,10 +3,12 @@ context "Put" do context "Stream Position Increases with Subsequent Writes" do stream_name = Controls::StreamName.example - write_message = Controls::MessageData::Write.example - position_1 = Put.(write_message, stream_name) - position_2 = Put.(write_message, stream_name) + write_message_1 = Controls::MessageData::Write.example + write_message_2 = Controls::MessageData::Write.example + + position_1 = Put.(write_message_1, stream_name) + position_2 = Put.(write_message_2, stream_name) test "First version is one less than the second version" do assert(position_1 + 1 == position_2) From 58b68d801355cd92db284d16c43fc83791e9e39f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 25 Jun 2018 10:43:57 -0500 Subject: [PATCH 405/591] Benchmark result for ID index unique constraint --- ...t Benchmark - 2018-06-25T15-41-28.825Z.txt | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z.txt diff --git a/test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z.txt new file mode 100644 index 0000000..08c651c --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z.txt @@ -0,0 +1,21 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 127165.967000ms +Mean Cycle Time: 1.271660ms (± 0.604510ms) +Cycles Per Second: 786.373920 +GC: off + +Note: ID index has a unique constraint + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0156.B00 +SMC Version (system): 2.42f10 From a0827d8880f90a0ac8e24234b93fec25fc8a7e88 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 13 Jul 2018 14:49:59 -0500 Subject: [PATCH 406/591] Sql command logging is tagged with :sql rather than :data --- lib/message_store/postgres/session.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index e1a4e87..2188c4e 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -82,7 +82,7 @@ def settings def execute(sql_command, params=nil) logger.trace { "Executing SQL command" } - logger.trace(tag: :data) { sql_command } + logger.trace(tag: :sql) { sql_command } logger.trace(tag: :data) { params.pretty_inspect } unless connected? From 5cf353904a7dd1d68c3284675df52a2bff0162fd Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 13 Jul 2018 14:53:04 -0500 Subject: [PATCH 407/591] Package version is increased from 0.10.0.0 to 0.10.0.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 985caa1..58fc96f 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.10.0.0' + s.version = '0.10.0.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From ce671a5193b8b070fcce631fe20a9e838aa36062 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 13 Jul 2018 16:25:25 -0500 Subject: [PATCH 408/591] Vestigial comment is removed --- lib/message_store/postgres/put.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index f57bbf5..de4a548 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -57,7 +57,6 @@ def destructure_message(write_message) return id, type, data, metadata end -## TODO Rename to put_messages def insert_message(id, stream_name, type, data, metadata, expected_version) serialized_data = serialized_data(data) serialized_metadata = serialized_metadata(metadata) From 3cbe478831a8cb793b604f31fd90a9b8e56e52b2 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 13 Jul 2018 16:53:23 -0500 Subject: [PATCH 409/591] Session open is an alias for connect --- lib/message_store/postgres/session.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index 2188c4e..6e67c4b 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -28,7 +28,7 @@ def self.configure(receiver, session: nil, attr_name: nil) instance end - def connect + def open logger.trace { "Connecting to database" } if connected? @@ -44,6 +44,7 @@ def connect connection end + alias :connect :open def self.build_connection(instance) settings = instance.settings From 3d7f797fcd82c42bbd03f09a99dccc6f46d51333 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 13 Jul 2018 16:54:03 -0500 Subject: [PATCH 410/591] Package version is increased from 0.10.0.1 to 0.10.1.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 58fc96f..6822aae 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.10.0.1' + s.version = '0.10.1.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 91e46c0d43cd71feee3b25f6f97a146e87ab05f4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 15 Jul 2018 15:18:33 -0500 Subject: [PATCH 411/591] Override settings file path with MESSAGE_STORE_SETTINGS_PATH environment variable --- lib/message_store/postgres/settings.rb | 8 +++++++- test/automated/session/settings.rb | 2 +- test/automated/settings/default_path.rb | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 test/automated/settings/default_path.rb diff --git a/lib/message_store/postgres/settings.rb b/lib/message_store/postgres/settings.rb index f7d775f..f1a4298 100644 --- a/lib/message_store/postgres/settings.rb +++ b/lib/message_store/postgres/settings.rb @@ -6,7 +6,7 @@ def self.instance end def self.data_source - 'settings/message_store_postgres.json' + Defaults.data_source || 'settings/message_store_postgres.json' end def self.names @@ -25,6 +25,12 @@ def self.names :service ] end + + class Defaults + def self.data_source + ENV['MESSAGE_STORE_SETTINGS_PATH'] + end + end end end end diff --git a/test/automated/session/settings.rb b/test/automated/session/settings.rb index 6676102..505f745 100644 --- a/test/automated/session/settings.rb +++ b/test/automated/session/settings.rb @@ -11,7 +11,7 @@ names.each do |name| test "#{name}" do - session_val = session.public_send name + session_val = session.public_send(name) settings_val = settings_hash[name] assert(session_val == settings_val) diff --git a/test/automated/settings/default_path.rb b/test/automated/settings/default_path.rb new file mode 100644 index 0000000..46ada36 --- /dev/null +++ b/test/automated/settings/default_path.rb @@ -0,0 +1,18 @@ +require_relative '../automated_init' + +context "Settings" do + context "Default Path" do + prior_default_settings_path = ENV['MESSAGE_STORE_SETTINGS_PATH'] + + overridden_path = 'some_path' + ENV['MESSAGE_STORE_SETTINGS_PATH'] = overridden_path + + settings_path = Postgres::Settings.data_source + + test "Overridden by MESSAGE_STORE_SETTINGS_PATH environment variable" do + assert(settings_path == overridden_path) + end + + ENV['MESSAGE_STORE_SETTINGS_PATH'] = prior_default_settings_path + end +end From d3c355e8d0a94b4ff22ea98316722835d258766d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 15 Jul 2018 15:19:24 -0500 Subject: [PATCH 412/591] Package version is increased from 0.10.1.0 to 0.10.2.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 6822aae..d6edda3 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.10.1.0' + s.version = '0.10.2.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 31626c23932fe601f4b80ccb95067c1cfc2df592 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 15 Jul 2018 21:26:22 -0500 Subject: [PATCH 413/591] Writer logs list of types written --- lib/message_store/postgres/write.rb | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/message_store/postgres/write.rb b/lib/message_store/postgres/write.rb index b1789e6..7b707f6 100644 --- a/lib/message_store/postgres/write.rb +++ b/lib/message_store/postgres/write.rb @@ -10,7 +10,10 @@ def configure(session: nil) end def write(batch, stream_name, expected_version: nil) - logger.trace(tag: :write) { "Writing batch (Stream Name: #{stream_name}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})" } + logger.trace(tag: :write) do + message_types = batch.map {|message_data| message_data.type }.uniq.join(', ') + "Writing batch (Stream Name: #{stream_name}, Types: #{message_types}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})" + end unless expected_version.nil? expected_version = ExpectedVersion.canonize(expected_version) @@ -27,18 +30,21 @@ def write(batch, stream_name, expected_version: nil) end end - logger.debug(tag: :write) { "Wrote batch (Stream Name: #{stream_name}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})" } + logger.debug(tag: :write) do + message_types = batch.map {|message_data| message_data.type }.uniq.join(', ') + "Wrote batch (Stream Name: #{stream_name}, Types: #{message_types}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})" + end last_position end def write_message_data(message_data, stream_name, expected_version: nil) logger.trace(tag: :write) { "Writing message data (Stream Name: #{stream_name}, Type: #{message_data.type}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :message_data, :write]) { message_data.pretty_inspect } + logger.trace(tags: [:data, :message_data]) { message_data.pretty_inspect } put.(message_data, stream_name, expected_version: expected_version).tap do logger.debug(tag: :write) { "Wrote message data (Stream Name: #{stream_name}, Type: #{message_data.type}, Expected Version: #{expected_version.inspect})" } - logger.debug(tags: [:data, :message_data, :write]) { message_data.pretty_inspect } + logger.debug(tags: [:data, :message_data]) { message_data.pretty_inspect } end end end From 85d6e5ece7fda86189a867ce9357c0bea8b0ac34 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 15 Jul 2018 21:27:06 -0500 Subject: [PATCH 414/591] Package version is increased from 0.10.2.0 to 0.10.2.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index d6edda3..8776b97 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.10.2.0' + s.version = '0.10.2.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 230bb2c16e576d939221ee878af2d44d3a864bf3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 16 Jul 2018 00:09:01 -0500 Subject: [PATCH 415/591] Logging --- lib/message_store/postgres/put.rb | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index de4a548..a7129d3 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -29,7 +29,7 @@ def self.call(write_message, stream_name, expected_version: nil, session: nil) end def call(write_message, stream_name, expected_version: nil) - logger.trace { "Putting message data (Stream Name: #{stream_name}, Type: #{write_message.type}, Expected Version: #{expected_version.inspect})" } + logger.trace { "Putting message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" } logger.trace(tags: [:data, :message_data]) { write_message.pretty_inspect } write_message.id ||= identifier.get @@ -38,7 +38,7 @@ def call(write_message, stream_name, expected_version: nil) expected_version = ExpectedVersion.canonize(expected_version) insert_message(id, stream_name, type, data, metadata, expected_version).tap do |position| - logger.info { "Put message data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_message.type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } + logger.info { "Put message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect}, Position: #{position})" } logger.info(tags: [:data, :message_data]) { write_message.pretty_inspect } end end @@ -58,21 +58,21 @@ def destructure_message(write_message) end def insert_message(id, stream_name, type, data, metadata, expected_version) - serialized_data = serialized_data(data) - serialized_metadata = serialized_metadata(metadata) - records = execute_query(id, stream_name, type, serialized_data, serialized_metadata, expected_version) + transformed_data = transformed_data(data) + transformed_metadata = transformed_metadata(metadata) + records = execute_query(id, stream_name, type, transformed_data, transformed_metadata, expected_version) position(records) end - def execute_query(id, stream_name, type, serialized_data, serialized_metadata, expected_version) + def execute_query(id, stream_name, type, transformed_data, transformed_metadata, expected_version) logger.trace { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } params = [ id, stream_name, type, - serialized_data, - serialized_metadata, + transformed_data, + transformed_metadata, expected_version ] @@ -82,7 +82,7 @@ def execute_query(id, stream_name, type, serialized_data, serialized_metadata, e raise_error e end - logger.debug { "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } + logger.debug { "Executed insert (Type: #{type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } records end @@ -91,36 +91,36 @@ def self.statement @statement ||= "SELECT write_message($1::varchar, $2::varchar, $3::varchar, $4::jsonb, $5::jsonb, $6::bigint);" end - def serialized_data(data) - serialized_data = nil + def transformed_data(data) + transformed_data = nil if data.is_a?(Hash) && data.empty? data = nil end unless data.nil? - serializable_data = MessageData::Hash[data] - serialized_data = Transform::Write.(serializable_data, :json) + transformable_data = MessageData::Hash[data] + transformed_data = Transform::Write.(transformable_data, :json) end - logger.debug(tags: [:data, :serialize]) { "Serialized Data: #{serialized_data.inspect}" } - serialized_data + logger.debug(tags: [:data, :serialize]) { "Transformed Data: #{transformed_data.inspect}" } + transformed_data end - def serialized_metadata(metadata) - serialized_metadata = nil + def transformed_metadata(metadata) + transformed_metadata = nil if metadata.is_a?(Hash) && metadata.empty? metadata = nil end unless metadata.nil? - serializable_metadata = MessageData::Hash[metadata] - serialized_metadata = Transform::Write.(serializable_metadata, :json) + transformable_metadata = MessageData::Hash[metadata] + transformed_metadata = Transform::Write.(transformable_metadata, :json) end - logger.debug(tags: [:data, :serialize]) { "Serialized Metadata: #{serialized_metadata.inspect}" } - serialized_metadata + logger.debug(tags: [:data, :serialize]) { "Transformed Metadata: #{transformed_metadata.inspect}" } + transformed_metadata end def position(records) From 2eeab9b6cb8ca68f7cd7507a878a3bfe9b48ca8e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 16 Jul 2018 00:09:34 -0500 Subject: [PATCH 416/591] Package version is increased from 0.10.2.1 to 0.10.2.2 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 8776b97..daf6d43 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.10.2.1' + s.version = '0.10.2.2' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 7453435c3cc34103a347310b31bcd9570ee9ceb0 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 13 Aug 2018 19:55:23 -0500 Subject: [PATCH 417/591] load_path.rb is updated with the latest version --- load_path.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/load_path.rb b/load_path.rb index 2257423..4f7ebf2 100644 --- a/load_path.rb +++ b/load_path.rb @@ -3,13 +3,13 @@ begin require_relative bundler_standalone_loader rescue LoadError - puts "WARNING: Standalone bundle loader is not at #{bundler_standalone_loader}. Using Bundler to load gems." + warn "WARNING: Standalone bundle loader is not at #{bundler_standalone_loader}. Using Bundler to load gems." require "bundler/setup" Bundler.require end -lib_dir = File.expand_path('../lib', __FILE__) -$LOAD_PATH.unshift lib_dir unless $LOAD_PATH.include?(lib_dir) +lib_dir = File.expand_path('lib', __dir__) +$LOAD_PATH.unshift(lib_dir) unless $LOAD_PATH.include?(lib_dir) libraries_dir = ENV['LIBRARIES_HOME'] unless libraries_dir.nil? From be8242606393669636ff012a22effceec404ce9d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 5 Sep 2018 11:37:50 -0500 Subject: [PATCH 418/591] Ruby description is included in benchmark output --- test/benchmark/record_result.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/benchmark/record_result.rb b/test/benchmark/record_result.rb index c51e817..32c5d30 100644 --- a/test/benchmark/record_result.rb +++ b/test/benchmark/record_result.rb @@ -47,6 +47,7 @@ def digest #{name} - - - #{content} + #{RUBY_DESCRIPTION} TEXT .chomp end From 2c7dd1a9d4b12584c8be7b3e419b807a6d052af3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 27 Sep 2018 14:55:44 -0500 Subject: [PATCH 419/591] Session test file names are clarified --- test/automated/session/{session.rb => connect_on_first_use.rb} | 0 .../{initial_state.rb => initial_state_is_not_connected.rb} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename test/automated/session/{session.rb => connect_on_first_use.rb} (100%) rename test/automated/session/{initial_state.rb => initial_state_is_not_connected.rb} (100%) diff --git a/test/automated/session/session.rb b/test/automated/session/connect_on_first_use.rb similarity index 100% rename from test/automated/session/session.rb rename to test/automated/session/connect_on_first_use.rb diff --git a/test/automated/session/initial_state.rb b/test/automated/session/initial_state_is_not_connected.rb similarity index 100% rename from test/automated/session/initial_state.rb rename to test/automated/session/initial_state_is_not_connected.rb From 88706f531bffbee03634ce87d91d99633000ac3a Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 27 Sep 2018 15:02:39 -0500 Subject: [PATCH 420/591] Session configuration also receives settings --- lib/message_store/postgres/session.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index 6e67c4b..93b3c9a 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -1,6 +1,8 @@ module MessageStore module Postgres class Session + Error = Class.new(RuntimeError) + include Log::Dependency def self.settings @@ -20,10 +22,14 @@ def self.build(settings: nil) end end - def self.configure(receiver, session: nil, attr_name: nil) + def self.configure(receiver, session: nil, settings: nil, attr_name: nil) attr_name ||= :session - instance = session || build + if session != nil && settings != nil + raise Error, "Session configured with both settings and session arguments. Use one or the other, but not both." + end + + instance = session || build(settings: settings) receiver.public_send "#{attr_name}=", instance instance end From 3841550f205c41038e91ffad8362a900efa085b3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 27 Sep 2018 15:18:50 -0500 Subject: [PATCH 421/591] Package version is increased from 0.10.2.2 to 0.10.3.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index daf6d43..7f06a58 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.10.2.2' + s.version = '0.10.3.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 4032e79344e24398c77b0a523d5937b37a39ed30 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 2 Oct 2018 19:26:52 -0500 Subject: [PATCH 422/591] Session and settings test coverage is increased --- test/automated/session/build.rb | 25 +++++++++++++++ test/automated/session/configure.rb | 47 +++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 test/automated/session/build.rb create mode 100644 test/automated/session/configure.rb diff --git a/test/automated/session/build.rb b/test/automated/session/build.rb new file mode 100644 index 0000000..b9e837e --- /dev/null +++ b/test/automated/session/build.rb @@ -0,0 +1,25 @@ +require_relative '../automated_init' + +context "Session" do + context "Build" do + context "Settings is specified" do + settings = MessageStore::Postgres::Settings.build + + session = Session.build(settings: settings) + + test "Specified settings is used" do + assert(session.user == settings.get(:user)) + end + end + + context "Settings is not specified" do + settings = MessageStore::Postgres::Settings.build + + session = Session.build + + test "Settings is built" do + assert(session.user == settings.get(:user)) + end + end + end +end diff --git a/test/automated/session/configure.rb b/test/automated/session/configure.rb new file mode 100644 index 0000000..03e9882 --- /dev/null +++ b/test/automated/session/configure.rb @@ -0,0 +1,47 @@ +require_relative '../automated_init' + +context "Session" do + context "Configure" do + context "Session is specified" do + receiver = OpenStruct.new + settings = Object.new + session = Object.new + + Session.configure(receiver, session: session) + + test "Specified session is used" do + assert(receiver.session == session) + end + end + + context "Settings is specified" do + receiver = OpenStruct.new + + user = SecureRandom.hex + + settings_data = { + user: user + } + + settings = MessageStore::Postgres::Settings.build(settings_data) + + Session.configure(receiver, settings: settings) + + test "Session is built from settings" do + assert(receiver.session.user == settings.get('user')) + end + end + + context "Specifying both the session and settings" do + receiver = OpenStruct.new + settings = Object.new + session = Object.new + + test "Is an error" do + assert proc { Session.configure(receiver, settings: settings, session: session) } do + raises_error? Session::Error + end + end + end + end +end From 53c10ed7888ee2907ce446ccd84d1b357be2275a Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Tue, 9 Oct 2018 14:54:43 -0500 Subject: [PATCH 423/591] Settings macro is no longer activated when loading code --- lib/message_store/postgres.rb | 2 +- lib/message_store/postgres/session.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index 0ebdf2f..2e6d4b5 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -4,7 +4,7 @@ require 'log' require 'telemetry' -require 'settings'; Settings.activate +require 'settings' require 'message_store/postgres/log' diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index 93b3c9a..45b45ca 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -3,6 +3,8 @@ module Postgres class Session Error = Class.new(RuntimeError) + Settings.activate(self) + include Log::Dependency def self.settings From f7b0582c83eb0913707bfe1d5103d9958965fdfc Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Tue, 9 Oct 2018 14:55:15 -0500 Subject: [PATCH 424/591] Package version is increased from 0.10.3.0 to 0.11.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 7f06a58..623b761 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.10.3.0' + s.version = '0.11.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 91aae43aaba3f2c818f50a1f65a6a47c7c990c60 Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Tue, 9 Oct 2018 15:22:50 -0500 Subject: [PATCH 425/591] Dependency macro is no longer expected to have been loaded --- lib/message_store/postgres/put.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index a7129d3..b303d0d 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -1,6 +1,8 @@ module MessageStore module Postgres class Put + Dependency.activate(self) + include Log::Dependency dependency :session, Session From 396845e5dd84b9fe5ee6a3f94d742ce483037f26 Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Tue, 9 Oct 2018 15:23:06 -0500 Subject: [PATCH 426/591] Package version is increased from 0.11.0.0 to 0.11.0.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 623b761..facbc2a 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.11.0.0' + s.version = '0.11.0.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 933143e8a1202de0a220970e1d7d6fdaafd5f440 Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Fri, 12 Oct 2018 10:28:09 -0500 Subject: [PATCH 427/591] Settings::Setting module is included instead of deprecated Settings.activate --- lib/message_store/postgres/session.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index 45b45ca..7e26a9a 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -3,7 +3,7 @@ module Postgres class Session Error = Class.new(RuntimeError) - Settings.activate(self) + include Settings::Setting include Log::Dependency From 19f2bb6f421b7cbe43e2fad733260d1443fb6bf4 Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Fri, 12 Oct 2018 10:37:50 -0500 Subject: [PATCH 428/591] Package version is increased from 0.11.0.1 to 0.11.0.2 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index facbc2a..149310b 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.11.0.1' + s.version = '0.11.0.2' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 966da1eea98ad5b2b00144fc32ea0871d4885e61 Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Fri, 12 Oct 2018 11:15:35 -0500 Subject: [PATCH 429/591] Libraries with activations are included --- lib/message_store/postgres/put.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index b303d0d..e7c354a 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -1,8 +1,7 @@ module MessageStore module Postgres class Put - Dependency.activate(self) - + include Dependency include Log::Dependency dependency :session, Session From fd22da594143534a2ff6845787776a78acce73fa Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Fri, 12 Oct 2018 11:15:45 -0500 Subject: [PATCH 430/591] Package version is increased from 0.11.0.2 to 0.11.0.3 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 149310b..e1a9c5c 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.11.0.2' + s.version = '0.11.0.3' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 15a1846ecdac0ebd7840dd9c267e1f8a8ccd4f49 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 23 Oct 2018 19:13:13 -0500 Subject: [PATCH 431/591] Read's default batch size is Get's default batch size --- lib/message_store/postgres/read.rb | 6 ++++++ test/automated/read/default_batch_size.rb | 11 +++++++++++ 2 files changed, 17 insertions(+) create mode 100644 test/automated/read/default_batch_size.rb diff --git a/lib/message_store/postgres/read.rb b/lib/message_store/postgres/read.rb index e6d499b..1198bf2 100644 --- a/lib/message_store/postgres/read.rb +++ b/lib/message_store/postgres/read.rb @@ -7,6 +7,12 @@ def configure(session: nil, condition: nil) Iterator.configure(self, stream_name, position: position) Get.configure(self.iterator, batch_size: batch_size, condition: condition, session: session) end + + module Defaults + def self.batch_size + Postgres::Get::Defaults.batch_size + end + end end end end diff --git a/test/automated/read/default_batch_size.rb b/test/automated/read/default_batch_size.rb new file mode 100644 index 0000000..821299f --- /dev/null +++ b/test/automated/read/default_batch_size.rb @@ -0,0 +1,11 @@ +require_relative '../automated_init' + +context "Read" do + context "Default batch Size" do + default_batch_size = MessageStore::Postgres::Read::Defaults.batch_size + + test "Is the Get implementation's default batch size" do + assert(default_batch_size == MessageStore::Postgres::Get::Defaults.batch_size) + end + end +end From 26c425e0410c1a7488f143f0b5b0309ff33ec394 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 23 Oct 2018 19:15:36 -0500 Subject: [PATCH 432/591] Package version is increased from 0.11.0.3 to 0.11.1.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index e1a9c5c..cf93b8e 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.11.0.3' + s.version = '0.11.1.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 1aa3030197084f0f3984772a2e5b2997f4368f76 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 29 Oct 2018 13:25:55 -0500 Subject: [PATCH 433/591] Benchmarks for text column --- test/benchmark/benchmark_init.rb | 3 +++ ...t Benchmark - 2018-10-29T17-46-05.986Z.txt | 23 +++++++++++++++++++ ...t Benchmark - 2018-10-29T18-06-45.490Z.txt | 23 +++++++++++++++++++ ...t Benchmark - 2018-10-29T16-49-53.149Z.txt | 23 +++++++++++++++++++ ...t Benchmark - 2018-10-29T18-01-54.527Z.txt | 23 +++++++++++++++++++ 5 files changed, 95 insertions(+) create mode 100644 test/benchmark/results/Get Benchmark - 2018-10-29T17-46-05.986Z.txt create mode 100644 test/benchmark/results/Get Benchmark - 2018-10-29T18-06-45.490Z.txt create mode 100644 test/benchmark/results/Put Benchmark - 2018-10-29T16-49-53.149Z.txt create mode 100644 test/benchmark/results/Put Benchmark - 2018-10-29T18-01-54.527Z.txt diff --git a/test/benchmark/benchmark_init.rb b/test/benchmark/benchmark_init.rb index 9226419..27195c8 100644 --- a/test/benchmark/benchmark_init.rb +++ b/test/benchmark/benchmark_init.rb @@ -1,5 +1,8 @@ require_relative '../test_init' +Dependency.activate +Initializer.activate + require 'diagnostics/sample' require 'fileutils' require 'pathname' diff --git a/test/benchmark/results/Get Benchmark - 2018-10-29T17-46-05.986Z.txt b/test/benchmark/results/Get Benchmark - 2018-10-29T17-46-05.986Z.txt new file mode 100644 index 0000000..258fe95 --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-10-29T17-46-05.986Z.txt @@ -0,0 +1,23 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 86420.050000ms +Mean Cycle Time: 0.864200ms (± 1.574568ms) +Cycles Per Second: 1157.138882 +GC: off + +Note: Pre-change of varchar(n) columns to text columns + +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Get Benchmark - 2018-10-29T18-06-45.490Z.txt b/test/benchmark/results/Get Benchmark - 2018-10-29T18-06-45.490Z.txt new file mode 100644 index 0000000..cda3fff --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2018-10-29T18-06-45.490Z.txt @@ -0,0 +1,23 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 86997.039000ms +Mean Cycle Time: 0.869970ms (± 1.669369ms) +Cycles Per Second: 1149.464409 +GC: off + +Note: Post-change of varchar(n) columns to text columns + +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2018-10-29T16-49-53.149Z.txt b/test/benchmark/results/Put Benchmark - 2018-10-29T16-49-53.149Z.txt new file mode 100644 index 0000000..80a388f --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-10-29T16-49-53.149Z.txt @@ -0,0 +1,23 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 136078.335000ms +Mean Cycle Time: 1.360783ms (± 0.736631ms) +Cycles Per Second: 734.870837 +GC: off + +Note: Pre-change of varchar(n) columns to text columns + +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2018-10-29T18-01-54.527Z.txt b/test/benchmark/results/Put Benchmark - 2018-10-29T18-01-54.527Z.txt new file mode 100644 index 0000000..58084bd --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2018-10-29T18-01-54.527Z.txt @@ -0,0 +1,23 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 144978.105000ms +Mean Cycle Time: 1.449781ms (± 1.533522ms) +Cycles Per Second: 689.759326 +GC: off + +Note: Post-change of varchar(n) columns to text columns + +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 From 435229325889f39a41f996d815160b6354757699 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 3 Nov 2018 15:26:44 -0500 Subject: [PATCH 434/591] Library-level log tags reduced to 'message_store' --- lib/message_store/postgres/log.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/message_store/postgres/log.rb b/lib/message_store/postgres/log.rb index 1c55997..5f40928 100644 --- a/lib/message_store/postgres/log.rb +++ b/lib/message_store/postgres/log.rb @@ -2,9 +2,7 @@ module MessageStore module Postgres class Log < ::Log def tag!(tags) - tags << :message_store_postgres - tags << :library - tags << :verbose + tags << :message_store end end end From 6b6775457d15e14918568180439637692af6051b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 3 Nov 2018 15:27:51 -0500 Subject: [PATCH 435/591] Package version is increased from 0.11.1.0 to 0.11.1.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index cf93b8e..aa7ce80 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.11.1.0' + s.version = '0.11.1.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 4d2a454dedf2f24abb5ddde880510337c0a5958f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 3 Nov 2018 16:15:25 -0500 Subject: [PATCH 436/591] Link to documentation site --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 6d24cb8..85a13db 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # message_store-postgres +Message store implementation for PostgreSQL. + +## Documentation + +See the [Eventide documentation site](http://docs.eventide-project.org) for more information, examples, and user guides. + ## License The `message_store-postgres` library is released under the [MIT License](https://github.com/eventide-project/message-store-postgres/blob/master/MIT-License.txt). From c1829bfe6575d87af675fccb0bd4c96a53adacba Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 3 Nov 2018 18:19:08 -0500 Subject: [PATCH 437/591] Log tags are standardized --- lib/message_store/postgres/get.rb | 12 ++++++------ lib/message_store/postgres/put.rb | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 77fee19..961ab70 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -33,7 +33,7 @@ def self.call(stream_name, position: nil, batch_size: nil, condition: nil, sess end def call(stream_name, position: nil) - logger.trace { "Getting message data (Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } + logger.trace(tag: :get) { "Getting message data (Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } position ||= Defaults.position @@ -41,14 +41,14 @@ def call(stream_name, position: nil) message_data = convert(result) - logger.info { "Finished getting message data (Count: #{message_data.length}, Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } + logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } message_data end def get_result(stream_name, position) - logger.trace { "Getting result (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } + logger.trace(tag: :get) { "Getting result (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } sql_command = self.class.sql_command(stream_name, position, batch_size, condition) @@ -63,7 +63,7 @@ def get_result(stream_name, position) result = session.execute(sql_command, params) - logger.debug { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } result end @@ -89,7 +89,7 @@ def self.category_stream?(stream_name) end def convert(result) - logger.trace { "Converting result to message data (Result Count: #{result.ntuples})" } + logger.trace(tag: :get) { "Converting result to message data (Result Count: #{result.ntuples})" } message_data = result.map do |record| record['data'] = Deserialize.data(record['data']) @@ -99,7 +99,7 @@ def convert(result) MessageData::Read.build record end - logger.debug { "Converted result to message data (Message Data Count: #{message_data.length})" } + logger.debug(tag: :get) { "Converted result to message data (Message Data Count: #{message_data.length})" } message_data end diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index e7c354a..d36fca2 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -30,7 +30,7 @@ def self.call(write_message, stream_name, expected_version: nil, session: nil) end def call(write_message, stream_name, expected_version: nil) - logger.trace { "Putting message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" } + logger.trace(tag: :put) { "Putting message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" } logger.trace(tags: [:data, :message_data]) { write_message.pretty_inspect } write_message.id ||= identifier.get @@ -39,7 +39,7 @@ def call(write_message, stream_name, expected_version: nil) expected_version = ExpectedVersion.canonize(expected_version) insert_message(id, stream_name, type, data, metadata, expected_version).tap do |position| - logger.info { "Put message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect}, Position: #{position})" } + logger.info(tag: :put) { "Put message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect}, Position: #{position})" } logger.info(tags: [:data, :message_data]) { write_message.pretty_inspect } end end @@ -66,7 +66,7 @@ def insert_message(id, stream_name, type, data, metadata, expected_version) end def execute_query(id, stream_name, type, transformed_data, transformed_metadata, expected_version) - logger.trace { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } + logger.trace(tag: :put) { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } params = [ id, @@ -83,7 +83,7 @@ def execute_query(id, stream_name, type, transformed_data, transformed_metadata, raise_error e end - logger.debug { "Executed insert (Type: #{type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } + logger.debug(tag: :put) { "Executed insert (Type: #{type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } records end From d1c7f4c9f4153d4052f231762c2fad88984648f4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 3 Nov 2018 18:19:41 -0500 Subject: [PATCH 438/591] Package version is increased from 0.11.1.1 to 0.11.1.2 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index aa7ce80..bf21b45 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.11.1.1' + s.version = '0.11.1.2' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 5a8fdd7a9d4ebfb46ed1e017fe0199c739115377 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 3 Nov 2018 18:31:01 -0500 Subject: [PATCH 439/591] Log tags are standardized --- lib/message_store/postgres/get/last.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/message_store/postgres/get/last.rb b/lib/message_store/postgres/get/last.rb index cb04b75..1a7c798 100644 --- a/lib/message_store/postgres/get/last.rb +++ b/lib/message_store/postgres/get/last.rb @@ -11,7 +11,7 @@ def configure(session: nil) end def call(stream_name) - logger.trace { "Getting last message data (Stream Name: #{stream_name})" } + logger.trace(tag: :get) { "Getting last message data (Stream Name: #{stream_name})" } result = get_result(stream_name) @@ -19,14 +19,14 @@ def call(stream_name) message_data = convert(result[0]) - logger.info { "Finished getting message data (Stream Name: #{stream_name})" } + logger.info(tag: :get) { "Finished getting message data (Stream Name: #{stream_name})" } logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } message_data end def get_result(stream_name) - logger.trace { "Getting last record (Stream: #{stream_name})" } + logger.trace(tag: :get) { "Getting last record (Stream: #{stream_name})" } sql_command = self.class.sql_command(stream_name) @@ -36,7 +36,7 @@ def get_result(stream_name) result = session.execute(sql_command, params) - logger.debug { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" } + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" } return nil if result.ntuples == 0 @@ -50,7 +50,7 @@ def self.sql_command(stream_name) end def convert(record) - logger.trace { "Converting record to message data" } + logger.trace(tag: :get) { "Converting record to message data" } record['data'] = Deserialize.data(record['data']) record['metadata'] = Deserialize.metadata(record['metadata']) @@ -58,7 +58,7 @@ def convert(record) message_data = MessageData::Read.build(record) - logger.debug { "Converted record to message data" } + logger.debug(tag: :get) { "Converted record to message data" } message_data end From cb1c46e6483296a6b49423ef3b34a322a51b3e5e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 3 Nov 2018 18:31:21 -0500 Subject: [PATCH 440/591] Package version is increased from 0.11.1.2 to 0.11.1.3 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index bf21b45..934fd78 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.11.1.2' + s.version = '0.11.1.3' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 1959d46300534ab644983c07678fd269a45c6415 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 3 Nov 2018 18:45:31 -0500 Subject: [PATCH 441/591] Log tags are standardized --- lib/message_store/postgres/session.rb | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index 7e26a9a..beaf08e 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -28,7 +28,9 @@ def self.configure(receiver, session: nil, settings: nil, attr_name: nil) attr_name ||= :session if session != nil && settings != nil - raise Error, "Session configured with both settings and session arguments. Use one or the other, but not both." + error_msg = "Session configured with both settings and session arguments. Use one or the other, but not both." + logger.error(tag: :session) { error_msg } + raise Error, error_msg end instance = session || build(settings: settings) @@ -37,18 +39,18 @@ def self.configure(receiver, session: nil, settings: nil, attr_name: nil) end def open - logger.trace { "Connecting to database" } + logger.trace(tag: :session) { "Connecting to database" } if connected? - logger.debug { "Already connected. A new connection will not be built." } + logger.debug(tag: :session) { "Already connected. A new connection will not be built." } return connection end - logger.debug { "Not connected. A new connection will be built." } + logger.debug(tag: :session) { "Not connected. A new connection will be built." } connection = self.class.build_connection(self) self.connection = connection - logger.debug { "Connected to database" } + logger.debug(tag: :session) { "Connected to database" } connection end @@ -56,12 +58,12 @@ def open def self.build_connection(instance) settings = instance.settings - logger.trace { "Building new connection to database (Settings: #{LogText.settings(settings).inspect})" } + logger.trace(tag: :session) { "Building new connection to database (Settings: #{LogText.settings(settings).inspect})" } connection = PG::Connection.open(settings) connection.type_map_for_results = PG::BasicTypeMapForResults.new(connection) - logger.trace { "Built new connection to database (Settings: #{LogText.settings(settings).inspect})" } + logger.debug(tag: :session) { "Built new connection to database (Settings: #{LogText.settings(settings).inspect})" } connection end @@ -90,7 +92,7 @@ def settings end def execute(sql_command, params=nil) - logger.trace { "Executing SQL command" } + logger.trace(tag: :session) { "Executing SQL command" } logger.trace(tag: :sql) { sql_command } logger.trace(tag: :data) { params.pretty_inspect } @@ -100,11 +102,11 @@ def execute(sql_command, params=nil) if params.nil? connection.exec(sql_command).tap do - logger.debug { "Executed SQL command (no params)" } + logger.debug(tag: :session) { "Executed SQL command (no params)" } end else connection.exec_params(sql_command, params).tap do - logger.debug { "Executed SQL command with params" } + logger.debug(tag: :session) { "Executed SQL command with params" } end end end From 47a94dc5bcc7463a68563652d39a554963f2eecb Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 3 Nov 2018 18:46:07 -0500 Subject: [PATCH 442/591] Package version is increased from 0.11.1.3 to 0.11.1.4 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 934fd78..20b7198 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.11.1.3' + s.version = '0.11.1.4' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From adbd7114d14441c9a751345d04c33f31b472041d Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Sun, 4 Nov 2018 13:22:12 -0600 Subject: [PATCH 443/591] Package version is increased from 0.11.1.4 to 1.0.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 20b7198..b90d799 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '0.11.1.4' + s.version = '1.0.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From c707e7cfe4cd44c85aa705019763fbc49e190a38 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 17 Jan 2019 21:54:25 -0500 Subject: [PATCH 444/591] Connection status check is protected when nil --- lib/message_store/postgres/session.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index beaf08e..387251b 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -69,7 +69,16 @@ def self.build_connection(instance) end def connected? - !connection.nil? && connection.status == PG::CONNECTION_OK + return false if connection.nil? + + status = PG::CONNECTION_OK + begin + status = connection.status + rescue PG::ConnectionBad + status = nil + end + + status == PG::CONNECTION_OK end alias :open? :connected? From 9f897ea12f32f81db39387fd575503ececf96677 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 17 Jan 2019 21:55:14 -0500 Subject: [PATCH 445/591] Package version is increased from 1.0.0.0 to 1.0.0.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index b90d799..c00cde5 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '1.0.0.0' + s.version = '1.0.0.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From d0e7c746ae2dd4eab68f87ecd89dd257030cf329 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Feb 2019 21:33:12 -0600 Subject: [PATCH 446/591] Benchmarks are updated, and results files include the Ruby description --- test/benchmark/record_result.rb | 2 +- ...2-25 revision 61468) [x86_64-darwin17].txt | 21 +++++++++++++++++++ ...1-30 revision 66950) [x86_64-darwin17].txt | 9 ++++++++ ...4.4, GraalVM CE Native [x86_64-darwin].txt | 9 ++++++++ ...2-25 revision 61468) [x86_64-darwin17].txt | 21 +++++++++++++++++++ ...1-30 revision 66950) [x86_64-darwin17].txt | 9 ++++++++ ...4.4, GraalVM CE Native [x86_64-darwin].txt | 9 ++++++++ 7 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 test/benchmark/results/Get Benchmark - 2019-02-06T01-21-21.133Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt create mode 100644 test/benchmark/results/Get Benchmark - 2019-02-06T02-29-48.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt create mode 100644 test/benchmark/results/Get Benchmark - 2019-02-06T03-28-10.452Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt create mode 100644 test/benchmark/results/Put Benchmark - 2019-02-06T01-13-04.354Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt create mode 100644 test/benchmark/results/Put Benchmark - 2019-02-06T02-33-29.071Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt create mode 100644 test/benchmark/results/Put Benchmark - 2019-02-06T03-07-01.152Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt diff --git a/test/benchmark/record_result.rb b/test/benchmark/record_result.rb index 32c5d30..ca0335c 100644 --- a/test/benchmark/record_result.rb +++ b/test/benchmark/record_result.rb @@ -37,7 +37,7 @@ def write_result_file def filename time = clock.iso8601.gsub(':','-') directory = Pathname.new('test/benchmark/tmp') - filename = directory.join("#{name} - #{time}.txt") + filename = directory.join("#{name} - #{time} - #{RUBY_DESCRIPTION}.txt") filename end diff --git a/test/benchmark/results/Get Benchmark - 2019-02-06T01-21-21.133Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt b/test/benchmark/results/Get Benchmark - 2019-02-06T01-21-21.133Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt new file mode 100644 index 0000000..f6db09a --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2019-02-06T01-21-21.133Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt @@ -0,0 +1,21 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 79444.591000ms +Mean Cycle Time: 0.794446ms (± 0.603666ms) +Cycles Per Second: 1258.738937 +GC: off + +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Get Benchmark - 2019-02-06T02-29-48.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt b/test/benchmark/results/Get Benchmark - 2019-02-06T02-29-48.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt new file mode 100644 index 0000000..f2f194f --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2019-02-06T02-29-48.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt @@ -0,0 +1,9 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 62712.511000ms +Mean Cycle Time: 0.627125ms (± 0.081911ms) +Cycles Per Second: 1594.578154 +GC: off + +ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] \ No newline at end of file diff --git a/test/benchmark/results/Get Benchmark - 2019-02-06T03-28-10.452Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt b/test/benchmark/results/Get Benchmark - 2019-02-06T03-28-10.452Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt new file mode 100644 index 0000000..112b24e --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2019-02-06T03-28-10.452Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt @@ -0,0 +1,9 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 196650.188980ms +Mean Cycle Time: 1.966502ms (± 23.192495ms) +Cycles Per Second: 508.517182 +GC: off + +truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin] \ No newline at end of file diff --git a/test/benchmark/results/Put Benchmark - 2019-02-06T01-13-04.354Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt b/test/benchmark/results/Put Benchmark - 2019-02-06T01-13-04.354Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt new file mode 100644 index 0000000..761ea02 --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2019-02-06T01-13-04.354Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt @@ -0,0 +1,21 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 145874.364000ms +Mean Cycle Time: 1.458744ms (± 4.047611ms) +Cycles Per Second: 685.521412 +GC: off + +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2019-02-06T02-33-29.071Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt b/test/benchmark/results/Put Benchmark - 2019-02-06T02-33-29.071Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt new file mode 100644 index 0000000..ac1dde1 --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2019-02-06T02-33-29.071Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt @@ -0,0 +1,9 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 137535.101000ms +Mean Cycle Time: 1.375351ms (± 0.955491ms) +Cycles Per Second: 727.087116 +GC: off + +ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] \ No newline at end of file diff --git a/test/benchmark/results/Put Benchmark - 2019-02-06T03-07-01.152Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt b/test/benchmark/results/Put Benchmark - 2019-02-06T03-07-01.152Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt new file mode 100644 index 0000000..eac15be --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2019-02-06T03-07-01.152Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt @@ -0,0 +1,9 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 187106.754654ms +Mean Cycle Time: 1.871068ms (± 15.991498ms) +Cycles Per Second: 534.454249 +GC: off + +truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin] \ No newline at end of file From 556fc2324bf3fa6e09b5df0aac468da36a9b0326 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Feb 2019 21:34:31 -0600 Subject: [PATCH 447/591] Machine information is included in the new benchmark results --- ...019-01-30 revision 66950) [x86_64-darwin17].txt | 14 +++++++++++++- ...by 2.4.4, GraalVM CE Native [x86_64-darwin].txt | 14 +++++++++++++- ...019-01-30 revision 66950) [x86_64-darwin17].txt | 14 +++++++++++++- ...by 2.4.4, GraalVM CE Native [x86_64-darwin].txt | 14 +++++++++++++- 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/test/benchmark/results/Get Benchmark - 2019-02-06T02-29-48.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt b/test/benchmark/results/Get Benchmark - 2019-02-06T02-29-48.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt index f2f194f..4e459a2 100644 --- a/test/benchmark/results/Get Benchmark - 2019-02-06T02-29-48.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt +++ b/test/benchmark/results/Get Benchmark - 2019-02-06T02-29-48.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt @@ -6,4 +6,16 @@ Mean Cycle Time: 0.627125ms (± 0.081911ms) Cycles Per Second: 1594.578154 GC: off -ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] \ No newline at end of file +ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Get Benchmark - 2019-02-06T03-28-10.452Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt b/test/benchmark/results/Get Benchmark - 2019-02-06T03-28-10.452Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt index 112b24e..632b197 100644 --- a/test/benchmark/results/Get Benchmark - 2019-02-06T03-28-10.452Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt +++ b/test/benchmark/results/Get Benchmark - 2019-02-06T03-28-10.452Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt @@ -6,4 +6,16 @@ Mean Cycle Time: 1.966502ms (± 23.192495ms) Cycles Per Second: 508.517182 GC: off -truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin] \ No newline at end of file +truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2019-02-06T02-33-29.071Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt b/test/benchmark/results/Put Benchmark - 2019-02-06T02-33-29.071Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt index ac1dde1..c727fa3 100644 --- a/test/benchmark/results/Put Benchmark - 2019-02-06T02-33-29.071Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt +++ b/test/benchmark/results/Put Benchmark - 2019-02-06T02-33-29.071Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt @@ -6,4 +6,16 @@ Mean Cycle Time: 1.375351ms (± 0.955491ms) Cycles Per Second: 727.087116 GC: off -ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] \ No newline at end of file +ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2019-02-06T03-07-01.152Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt b/test/benchmark/results/Put Benchmark - 2019-02-06T03-07-01.152Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt index eac15be..3244702 100644 --- a/test/benchmark/results/Put Benchmark - 2019-02-06T03-07-01.152Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt +++ b/test/benchmark/results/Put Benchmark - 2019-02-06T03-07-01.152Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt @@ -6,4 +6,16 @@ Mean Cycle Time: 1.871068ms (± 15.991498ms) Cycles Per Second: 534.454249 GC: off -truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin] \ No newline at end of file +truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 From 7127e27d893db16b77b6a00d722f78bd6eb313a5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Feb 2019 21:36:36 -0600 Subject: [PATCH 448/591] Benchmark results file names include Ruby description --- ...uby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} | 0 ...uby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} | 0 ...uby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} | 0 ...uby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} | 0 ...uby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} | 0 ...uby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} | 0 ...uby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} | 0 ...uby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} | 0 ...uby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} | 0 ...uby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename test/benchmark/results/{Get Benchmark - 2018-06-05T02-03-42.080Z.txt => Get Benchmark - 2018-06-05T02-03-42.080Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} (100%) rename test/benchmark/results/{Get Benchmark - 2018-06-05T02-27-02.253Z.txt => Get Benchmark - 2018-06-05T02-27-02.253Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} (100%) rename test/benchmark/results/{Get Benchmark - 2018-06-08T06-42-47.108Z.txt => Get Benchmark - 2018-06-08T06-42-47.108Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} (100%) rename test/benchmark/results/{Get Benchmark - 2018-10-29T17-46-05.986Z.txt => Get Benchmark - 2018-10-29T17-46-05.986Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} (100%) rename test/benchmark/results/{Get Benchmark - 2018-10-29T18-06-45.490Z.txt => Get Benchmark - 2018-10-29T18-06-45.490Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} (100%) rename test/benchmark/results/{Put Benchmark - 2018-06-05T01-59-19.666Z.txt => Put Benchmark - 2018-06-05T01-59-19.666Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} (100%) rename test/benchmark/results/{Put Benchmark - 2018-06-05T02-14-58.644Z.txt => Put Benchmark - 2018-06-05T02-14-58.644Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} (100%) rename test/benchmark/results/{Put Benchmark - 2018-06-25T15-41-28.825Z.txt => Put Benchmark - 2018-06-25T15-41-28.825Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} (100%) rename test/benchmark/results/{Put Benchmark - 2018-10-29T16-49-53.149Z.txt => Put Benchmark - 2018-10-29T16-49-53.149Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} (100%) rename test/benchmark/results/{Put Benchmark - 2018-10-29T18-01-54.527Z.txt => Put Benchmark - 2018-10-29T18-01-54.527Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt} (100%) diff --git a/test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt similarity index 100% rename from test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z.txt rename to test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt similarity index 100% rename from test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z.txt rename to test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z.txt b/test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt similarity index 100% rename from test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z.txt rename to test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Get Benchmark - 2018-10-29T17-46-05.986Z.txt b/test/benchmark/results/Get Benchmark - 2018-10-29T17-46-05.986Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt similarity index 100% rename from test/benchmark/results/Get Benchmark - 2018-10-29T17-46-05.986Z.txt rename to test/benchmark/results/Get Benchmark - 2018-10-29T17-46-05.986Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Get Benchmark - 2018-10-29T18-06-45.490Z.txt b/test/benchmark/results/Get Benchmark - 2018-10-29T18-06-45.490Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt similarity index 100% rename from test/benchmark/results/Get Benchmark - 2018-10-29T18-06-45.490Z.txt rename to test/benchmark/results/Get Benchmark - 2018-10-29T18-06-45.490Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt similarity index 100% rename from test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z.txt rename to test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt similarity index 100% rename from test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z.txt rename to test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z.txt b/test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt similarity index 100% rename from test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z.txt rename to test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Put Benchmark - 2018-10-29T16-49-53.149Z.txt b/test/benchmark/results/Put Benchmark - 2018-10-29T16-49-53.149Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt similarity index 100% rename from test/benchmark/results/Put Benchmark - 2018-10-29T16-49-53.149Z.txt rename to test/benchmark/results/Put Benchmark - 2018-10-29T16-49-53.149Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Put Benchmark - 2018-10-29T18-01-54.527Z.txt b/test/benchmark/results/Put Benchmark - 2018-10-29T18-01-54.527Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt similarity index 100% rename from test/benchmark/results/Put Benchmark - 2018-10-29T18-01-54.527Z.txt rename to test/benchmark/results/Put Benchmark - 2018-10-29T18-01-54.527Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt From 8603d8b0100d16c3c27adee955b55c583e2478d1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Feb 2019 21:38:47 -0600 Subject: [PATCH 449/591] Ruby description is noted in all benchmark results files --- ...by 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt | 2 ++ ...by 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt | 2 ++ ...by 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt | 2 ++ ...by 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt | 2 ++ ...by 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt | 2 ++ ...by 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt | 2 ++ 6 files changed, 12 insertions(+) diff --git a/test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt index 5d8edf9..bf7866e 100644 --- a/test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt +++ b/test/benchmark/results/Get Benchmark - 2018-06-05T02-03-42.080Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt @@ -8,6 +8,8 @@ GC: off Note: Pre-advisory lock implementation +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + Model Name: MacBook Model Identifier: MacBook10,1 Processor Name: Intel Core i7 diff --git a/test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt b/test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt index 70d5376..4c05dfa 100644 --- a/test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt +++ b/test/benchmark/results/Get Benchmark - 2018-06-05T02-27-02.253Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt @@ -8,6 +8,8 @@ GC: off Note: Post-advisory lock implementation +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + Model Name: MacBook Model Identifier: MacBook10,1 Processor Name: Intel Core i7 diff --git a/test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt b/test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt index 22a5fec..efb8d67 100644 --- a/test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt +++ b/test/benchmark/results/Get Benchmark - 2018-06-08T06-42-47.108Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt @@ -8,6 +8,8 @@ GC: off Note: Post-retrieval server function implementation +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + Model Name: MacBook Model Identifier: MacBook10,1 Processor Name: Intel Core i7 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt b/test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt index 4f89f45..f28411b 100644 --- a/test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt +++ b/test/benchmark/results/Put Benchmark - 2018-06-05T01-59-19.666Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt @@ -8,6 +8,8 @@ GC: off Note: Pre-advisory lock implementation +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + Model Name: MacBook Model Identifier: MacBook10,1 Processor Name: Intel Core i7 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt b/test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt index d8297eb..4e5440f 100644 --- a/test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt +++ b/test/benchmark/results/Put Benchmark - 2018-06-05T02-14-58.644Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt @@ -8,6 +8,8 @@ GC: off Note: Post-advisory lock implementation +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + Model Name: MacBook Model Identifier: MacBook10,1 Processor Name: Intel Core i7 diff --git a/test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt b/test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt index 08c651c..866cc0d 100644 --- a/test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt +++ b/test/benchmark/results/Put Benchmark - 2018-06-25T15-41-28.825Z - ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17].txt @@ -8,6 +8,8 @@ GC: off Note: ID index has a unique constraint +ruby 2.5.0p0 (2017-12-25 revision 61468) [x86_64-darwin17] + Model Name: MacBook Model Identifier: MacBook10,1 Processor Name: Intel Core i7 From 968f32daa32642aa8c7fab4921f22a3b7f2c0b1f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 5 Feb 2019 22:02:35 -0600 Subject: [PATCH 450/591] TruffleRuby benchmarks with GC on --- ...4.4, GraalVM CE Native [x86_64-darwin].txt | 21 +++++++++++++++++++ ...4.4, GraalVM CE Native [x86_64-darwin].txt | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/benchmark/results/Get Benchmark - 2019-02-06T04-01-19.164Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt create mode 100644 test/benchmark/results/Put Benchmark - 2019-02-06T03-52-34.879Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt diff --git a/test/benchmark/results/Get Benchmark - 2019-02-06T04-01-19.164Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt b/test/benchmark/results/Get Benchmark - 2019-02-06T04-01-19.164Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt new file mode 100644 index 0000000..4dbb9b8 --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2019-02-06T04-01-19.164Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt @@ -0,0 +1,21 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 207754.828951ms +Mean Cycle Time: 2.077548ms (± 24.335594ms) +Cycles Per Second: 481.336586 +GC: on + +truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2019-02-06T03-52-34.879Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt b/test/benchmark/results/Put Benchmark - 2019-02-06T03-52-34.879Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt new file mode 100644 index 0000000..a07db7a --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2019-02-06T03-52-34.879Z - truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin].txt @@ -0,0 +1,21 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 199665.528250ms +Mean Cycle Time: 1.996655ms (± 17.571019ms) +Cycles Per Second: 500.837580 +GC: on + +truffleruby 1.0.0-rc12, like ruby 2.4.4, GraalVM CE Native [x86_64-darwin] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 From cfa530804422d96ce59f8a1e1ed4949c09fbd8c9 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 20 Feb 2019 22:33:09 -0600 Subject: [PATCH 451/591] Benchmark cycles are increased from 25,000 to 100,000 --- test/benchmark/defaults.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark/defaults.rb b/test/benchmark/defaults.rb index dc178e3..2950db4 100644 --- a/test/benchmark/defaults.rb +++ b/test/benchmark/defaults.rb @@ -20,7 +20,7 @@ def self.build end def self.cycles - Integer(ENV['CYCLES'] || 2500) + Integer(ENV['CYCLES'] || 100_000) end def self.warmup_cycles From bed8f567e62f95ef886ceefbf5d921ea496c5f73 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 20 Feb 2019 22:46:30 -0600 Subject: [PATCH 452/591] Put benchmark results post category advisory lock --- ...1-30 revision 66950) [x86_64-darwin17].txt | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 test/benchmark/results/Put Benchmark - 2019-02-21T04-36-28.780Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Put Benchmark - 2019-02-21T04-36-28.780Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt b/test/benchmark/results/Put Benchmark - 2019-02-21T04-36-28.780Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt new file mode 100644 index 0000000..b7c52c3 --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2019-02-21T04-36-28.780Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt @@ -0,0 +1,23 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 141589.302000ms +Mean Cycle Time: 1.415893ms (± 1.068947ms) +Cycles Per Second: 706.268048 +GC: off + +Note: Post category advisory lock + +ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 From 3a92d26d5198abf96f7875215ea7f83dba51ff4d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 20 Feb 2019 22:59:26 -0600 Subject: [PATCH 453/591] Benchmark allows from destination stream name to be specified by environment variable --- test/benchmark/defaults.rb | 9 +++++++-- test/benchmark/get.rb | 6 ++++-- test/benchmark/put.rb | 3 ++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/test/benchmark/defaults.rb b/test/benchmark/defaults.rb index 2950db4..2aea3b3 100644 --- a/test/benchmark/defaults.rb +++ b/test/benchmark/defaults.rb @@ -1,7 +1,7 @@ module Test module Benchmark class Defaults - initializer :cycles, :warmup_cycles, :gc + initializer :cycles, :warmup_cycles, :gc, :stream_name def total_cycles cycles + warmup_cycles @@ -12,11 +12,12 @@ def to_s Cycles: #{cycles} Warmup Cycles: #{warmup_cycles} GC: #{gc} + Stream Name: #{stream_name.inspect} TEXT end def self.build - new(cycles, warmup_cycles, gc) + new(cycles, warmup_cycles, gc, stream_name) end def self.cycles @@ -30,6 +31,10 @@ def self.warmup_cycles def self.gc ['on', 'true'].include?(ENV['GC']) ? true : false end + + def self.stream_name + ENV['STREAM_NAME'] + end end end end diff --git a/test/benchmark/get.rb b/test/benchmark/get.rb index b650099..daa578c 100644 --- a/test/benchmark/get.rb +++ b/test/benchmark/get.rb @@ -7,14 +7,16 @@ put = Put.build list.each do |entry| - put.(entry.message_data, entry.stream_name) + stream_name = defaults.stream_name || entry.stream_name + put.(entry.message_data, stream_name) end get = Get.build result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] - get.(entry.stream_name) + stream_name = defaults.stream_name || entry.stream_name + get.(stream_name) end puts diff --git a/test/benchmark/put.rb b/test/benchmark/put.rb index 0332464..e60124d 100644 --- a/test/benchmark/put.rb +++ b/test/benchmark/put.rb @@ -8,7 +8,8 @@ result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] - put.(entry.message_data, entry.stream_name) + stream_name = defaults.stream_name || entry.stream_name + put.(entry.message_data, stream_name) end puts From 5e82972dc9bfc3e1645fb58ae113ab2e08f21bf1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 20 Feb 2019 23:30:25 -0600 Subject: [PATCH 454/591] Benchmark results for concurrent writers to the same category --- ...1-30 revision 66950) [x86_64-darwin17].txt | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 test/benchmark/results/Put Benchmark - 2019-02-21T05-11-28.525Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Put Benchmark - 2019-02-21T05-11-28.525Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt b/test/benchmark/results/Put Benchmark - 2019-02-21T05-11-28.525Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt new file mode 100644 index 0000000..191b501 --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2019-02-21T05-11-28.525Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt @@ -0,0 +1,23 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 306687.095000ms +Mean Cycle Time: 3.066871ms (± 3.746017ms) +Cycles Per Second: 326.065236 +GC: off + +Note: Tested with 5 concurrent writers on the same category + +ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 From f0aca7bda933cf73cfae6e31f51f840b1a5a3cb7 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 12 Jul 2019 14:57:02 -0500 Subject: [PATCH 455/591] Tests are corrected for standardization of settings keys to string data --- test/automated/session/settings.rb | 2 +- test/automated/settings.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/automated/session/settings.rb b/test/automated/session/settings.rb index 505f745..deb2fde 100644 --- a/test/automated/session/settings.rb +++ b/test/automated/session/settings.rb @@ -12,7 +12,7 @@ names.each do |name| test "#{name}" do session_val = session.public_send(name) - settings_val = settings_hash[name] + settings_val = settings_hash[name.to_s] assert(session_val == settings_val) end diff --git a/test/automated/settings.rb b/test/automated/settings.rb index 5e90030..fa0a521 100644 --- a/test/automated/settings.rb +++ b/test/automated/settings.rb @@ -10,7 +10,7 @@ names.each do |name| test "#{name}" do - assert(settings_hash.has_key? name) + assert(settings_hash.has_key? name.to_s) end end end From b1ca0e1f2efc01876d22597153a45ef03d244b49 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 13 Sep 2019 08:47:28 -0500 Subject: [PATCH 456/591] New TruffleRuby measurements --- ...6.2, GraalVM CE Native [x86_64-darwin].txt | 21 +++++++++++++++++++ ...6.2, GraalVM CE Native [x86_64-darwin].txt | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/benchmark/results/Get Benchmark - 2019-09-12T20-34-51.223Z - truffleruby 19.2.0, like ruby 2.6.2, GraalVM CE Native [x86_64-darwin].txt create mode 100644 test/benchmark/results/Put Benchmark - 2019-09-12T16-54-47.909Z - truffleruby 19.2.0, like ruby 2.6.2, GraalVM CE Native [x86_64-darwin].txt diff --git a/test/benchmark/results/Get Benchmark - 2019-09-12T20-34-51.223Z - truffleruby 19.2.0, like ruby 2.6.2, GraalVM CE Native [x86_64-darwin].txt b/test/benchmark/results/Get Benchmark - 2019-09-12T20-34-51.223Z - truffleruby 19.2.0, like ruby 2.6.2, GraalVM CE Native [x86_64-darwin].txt new file mode 100644 index 0000000..f7e98e6 --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2019-09-12T20-34-51.223Z - truffleruby 19.2.0, like ruby 2.6.2, GraalVM CE Native [x86_64-darwin].txt @@ -0,0 +1,21 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 195429.244053ms +Mean Cycle Time: 1.954292ms (± 25.676337ms) +Cycles Per Second: 511.694145 +GC: off + +truffleruby 19.2.0, like ruby 2.6.2, GraalVM CE Native [x86_64-darwin] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2019-09-12T16-54-47.909Z - truffleruby 19.2.0, like ruby 2.6.2, GraalVM CE Native [x86_64-darwin].txt b/test/benchmark/results/Put Benchmark - 2019-09-12T16-54-47.909Z - truffleruby 19.2.0, like ruby 2.6.2, GraalVM CE Native [x86_64-darwin].txt new file mode 100644 index 0000000..5ff7894 --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2019-09-12T16-54-47.909Z - truffleruby 19.2.0, like ruby 2.6.2, GraalVM CE Native [x86_64-darwin].txt @@ -0,0 +1,21 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 288567.912765ms +Mean Cycle Time: 2.885679ms (± 18.626377ms) +Cycles Per Second: 346.538876 +GC: off + +truffleruby 19.2.0, like ruby 2.6.2, GraalVM CE Native [x86_64-darwin] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 From e9612e7174536c8d1e7ea373886b802dc86f63f4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 23 Sep 2019 20:44:48 -0500 Subject: [PATCH 457/591] Iterator tests for both stream and category --- test/automated/iterator/category/next.rb | 27 +++++++++++++++++++ .../category/no_further_message_data.rb | 24 +++++++++++++++++ test/automated/iterator/next.rb | 21 --------------- .../iterator/no_further_message_data.rb | 18 ------------- test/automated/iterator/stream/next.rb | 23 ++++++++++++++++ .../stream/no_further_message_data.rb | 20 ++++++++++++++ 6 files changed, 94 insertions(+), 39 deletions(-) create mode 100644 test/automated/iterator/category/next.rb create mode 100644 test/automated/iterator/category/no_further_message_data.rb delete mode 100644 test/automated/iterator/next.rb delete mode 100644 test/automated/iterator/no_further_message_data.rb create mode 100644 test/automated/iterator/stream/next.rb create mode 100644 test/automated/iterator/stream/no_further_message_data.rb diff --git a/test/automated/iterator/category/next.rb b/test/automated/iterator/category/next.rb new file mode 100644 index 0000000..a9c2cc5 --- /dev/null +++ b/test/automated/iterator/category/next.rb @@ -0,0 +1,27 @@ +require_relative '../../automated_init' + +context "Iterator" do + context "Category" do + context "Next" do + stream_name, _ = Controls::Put.() + + category = StreamName.get_category(stream_name) + + Controls::Put.(category: category) + + iterator = Read::Iterator.build(category) + Get.configure(iterator, batch_size: 1) + + batch = [] + + 2.times do + message_data = iterator.next + batch << message_data unless message_data.nil? + end + + test "Gets each message" do + assert(batch.length == 2) + end + end + end +end diff --git a/test/automated/iterator/category/no_further_message_data.rb b/test/automated/iterator/category/no_further_message_data.rb new file mode 100644 index 0000000..0b47418 --- /dev/null +++ b/test/automated/iterator/category/no_further_message_data.rb @@ -0,0 +1,24 @@ +require_relative '../../automated_init' + +context "Iterator" do + context "Category" do + context "No Further Message Data" do + stream_name, _ = Controls::Put.() + + category = StreamName.get_category(stream_name) + + Controls::Put.(category: category) + + iterator = Read::Iterator.build(category) + Get.configure(iterator, batch_size: 1) + + 2.times { iterator.next } + + last = iterator.next + + test "Results in nil" do + assert(last.nil?) + end + end + end +end diff --git a/test/automated/iterator/next.rb b/test/automated/iterator/next.rb deleted file mode 100644 index b133bec..0000000 --- a/test/automated/iterator/next.rb +++ /dev/null @@ -1,21 +0,0 @@ -require_relative '../automated_init' - -context "Iterator" do - context "Next" do - stream_name, _ = Controls::Put.(instances: 2) - - iterator = Read::Iterator.build(stream_name) - Get.configure(iterator, batch_size: 1) - - batch = [] - - 2.times do - message_data = iterator.next - batch << message_data unless message_data.nil? - end - - test "Gets each message" do - assert(batch.length == 2) - end - end -end diff --git a/test/automated/iterator/no_further_message_data.rb b/test/automated/iterator/no_further_message_data.rb deleted file mode 100644 index 9cdf5c4..0000000 --- a/test/automated/iterator/no_further_message_data.rb +++ /dev/null @@ -1,18 +0,0 @@ -require_relative '../automated_init' - -context "Iterator" do - context "No further message data" do - stream_name, _ = Controls::Put.(instances: 2) - - iterator = Read::Iterator.build(stream_name) - Get.configure(iterator, batch_size: 1) - - 2.times { iterator.next } - - last = iterator.next - - test "Results in nil" do - assert(last.nil?) - end - end -end diff --git a/test/automated/iterator/stream/next.rb b/test/automated/iterator/stream/next.rb new file mode 100644 index 0000000..ef818f5 --- /dev/null +++ b/test/automated/iterator/stream/next.rb @@ -0,0 +1,23 @@ +require_relative '../../automated_init' + +context "Iterator" do + context "Stream" do + context "Next" do + stream_name, _ = Controls::Put.(instances: 2) + + iterator = Read::Iterator.build(stream_name) + Get.configure(iterator, batch_size: 1) + + batch = [] + + 2.times do + message_data = iterator.next + batch << message_data unless message_data.nil? + end + + test "Gets each message" do + assert(batch.length == 2) + end + end + end +end diff --git a/test/automated/iterator/stream/no_further_message_data.rb b/test/automated/iterator/stream/no_further_message_data.rb new file mode 100644 index 0000000..4f80077 --- /dev/null +++ b/test/automated/iterator/stream/no_further_message_data.rb @@ -0,0 +1,20 @@ +require_relative '../../automated_init' + +context "Iterator" do + context "Stream" do + context "No Further Message Data" do + stream_name, _ = Controls::Put.(instances: 2) + + iterator = Read::Iterator.build(stream_name) + Get.configure(iterator, batch_size: 1) + + 2.times { iterator.next } + + last = iterator.next + + test "Results in nil" do + assert(last.nil?) + end + end + end +end From 87c4adaf5f8edae1886c7e1a3d36c0dc617e906b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 16:25:33 -0500 Subject: [PATCH 458/591] Notes on implementation --- lib/message_store/postgres/get.rb | 1 + lib/message_store/postgres/get/category.rb | 0 lib/message_store/postgres/get/stream.rb | 0 lib/message_store/postgres/read.rb | 3 +++ lib/message_store/postgres/read/iterator.rb | 7 +++++++ test/automated/get/stream/implementation.rb | 0 6 files changed, 11 insertions(+) create mode 100644 lib/message_store/postgres/get/category.rb create mode 100644 lib/message_store/postgres/get/stream.rb create mode 100644 test/automated/get/stream/implementation.rb diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 961ab70..16dc577 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -11,6 +11,7 @@ def batch_size @batch_size ||= Defaults.batch_size end +## Must receive stream name def self.build(batch_size: nil, session: nil, condition: nil) new(batch_size, condition).tap do |instance| instance.configure(session: session) diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb new file mode 100644 index 0000000..e69de29 diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb new file mode 100644 index 0000000..e69de29 diff --git a/lib/message_store/postgres/read.rb b/lib/message_store/postgres/read.rb index 1198bf2..535116e 100644 --- a/lib/message_store/postgres/read.rb +++ b/lib/message_store/postgres/read.rb @@ -5,6 +5,9 @@ class Read def configure(session: nil, condition: nil) Iterator.configure(self, stream_name, position: position) + +## Configure method should select the proper implementation based on +## The stream name Get.configure(self.iterator, batch_size: batch_size, condition: condition, session: session) end diff --git a/lib/message_store/postgres/read/iterator.rb b/lib/message_store/postgres/read/iterator.rb index 5b197c5..f972a17 100644 --- a/lib/message_store/postgres/read/iterator.rb +++ b/lib/message_store/postgres/read/iterator.rb @@ -1,3 +1,10 @@ +## Specialize Get +# - This could be injected by reader into iterator +# - Reader knows whether category or not +# - Reader can tell iterator how to get last message +# - Use a block to determine how to get last message position + + module MessageStore module Postgres class Read diff --git a/test/automated/get/stream/implementation.rb b/test/automated/get/stream/implementation.rb new file mode 100644 index 0000000..e69de29 From 33fc7cfaed6e17197adf889170ed7cdc70d9c602 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 17:24:01 -0500 Subject: [PATCH 459/591] Stream specialization is constructed if stream name is not a category --- lib/message_store/postgres.rb | 2 ++ lib/message_store/postgres/get.rb | 25 +++++++++++++++++---- lib/message_store/postgres/get/stream.rb | 8 +++++++ test/automated/get/stream/implementation.rb | 0 test/automated/get/stream/specialization.rb | 15 +++++++++++++ 5 files changed, 46 insertions(+), 4 deletions(-) delete mode 100644 test/automated/get/stream/implementation.rb create mode 100644 test/automated/get/stream/specialization.rb diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index 2e6d4b5..29b6341 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -17,6 +17,8 @@ require 'message_store/postgres/write' require 'message_store/postgres/get' +require 'message_store/postgres/get/stream' +require 'message_store/postgres/get/category' require 'message_store/postgres/get/last' require 'message_store/postgres/read/iterator' require 'message_store/postgres/read' diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 16dc577..e9a9261 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -1,23 +1,40 @@ module MessageStore module Postgres + ## Make into module class Get include MessageStore::Get dependency :session, Session - initializer na(:batch_size), :condition +## initializer na(:batch_size), :condition + initializer :stream_name, na(:batch_size), :condition def batch_size @batch_size ||= Defaults.batch_size end -## Must receive stream name - def self.build(batch_size: nil, session: nil, condition: nil) - new(batch_size, condition).tap do |instance| +## def self.__build(batch_size: nil, session: nil, condition: nil) +## new(batch_size, condition).tap do |instance| +## instance.configure(session: session) +## end +## end + + def self.build(stream_name, batch_size: nil, session: nil, condition: nil) + specialization = specialization(stream_name) + + specialization.new(stream_name, batch_size, condition).tap do |instance| instance.configure(session: session) end end + def self.specialization(stream_name) + if not category_stream?(stream_name) + Stream + else + ## Category + end + end + def self.configure(receiver, attr_name: nil, batch_size: nil, condition: nil, session: nil) attr_name ||= :get instance = build(batch_size: batch_size, condition: condition, session: session) diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index e69de29..775f96f 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -0,0 +1,8 @@ +module MessageStore + module Postgres + class Get + class Stream < Get + end + end + end +end diff --git a/test/automated/get/stream/implementation.rb b/test/automated/get/stream/implementation.rb deleted file mode 100644 index e69de29..0000000 diff --git a/test/automated/get/stream/specialization.rb b/test/automated/get/stream/specialization.rb new file mode 100644 index 0000000..ea0fd69 --- /dev/null +++ b/test/automated/get/stream/specialization.rb @@ -0,0 +1,15 @@ +require_relative '../../automated_init' + +context "Get" do + context "Stream" do + context "Specialization" do + stream_name = Controls::StreamName.example + + get = Get.build(stream_name) + + test "Get::Stream" do + assert(get.instance_of?(Get::Stream)) + end + end + end +end From 9e9d71c4dd71e30ffacfd30d9ff88103eaffaf37 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 17:31:02 -0500 Subject: [PATCH 460/591] Category specialization is constructed if stream name is a category --- lib/message_store/postgres/get.rb | 6 +++--- lib/message_store/postgres/get/category.rb | 8 ++++++++ test/automated/get/category/specialization.rb | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 test/automated/get/category/specialization.rb diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index e9a9261..a221623 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -28,10 +28,10 @@ def self.build(stream_name, batch_size: nil, session: nil, condition: nil) end def self.specialization(stream_name) - if not category_stream?(stream_name) - Stream + if category_stream?(stream_name) + Category else - ## Category + Stream end end diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index e69de29..1fef6c7 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -0,0 +1,8 @@ +module MessageStore + module Postgres + class Get + class Category < Get + end + end + end +end diff --git a/test/automated/get/category/specialization.rb b/test/automated/get/category/specialization.rb new file mode 100644 index 0000000..a372f0f --- /dev/null +++ b/test/automated/get/category/specialization.rb @@ -0,0 +1,15 @@ +require_relative '../../automated_init' + +context "Get" do + context "Category" do + context "Specialization" do + stream_name = Controls::Category.example + + get = Get.build(stream_name) + + test "Get::Stream" do + assert(get.instance_of?(Get::Category)) + end + end + end +end From 6c5da5af91923586a9debd986e014602d2fdf3f9 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 20:46:14 -0500 Subject: [PATCH 461/591] Specialized retrieval is tested --- test/automated/get/category/get_messages.rb | 22 +++++++++++++++++++++ test/automated/get/stream/get_message.rb | 22 +++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 test/automated/get/category/get_messages.rb create mode 100644 test/automated/get/stream/get_message.rb diff --git a/test/automated/get/category/get_messages.rb b/test/automated/get/category/get_messages.rb new file mode 100644 index 0000000..f5dbe60 --- /dev/null +++ b/test/automated/get/category/get_messages.rb @@ -0,0 +1,22 @@ +require_relative '../../automated_init' + +context "Get" do + context "Category" do + context "Get Messages" do + write_message = Controls::MessageData::Write.example + + category = Controls::Category.example + + stream_name_1, _ = Controls::Put.(category: category) + stream_name_2, _ = Controls::Put.(category: category) + + messages = Get.(category) + + context "Messages Retrieved" do + test "Messages from the all streams in the category" do + assert(messages.length == 2) + end + end + end + end +end diff --git a/test/automated/get/stream/get_message.rb b/test/automated/get/stream/get_message.rb new file mode 100644 index 0000000..2caf634 --- /dev/null +++ b/test/automated/get/stream/get_message.rb @@ -0,0 +1,22 @@ +require_relative '../../automated_init' + +context "Get" do + context "Stream" do + context "Get Message" do + write_message = Controls::MessageData::Write.example + + category = Controls::Category.example + + stream_name_1, _ = Controls::Put.(category: category) + stream_name_2, _ = Controls::Put.(category: category) + + messages = Get.(stream_name_1) + + context "Messages Retrieved" do + test "Only messages from the specific stream" do + assert(messages.length == 1) + end + end + end + end +end From 7f61307a62eaee6f99c52afceec9d79d8a4377b3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 20:51:47 -0500 Subject: [PATCH 462/591] Obsolete get tests are removed --- test/automated/get/category.rb | 17 ----------------- test/automated/{get/get.rb => put_and_get.rb} | 2 +- 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 test/automated/get/category.rb rename test/automated/{get/get.rb => put_and_get.rb} (96%) diff --git a/test/automated/get/category.rb b/test/automated/get/category.rb deleted file mode 100644 index e9edc95..0000000 --- a/test/automated/get/category.rb +++ /dev/null @@ -1,17 +0,0 @@ -require_relative '../automated_init' - -context "Get" do - context "Category" do - category = Controls::Category.example - - stream_name, _ = Controls::Put.(instances: 2, category: category) - - messages = Get.(stream_name) - - number_of_messages = messages.length - - test "Number of messages retrieved is the number written to the category" do - assert(number_of_messages == 2) - end - end -end diff --git a/test/automated/get/get.rb b/test/automated/put_and_get.rb similarity index 96% rename from test/automated/get/get.rb rename to test/automated/put_and_get.rb index 4bdcbbf..9f3fdc0 100644 --- a/test/automated/get/get.rb +++ b/test/automated/put_and_get.rb @@ -1,4 +1,4 @@ -require_relative '../automated_init' +require_relative 'automated_init' context "Put and Get" do stream_name = Controls::StreamName.example From 7b73e51b387642488eabb3649c3e4085d53c3eb2 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 21:13:58 -0500 Subject: [PATCH 463/591] Stream name is a constructor argument --- lib/message_store/postgres/get.rb | 2 +- test/automated/get/batch_size.rb | 15 --------------- test/automated/get/category/batch_size.rb | 21 +++++++++++++++++++++ test/automated/get/category/get_messages.rb | 6 ++---- test/automated/get/stream/batch_size.rb | 17 +++++++++++++++++ test/automated/get/stream/get_message.rb | 6 +++--- 6 files changed, 44 insertions(+), 23 deletions(-) delete mode 100644 test/automated/get/batch_size.rb create mode 100644 test/automated/get/category/batch_size.rb create mode 100644 test/automated/get/stream/batch_size.rb diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index a221623..4e51a96 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -46,7 +46,7 @@ def configure(session: nil) end def self.call(stream_name, position: nil, batch_size: nil, condition: nil, session: nil) - instance = build(batch_size: batch_size, condition: condition, session: session) + instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) instance.(stream_name, position: position) end diff --git a/test/automated/get/batch_size.rb b/test/automated/get/batch_size.rb deleted file mode 100644 index 72e338f..0000000 --- a/test/automated/get/batch_size.rb +++ /dev/null @@ -1,15 +0,0 @@ -require_relative '../automated_init' - -context "Get" do - context "Batch Size" do - stream_name, _ = Controls::Put.(instances: 3) - - messages = Get.(stream_name, batch_size: 2) - - number_of_messages = messages.length - - test "Number of messages retrieved is the specified batch size" do - assert(number_of_messages == 2) - end - end -end diff --git a/test/automated/get/category/batch_size.rb b/test/automated/get/category/batch_size.rb new file mode 100644 index 0000000..2f08b96 --- /dev/null +++ b/test/automated/get/category/batch_size.rb @@ -0,0 +1,21 @@ +require_relative '../../automated_init' + +context "Get" do + context "Category" do + context "Batch Size" do + category = Controls::Category.example + + Controls::Put.(category: category) + Controls::Put.(category: category) + Controls::Put.(category: category) + + messages = Get.(category, batch_size: 2) + + number_of_messages = messages.length + + test "Number of messages retrieved is the specified batch size" do + assert(number_of_messages == 2) + end + end + end +end diff --git a/test/automated/get/category/get_messages.rb b/test/automated/get/category/get_messages.rb index f5dbe60..4bbc6ed 100644 --- a/test/automated/get/category/get_messages.rb +++ b/test/automated/get/category/get_messages.rb @@ -3,12 +3,10 @@ context "Get" do context "Category" do context "Get Messages" do - write_message = Controls::MessageData::Write.example - category = Controls::Category.example - stream_name_1, _ = Controls::Put.(category: category) - stream_name_2, _ = Controls::Put.(category: category) + Controls::Put.(category: category) + Controls::Put.(category: category) messages = Get.(category) diff --git a/test/automated/get/stream/batch_size.rb b/test/automated/get/stream/batch_size.rb new file mode 100644 index 0000000..60d7d2c --- /dev/null +++ b/test/automated/get/stream/batch_size.rb @@ -0,0 +1,17 @@ +require_relative '../../automated_init' + +context "Get" do + context "Stream" do + context "Batch Size" do + stream_name, _ = Controls::Put.(instances: 3) + + messages = Get.(stream_name, batch_size: 2) + + number_of_messages = messages.length + + test "Number of messages retrieved is the specified batch size" do + assert(number_of_messages == 2) + end + end + end +end diff --git a/test/automated/get/stream/get_message.rb b/test/automated/get/stream/get_message.rb index 2caf634..dd6ebae 100644 --- a/test/automated/get/stream/get_message.rb +++ b/test/automated/get/stream/get_message.rb @@ -7,10 +7,10 @@ category = Controls::Category.example - stream_name_1, _ = Controls::Put.(category: category) - stream_name_2, _ = Controls::Put.(category: category) + stream_name, _ = Controls::Put.(category: category) + Controls::Put.(category: category) - messages = Get.(stream_name_1) + messages = Get.(stream_name) context "Messages Retrieved" do test "Only messages from the specific stream" do From 5f2a4c5fde5b14559f3065fc6f209753106c4412 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 21:15:15 -0500 Subject: [PATCH 464/591] Obsolete code is removed --- lib/message_store/postgres/get.rb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 4e51a96..6f6a481 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -6,19 +6,12 @@ class Get dependency :session, Session -## initializer na(:batch_size), :condition initializer :stream_name, na(:batch_size), :condition def batch_size @batch_size ||= Defaults.batch_size end -## def self.__build(batch_size: nil, session: nil, condition: nil) -## new(batch_size, condition).tap do |instance| -## instance.configure(session: session) -## end -## end - def self.build(stream_name, batch_size: nil, session: nil, condition: nil) specialization = specialization(stream_name) From cf8f26807edc9fd42ee908546498d8015b00505f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 21:22:20 -0500 Subject: [PATCH 465/591] Stream name is no longer an actuation parameter, and position is a positional parameter --- lib/message_store/postgres/get.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 6f6a481..07c8491 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -13,9 +13,9 @@ def batch_size end def self.build(stream_name, batch_size: nil, session: nil, condition: nil) - specialization = specialization(stream_name) + cls = specialization(stream_name) - specialization.new(stream_name, batch_size, condition).tap do |instance| + cls.new(stream_name, batch_size, condition).tap do |instance| instance.configure(session: session) end end @@ -40,11 +40,11 @@ def configure(session: nil) def self.call(stream_name, position: nil, batch_size: nil, condition: nil, session: nil) instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) - instance.(stream_name, position: position) + instance.(position) end - def call(stream_name, position: nil) - logger.trace(tag: :get) { "Getting message data (Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } + def call(position) + logger.trace(tag: :get) { "Getting message data (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } position ||= Defaults.position @@ -52,14 +52,14 @@ def call(stream_name, position: nil) message_data = convert(result) - logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect})" } + logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } message_data end def get_result(stream_name, position) - logger.trace(tag: :get) { "Getting result (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } + logger.trace(tag: :get) { "Getting result (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } sql_command = self.class.sql_command(stream_name, position, batch_size, condition) @@ -74,7 +74,7 @@ def get_result(stream_name, position) result = session.execute(sql_command, params) - logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } result end From 2dceebf5e9e6a082f7b04d254d20215c9017604c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 21:24:16 -0500 Subject: [PATCH 466/591] Stream name is passed to constructor from configuration --- lib/message_store/postgres/get.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 07c8491..11ccc58 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -30,7 +30,7 @@ def self.specialization(stream_name) def self.configure(receiver, attr_name: nil, batch_size: nil, condition: nil, session: nil) attr_name ||= :get - instance = build(batch_size: batch_size, condition: condition, session: session) + instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) receiver.public_send "#{attr_name}=", instance end From 59e06458425348cd35f6f83bfcea7dd0ed1ab3f2 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 21:53:11 -0500 Subject: [PATCH 467/591] Tests are compatible with changes to Get interface --- test/automated/iterator/category/next.rb | 3 ++- test/automated/iterator/stream/next.rb | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/automated/iterator/category/next.rb b/test/automated/iterator/category/next.rb index a9c2cc5..d4e89e2 100644 --- a/test/automated/iterator/category/next.rb +++ b/test/automated/iterator/category/next.rb @@ -9,8 +9,9 @@ Controls::Put.(category: category) +## build parm not needed. get will have it iterator = Read::Iterator.build(category) - Get.configure(iterator, batch_size: 1) + Get.configure(iterator, category, batch_size: 1) batch = [] diff --git a/test/automated/iterator/stream/next.rb b/test/automated/iterator/stream/next.rb index ef818f5..7b5e2f9 100644 --- a/test/automated/iterator/stream/next.rb +++ b/test/automated/iterator/stream/next.rb @@ -5,8 +5,9 @@ context "Next" do stream_name, _ = Controls::Put.(instances: 2) +## build parm not needed. get will have it iterator = Read::Iterator.build(stream_name) - Get.configure(iterator, batch_size: 1) + Get.configure(iterator, stream_name, batch_size: 1) batch = [] From 00929367016a4649f403e9ee7a3609b1d35c0ba0 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 21:53:26 -0500 Subject: [PATCH 468/591] Get is configured with stream name --- lib/message_store/postgres/get.rb | 2 +- lib/message_store/postgres/read.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 11ccc58..f084254 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -28,7 +28,7 @@ def self.specialization(stream_name) end end - def self.configure(receiver, attr_name: nil, batch_size: nil, condition: nil, session: nil) + def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, condition: nil, session: nil) attr_name ||= :get instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) receiver.public_send "#{attr_name}=", instance diff --git a/lib/message_store/postgres/read.rb b/lib/message_store/postgres/read.rb index 535116e..c022714 100644 --- a/lib/message_store/postgres/read.rb +++ b/lib/message_store/postgres/read.rb @@ -8,7 +8,7 @@ def configure(session: nil, condition: nil) ## Configure method should select the proper implementation based on ## The stream name - Get.configure(self.iterator, batch_size: batch_size, condition: condition, session: session) + Get.configure(self.iterator, self.stream_name, batch_size: batch_size, condition: condition, session: session) end module Defaults From a1db2fe6216f31156697b8d607b0944d3ef7a002 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 28 Sep 2019 22:14:09 -0500 Subject: [PATCH 469/591] Iterator no longer accepts stream name as get no longer receives it as actuation argument --- lib/message_store/postgres/read.rb | 3 ++- lib/message_store/postgres/read/iterator.rb | 5 ++++- test/automated/iterator/category/next.rb | 3 ++- test/automated/iterator/category/no_further_message_data.rb | 4 +++- test/automated/iterator/stream/next.rb | 3 ++- test/automated/iterator/stream/no_further_message_data.rb | 4 +++- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/message_store/postgres/read.rb b/lib/message_store/postgres/read.rb index c022714..d828d20 100644 --- a/lib/message_store/postgres/read.rb +++ b/lib/message_store/postgres/read.rb @@ -4,7 +4,8 @@ class Read include MessageStore::Read def configure(session: nil, condition: nil) - Iterator.configure(self, stream_name, position: position) +## Iterator.configure(self, stream_name, position: position) + Iterator.configure(self, position) ## Configure method should select the proper implementation based on ## The stream name diff --git a/lib/message_store/postgres/read/iterator.rb b/lib/message_store/postgres/read/iterator.rb index f972a17..fd279fb 100644 --- a/lib/message_store/postgres/read/iterator.rb +++ b/lib/message_store/postgres/read/iterator.rb @@ -12,7 +12,10 @@ class Iterator include MessageStore::Read::Iterator def last_position - unless MessageStore::Postgres::StreamName.category?(stream_name) +## Using get's stream name to figure out how to access last position +## Note: Needs strategy injection +## unless MessageStore::Postgres::StreamName.category?(stream_name) + unless MessageStore::Postgres::StreamName.category?(get.stream_name) batch.last.position else batch.last.global_position diff --git a/test/automated/iterator/category/next.rb b/test/automated/iterator/category/next.rb index d4e89e2..b68ef4c 100644 --- a/test/automated/iterator/category/next.rb +++ b/test/automated/iterator/category/next.rb @@ -10,7 +10,8 @@ Controls::Put.(category: category) ## build parm not needed. get will have it - iterator = Read::Iterator.build(category) +## iterator = Read::Iterator.build(category) + iterator = Read::Iterator.build Get.configure(iterator, category, batch_size: 1) batch = [] diff --git a/test/automated/iterator/category/no_further_message_data.rb b/test/automated/iterator/category/no_further_message_data.rb index 0b47418..8cedfe2 100644 --- a/test/automated/iterator/category/no_further_message_data.rb +++ b/test/automated/iterator/category/no_further_message_data.rb @@ -9,7 +9,9 @@ Controls::Put.(category: category) - iterator = Read::Iterator.build(category) +## build parm not needed. get will have it +## iterator = Read::Iterator.build(category) + iterator = Read::Iterator.build Get.configure(iterator, batch_size: 1) 2.times { iterator.next } diff --git a/test/automated/iterator/stream/next.rb b/test/automated/iterator/stream/next.rb index 7b5e2f9..9894641 100644 --- a/test/automated/iterator/stream/next.rb +++ b/test/automated/iterator/stream/next.rb @@ -6,7 +6,8 @@ stream_name, _ = Controls::Put.(instances: 2) ## build parm not needed. get will have it - iterator = Read::Iterator.build(stream_name) +## iterator = Read::Iterator.build(stream_name) + iterator = Read::Iterator.build Get.configure(iterator, stream_name, batch_size: 1) batch = [] diff --git a/test/automated/iterator/stream/no_further_message_data.rb b/test/automated/iterator/stream/no_further_message_data.rb index 4f80077..bc90254 100644 --- a/test/automated/iterator/stream/no_further_message_data.rb +++ b/test/automated/iterator/stream/no_further_message_data.rb @@ -5,7 +5,9 @@ context "No Further Message Data" do stream_name, _ = Controls::Put.(instances: 2) - iterator = Read::Iterator.build(stream_name) +## build parm not needed. get will have it +## iterator = Read::Iterator.build(stream_name) + iterator = Read::Iterator.build Get.configure(iterator, batch_size: 1) 2.times { iterator.next } From 60c22048bb23a89540abf709bedf11c0e93b9467 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 29 Sep 2019 20:11:14 -0500 Subject: [PATCH 470/591] Reminder on specialization --- lib/message_store/postgres/get.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index f084254..2e8f4f0 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -1,3 +1,7 @@ +## Needs abstract method for getting last position from batch +## Put it on the specialized get classes (as a class method) +## Pass in the batch, get the last position back + module MessageStore module Postgres ## Make into module @@ -95,6 +99,7 @@ def self.sql_command(stream_name, position, batch_size, condition) end end +## Can be generalized to base module def self.category_stream?(stream_name) StreamName.category?(stream_name) end From a08fa07710c1e62c4e013dffc38467f351a1e609 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 29 Sep 2019 22:09:42 -0500 Subject: [PATCH 471/591] Get last position is a member of Get --- lib/message_store/postgres/get.rb | 18 +++++++++++++----- lib/message_store/postgres/read/iterator.rb | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 2e8f4f0..81fd489 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -99,11 +99,6 @@ def self.sql_command(stream_name, position, batch_size, condition) end end -## Can be generalized to base module - def self.category_stream?(stream_name) - StreamName.category?(stream_name) - end - def convert(result) logger.trace(tag: :get) { "Converting result to message data (Result Count: #{result.ntuples})" } @@ -120,6 +115,19 @@ def convert(result) message_data end + def last_position(batch) + if self.class.category_stream?(stream_name) + batch.last.global_position + else + batch.last.position + end + end + +## Can be generalized to base module + def self.category_stream?(stream_name) + StreamName.category?(stream_name) + end + module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? diff --git a/lib/message_store/postgres/read/iterator.rb b/lib/message_store/postgres/read/iterator.rb index fd279fb..02e19cd 100644 --- a/lib/message_store/postgres/read/iterator.rb +++ b/lib/message_store/postgres/read/iterator.rb @@ -11,7 +11,7 @@ class Read class Iterator include MessageStore::Read::Iterator - def last_position + def _last_position ## Using get's stream name to figure out how to access last position ## Note: Needs strategy injection ## unless MessageStore::Postgres::StreamName.category?(stream_name) From 968a3694158d380f019019e18543c40b5ed53285 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 29 Sep 2019 22:56:21 -0500 Subject: [PATCH 472/591] Iterator is a concrete class in the generalized message store library --- lib/message_store/postgres.rb | 1 - lib/message_store/postgres/read.rb | 3 ++- lib/message_store/postgres/read/iterator.rb | 27 --------------------- 3 files changed, 2 insertions(+), 29 deletions(-) delete mode 100644 lib/message_store/postgres/read/iterator.rb diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index 29b6341..b4acfef 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -20,5 +20,4 @@ require 'message_store/postgres/get/stream' require 'message_store/postgres/get/category' require 'message_store/postgres/get/last' -require 'message_store/postgres/read/iterator' require 'message_store/postgres/read' diff --git a/lib/message_store/postgres/read.rb b/lib/message_store/postgres/read.rb index d828d20..ccc05d7 100644 --- a/lib/message_store/postgres/read.rb +++ b/lib/message_store/postgres/read.rb @@ -5,7 +5,8 @@ class Read def configure(session: nil, condition: nil) ## Iterator.configure(self, stream_name, position: position) - Iterator.configure(self, position) +## Iterator is concrete +## Iterator.configure(self, position) ## Configure method should select the proper implementation based on ## The stream name diff --git a/lib/message_store/postgres/read/iterator.rb b/lib/message_store/postgres/read/iterator.rb deleted file mode 100644 index 02e19cd..0000000 --- a/lib/message_store/postgres/read/iterator.rb +++ /dev/null @@ -1,27 +0,0 @@ -## Specialize Get -# - This could be injected by reader into iterator -# - Reader knows whether category or not -# - Reader can tell iterator how to get last message -# - Use a block to determine how to get last message position - - -module MessageStore - module Postgres - class Read - class Iterator - include MessageStore::Read::Iterator - - def _last_position -## Using get's stream name to figure out how to access last position -## Note: Needs strategy injection -## unless MessageStore::Postgres::StreamName.category?(stream_name) - unless MessageStore::Postgres::StreamName.category?(get.stream_name) - batch.last.position - else - batch.last.global_position - end - end - end - end - end -end From 9324aafdab1e6ff3bd2cad97bacb5b00b1915e2a Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 29 Sep 2019 23:02:26 -0500 Subject: [PATCH 473/591] Vestigial code is removed --- lib/message_store/postgres/read.rb | 6 ------ test/automated/iterator/category/next.rb | 2 -- test/automated/iterator/category/no_further_message_data.rb | 2 -- test/automated/iterator/stream/next.rb | 2 -- test/automated/iterator/stream/no_further_message_data.rb | 2 -- 5 files changed, 14 deletions(-) diff --git a/lib/message_store/postgres/read.rb b/lib/message_store/postgres/read.rb index ccc05d7..1c61e54 100644 --- a/lib/message_store/postgres/read.rb +++ b/lib/message_store/postgres/read.rb @@ -4,12 +4,6 @@ class Read include MessageStore::Read def configure(session: nil, condition: nil) -## Iterator.configure(self, stream_name, position: position) -## Iterator is concrete -## Iterator.configure(self, position) - -## Configure method should select the proper implementation based on -## The stream name Get.configure(self.iterator, self.stream_name, batch_size: batch_size, condition: condition, session: session) end diff --git a/test/automated/iterator/category/next.rb b/test/automated/iterator/category/next.rb index b68ef4c..ef470e1 100644 --- a/test/automated/iterator/category/next.rb +++ b/test/automated/iterator/category/next.rb @@ -9,8 +9,6 @@ Controls::Put.(category: category) -## build parm not needed. get will have it -## iterator = Read::Iterator.build(category) iterator = Read::Iterator.build Get.configure(iterator, category, batch_size: 1) diff --git a/test/automated/iterator/category/no_further_message_data.rb b/test/automated/iterator/category/no_further_message_data.rb index 8cedfe2..3be6b78 100644 --- a/test/automated/iterator/category/no_further_message_data.rb +++ b/test/automated/iterator/category/no_further_message_data.rb @@ -9,8 +9,6 @@ Controls::Put.(category: category) -## build parm not needed. get will have it -## iterator = Read::Iterator.build(category) iterator = Read::Iterator.build Get.configure(iterator, batch_size: 1) diff --git a/test/automated/iterator/stream/next.rb b/test/automated/iterator/stream/next.rb index 9894641..74c44b5 100644 --- a/test/automated/iterator/stream/next.rb +++ b/test/automated/iterator/stream/next.rb @@ -5,8 +5,6 @@ context "Next" do stream_name, _ = Controls::Put.(instances: 2) -## build parm not needed. get will have it -## iterator = Read::Iterator.build(stream_name) iterator = Read::Iterator.build Get.configure(iterator, stream_name, batch_size: 1) diff --git a/test/automated/iterator/stream/no_further_message_data.rb b/test/automated/iterator/stream/no_further_message_data.rb index bc90254..4688b26 100644 --- a/test/automated/iterator/stream/no_further_message_data.rb +++ b/test/automated/iterator/stream/no_further_message_data.rb @@ -5,8 +5,6 @@ context "No Further Message Data" do stream_name, _ = Controls::Put.(instances: 2) -## build parm not needed. get will have it -## iterator = Read::Iterator.build(stream_name) iterator = Read::Iterator.build Get.configure(iterator, batch_size: 1) From 058517d881ce8e36f850b6470fc3e75e3c3b3134 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 29 Sep 2019 23:09:54 -0500 Subject: [PATCH 474/591] Get is abstract and stream and category specializations are concrete --- lib/message_store/postgres/get.rb | 73 ++++++++++------------ lib/message_store/postgres/get/category.rb | 13 +++- lib/message_store/postgres/get/last.rb | 2 +- lib/message_store/postgres/get/stream.rb | 13 +++- 4 files changed, 55 insertions(+), 46 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 81fd489..1b3b2ea 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -1,19 +1,24 @@ -## Needs abstract method for getting last position from batch -## Put it on the specialized get classes (as a class method) -## Pass in the batch, get the last position back - module MessageStore module Postgres - ## Make into module - class Get - include MessageStore::Get + module Get + def self.included(cls) + cls.class_exec do + include MessageStore::Get + prepend Call + prepend BatchSize + + extend SQLCommand - dependency :session, Session + dependency :session, Session - initializer :stream_name, na(:batch_size), :condition + initializer :stream_name, na(:batch_size), :condition + end + end - def batch_size - @batch_size ||= Defaults.batch_size + module BatchSize + def batch_size + @batch_size ||= Defaults.batch_size + end end def self.build(stream_name, batch_size: nil, session: nil, condition: nil) @@ -25,7 +30,7 @@ def self.build(stream_name, batch_size: nil, session: nil, condition: nil) end def self.specialization(stream_name) - if category_stream?(stream_name) + if StreamName.category?(stream_name) Category else Stream @@ -47,19 +52,21 @@ def self.call(stream_name, position: nil, batch_size: nil, condition: nil, sess instance.(position) end - def call(position) - logger.trace(tag: :get) { "Getting message data (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } + module Call + def call(position) + logger.trace(tag: :get) { "Getting message data (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } - position ||= Defaults.position + position ||= Defaults.position - result = get_result(stream_name, position) + result = get_result(stream_name, position) - message_data = convert(result) + message_data = convert(result) - logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } - logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } + logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } + logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } - message_data + message_data + end end def get_result(stream_name, position) @@ -67,7 +74,7 @@ def get_result(stream_name, position) sql_command = self.class.sql_command(stream_name, position, batch_size, condition) - cond = self.class.constrain_condition(condition) + cond = Get.constrain_condition(condition) params = [ stream_name, @@ -89,13 +96,10 @@ def self.constrain_condition(condition) "(#{condition})" end - def self.sql_command(stream_name, position, batch_size, condition) - parameters = '$1::varchar, $2::bigint, $3::bigint, $4::varchar' - - if category_stream?(stream_name) - return "SELECT * FROM get_category_messages(#{parameters});" - else - return "SELECT * FROM get_stream_messages(#{parameters});" + module SQLCommand + def sql_command(stream_name, position, batch_size, condition) + parameters = '$1::varchar, $2::bigint, $3::bigint, $4::varchar' + command_text(parameters) end end @@ -115,19 +119,6 @@ def convert(result) message_data end - def last_position(batch) - if self.class.category_stream?(stream_name) - batch.last.global_position - else - batch.last.position - end - end - -## Can be generalized to base module - def self.category_stream?(stream_name) - StreamName.category?(stream_name) - end - module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 1fef6c7..1e10518 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -1,7 +1,16 @@ module MessageStore module Postgres - class Get - class Category < Get + module Get + class Category + include Get + + def self.command_text(parameters) + "SELECT * FROM get_category_messages(#{parameters});" + end + + def last_position(batch) + batch.last.global_position + end end end end diff --git a/lib/message_store/postgres/get/last.rb b/lib/message_store/postgres/get/last.rb index 1a7c798..830a1f1 100644 --- a/lib/message_store/postgres/get/last.rb +++ b/lib/message_store/postgres/get/last.rb @@ -1,6 +1,6 @@ module MessageStore module Postgres - class Get + module Get class Last include MessageStore::Get::Last diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index 775f96f..7d4e9a1 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -1,7 +1,16 @@ module MessageStore module Postgres - class Get - class Stream < Get + module Get + class Stream + include Get + + def self.command_text(parameters) + "SELECT * FROM get_stream_messages(#{parameters});" + end + + def last_position(batch) + batch.last.position + end end end end From 7b781a90a0feb773d1f3a5e2ead03c9adc1ea542 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 29 Sep 2019 23:09:54 -0500 Subject: [PATCH 475/591] Get is abstract and stream and category specializations are concrete --- lib/message_store/postgres/get.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 1b3b2ea..fd284a0 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -111,7 +111,7 @@ def convert(result) record['metadata'] = Deserialize.metadata(record['metadata']) record['time'] = Time.utc_coerced(record['time']) - MessageData::Read.build record + MessageData::Read.build(record) end logger.debug(tag: :get) { "Converted result to message data (Message Data Count: #{message_data.length})" } From cee6210c61f631b7785ecd21567c816b2cbe762c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 30 Sep 2019 00:45:05 -0500 Subject: [PATCH 476/591] GetLast is in the stream namespace --- lib/message_store/postgres.rb | 2 +- lib/message_store/postgres/get/last.rb | 86 ------------------ lib/message_store/postgres/get/stream/last.rb | 88 +++++++++++++++++++ test/automated/get_last/get_last.rb | 2 +- test/automated/get_last/no_messages.rb | 2 +- 5 files changed, 91 insertions(+), 89 deletions(-) delete mode 100644 lib/message_store/postgres/get/last.rb create mode 100644 lib/message_store/postgres/get/stream/last.rb diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index b4acfef..5c1baf4 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -19,5 +19,5 @@ require 'message_store/postgres/get' require 'message_store/postgres/get/stream' require 'message_store/postgres/get/category' -require 'message_store/postgres/get/last' +require 'message_store/postgres/get/stream/last' require 'message_store/postgres/read' diff --git a/lib/message_store/postgres/get/last.rb b/lib/message_store/postgres/get/last.rb deleted file mode 100644 index 830a1f1..0000000 --- a/lib/message_store/postgres/get/last.rb +++ /dev/null @@ -1,86 +0,0 @@ -module MessageStore - module Postgres - module Get - class Last - include MessageStore::Get::Last - - dependency :session, Session - - def configure(session: nil) - Session.configure(self, session: session) - end - - def call(stream_name) - logger.trace(tag: :get) { "Getting last message data (Stream Name: #{stream_name})" } - - result = get_result(stream_name) - - return nil if result.nil? - - message_data = convert(result[0]) - - logger.info(tag: :get) { "Finished getting message data (Stream Name: #{stream_name})" } - logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } - - message_data - end - - def get_result(stream_name) - logger.trace(tag: :get) { "Getting last record (Stream: #{stream_name})" } - - sql_command = self.class.sql_command(stream_name) - - params = [ - stream_name - ] - - result = session.execute(sql_command, params) - - logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" } - - return nil if result.ntuples == 0 - - result - end - - def self.sql_command(stream_name) - parameters = '$1::varchar' - - "SELECT * FROM get_last_message(#{parameters});" - end - - def convert(record) - logger.trace(tag: :get) { "Converting record to message data" } - - record['data'] = Deserialize.data(record['data']) - record['metadata'] = Deserialize.metadata(record['metadata']) - record['time'] = Time.utc_coerced(record['time']) - - message_data = MessageData::Read.build(record) - - logger.debug(tag: :get) { "Converted record to message data" } - - message_data - end - - module Deserialize - def self.data(serialized_data) - return nil if serialized_data.nil? - Transform::Read.(serialized_data, :json, MessageData::Hash) - end - - def self.metadata(serialized_metadata) - return nil if serialized_metadata.nil? - Transform::Read.(serialized_metadata, :json, MessageData::Hash) - end - end - - module Time - def self.utc_coerced(local_time) - Clock::UTC.coerce(local_time) - end - end - end - end - end -end diff --git a/lib/message_store/postgres/get/stream/last.rb b/lib/message_store/postgres/get/stream/last.rb new file mode 100644 index 0000000..384bf41 --- /dev/null +++ b/lib/message_store/postgres/get/stream/last.rb @@ -0,0 +1,88 @@ +module MessageStore + module Postgres + module Get + class Stream + class Last + include MessageStore::Get::Stream::Last + + dependency :session, Session + + def configure(session: nil) + Session.configure(self, session: session) + end + + def call(stream_name) + logger.trace(tag: :get) { "Getting last message data (Stream Name: #{stream_name})" } + + result = get_result(stream_name) + + return nil if result.nil? + + message_data = convert(result[0]) + + logger.info(tag: :get) { "Finished getting message data (Stream Name: #{stream_name})" } + logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } + + message_data + end + + def get_result(stream_name) + logger.trace(tag: :get) { "Getting last record (Stream: #{stream_name})" } + + sql_command = self.class.sql_command(stream_name) + + params = [ + stream_name + ] + + result = session.execute(sql_command, params) + + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" } + + return nil if result.ntuples == 0 + + result + end + + def self.sql_command(stream_name) + parameters = '$1::varchar' + + "SELECT * FROM get_last_message(#{parameters});" + end + + def convert(record) + logger.trace(tag: :get) { "Converting record to message data" } + + record['data'] = Deserialize.data(record['data']) + record['metadata'] = Deserialize.metadata(record['metadata']) + record['time'] = Time.utc_coerced(record['time']) + + message_data = MessageData::Read.build(record) + + logger.debug(tag: :get) { "Converted record to message data" } + + message_data + end + + module Deserialize + def self.data(serialized_data) + return nil if serialized_data.nil? + Transform::Read.(serialized_data, :json, MessageData::Hash) + end + + def self.metadata(serialized_metadata) + return nil if serialized_metadata.nil? + Transform::Read.(serialized_metadata, :json, MessageData::Hash) + end + end + + module Time + def self.utc_coerced(local_time) + Clock::UTC.coerce(local_time) + end + end + end + end + end + end +end diff --git a/test/automated/get_last/get_last.rb b/test/automated/get_last/get_last.rb index 86c4e8f..15ede65 100644 --- a/test/automated/get_last/get_last.rb +++ b/test/automated/get_last/get_last.rb @@ -7,7 +7,7 @@ position = Put.(write_message, stream_name) - last_message = Get::Last.(stream_name) + last_message = Get::Stream::Last.(stream_name) test "Gets the last message in the stream" do assert(last_message.data == write_message.data) diff --git a/test/automated/get_last/no_messages.rb b/test/automated/get_last/no_messages.rb index 68bd0af..344dd4c 100644 --- a/test/automated/get_last/no_messages.rb +++ b/test/automated/get_last/no_messages.rb @@ -4,7 +4,7 @@ context "No Messages" do stream_name = Controls::StreamName.example - last_message = Get::Last.(stream_name) + last_message = Get::Stream::Last.(stream_name) test "Nil message" do assert(last_message.nil?) From 03580bb353f51c908a61b3bc77800ab173ac4dda Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 30 Sep 2019 13:25:54 -0500 Subject: [PATCH 477/591] Less significant code is moved to a less emphatic place in the file --- lib/message_store/postgres/get.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index fd284a0..14fc051 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -29,14 +29,6 @@ def self.build(stream_name, batch_size: nil, session: nil, condition: nil) end end - def self.specialization(stream_name) - if StreamName.category?(stream_name) - Category - else - Stream - end - end - def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, condition: nil, session: nil) attr_name ||= :get instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) @@ -119,6 +111,14 @@ def convert(result) message_data end + def self.specialization(stream_name) + if StreamName.category?(stream_name) + Category + else + Stream + end + end + module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? From e1e43728f6d2fb69a7f979f2549811349c4e060e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 2 Oct 2019 12:24:34 -0500 Subject: [PATCH 478/591] Package version is increased from 1.0.0.1 to 1.1.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index c00cde5..31789ce 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '1.0.0.1' + s.version = '1.1.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 8de303a7fec251b493b50cd61678ed97da0bc8cd Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Mon, 14 Oct 2019 14:45:27 -0500 Subject: [PATCH 479/591] Test files are compatible with TestBench 2.0 --- test/automated.rb | 9 ++++----- test/automated/put/expected_version/error.rb | 4 ++-- test/automated/put/no_stream/existing_stream.rb | 6 ++---- test/automated/session/configure.rb | 4 ++-- test/automated/session/connect_on_first_use.rb | 4 ++-- test/automated/write/batch/expected_version/error.rb | 4 ++-- test/automated/write/message/expected_version/error.rb | 4 ++-- 7 files changed, 16 insertions(+), 19 deletions(-) diff --git a/test/automated.rb b/test/automated.rb index de477e0..4d17efc 100644 --- a/test/automated.rb +++ b/test/automated.rb @@ -1,6 +1,5 @@ -require_relative 'test_init' +ENV['TEST_BENCH_EXCLUDE_FILE_PATTERN'] ||= '/_|sketch|(_init\.rb|_tests\.rb)\z' -TestBench::Runner.( - 'automated/**/*.rb', - exclude_pattern: %r{\/_|sketch|(_init\.rb|_tests\.rb)\z} -) or exit 1 +require_relative './test_init' + +TestBench::CLI.() diff --git a/test/automated/put/expected_version/error.rb b/test/automated/put/expected_version/error.rb index ecfcbf8..75017c3 100644 --- a/test/automated/put/expected_version/error.rb +++ b/test/automated/put/expected_version/error.rb @@ -11,8 +11,8 @@ incorrect_stream_version = position + 1 test "Is an error" do - assert proc { Put.(write_message, stream_name, expected_version: incorrect_stream_version ) } do - raises_error? ExpectedVersion::Error + assert_raises ExpectedVersion::Error do + Put.(write_message, stream_name, expected_version: incorrect_stream_version ) end end diff --git a/test/automated/put/no_stream/existing_stream.rb b/test/automated/put/no_stream/existing_stream.rb index 1f654db..a814816 100644 --- a/test/automated/put/no_stream/existing_stream.rb +++ b/test/automated/put/no_stream/existing_stream.rb @@ -10,11 +10,9 @@ Put.(write_message_1, stream_name) - erroneous = proc { Put.(write_message_2, stream_name, expected_version: NoStream.name) } - test "Is an error" do - assert erroneous do - raises_error? ExpectedVersion::Error + assert_raises ExpectedVersion::Error do + Put.(write_message_2, stream_name, expected_version: NoStream.name) end end end diff --git a/test/automated/session/configure.rb b/test/automated/session/configure.rb index 03e9882..c745ad1 100644 --- a/test/automated/session/configure.rb +++ b/test/automated/session/configure.rb @@ -38,8 +38,8 @@ session = Object.new test "Is an error" do - assert proc { Session.configure(receiver, settings: settings, session: session) } do - raises_error? Session::Error + assert_raises Session::Error do + Session.configure(receiver, settings: settings, session: session) end end end diff --git a/test/automated/session/connect_on_first_use.rb b/test/automated/session/connect_on_first_use.rb index 9f4301d..3d7cae9 100644 --- a/test/automated/session/connect_on_first_use.rb +++ b/test/automated/session/connect_on_first_use.rb @@ -7,8 +7,8 @@ refute(session.connected?) test "Connects" do - refute proc { session.execute('SELECT 1;') } do - raises_error? + refute_raises do + session.execute('SELECT 1;') end end end diff --git a/test/automated/write/batch/expected_version/error.rb b/test/automated/write/batch/expected_version/error.rb index 4c4e6c8..e097929 100644 --- a/test/automated/write/batch/expected_version/error.rb +++ b/test/automated/write/batch/expected_version/error.rb @@ -17,8 +17,8 @@ batch = [write_message_1, write_message_2] test "Is an error" do - assert proc { Write.(batch, stream_name, expected_version: incorrect_stream_version ) } do - raises_error? ExpectedVersion::Error + assert_raises ExpectedVersion::Error do + Write.(batch, stream_name, expected_version: incorrect_stream_version) end end diff --git a/test/automated/write/message/expected_version/error.rb b/test/automated/write/message/expected_version/error.rb index 4651e44..86335f7 100644 --- a/test/automated/write/message/expected_version/error.rb +++ b/test/automated/write/message/expected_version/error.rb @@ -12,8 +12,8 @@ incorrect_stream_version = position + 1 test "Is an error" do - assert proc { Write.(write_message, stream_name, expected_version: incorrect_stream_version ) } do - raises_error? ExpectedVersion::Error + assert_raises ExpectedVersion::Error do + Write.(write_message, stream_name, expected_version: incorrect_stream_version ) end end From 622df216114f332f9a8f7ece6943885a0a3e4513 Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Thu, 17 Oct 2019 16:32:11 -0500 Subject: [PATCH 480/591] Automated test runner supplies exclude file pattern directly into CLI --- test/automated.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/automated.rb b/test/automated.rb index 4d17efc..5bdd0b4 100644 --- a/test/automated.rb +++ b/test/automated.rb @@ -1,5 +1,5 @@ -ENV['TEST_BENCH_EXCLUDE_FILE_PATTERN'] ||= '/_|sketch|(_init\.rb|_tests\.rb)\z' - require_relative './test_init' -TestBench::CLI.() +TestBench::CLI.( + exclude_file_pattern: %r{/_|sketch|(_init\.rb|_tests\.rb)\z} +) From 4819be12405411cd18c46a1533fc89380e95570b Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Thu, 17 Oct 2019 16:32:12 -0500 Subject: [PATCH 481/591] Parenthesis are added to assert_raises and refute_raises --- test/automated/put/expected_version/error.rb | 2 +- test/automated/put/no_stream/existing_stream.rb | 2 +- test/automated/session/configure.rb | 2 +- test/automated/write/batch/expected_version/error.rb | 2 +- test/automated/write/message/expected_version/error.rb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/automated/put/expected_version/error.rb b/test/automated/put/expected_version/error.rb index 75017c3..2a257d0 100644 --- a/test/automated/put/expected_version/error.rb +++ b/test/automated/put/expected_version/error.rb @@ -11,7 +11,7 @@ incorrect_stream_version = position + 1 test "Is an error" do - assert_raises ExpectedVersion::Error do + assert_raises(ExpectedVersion::Error) do Put.(write_message, stream_name, expected_version: incorrect_stream_version ) end end diff --git a/test/automated/put/no_stream/existing_stream.rb b/test/automated/put/no_stream/existing_stream.rb index a814816..a2989a2 100644 --- a/test/automated/put/no_stream/existing_stream.rb +++ b/test/automated/put/no_stream/existing_stream.rb @@ -11,7 +11,7 @@ Put.(write_message_1, stream_name) test "Is an error" do - assert_raises ExpectedVersion::Error do + assert_raises(ExpectedVersion::Error) do Put.(write_message_2, stream_name, expected_version: NoStream.name) end end diff --git a/test/automated/session/configure.rb b/test/automated/session/configure.rb index c745ad1..1e748cf 100644 --- a/test/automated/session/configure.rb +++ b/test/automated/session/configure.rb @@ -38,7 +38,7 @@ session = Object.new test "Is an error" do - assert_raises Session::Error do + assert_raises(Session::Error) do Session.configure(receiver, settings: settings, session: session) end end diff --git a/test/automated/write/batch/expected_version/error.rb b/test/automated/write/batch/expected_version/error.rb index e097929..adca25a 100644 --- a/test/automated/write/batch/expected_version/error.rb +++ b/test/automated/write/batch/expected_version/error.rb @@ -17,7 +17,7 @@ batch = [write_message_1, write_message_2] test "Is an error" do - assert_raises ExpectedVersion::Error do + assert_raises(ExpectedVersion::Error) do Write.(batch, stream_name, expected_version: incorrect_stream_version) end end diff --git a/test/automated/write/message/expected_version/error.rb b/test/automated/write/message/expected_version/error.rb index 86335f7..54603d6 100644 --- a/test/automated/write/message/expected_version/error.rb +++ b/test/automated/write/message/expected_version/error.rb @@ -12,7 +12,7 @@ incorrect_stream_version = position + 1 test "Is an error" do - assert_raises ExpectedVersion::Error do + assert_raises(ExpectedVersion::Error) do Write.(write_message, stream_name, expected_version: incorrect_stream_version ) end end From 7aaa3ac56004e21fd3ec70f3aea0f9eae6a2dc6b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 26 Oct 2019 18:00:22 -0500 Subject: [PATCH 482/591] Sketch is ignored --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 740a161..9401526 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ gems loader.rb /test/package/installed /test/benchmark/tmp/**/* +_sketch.rb From 77196651e1d3f2bab2515022192f408148298d01 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 26 Oct 2019 18:07:06 -0500 Subject: [PATCH 483/591] Get used in iterator test is configured with stream name or category name --- test/automated/iterator/category/no_further_message_data.rb | 2 +- test/automated/iterator/stream/no_further_message_data.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/automated/iterator/category/no_further_message_data.rb b/test/automated/iterator/category/no_further_message_data.rb index 3be6b78..1ae7b32 100644 --- a/test/automated/iterator/category/no_further_message_data.rb +++ b/test/automated/iterator/category/no_further_message_data.rb @@ -10,7 +10,7 @@ Controls::Put.(category: category) iterator = Read::Iterator.build - Get.configure(iterator, batch_size: 1) + Get.configure(iterator, category, batch_size: 1) 2.times { iterator.next } diff --git a/test/automated/iterator/stream/no_further_message_data.rb b/test/automated/iterator/stream/no_further_message_data.rb index 4688b26..99ffa10 100644 --- a/test/automated/iterator/stream/no_further_message_data.rb +++ b/test/automated/iterator/stream/no_further_message_data.rb @@ -6,7 +6,7 @@ stream_name, _ = Controls::Put.(instances: 2) iterator = Read::Iterator.build - Get.configure(iterator, batch_size: 1) + Get.configure(iterator, stream_name, batch_size: 1) 2.times { iterator.next } From 5ec29de5cf2a13b53b91eabe577e99938f446310 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 26 Oct 2019 22:37:39 -0500 Subject: [PATCH 484/591] Separate condition tests for both category and stream --- test/automated/get/category/condition.rb | 23 +++++++++++++++++++++++ test/automated/get/condition.rb | 19 ------------------- test/automated/get/stream/condition.rb | 21 +++++++++++++++++++++ 3 files changed, 44 insertions(+), 19 deletions(-) create mode 100644 test/automated/get/category/condition.rb delete mode 100644 test/automated/get/condition.rb create mode 100644 test/automated/get/stream/condition.rb diff --git a/test/automated/get/category/condition.rb b/test/automated/get/category/condition.rb new file mode 100644 index 0000000..2e1ffd6 --- /dev/null +++ b/test/automated/get/category/condition.rb @@ -0,0 +1,23 @@ +require_relative '../../automated_init' + +context "Get" do + context "Category" do + context "Condition" do + category = Controls::Category.example + + stream_name, _ = Controls::Put.(instances: 3, category: category) + + condition = 'position = 0 OR position = 2' + + messages = Get.(category, batch_size: 3, condition: condition) + + message_positions = messages.map do |message| + message.position + end + + test "Returns messages that meet the condition" do + assert(message_positions == [0, 2]) + end + end + end +end diff --git a/test/automated/get/condition.rb b/test/automated/get/condition.rb deleted file mode 100644 index 1c616b9..0000000 --- a/test/automated/get/condition.rb +++ /dev/null @@ -1,19 +0,0 @@ -require_relative '../automated_init' - -context "Get" do - context "Condition" do - stream_name, _ = Controls::Put.(instances: 3) - - condition = 'position = 0 OR position = 2' - - messages = Get.(stream_name, batch_size: 3, condition: condition) - - message_positions = messages.map do |message| - message.position - end - - test "Returns messages that meet the condition" do - assert(message_positions == [0, 2]) - end - end -end diff --git a/test/automated/get/stream/condition.rb b/test/automated/get/stream/condition.rb new file mode 100644 index 0000000..e16539c --- /dev/null +++ b/test/automated/get/stream/condition.rb @@ -0,0 +1,21 @@ +require_relative '../../automated_init' + +context "Get" do + context "Stream" do + context "Condition" do + stream_name, _ = Controls::Put.(instances: 3) + + condition = 'position = 0 OR position = 2' + + messages = Get.(stream_name, batch_size: 3, condition: condition) + + message_positions = messages.map do |message| + message.position + end + + test "Returns messages that meet the condition" do + assert(message_positions == [0, 2]) + end + end + end +end From 8c55fe7efbec88955e2c48c3442d29d36046245e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 26 Oct 2019 22:50:19 -0500 Subject: [PATCH 485/591] Category get with condition test is suspended --- test/automated/get/category/{condition.rb => _condition.rb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/automated/get/category/{condition.rb => _condition.rb} (100%) diff --git a/test/automated/get/category/condition.rb b/test/automated/get/category/_condition.rb similarity index 100% rename from test/automated/get/category/condition.rb rename to test/automated/get/category/_condition.rb From 7fce05d9edaa0639c67b176d58ca8070bce12f2e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 26 Oct 2019 22:57:48 -0500 Subject: [PATCH 486/591] Superfluous, vestigial parameters parameter is removed from SQL text generation method --- lib/message_store/postgres/get.rb | 11 +++++------ lib/message_store/postgres/get/category.rb | 2 +- lib/message_store/postgres/get/stream.rb | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 14fc051..3753bd0 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -7,7 +7,7 @@ def self.included(cls) prepend Call prepend BatchSize - extend SQLCommand + extend SQLParameters dependency :session, Session @@ -64,7 +64,7 @@ def call(position) def get_result(stream_name, position) logger.trace(tag: :get) { "Getting result (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } - sql_command = self.class.sql_command(stream_name, position, batch_size, condition) + sql_command = self.class.sql_command cond = Get.constrain_condition(condition) @@ -88,10 +88,9 @@ def self.constrain_condition(condition) "(#{condition})" end - module SQLCommand - def sql_command(stream_name, position, batch_size, condition) - parameters = '$1::varchar, $2::bigint, $3::bigint, $4::varchar' - command_text(parameters) + module SQLParameters + def parameters + '$1::varchar, $2::bigint, $3::bigint, $4::varchar' end end diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 1e10518..fd802bc 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -4,7 +4,7 @@ module Get class Category include Get - def self.command_text(parameters) + def self.sql_command "SELECT * FROM get_category_messages(#{parameters});" end diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index 7d4e9a1..788d0f2 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -4,7 +4,7 @@ module Get class Stream include Get - def self.command_text(parameters) + def self.sql_command "SELECT * FROM get_stream_messages(#{parameters});" end From 9e4cf02a450d99ff3821a1a3e10f29fe0b5d3c07 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 26 Oct 2019 23:09:08 -0500 Subject: [PATCH 487/591] Condition test is reactivated --- test/automated/get/category/{_condition.rb => condition.rb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/automated/get/category/{_condition.rb => condition.rb} (100%) diff --git a/test/automated/get/category/_condition.rb b/test/automated/get/category/condition.rb similarity index 100% rename from test/automated/get/category/_condition.rb rename to test/automated/get/category/condition.rb From 734800bbcbe4aca63400d8521b0e28c00522bf1f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 26 Oct 2019 23:10:07 -0500 Subject: [PATCH 488/591] Module name is clarified --- lib/message_store/postgres/get.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 3753bd0..84dbd9c 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -7,7 +7,7 @@ def self.included(cls) prepend Call prepend BatchSize - extend SQLParameters + extend Parameters dependency :session, Session @@ -88,7 +88,7 @@ def self.constrain_condition(condition) "(#{condition})" end - module SQLParameters + module Parameters def parameters '$1::varchar, $2::bigint, $3::bigint, $4::varchar' end From 7d873f2d8e6c4e650eec887b46ccec24da05f6d2 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 26 Oct 2019 23:37:08 -0500 Subject: [PATCH 489/591] Correlation parameter is received by Get --- lib/message_store/postgres/get.rb | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 84dbd9c..2752e25 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -11,7 +11,7 @@ def self.included(cls) dependency :session, Session - initializer :stream_name, na(:batch_size), :condition + initializer :stream_name, na(:batch_size), :correlation, :condition end end @@ -21,32 +21,32 @@ def batch_size end end - def self.build(stream_name, batch_size: nil, session: nil, condition: nil) + def self.build(stream_name, batch_size: nil, session: nil, correlation: nil, condition: nil) cls = specialization(stream_name) - cls.new(stream_name, batch_size, condition).tap do |instance| + cls.new(stream_name, batch_size, correlation, condition).tap do |instance| instance.configure(session: session) end end - def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, condition: nil, session: nil) + def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, correlation: nil, condition: nil, session: nil) attr_name ||= :get - instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) - receiver.public_send "#{attr_name}=", instance + instance = build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) + receiver.public_send("#{attr_name}=", instance) end def configure(session: nil) - Session.configure self, session: session + Session.configure(self, session: session) end - def self.call(stream_name, position: nil, batch_size: nil, condition: nil, session: nil) - instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) + def self.call(stream_name, position: nil, batch_size: nil, correlation: nil, condition: nil, session: nil) + instance = build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) instance.(position) end module Call def call(position) - logger.trace(tag: :get) { "Getting message data (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } + logger.trace(tag: :get) { "Getting message data (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } position ||= Defaults.position @@ -54,7 +54,7 @@ def call(position) message_data = convert(result) - logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect})" } + logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } message_data @@ -62,7 +62,7 @@ def call(position) end def get_result(stream_name, position) - logger.trace(tag: :get) { "Getting result (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } + logger.trace(tag: :get) { "Getting result (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } sql_command = self.class.sql_command @@ -72,12 +72,13 @@ def get_result(stream_name, position) stream_name, position, batch_size, + correlation, cond ] result = session.execute(sql_command, params) - logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'})" } + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } result end @@ -90,7 +91,7 @@ def self.constrain_condition(condition) module Parameters def parameters - '$1::varchar, $2::bigint, $3::bigint, $4::varchar' + '$1::varchar, $2::bigint, $3::bigint, $4::varchar, $5::varchar' end end From 90c082aa80e5528c60eb23e5640d37994609b536 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 27 Oct 2019 00:23:27 -0500 Subject: [PATCH 490/591] Test runner explicitly specifies the tests directory --- test/automated.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/automated.rb b/test/automated.rb index 5bdd0b4..0201c1d 100644 --- a/test/automated.rb +++ b/test/automated.rb @@ -1,5 +1,6 @@ require_relative './test_init' TestBench::CLI.( + tests_directory: 'test/automated', exclude_file_pattern: %r{/_|sketch|(_init\.rb|_tests\.rb)\z} ) From 6083078d1b9533fb8d38c6f69e9b55623e8c5599 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 27 Oct 2019 01:19:52 -0500 Subject: [PATCH 491/591] Put control accepts message_data as well as message for backward compatibility --- lib/message_store/postgres/controls/put.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/message_store/postgres/controls/put.rb b/lib/message_store/postgres/controls/put.rb index 3822625..13c5368 100644 --- a/lib/message_store/postgres/controls/put.rb +++ b/lib/message_store/postgres/controls/put.rb @@ -2,20 +2,21 @@ module MessageStore module Postgres module Controls module Put - def self.call(instances: nil, stream_name: nil, message: nil, category: nil) + def self.call(instances: nil, stream_name: nil, message_data: nil, message: nil, category: nil) instances ||= 1 stream_name ||= StreamName.example(category: category) + message_data ||= message - message_specified = !message.nil? + message_specified = !message_data.nil? - message ||= MessageData::Write.example + message_data ||= MessageData::Write.example position = nil instances.times do - position = MessageStore::Postgres::Put.(message, stream_name) + position = MessageStore::Postgres::Put.(message_data, stream_name) unless message_specified - message.id = MessageData::Write.id + message_data.id = MessageData::Write.id end end From a832648e041db931af576d90b89d98ef17ee84c2 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 27 Oct 2019 01:51:55 -0500 Subject: [PATCH 492/591] Correlation category get is tested --- test/automated/get/category/correlation.rb | 40 ++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 test/automated/get/category/correlation.rb diff --git a/test/automated/get/category/correlation.rb b/test/automated/get/category/correlation.rb new file mode 100644 index 0000000..22471a3 --- /dev/null +++ b/test/automated/get/category/correlation.rb @@ -0,0 +1,40 @@ +require_relative '../../automated_init' + +context "Get" do + context "Category" do + context "Condition" do + correlation_category = Controls::Category.example + + correlation_stream_name = Controls::StreamName.example(category: correlation_category) + + correlation_metadata = { + correlation_stream_name: correlation_stream_name + } + + stream_name = Controls::StreamName.example + + message_data = Controls::MessageData::Write.example + + message_data.metadata = { + correlation_stream_name: SecureRandom.hex + } + + Put.(message_data, stream_name) + + 2.times do + message_data = Controls::MessageData::Write.example(metadata: correlation_metadata) + Put.(message_data, stream_name) + end + + message_datas = Get.(stream_name, correlation: correlation_category) + + correlation_stream_names = message_datas.map do |message_data| + message_data.metadata[:correlation_stream_name] + end + + test "Retrieves messages that meet the condition" do + assert(correlation_stream_names == [correlation_stream_name, correlation_stream_name]) + end + end + end +end From 06dbd5af459e72d417d79a737eb1c12a0c0066f6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 27 Oct 2019 01:52:51 -0500 Subject: [PATCH 493/591] Test language is clarified --- test/automated/get/category/condition.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/automated/get/category/condition.rb b/test/automated/get/category/condition.rb index 2e1ffd6..8778a79 100644 --- a/test/automated/get/category/condition.rb +++ b/test/automated/get/category/condition.rb @@ -15,7 +15,7 @@ message.position end - test "Returns messages that meet the condition" do + test "Retrieves messages that meet the condition" do assert(message_positions == [0, 2]) end end From 42aa940250e7a8a875f285d2baef693801064812 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 27 Oct 2019 01:54:53 -0500 Subject: [PATCH 494/591] Correlation stream get is tested --- test/automated/get/stream/correlation.rb | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 test/automated/get/stream/correlation.rb diff --git a/test/automated/get/stream/correlation.rb b/test/automated/get/stream/correlation.rb new file mode 100644 index 0000000..4befb61 --- /dev/null +++ b/test/automated/get/stream/correlation.rb @@ -0,0 +1,40 @@ +require_relative '../../automated_init' + +context "Get" do + context "Stream" do + context "Condition" do + correlation_category = Controls::Category.example + + correlation_stream_name = Controls::StreamName.example(category: correlation_category) + + correlation_metadata = { + correlation_stream_name: correlation_stream_name + } + + stream_name = Controls::StreamName.example + + message_data = Controls::MessageData::Write.example + + message_data.metadata = { + correlation_stream_name: SecureRandom.hex + } + + Put.(message_data, stream_name) + + 2.times do + message_data = Controls::MessageData::Write.example(metadata: correlation_metadata) + Put.(message_data, stream_name) + end + + message_datas = Get.(stream_name, correlation: correlation_category) + + correlation_stream_names = message_datas.map do |message_data| + message_data.metadata[:correlation_stream_name] + end + + test "Retrieves messages that meet the condition" do + assert(correlation_stream_names == [correlation_stream_name, correlation_stream_name]) + end + end + end +end From 916a7d88cc947af4f3037326d09e42e7b925c744 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 27 Oct 2019 02:06:44 -0500 Subject: [PATCH 495/591] Correlation stream get test is corrected --- test/automated/get/category/correlation.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/automated/get/category/correlation.rb b/test/automated/get/category/correlation.rb index 22471a3..44ed938 100644 --- a/test/automated/get/category/correlation.rb +++ b/test/automated/get/category/correlation.rb @@ -11,7 +11,7 @@ correlation_stream_name: correlation_stream_name } - stream_name = Controls::StreamName.example + category = Controls::Category.example message_data = Controls::MessageData::Write.example @@ -19,14 +19,16 @@ correlation_stream_name: SecureRandom.hex } + stream_name = Controls::StreamName.example(category: category) Put.(message_data, stream_name) 2.times do message_data = Controls::MessageData::Write.example(metadata: correlation_metadata) + stream_name = Controls::StreamName.example(category: category) Put.(message_data, stream_name) end - message_datas = Get.(stream_name, correlation: correlation_category) + message_datas = Get.(category, correlation: correlation_category) correlation_stream_names = message_datas.map do |message_data| message_data.metadata[:correlation_stream_name] From 931f7de041a98febd17612893cf18e8129755dd2 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 27 Oct 2019 12:22:34 -0500 Subject: [PATCH 496/591] Specialized starting position tests for both category and stream --- test/automated/get/category/position.rb | 18 ++++++++++++++++++ test/automated/get/position.rb | 13 ------------- test/automated/get/stream/position.rb | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 13 deletions(-) create mode 100644 test/automated/get/category/position.rb delete mode 100644 test/automated/get/position.rb create mode 100644 test/automated/get/stream/position.rb diff --git a/test/automated/get/category/position.rb b/test/automated/get/category/position.rb new file mode 100644 index 0000000..9e17c66 --- /dev/null +++ b/test/automated/get/category/position.rb @@ -0,0 +1,18 @@ +require_relative '../../automated_init' + +context "Get" do + context "Category" do + context "Position" do + category = Controls::Category.example + stream_name = Controls::StreamName.example(category: category) + + Controls::Put.(instances: 2, stream_name: stream_name) + + batch = Get.(category, position: 1, batch_size: 1) + + test "Retrieves messages from the starting position" do + assert(batch.length == 1) + end + end + end +end diff --git a/test/automated/get/position.rb b/test/automated/get/position.rb deleted file mode 100644 index b3eed96..0000000 --- a/test/automated/get/position.rb +++ /dev/null @@ -1,13 +0,0 @@ -require_relative '../automated_init' - -context "Get" do - context "Position" do - stream_name, _ = Controls::Put.(instances: 2) - - batch = Get.(stream_name, position: 1, batch_size: 1) - - test "Gets from the starting position" do - assert(batch.length == 1) - end - end -end diff --git a/test/automated/get/stream/position.rb b/test/automated/get/stream/position.rb new file mode 100644 index 0000000..3e6bad8 --- /dev/null +++ b/test/automated/get/stream/position.rb @@ -0,0 +1,15 @@ +require_relative '../../automated_init' + +context "Get" do + context "Stream" do + context "Position" do + stream_name, _ = Controls::Put.(instances: 2) + + batch = Get.(stream_name, position: 1, batch_size: 1) + + test "Retrieves messages from the starting position" do + assert(batch.length == 1) + end + end + end +end From 86d08d8aa93dfe7c5a51108810e1032e0844358f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 27 Oct 2019 12:43:19 -0500 Subject: [PATCH 497/591] Specialized no message tests for both category and stream --- test/automated/get/category/no_messages.rb | 15 +++++++++++++++ test/automated/get/no_messages.rb | 13 ------------- test/automated/get/stream/no_messages.rb | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 13 deletions(-) create mode 100644 test/automated/get/category/no_messages.rb delete mode 100644 test/automated/get/no_messages.rb create mode 100644 test/automated/get/stream/no_messages.rb diff --git a/test/automated/get/category/no_messages.rb b/test/automated/get/category/no_messages.rb new file mode 100644 index 0000000..9d74ad4 --- /dev/null +++ b/test/automated/get/category/no_messages.rb @@ -0,0 +1,15 @@ +require_relative '../../automated_init' + +context "Get" do + context "Category" do + context "No Messages" do + category = Controls::Category.example + + batch = Get.(category) + + test "Retrieves no messages" do + assert(batch == []) + end + end + end +end diff --git a/test/automated/get/no_messages.rb b/test/automated/get/no_messages.rb deleted file mode 100644 index 79cefa8..0000000 --- a/test/automated/get/no_messages.rb +++ /dev/null @@ -1,13 +0,0 @@ -require_relative '../automated_init' - -context "Get" do - context "No Messages" do - stream_name = Controls::StreamName.example - - batch = Get.(stream_name) - - test "Empty array" do - assert(batch == []) - end - end -end diff --git a/test/automated/get/stream/no_messages.rb b/test/automated/get/stream/no_messages.rb new file mode 100644 index 0000000..14fa4d5 --- /dev/null +++ b/test/automated/get/stream/no_messages.rb @@ -0,0 +1,15 @@ +require_relative '../../automated_init' + +context "Get" do + context "Stream" do + context "No Messages" do + stream_name = Controls::StreamName.example + + batch = Get.(stream_name) + + test "Retrieves no messages" do + assert(batch == []) + end + end + end +end From 1449f51e412969a45cc0a0bd9491661a3ced9028 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 29 Oct 2019 23:21:37 -0500 Subject: [PATCH 498/591] Parameter order is corrected --- lib/message_store/postgres/get.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 2752e25..4a30c08 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -21,7 +21,7 @@ def batch_size end end - def self.build(stream_name, batch_size: nil, session: nil, correlation: nil, condition: nil) + def self.build(stream_name, batch_size: nil, correlation: nil, condition: nil, session: nil) cls = specialization(stream_name) cls.new(stream_name, batch_size, correlation, condition).tap do |instance| From 1af076260fecfb67a60fcae614a56c3c35003032 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Nov 2019 01:05:58 -0500 Subject: [PATCH 499/591] StreamName module is vestigial. It is removed. --- lib/message_store/postgres.rb | 2 - lib/message_store/postgres/stream_name.rb | 52 ----------------- test/automated/stream_name/get_category.rb | 57 ------------------- test/automated/stream_name/get_entity_name.rb | 55 ------------------ test/automated/stream_name/get_id.rb | 19 ------- test/automated/stream_name/get_types.rb | 30 ---------- test/automated/stream_name/is_category.rb | 26 --------- test/automated/stream_name/stream_name.rb | 51 ----------------- 8 files changed, 292 deletions(-) delete mode 100644 lib/message_store/postgres/stream_name.rb delete mode 100644 test/automated/stream_name/get_category.rb delete mode 100644 test/automated/stream_name/get_entity_name.rb delete mode 100644 test/automated/stream_name/get_id.rb delete mode 100644 test/automated/stream_name/get_types.rb delete mode 100644 test/automated/stream_name/is_category.rb delete mode 100644 test/automated/stream_name/stream_name.rb diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index 5c1baf4..52a4559 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -11,8 +11,6 @@ require 'message_store/postgres/settings' require 'message_store/postgres/session' -require 'message_store/postgres/stream_name' - require 'message_store/postgres/put' require 'message_store/postgres/write' diff --git a/lib/message_store/postgres/stream_name.rb b/lib/message_store/postgres/stream_name.rb deleted file mode 100644 index 29e6cd6..0000000 --- a/lib/message_store/postgres/stream_name.rb +++ /dev/null @@ -1,52 +0,0 @@ -module MessageStore - module Postgres - module StreamName - def self.stream_name(category_name, id=nil, type: nil, types: nil) - types = Array(types) - types.unshift(type) unless type.nil? - - type_list = nil - type_list = types.join('+') unless types.empty? - - stream_name = category_name - stream_name = "#{stream_name}:#{type_list}" unless type_list.nil? - stream_name = "#{stream_name}-#{id}" unless id.nil? - - stream_name - end - - def self.get_id(stream_name) - id = stream_name.partition('-').last - id.empty? ? nil : id - end - - def self.get_category(stream_name) - stream_name.split('-').first - end - - def self.category?(stream_name) - !stream_name.include?('-') - end - - def self.get_type_list(stream_name) - type = stream_name.split(':').last.split('-').first - - return nil if stream_name.start_with?(type) - - type - end - - def self.get_types(stream_name) - type_list = get_type_list(stream_name) - - return [] if type_list.nil? - - type_list.split('+') - end - - def self.get_entity_name(stream_name) - get_category(stream_name).split(':').first - end - end - end -end diff --git a/test/automated/stream_name/get_category.rb b/test/automated/stream_name/get_category.rb deleted file mode 100644 index e13742c..0000000 --- a/test/automated/stream_name/get_category.rb +++ /dev/null @@ -1,57 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Get Category" do - category = 'someStream' - - context "Stream Name Contains an ID" do - id = Identifier::UUID.random - stream_name = "#{category}-#{id}" - - stream_category = StreamName.get_category(stream_name) - - test "Category name is the part of the stream name before the first dash" do - assert(stream_category == category) - end - end - - context "Stream Name Contains no ID" do - stream_name = category - stream_category = StreamName.get_category(stream_name) - - test "Category name is the stream name" do - assert(stream_category == category) - end - end - - context "Stream Name Contains Type" do - stream_name = "#{category}:someType" - stream_category = StreamName.get_category(stream_name) - - test "Category name is the stream name" do - assert(stream_category == stream_name) - end - end - - context "Stream Name Contains Types" do - stream_name = "#{category}:someType+someOtherType" - stream_category = StreamName.get_category(stream_name) - - test "Category name is the stream name" do - assert(stream_category == stream_name) - end - end - - context "Stream Name Contains ID and Types" do - id = Identifier::UUID.random - category_and_types = "#{category}:someType+someOtherType" - stream_name = "#{category_and_types}-#{id}" - - stream_category = StreamName.get_category(stream_name) - - test "Category name is the stream name" do - assert(stream_category == category_and_types) - end - end - end -end diff --git a/test/automated/stream_name/get_entity_name.rb b/test/automated/stream_name/get_entity_name.rb deleted file mode 100644 index 67292eb..0000000 --- a/test/automated/stream_name/get_entity_name.rb +++ /dev/null @@ -1,55 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Get Entity Name" do - category = 'someStream' - - context "Stream Name Contains an ID" do - id = Identifier::UUID.random - stream_name = "#{category}-#{id}" - - entity_name = StreamName.get_entity_name(stream_name) - - test "Entity name is the part of the stream name before the first dash" do - assert(entity_name == category) - end - end - - context "Stream Name Contains no ID" do - stream_name = category - entity_name = StreamName.get_entity_name(stream_name) - - test "Entity name is the stream name" do - assert(entity_name == category) - end - end - - context "Stream Name Contains Type" do - stream_name = "#{category}:someType" - entity_name = StreamName.get_entity_name(stream_name) - - test "Entity name is the category without any types" do - assert(entity_name == category) - end - end - - context "Stream Name Contains Types" do - stream_name = "#{category}:someType+someOtherType" - entity_name = StreamName.get_entity_name(stream_name) - - test "Entity name is the category without any types" do - assert(entity_name == category) - end - end - - context "Stream Name Contains ID and Types" do - id = Identifier::UUID.random - stream_name = "#{category}:someType+someOtherType-#{id}" - entity_name = StreamName.get_entity_name(stream_name) - - test "Entity name is the category without any types" do - assert(entity_name == category) - end - end - end -end diff --git a/test/automated/stream_name/get_id.rb b/test/automated/stream_name/get_id.rb deleted file mode 100644 index e8fa9d8..0000000 --- a/test/automated/stream_name/get_id.rb +++ /dev/null @@ -1,19 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Get ID" do - test "Is the part of a stream name after the first dash" do - id = Identifier::UUID.random - stream_name = "someStream-#{id}" - - stream_id = StreamName.get_id(stream_name) - - assert(stream_id == id) - end - - test "Is nil if there is no ID part in the stream name" do - stream_id = StreamName.get_id('someStream') - assert(stream_id.nil?) - end - end -end diff --git a/test/automated/stream_name/get_types.rb b/test/automated/stream_name/get_types.rb deleted file mode 100644 index 2d00211..0000000 --- a/test/automated/stream_name/get_types.rb +++ /dev/null @@ -1,30 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Get Types" do - context "Many Types" do - test "Types are the list of elements following a colon separator and preceding the ID" do - stream_name = "someStream:someType+someOtherType" - - types = StreamName.get_types(stream_name) - - assert(types == ['someType', 'someOtherType']) - end - end - - context "Single Type" do - test "Types are the list of elements following a colon separator and preceding the ID" do - stream_name = "someStream:someType" - - types = StreamName.get_types(stream_name) - - assert(types == ['someType']) - end - end - - test "Is empty if there is no type list in the stream name" do - types = StreamName.get_types('someStream') - assert(types.empty?) - end - end -end diff --git a/test/automated/stream_name/is_category.rb b/test/automated/stream_name/is_category.rb deleted file mode 100644 index f4032cd..0000000 --- a/test/automated/stream_name/is_category.rb +++ /dev/null @@ -1,26 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Is Category" do - category = 'someStream' - - context "Stream Name Contains a Dash (-)" do - id = Identifier::UUID.random - stream_name = "#{category}-#{id}" - - is_category = StreamName.category?(stream_name) - - test "Not a category" do - refute(is_category) - end - end - - context "Stream Name Contains no Dash (-)" do - is_category = StreamName.category?(category) - - test "Is a category" do - assert(is_category) - end - end - end -end diff --git a/test/automated/stream_name/stream_name.rb b/test/automated/stream_name/stream_name.rb deleted file mode 100644 index 82475db..0000000 --- a/test/automated/stream_name/stream_name.rb +++ /dev/null @@ -1,51 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Category" do - stream_name = StreamName.stream_name('someCategory') - - test "Stream name is the category" do - assert(stream_name == 'someCategory') - end - end - - context "Category and ID" do - stream_name = StreamName.stream_name('someCategory', 'some_id') - - test "Stream name is the category and the ID" do - assert(stream_name == 'someCategory-some_id') - end - end - - context "Category and Type" do - stream_name = StreamName.stream_name('someCategory', type: 'someType') - - test "Stream name is the category and the Type" do - assert(stream_name == 'someCategory:someType') - end - end - - context "Category and Types" do - stream_name = StreamName.stream_name('someCategory', types: ['someType', 'someOtherType']) - - test "Stream name is the category and the types delimited by the plus (+) sign" do - assert(stream_name == 'someCategory:someType+someOtherType') - end - end - - context "Category, Type, and Types" do - stream_name = StreamName.stream_name('someCategory', type: 'someType', types: ['someOtherType', 'yetAnotherYet']) - - test "Stream name is the category and the types delimited by the plus (+) sign" do - assert(stream_name == 'someCategory:someType+someOtherType+yetAnotherYet') - end - end - - context "Category, Type and ID" do - stream_name = StreamName.stream_name('someCategory', 'some_id', type: 'someType') - - test "Stream name is the category, type and ID" do - assert(stream_name == 'someCategory:someType-some_id') - end - end -end From 7657d9d5365f8ca7e8b37eedeab9cc6c6f491140 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Nov 2019 01:07:13 -0500 Subject: [PATCH 500/591] Package version is increased from 1.1.0.0 to 1.2.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 31789ce..9d4e6ed 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '1.1.0.0' + s.version = '1.2.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 989eab4835d3cc9cf80e68a5e686cbc3e3df4345 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Nov 2019 02:40:17 -0600 Subject: [PATCH 501/591] Get benchmark is corrected for new Get interface --- lib/message_store/postgres/get.rb | 8 +++++--- test/benchmark/defaults.rb | 2 +- test/benchmark/get.rb | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 4a30c08..16e1b7f 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -45,11 +45,13 @@ def self.call(stream_name, position: nil, batch_size: nil, correlation: nil, con end module Call - def call(position) - logger.trace(tag: :get) { "Getting message data (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } - + def call(position=nil, stream_name: nil) position ||= Defaults.position + stream_name ||= self.stream_name + + logger.trace(tag: :get) { "Getting message data (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } + result = get_result(stream_name, position) message_data = convert(result) diff --git a/test/benchmark/defaults.rb b/test/benchmark/defaults.rb index 2aea3b3..7d6e8f5 100644 --- a/test/benchmark/defaults.rb +++ b/test/benchmark/defaults.rb @@ -33,7 +33,7 @@ def self.gc end def self.stream_name - ENV['STREAM_NAME'] + ENV['STREAM_NAME'] || Controls::StreamName.example end end end diff --git a/test/benchmark/get.rb b/test/benchmark/get.rb index daa578c..1287995 100644 --- a/test/benchmark/get.rb +++ b/test/benchmark/get.rb @@ -11,12 +11,12 @@ put.(entry.message_data, stream_name) end -get = Get.build +get = Get.build('something') result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] stream_name = defaults.stream_name || entry.stream_name - get.(stream_name) + get.(0, stream_name: stream_name) end puts From 032ba66e9a1e4c5149d3a359a6883e017101fb2b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 6 Nov 2019 02:56:01 -0600 Subject: [PATCH 502/591] Benchmarks --- ...1-30 revision 66950) [x86_64-darwin17].txt | 23 +++++++++++++++++++ ...1-30 revision 66950) [x86_64-darwin17].txt | 23 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 test/benchmark/results/Get Benchmark - 2019-11-06T08-49-54.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt create mode 100644 test/benchmark/results/Put Benchmark - 2019-11-06T07-53-35.642Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Get Benchmark - 2019-11-06T08-49-54.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt b/test/benchmark/results/Get Benchmark - 2019-11-06T08-49-54.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt new file mode 100644 index 0000000..d22ea2e --- /dev/null +++ b/test/benchmark/results/Get Benchmark - 2019-11-06T08-49-54.515Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt @@ -0,0 +1,23 @@ +Get Benchmark +- - - +Cycles: 100000 +Time: 61260.112000ms +Mean Cycle Time: 0.612601ms (± 0.939958ms) +Cycles Per Second: 1632.383565 +GC: off + +Note: After implementation of correlation in the message store + +ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2019-11-06T07-53-35.642Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt b/test/benchmark/results/Put Benchmark - 2019-11-06T07-53-35.642Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt new file mode 100644 index 0000000..1830a6a --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2019-11-06T07-53-35.642Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt @@ -0,0 +1,23 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 139714.202000ms +Mean Cycle Time: 1.397142ms (± 0.755262ms) +Cycles Per Second: 715.746850 +GC: off + +Note: After implementation of correlation in the message store + +ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 From 2cf8fbf76d2724e2524a24c8d82dab58ce99b334 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 9 Nov 2019 00:07:17 -0600 Subject: [PATCH 503/591] Test context name is corrected --- test/automated/get/category/correlation.rb | 2 +- test/automated/get/stream/correlation.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/automated/get/category/correlation.rb b/test/automated/get/category/correlation.rb index 44ed938..869f4b7 100644 --- a/test/automated/get/category/correlation.rb +++ b/test/automated/get/category/correlation.rb @@ -2,7 +2,7 @@ context "Get" do context "Category" do - context "Condition" do + context "Correlation" do correlation_category = Controls::Category.example correlation_stream_name = Controls::StreamName.example(category: correlation_category) diff --git a/test/automated/get/stream/correlation.rb b/test/automated/get/stream/correlation.rb index 4befb61..c333bdf 100644 --- a/test/automated/get/stream/correlation.rb +++ b/test/automated/get/stream/correlation.rb @@ -2,7 +2,7 @@ context "Get" do context "Stream" do - context "Condition" do + context "Correlation" do correlation_category = Controls::Category.example correlation_stream_name = Controls::StreamName.example(category: correlation_category) From 81112dce6bcaa5505a3a6250d63e3855f16cbffd Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 9 Nov 2019 00:26:43 -0600 Subject: [PATCH 504/591] Correlation error raised by the database is caught and converted to a correlation error --- lib/message_store/postgres/get.rb | 16 +++++++++++++++- .../get/category/correlation_error.rb | 19 +++++++++++++++++++ .../automated/get/stream/correlation_error.rb | 19 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 test/automated/get/category/correlation_error.rb create mode 100644 test/automated/get/stream/correlation_error.rb diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 16e1b7f..e81cf64 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -78,7 +78,11 @@ def get_result(stream_name, position) cond ] - result = session.execute(sql_command, params) + begin + result = session.execute(sql_command, params) + rescue PG::RaiseException => e + raise_error(e) + end logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } @@ -113,6 +117,16 @@ def convert(result) message_data end + def raise_error(pg_error) + error_message = pg_error.message + if error_message.include?('Correlation must be a category') + error_message.gsub!('ERROR:', '').strip! + logger.error { error_message } + raise Correlation::Error, error_message + end + raise pg_error + end + def self.specialization(stream_name) if StreamName.category?(stream_name) Category diff --git a/test/automated/get/category/correlation_error.rb b/test/automated/get/category/correlation_error.rb new file mode 100644 index 0000000..aadb689 --- /dev/null +++ b/test/automated/get/category/correlation_error.rb @@ -0,0 +1,19 @@ +require_relative '../../automated_init' + +context "Get" do + context "Category" do + context "Correlation" do + context "Not a Category" do + correlation = Controls::StreamName.example + + stream_name = Controls::StreamName.example + + test "Is an error" do + assert_raises MessageStore::Correlation::Error do + Get.(stream_name, correlation: correlation) + end + end + end + end + end +end diff --git a/test/automated/get/stream/correlation_error.rb b/test/automated/get/stream/correlation_error.rb new file mode 100644 index 0000000..298eadb --- /dev/null +++ b/test/automated/get/stream/correlation_error.rb @@ -0,0 +1,19 @@ +require_relative '../../automated_init' + +context "Get" do + context "Stream" do + context "Correlation" do + context "Not a Category" do + correlation = Controls::StreamName.example + + stream_name = Controls::StreamName.example + + test "Is an error" do + assert_raises MessageStore::Correlation::Error do + Get.(stream_name, correlation: correlation) + end + end + end + end + end +end From e8eb22965fc6157ce66dd55fe39cb8982ced4ef9 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 9 Nov 2019 13:49:45 -0600 Subject: [PATCH 505/591] Get::Last tests are in the stream namespace --- .../automated/get/stream/get_last/get_last.rb | 19 +++++++++++++++++++ .../get/stream/get_last/no_messages.rb | 17 +++++++++++++++++ test/automated/get_last/get_last.rb | 15 --------------- test/automated/get_last/no_messages.rb | 13 ------------- 4 files changed, 36 insertions(+), 28 deletions(-) create mode 100644 test/automated/get/stream/get_last/get_last.rb create mode 100644 test/automated/get/stream/get_last/no_messages.rb delete mode 100644 test/automated/get_last/get_last.rb delete mode 100644 test/automated/get_last/no_messages.rb diff --git a/test/automated/get/stream/get_last/get_last.rb b/test/automated/get/stream/get_last/get_last.rb new file mode 100644 index 0000000..e94becc --- /dev/null +++ b/test/automated/get/stream/get_last/get_last.rb @@ -0,0 +1,19 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Last" do + stream_name, _ = Controls::Put.(instances: 2) + + write_message = Controls::MessageData::Write.example + + position = Put.(write_message, stream_name) + + last_message = Get::Stream::Last.(stream_name) + + test "Gets the last message in the stream" do + assert(last_message.data == write_message.data) + end + end + end +end diff --git a/test/automated/get/stream/get_last/no_messages.rb b/test/automated/get/stream/get_last/no_messages.rb new file mode 100644 index 0000000..7c07934 --- /dev/null +++ b/test/automated/get/stream/get_last/no_messages.rb @@ -0,0 +1,17 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Last" do + context "No Messages" do + stream_name = Controls::StreamName.example + + last_message = Get::Stream::Last.(stream_name) + + test "Nil message" do + assert(last_message.nil?) + end + end + end + end +end diff --git a/test/automated/get_last/get_last.rb b/test/automated/get_last/get_last.rb deleted file mode 100644 index 15ede65..0000000 --- a/test/automated/get_last/get_last.rb +++ /dev/null @@ -1,15 +0,0 @@ -require_relative '../automated_init' - -context "Get Last" do - stream_name, _ = Controls::Put.(instances: 2) - - write_message = Controls::MessageData::Write.example - - position = Put.(write_message, stream_name) - - last_message = Get::Stream::Last.(stream_name) - - test "Gets the last message in the stream" do - assert(last_message.data == write_message.data) - end -end diff --git a/test/automated/get_last/no_messages.rb b/test/automated/get_last/no_messages.rb deleted file mode 100644 index 344dd4c..0000000 --- a/test/automated/get_last/no_messages.rb +++ /dev/null @@ -1,13 +0,0 @@ -require_relative '../automated_init' - -context "Get Last" do - context "No Messages" do - stream_name = Controls::StreamName.example - - last_message = Get::Stream::Last.(stream_name) - - test "Nil message" do - assert(last_message.nil?) - end - end -end From 90f3d9e62727ad7b0d522b734f802e1b10f6832d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 9 Nov 2019 13:54:52 -0600 Subject: [PATCH 506/591] Forcing merge --- test/automated/get/stream/get_last/get_last.rb | 1 + test/automated/get/stream/get_last/no_messages.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/test/automated/get/stream/get_last/get_last.rb b/test/automated/get/stream/get_last/get_last.rb index e94becc..612da6e 100644 --- a/test/automated/get/stream/get_last/get_last.rb +++ b/test/automated/get/stream/get_last/get_last.rb @@ -17,3 +17,4 @@ end end end + diff --git a/test/automated/get/stream/get_last/no_messages.rb b/test/automated/get/stream/get_last/no_messages.rb index 7c07934..998d03d 100644 --- a/test/automated/get/stream/get_last/no_messages.rb +++ b/test/automated/get/stream/get_last/no_messages.rb @@ -15,3 +15,4 @@ end end end + From d513c9485004b826718004850e3ad1e53d38947b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 9 Nov 2019 15:17:33 -0600 Subject: [PATCH 507/591] Position defaults for category and stream are provided by their individual implementations --- lib/message_store/postgres/get.rb | 6 +----- lib/message_store/postgres/get/category.rb | 6 ++++++ lib/message_store/postgres/get/stream.rb | 6 ++++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index e81cf64..4ea8a9f 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -46,7 +46,7 @@ def self.call(stream_name, position: nil, batch_size: nil, correlation: nil, con module Call def call(position=nil, stream_name: nil) - position ||= Defaults.position + position ||= self.class::Defaults.position stream_name ||= self.stream_name @@ -154,10 +154,6 @@ def self.utc_coerced(local_time) end module Defaults - def self.position - 0 - end - def self.batch_size 1000 end diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index fd802bc..5992640 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -11,6 +11,12 @@ def self.sql_command def last_position(batch) batch.last.global_position end + + module Defaults + def self.position + 1 + end + end end end end diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index 788d0f2..a87caa4 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -11,6 +11,12 @@ def self.sql_command def last_position(batch) batch.last.position end + + module Defaults + def self.position + 0 + end + end end end end From 71b4732bc3944b4617cbf90a4b9979e9d5bbfd73 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 17 Nov 2019 12:53:24 -0600 Subject: [PATCH 508/591] SQL command and parameters are specialized per strategy --- lib/message_store/postgres/get.rb | 46 ++++++++++--------- lib/message_store/postgres/get/category.rb | 19 +++++++- lib/message_store/postgres/get/stream.rb | 18 +++++++- .../{get_message.rb => get_messages.rb} | 0 4 files changed, 58 insertions(+), 25 deletions(-) rename test/automated/get/stream/{get_message.rb => get_messages.rb} (100%) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 4ea8a9f..177a63e 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -7,11 +7,14 @@ def self.included(cls) prepend Call prepend BatchSize - extend Parameters - dependency :session, Session initializer :stream_name, na(:batch_size), :correlation, :condition + + abstract :sql_command + abstract :parameter_names + abstract :parameter_values + abstract :last_position end end @@ -66,20 +69,24 @@ def call(position=nil, stream_name: nil) def get_result(stream_name, position) logger.trace(tag: :get) { "Getting result (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } - sql_command = self.class.sql_command +## sql_command = self.class.sql_command - cond = Get.constrain_condition(condition) +## cond = Get.constrain_condition(condition) - params = [ - stream_name, - position, - batch_size, - correlation, - cond - ] +## + # params = [ + # stream_name, + # position, + # batch_size, + # correlation, + # cond + # ] + + parameter_values = parameter_values(stream_name, position) begin - result = session.execute(sql_command, params) +## result = session.execute(sql_command, params) + result = session.execute(sql_command, parameter_values) rescue PG::RaiseException => e raise_error(e) end @@ -89,17 +96,12 @@ def get_result(stream_name, position) result end - def self.constrain_condition(condition) - return nil if condition.nil? +## + # def self.constrain_condition(condition) + # return nil if condition.nil? - "(#{condition})" - end - - module Parameters - def parameters - '$1::varchar, $2::bigint, $3::bigint, $4::varchar, $5::varchar' - end - end + # "(#{condition})" + # end def convert(result) logger.trace(tag: :get) { "Converting result to message data (Result Count: #{result.ntuples})" } diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 5992640..8b7714c 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -4,10 +4,27 @@ module Get class Category include Get - def self.sql_command +## def self.sql_command + def sql_command "SELECT * FROM get_category_messages(#{parameters});" end + def parameter_names + '$1::varchar, $2::bigint, $3::bigint, $4::varchar, $5::bigint, $6::bigint, $7::varchar' + end + + def parameter_values(stream_name, position) + [ + stream_name, + position, + batch_size, + correlation, + consumer_group_member, + consumer_group_size, + condition + ] + end + def last_position(batch) batch.last.global_position end diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index a87caa4..c6b994b 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -4,8 +4,22 @@ module Get class Stream include Get - def self.sql_command - "SELECT * FROM get_stream_messages(#{parameters});" + def sql_command + "SELECT * FROM get_stream_messages(#{parameter_names});" + end + + def parameter_names + '$1::varchar, $2::bigint, $3::bigint, $4::varchar, $5::varchar' + end + + def parameter_values(stream_name, position) + [ + stream_name, + position, + batch_size, + correlation, + condition + ] end def last_position(batch) diff --git a/test/automated/get/stream/get_message.rb b/test/automated/get/stream/get_messages.rb similarity index 100% rename from test/automated/get/stream/get_message.rb rename to test/automated/get/stream/get_messages.rb From 665f7d6b8a0ed031158209acf3398f44971acbd4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 17 Nov 2019 17:27:37 -0600 Subject: [PATCH 509/591] Get implementations are specialized for variant parameters --- lib/message_store/postgres/get.rb | 41 +++++----------------- lib/message_store/postgres/get/category.rb | 9 +++-- lib/message_store/postgres/get/stream.rb | 10 ++++-- 3 files changed, 24 insertions(+), 36 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 177a63e..11f8759 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -9,10 +9,8 @@ def self.included(cls) dependency :session, Session - initializer :stream_name, na(:batch_size), :correlation, :condition - abstract :sql_command - abstract :parameter_names + abstract :parameters abstract :parameter_values abstract :last_position end @@ -24,17 +22,17 @@ def batch_size end end - def self.build(stream_name, batch_size: nil, correlation: nil, condition: nil, session: nil) + def self.build(stream_name, session: nil, **args) cls = specialization(stream_name) - cls.new(stream_name, batch_size, correlation, condition).tap do |instance| + cls.build(stream_name, **args).tap do |instance| instance.configure(session: session) end end - def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, correlation: nil, condition: nil, session: nil) + def self.configure(receiver, stream_name, attr_name: nil, session: nil, **args) attr_name ||= :get - instance = build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) + instance = build(stream_name, session: session, **args) receiver.public_send("#{attr_name}=", instance) end @@ -53,13 +51,13 @@ def call(position=nil, stream_name: nil) stream_name ||= self.stream_name - logger.trace(tag: :get) { "Getting message data (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } + logger.trace(tag: :get) { "Getting message data (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" } result = get_result(stream_name, position) message_data = convert(result) - logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } + logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" } logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } message_data @@ -67,42 +65,21 @@ def call(position=nil, stream_name: nil) end def get_result(stream_name, position) - logger.trace(tag: :get) { "Getting result (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } - -## sql_command = self.class.sql_command - -## cond = Get.constrain_condition(condition) - -## - # params = [ - # stream_name, - # position, - # batch_size, - # correlation, - # cond - # ] + logger.trace(tag: :get) { "Getting result (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" } parameter_values = parameter_values(stream_name, position) begin -## result = session.execute(sql_command, params) result = session.execute(sql_command, parameter_values) rescue PG::RaiseException => e raise_error(e) end - logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition || '(none)'}, Correlation: #{correlation || '(none)'})" } + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" } result end -## - # def self.constrain_condition(condition) - # return nil if condition.nil? - - # "(#{condition})" - # end - def convert(result) logger.trace(tag: :get) { "Converting result to message data (Result Count: #{result.ntuples})" } diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 8b7714c..6a32797 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -4,12 +4,17 @@ module Get class Category include Get -## def self.sql_command + initializer :stream_name, na(:batch_size), :correlation, :consumer_group_member, :consumer_group_size, :condition + + def self.build(stream_name, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil) + new(stream_name, batch_size, correlation, consumer_group_member, consumer_group_size, condition) + end + def sql_command "SELECT * FROM get_category_messages(#{parameters});" end - def parameter_names + def parameters '$1::varchar, $2::bigint, $3::bigint, $4::varchar, $5::bigint, $6::bigint, $7::varchar' end diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index c6b994b..2288b82 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -4,11 +4,17 @@ module Get class Stream include Get + initializer :stream_name, na(:batch_size), :correlation, :condition + + def self.build(stream_name, batch_size: nil, correlation: nil, condition: nil) + new(stream_name, batch_size, correlation, condition) + end + def sql_command - "SELECT * FROM get_stream_messages(#{parameter_names});" + "SELECT * FROM get_stream_messages(#{parameters});" end - def parameter_names + def parameters '$1::varchar, $2::bigint, $3::bigint, $4::varchar, $5::varchar' end From 225851292da299eaeeb42c17fcfe8d4580955f29 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 17 Nov 2019 18:25:46 -0600 Subject: [PATCH 510/591] Get implementation logging is specialized --- lib/message_store/postgres/get.rb | 9 +++++---- lib/message_store/postgres/get/category.rb | 4 ++++ lib/message_store/postgres/get/stream.rb | 4 ++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 11f8759..234ae51 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -13,6 +13,7 @@ def self.included(cls) abstract :parameters abstract :parameter_values abstract :last_position + abstract :log_text end end @@ -51,13 +52,13 @@ def call(position=nil, stream_name: nil) stream_name ||= self.stream_name - logger.trace(tag: :get) { "Getting message data (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" } + logger.trace(tag: :get) { "Getting message data (#{log_text(position, stream_name)})" } result = get_result(stream_name, position) message_data = convert(result) - logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" } + logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, #{log_text(position, stream_name)})" } logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } message_data @@ -65,7 +66,7 @@ def call(position=nil, stream_name: nil) end def get_result(stream_name, position) - logger.trace(tag: :get) { "Getting result (Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" } + logger.trace(tag: :get) { "Getting result (#{log_text(position, stream_name)})" } parameter_values = parameter_values(stream_name, position) @@ -75,7 +76,7 @@ def get_result(stream_name, position) raise_error(e) end - logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" } + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, #{log_text(position, stream_name)})" } result end diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 6a32797..1663df2 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -34,6 +34,10 @@ def last_position(batch) batch.last.global_position end + def log_text(position, stream_name) + "Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Consumer Group Member: #{consumer_group_member.inspect}, Consumer Group Size: #{consumer_group_size.inspect}, Condition: #{condition.inspect})" + end + module Defaults def self.position 1 diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index 2288b82..d3bd56c 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -32,6 +32,10 @@ def last_position(batch) batch.last.position end + def log_text(position, stream_name) + "Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" + end + module Defaults def self.position 0 From 068d2da8e3fa37944dffe686fdb63a514fe2c62f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 18 Nov 2019 20:10:53 -0600 Subject: [PATCH 511/591] Batch retrieval tests are clarified --- .../category/consumer_group/consumer_group.rb | 18 ++++++++++++++++++ .../category/{ => correlation}/correlation.rb | 2 +- .../error.rb} | 2 +- test/automated/get/category/get_messages.rb | 14 ++++++++++++-- .../stream/{ => correlation}/correlation.rb | 2 +- .../error.rb} | 2 +- test/automated/get/stream/get_messages.rb | 14 ++++++++++++-- 7 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 test/automated/get/category/consumer_group/consumer_group.rb rename test/automated/get/category/{ => correlation}/correlation.rb (96%) rename test/automated/get/category/{correlation_error.rb => correlation/error.rb} (90%) rename test/automated/get/stream/{ => correlation}/correlation.rb (96%) rename test/automated/get/stream/{correlation_error.rb => correlation/error.rb} (90%) diff --git a/test/automated/get/category/consumer_group/consumer_group.rb b/test/automated/get/category/consumer_group/consumer_group.rb new file mode 100644 index 0000000..5a6d607 --- /dev/null +++ b/test/automated/get/category/consumer_group/consumer_group.rb @@ -0,0 +1,18 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Category" do + context "Consumer Group" do + + + message_data_group_0 = Get.(category, consumer_group_member: 0, consumer_group_size: 2) + message_data_group_1 = Get.(category, consumer_group_member: 1, consumer_group_size: 2) + + + + + + + end + end +end diff --git a/test/automated/get/category/correlation.rb b/test/automated/get/category/correlation/correlation.rb similarity index 96% rename from test/automated/get/category/correlation.rb rename to test/automated/get/category/correlation/correlation.rb index 869f4b7..fe38593 100644 --- a/test/automated/get/category/correlation.rb +++ b/test/automated/get/category/correlation/correlation.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Category" do diff --git a/test/automated/get/category/correlation_error.rb b/test/automated/get/category/correlation/error.rb similarity index 90% rename from test/automated/get/category/correlation_error.rb rename to test/automated/get/category/correlation/error.rb index aadb689..5355419 100644 --- a/test/automated/get/category/correlation_error.rb +++ b/test/automated/get/category/correlation/error.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Category" do diff --git a/test/automated/get/category/get_messages.rb b/test/automated/get/category/get_messages.rb index 4bbc6ed..6adfac6 100644 --- a/test/automated/get/category/get_messages.rb +++ b/test/automated/get/category/get_messages.rb @@ -8,11 +8,21 @@ Controls::Put.(category: category) Controls::Put.(category: category) - messages = Get.(category) + message_data = Get.(category) context "Messages Retrieved" do test "Messages from the all streams in the category" do - assert(messages.length == 2) + assert(message_data.length == 2) + end + + context "Message category is the category written" do + message_data.each do |md| + message_cateogry = StreamName.get_category(md.stream_name) + + test do + assert(message_cateogry == category) + end + end end end end diff --git a/test/automated/get/stream/correlation.rb b/test/automated/get/stream/correlation/correlation.rb similarity index 96% rename from test/automated/get/stream/correlation.rb rename to test/automated/get/stream/correlation/correlation.rb index c333bdf..203eb76 100644 --- a/test/automated/get/stream/correlation.rb +++ b/test/automated/get/stream/correlation/correlation.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Stream" do diff --git a/test/automated/get/stream/correlation_error.rb b/test/automated/get/stream/correlation/error.rb similarity index 90% rename from test/automated/get/stream/correlation_error.rb rename to test/automated/get/stream/correlation/error.rb index 298eadb..bfc3b74 100644 --- a/test/automated/get/stream/correlation_error.rb +++ b/test/automated/get/stream/correlation/error.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Stream" do diff --git a/test/automated/get/stream/get_messages.rb b/test/automated/get/stream/get_messages.rb index dd6ebae..e68ee53 100644 --- a/test/automated/get/stream/get_messages.rb +++ b/test/automated/get/stream/get_messages.rb @@ -10,11 +10,21 @@ stream_name, _ = Controls::Put.(category: category) Controls::Put.(category: category) - messages = Get.(stream_name) + message_data = Get.(stream_name) context "Messages Retrieved" do test "Only messages from the specific stream" do - assert(messages.length == 1) + assert(message_data.length == 1) + end + + context "Message stream is the stream written" do + message_data.each do |md| + message_stream_name = md.stream_name + + test do + assert(message_stream_name == stream_name) + end + end end end end From c64fb1982a8ee1b2aed939550321c8882e76cabc Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 18 Nov 2019 21:58:35 -0600 Subject: [PATCH 512/591] Consumer group test --- lib/message_store/postgres/get.rb | 6 +++-- .../category/consumer_group/consumer_group.rb | 18 ------------- .../consumer_groups/consumer_groups.rb | 26 +++++++++++++++++++ 3 files changed, 30 insertions(+), 20 deletions(-) delete mode 100644 test/automated/get/category/consumer_group/consumer_group.rb create mode 100644 test/automated/get/category/consumer_groups/consumer_groups.rb diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 234ae51..ca32cdf 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -41,8 +41,10 @@ def configure(session: nil) Session.configure(self, session: session) end - def self.call(stream_name, position: nil, batch_size: nil, correlation: nil, condition: nil, session: nil) - instance = build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) + # def self.call(stream_name, position: nil, batch_size: nil, correlation: nil, condition: nil, session: nil) + # def self.build(stream_name, session: nil, **args) + def self.call(stream_name, position: nil, batch_size: nil, correlation: nil, condition: nil, session: nil, **args) + instance = build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session, **args) instance.(position) end diff --git a/test/automated/get/category/consumer_group/consumer_group.rb b/test/automated/get/category/consumer_group/consumer_group.rb deleted file mode 100644 index 5a6d607..0000000 --- a/test/automated/get/category/consumer_group/consumer_group.rb +++ /dev/null @@ -1,18 +0,0 @@ -require_relative '../../../automated_init' - -context "Get" do - context "Category" do - context "Consumer Group" do - - - message_data_group_0 = Get.(category, consumer_group_member: 0, consumer_group_size: 2) - message_data_group_1 = Get.(category, consumer_group_member: 1, consumer_group_size: 2) - - - - - - - end - end -end diff --git a/test/automated/get/category/consumer_groups/consumer_groups.rb b/test/automated/get/category/consumer_groups/consumer_groups.rb new file mode 100644 index 0000000..b13fd7d --- /dev/null +++ b/test/automated/get/category/consumer_groups/consumer_groups.rb @@ -0,0 +1,26 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Category" do + context "Consumer Groups" do + category = Controls::Category.example + + instances = 11 + partitions = 3 + + instances.times do + Controls::Put.(category: category) + end + + retrieved_count = 0 + partitions.times do |i| + messages = Get.(category, consumer_group_member: i, consumer_group_size: partitions) + retrieved_count += messages.count + end + + test "Retrieve messages in partitions" do + assert(retrieved_count == instances) + end + end + end +end From 3abcc5c855ebf2d139b9166d1a7b2cd64afb01d8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 18 Nov 2019 22:23:09 -0600 Subject: [PATCH 513/591] Use of keyword args is harmonized --- lib/message_store/postgres/get.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index ca32cdf..4da7a69 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -23,17 +23,21 @@ def batch_size end end - def self.build(stream_name, session: nil, **args) + def self.build(stream_name, **args) cls = specialization(stream_name) + session = args.delete(:session) + cls.build(stream_name, **args).tap do |instance| instance.configure(session: session) end end - def self.configure(receiver, stream_name, attr_name: nil, session: nil, **args) + def self.configure(receiver, stream_name, **args) + attr_name = args.delete(:attr_name) attr_name ||= :get - instance = build(stream_name, session: session, **args) + + instance = build(stream_name, **args) receiver.public_send("#{attr_name}=", instance) end @@ -41,10 +45,9 @@ def configure(session: nil) Session.configure(self, session: session) end - # def self.call(stream_name, position: nil, batch_size: nil, correlation: nil, condition: nil, session: nil) - # def self.build(stream_name, session: nil, **args) - def self.call(stream_name, position: nil, batch_size: nil, correlation: nil, condition: nil, session: nil, **args) - instance = build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session, **args) + def self.call(stream_name, **args) + position = args.delete(:position) + instance = build(stream_name, **args) instance.(position) end From 85d104777428cedef73c7191011eb9aa0753e93c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 19 Nov 2019 07:42:58 -0600 Subject: [PATCH 514/591] Consumer group test is more thorough --- .../consumer_groups/consumer_groups.rb | 23 +++++++++++++++---- .../get/category/consumer_groups/error.rb | 1 + 2 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 test/automated/get/category/consumer_groups/error.rb diff --git a/test/automated/get/category/consumer_groups/consumer_groups.rb b/test/automated/get/category/consumer_groups/consumer_groups.rb index b13fd7d..d23b0f6 100644 --- a/test/automated/get/category/consumer_groups/consumer_groups.rb +++ b/test/automated/get/category/consumer_groups/consumer_groups.rb @@ -12,14 +12,27 @@ Controls::Put.(category: category) end - retrieved_count = 0 + retrieved_message_data = [] partitions.times do |i| - messages = Get.(category, consumer_group_member: i, consumer_group_size: partitions) - retrieved_count += messages.count + retrieved_message_data += Get.(category, consumer_group_member: i, consumer_group_size: partitions) end - test "Retrieve messages in partitions" do - assert(retrieved_count == instances) + context "Message Partitions" do + context "Distribution" do + retrieved_count = retrieved_message_data.count + + test "Across partitions" do + assert(retrieved_count == instances) + end + end + end + + context "Messages in all partitions" do + test "Are of the same category" do + retrieved_message_data.each do |message_data| + assert(StreamName.get_category(message_data.stream_name) == category) + end + end end end end diff --git a/test/automated/get/category/consumer_groups/error.rb b/test/automated/get/category/consumer_groups/error.rb new file mode 100644 index 0000000..0569fb0 --- /dev/null +++ b/test/automated/get/category/consumer_groups/error.rb @@ -0,0 +1 @@ +fail 'To do' From d12db43f133e6bceed36ebf7296e2e58cbcf6fd1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 19 Nov 2019 08:30:18 -0600 Subject: [PATCH 515/591] Consumer group error --- lib/message_store/postgres.rb | 1 + lib/message_store/postgres/get.rb | 33 ++++++++++-- lib/message_store/postgres/get/category.rb | 2 + .../postgres/get/category/consumer_group.rb | 11 ++++ lib/message_store/postgres/get/stream.rb | 6 +++ .../get/category/consumer_groups/error.rb | 51 ++++++++++++++++++- test/automated/get/stream/consumer_groups.rb | 17 +++++++ 7 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 lib/message_store/postgres/get/category/consumer_group.rb create mode 100644 test/automated/get/stream/consumer_groups.rb diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index 5c1baf4..20c1433 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -18,6 +18,7 @@ require 'message_store/postgres/get' require 'message_store/postgres/get/stream' +require 'message_store/postgres/get/category/consumer_group' require 'message_store/postgres/get/category' require 'message_store/postgres/get/stream/last' require 'message_store/postgres/read' diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 4da7a69..8041cdc 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -1,9 +1,14 @@ module MessageStore module Postgres module Get + Error = Class.new(RuntimeError) + def self.included(cls) cls.class_exec do include MessageStore::Get + + extend Assure + prepend Call prepend BatchSize @@ -26,6 +31,8 @@ def batch_size def self.build(stream_name, **args) cls = specialization(stream_name) + cls.assure(stream_name, args) + session = args.delete(:session) cls.build(stream_name, **args).tap do |instance| @@ -103,12 +110,25 @@ def convert(result) end def raise_error(pg_error) - error_message = pg_error.message - if error_message.include?('Correlation must be a category') - error_message.gsub!('ERROR:', '').strip! + error_message = pg_error.message.gsub('ERROR:', '').strip + + error_class = nil + + case + when error_message.start_with?('Correlation must be a category') + error_class = Correlation::Error + when error_message.start_with?('Consumer group size must not be less than 1') || + error_message.start_with?('Consumer group member must be less than the group size') || + error_message.start_with?('Consumer group member must not be less than 0') || + error_message.start_with?('Consumer group member and size must be specified') + error_class = Get::Category::ConsumerGroup::Error + end + + if not error_message.nil? logger.error { error_message } - raise Correlation::Error, error_message + raise error_class, error_message end + raise pg_error end @@ -138,6 +158,11 @@ def self.utc_coerced(local_time) end end + module Assure + def assure(*) + end + end + module Defaults def self.batch_size 1000 diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 1663df2..be916c1 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -2,6 +2,8 @@ module MessageStore module Postgres module Get class Category + Error = Class.new(RuntimeError) + include Get initializer :stream_name, na(:batch_size), :correlation, :consumer_group_member, :consumer_group_size, :condition diff --git a/lib/message_store/postgres/get/category/consumer_group.rb b/lib/message_store/postgres/get/category/consumer_group.rb new file mode 100644 index 0000000..457d1cc --- /dev/null +++ b/lib/message_store/postgres/get/category/consumer_group.rb @@ -0,0 +1,11 @@ +module MessageStore + module Postgres + module Get + class Category + module ConsumerGroup + Error = Class.new(RuntimeError) + end + end + end + end +end diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index d3bd56c..2040619 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -36,6 +36,12 @@ def log_text(position, stream_name) "Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" end + def self.assure(stream_name, args) + if args.include?(:consumer_group_member) || args.include?(:consumer_group_size) + raise Error, "Consumer groups are only supported for category retrieval (Stream Name: #{stream_name})" + end + end + module Defaults def self.position 0 diff --git a/test/automated/get/category/consumer_groups/error.rb b/test/automated/get/category/consumer_groups/error.rb index 0569fb0..7ad7520 100644 --- a/test/automated/get/category/consumer_groups/error.rb +++ b/test/automated/get/category/consumer_groups/error.rb @@ -1 +1,50 @@ -fail 'To do' +require_relative '../../../automated_init' + +context "Get" do + context "Category" do + context "Consumer Groups" do + context "Error" do + + context "Consumer Group Size Is Less than 1" do + test "Is an error" do + assert_raises(Get::Category::ConsumerGroup::Error) do + Get.('someCategory', consumer_group_member: 0, consumer_group_size: 0) + end + end + end + + context "Consumer Group Member Is Greater than the Consumer Group Size" do + test "Is an error" do + assert_raises(Get::Category::ConsumerGroup::Error) do + Get.('someCategory', consumer_group_member: 2, consumer_group_size: 1) + end + end + end + + context "Consumer Group Member Is Less than 0" do + test "Is an error" do + assert_raises(Get::Category::ConsumerGroup::Error) do + Get.('someCategory', consumer_group_member: -1, consumer_group_size: 1) + end + end + end + + context "Consumer Group Size is Missing" do + test "Is an error" do + assert_raises(Get::Category::ConsumerGroup::Error) do + Get.('someCategory', consumer_group_member: 0) + end + end + end + + context "Consumer Group Member is Missing" do + test "Is an error" do + assert_raises(Get::Category::ConsumerGroup::Error) do + Get.('someCategory', consumer_group_size: 1) + end + end + end + end + end + end +end diff --git a/test/automated/get/stream/consumer_groups.rb b/test/automated/get/stream/consumer_groups.rb new file mode 100644 index 0000000..caf2ded --- /dev/null +++ b/test/automated/get/stream/consumer_groups.rb @@ -0,0 +1,17 @@ +require_relative '../../automated_init' + +context "Get" do + context "Stream" do + context "Consumer Groups" do + context "Retrieving from a Stream Using Consumer Groups" do + stream_name = Controls::StreamName.example + + test "Is an error" do + assert_raises(Get::Error) do + Get.(stream_name, consumer_group_member: 0, consumer_group_size: 1) + end + end + end + end + end +end From 93c1b23778e465f65c1cbe07d18ac80a819ee980 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 20 Nov 2019 10:57:09 -0600 Subject: [PATCH 516/591] Argument assurance for stream get removes superfluous arguments --- lib/message_store/postgres/get/stream.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index 2040619..edb4350 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -37,8 +37,11 @@ def log_text(position, stream_name) end def self.assure(stream_name, args) - if args.include?(:consumer_group_member) || args.include?(:consumer_group_size) - raise Error, "Consumer groups are only supported for category retrieval (Stream Name: #{stream_name})" + consumer_group_member = args.delete(:consumer_group_member) + consumer_group_size = args.delete(:consumer_group_size) + + if [consumer_group_member, consumer_group_size].compact.length > 0 + raise Error, "Consumer groups are supported only for category retrieval (Stream Name: #{stream_name})" end end From 4d22edbeca8a9f02c4784cb6051dc28c87670b85 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Nov 2019 01:05:58 -0500 Subject: [PATCH 517/591] StreamName module is vestigial. It is removed. --- lib/message_store/postgres.rb | 2 - lib/message_store/postgres/stream_name.rb | 52 ----------------- test/automated/stream_name/get_category.rb | 57 ------------------- test/automated/stream_name/get_entity_name.rb | 55 ------------------ test/automated/stream_name/get_id.rb | 19 ------- test/automated/stream_name/get_types.rb | 30 ---------- test/automated/stream_name/is_category.rb | 26 --------- test/automated/stream_name/stream_name.rb | 51 ----------------- 8 files changed, 292 deletions(-) delete mode 100644 lib/message_store/postgres/stream_name.rb delete mode 100644 test/automated/stream_name/get_category.rb delete mode 100644 test/automated/stream_name/get_entity_name.rb delete mode 100644 test/automated/stream_name/get_id.rb delete mode 100644 test/automated/stream_name/get_types.rb delete mode 100644 test/automated/stream_name/is_category.rb delete mode 100644 test/automated/stream_name/stream_name.rb diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index 20c1433..63fb6f4 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -11,8 +11,6 @@ require 'message_store/postgres/settings' require 'message_store/postgres/session' -require 'message_store/postgres/stream_name' - require 'message_store/postgres/put' require 'message_store/postgres/write' diff --git a/lib/message_store/postgres/stream_name.rb b/lib/message_store/postgres/stream_name.rb deleted file mode 100644 index 29e6cd6..0000000 --- a/lib/message_store/postgres/stream_name.rb +++ /dev/null @@ -1,52 +0,0 @@ -module MessageStore - module Postgres - module StreamName - def self.stream_name(category_name, id=nil, type: nil, types: nil) - types = Array(types) - types.unshift(type) unless type.nil? - - type_list = nil - type_list = types.join('+') unless types.empty? - - stream_name = category_name - stream_name = "#{stream_name}:#{type_list}" unless type_list.nil? - stream_name = "#{stream_name}-#{id}" unless id.nil? - - stream_name - end - - def self.get_id(stream_name) - id = stream_name.partition('-').last - id.empty? ? nil : id - end - - def self.get_category(stream_name) - stream_name.split('-').first - end - - def self.category?(stream_name) - !stream_name.include?('-') - end - - def self.get_type_list(stream_name) - type = stream_name.split(':').last.split('-').first - - return nil if stream_name.start_with?(type) - - type - end - - def self.get_types(stream_name) - type_list = get_type_list(stream_name) - - return [] if type_list.nil? - - type_list.split('+') - end - - def self.get_entity_name(stream_name) - get_category(stream_name).split(':').first - end - end - end -end diff --git a/test/automated/stream_name/get_category.rb b/test/automated/stream_name/get_category.rb deleted file mode 100644 index e13742c..0000000 --- a/test/automated/stream_name/get_category.rb +++ /dev/null @@ -1,57 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Get Category" do - category = 'someStream' - - context "Stream Name Contains an ID" do - id = Identifier::UUID.random - stream_name = "#{category}-#{id}" - - stream_category = StreamName.get_category(stream_name) - - test "Category name is the part of the stream name before the first dash" do - assert(stream_category == category) - end - end - - context "Stream Name Contains no ID" do - stream_name = category - stream_category = StreamName.get_category(stream_name) - - test "Category name is the stream name" do - assert(stream_category == category) - end - end - - context "Stream Name Contains Type" do - stream_name = "#{category}:someType" - stream_category = StreamName.get_category(stream_name) - - test "Category name is the stream name" do - assert(stream_category == stream_name) - end - end - - context "Stream Name Contains Types" do - stream_name = "#{category}:someType+someOtherType" - stream_category = StreamName.get_category(stream_name) - - test "Category name is the stream name" do - assert(stream_category == stream_name) - end - end - - context "Stream Name Contains ID and Types" do - id = Identifier::UUID.random - category_and_types = "#{category}:someType+someOtherType" - stream_name = "#{category_and_types}-#{id}" - - stream_category = StreamName.get_category(stream_name) - - test "Category name is the stream name" do - assert(stream_category == category_and_types) - end - end - end -end diff --git a/test/automated/stream_name/get_entity_name.rb b/test/automated/stream_name/get_entity_name.rb deleted file mode 100644 index 67292eb..0000000 --- a/test/automated/stream_name/get_entity_name.rb +++ /dev/null @@ -1,55 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Get Entity Name" do - category = 'someStream' - - context "Stream Name Contains an ID" do - id = Identifier::UUID.random - stream_name = "#{category}-#{id}" - - entity_name = StreamName.get_entity_name(stream_name) - - test "Entity name is the part of the stream name before the first dash" do - assert(entity_name == category) - end - end - - context "Stream Name Contains no ID" do - stream_name = category - entity_name = StreamName.get_entity_name(stream_name) - - test "Entity name is the stream name" do - assert(entity_name == category) - end - end - - context "Stream Name Contains Type" do - stream_name = "#{category}:someType" - entity_name = StreamName.get_entity_name(stream_name) - - test "Entity name is the category without any types" do - assert(entity_name == category) - end - end - - context "Stream Name Contains Types" do - stream_name = "#{category}:someType+someOtherType" - entity_name = StreamName.get_entity_name(stream_name) - - test "Entity name is the category without any types" do - assert(entity_name == category) - end - end - - context "Stream Name Contains ID and Types" do - id = Identifier::UUID.random - stream_name = "#{category}:someType+someOtherType-#{id}" - entity_name = StreamName.get_entity_name(stream_name) - - test "Entity name is the category without any types" do - assert(entity_name == category) - end - end - end -end diff --git a/test/automated/stream_name/get_id.rb b/test/automated/stream_name/get_id.rb deleted file mode 100644 index e8fa9d8..0000000 --- a/test/automated/stream_name/get_id.rb +++ /dev/null @@ -1,19 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Get ID" do - test "Is the part of a stream name after the first dash" do - id = Identifier::UUID.random - stream_name = "someStream-#{id}" - - stream_id = StreamName.get_id(stream_name) - - assert(stream_id == id) - end - - test "Is nil if there is no ID part in the stream name" do - stream_id = StreamName.get_id('someStream') - assert(stream_id.nil?) - end - end -end diff --git a/test/automated/stream_name/get_types.rb b/test/automated/stream_name/get_types.rb deleted file mode 100644 index 2d00211..0000000 --- a/test/automated/stream_name/get_types.rb +++ /dev/null @@ -1,30 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Get Types" do - context "Many Types" do - test "Types are the list of elements following a colon separator and preceding the ID" do - stream_name = "someStream:someType+someOtherType" - - types = StreamName.get_types(stream_name) - - assert(types == ['someType', 'someOtherType']) - end - end - - context "Single Type" do - test "Types are the list of elements following a colon separator and preceding the ID" do - stream_name = "someStream:someType" - - types = StreamName.get_types(stream_name) - - assert(types == ['someType']) - end - end - - test "Is empty if there is no type list in the stream name" do - types = StreamName.get_types('someStream') - assert(types.empty?) - end - end -end diff --git a/test/automated/stream_name/is_category.rb b/test/automated/stream_name/is_category.rb deleted file mode 100644 index f4032cd..0000000 --- a/test/automated/stream_name/is_category.rb +++ /dev/null @@ -1,26 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Is Category" do - category = 'someStream' - - context "Stream Name Contains a Dash (-)" do - id = Identifier::UUID.random - stream_name = "#{category}-#{id}" - - is_category = StreamName.category?(stream_name) - - test "Not a category" do - refute(is_category) - end - end - - context "Stream Name Contains no Dash (-)" do - is_category = StreamName.category?(category) - - test "Is a category" do - assert(is_category) - end - end - end -end diff --git a/test/automated/stream_name/stream_name.rb b/test/automated/stream_name/stream_name.rb deleted file mode 100644 index 82475db..0000000 --- a/test/automated/stream_name/stream_name.rb +++ /dev/null @@ -1,51 +0,0 @@ -require_relative '../automated_init' - -context "Stream Name" do - context "Category" do - stream_name = StreamName.stream_name('someCategory') - - test "Stream name is the category" do - assert(stream_name == 'someCategory') - end - end - - context "Category and ID" do - stream_name = StreamName.stream_name('someCategory', 'some_id') - - test "Stream name is the category and the ID" do - assert(stream_name == 'someCategory-some_id') - end - end - - context "Category and Type" do - stream_name = StreamName.stream_name('someCategory', type: 'someType') - - test "Stream name is the category and the Type" do - assert(stream_name == 'someCategory:someType') - end - end - - context "Category and Types" do - stream_name = StreamName.stream_name('someCategory', types: ['someType', 'someOtherType']) - - test "Stream name is the category and the types delimited by the plus (+) sign" do - assert(stream_name == 'someCategory:someType+someOtherType') - end - end - - context "Category, Type, and Types" do - stream_name = StreamName.stream_name('someCategory', type: 'someType', types: ['someOtherType', 'yetAnotherYet']) - - test "Stream name is the category and the types delimited by the plus (+) sign" do - assert(stream_name == 'someCategory:someType+someOtherType+yetAnotherYet') - end - end - - context "Category, Type and ID" do - stream_name = StreamName.stream_name('someCategory', 'some_id', type: 'someType') - - test "Stream name is the category, type and ID" do - assert(stream_name == 'someCategory:someType-some_id') - end - end -end From 7a3c7c29eefe5ce263ba329473187dda9ea0a1d5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 1 Nov 2019 01:07:13 -0500 Subject: [PATCH 518/591] Package version is increased from 1.1.0.0 to 1.2.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 31789ce..9d4e6ed 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '1.1.0.0' + s.version = '1.2.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 47b7e1fd44be956e839e88392cc5f67961d19e96 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 23 Nov 2019 14:35:51 -0600 Subject: [PATCH 519/591] Get with condition requires activation --- lib/message_store/postgres.rb | 1 + lib/message_store/postgres/get.rb | 2 ++ lib/message_store/postgres/get/condition.rb | 9 +++++++ .../get/category/{ => condition}/condition.rb | 9 +++++-- .../get/category/condition/not_activated.rb | 24 +++++++++++++++++++ .../get/stream/{ => condition}/condition.rb | 9 +++++-- .../get/stream/condition/not_activated.rb | 24 +++++++++++++++++++ test/automated/read/condition.rb | 7 +++++- 8 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 lib/message_store/postgres/get/condition.rb rename test/automated/get/category/{ => condition}/condition.rb (69%) create mode 100644 test/automated/get/category/condition/not_activated.rb rename test/automated/get/stream/{ => condition}/condition.rb (66%) create mode 100644 test/automated/get/stream/condition/not_activated.rb diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index 63fb6f4..2a8b9c1 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -15,6 +15,7 @@ require 'message_store/postgres/write' require 'message_store/postgres/get' +require 'message_store/postgres/get/condition' require 'message_store/postgres/get/stream' require 'message_store/postgres/get/category/consumer_group' require 'message_store/postgres/get/category' diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 8041cdc..3392c8b 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -122,6 +122,8 @@ def raise_error(pg_error) error_message.start_with?('Consumer group member must not be less than 0') || error_message.start_with?('Consumer group member and size must be specified') error_class = Get::Category::ConsumerGroup::Error + when error_message.start_with?('Retrieval with SQL condition is not activated') + error_class = Get::Condition::Error end if not error_message.nil? diff --git a/lib/message_store/postgres/get/condition.rb b/lib/message_store/postgres/get/condition.rb new file mode 100644 index 0000000..7657c78 --- /dev/null +++ b/lib/message_store/postgres/get/condition.rb @@ -0,0 +1,9 @@ +module MessageStore + module Postgres + module Get + module Condition + Error = Class.new(RuntimeError) + end + end + end +end diff --git a/test/automated/get/category/condition.rb b/test/automated/get/category/condition/condition.rb similarity index 69% rename from test/automated/get/category/condition.rb rename to test/automated/get/category/condition/condition.rb index 8778a79..5fd4839 100644 --- a/test/automated/get/category/condition.rb +++ b/test/automated/get/category/condition/condition.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Category" do @@ -9,7 +9,12 @@ condition = 'position = 0 OR position = 2' - messages = Get.(category, batch_size: 3, condition: condition) + settings = Postgres::Settings.build + session = Session.new + settings.set(session) + session.options = '-c message_store.sql_condition=on' + + messages = Get.(category, batch_size: 3, condition: condition, session: session) message_positions = messages.map do |message| message.position diff --git a/test/automated/get/category/condition/not_activated.rb b/test/automated/get/category/condition/not_activated.rb new file mode 100644 index 0000000..9bbf27f --- /dev/null +++ b/test/automated/get/category/condition/not_activated.rb @@ -0,0 +1,24 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Category" do + context "Condition" do + context "Not Activated" do + category = Controls::Category.example + + condition = 'some condition' + + settings = Postgres::Settings.build + session = Session.new + settings.set(session) + session.options = nil + + test "Is an error" do + assert_raises(Get::Condition::Error) do + Get.(category, batch_size: 3, condition: condition, session: session) + end + end + end + end + end +end diff --git a/test/automated/get/stream/condition.rb b/test/automated/get/stream/condition/condition.rb similarity index 66% rename from test/automated/get/stream/condition.rb rename to test/automated/get/stream/condition/condition.rb index e16539c..6e6ff44 100644 --- a/test/automated/get/stream/condition.rb +++ b/test/automated/get/stream/condition/condition.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Stream" do @@ -7,7 +7,12 @@ condition = 'position = 0 OR position = 2' - messages = Get.(stream_name, batch_size: 3, condition: condition) + settings = Postgres::Settings.build + session = Session.new + settings.set(session) + session.options = '-c message_store.sql_condition=on' + + messages = Get.(stream_name, batch_size: 3, condition: condition, session: session) message_positions = messages.map do |message| message.position diff --git a/test/automated/get/stream/condition/not_activated.rb b/test/automated/get/stream/condition/not_activated.rb new file mode 100644 index 0000000..f5087d6 --- /dev/null +++ b/test/automated/get/stream/condition/not_activated.rb @@ -0,0 +1,24 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Category" do + context "Condition" do + context "Not Activated" do + stream_name = Controls::StreamName.example + + condition = 'some condition' + + settings = Postgres::Settings.build + session = Session.new + settings.set(session) + session.options = nil + + test "Is an error" do + assert_raises(Get::Condition::Error) do + Get.(stream_name, batch_size: 3, condition: condition, session: session) + end + end + end + end + end +end diff --git a/test/automated/read/condition.rb b/test/automated/read/condition.rb index 44a8214..9113e67 100644 --- a/test/automated/read/condition.rb +++ b/test/automated/read/condition.rb @@ -8,7 +8,12 @@ message_count = 0 - Read.(stream_name, condition: condition) do + settings = Postgres::Settings.build + session = Session.new + settings.set(session) + session.options = '-c message_store.sql_condition=on' + + Read.(stream_name, condition: condition, session: session) do message_count += 1 end From b95a221c07f6748aaff82e1f3d4def97114e846e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 26 Nov 2019 13:38:46 -0600 Subject: [PATCH 520/591] The get_last_message server function is renamed to get_last_stream_message --- lib/message_store/postgres/get/stream/last.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/get/stream/last.rb b/lib/message_store/postgres/get/stream/last.rb index 384bf41..b807ec5 100644 --- a/lib/message_store/postgres/get/stream/last.rb +++ b/lib/message_store/postgres/get/stream/last.rb @@ -47,7 +47,7 @@ def get_result(stream_name) def self.sql_command(stream_name) parameters = '$1::varchar' - "SELECT * FROM get_last_message(#{parameters});" + "SELECT * FROM get_last_stream_message(#{parameters});" end def convert(record) From d07bc55fd24ef8f9f273880c2e1ff33a730e11a1 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 27 Nov 2019 14:37:27 -0600 Subject: [PATCH 521/591] Position and stream name parameter order is reversed --- lib/message_store/postgres/get.rb | 8 ++++---- lib/message_store/postgres/get/category.rb | 2 +- lib/message_store/postgres/get/stream.rb | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 3392c8b..13f86b4 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -64,13 +64,13 @@ def call(position=nil, stream_name: nil) stream_name ||= self.stream_name - logger.trace(tag: :get) { "Getting message data (#{log_text(position, stream_name)})" } + logger.trace(tag: :get) { "Getting message data (#{log_text(stream_name, position)})" } result = get_result(stream_name, position) message_data = convert(result) - logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, #{log_text(position, stream_name)})" } + logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, #{log_text(stream_name, position)})" } logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } message_data @@ -78,7 +78,7 @@ def call(position=nil, stream_name: nil) end def get_result(stream_name, position) - logger.trace(tag: :get) { "Getting result (#{log_text(position, stream_name)})" } + logger.trace(tag: :get) { "Getting result (#{log_text(stream_name, position)})" } parameter_values = parameter_values(stream_name, position) @@ -88,7 +88,7 @@ def get_result(stream_name, position) raise_error(e) end - logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, #{log_text(position, stream_name)})" } + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, #{log_text(stream_name, position)})" } result end diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index be916c1..4d54950 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -36,7 +36,7 @@ def last_position(batch) batch.last.global_position end - def log_text(position, stream_name) + def log_text(stream_name, position) "Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Consumer Group Member: #{consumer_group_member.inspect}, Consumer Group Size: #{consumer_group_size.inspect}, Condition: #{condition.inspect})" end diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index edb4350..239f5a1 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -32,7 +32,7 @@ def last_position(batch) batch.last.position end - def log_text(position, stream_name) + def log_text(stream_name, position) "Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" end From 78de279930b395b1c86ae5d2f14a4807e4f213c7 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 27 Nov 2019 14:43:37 -0600 Subject: [PATCH 522/591] Get category's internal state is category rather than stream_name --- lib/message_store/postgres/get.rb | 1 + lib/message_store/postgres/get/category.rb | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 13f86b4..effa418 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -14,6 +14,7 @@ def self.included(cls) dependency :session, Session + abstract :stream_name abstract :sql_command abstract :parameters abstract :parameter_values diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 4d54950..007ab53 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -6,10 +6,11 @@ class Category include Get - initializer :stream_name, na(:batch_size), :correlation, :consumer_group_member, :consumer_group_size, :condition + initializer :category, na(:batch_size), :correlation, :consumer_group_member, :consumer_group_size, :condition + alias :stream_name :category - def self.build(stream_name, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil) - new(stream_name, batch_size, correlation, consumer_group_member, consumer_group_size, condition) + def self.build(category, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil) + new(category, batch_size, correlation, consumer_group_member, consumer_group_size, condition) end def sql_command @@ -20,9 +21,9 @@ def parameters '$1::varchar, $2::bigint, $3::bigint, $4::varchar, $5::bigint, $6::bigint, $7::varchar' end - def parameter_values(stream_name, position) + def parameter_values(category, position) [ - stream_name, + category, position, batch_size, correlation, @@ -36,8 +37,8 @@ def last_position(batch) batch.last.global_position end - def log_text(stream_name, position) - "Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Consumer Group Member: #{consumer_group_member.inspect}, Consumer Group Size: #{consumer_group_size.inspect}, Condition: #{condition.inspect})" + def log_text(category, position) + "Category: #{category}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Consumer Group Member: #{consumer_group_member.inspect}, Consumer Group Size: #{consumer_group_size.inspect}, Condition: #{condition.inspect})" end module Defaults From 71d6c18f3e4a18853832bd6c9f6175650f4632e8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 27 Nov 2019 21:34:24 -0600 Subject: [PATCH 523/591] Concrete category and stream implementations --- lib/message_store/postgres/get.rb | 1 + lib/message_store/postgres/get/category.rb | 21 +++++++++- lib/message_store/postgres/get/stream.rb | 23 ++++++++-- .../get/category/concrete/dependency.rb | 38 +++++++++++++++++ .../get/category/concrete/get_messages.rb | 32 ++++++++++++++ .../get/stream/concrete/dependency.rb | 42 +++++++++++++++++++ .../get/stream/concrete/get_messages.rb | 34 +++++++++++++++ ...mer_groups.rb => consumer_groups_error.rb} | 0 8 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 test/automated/get/category/concrete/dependency.rb create mode 100644 test/automated/get/category/concrete/get_messages.rb create mode 100644 test/automated/get/stream/concrete/dependency.rb create mode 100644 test/automated/get/stream/concrete/get_messages.rb rename test/automated/get/stream/{consumer_groups.rb => consumer_groups_error.rb} (100%) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index effa418..9f9222d 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -14,6 +14,7 @@ def self.included(cls) dependency :session, Session + abstract :configure abstract :stream_name abstract :sql_command abstract :parameters diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 007ab53..654a4f7 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -9,8 +9,25 @@ class Category initializer :category, na(:batch_size), :correlation, :consumer_group_member, :consumer_group_size, :condition alias :stream_name :category - def self.build(category, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil) - new(category, batch_size, correlation, consumer_group_member, consumer_group_size, condition) + def self.call(category, position: nil, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil) + instance = build(category, batch_size: batch_size, correlation: correlation, consumer_group_member: consumer_group_member, consumer_group_size: consumer_group_size, condition: condition, session: session) + instance.(position) + end + + def self.build(category, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil) + instance = new(category, batch_size, correlation, consumer_group_member, consumer_group_size, condition) + instance.configure(session: session) + instance + end + + def self.configure(receiver, category, attr_name: nil, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil) + attr_name ||= :get + instance = build(category, batch_size: batch_size, correlation: correlation, consumer_group_member: consumer_group_member, consumer_group_size: consumer_group_size, condition: condition, session: session) + receiver.public_send("#{attr_name}=", instance) + end + + def configure(session: nil) + Session.configure(self, session: session) end def sql_command diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index 239f5a1..a671667 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -6,8 +6,25 @@ class Stream initializer :stream_name, na(:batch_size), :correlation, :condition - def self.build(stream_name, batch_size: nil, correlation: nil, condition: nil) - new(stream_name, batch_size, correlation, condition) + def self.call(stream_name, position: nil, batch_size: nil, correlation: nil, condition: nil, session: nil) + instance = build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) + instance.(position) + end + + def self.build(stream_name, batch_size: nil, correlation: nil, condition: nil, session: nil) + instance = new(stream_name, batch_size, correlation, condition) + instance.configure(session: session) + instance + end + + def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, correlation: nil, condition: nil, session: nil) + attr_name ||= :get + instance = build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) + receiver.public_send("#{attr_name}=", instance) + end + + def configure(session: nil) + Session.configure(self, session: session) end def sql_command @@ -41,7 +58,7 @@ def self.assure(stream_name, args) consumer_group_size = args.delete(:consumer_group_size) if [consumer_group_member, consumer_group_size].compact.length > 0 - raise Error, "Consumer groups are supported only for category retrieval (Stream Name: #{stream_name})" + raise Error, "Consumer groups are supported exclusively for category retrieval (Stream Name: #{stream_name})" end end diff --git a/test/automated/get/category/concrete/dependency.rb b/test/automated/get/category/concrete/dependency.rb new file mode 100644 index 0000000..73161d2 --- /dev/null +++ b/test/automated/get/category/concrete/dependency.rb @@ -0,0 +1,38 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Category" do + context "Concrete" do + context "Dependency" do + category = Controls::Category.example + + Controls::Put.(category: category) + Controls::Put.(category: category) + + receiver = OpenStruct.new + + Get::Category.configure(receiver, category) + + get = receiver.get + + message_data = get.() + + context "Messages Retrieved" do + test "Messages from the all streams in the category" do + assert(message_data.length == 2) + end + + context "Message category is the category written" do + message_data.each do |md| + message_cateogry = StreamName.get_category(md.stream_name) + + test do + assert(message_cateogry == category) + end + end + end + end + end + end + end +end diff --git a/test/automated/get/category/concrete/get_messages.rb b/test/automated/get/category/concrete/get_messages.rb new file mode 100644 index 0000000..7302558 --- /dev/null +++ b/test/automated/get/category/concrete/get_messages.rb @@ -0,0 +1,32 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Category" do + context "Concrete" do + context "Get Messages" do + category = Controls::Category.example + + Controls::Put.(category: category) + Controls::Put.(category: category) + + message_data = Get::Category.(category) + + context "Messages Retrieved" do + test "Messages from the all streams in the category" do + assert(message_data.length == 2) + end + + context "Message category is the category written" do + message_data.each do |md| + message_cateogry = StreamName.get_category(md.stream_name) + + test do + assert(message_cateogry == category) + end + end + end + end + end + end + end +end diff --git a/test/automated/get/stream/concrete/dependency.rb b/test/automated/get/stream/concrete/dependency.rb new file mode 100644 index 0000000..b97fe82 --- /dev/null +++ b/test/automated/get/stream/concrete/dependency.rb @@ -0,0 +1,42 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Concrete" do + context "Dependency" do + write_message = Controls::MessageData::Write.example + + category = Controls::Category.example + + stream_name, _ = Controls::Put.(category: category) + Controls::Put.(category: category) + + receiver = OpenStruct.new + + Get::Stream.configure(receiver, stream_name) + + get = receiver.get + + message_data = get.() + + # message_data = Get::Stream.(stream_name) + + context "Messages Retrieved" do + test "Only messages from the specific stream" do + assert(message_data.length == 1) + end + + context "Message stream is the stream written" do + message_data.each do |md| + message_stream_name = md.stream_name + + test do + assert(message_stream_name == stream_name) + end + end + end + end + end + end + end +end diff --git a/test/automated/get/stream/concrete/get_messages.rb b/test/automated/get/stream/concrete/get_messages.rb new file mode 100644 index 0000000..8d11609 --- /dev/null +++ b/test/automated/get/stream/concrete/get_messages.rb @@ -0,0 +1,34 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Concrete" do + context "Get Message" do + write_message = Controls::MessageData::Write.example + + category = Controls::Category.example + + stream_name, _ = Controls::Put.(category: category) + Controls::Put.(category: category) + + message_data = Get::Stream.(stream_name) + + context "Messages Retrieved" do + test "Only messages from the specific stream" do + assert(message_data.length == 1) + end + + context "Message stream is the stream written" do + message_data.each do |md| + message_stream_name = md.stream_name + + test do + assert(message_stream_name == stream_name) + end + end + end + end + end + end + end +end diff --git a/test/automated/get/stream/consumer_groups.rb b/test/automated/get/stream/consumer_groups_error.rb similarity index 100% rename from test/automated/get/stream/consumer_groups.rb rename to test/automated/get/stream/consumer_groups_error.rb From e56f62e7fb93f6ba58d76a0e1f681b8f17bcfefe Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 28 Nov 2019 16:58:44 -0600 Subject: [PATCH 524/591] Specializations are tested --- test/automated/get/category/specialization.rb | 46 +++++++++++++++++-- test/automated/get/stream/specialization.rb | 37 +++++++++++++-- 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/test/automated/get/category/specialization.rb b/test/automated/get/category/specialization.rb index a372f0f..cd94fb9 100644 --- a/test/automated/get/category/specialization.rb +++ b/test/automated/get/category/specialization.rb @@ -3,12 +3,50 @@ context "Get" do context "Category" do context "Specialization" do - stream_name = Controls::Category.example + category = Controls::Category.example + batch_size = 1 + correlation = 'someCorrelation' + consumer_group_member = 0 + consumer_group_size = 1 + condition = 'global_position >= 1' + session = Session.build - get = Get.build(stream_name) + get = Get.build(category, batch_size: batch_size, correlation: correlation, consumer_group_member: consumer_group_member, consumer_group_size: consumer_group_size, condition: condition, session: session) - test "Get::Stream" do - assert(get.instance_of?(Get::Category)) + context "Type" do + test "Get::Category" do + assert(get.instance_of?(Get::Category)) + end + end + + context "Attributes" do + test "category" do + assert(get.category == category) + end + + test "batch_size" do + assert(get.batch_size == batch_size) + end + + test "correlation" do + assert(get.correlation == correlation) + end + + test "consumer_group_member" do + assert(get.consumer_group_member == consumer_group_member) + end + + test "consumer_group_size" do + assert(get.consumer_group_size == consumer_group_size) + end + + test "condition" do + assert(get.condition == condition) + end + + test "session" do + assert(get.session == session) + end end end end diff --git a/test/automated/get/stream/specialization.rb b/test/automated/get/stream/specialization.rb index ea0fd69..ec1112d 100644 --- a/test/automated/get/stream/specialization.rb +++ b/test/automated/get/stream/specialization.rb @@ -5,10 +5,41 @@ context "Specialization" do stream_name = Controls::StreamName.example - get = Get.build(stream_name) + batch_size = 1 + correlation = 'someCorrelation' + consumer_group_member = 0 + consumer_group_size = 1 + condition = 'global_position >= 1' + session = Session.build - test "Get::Stream" do - assert(get.instance_of?(Get::Stream)) + get = Get.build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) + + context "Type" do + test "Get::Stream" do + assert(get.instance_of?(Get::Stream)) + end + end + + context "Attributes" do + test "stream_name" do + assert(get.stream_name == stream_name) + end + + test "batch_size" do + assert(get.batch_size == batch_size) + end + + test "correlation" do + assert(get.correlation == correlation) + end + + test "condition" do + assert(get.condition == condition) + end + + test "session" do + assert(get.session == session) + end end end end From f8f990cb6e90ebbba1c126584d211a0fb9a62ed3 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 30 Nov 2019 17:54:48 -0600 Subject: [PATCH 525/591] Generalized and specialized category retrieval tests are separated --- .../get/category/{ => generalized}/get_messages.rb | 2 +- .../get/category/{ => generalized}/specialization.rb | 2 +- .../get/category/{ => specialized}/batch_size.rb | 4 ++-- .../{ => specialized}/condition/condition.rb | 4 ++-- .../{ => specialized}/condition/not_activated.rb | 4 ++-- .../consumer_groups/consumer_groups.rb | 4 ++-- .../{ => specialized}/consumer_groups/error.rb | 12 ++++++------ .../{ => specialized}/correlation/correlation.rb | 4 ++-- .../category/{ => specialized}/correlation/error.rb | 8 ++++---- .../category/{concrete => specialized}/dependency.rb | 0 .../{concrete => specialized}/get_messages.rb | 0 .../get/category/{ => specialized}/no_messages.rb | 4 ++-- .../get/category/{ => specialized}/position.rb | 4 ++-- 13 files changed, 26 insertions(+), 26 deletions(-) rename test/automated/get/category/{ => generalized}/get_messages.rb (94%) rename test/automated/get/category/{ => generalized}/specialization.rb (96%) rename test/automated/get/category/{ => specialized}/batch_size.rb (80%) rename test/automated/get/category/{ => specialized}/condition/condition.rb (81%) rename test/automated/get/category/{ => specialized}/condition/not_activated.rb (76%) rename test/automated/get/category/{ => specialized}/consumer_groups/consumer_groups.rb (83%) rename test/automated/get/category/{ => specialized}/consumer_groups/error.rb (70%) rename test/automated/get/category/{ => specialized}/correlation/correlation.rb (90%) rename test/automated/get/category/{ => specialized}/correlation/error.rb (52%) rename test/automated/get/category/{concrete => specialized}/dependency.rb (100%) rename test/automated/get/category/{concrete => specialized}/get_messages.rb (100%) rename test/automated/get/category/{ => specialized}/no_messages.rb (72%) rename test/automated/get/category/{ => specialized}/position.rb (77%) diff --git a/test/automated/get/category/get_messages.rb b/test/automated/get/category/generalized/get_messages.rb similarity index 94% rename from test/automated/get/category/get_messages.rb rename to test/automated/get/category/generalized/get_messages.rb index 6adfac6..ed47c35 100644 --- a/test/automated/get/category/get_messages.rb +++ b/test/automated/get/category/generalized/get_messages.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Category" do diff --git a/test/automated/get/category/specialization.rb b/test/automated/get/category/generalized/specialization.rb similarity index 96% rename from test/automated/get/category/specialization.rb rename to test/automated/get/category/generalized/specialization.rb index cd94fb9..d308e24 100644 --- a/test/automated/get/category/specialization.rb +++ b/test/automated/get/category/generalized/specialization.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Category" do diff --git a/test/automated/get/category/batch_size.rb b/test/automated/get/category/specialized/batch_size.rb similarity index 80% rename from test/automated/get/category/batch_size.rb rename to test/automated/get/category/specialized/batch_size.rb index 2f08b96..6c132ea 100644 --- a/test/automated/get/category/batch_size.rb +++ b/test/automated/get/category/specialized/batch_size.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Category" do @@ -9,7 +9,7 @@ Controls::Put.(category: category) Controls::Put.(category: category) - messages = Get.(category, batch_size: 2) + messages = Get::Category.(category, batch_size: 2) number_of_messages = messages.length diff --git a/test/automated/get/category/condition/condition.rb b/test/automated/get/category/specialized/condition/condition.rb similarity index 81% rename from test/automated/get/category/condition/condition.rb rename to test/automated/get/category/specialized/condition/condition.rb index 5fd4839..814b0d2 100644 --- a/test/automated/get/category/condition/condition.rb +++ b/test/automated/get/category/specialized/condition/condition.rb @@ -1,4 +1,4 @@ -require_relative '../../../automated_init' +require_relative '../../../../automated_init' context "Get" do context "Category" do @@ -14,7 +14,7 @@ settings.set(session) session.options = '-c message_store.sql_condition=on' - messages = Get.(category, batch_size: 3, condition: condition, session: session) + messages = Get::Category.(category, batch_size: 3, condition: condition, session: session) message_positions = messages.map do |message| message.position diff --git a/test/automated/get/category/condition/not_activated.rb b/test/automated/get/category/specialized/condition/not_activated.rb similarity index 76% rename from test/automated/get/category/condition/not_activated.rb rename to test/automated/get/category/specialized/condition/not_activated.rb index 9bbf27f..b2ca22d 100644 --- a/test/automated/get/category/condition/not_activated.rb +++ b/test/automated/get/category/specialized/condition/not_activated.rb @@ -1,4 +1,4 @@ -require_relative '../../../automated_init' +require_relative '../../../../automated_init' context "Get" do context "Category" do @@ -15,7 +15,7 @@ test "Is an error" do assert_raises(Get::Condition::Error) do - Get.(category, batch_size: 3, condition: condition, session: session) + Get::Category.(category, batch_size: 3, condition: condition, session: session) end end end diff --git a/test/automated/get/category/consumer_groups/consumer_groups.rb b/test/automated/get/category/specialized/consumer_groups/consumer_groups.rb similarity index 83% rename from test/automated/get/category/consumer_groups/consumer_groups.rb rename to test/automated/get/category/specialized/consumer_groups/consumer_groups.rb index d23b0f6..12923ad 100644 --- a/test/automated/get/category/consumer_groups/consumer_groups.rb +++ b/test/automated/get/category/specialized/consumer_groups/consumer_groups.rb @@ -1,4 +1,4 @@ -require_relative '../../../automated_init' +require_relative '../../../../automated_init' context "Get" do context "Category" do @@ -14,7 +14,7 @@ retrieved_message_data = [] partitions.times do |i| - retrieved_message_data += Get.(category, consumer_group_member: i, consumer_group_size: partitions) + retrieved_message_data += Get::Category.(category, consumer_group_member: i, consumer_group_size: partitions) end context "Message Partitions" do diff --git a/test/automated/get/category/consumer_groups/error.rb b/test/automated/get/category/specialized/consumer_groups/error.rb similarity index 70% rename from test/automated/get/category/consumer_groups/error.rb rename to test/automated/get/category/specialized/consumer_groups/error.rb index 7ad7520..31ec3d0 100644 --- a/test/automated/get/category/consumer_groups/error.rb +++ b/test/automated/get/category/specialized/consumer_groups/error.rb @@ -1,4 +1,4 @@ -require_relative '../../../automated_init' +require_relative '../../../../automated_init' context "Get" do context "Category" do @@ -8,7 +8,7 @@ context "Consumer Group Size Is Less than 1" do test "Is an error" do assert_raises(Get::Category::ConsumerGroup::Error) do - Get.('someCategory', consumer_group_member: 0, consumer_group_size: 0) + Get::Category.('someCategory', consumer_group_member: 0, consumer_group_size: 0) end end end @@ -16,7 +16,7 @@ context "Consumer Group Member Is Greater than the Consumer Group Size" do test "Is an error" do assert_raises(Get::Category::ConsumerGroup::Error) do - Get.('someCategory', consumer_group_member: 2, consumer_group_size: 1) + Get::Category.('someCategory', consumer_group_member: 2, consumer_group_size: 1) end end end @@ -24,7 +24,7 @@ context "Consumer Group Member Is Less than 0" do test "Is an error" do assert_raises(Get::Category::ConsumerGroup::Error) do - Get.('someCategory', consumer_group_member: -1, consumer_group_size: 1) + Get::Category.('someCategory', consumer_group_member: -1, consumer_group_size: 1) end end end @@ -32,7 +32,7 @@ context "Consumer Group Size is Missing" do test "Is an error" do assert_raises(Get::Category::ConsumerGroup::Error) do - Get.('someCategory', consumer_group_member: 0) + Get::Category.('someCategory', consumer_group_member: 0) end end end @@ -40,7 +40,7 @@ context "Consumer Group Member is Missing" do test "Is an error" do assert_raises(Get::Category::ConsumerGroup::Error) do - Get.('someCategory', consumer_group_size: 1) + Get::Category.('someCategory', consumer_group_size: 1) end end end diff --git a/test/automated/get/category/correlation/correlation.rb b/test/automated/get/category/specialized/correlation/correlation.rb similarity index 90% rename from test/automated/get/category/correlation/correlation.rb rename to test/automated/get/category/specialized/correlation/correlation.rb index fe38593..5ca4495 100644 --- a/test/automated/get/category/correlation/correlation.rb +++ b/test/automated/get/category/specialized/correlation/correlation.rb @@ -1,4 +1,4 @@ -require_relative '../../../automated_init' +require_relative '../../../../automated_init' context "Get" do context "Category" do @@ -28,7 +28,7 @@ Put.(message_data, stream_name) end - message_datas = Get.(category, correlation: correlation_category) + message_datas = Get::Category.(category, correlation: correlation_category) correlation_stream_names = message_datas.map do |message_data| message_data.metadata[:correlation_stream_name] diff --git a/test/automated/get/category/correlation/error.rb b/test/automated/get/category/specialized/correlation/error.rb similarity index 52% rename from test/automated/get/category/correlation/error.rb rename to test/automated/get/category/specialized/correlation/error.rb index 5355419..fa8ebd9 100644 --- a/test/automated/get/category/correlation/error.rb +++ b/test/automated/get/category/specialized/correlation/error.rb @@ -1,4 +1,4 @@ -require_relative '../../../automated_init' +require_relative '../../../../automated_init' context "Get" do context "Category" do @@ -6,11 +6,11 @@ context "Not a Category" do correlation = Controls::StreamName.example - stream_name = Controls::StreamName.example + category = Controls::Category.example test "Is an error" do - assert_raises MessageStore::Correlation::Error do - Get.(stream_name, correlation: correlation) + assert_raises(MessageStore::Correlation::Error) do + Get::Category.(category, correlation: correlation) end end end diff --git a/test/automated/get/category/concrete/dependency.rb b/test/automated/get/category/specialized/dependency.rb similarity index 100% rename from test/automated/get/category/concrete/dependency.rb rename to test/automated/get/category/specialized/dependency.rb diff --git a/test/automated/get/category/concrete/get_messages.rb b/test/automated/get/category/specialized/get_messages.rb similarity index 100% rename from test/automated/get/category/concrete/get_messages.rb rename to test/automated/get/category/specialized/get_messages.rb diff --git a/test/automated/get/category/no_messages.rb b/test/automated/get/category/specialized/no_messages.rb similarity index 72% rename from test/automated/get/category/no_messages.rb rename to test/automated/get/category/specialized/no_messages.rb index 9d74ad4..2b72ea3 100644 --- a/test/automated/get/category/no_messages.rb +++ b/test/automated/get/category/specialized/no_messages.rb @@ -1,11 +1,11 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Category" do context "No Messages" do category = Controls::Category.example - batch = Get.(category) + batch = Get::Category.(category) test "Retrieves no messages" do assert(batch == []) diff --git a/test/automated/get/category/position.rb b/test/automated/get/category/specialized/position.rb similarity index 77% rename from test/automated/get/category/position.rb rename to test/automated/get/category/specialized/position.rb index 9e17c66..1b819c0 100644 --- a/test/automated/get/category/position.rb +++ b/test/automated/get/category/specialized/position.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Category" do @@ -8,7 +8,7 @@ Controls::Put.(instances: 2, stream_name: stream_name) - batch = Get.(category, position: 1, batch_size: 1) + batch = Get::Category.(category, position: 1, batch_size: 1) test "Retrieves messages from the starting position" do assert(batch.length == 1) From 95ebfe04748af24a7b99415b55d57e7404f0323f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 30 Nov 2019 18:13:57 -0600 Subject: [PATCH 526/591] Context namespaces reflect file system organization --- .../get/category/generalized/get_messages.rb | 30 ++++--- .../category/generalized/specialization.rb | 90 ++++++++++--------- .../get/category/specialized/batch_size.rb | 20 +++-- .../specialized/condition/condition.rb | 30 ++++--- .../specialized/condition/not_activated.rb | 24 ++--- .../consumer_groups/consumer_groups.rb | 44 ++++----- .../specialized/consumer_groups/error.rb | 54 +++++------ .../specialized/correlation/correlation.rb | 48 +++++----- .../category/specialized/correlation/error.rb | 16 ++-- .../get/category/specialized/dependency.rb | 2 +- .../get/category/specialized/get_messages.rb | 2 +- .../get/category/specialized/no_messages.rb | 12 +-- .../get/category/specialized/position.rb | 16 ++-- test/automated/get/stream/batch_size.rb | 17 ---- .../get/stream/condition/condition.rb | 26 ------ .../get/stream/condition/not_activated.rb | 24 ----- .../get/stream/consumer_groups_error.rb | 17 ---- .../get/stream/correlation/correlation.rb | 40 --------- .../automated/get/stream/correlation/error.rb | 19 ---- .../get/stream/generalized/get_messages.rb | 34 +++++++ .../get/stream/generalized/specialization.rb | 48 ++++++++++ test/automated/get/stream/get_messages.rb | 32 ------- test/automated/get/stream/no_messages.rb | 15 ---- test/automated/get/stream/specialization.rb | 46 ---------- .../get/stream/specialized/batch_size.rb | 19 ++++ .../stream/specialized/condition/condition.rb | 28 ++++++ .../specialized/condition/not_activated.rb | 26 ++++++ .../specialized/consumer_groups_error.rb | 19 ++++ .../specialized/correlation/correlation.rb | 42 +++++++++ .../stream/specialized/correlation/error.rb | 21 +++++ .../{concrete => specialized}/dependency.rb | 2 +- .../{concrete => specialized}/get_messages.rb | 2 +- .../get/stream/specialized/no_messages.rb | 17 ++++ .../get/stream/{ => specialized}/position.rb | 2 +- 34 files changed, 462 insertions(+), 422 deletions(-) delete mode 100644 test/automated/get/stream/batch_size.rb delete mode 100644 test/automated/get/stream/condition/condition.rb delete mode 100644 test/automated/get/stream/condition/not_activated.rb delete mode 100644 test/automated/get/stream/consumer_groups_error.rb delete mode 100644 test/automated/get/stream/correlation/correlation.rb delete mode 100644 test/automated/get/stream/correlation/error.rb create mode 100644 test/automated/get/stream/generalized/get_messages.rb create mode 100644 test/automated/get/stream/generalized/specialization.rb delete mode 100644 test/automated/get/stream/get_messages.rb delete mode 100644 test/automated/get/stream/no_messages.rb delete mode 100644 test/automated/get/stream/specialization.rb create mode 100644 test/automated/get/stream/specialized/batch_size.rb create mode 100644 test/automated/get/stream/specialized/condition/condition.rb create mode 100644 test/automated/get/stream/specialized/condition/not_activated.rb create mode 100644 test/automated/get/stream/specialized/consumer_groups_error.rb create mode 100644 test/automated/get/stream/specialized/correlation/correlation.rb create mode 100644 test/automated/get/stream/specialized/correlation/error.rb rename test/automated/get/stream/{concrete => specialized}/dependency.rb (97%) rename test/automated/get/stream/{concrete => specialized}/get_messages.rb (96%) create mode 100644 test/automated/get/stream/specialized/no_messages.rb rename test/automated/get/stream/{ => specialized}/position.rb (87%) diff --git a/test/automated/get/category/generalized/get_messages.rb b/test/automated/get/category/generalized/get_messages.rb index ed47c35..9e08b35 100644 --- a/test/automated/get/category/generalized/get_messages.rb +++ b/test/automated/get/category/generalized/get_messages.rb @@ -2,25 +2,27 @@ context "Get" do context "Category" do - context "Get Messages" do - category = Controls::Category.example + context "Generalized" do + context "Get Messages" do + category = Controls::Category.example - Controls::Put.(category: category) - Controls::Put.(category: category) + Controls::Put.(category: category) + Controls::Put.(category: category) - message_data = Get.(category) + message_data = Get.(category) - context "Messages Retrieved" do - test "Messages from the all streams in the category" do - assert(message_data.length == 2) - end + context "Messages Retrieved" do + test "Messages from the all streams in the category" do + assert(message_data.length == 2) + end - context "Message category is the category written" do - message_data.each do |md| - message_cateogry = StreamName.get_category(md.stream_name) + context "Message category is the category written" do + message_data.each do |md| + message_cateogry = StreamName.get_category(md.stream_name) - test do - assert(message_cateogry == category) + test do + assert(message_cateogry == category) + end end end end diff --git a/test/automated/get/category/generalized/specialization.rb b/test/automated/get/category/generalized/specialization.rb index d308e24..7c37ea4 100644 --- a/test/automated/get/category/generalized/specialization.rb +++ b/test/automated/get/category/generalized/specialization.rb @@ -2,50 +2,52 @@ context "Get" do context "Category" do - context "Specialization" do - category = Controls::Category.example - batch_size = 1 - correlation = 'someCorrelation' - consumer_group_member = 0 - consumer_group_size = 1 - condition = 'global_position >= 1' - session = Session.build - - get = Get.build(category, batch_size: batch_size, correlation: correlation, consumer_group_member: consumer_group_member, consumer_group_size: consumer_group_size, condition: condition, session: session) - - context "Type" do - test "Get::Category" do - assert(get.instance_of?(Get::Category)) - end - end - - context "Attributes" do - test "category" do - assert(get.category == category) - end - - test "batch_size" do - assert(get.batch_size == batch_size) - end - - test "correlation" do - assert(get.correlation == correlation) - end - - test "consumer_group_member" do - assert(get.consumer_group_member == consumer_group_member) - end - - test "consumer_group_size" do - assert(get.consumer_group_size == consumer_group_size) - end - - test "condition" do - assert(get.condition == condition) - end - - test "session" do - assert(get.session == session) + context "Generalized" do + context "Specialization" do + category = Controls::Category.example + batch_size = 1 + correlation = 'someCorrelation' + consumer_group_member = 0 + consumer_group_size = 1 + condition = 'global_position >= 1' + session = Session.build + + get = Get.build(category, batch_size: batch_size, correlation: correlation, consumer_group_member: consumer_group_member, consumer_group_size: consumer_group_size, condition: condition, session: session) + + context "Type" do + test "Get::Category" do + assert(get.instance_of?(Get::Category)) + end + end + + context "Attributes" do + test "category" do + assert(get.category == category) + end + + test "batch_size" do + assert(get.batch_size == batch_size) + end + + test "correlation" do + assert(get.correlation == correlation) + end + + test "consumer_group_member" do + assert(get.consumer_group_member == consumer_group_member) + end + + test "consumer_group_size" do + assert(get.consumer_group_size == consumer_group_size) + end + + test "condition" do + assert(get.condition == condition) + end + + test "session" do + assert(get.session == session) + end end end end diff --git a/test/automated/get/category/specialized/batch_size.rb b/test/automated/get/category/specialized/batch_size.rb index 6c132ea..2d4de98 100644 --- a/test/automated/get/category/specialized/batch_size.rb +++ b/test/automated/get/category/specialized/batch_size.rb @@ -2,19 +2,21 @@ context "Get" do context "Category" do - context "Batch Size" do - category = Controls::Category.example + context "Specialized" do + context "Batch Size" do + category = Controls::Category.example - Controls::Put.(category: category) - Controls::Put.(category: category) - Controls::Put.(category: category) + Controls::Put.(category: category) + Controls::Put.(category: category) + Controls::Put.(category: category) - messages = Get::Category.(category, batch_size: 2) + messages = Get::Category.(category, batch_size: 2) - number_of_messages = messages.length + number_of_messages = messages.length - test "Number of messages retrieved is the specified batch size" do - assert(number_of_messages == 2) + test "Number of messages retrieved is the specified batch size" do + assert(number_of_messages == 2) + end end end end diff --git a/test/automated/get/category/specialized/condition/condition.rb b/test/automated/get/category/specialized/condition/condition.rb index 814b0d2..b6664bc 100644 --- a/test/automated/get/category/specialized/condition/condition.rb +++ b/test/automated/get/category/specialized/condition/condition.rb @@ -2,26 +2,28 @@ context "Get" do context "Category" do - context "Condition" do - category = Controls::Category.example + context "Specialized" do + context "Condition" do + category = Controls::Category.example - stream_name, _ = Controls::Put.(instances: 3, category: category) + stream_name, _ = Controls::Put.(instances: 3, category: category) - condition = 'position = 0 OR position = 2' + condition = 'position = 0 OR position = 2' - settings = Postgres::Settings.build - session = Session.new - settings.set(session) - session.options = '-c message_store.sql_condition=on' + settings = Postgres::Settings.build + session = Session.new + settings.set(session) + session.options = '-c message_store.sql_condition=on' - messages = Get::Category.(category, batch_size: 3, condition: condition, session: session) + messages = Get::Category.(category, batch_size: 3, condition: condition, session: session) - message_positions = messages.map do |message| - message.position - end + message_positions = messages.map do |message| + message.position + end - test "Retrieves messages that meet the condition" do - assert(message_positions == [0, 2]) + test "Retrieves messages that meet the condition" do + assert(message_positions == [0, 2]) + end end end end diff --git a/test/automated/get/category/specialized/condition/not_activated.rb b/test/automated/get/category/specialized/condition/not_activated.rb index b2ca22d..c85c009 100644 --- a/test/automated/get/category/specialized/condition/not_activated.rb +++ b/test/automated/get/category/specialized/condition/not_activated.rb @@ -2,20 +2,22 @@ context "Get" do context "Category" do - context "Condition" do - context "Not Activated" do - category = Controls::Category.example + context "Specialized" do + context "Condition" do + context "Not Activated" do + category = Controls::Category.example - condition = 'some condition' + condition = 'some condition' - settings = Postgres::Settings.build - session = Session.new - settings.set(session) - session.options = nil + settings = Postgres::Settings.build + session = Session.new + settings.set(session) + session.options = nil - test "Is an error" do - assert_raises(Get::Condition::Error) do - Get::Category.(category, batch_size: 3, condition: condition, session: session) + test "Is an error" do + assert_raises(Get::Condition::Error) do + Get::Category.(category, batch_size: 3, condition: condition, session: session) + end end end end diff --git a/test/automated/get/category/specialized/consumer_groups/consumer_groups.rb b/test/automated/get/category/specialized/consumer_groups/consumer_groups.rb index 12923ad..2d80f59 100644 --- a/test/automated/get/category/specialized/consumer_groups/consumer_groups.rb +++ b/test/automated/get/category/specialized/consumer_groups/consumer_groups.rb @@ -2,35 +2,37 @@ context "Get" do context "Category" do - context "Consumer Groups" do - category = Controls::Category.example + context "Specialized" do + context "Consumer Groups" do + category = Controls::Category.example - instances = 11 - partitions = 3 + instances = 11 + partitions = 3 - instances.times do - Controls::Put.(category: category) - end + instances.times do + Controls::Put.(category: category) + end - retrieved_message_data = [] - partitions.times do |i| - retrieved_message_data += Get::Category.(category, consumer_group_member: i, consumer_group_size: partitions) - end + retrieved_message_data = [] + partitions.times do |i| + retrieved_message_data += Get::Category.(category, consumer_group_member: i, consumer_group_size: partitions) + end - context "Message Partitions" do - context "Distribution" do - retrieved_count = retrieved_message_data.count + context "Message Partitions" do + context "Distribution" do + retrieved_count = retrieved_message_data.count - test "Across partitions" do - assert(retrieved_count == instances) + test "Across partitions" do + assert(retrieved_count == instances) + end end end - end - context "Messages in all partitions" do - test "Are of the same category" do - retrieved_message_data.each do |message_data| - assert(StreamName.get_category(message_data.stream_name) == category) + context "Messages in all partitions" do + test "Are of the same category" do + retrieved_message_data.each do |message_data| + assert(StreamName.get_category(message_data.stream_name) == category) + end end end end diff --git a/test/automated/get/category/specialized/consumer_groups/error.rb b/test/automated/get/category/specialized/consumer_groups/error.rb index 31ec3d0..c8f03d0 100644 --- a/test/automated/get/category/specialized/consumer_groups/error.rb +++ b/test/automated/get/category/specialized/consumer_groups/error.rb @@ -2,45 +2,47 @@ context "Get" do context "Category" do - context "Consumer Groups" do - context "Error" do + context "Specialized" do + context "Consumer Groups" do + context "Error" do - context "Consumer Group Size Is Less than 1" do - test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do - Get::Category.('someCategory', consumer_group_member: 0, consumer_group_size: 0) + context "Consumer Group Size Is Less than 1" do + test "Is an error" do + assert_raises(Get::Category::ConsumerGroup::Error) do + Get::Category.('someCategory', consumer_group_member: 0, consumer_group_size: 0) + end end end - end - context "Consumer Group Member Is Greater than the Consumer Group Size" do - test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do - Get::Category.('someCategory', consumer_group_member: 2, consumer_group_size: 1) + context "Consumer Group Member Is Greater than the Consumer Group Size" do + test "Is an error" do + assert_raises(Get::Category::ConsumerGroup::Error) do + Get::Category.('someCategory', consumer_group_member: 2, consumer_group_size: 1) + end end end - end - context "Consumer Group Member Is Less than 0" do - test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do - Get::Category.('someCategory', consumer_group_member: -1, consumer_group_size: 1) + context "Consumer Group Member Is Less than 0" do + test "Is an error" do + assert_raises(Get::Category::ConsumerGroup::Error) do + Get::Category.('someCategory', consumer_group_member: -1, consumer_group_size: 1) + end end end - end - context "Consumer Group Size is Missing" do - test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do - Get::Category.('someCategory', consumer_group_member: 0) + context "Consumer Group Size is Missing" do + test "Is an error" do + assert_raises(Get::Category::ConsumerGroup::Error) do + Get::Category.('someCategory', consumer_group_member: 0) + end end end - end - context "Consumer Group Member is Missing" do - test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do - Get::Category.('someCategory', consumer_group_size: 1) + context "Consumer Group Member is Missing" do + test "Is an error" do + assert_raises(Get::Category::ConsumerGroup::Error) do + Get::Category.('someCategory', consumer_group_size: 1) + end end end end diff --git a/test/automated/get/category/specialized/correlation/correlation.rb b/test/automated/get/category/specialized/correlation/correlation.rb index 5ca4495..69d131a 100644 --- a/test/automated/get/category/specialized/correlation/correlation.rb +++ b/test/automated/get/category/specialized/correlation/correlation.rb @@ -2,40 +2,42 @@ context "Get" do context "Category" do - context "Correlation" do - correlation_category = Controls::Category.example + context "Specialized" do + context "Correlation" do + correlation_category = Controls::Category.example - correlation_stream_name = Controls::StreamName.example(category: correlation_category) + correlation_stream_name = Controls::StreamName.example(category: correlation_category) - correlation_metadata = { - correlation_stream_name: correlation_stream_name - } + correlation_metadata = { + correlation_stream_name: correlation_stream_name + } - category = Controls::Category.example + category = Controls::Category.example - message_data = Controls::MessageData::Write.example + message_data = Controls::MessageData::Write.example - message_data.metadata = { - correlation_stream_name: SecureRandom.hex - } + message_data.metadata = { + correlation_stream_name: SecureRandom.hex + } - stream_name = Controls::StreamName.example(category: category) - Put.(message_data, stream_name) - - 2.times do - message_data = Controls::MessageData::Write.example(metadata: correlation_metadata) stream_name = Controls::StreamName.example(category: category) Put.(message_data, stream_name) - end - message_datas = Get::Category.(category, correlation: correlation_category) + 2.times do + message_data = Controls::MessageData::Write.example(metadata: correlation_metadata) + stream_name = Controls::StreamName.example(category: category) + Put.(message_data, stream_name) + end - correlation_stream_names = message_datas.map do |message_data| - message_data.metadata[:correlation_stream_name] - end + message_datas = Get::Category.(category, correlation: correlation_category) + + correlation_stream_names = message_datas.map do |message_data| + message_data.metadata[:correlation_stream_name] + end - test "Retrieves messages that meet the condition" do - assert(correlation_stream_names == [correlation_stream_name, correlation_stream_name]) + test "Retrieves messages that meet the condition" do + assert(correlation_stream_names == [correlation_stream_name, correlation_stream_name]) + end end end end diff --git a/test/automated/get/category/specialized/correlation/error.rb b/test/automated/get/category/specialized/correlation/error.rb index fa8ebd9..c7e3490 100644 --- a/test/automated/get/category/specialized/correlation/error.rb +++ b/test/automated/get/category/specialized/correlation/error.rb @@ -2,15 +2,17 @@ context "Get" do context "Category" do - context "Correlation" do - context "Not a Category" do - correlation = Controls::StreamName.example + context "Specialized" do + context "Correlation" do + context "Not a Category" do + correlation = Controls::StreamName.example - category = Controls::Category.example + category = Controls::Category.example - test "Is an error" do - assert_raises(MessageStore::Correlation::Error) do - Get::Category.(category, correlation: correlation) + test "Is an error" do + assert_raises(MessageStore::Correlation::Error) do + Get::Category.(category, correlation: correlation) + end end end end diff --git a/test/automated/get/category/specialized/dependency.rb b/test/automated/get/category/specialized/dependency.rb index 73161d2..5b7170e 100644 --- a/test/automated/get/category/specialized/dependency.rb +++ b/test/automated/get/category/specialized/dependency.rb @@ -2,7 +2,7 @@ context "Get" do context "Category" do - context "Concrete" do + context "Specialized" do context "Dependency" do category = Controls::Category.example diff --git a/test/automated/get/category/specialized/get_messages.rb b/test/automated/get/category/specialized/get_messages.rb index 7302558..a353950 100644 --- a/test/automated/get/category/specialized/get_messages.rb +++ b/test/automated/get/category/specialized/get_messages.rb @@ -2,7 +2,7 @@ context "Get" do context "Category" do - context "Concrete" do + context "Specialized" do context "Get Messages" do category = Controls::Category.example diff --git a/test/automated/get/category/specialized/no_messages.rb b/test/automated/get/category/specialized/no_messages.rb index 2b72ea3..422dabd 100644 --- a/test/automated/get/category/specialized/no_messages.rb +++ b/test/automated/get/category/specialized/no_messages.rb @@ -2,13 +2,15 @@ context "Get" do context "Category" do - context "No Messages" do - category = Controls::Category.example + context "Specialized" do + context "No Messages" do + category = Controls::Category.example - batch = Get::Category.(category) + batch = Get::Category.(category) - test "Retrieves no messages" do - assert(batch == []) + test "Retrieves no messages" do + assert(batch == []) + end end end end diff --git a/test/automated/get/category/specialized/position.rb b/test/automated/get/category/specialized/position.rb index 1b819c0..eaec3eb 100644 --- a/test/automated/get/category/specialized/position.rb +++ b/test/automated/get/category/specialized/position.rb @@ -2,16 +2,18 @@ context "Get" do context "Category" do - context "Position" do - category = Controls::Category.example - stream_name = Controls::StreamName.example(category: category) + context "Specialized" do + context "Position" do + category = Controls::Category.example + stream_name = Controls::StreamName.example(category: category) - Controls::Put.(instances: 2, stream_name: stream_name) + Controls::Put.(instances: 2, stream_name: stream_name) - batch = Get::Category.(category, position: 1, batch_size: 1) + batch = Get::Category.(category, position: 1, batch_size: 1) - test "Retrieves messages from the starting position" do - assert(batch.length == 1) + test "Retrieves messages from the starting position" do + assert(batch.length == 1) + end end end end diff --git a/test/automated/get/stream/batch_size.rb b/test/automated/get/stream/batch_size.rb deleted file mode 100644 index 60d7d2c..0000000 --- a/test/automated/get/stream/batch_size.rb +++ /dev/null @@ -1,17 +0,0 @@ -require_relative '../../automated_init' - -context "Get" do - context "Stream" do - context "Batch Size" do - stream_name, _ = Controls::Put.(instances: 3) - - messages = Get.(stream_name, batch_size: 2) - - number_of_messages = messages.length - - test "Number of messages retrieved is the specified batch size" do - assert(number_of_messages == 2) - end - end - end -end diff --git a/test/automated/get/stream/condition/condition.rb b/test/automated/get/stream/condition/condition.rb deleted file mode 100644 index 6e6ff44..0000000 --- a/test/automated/get/stream/condition/condition.rb +++ /dev/null @@ -1,26 +0,0 @@ -require_relative '../../../automated_init' - -context "Get" do - context "Stream" do - context "Condition" do - stream_name, _ = Controls::Put.(instances: 3) - - condition = 'position = 0 OR position = 2' - - settings = Postgres::Settings.build - session = Session.new - settings.set(session) - session.options = '-c message_store.sql_condition=on' - - messages = Get.(stream_name, batch_size: 3, condition: condition, session: session) - - message_positions = messages.map do |message| - message.position - end - - test "Returns messages that meet the condition" do - assert(message_positions == [0, 2]) - end - end - end -end diff --git a/test/automated/get/stream/condition/not_activated.rb b/test/automated/get/stream/condition/not_activated.rb deleted file mode 100644 index f5087d6..0000000 --- a/test/automated/get/stream/condition/not_activated.rb +++ /dev/null @@ -1,24 +0,0 @@ -require_relative '../../../automated_init' - -context "Get" do - context "Category" do - context "Condition" do - context "Not Activated" do - stream_name = Controls::StreamName.example - - condition = 'some condition' - - settings = Postgres::Settings.build - session = Session.new - settings.set(session) - session.options = nil - - test "Is an error" do - assert_raises(Get::Condition::Error) do - Get.(stream_name, batch_size: 3, condition: condition, session: session) - end - end - end - end - end -end diff --git a/test/automated/get/stream/consumer_groups_error.rb b/test/automated/get/stream/consumer_groups_error.rb deleted file mode 100644 index caf2ded..0000000 --- a/test/automated/get/stream/consumer_groups_error.rb +++ /dev/null @@ -1,17 +0,0 @@ -require_relative '../../automated_init' - -context "Get" do - context "Stream" do - context "Consumer Groups" do - context "Retrieving from a Stream Using Consumer Groups" do - stream_name = Controls::StreamName.example - - test "Is an error" do - assert_raises(Get::Error) do - Get.(stream_name, consumer_group_member: 0, consumer_group_size: 1) - end - end - end - end - end -end diff --git a/test/automated/get/stream/correlation/correlation.rb b/test/automated/get/stream/correlation/correlation.rb deleted file mode 100644 index 203eb76..0000000 --- a/test/automated/get/stream/correlation/correlation.rb +++ /dev/null @@ -1,40 +0,0 @@ -require_relative '../../../automated_init' - -context "Get" do - context "Stream" do - context "Correlation" do - correlation_category = Controls::Category.example - - correlation_stream_name = Controls::StreamName.example(category: correlation_category) - - correlation_metadata = { - correlation_stream_name: correlation_stream_name - } - - stream_name = Controls::StreamName.example - - message_data = Controls::MessageData::Write.example - - message_data.metadata = { - correlation_stream_name: SecureRandom.hex - } - - Put.(message_data, stream_name) - - 2.times do - message_data = Controls::MessageData::Write.example(metadata: correlation_metadata) - Put.(message_data, stream_name) - end - - message_datas = Get.(stream_name, correlation: correlation_category) - - correlation_stream_names = message_datas.map do |message_data| - message_data.metadata[:correlation_stream_name] - end - - test "Retrieves messages that meet the condition" do - assert(correlation_stream_names == [correlation_stream_name, correlation_stream_name]) - end - end - end -end diff --git a/test/automated/get/stream/correlation/error.rb b/test/automated/get/stream/correlation/error.rb deleted file mode 100644 index bfc3b74..0000000 --- a/test/automated/get/stream/correlation/error.rb +++ /dev/null @@ -1,19 +0,0 @@ -require_relative '../../../automated_init' - -context "Get" do - context "Stream" do - context "Correlation" do - context "Not a Category" do - correlation = Controls::StreamName.example - - stream_name = Controls::StreamName.example - - test "Is an error" do - assert_raises MessageStore::Correlation::Error do - Get.(stream_name, correlation: correlation) - end - end - end - end - end -end diff --git a/test/automated/get/stream/generalized/get_messages.rb b/test/automated/get/stream/generalized/get_messages.rb new file mode 100644 index 0000000..4fc5745 --- /dev/null +++ b/test/automated/get/stream/generalized/get_messages.rb @@ -0,0 +1,34 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Generalized" do + context "Get Message" do + write_message = Controls::MessageData::Write.example + + category = Controls::Category.example + + stream_name, _ = Controls::Put.(category: category) + Controls::Put.(category: category) + + message_data = Get.(stream_name) + + context "Messages Retrieved" do + test "Only messages from the specific stream" do + assert(message_data.length == 1) + end + + context "Message stream is the stream written" do + message_data.each do |md| + message_stream_name = md.stream_name + + test do + assert(message_stream_name == stream_name) + end + end + end + end + end + end + end +end diff --git a/test/automated/get/stream/generalized/specialization.rb b/test/automated/get/stream/generalized/specialization.rb new file mode 100644 index 0000000..4735a29 --- /dev/null +++ b/test/automated/get/stream/generalized/specialization.rb @@ -0,0 +1,48 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Generalized" do + context "Specialization" do + stream_name = Controls::StreamName.example + + batch_size = 1 + correlation = 'someCorrelation' + consumer_group_member = 0 + consumer_group_size = 1 + condition = 'global_position >= 1' + session = Session.build + + get = Get.build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) + + context "Type" do + test "Get::Stream" do + assert(get.instance_of?(Get::Stream)) + end + end + + context "Attributes" do + test "stream_name" do + assert(get.stream_name == stream_name) + end + + test "batch_size" do + assert(get.batch_size == batch_size) + end + + test "correlation" do + assert(get.correlation == correlation) + end + + test "condition" do + assert(get.condition == condition) + end + + test "session" do + assert(get.session == session) + end + end + end + end + end +end diff --git a/test/automated/get/stream/get_messages.rb b/test/automated/get/stream/get_messages.rb deleted file mode 100644 index e68ee53..0000000 --- a/test/automated/get/stream/get_messages.rb +++ /dev/null @@ -1,32 +0,0 @@ -require_relative '../../automated_init' - -context "Get" do - context "Stream" do - context "Get Message" do - write_message = Controls::MessageData::Write.example - - category = Controls::Category.example - - stream_name, _ = Controls::Put.(category: category) - Controls::Put.(category: category) - - message_data = Get.(stream_name) - - context "Messages Retrieved" do - test "Only messages from the specific stream" do - assert(message_data.length == 1) - end - - context "Message stream is the stream written" do - message_data.each do |md| - message_stream_name = md.stream_name - - test do - assert(message_stream_name == stream_name) - end - end - end - end - end - end -end diff --git a/test/automated/get/stream/no_messages.rb b/test/automated/get/stream/no_messages.rb deleted file mode 100644 index 14fa4d5..0000000 --- a/test/automated/get/stream/no_messages.rb +++ /dev/null @@ -1,15 +0,0 @@ -require_relative '../../automated_init' - -context "Get" do - context "Stream" do - context "No Messages" do - stream_name = Controls::StreamName.example - - batch = Get.(stream_name) - - test "Retrieves no messages" do - assert(batch == []) - end - end - end -end diff --git a/test/automated/get/stream/specialization.rb b/test/automated/get/stream/specialization.rb deleted file mode 100644 index ec1112d..0000000 --- a/test/automated/get/stream/specialization.rb +++ /dev/null @@ -1,46 +0,0 @@ -require_relative '../../automated_init' - -context "Get" do - context "Stream" do - context "Specialization" do - stream_name = Controls::StreamName.example - - batch_size = 1 - correlation = 'someCorrelation' - consumer_group_member = 0 - consumer_group_size = 1 - condition = 'global_position >= 1' - session = Session.build - - get = Get.build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) - - context "Type" do - test "Get::Stream" do - assert(get.instance_of?(Get::Stream)) - end - end - - context "Attributes" do - test "stream_name" do - assert(get.stream_name == stream_name) - end - - test "batch_size" do - assert(get.batch_size == batch_size) - end - - test "correlation" do - assert(get.correlation == correlation) - end - - test "condition" do - assert(get.condition == condition) - end - - test "session" do - assert(get.session == session) - end - end - end - end -end diff --git a/test/automated/get/stream/specialized/batch_size.rb b/test/automated/get/stream/specialized/batch_size.rb new file mode 100644 index 0000000..bbec381 --- /dev/null +++ b/test/automated/get/stream/specialized/batch_size.rb @@ -0,0 +1,19 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Specialized" do + context "Batch Size" do + stream_name, _ = Controls::Put.(instances: 3) + + messages = Get.(stream_name, batch_size: 2) + + number_of_messages = messages.length + + test "Number of messages retrieved is the specified batch size" do + assert(number_of_messages == 2) + end + end + end + end +end diff --git a/test/automated/get/stream/specialized/condition/condition.rb b/test/automated/get/stream/specialized/condition/condition.rb new file mode 100644 index 0000000..e0f6be7 --- /dev/null +++ b/test/automated/get/stream/specialized/condition/condition.rb @@ -0,0 +1,28 @@ +require_relative '../../../../automated_init' + +context "Get" do + context "Stream" do + context "Specialized" do + context "Condition" do + stream_name, _ = Controls::Put.(instances: 3) + + condition = 'position = 0 OR position = 2' + + settings = Postgres::Settings.build + session = Session.new + settings.set(session) + session.options = '-c message_store.sql_condition=on' + + messages = Get.(stream_name, batch_size: 3, condition: condition, session: session) + + message_positions = messages.map do |message| + message.position + end + + test "Returns messages that meet the condition" do + assert(message_positions == [0, 2]) + end + end + end + end +end diff --git a/test/automated/get/stream/specialized/condition/not_activated.rb b/test/automated/get/stream/specialized/condition/not_activated.rb new file mode 100644 index 0000000..0a34e4f --- /dev/null +++ b/test/automated/get/stream/specialized/condition/not_activated.rb @@ -0,0 +1,26 @@ +require_relative '../../../../automated_init' + +context "Get" do + context "Category" do + context "Specialized" do + context "Condition" do + context "Not Activated" do + stream_name = Controls::StreamName.example + + condition = 'some condition' + + settings = Postgres::Settings.build + session = Session.new + settings.set(session) + session.options = nil + + test "Is an error" do + assert_raises(Get::Condition::Error) do + Get.(stream_name, batch_size: 3, condition: condition, session: session) + end + end + end + end + end + end +end diff --git a/test/automated/get/stream/specialized/consumer_groups_error.rb b/test/automated/get/stream/specialized/consumer_groups_error.rb new file mode 100644 index 0000000..77719cf --- /dev/null +++ b/test/automated/get/stream/specialized/consumer_groups_error.rb @@ -0,0 +1,19 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Specialized" do + context "Consumer Groups" do + context "Retrieving from a Stream Using Consumer Groups" do + stream_name = Controls::StreamName.example + + test "Is an error" do + assert_raises(Get::Error) do + Get.(stream_name, consumer_group_member: 0, consumer_group_size: 1) + end + end + end + end + end + end +end diff --git a/test/automated/get/stream/specialized/correlation/correlation.rb b/test/automated/get/stream/specialized/correlation/correlation.rb new file mode 100644 index 0000000..2cbcd68 --- /dev/null +++ b/test/automated/get/stream/specialized/correlation/correlation.rb @@ -0,0 +1,42 @@ +require_relative '../../../../automated_init' + +context "Get" do + context "Stream" do + context "Specialized" do + context "Correlation" do + correlation_category = Controls::Category.example + + correlation_stream_name = Controls::StreamName.example(category: correlation_category) + + correlation_metadata = { + correlation_stream_name: correlation_stream_name + } + + stream_name = Controls::StreamName.example + + message_data = Controls::MessageData::Write.example + + message_data.metadata = { + correlation_stream_name: SecureRandom.hex + } + + Put.(message_data, stream_name) + + 2.times do + message_data = Controls::MessageData::Write.example(metadata: correlation_metadata) + Put.(message_data, stream_name) + end + + message_datas = Get.(stream_name, correlation: correlation_category) + + correlation_stream_names = message_datas.map do |message_data| + message_data.metadata[:correlation_stream_name] + end + + test "Retrieves messages that meet the condition" do + assert(correlation_stream_names == [correlation_stream_name, correlation_stream_name]) + end + end + end + end +end diff --git a/test/automated/get/stream/specialized/correlation/error.rb b/test/automated/get/stream/specialized/correlation/error.rb new file mode 100644 index 0000000..56548bc --- /dev/null +++ b/test/automated/get/stream/specialized/correlation/error.rb @@ -0,0 +1,21 @@ +require_relative '../../../../automated_init' + +context "Get" do + context "Stream" do + context "Specialized" do + context "Correlation" do + context "Not a Category" do + correlation = Controls::StreamName.example + + stream_name = Controls::StreamName.example + + test "Is an error" do + assert_raises MessageStore::Correlation::Error do + Get.(stream_name, correlation: correlation) + end + end + end + end + end + end +end diff --git a/test/automated/get/stream/concrete/dependency.rb b/test/automated/get/stream/specialized/dependency.rb similarity index 97% rename from test/automated/get/stream/concrete/dependency.rb rename to test/automated/get/stream/specialized/dependency.rb index b97fe82..4f2c297 100644 --- a/test/automated/get/stream/concrete/dependency.rb +++ b/test/automated/get/stream/specialized/dependency.rb @@ -2,7 +2,7 @@ context "Get" do context "Stream" do - context "Concrete" do + context "Specialized" do context "Dependency" do write_message = Controls::MessageData::Write.example diff --git a/test/automated/get/stream/concrete/get_messages.rb b/test/automated/get/stream/specialized/get_messages.rb similarity index 96% rename from test/automated/get/stream/concrete/get_messages.rb rename to test/automated/get/stream/specialized/get_messages.rb index 8d11609..dcc2c89 100644 --- a/test/automated/get/stream/concrete/get_messages.rb +++ b/test/automated/get/stream/specialized/get_messages.rb @@ -2,7 +2,7 @@ context "Get" do context "Stream" do - context "Concrete" do + context "Specialized" do context "Get Message" do write_message = Controls::MessageData::Write.example diff --git a/test/automated/get/stream/specialized/no_messages.rb b/test/automated/get/stream/specialized/no_messages.rb new file mode 100644 index 0000000..e275cca --- /dev/null +++ b/test/automated/get/stream/specialized/no_messages.rb @@ -0,0 +1,17 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Specialized" do + context "No Messages" do + stream_name = Controls::StreamName.example + + batch = Get.(stream_name) + + test "Retrieves no messages" do + assert(batch == []) + end + end + end + end +end diff --git a/test/automated/get/stream/position.rb b/test/automated/get/stream/specialized/position.rb similarity index 87% rename from test/automated/get/stream/position.rb rename to test/automated/get/stream/specialized/position.rb index 3e6bad8..389490a 100644 --- a/test/automated/get/stream/position.rb +++ b/test/automated/get/stream/specialized/position.rb @@ -1,4 +1,4 @@ -require_relative '../../automated_init' +require_relative '../../../automated_init' context "Get" do context "Stream" do From 0230550c7d47c7b7365cd5a1ab36f0301e54f664 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 30 Nov 2019 18:48:20 -0600 Subject: [PATCH 527/591] Error handling is specialized --- lib/message_store/postgres/get.rb | 27 +++++++++++----------- lib/message_store/postgres/get/category.rb | 13 +++++++++++ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 9f9222d..14e480f 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -21,6 +21,8 @@ def self.included(cls) abstract :parameter_values abstract :last_position abstract :log_text + + virtual :specialize_error end end @@ -114,21 +116,20 @@ def convert(result) def raise_error(pg_error) error_message = pg_error.message.gsub('ERROR:', '').strip - error_class = nil - - case - when error_message.start_with?('Correlation must be a category') - error_class = Correlation::Error - when error_message.start_with?('Consumer group size must not be less than 1') || - error_message.start_with?('Consumer group member must be less than the group size') || - error_message.start_with?('Consumer group member must not be less than 0') || - error_message.start_with?('Consumer group member and size must be specified') - error_class = Get::Category::ConsumerGroup::Error - when error_message.start_with?('Retrieval with SQL condition is not activated') - error_class = Get::Condition::Error + error_class = specialize_error(error_message) + + if error_class.nil? + if error_message.start_with?('Retrieval with SQL condition is not activated') + error_class = Get::Condition::Error + end + +## remove once correlation is removed from stream get + if error_message.start_with?('Correlation must be a category') + error_class = Correlation::Error + end end - if not error_message.nil? + if not error_class.nil? logger.error { error_message } raise error_class, error_message end diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 654a4f7..176fa12 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -54,6 +54,19 @@ def last_position(batch) batch.last.global_position end + def specialize_error(error_message) + if error_message.start_with?('Correlation must be a category') + return Correlation::Error + end + + if error_message.start_with?('Consumer group size must not be less than 1') || + error_message.start_with?('Consumer group member must be less than the group size') || + error_message.start_with?('Consumer group member must not be less than 0') || + error_message.start_with?('Consumer group member and size must be specified') + return Get::Category::ConsumerGroup::Error + end + end + def log_text(category, position) "Category: #{category}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Consumer Group Member: #{consumer_group_member.inspect}, Consumer Group Size: #{consumer_group_size.inspect}, Condition: #{condition.inspect})" end From 26929b92a2b1f430df01ab541d50a5ccd96088e6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 1 Dec 2019 10:38:36 -0600 Subject: [PATCH 528/591] Category control is control from base library --- .../postgres/controls/category.rb | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/lib/message_store/postgres/controls/category.rb b/lib/message_store/postgres/controls/category.rb index 9678269..15f1667 100644 --- a/lib/message_store/postgres/controls/category.rb +++ b/lib/message_store/postgres/controls/category.rb @@ -1,25 +1,7 @@ module MessageStore module Postgres module Controls - module Category - def self.example(category: nil, randomize_category: nil) - if randomize_category.nil? - if !category.nil? - randomize_category = false - end - end - - randomize_category = true if randomize_category.nil? - - category ||= 'test' - - if randomize_category - category = "#{category}#{SecureRandom.hex(16)}XX" - end - - category - end - end + Category = MessageStore::Controls::Category end end end From d15893e71b94655a05eaacfd20945410b3eac0ea Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 1 Dec 2019 15:21:20 -0600 Subject: [PATCH 529/591] Get specializations assure the input stream name --- lib/message_store/postgres/get.rb | 7 +++---- lib/message_store/postgres/get/category.rb | 6 ++++++ lib/message_store/postgres/get/stream.rb | 11 +++++------ .../category/specialized/stream_name_error.rb | 19 +++++++++++++++++++ .../specialized/consumer_groups_error.rb | 19 ------------------- .../stream/specialized/stream_name_error.rb | 19 +++++++++++++++++++ 6 files changed, 52 insertions(+), 29 deletions(-) create mode 100644 test/automated/get/category/specialized/stream_name_error.rb delete mode 100644 test/automated/get/stream/specialized/consumer_groups_error.rb create mode 100644 test/automated/get/stream/specialized/stream_name_error.rb diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 14e480f..e869d15 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -1,8 +1,6 @@ module MessageStore module Postgres module Get - Error = Class.new(RuntimeError) - def self.included(cls) cls.class_exec do include MessageStore::Get @@ -23,6 +21,7 @@ def self.included(cls) abstract :log_text virtual :specialize_error + virtual :assure end end @@ -35,8 +34,6 @@ def batch_size def self.build(stream_name, **args) cls = specialization(stream_name) - cls.assure(stream_name, args) - session = args.delete(:session) cls.build(stream_name, **args).tap do |instance| @@ -68,6 +65,8 @@ def call(position=nil, stream_name: nil) stream_name ||= self.stream_name + assure + logger.trace(tag: :get) { "Getting message data (#{log_text(stream_name, position)})" } result = get_result(stream_name, position) diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 176fa12..480d3a5 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -71,6 +71,12 @@ def log_text(category, position) "Category: #{category}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Consumer Group Member: #{consumer_group_member.inspect}, Consumer Group Size: #{consumer_group_size.inspect}, Condition: #{condition.inspect})" end + def assure + if not MessageStore::StreamName.category?(category) + raise Error, "Must be a category (Stream Name: #{category})" + end + end + module Defaults def self.position 1 diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index a671667..26aa707 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -2,6 +2,8 @@ module MessageStore module Postgres module Get class Stream + Error = Class.new(RuntimeError) + include Get initializer :stream_name, na(:batch_size), :correlation, :condition @@ -53,12 +55,9 @@ def log_text(stream_name, position) "Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" end - def self.assure(stream_name, args) - consumer_group_member = args.delete(:consumer_group_member) - consumer_group_size = args.delete(:consumer_group_size) - - if [consumer_group_member, consumer_group_size].compact.length > 0 - raise Error, "Consumer groups are supported exclusively for category retrieval (Stream Name: #{stream_name})" + def assure + if MessageStore::StreamName.category?(stream_name) + raise Error, "Must be a stream name (Category: #{stream_name})" end end diff --git a/test/automated/get/category/specialized/stream_name_error.rb b/test/automated/get/category/specialized/stream_name_error.rb new file mode 100644 index 0000000..e8a521d --- /dev/null +++ b/test/automated/get/category/specialized/stream_name_error.rb @@ -0,0 +1,19 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Category" do + context "Specialized" do + context "Stream Name Error" do + context "Retrieving from a Stream Using Stream Name" do + stream_name = Controls::StreamName.example + + test "Is an error" do + assert_raises(Get::Category::Error) do + Get::Category.(stream_name) + end + end + end + end + end + end +end diff --git a/test/automated/get/stream/specialized/consumer_groups_error.rb b/test/automated/get/stream/specialized/consumer_groups_error.rb deleted file mode 100644 index 77719cf..0000000 --- a/test/automated/get/stream/specialized/consumer_groups_error.rb +++ /dev/null @@ -1,19 +0,0 @@ -require_relative '../../../automated_init' - -context "Get" do - context "Stream" do - context "Specialized" do - context "Consumer Groups" do - context "Retrieving from a Stream Using Consumer Groups" do - stream_name = Controls::StreamName.example - - test "Is an error" do - assert_raises(Get::Error) do - Get.(stream_name, consumer_group_member: 0, consumer_group_size: 1) - end - end - end - end - end - end -end diff --git a/test/automated/get/stream/specialized/stream_name_error.rb b/test/automated/get/stream/specialized/stream_name_error.rb new file mode 100644 index 0000000..a06ca92 --- /dev/null +++ b/test/automated/get/stream/specialized/stream_name_error.rb @@ -0,0 +1,19 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Specialized" do + context "Stream Name Error" do + context "Retrieving from a Stream Using Category Name" do + category = Controls::Category.example + + test "Is an error" do + assert_raises(Get::Stream::Error) do + Get::Stream.(category) + end + end + end + end + end + end +end From e7fa88b77d3df0b25430c5fd4181f798b12be6ab Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 1 Dec 2019 15:22:50 -0600 Subject: [PATCH 530/591] Obsolete module is removed --- lib/message_store/postgres/get.rb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index e869d15..5271101 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -5,8 +5,6 @@ def self.included(cls) cls.class_exec do include MessageStore::Get - extend Assure - prepend Call prepend BatchSize @@ -162,11 +160,6 @@ def self.utc_coerced(local_time) end end - module Assure - def assure(*) - end - end - module Defaults def self.batch_size 1000 From 10c6018eaeb80949257cacdf4e2df02ec83f13fd Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 1 Dec 2019 22:20:13 -0600 Subject: [PATCH 531/591] Correlation is no longer supported for stream retrieval --- lib/message_store/postgres/get/stream.rb | 17 ++++++++--------- .../get/stream/generalized/specialization.rb | 7 +------ .../correlation.rb | 0 .../{correlation => _correlation}/error.rb | 0 4 files changed, 9 insertions(+), 15 deletions(-) rename test/automated/get/stream/specialized/{correlation => _correlation}/correlation.rb (100%) rename test/automated/get/stream/specialized/{correlation => _correlation}/error.rb (100%) diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index 26aa707..5b0da42 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -6,22 +6,22 @@ class Stream include Get - initializer :stream_name, na(:batch_size), :correlation, :condition + initializer :stream_name, na(:batch_size), :condition - def self.call(stream_name, position: nil, batch_size: nil, correlation: nil, condition: nil, session: nil) - instance = build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) + def self.call(stream_name, position: nil, batch_size: nil, condition: nil, session: nil) + instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) instance.(position) end - def self.build(stream_name, batch_size: nil, correlation: nil, condition: nil, session: nil) - instance = new(stream_name, batch_size, correlation, condition) + def self.build(stream_name, batch_size: nil, condition: nil, session: nil) + instance = new(stream_name, batch_size, condition) instance.configure(session: session) instance end - def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, correlation: nil, condition: nil, session: nil) + def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, condition: nil, session: nil) attr_name ||= :get - instance = build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) + instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) receiver.public_send("#{attr_name}=", instance) end @@ -34,7 +34,7 @@ def sql_command end def parameters - '$1::varchar, $2::bigint, $3::bigint, $4::varchar, $5::varchar' + '$1::varchar, $2::bigint, $3::bigint, $4::varchar' end def parameter_values(stream_name, position) @@ -42,7 +42,6 @@ def parameter_values(stream_name, position) stream_name, position, batch_size, - correlation, condition ] end diff --git a/test/automated/get/stream/generalized/specialization.rb b/test/automated/get/stream/generalized/specialization.rb index 4735a29..30c6972 100644 --- a/test/automated/get/stream/generalized/specialization.rb +++ b/test/automated/get/stream/generalized/specialization.rb @@ -7,13 +7,12 @@ stream_name = Controls::StreamName.example batch_size = 1 - correlation = 'someCorrelation' consumer_group_member = 0 consumer_group_size = 1 condition = 'global_position >= 1' session = Session.build - get = Get.build(stream_name, batch_size: batch_size, correlation: correlation, condition: condition, session: session) + get = Get.build(stream_name, batch_size: batch_size, condition: condition, session: session) context "Type" do test "Get::Stream" do @@ -30,10 +29,6 @@ assert(get.batch_size == batch_size) end - test "correlation" do - assert(get.correlation == correlation) - end - test "condition" do assert(get.condition == condition) end diff --git a/test/automated/get/stream/specialized/correlation/correlation.rb b/test/automated/get/stream/specialized/_correlation/correlation.rb similarity index 100% rename from test/automated/get/stream/specialized/correlation/correlation.rb rename to test/automated/get/stream/specialized/_correlation/correlation.rb diff --git a/test/automated/get/stream/specialized/correlation/error.rb b/test/automated/get/stream/specialized/_correlation/error.rb similarity index 100% rename from test/automated/get/stream/specialized/correlation/error.rb rename to test/automated/get/stream/specialized/_correlation/error.rb From 714d271cffaecdb0c4c65899e57362fc64fb5424 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 1 Dec 2019 23:00:11 -0600 Subject: [PATCH 532/591] Stream correlation tests are removed --- .../specialized/_correlation/correlation.rb | 42 ------------------- .../stream/specialized/_correlation/error.rb | 21 ---------- 2 files changed, 63 deletions(-) delete mode 100644 test/automated/get/stream/specialized/_correlation/correlation.rb delete mode 100644 test/automated/get/stream/specialized/_correlation/error.rb diff --git a/test/automated/get/stream/specialized/_correlation/correlation.rb b/test/automated/get/stream/specialized/_correlation/correlation.rb deleted file mode 100644 index 2cbcd68..0000000 --- a/test/automated/get/stream/specialized/_correlation/correlation.rb +++ /dev/null @@ -1,42 +0,0 @@ -require_relative '../../../../automated_init' - -context "Get" do - context "Stream" do - context "Specialized" do - context "Correlation" do - correlation_category = Controls::Category.example - - correlation_stream_name = Controls::StreamName.example(category: correlation_category) - - correlation_metadata = { - correlation_stream_name: correlation_stream_name - } - - stream_name = Controls::StreamName.example - - message_data = Controls::MessageData::Write.example - - message_data.metadata = { - correlation_stream_name: SecureRandom.hex - } - - Put.(message_data, stream_name) - - 2.times do - message_data = Controls::MessageData::Write.example(metadata: correlation_metadata) - Put.(message_data, stream_name) - end - - message_datas = Get.(stream_name, correlation: correlation_category) - - correlation_stream_names = message_datas.map do |message_data| - message_data.metadata[:correlation_stream_name] - end - - test "Retrieves messages that meet the condition" do - assert(correlation_stream_names == [correlation_stream_name, correlation_stream_name]) - end - end - end - end -end diff --git a/test/automated/get/stream/specialized/_correlation/error.rb b/test/automated/get/stream/specialized/_correlation/error.rb deleted file mode 100644 index 56548bc..0000000 --- a/test/automated/get/stream/specialized/_correlation/error.rb +++ /dev/null @@ -1,21 +0,0 @@ -require_relative '../../../../automated_init' - -context "Get" do - context "Stream" do - context "Specialized" do - context "Correlation" do - context "Not a Category" do - correlation = Controls::StreamName.example - - stream_name = Controls::StreamName.example - - test "Is an error" do - assert_raises MessageStore::Correlation::Error do - Get.(stream_name, correlation: correlation) - end - end - end - end - end - end -end From 664271a7099fb87b9a396f60dec2c8b2fbc80771 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 3 Dec 2019 09:52:22 -0600 Subject: [PATCH 533/591] Consumer group and correlation is moved to generalized namespace --- lib/message_store/postgres.rb | 3 +-- .../postgres/get/category/consumer_group.rb | 11 ----------- .../get/category/specialized/correlation/error.rb | 2 +- 3 files changed, 2 insertions(+), 14 deletions(-) delete mode 100644 lib/message_store/postgres/get/category/consumer_group.rb diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index 2a8b9c1..f00a211 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -17,7 +17,6 @@ require 'message_store/postgres/get' require 'message_store/postgres/get/condition' require 'message_store/postgres/get/stream' -require 'message_store/postgres/get/category/consumer_group' -require 'message_store/postgres/get/category' require 'message_store/postgres/get/stream/last' +require 'message_store/postgres/get/category' require 'message_store/postgres/read' diff --git a/lib/message_store/postgres/get/category/consumer_group.rb b/lib/message_store/postgres/get/category/consumer_group.rb deleted file mode 100644 index 457d1cc..0000000 --- a/lib/message_store/postgres/get/category/consumer_group.rb +++ /dev/null @@ -1,11 +0,0 @@ -module MessageStore - module Postgres - module Get - class Category - module ConsumerGroup - Error = Class.new(RuntimeError) - end - end - end - end -end diff --git a/test/automated/get/category/specialized/correlation/error.rb b/test/automated/get/category/specialized/correlation/error.rb index c7e3490..eed483f 100644 --- a/test/automated/get/category/specialized/correlation/error.rb +++ b/test/automated/get/category/specialized/correlation/error.rb @@ -10,7 +10,7 @@ category = Controls::Category.example test "Is an error" do - assert_raises(MessageStore::Correlation::Error) do + assert_raises(MessageStore::Postgres::Correlation::Error) do Get::Category.(category, correlation: correlation) end end From e177038dff3cfcd1a5ebb8e894099fdd21d087e5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 3 Dec 2019 11:14:04 -0600 Subject: [PATCH 534/591] Category retrieval errors --- lib/message_store/postgres.rb | 2 ++ lib/message_store/postgres/get.rb | 6 +++--- lib/message_store/postgres/get/category.rb | 4 ++-- .../postgres/get/category/consumer_group.rb | 11 +++++++++++ .../postgres/get/category/correlation.rb | 11 +++++++++++ .../get/category/specialized/consumer_groups/error.rb | 11 +++++------ .../get/category/specialized/correlation/error.rb | 2 +- 7 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 lib/message_store/postgres/get/category/consumer_group.rb create mode 100644 lib/message_store/postgres/get/category/correlation.rb diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index f00a211..fef1e16 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -19,4 +19,6 @@ require 'message_store/postgres/get/stream' require 'message_store/postgres/get/stream/last' require 'message_store/postgres/get/category' +require 'message_store/postgres/get/category/correlation' +require 'message_store/postgres/get/category/consumer_group' require 'message_store/postgres/read' diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 5271101..da3cb4b 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -121,9 +121,9 @@ def raise_error(pg_error) end ## remove once correlation is removed from stream get - if error_message.start_with?('Correlation must be a category') - error_class = Correlation::Error - end + # if error_message.start_with?('Correlation must be a category') + # error_class = Correlation::Error + # end end if not error_class.nil? diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 480d3a5..63c46f9 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -56,14 +56,14 @@ def last_position(batch) def specialize_error(error_message) if error_message.start_with?('Correlation must be a category') - return Correlation::Error + return MessageStore::Postgres::Get::Category::Correlation::Error end if error_message.start_with?('Consumer group size must not be less than 1') || error_message.start_with?('Consumer group member must be less than the group size') || error_message.start_with?('Consumer group member must not be less than 0') || error_message.start_with?('Consumer group member and size must be specified') - return Get::Category::ConsumerGroup::Error + return MessageStore::Postgres::Get::Category::ConsumerGroup::Error end end diff --git a/lib/message_store/postgres/get/category/consumer_group.rb b/lib/message_store/postgres/get/category/consumer_group.rb new file mode 100644 index 0000000..457d1cc --- /dev/null +++ b/lib/message_store/postgres/get/category/consumer_group.rb @@ -0,0 +1,11 @@ +module MessageStore + module Postgres + module Get + class Category + module ConsumerGroup + Error = Class.new(RuntimeError) + end + end + end + end +end diff --git a/lib/message_store/postgres/get/category/correlation.rb b/lib/message_store/postgres/get/category/correlation.rb new file mode 100644 index 0000000..8d70b07 --- /dev/null +++ b/lib/message_store/postgres/get/category/correlation.rb @@ -0,0 +1,11 @@ +module MessageStore + module Postgres + module Get + class Category + module Correlation + Error = Class.new(RuntimeError) + end + end + end + end +end diff --git a/test/automated/get/category/specialized/consumer_groups/error.rb b/test/automated/get/category/specialized/consumer_groups/error.rb index c8f03d0..46ae2b2 100644 --- a/test/automated/get/category/specialized/consumer_groups/error.rb +++ b/test/automated/get/category/specialized/consumer_groups/error.rb @@ -5,10 +5,9 @@ context "Specialized" do context "Consumer Groups" do context "Error" do - context "Consumer Group Size Is Less than 1" do test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: 0, consumer_group_size: 0) end end @@ -16,7 +15,7 @@ context "Consumer Group Member Is Greater than the Consumer Group Size" do test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: 2, consumer_group_size: 1) end end @@ -24,7 +23,7 @@ context "Consumer Group Member Is Less than 0" do test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: -1, consumer_group_size: 1) end end @@ -32,7 +31,7 @@ context "Consumer Group Size is Missing" do test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: 0) end end @@ -40,7 +39,7 @@ context "Consumer Group Member is Missing" do test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_size: 1) end end diff --git a/test/automated/get/category/specialized/correlation/error.rb b/test/automated/get/category/specialized/correlation/error.rb index eed483f..178adff 100644 --- a/test/automated/get/category/specialized/correlation/error.rb +++ b/test/automated/get/category/specialized/correlation/error.rb @@ -10,7 +10,7 @@ category = Controls::Category.example test "Is an error" do - assert_raises(MessageStore::Postgres::Correlation::Error) do + assert_raises(MessageStore::Postgres::Get::Category::Correlation::Error) do Get::Category.(category, correlation: correlation) end end From db1c3640d35e251f1179f2c6fabde6229b87475a Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 3 Dec 2019 11:14:04 -0600 Subject: [PATCH 535/591] Category retrieval errors --- lib/message_store/postgres.rb | 2 ++ lib/message_store/postgres/get.rb | 17 +++++----------- lib/message_store/postgres/get/category.rb | 13 +++++------- .../postgres/get/category/consumer_group.rb | 20 +++++++++++++++++++ .../postgres/get/category/correlation.rb | 17 ++++++++++++++++ lib/message_store/postgres/get/condition.rb | 6 ++++++ ...ot_activated.rb => not_activated_error.rb} | 2 +- .../specialized/consumer_groups/error.rb | 11 +++++----- .../category/specialized/correlation/error.rb | 2 +- 9 files changed, 62 insertions(+), 28 deletions(-) create mode 100644 lib/message_store/postgres/get/category/consumer_group.rb create mode 100644 lib/message_store/postgres/get/category/correlation.rb rename test/automated/get/category/specialized/condition/{not_activated.rb => not_activated_error.rb} (93%) diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb index f00a211..fef1e16 100644 --- a/lib/message_store/postgres.rb +++ b/lib/message_store/postgres.rb @@ -19,4 +19,6 @@ require 'message_store/postgres/get/stream' require 'message_store/postgres/get/stream/last' require 'message_store/postgres/get/category' +require 'message_store/postgres/get/category/correlation' +require 'message_store/postgres/get/category/consumer_group' require 'message_store/postgres/read' diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 5271101..8c5f0f1 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -113,22 +113,15 @@ def convert(result) def raise_error(pg_error) error_message = pg_error.message.gsub('ERROR:', '').strip - error_class = specialize_error(error_message) + error = Condition.error(error_message) - if error_class.nil? - if error_message.start_with?('Retrieval with SQL condition is not activated') - error_class = Get::Condition::Error - end - -## remove once correlation is removed from stream get - if error_message.start_with?('Correlation must be a category') - error_class = Correlation::Error - end + if error.nil? + error = specialize_error(error_message) end - if not error_class.nil? + if not error.nil? logger.error { error_message } - raise error_class, error_message + raise error end raise pg_error diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 480d3a5..0d5a60e 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -55,16 +55,13 @@ def last_position(batch) end def specialize_error(error_message) - if error_message.start_with?('Correlation must be a category') - return Correlation::Error - end + error = Correlation.error(error_message) - if error_message.start_with?('Consumer group size must not be less than 1') || - error_message.start_with?('Consumer group member must be less than the group size') || - error_message.start_with?('Consumer group member must not be less than 0') || - error_message.start_with?('Consumer group member and size must be specified') - return Get::Category::ConsumerGroup::Error + if error.nil? + error = ConsumerGroup.error(error_message) end + + error end def log_text(category, position) diff --git a/lib/message_store/postgres/get/category/consumer_group.rb b/lib/message_store/postgres/get/category/consumer_group.rb new file mode 100644 index 0000000..fc5bd0b --- /dev/null +++ b/lib/message_store/postgres/get/category/consumer_group.rb @@ -0,0 +1,20 @@ +module MessageStore + module Postgres + module Get + class Category + module ConsumerGroup + Error = Class.new(RuntimeError) + + def self.error(error_message) + if error_message.start_with?('Consumer group size must not be less than 1') || + error_message.start_with?('Consumer group member must be less than the group size') || + error_message.start_with?('Consumer group member must not be less than 0') || + error_message.start_with?('Consumer group member and size must be specified') + Error.new(error_message) + end + end + end + end + end + end +end diff --git a/lib/message_store/postgres/get/category/correlation.rb b/lib/message_store/postgres/get/category/correlation.rb new file mode 100644 index 0000000..c857077 --- /dev/null +++ b/lib/message_store/postgres/get/category/correlation.rb @@ -0,0 +1,17 @@ +module MessageStore + module Postgres + module Get + class Category + module Correlation + Error = Class.new(RuntimeError) + + def self.error(error_message) + if error_message.start_with?('Correlation must be a category') + Error.new(error_message) + end + end + end + end + end + end +end diff --git a/lib/message_store/postgres/get/condition.rb b/lib/message_store/postgres/get/condition.rb index 7657c78..7e0cb99 100644 --- a/lib/message_store/postgres/get/condition.rb +++ b/lib/message_store/postgres/get/condition.rb @@ -3,6 +3,12 @@ module Postgres module Get module Condition Error = Class.new(RuntimeError) + + def self.error(error_message) + if error_message.start_with?('Retrieval with SQL condition is not activated') + Get::Condition::Error.new(error_message) + end + end end end end diff --git a/test/automated/get/category/specialized/condition/not_activated.rb b/test/automated/get/category/specialized/condition/not_activated_error.rb similarity index 93% rename from test/automated/get/category/specialized/condition/not_activated.rb rename to test/automated/get/category/specialized/condition/not_activated_error.rb index c85c009..5b1dad2 100644 --- a/test/automated/get/category/specialized/condition/not_activated.rb +++ b/test/automated/get/category/specialized/condition/not_activated_error.rb @@ -4,7 +4,7 @@ context "Category" do context "Specialized" do context "Condition" do - context "Not Activated" do + context "Not Activated Error" do category = Controls::Category.example condition = 'some condition' diff --git a/test/automated/get/category/specialized/consumer_groups/error.rb b/test/automated/get/category/specialized/consumer_groups/error.rb index c8f03d0..46ae2b2 100644 --- a/test/automated/get/category/specialized/consumer_groups/error.rb +++ b/test/automated/get/category/specialized/consumer_groups/error.rb @@ -5,10 +5,9 @@ context "Specialized" do context "Consumer Groups" do context "Error" do - context "Consumer Group Size Is Less than 1" do test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: 0, consumer_group_size: 0) end end @@ -16,7 +15,7 @@ context "Consumer Group Member Is Greater than the Consumer Group Size" do test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: 2, consumer_group_size: 1) end end @@ -24,7 +23,7 @@ context "Consumer Group Member Is Less than 0" do test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: -1, consumer_group_size: 1) end end @@ -32,7 +31,7 @@ context "Consumer Group Size is Missing" do test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: 0) end end @@ -40,7 +39,7 @@ context "Consumer Group Member is Missing" do test "Is an error" do - assert_raises(Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_size: 1) end end diff --git a/test/automated/get/category/specialized/correlation/error.rb b/test/automated/get/category/specialized/correlation/error.rb index eed483f..178adff 100644 --- a/test/automated/get/category/specialized/correlation/error.rb +++ b/test/automated/get/category/specialized/correlation/error.rb @@ -10,7 +10,7 @@ category = Controls::Category.example test "Is an error" do - assert_raises(MessageStore::Postgres::Correlation::Error) do + assert_raises(MessageStore::Postgres::Get::Category::Correlation::Error) do Get::Category.(category, correlation: correlation) end end From 193f97cd1c422dadca41605c13796eb36e22cf73 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 4 Dec 2019 22:34:28 -0600 Subject: [PATCH 536/591] Getting last message raises error when the stream name is a category --- lib/message_store/postgres/get/stream/last.rb | 27 +++++++++++++++++-- .../automated/get/stream/get_last/get_last.rb | 1 - .../get/stream/get_last/stream_name_error.rb | 17 ++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 test/automated/get/stream/get_last/stream_name_error.rb diff --git a/lib/message_store/postgres/get/stream/last.rb b/lib/message_store/postgres/get/stream/last.rb index b807ec5..1949553 100644 --- a/lib/message_store/postgres/get/stream/last.rb +++ b/lib/message_store/postgres/get/stream/last.rb @@ -3,6 +3,8 @@ module Postgres module Get class Stream class Last + Error = Class.new(RuntimeError) + include MessageStore::Get::Stream::Last dependency :session, Session @@ -31,11 +33,21 @@ def get_result(stream_name) sql_command = self.class.sql_command(stream_name) - params = [ + parameter_values = [ stream_name ] - result = session.execute(sql_command, params) + +## result = session.execute(sql_command, params) + + + begin + result = session.execute(sql_command, parameter_values) + rescue PG::RaiseException => e + raise_error(e) + end + + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" } @@ -64,6 +76,17 @@ def convert(record) message_data end + def raise_error(pg_error) + error_message = pg_error.message.gsub('ERROR:', '').strip + + if error_message.start_with?('Must be a stream name') + logger.error { error_message } + raise Error, error_message + end + + raise pg_error + end + module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? diff --git a/test/automated/get/stream/get_last/get_last.rb b/test/automated/get/stream/get_last/get_last.rb index 612da6e..e94becc 100644 --- a/test/automated/get/stream/get_last/get_last.rb +++ b/test/automated/get/stream/get_last/get_last.rb @@ -17,4 +17,3 @@ end end end - diff --git a/test/automated/get/stream/get_last/stream_name_error.rb b/test/automated/get/stream/get_last/stream_name_error.rb new file mode 100644 index 0000000..070a46b --- /dev/null +++ b/test/automated/get/stream/get_last/stream_name_error.rb @@ -0,0 +1,17 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Last" do + context "Not a Stream Name" do + category = Controls::Category.example + + test "Is an error" do + assert_raises(MessageStore::Postgres::Get::Stream::Last::Error) do + Get::Stream::Last.(category) + end + end + end + end + end +end From a199153296a23db9ae1a160b77948a2bb591ac36 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 3 Dec 2019 11:14:04 -0600 Subject: [PATCH 537/591] Category retrieval errors --- lib/message_store/postgres/get.rb | 17 +++++------------ lib/message_store/postgres/get/category.rb | 13 +++++-------- .../postgres/get/category/consumer_group.rb | 9 +++++++++ .../postgres/get/category/correlation.rb | 6 ++++++ lib/message_store/postgres/get/condition.rb | 6 ++++++ ...{not_activated.rb => not_activated_error.rb} | 2 +- 6 files changed, 32 insertions(+), 21 deletions(-) rename test/automated/get/category/specialized/condition/{not_activated.rb => not_activated_error.rb} (93%) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index da3cb4b..8c5f0f1 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -113,22 +113,15 @@ def convert(result) def raise_error(pg_error) error_message = pg_error.message.gsub('ERROR:', '').strip - error_class = specialize_error(error_message) + error = Condition.error(error_message) - if error_class.nil? - if error_message.start_with?('Retrieval with SQL condition is not activated') - error_class = Get::Condition::Error - end - -## remove once correlation is removed from stream get - # if error_message.start_with?('Correlation must be a category') - # error_class = Correlation::Error - # end + if error.nil? + error = specialize_error(error_message) end - if not error_class.nil? + if not error.nil? logger.error { error_message } - raise error_class, error_message + raise error end raise pg_error diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 63c46f9..0d5a60e 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -55,16 +55,13 @@ def last_position(batch) end def specialize_error(error_message) - if error_message.start_with?('Correlation must be a category') - return MessageStore::Postgres::Get::Category::Correlation::Error - end + error = Correlation.error(error_message) - if error_message.start_with?('Consumer group size must not be less than 1') || - error_message.start_with?('Consumer group member must be less than the group size') || - error_message.start_with?('Consumer group member must not be less than 0') || - error_message.start_with?('Consumer group member and size must be specified') - return MessageStore::Postgres::Get::Category::ConsumerGroup::Error + if error.nil? + error = ConsumerGroup.error(error_message) end + + error end def log_text(category, position) diff --git a/lib/message_store/postgres/get/category/consumer_group.rb b/lib/message_store/postgres/get/category/consumer_group.rb index 457d1cc..fc5bd0b 100644 --- a/lib/message_store/postgres/get/category/consumer_group.rb +++ b/lib/message_store/postgres/get/category/consumer_group.rb @@ -4,6 +4,15 @@ module Get class Category module ConsumerGroup Error = Class.new(RuntimeError) + + def self.error(error_message) + if error_message.start_with?('Consumer group size must not be less than 1') || + error_message.start_with?('Consumer group member must be less than the group size') || + error_message.start_with?('Consumer group member must not be less than 0') || + error_message.start_with?('Consumer group member and size must be specified') + Error.new(error_message) + end + end end end end diff --git a/lib/message_store/postgres/get/category/correlation.rb b/lib/message_store/postgres/get/category/correlation.rb index 8d70b07..c857077 100644 --- a/lib/message_store/postgres/get/category/correlation.rb +++ b/lib/message_store/postgres/get/category/correlation.rb @@ -4,6 +4,12 @@ module Get class Category module Correlation Error = Class.new(RuntimeError) + + def self.error(error_message) + if error_message.start_with?('Correlation must be a category') + Error.new(error_message) + end + end end end end diff --git a/lib/message_store/postgres/get/condition.rb b/lib/message_store/postgres/get/condition.rb index 7657c78..7e0cb99 100644 --- a/lib/message_store/postgres/get/condition.rb +++ b/lib/message_store/postgres/get/condition.rb @@ -3,6 +3,12 @@ module Postgres module Get module Condition Error = Class.new(RuntimeError) + + def self.error(error_message) + if error_message.start_with?('Retrieval with SQL condition is not activated') + Get::Condition::Error.new(error_message) + end + end end end end diff --git a/test/automated/get/category/specialized/condition/not_activated.rb b/test/automated/get/category/specialized/condition/not_activated_error.rb similarity index 93% rename from test/automated/get/category/specialized/condition/not_activated.rb rename to test/automated/get/category/specialized/condition/not_activated_error.rb index c85c009..5b1dad2 100644 --- a/test/automated/get/category/specialized/condition/not_activated.rb +++ b/test/automated/get/category/specialized/condition/not_activated_error.rb @@ -4,7 +4,7 @@ context "Category" do context "Specialized" do context "Condition" do - context "Not Activated" do + context "Not Activated Error" do category = Controls::Category.example condition = 'some condition' From 94d8e23a87724251d708eee9356c9af3ebec2e85 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 4 Dec 2019 22:34:28 -0600 Subject: [PATCH 538/591] Getting last message raises error when the stream name is a category --- lib/message_store/postgres/get/stream/last.rb | 27 +++++++++++++++++-- .../automated/get/stream/get_last/get_last.rb | 1 - .../get/stream/get_last/stream_name_error.rb | 17 ++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 test/automated/get/stream/get_last/stream_name_error.rb diff --git a/lib/message_store/postgres/get/stream/last.rb b/lib/message_store/postgres/get/stream/last.rb index b807ec5..1949553 100644 --- a/lib/message_store/postgres/get/stream/last.rb +++ b/lib/message_store/postgres/get/stream/last.rb @@ -3,6 +3,8 @@ module Postgres module Get class Stream class Last + Error = Class.new(RuntimeError) + include MessageStore::Get::Stream::Last dependency :session, Session @@ -31,11 +33,21 @@ def get_result(stream_name) sql_command = self.class.sql_command(stream_name) - params = [ + parameter_values = [ stream_name ] - result = session.execute(sql_command, params) + +## result = session.execute(sql_command, params) + + + begin + result = session.execute(sql_command, parameter_values) + rescue PG::RaiseException => e + raise_error(e) + end + + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" } @@ -64,6 +76,17 @@ def convert(record) message_data end + def raise_error(pg_error) + error_message = pg_error.message.gsub('ERROR:', '').strip + + if error_message.start_with?('Must be a stream name') + logger.error { error_message } + raise Error, error_message + end + + raise pg_error + end + module Deserialize def self.data(serialized_data) return nil if serialized_data.nil? diff --git a/test/automated/get/stream/get_last/get_last.rb b/test/automated/get/stream/get_last/get_last.rb index 612da6e..e94becc 100644 --- a/test/automated/get/stream/get_last/get_last.rb +++ b/test/automated/get/stream/get_last/get_last.rb @@ -17,4 +17,3 @@ end end end - diff --git a/test/automated/get/stream/get_last/stream_name_error.rb b/test/automated/get/stream/get_last/stream_name_error.rb new file mode 100644 index 0000000..070a46b --- /dev/null +++ b/test/automated/get/stream/get_last/stream_name_error.rb @@ -0,0 +1,17 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Last" do + context "Not a Stream Name" do + category = Controls::Category.example + + test "Is an error" do + assert_raises(MessageStore::Postgres::Get::Stream::Last::Error) do + Get::Stream::Last.(category) + end + end + end + end + end +end From 7c96661d3939f93de5529b6a40adacd75b67a2ac Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 4 Dec 2019 22:47:52 -0600 Subject: [PATCH 539/591] Vestigial code is removed --- lib/message_store/postgres/get/stream/last.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/message_store/postgres/get/stream/last.rb b/lib/message_store/postgres/get/stream/last.rb index 1949553..8d9540d 100644 --- a/lib/message_store/postgres/get/stream/last.rb +++ b/lib/message_store/postgres/get/stream/last.rb @@ -37,18 +37,12 @@ def get_result(stream_name) stream_name ] - -## result = session.execute(sql_command, params) - - begin result = session.execute(sql_command, parameter_values) rescue PG::RaiseException => e raise_error(e) end - - logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" } return nil if result.ntuples == 0 From bc4f0dd349aeb5b411a453ab9e5239ff71c78132 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 5 Dec 2019 18:01:08 -0600 Subject: [PATCH 540/591] Duplicated implementation is replaced with use of generalizations --- lib/message_store/postgres/get.rb | 20 +++++++++----- lib/message_store/postgres/get/stream/last.rb | 26 ++----------------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 8c5f0f1..31a5caa 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -98,11 +98,7 @@ def convert(result) logger.trace(tag: :get) { "Converting result to message data (Result Count: #{result.ntuples})" } message_data = result.map do |record| - record['data'] = Deserialize.data(record['data']) - record['metadata'] = Deserialize.metadata(record['metadata']) - record['time'] = Time.utc_coerced(record['time']) - - MessageData::Read.build(record) + Get.message_data(record) end logger.debug(tag: :get) { "Converted result to message data (Message Data Count: #{message_data.length})" } @@ -110,8 +106,16 @@ def convert(result) message_data end + def self.message_data(record) + record['data'] = Get::Deserialize.data(record['data']) + record['metadata'] = Get::Deserialize.metadata(record['metadata']) + record['time'] = Get::Time.utc_coerced(record['time']) + + MessageData::Read.build(record) + end + def raise_error(pg_error) - error_message = pg_error.message.gsub('ERROR:', '').strip + error_message = Get.error_message(pg_error) error = Condition.error(error_message) @@ -127,6 +131,10 @@ def raise_error(pg_error) raise pg_error end + def self.error_message(pg_error) + pg_error.message.gsub('ERROR:', '').strip + end + def self.specialization(stream_name) if StreamName.category?(stream_name) Category diff --git a/lib/message_store/postgres/get/stream/last.rb b/lib/message_store/postgres/get/stream/last.rb index 8d9540d..51c903b 100644 --- a/lib/message_store/postgres/get/stream/last.rb +++ b/lib/message_store/postgres/get/stream/last.rb @@ -59,11 +59,7 @@ def self.sql_command(stream_name) def convert(record) logger.trace(tag: :get) { "Converting record to message data" } - record['data'] = Deserialize.data(record['data']) - record['metadata'] = Deserialize.metadata(record['metadata']) - record['time'] = Time.utc_coerced(record['time']) - - message_data = MessageData::Read.build(record) + message_data = Get.message_data(record) logger.debug(tag: :get) { "Converted record to message data" } @@ -71,7 +67,7 @@ def convert(record) end def raise_error(pg_error) - error_message = pg_error.message.gsub('ERROR:', '').strip + error_message = Get.error_message(pg_error) if error_message.start_with?('Must be a stream name') logger.error { error_message } @@ -80,24 +76,6 @@ def raise_error(pg_error) raise pg_error end - - module Deserialize - def self.data(serialized_data) - return nil if serialized_data.nil? - Transform::Read.(serialized_data, :json, MessageData::Hash) - end - - def self.metadata(serialized_metadata) - return nil if serialized_metadata.nil? - Transform::Read.(serialized_metadata, :json, MessageData::Hash) - end - end - - module Time - def self.utc_coerced(local_time) - Clock::UTC.coerce(local_time) - end - end end end end From ca60deb0e8d4d95860b96f46d544cc8a153fb00c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sat, 7 Dec 2019 17:50:00 -0600 Subject: [PATCH 541/591] Dependency on evt-message_store-postgres-database is replaced with message-db --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 9d4e6ed..1c799fb 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -19,10 +19,10 @@ Gem::Specification.new do |s| s.bindir = 'scripts' s.add_runtime_dependency 'evt-message_store' - s.add_runtime_dependency 'evt-message_store-postgres-database' s.add_runtime_dependency 'evt-log' s.add_runtime_dependency 'evt-settings' + s.add_runtime_dependency 'message-db' s.add_runtime_dependency 'pg' s.add_development_dependency 'test_bench' From c6cdb14ea8bcbac07a48c7757412cb8394384325 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 8 Dec 2019 17:23:12 -0600 Subject: [PATCH 542/591] Category is a legitimate input to last message retrieval --- lib/message_store/postgres/get/stream/last.rb | 19 +-------------- .../automated/get/stream/get_last/category.rb | 23 +++++++++++++++++++ .../automated/get/stream/get_last/get_last.rb | 19 --------------- .../get/stream/get_last/stream_name.rb | 21 +++++++++++++++++ .../get/stream/get_last/stream_name_error.rb | 17 -------------- 5 files changed, 45 insertions(+), 54 deletions(-) create mode 100644 test/automated/get/stream/get_last/category.rb delete mode 100644 test/automated/get/stream/get_last/get_last.rb create mode 100644 test/automated/get/stream/get_last/stream_name.rb delete mode 100644 test/automated/get/stream/get_last/stream_name_error.rb diff --git a/lib/message_store/postgres/get/stream/last.rb b/lib/message_store/postgres/get/stream/last.rb index 51c903b..865e7f8 100644 --- a/lib/message_store/postgres/get/stream/last.rb +++ b/lib/message_store/postgres/get/stream/last.rb @@ -3,8 +3,6 @@ module Postgres module Get class Stream class Last - Error = Class.new(RuntimeError) - include MessageStore::Get::Stream::Last dependency :session, Session @@ -37,11 +35,7 @@ def get_result(stream_name) stream_name ] - begin - result = session.execute(sql_command, parameter_values) - rescue PG::RaiseException => e - raise_error(e) - end + result = session.execute(sql_command, parameter_values) logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" } @@ -65,17 +59,6 @@ def convert(record) message_data end - - def raise_error(pg_error) - error_message = Get.error_message(pg_error) - - if error_message.start_with?('Must be a stream name') - logger.error { error_message } - raise Error, error_message - end - - raise pg_error - end end end end diff --git a/test/automated/get/stream/get_last/category.rb b/test/automated/get/stream/get_last/category.rb new file mode 100644 index 0000000..8355cb9 --- /dev/null +++ b/test/automated/get/stream/get_last/category.rb @@ -0,0 +1,23 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Last" do + context "Category" do + category = Controls::Category.example + + Controls::Put.(instances: 2, stream_name: category) + + write_message = Controls::MessageData::Write.example + + Put.(write_message, category) + + last_message = Get::Stream::Last.(category) + + test "Gets the last message in the stream" do + assert(last_message.data == write_message.data) + end + end + end + end +end diff --git a/test/automated/get/stream/get_last/get_last.rb b/test/automated/get/stream/get_last/get_last.rb deleted file mode 100644 index e94becc..0000000 --- a/test/automated/get/stream/get_last/get_last.rb +++ /dev/null @@ -1,19 +0,0 @@ -require_relative '../../../automated_init' - -context "Get" do - context "Stream" do - context "Last" do - stream_name, _ = Controls::Put.(instances: 2) - - write_message = Controls::MessageData::Write.example - - position = Put.(write_message, stream_name) - - last_message = Get::Stream::Last.(stream_name) - - test "Gets the last message in the stream" do - assert(last_message.data == write_message.data) - end - end - end -end diff --git a/test/automated/get/stream/get_last/stream_name.rb b/test/automated/get/stream/get_last/stream_name.rb new file mode 100644 index 0000000..d62c3cd --- /dev/null +++ b/test/automated/get/stream/get_last/stream_name.rb @@ -0,0 +1,21 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Last" do + context "Stream Name" do + stream_name, _ = Controls::Put.(instances: 2) + + write_message = Controls::MessageData::Write.example + + Put.(write_message, stream_name) + + last_message = Get::Stream::Last.(stream_name) + + test "Gets the last message in the stream" do + assert(last_message.data == write_message.data) + end + end + end + end +end diff --git a/test/automated/get/stream/get_last/stream_name_error.rb b/test/automated/get/stream/get_last/stream_name_error.rb deleted file mode 100644 index 070a46b..0000000 --- a/test/automated/get/stream/get_last/stream_name_error.rb +++ /dev/null @@ -1,17 +0,0 @@ -require_relative '../../../automated_init' - -context "Get" do - context "Stream" do - context "Last" do - context "Not a Stream Name" do - category = Controls::Category.example - - test "Is an error" do - assert_raises(MessageStore::Postgres::Get::Stream::Last::Error) do - Get::Stream::Last.(category) - end - end - end - end - end -end From cfa24a85443b8789e4d424a01cea34eb7b775408 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 8 Dec 2019 18:18:25 -0600 Subject: [PATCH 543/591] Test name is clarified --- .../get/category/specialized/condition/not_activated_error.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/automated/get/category/specialized/condition/not_activated_error.rb b/test/automated/get/category/specialized/condition/not_activated_error.rb index 5b1dad2..41dd97b 100644 --- a/test/automated/get/category/specialized/condition/not_activated_error.rb +++ b/test/automated/get/category/specialized/condition/not_activated_error.rb @@ -4,7 +4,7 @@ context "Category" do context "Specialized" do context "Condition" do - context "Not Activated Error" do + context "Condition Is Not Activated" do category = Controls::Category.example condition = 'some condition' From 60e195b44c501ec10d1104dc76a632cb5488f515 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 10 Dec 2019 15:13:20 -0600 Subject: [PATCH 544/591] Package version is increased from 1.2.0.0 to 2.3.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 1c799fb..5e72238 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '1.2.0.0' + s.version = '2.3.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 26dc3e9e19a4c5931967a7a97f660d02531a9da6 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 10 Dec 2019 20:31:03 -0600 Subject: [PATCH 545/591] Obsolete session configuration implementation is removed --- lib/message_store/postgres/get.rb | 8 +------- lib/message_store/postgres/get/category.rb | 4 ---- lib/message_store/postgres/get/stream.rb | 4 ---- 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb index 31a5caa..ce440d8 100644 --- a/lib/message_store/postgres/get.rb +++ b/lib/message_store/postgres/get.rb @@ -10,7 +10,6 @@ def self.included(cls) dependency :session, Session - abstract :configure abstract :stream_name abstract :sql_command abstract :parameters @@ -31,12 +30,7 @@ def batch_size def self.build(stream_name, **args) cls = specialization(stream_name) - - session = args.delete(:session) - - cls.build(stream_name, **args).tap do |instance| - instance.configure(session: session) - end + cls.build(stream_name, **args) end def self.configure(receiver, stream_name, **args) diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb index 0d5a60e..d226735 100644 --- a/lib/message_store/postgres/get/category.rb +++ b/lib/message_store/postgres/get/category.rb @@ -26,10 +26,6 @@ def self.configure(receiver, category, attr_name: nil, batch_size: nil, correlat receiver.public_send("#{attr_name}=", instance) end - def configure(session: nil) - Session.configure(self, session: session) - end - def sql_command "SELECT * FROM get_category_messages(#{parameters});" end diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index 5b0da42..350a9d8 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -25,10 +25,6 @@ def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, condi receiver.public_send("#{attr_name}=", instance) end - def configure(session: nil) - Session.configure(self, session: session) - end - def sql_command "SELECT * FROM get_stream_messages(#{parameters});" end From 2494949df49c7449284d728cd2a6c23945987892 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 10 Dec 2019 20:36:31 -0600 Subject: [PATCH 546/591] Package version is increased from 2.3.0.0 to 2.4.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 5e72238..a3efc76 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '2.3.0.0' + s.version = '2.4.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 6fa7f256f2d105dbc4272bb570df396902e07668 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 09:27:59 -0600 Subject: [PATCH 547/591] Vestigial logging of correlation in the stream get is removed --- lib/message_store/postgres/get/stream.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb index 350a9d8..f769e9b 100644 --- a/lib/message_store/postgres/get/stream.rb +++ b/lib/message_store/postgres/get/stream.rb @@ -47,7 +47,7 @@ def last_position(batch) end def log_text(stream_name, position) - "Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Condition: #{condition.inspect})" + "Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition.inspect})" end def assure From a68c2191bea332b29f4d6ed1e76a9057bb2b69a7 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 09:30:13 -0600 Subject: [PATCH 548/591] Package version is increased from 2.4.0.0 to 2.4.0.1 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index a3efc76..51c2f4f 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '2.4.0.0' + s.version = '2.4.0.1' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 6fe46c2d993f5acee518cfd17e3703913fa78722 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 09:33:54 -0600 Subject: [PATCH 549/591] Change log is updated as of v2.4.0.1 --- CHANGES.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 CHANGES.md diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..e14e8ee --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,7 @@ +# Changes + +## 2.4.0.1 + +Thu Dec 19 2019 + +- Vestigial logging of correlation in Get::Stream is removed (It no longer has correlation handling as of v2) From ae074158f37dd88b24fdba8a6763819f16bcf5c5 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 09:56:14 -0600 Subject: [PATCH 550/591] Get benchmark is brought up to date with implementation --- test/benchmark/get.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/benchmark/get.rb b/test/benchmark/get.rb index 1287995..897773e 100644 --- a/test/benchmark/get.rb +++ b/test/benchmark/get.rb @@ -2,6 +2,10 @@ defaults = Test::Benchmark::Defaults.build +puts +puts "Running benchmark (#{__FILE__})..." +puts + list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) put = Put.build @@ -11,7 +15,7 @@ put.(entry.message_data, stream_name) end -get = Get.build('something') +get = Get.build(defaults.stream_name) result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] From 45291e41278944567b2e9ca9ec84147667dbfd0e Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 09:56:19 -0600 Subject: [PATCH 551/591] Put benchmark prints header --- test/benchmark/put.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/benchmark/put.rb b/test/benchmark/put.rb index e60124d..74c3e33 100644 --- a/test/benchmark/put.rb +++ b/test/benchmark/put.rb @@ -2,6 +2,10 @@ defaults = Test::Benchmark::Defaults.build +puts +puts "Running benchmark (#{__FILE__})..." +puts + list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) put = Put.build From 32bd981c10508151be37ba63602260e3374cd8b8 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 09:58:38 -0600 Subject: [PATCH 552/591] Get benchmark uses dummy stream name --- test/benchmark/get.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark/get.rb b/test/benchmark/get.rb index 897773e..62dbaab 100644 --- a/test/benchmark/get.rb +++ b/test/benchmark/get.rb @@ -15,7 +15,7 @@ put.(entry.message_data, stream_name) end -get = Get.build(defaults.stream_name) +get = Get.build(Controls::StreamName.example) result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] From d1aeda0c918008ac2be27e24d3dca26ccec5d667 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 10:52:11 -0600 Subject: [PATCH 553/591] Separate benchmark for get category --- test/benchmark/get_category.rb | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/benchmark/get_category.rb diff --git a/test/benchmark/get_category.rb b/test/benchmark/get_category.rb new file mode 100644 index 0000000..e7b8e72 --- /dev/null +++ b/test/benchmark/get_category.rb @@ -0,0 +1,46 @@ +require_relative 'benchmark_init' + +defaults = Test::Benchmark::Defaults.build + +puts +puts "Running benchmark (#{__FILE__})..." +puts + + +total_cycles = defaults.total_cycles + +puts "- constructing #{total_cycles} entries" +list = Controls::MessageData::Write::List.get(instances: total_cycles) + +## +pp list + +put = Put.build + +puts "- writing entries" +list.each do |entry| + stream_name = defaults.stream_name || entry.stream_name + put.(entry.message_data, stream_name) +end + +category = Controls::Category.example + +puts "- constructing Get::Category for #{category}" +get = Get::Category.build(category) + +puts "- executing and sampling #{total_cycles} cycles" +result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| + entry = list[i] + stream_name = MessageStore::StreamName.get_category(defaults.stream_name || entry.stream_name) + +## +puts "- getting #{stream_name}" + + get.(0, stream_name: stream_name) +end + +puts +filename = Benchmark::RecordResult.('Get Benchmark', result) +puts +puts filename +puts From 9c9ea398d8d1801a7cccceb9eb4079bf374a8004 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 12:50:09 -0600 Subject: [PATCH 554/591] Category benchmark is operational --- test/benchmark/defaults.rb | 11 ++++++++--- test/benchmark/get_category.rb | 25 ++++++++++++++---------- test/benchmark/{get.rb => get_stream.rb} | 13 +++++++++--- 3 files changed, 33 insertions(+), 16 deletions(-) rename test/benchmark/{get.rb => get_stream.rb} (61%) diff --git a/test/benchmark/defaults.rb b/test/benchmark/defaults.rb index 7d6e8f5..cc0e0ad 100644 --- a/test/benchmark/defaults.rb +++ b/test/benchmark/defaults.rb @@ -1,7 +1,7 @@ module Test module Benchmark class Defaults - initializer :cycles, :warmup_cycles, :gc, :stream_name + initializer :cycles, :warmup_cycles, :gc, :stream_name, :verbose def total_cycles cycles + warmup_cycles @@ -17,7 +17,7 @@ def to_s end def self.build - new(cycles, warmup_cycles, gc, stream_name) + new(cycles, warmup_cycles, gc, stream_name, verbose) end def self.cycles @@ -32,8 +32,13 @@ def self.gc ['on', 'true'].include?(ENV['GC']) ? true : false end + def self.verbose + ['on', 'true'].include?(ENV['VERBOSE']) ? true : false + end + def self.stream_name - ENV['STREAM_NAME'] || Controls::StreamName.example +## ENV['STREAM_NAME'] || Controls::StreamName.example + ENV['STREAM_NAME'] end end end diff --git a/test/benchmark/get_category.rb b/test/benchmark/get_category.rb index e7b8e72..cf49881 100644 --- a/test/benchmark/get_category.rb +++ b/test/benchmark/get_category.rb @@ -3,32 +3,37 @@ defaults = Test::Benchmark::Defaults.build puts -puts "Running benchmark (#{__FILE__})..." +puts "Get Category Benchmark (#{__FILE__})" puts +puts "» defaults" +puts defaults.to_s total_cycles = defaults.total_cycles +puts -puts "- constructing #{total_cycles} entries" +puts "» constructing #{total_cycles} entries" list = Controls::MessageData::Write::List.get(instances: total_cycles) -## -pp list - put = Put.build -puts "- writing entries" +puts "» writing entries" list.each do |entry| stream_name = defaults.stream_name || entry.stream_name + + if defaults.verbose + puts "Stream: #{stream_name}" + end + put.(entry.message_data, stream_name) end category = Controls::Category.example -puts "- constructing Get::Category for #{category}" -get = Get::Category.build(category) +puts "» constructing Get::Category" +get = Get::Category.build('') -puts "- executing and sampling #{total_cycles} cycles" +puts "» executing and sampling #{total_cycles} cycles" result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] stream_name = MessageStore::StreamName.get_category(defaults.stream_name || entry.stream_name) @@ -40,7 +45,7 @@ end puts -filename = Benchmark::RecordResult.('Get Benchmark', result) +filename = Benchmark::RecordResult.('Get Category Benchmark', result) puts puts filename puts diff --git a/test/benchmark/get.rb b/test/benchmark/get_stream.rb similarity index 61% rename from test/benchmark/get.rb rename to test/benchmark/get_stream.rb index 62dbaab..2a07e3c 100644 --- a/test/benchmark/get.rb +++ b/test/benchmark/get_stream.rb @@ -3,20 +3,27 @@ defaults = Test::Benchmark::Defaults.build puts -puts "Running benchmark (#{__FILE__})..." +puts "Get Stream Benchmark (#{__FILE__})" puts +puts defaults.to_s +puts + +puts "» constructing #{total_cycles} entries" list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) put = Put.build +puts "» writing entries" list.each do |entry| stream_name = defaults.stream_name || entry.stream_name put.(entry.message_data, stream_name) end -get = Get.build(Controls::StreamName.example) +puts "» constructing Get::Stream" +get = Get::Stream.build(Controls::StreamName.example) +puts "» executing and sampling #{total_cycles} cycles" result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] stream_name = defaults.stream_name || entry.stream_name @@ -24,7 +31,7 @@ end puts -filename = Benchmark::RecordResult.('Get Benchmark', result) +filename = Benchmark::RecordResult.('Get Stream Benchmark', result) puts puts filename puts From f7ddebcea0712196097d80dcdefb9836c25fa53c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 12:54:11 -0600 Subject: [PATCH 555/591] Category is pre-computed --- lib/message_store/postgres/controls/message_data.rb | 5 +++-- test/benchmark/get_category.rb | 10 ++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/message_store/postgres/controls/message_data.rb b/lib/message_store/postgres/controls/message_data.rb index b819973..b486417 100644 --- a/lib/message_store/postgres/controls/message_data.rb +++ b/lib/message_store/postgres/controls/message_data.rb @@ -6,7 +6,7 @@ module Controls module MessageData module Write module List - Entry = Struct.new(:stream_name, :message_data) + Entry = Struct.new(:stream_name, :category, :message_data) def self.get(instances: nil) instances ||= 1 @@ -16,9 +16,10 @@ def self.get(instances: nil) list = [] instances.times do stream_name = Controls::StreamName.example + category = MessageStore::StreamName.get_category(stream_name) write_message = Controls::MessageData::Write.example - list << Entry.new(stream_name, write_message) + list << Entry.new(stream_name, category, write_message) end list diff --git a/test/benchmark/get_category.rb b/test/benchmark/get_category.rb index cf49881..a4231f5 100644 --- a/test/benchmark/get_category.rb +++ b/test/benchmark/get_category.rb @@ -28,20 +28,18 @@ put.(entry.message_data, stream_name) end -category = Controls::Category.example - puts "» constructing Get::Category" get = Get::Category.build('') puts "» executing and sampling #{total_cycles} cycles" result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] - stream_name = MessageStore::StreamName.get_category(defaults.stream_name || entry.stream_name) -## -puts "- getting #{stream_name}" + if defaults.verbose + puts "Getting: #{entry.category}" + end - get.(0, stream_name: stream_name) + get.(0, stream_name: entry.category) end puts From 659ec9fbd0811df719d3a60534f40d2a430df99d Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 13:02:30 -0600 Subject: [PATCH 556/591] Get benchmark scripts are consistent with each other --- test/benchmark/get_category.rb | 2 +- test/benchmark/get_stream.rb | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/benchmark/get_category.rb b/test/benchmark/get_category.rb index a4231f5..ee3cd4a 100644 --- a/test/benchmark/get_category.rb +++ b/test/benchmark/get_category.rb @@ -8,9 +8,9 @@ puts "» defaults" puts defaults.to_s +puts total_cycles = defaults.total_cycles -puts puts "» constructing #{total_cycles} entries" list = Controls::MessageData::Write::List.get(instances: total_cycles) diff --git a/test/benchmark/get_stream.rb b/test/benchmark/get_stream.rb index 2a07e3c..154c77a 100644 --- a/test/benchmark/get_stream.rb +++ b/test/benchmark/get_stream.rb @@ -6,22 +6,30 @@ puts "Get Stream Benchmark (#{__FILE__})" puts +puts "» defaults" puts defaults.to_s puts +total_cycles = defaults.total_cycles + puts "» constructing #{total_cycles} entries" -list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) +list = Controls::MessageData::Write::List.get(instances: total_cycles) put = Put.build puts "» writing entries" list.each do |entry| stream_name = defaults.stream_name || entry.stream_name + + if defaults.verbose + puts "Stream: #{stream_name}" + end + put.(entry.message_data, stream_name) end puts "» constructing Get::Stream" -get = Get::Stream.build(Controls::StreamName.example) +get = Get::Stream.build('-') puts "» executing and sampling #{total_cycles} cycles" result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| From d4b384a579251e2e24f28e9ffcf5af4bc12bd808 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 13:10:19 -0600 Subject: [PATCH 557/591] Verbose output for get stream benchmark --- test/benchmark/get_stream.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/benchmark/get_stream.rb b/test/benchmark/get_stream.rb index 154c77a..015aaba 100644 --- a/test/benchmark/get_stream.rb +++ b/test/benchmark/get_stream.rb @@ -34,6 +34,11 @@ puts "» executing and sampling #{total_cycles} cycles" result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] + + if defaults.verbose + puts "Getting: #{entry.category}" + end + stream_name = defaults.stream_name || entry.stream_name get.(0, stream_name: stream_name) end From ab0a1a6306f5f3c4ae2c81aaaba15221e0638112 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 13:30:22 -0600 Subject: [PATCH 558/591] Benchmark controls receive stream_name and category --- lib/message_store/postgres/controls/message_data.rb | 11 +++++------ test/benchmark/get_category.rb | 2 +- test/benchmark/get_stream.rb | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/message_store/postgres/controls/message_data.rb b/lib/message_store/postgres/controls/message_data.rb index b486417..addfb6a 100644 --- a/lib/message_store/postgres/controls/message_data.rb +++ b/lib/message_store/postgres/controls/message_data.rb @@ -8,18 +8,17 @@ module Write module List Entry = Struct.new(:stream_name, :category, :message_data) - def self.get(instances: nil) + def self.get(instances: nil, stream_name: nil, category: nil) instances ||= 1 - stream_name = StreamName.example - list = [] instances.times do - stream_name = Controls::StreamName.example - category = MessageStore::StreamName.get_category(stream_name) + instance_stream_name = stream_name || StreamName.example(category: category) + instance_category = MessageStore::StreamName.get_category(instance_stream_name) + write_message = Controls::MessageData::Write.example - list << Entry.new(stream_name, category, write_message) + list << Entry.new(instance_stream_name, instance_category, write_message) end list diff --git a/test/benchmark/get_category.rb b/test/benchmark/get_category.rb index ee3cd4a..dcbff21 100644 --- a/test/benchmark/get_category.rb +++ b/test/benchmark/get_category.rb @@ -13,7 +13,7 @@ total_cycles = defaults.total_cycles puts "» constructing #{total_cycles} entries" -list = Controls::MessageData::Write::List.get(instances: total_cycles) +list = Controls::MessageData::Write::List.get(instances: total_cycles, category: defaults.stream_name) put = Put.build diff --git a/test/benchmark/get_stream.rb b/test/benchmark/get_stream.rb index 015aaba..4de51d8 100644 --- a/test/benchmark/get_stream.rb +++ b/test/benchmark/get_stream.rb @@ -13,7 +13,7 @@ total_cycles = defaults.total_cycles puts "» constructing #{total_cycles} entries" -list = Controls::MessageData::Write::List.get(instances: total_cycles) +list = Controls::MessageData::Write::List.get(instances: total_cycles, stream_name: defaults.stream_name) put = Put.build @@ -36,7 +36,7 @@ entry = list[i] if defaults.verbose - puts "Getting: #{entry.category}" + puts "Getting: #{entry.stream_name}" end stream_name = defaults.stream_name || entry.stream_name From 421715d98ddf3f5d41260953476f22472da689ee Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 13:53:35 -0600 Subject: [PATCH 559/591] Batch sizes are limited to 1 in get benchmarks --- test/benchmark/get_category.rb | 10 ++++------ test/benchmark/get_stream.rb | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/test/benchmark/get_category.rb b/test/benchmark/get_category.rb index dcbff21..d137f6b 100644 --- a/test/benchmark/get_category.rb +++ b/test/benchmark/get_category.rb @@ -19,24 +19,22 @@ puts "» writing entries" list.each do |entry| - stream_name = defaults.stream_name || entry.stream_name - if defaults.verbose - puts "Stream: #{stream_name}" + puts "Stream: #{entry.stream_name}, Category: #{entry.category}" end - put.(entry.message_data, stream_name) + put.(entry.message_data, entry.stream_name) end puts "» constructing Get::Category" -get = Get::Category.build('') +get = Get::Category.build('', batch_size: 1) puts "» executing and sampling #{total_cycles} cycles" result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] if defaults.verbose - puts "Getting: #{entry.category}" + puts "Getting Category: #{entry.category}" end get.(0, stream_name: entry.category) diff --git a/test/benchmark/get_stream.rb b/test/benchmark/get_stream.rb index 4de51d8..bceb488 100644 --- a/test/benchmark/get_stream.rb +++ b/test/benchmark/get_stream.rb @@ -19,24 +19,22 @@ puts "» writing entries" list.each do |entry| - stream_name = defaults.stream_name || entry.stream_name - if defaults.verbose - puts "Stream: #{stream_name}" + puts "Stream: #{entry.stream_name}, Category: #{entry.category}" end - put.(entry.message_data, stream_name) + put.(entry.message_data, entry.stream_name) end puts "» constructing Get::Stream" -get = Get::Stream.build('-') +get = Get::Stream.build('-', batch_size: 1) puts "» executing and sampling #{total_cycles} cycles" result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] if defaults.verbose - puts "Getting: #{entry.stream_name}" + puts "Getting Stream: #{entry.stream_name}" end stream_name = defaults.stream_name || entry.stream_name From 0853eccdcc753b3274eb9d94aa7ae033f90d7276 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 14:03:31 -0600 Subject: [PATCH 560/591] Benchmarks are consistent --- test/benchmark/get_stream.rb | 3 +-- test/benchmark/put.rb | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/test/benchmark/get_stream.rb b/test/benchmark/get_stream.rb index bceb488..486332d 100644 --- a/test/benchmark/get_stream.rb +++ b/test/benchmark/get_stream.rb @@ -37,8 +37,7 @@ puts "Getting Stream: #{entry.stream_name}" end - stream_name = defaults.stream_name || entry.stream_name - get.(0, stream_name: stream_name) + get.(0, stream_name: entry.stream_name) end puts diff --git a/test/benchmark/put.rb b/test/benchmark/put.rb index 74c3e33..e30d219 100644 --- a/test/benchmark/put.rb +++ b/test/benchmark/put.rb @@ -3,17 +3,28 @@ defaults = Test::Benchmark::Defaults.build puts -puts "Running benchmark (#{__FILE__})..." +puts "Put Benchmark (#{__FILE__})" puts -list = Controls::MessageData::Write::List.get(instances: defaults.total_cycles) +puts "» defaults" +puts defaults.to_s +puts + +total_cycles = defaults.total_cycles + +puts "» constructing #{total_cycles} entries" +list = Controls::MessageData::Write::List.get(instances: total_cycles, stream_name: defaults.stream_name) put = Put.build result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] - stream_name = defaults.stream_name || entry.stream_name - put.(entry.message_data, stream_name) + + if defaults.verbose + puts "Putting Stream: #{entry.stream_name}" + end + + put.(entry.message_data, entry.stream_name) end puts From 45786001a51160a08b288fec7dd82b94e92b82c2 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 14:17:08 -0600 Subject: [PATCH 561/591] Benchmark results --- ...1-30 revision 66950) [x86_64-darwin17].txt | 23 +++++++++++++++++++ ...1-30 revision 66950) [x86_64-darwin17].txt | 23 +++++++++++++++++++ ...1-30 revision 66950) [x86_64-darwin17].txt | 23 +++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 test/benchmark/results/Get Category Benchmark - 2019-12-19T19-45-12.412Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt create mode 100644 test/benchmark/results/Get Stream Benchmark - 2019-12-19T19-50-34.958Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt create mode 100644 test/benchmark/results/Put Benchmark - 2019-12-19T20-11-52.599Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt diff --git a/test/benchmark/results/Get Category Benchmark - 2019-12-19T19-45-12.412Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt b/test/benchmark/results/Get Category Benchmark - 2019-12-19T19-45-12.412Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt new file mode 100644 index 0000000..251c86b --- /dev/null +++ b/test/benchmark/results/Get Category Benchmark - 2019-12-19T19-45-12.412Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt @@ -0,0 +1,23 @@ +Get Category Benchmark +- - - +Cycles: 100000 +Time: 97735.090000ms +Mean Cycle Time: 0.977351ms (± 0.619131ms) +Cycles Per Second: 1023.173970 +GC: off + +Note: After after stream and category get benchmarks are separated + +ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Get Stream Benchmark - 2019-12-19T19-50-34.958Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt b/test/benchmark/results/Get Stream Benchmark - 2019-12-19T19-50-34.958Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt new file mode 100644 index 0000000..4f371d4 --- /dev/null +++ b/test/benchmark/results/Get Stream Benchmark - 2019-12-19T19-50-34.958Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt @@ -0,0 +1,23 @@ +Get Stream Benchmark +- - - +Cycles: 100000 +Time: 90234.657000ms +Mean Cycle Time: 0.902347ms (± 0.656719ms) +Cycles Per Second: 1108.221645 +GC: off + +Note: After after stream and category get benchmarks are separated + +ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2019-12-19T20-11-52.599Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt b/test/benchmark/results/Put Benchmark - 2019-12-19T20-11-52.599Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt new file mode 100644 index 0000000..5f17994 --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2019-12-19T20-11-52.599Z - ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17].txt @@ -0,0 +1,23 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 144316.699000ms +Mean Cycle Time: 1.443167ms (± 0.909076ms) +Cycles Per Second: 692.920505 +GC: off + +Note: After after stream and category get benchmarks are separated + +ruby 2.6.1p33 (2019-01-30 revision 66950) [x86_64-darwin17] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 From c24c6c0d3aa01737ded3a832d1f5e9ac83029acc Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Dec 2019 14:30:38 -0600 Subject: [PATCH 562/591] Benchmarks have consistent output --- test/benchmark/put.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/benchmark/put.rb b/test/benchmark/put.rb index e30d219..d42b806 100644 --- a/test/benchmark/put.rb +++ b/test/benchmark/put.rb @@ -15,8 +15,10 @@ puts "» constructing #{total_cycles} entries" list = Controls::MessageData::Write::List.get(instances: total_cycles, stream_name: defaults.stream_name) +puts "» constructing Put" put = Put.build +puts "» executing and sampling #{total_cycles} cycles" result = Diagnostics::Sample.(defaults.cycles, warmup_cycles: defaults.warmup_cycles, gc: defaults.gc) do |i| entry = list[i] From a41a7920b80b80a42ced2d68c7fcede24aabf161 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 2 Feb 2020 15:38:31 -0600 Subject: [PATCH 563/591] Settings default file path is defined by the defaults module --- lib/message_store/postgres/settings.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/message_store/postgres/settings.rb b/lib/message_store/postgres/settings.rb index f1a4298..ffee3df 100644 --- a/lib/message_store/postgres/settings.rb +++ b/lib/message_store/postgres/settings.rb @@ -6,7 +6,7 @@ def self.instance end def self.data_source - Defaults.data_source || 'settings/message_store_postgres.json' + Defaults.data_source end def self.names @@ -28,7 +28,7 @@ def self.names class Defaults def self.data_source - ENV['MESSAGE_STORE_SETTINGS_PATH'] + ENV['MESSAGE_STORE_SETTINGS_PATH'] || 'settings/message_store_postgres.json' end end end From cc06fbe83c3c6dc4314b1b9676aa03964349e59f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 2 Feb 2020 15:40:13 -0600 Subject: [PATCH 564/591] Package version is increased from 2.4.0.1 to 2.4.0.2 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 51c2f4f..5253edc 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '2.4.0.1' + s.version = '2.4.0.2' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 5e505a30c757a5d88ddf004c3c07d54b6254c7cc Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Sun, 8 Mar 2020 14:20:55 -0500 Subject: [PATCH 565/591] CLI actuator is corrected --- test/automated.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/automated.rb b/test/automated.rb index 0201c1d..5bdd0b4 100644 --- a/test/automated.rb +++ b/test/automated.rb @@ -1,6 +1,5 @@ require_relative './test_init' TestBench::CLI.( - tests_directory: 'test/automated', exclude_file_pattern: %r{/_|sketch|(_init\.rb|_tests\.rb)\z} ) From 860ed1ae7d653f1279516126467191ae9f79b440 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 21 Jul 2020 11:32:11 -0500 Subject: [PATCH 566/591] Identifier dependency type is corrected --- lib/message_store/postgres/put.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index d36fca2..1215fbc 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -5,7 +5,7 @@ class Put include Log::Dependency dependency :session, Session - dependency :identifier, Session + dependency :identifier, Identifier::UUID::RandomIdentifier dependency is corrected def self.build(session: nil) new.tap do |instance| From 265fb79657644118b2f685b6444c3dd155ec9f73 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 21 Jul 2020 11:33:03 -0500 Subject: [PATCH 567/591] Package version is increased from 2.4.0.2 to 2.4.0.3 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 5253edc..356c92a 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '2.4.0.2' + s.version = '2.4.0.3' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 38ab38e5a87fd9b68c4c7c1ceb95f6e4dbad2167 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 21 Jul 2020 12:00:16 -0500 Subject: [PATCH 568/591] Identifier dependency is corrected --- lib/message_store/postgres/put.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index 1215fbc..262e31b 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -5,7 +5,7 @@ class Put include Log::Dependency dependency :session, Session - dependency :identifier, Identifier::UUID::RandomIdentifier dependency is corrected + dependency :identifier, Identifier::UUID::RandomIdentifier def self.build(session: nil) new.tap do |instance| From d763a407c968d6839a7740ee9a8390f7c7891b5c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 21 Jul 2020 12:00:49 -0500 Subject: [PATCH 569/591] Package version is increased from 2.4.0.3 to 2.4.0.4 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 356c92a..c4b23ea 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '2.4.0.3' + s.version = '2.4.0.4' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 9c10889888283770bd5bfad550ab1bf1807feb3b Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 21 Jul 2020 16:56:40 -0500 Subject: [PATCH 570/591] Identifier dependency is corrected --- lib/message_store/postgres/put.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb index 262e31b..55005f6 100644 --- a/lib/message_store/postgres/put.rb +++ b/lib/message_store/postgres/put.rb @@ -5,7 +5,7 @@ class Put include Log::Dependency dependency :session, Session - dependency :identifier, Identifier::UUID::RandomIdentifier + dependency :identifier, Identifier::UUID::Random def self.build(session: nil) new.tap do |instance| From f48b89b3f9cd742910ea18d60fc1861d7f766b7f Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Tue, 21 Jul 2020 16:57:20 -0500 Subject: [PATCH 571/591] Package version is increased from 2.4.0.4 to 2.4.0.5 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index c4b23ea..c505b4d 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '2.4.0.4' + s.version = '2.4.0.5' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 58a17aaf538c3d73eb6dab36c87506e725747b5a Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Thu, 23 Jul 2020 14:34:05 -0500 Subject: [PATCH 572/591] Iterator tests suite runner is removed --- test/automated/iterator_tests.rb | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 test/automated/iterator_tests.rb diff --git a/test/automated/iterator_tests.rb b/test/automated/iterator_tests.rb deleted file mode 100644 index adf8d20..0000000 --- a/test/automated/iterator_tests.rb +++ /dev/null @@ -1,5 +0,0 @@ -require_relative '../test_init' - -TestBench::Runner.( - 'iterator/**/*.rb' -) or exit 1 From c6c288b3bf196cc58a872aa88971d7870e21d44c Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 19 Nov 2020 21:24:19 -0600 Subject: [PATCH 573/591] Benchmark results --- ...revision a0c7c23c9c) [x86_64-darwin19].txt | 21 +++++++++++++++++++ ...revision a0c7c23c9c) [x86_64-darwin19].txt | 21 +++++++++++++++++++ ...revision a0c7c23c9c) [x86_64-darwin19].txt | 21 +++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 test/benchmark/results/Get Category Benchmark - 2020-11-20T01-25-27.278Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt create mode 100644 test/benchmark/results/Get Stream Benchmark - 2020-11-20T03-10-44.280Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt create mode 100644 test/benchmark/results/Put Benchmark - 2020-11-20T01-14-19.439Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt diff --git a/test/benchmark/results/Get Category Benchmark - 2020-11-20T01-25-27.278Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt b/test/benchmark/results/Get Category Benchmark - 2020-11-20T01-25-27.278Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt new file mode 100644 index 0000000..706f826 --- /dev/null +++ b/test/benchmark/results/Get Category Benchmark - 2020-11-20T01-25-27.278Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt @@ -0,0 +1,21 @@ +Get Category Benchmark +- - - +Cycles: 100000 +Time: 73931.646000ms +Mean Cycle Time: 0.739316ms (± 0.249903ms) +Cycles Per Second: 1352.600752 +GC: off + +ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Get Stream Benchmark - 2020-11-20T03-10-44.280Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt b/test/benchmark/results/Get Stream Benchmark - 2020-11-20T03-10-44.280Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt new file mode 100644 index 0000000..8786685 --- /dev/null +++ b/test/benchmark/results/Get Stream Benchmark - 2020-11-20T03-10-44.280Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt @@ -0,0 +1,21 @@ +Get Stream Benchmark +- - - +Cycles: 100000 +Time: 70176.616000ms +Mean Cycle Time: 0.701766ms (± 0.201194ms) +Cycles Per Second: 1424.976092 +GC: off + +ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 diff --git a/test/benchmark/results/Put Benchmark - 2020-11-20T01-14-19.439Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt b/test/benchmark/results/Put Benchmark - 2020-11-20T01-14-19.439Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt new file mode 100644 index 0000000..85308ad --- /dev/null +++ b/test/benchmark/results/Put Benchmark - 2020-11-20T01-14-19.439Z - ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19].txt @@ -0,0 +1,21 @@ +Put Benchmark +- - - +Cycles: 100000 +Time: 140985.021000ms +Mean Cycle Time: 1.409850ms (± 0.395411ms) +Cycles Per Second: 709.295209 +GC: off + +ruby 2.7.1p83 (2020-03-31 revision a0c7c23c9c) [x86_64-darwin19] + +Model Name: MacBook +Model Identifier: MacBook10,1 +Processor Name: Intel Core i7 +Processor Speed: 1.4 GHz +Number of Processors: 1 +Total Number of Cores: 2 +L2 Cache (per Core): 256 KB +L3 Cache: 4 MB +Memory: 16 GB +Boot ROM Version: MB101.0163.B00 +SMC Version (system): 2.42f10 From 31a970ef18a0b3368605d7d72bfa49bf8d7ee620 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 14 Dec 2020 11:18:41 -0600 Subject: [PATCH 574/591] Session escapes data --- lib/message_store/postgres/session.rb | 8 ++++++++ test/automated/session/escape.rb | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 test/automated/session/escape.rb diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index 387251b..bcb9c53 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -128,6 +128,14 @@ def transaction(&blk) connection.transaction(&blk) end + def escape(data) + connection = connect + + escaped_data = connection.escape(data) + + escaped_data + end + def self.logger @logger ||= Log.get self end diff --git a/test/automated/session/escape.rb b/test/automated/session/escape.rb new file mode 100644 index 0000000..d632637 --- /dev/null +++ b/test/automated/session/escape.rb @@ -0,0 +1,22 @@ +require_relative '../automated_init' + +context "Session" do + context "Escape" do + session = Session.build + + unescaped_data = "'" + control_data = "''" + + escaped_data = session.escape(unescaped_data) + + context "Escaped Data" do + comment escaped_data.inspect + detail "Control Data: #{control_data.inspect}" + detail "Unescaped Data: #{unescaped_data.inspect}" + + test "escaped_Data is escaped" do + assert(escaped_data == control_data) + end + end + end +end From 731cd2dcd6f67dcd84aa5e81f1ceaab9747b68ee Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 14 Dec 2020 11:19:12 -0600 Subject: [PATCH 575/591] Package version is increased from 2.4.0.5 to 2.4.1.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index c505b4d..527b310 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '2.4.0.5' + s.version = '2.4.1.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 4b4761549e9a9cee5e6449dfb3e7c95aa6f66284 Mon Sep 17 00:00:00 2001 From: Aaron Jensen Date: Thu, 25 Feb 2021 14:34:21 -0600 Subject: [PATCH 576/591] Keepalive and tcp timeout settings are supported --- lib/message_store/postgres/settings.rb | 7 ++++++- settings/message_store_postgres.json | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/message_store/postgres/settings.rb b/lib/message_store/postgres/settings.rb index ffee3df..e45b6ea 100644 --- a/lib/message_store/postgres/settings.rb +++ b/lib/message_store/postgres/settings.rb @@ -22,7 +22,12 @@ def self.names :sslmode, :krbsrvname, :gsslib, - :service + :service, + :keepalives, + :keepalives_idle, + :keepalives_interval, + :keepalives_count, + :tcp_user_timeout ] end diff --git a/settings/message_store_postgres.json b/settings/message_store_postgres.json index 96139f8..2c62f9a 100644 --- a/settings/message_store_postgres.json +++ b/settings/message_store_postgres.json @@ -10,5 +10,10 @@ "sslmode": null, "krbsrvname": null, "gsslib": null, - "service": null + "service": null, + "keepalives": null, + "keepalives_idle": null, + "keepalives_interval": null, + "keepalives_count": null, + "tcp_user_timeout": null } From 6683df540fb27e81dd9652469a7cedfee5cf1b47 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Thu, 4 Mar 2021 21:13:27 -0600 Subject: [PATCH 577/591] Package version is increased from 2.4.1.0 to 2.4.2.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 527b310..0a4f313 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '2.4.1.0' + s.version = '2.4.2.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 115108a616e7167d8ec761208626e25ffa02d7e4 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 13 Feb 2022 22:04:12 -0600 Subject: [PATCH 578/591] Connection test sends a semicolon rather than a complete statement --- test/automated/session/connect_on_first_use.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/automated/session/connect_on_first_use.rb b/test/automated/session/connect_on_first_use.rb index 3d7cae9..4e6db9b 100644 --- a/test/automated/session/connect_on_first_use.rb +++ b/test/automated/session/connect_on_first_use.rb @@ -8,7 +8,7 @@ test "Connects" do refute_raises do - session.execute('SELECT 1;') + session.execute(';') end end end From 78187abb93f35d273c735970fc1f88699fc20d03 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 13 Feb 2022 22:03:04 -0600 Subject: [PATCH 579/591] Connection records time of last use --- test/automated/session/connection_time.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test/automated/session/connection_time.rb diff --git a/test/automated/session/connection_time.rb b/test/automated/session/connection_time.rb new file mode 100644 index 0000000..f2b6b86 --- /dev/null +++ b/test/automated/session/connection_time.rb @@ -0,0 +1,19 @@ +require_relative '../automated_init' + +context "Session" do + context "Connection Time" do + session = Session.new + + + + + + refute(session.connected?) + + test "Connects" do + refute_raises do + session.execute('SELECT 1;') + end + end + end +end From bacff978cd3c31b50299d2b02ebba74cbccb5d50 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Sun, 13 Feb 2022 23:02:34 -0600 Subject: [PATCH 580/591] Connection reports elapsed milliseconds since last execution --- lib/message_store/postgres/controls.rb | 1 + lib/message_store/postgres/controls/time.rb | 7 +++ lib/message_store/postgres/session.rb | 44 +++++++++++++------ .../session/connection_executed_time.rb | 27 ++++++++++++ ...tion_executed_time_elapsed_milliseconds.rb | 29 ++++++++++++ test/automated/session/connection_time.rb | 19 -------- 6 files changed, 95 insertions(+), 32 deletions(-) create mode 100644 lib/message_store/postgres/controls/time.rb create mode 100644 test/automated/session/connection_executed_time.rb create mode 100644 test/automated/session/connection_executed_time_elapsed_milliseconds.rb delete mode 100644 test/automated/session/connection_time.rb diff --git a/lib/message_store/postgres/controls.rb b/lib/message_store/postgres/controls.rb index 847c5d0..3560cdc 100644 --- a/lib/message_store/postgres/controls.rb +++ b/lib/message_store/postgres/controls.rb @@ -1,5 +1,6 @@ require 'message_store/controls' +require 'message_store/postgres/controls/time' require 'message_store/postgres/controls/position' require 'message_store/postgres/controls/category' require 'message_store/postgres/controls/stream_name' diff --git a/lib/message_store/postgres/controls/time.rb b/lib/message_store/postgres/controls/time.rb new file mode 100644 index 0000000..4d99aad --- /dev/null +++ b/lib/message_store/postgres/controls/time.rb @@ -0,0 +1,7 @@ +module MessageStore + module Postgres + module Controls + Time = MessageStore::Controls::Time + end + end +end diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index bcb9c53..79299bd 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -3,10 +3,13 @@ module Postgres class Session Error = Class.new(RuntimeError) + include Dependency include Settings::Setting include Log::Dependency + dependency :clock, Clock::UTC + def self.settings Settings.names end @@ -16,12 +19,17 @@ def self.settings end attr_accessor :connection + attr_accessor :executed_time def self.build(settings: nil) - new.tap do |instance| - settings ||= Settings.instance - settings.set(instance) - end + instance = new + + settings ||= Settings.instance + settings.set(instance) + + Clock::UTC.configure(instance) + + instance end def self.configure(receiver, session: nil, settings: nil, attr_name: nil) @@ -91,15 +99,6 @@ def reset connection.reset end - def settings - settings = {} - self.class.settings.each do |s| - val = public_send(s) - settings[s] = val unless val.nil? - end - settings - end - def execute(sql_command, params=nil) logger.trace(tag: :session) { "Executing SQL command" } logger.trace(tag: :sql) { sql_command } @@ -109,17 +108,27 @@ def execute(sql_command, params=nil) connect end + executed_time = nil + if params.nil? connection.exec(sql_command).tap do + self.executed_time = clock.now logger.debug(tag: :session) { "Executed SQL command (no params)" } end else connection.exec_params(sql_command, params).tap do + self.executed_time = clock.now logger.debug(tag: :session) { "Executed SQL command with params" } end end end + def executed_time_elapsed_milliseconds + return nil if executed_time.nil? + + (clock.now - executed_time) * 1000 + end + def transaction(&blk) unless connected? connect @@ -136,6 +145,15 @@ def escape(data) escaped_data end + def settings + settings = {} + self.class.settings.each do |s| + val = public_send(s) + settings[s] = val unless val.nil? + end + settings + end + def self.logger @logger ||= Log.get self end diff --git a/test/automated/session/connection_executed_time.rb b/test/automated/session/connection_executed_time.rb new file mode 100644 index 0000000..2d1bfdc --- /dev/null +++ b/test/automated/session/connection_executed_time.rb @@ -0,0 +1,27 @@ +require_relative '../automated_init' + +context "Session" do + context "Connection Executed Time" do + context "Before First Execution" do + session = Session.new + + test "Executed time isn't set" do + assert(session.executed_time.nil?) + end + end + + context "After Execution" do + session = Session.new + + time = Controls::Time::Raw.example + + session.clock.now = time + + session.execute(';') + + test "Executed time is set to the clock time" do + assert(session.executed_time == time) + end + end + end +end diff --git a/test/automated/session/connection_executed_time_elapsed_milliseconds.rb b/test/automated/session/connection_executed_time_elapsed_milliseconds.rb new file mode 100644 index 0000000..e996241 --- /dev/null +++ b/test/automated/session/connection_executed_time_elapsed_milliseconds.rb @@ -0,0 +1,29 @@ +require_relative '../automated_init' + +context "Session" do + context "Connection Executed Time Elapsed Milliseconds" do + context "Before First Execution" do + session = Session.new + + test "Is nil" do + assert(session.executed_time_elapsed_milliseconds.nil?) + end + end + + context "After Execution" do + session = Session.new + + start_time = Controls::Time::Raw.example + end_time = start_time + 1 + + session.executed_time = start_time + session.clock.now = end_time + + elapsed_milliseconds = (end_time - start_time) * 1000 + + test "Is difference between current time and executed time" do + assert(session.executed_time_elapsed_milliseconds == elapsed_milliseconds) + end + end + end +end diff --git a/test/automated/session/connection_time.rb b/test/automated/session/connection_time.rb deleted file mode 100644 index f2b6b86..0000000 --- a/test/automated/session/connection_time.rb +++ /dev/null @@ -1,19 +0,0 @@ -require_relative '../automated_init' - -context "Session" do - context "Connection Time" do - session = Session.new - - - - - - refute(session.connected?) - - test "Connects" do - refute_raises do - session.execute('SELECT 1;') - end - end - end -end From 74127ae07d503e42e9a754989334c71da566e403 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Mon, 14 Feb 2022 00:01:03 -0600 Subject: [PATCH 581/591] Package version is increased from 2.4.2.0 to 2.4.3.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 0a4f313..9918a45 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '2.4.2.0' + s.version = '2.4.3.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From be88edf5b7cde54178b834369751b54cd497f71e Mon Sep 17 00:00:00 2001 From: Aaron Jensen Date: Mon, 14 Feb 2022 05:10:46 -0500 Subject: [PATCH 582/591] Session connection executed time is corrected --- test/automated/session/connection_executed_time.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/automated/session/connection_executed_time.rb b/test/automated/session/connection_executed_time.rb index 2d1bfdc..ec0e5f9 100644 --- a/test/automated/session/connection_executed_time.rb +++ b/test/automated/session/connection_executed_time.rb @@ -3,7 +3,7 @@ context "Session" do context "Connection Executed Time" do context "Before First Execution" do - session = Session.new + session = Session.build test "Executed time isn't set" do assert(session.executed_time.nil?) @@ -11,10 +11,10 @@ end context "After Execution" do - session = Session.new + session = Session.build + Dependency::Substitute.(:clock, session) time = Controls::Time::Raw.example - session.clock.now = time session.execute(';') From ef28d77748c6c391993d6fb340c340f00053120d Mon Sep 17 00:00:00 2001 From: Nathan Ladd Date: Thu, 17 Feb 2022 09:46:16 -0600 Subject: [PATCH 583/591] Session closing is tested --- lib/message_store/postgres/session.rb | 2 +- test/automated/session/close.rb | 46 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 test/automated/session/close.rb diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index 79299bd..47454fe 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -92,7 +92,7 @@ def connected? def close connection.close - connection = nil + self.connection = nil end def reset diff --git a/test/automated/session/close.rb b/test/automated/session/close.rb new file mode 100644 index 0000000..83d3ae2 --- /dev/null +++ b/test/automated/session/close.rb @@ -0,0 +1,46 @@ +require_relative "../automated_init" + +context "Session" do + context "Close" do + context "Session is Open" do + session = Session.build + session.open + + assert(session.open?) + + session.close + + test "No longer connected" do + refute(session.connected?) + end + + context "Connection Attribute" do + connection = session.connection + + test "Not set" do + assert(connection.nil?) + end + end + end + + context "Session is Closed" do + session = Session.build + + refute(session.open?) + + session.close + + test "Not connected" do + refute(session.connected?) + end + + context "Connection Attribute" do + connection = session.connection + + test "Not set" do + assert(connection.nil?) + end + end + end + end +end From e97e2f15ee8f77a940e5b6b5165a49657949500e Mon Sep 17 00:00:00 2001 From: Aaron Jensen Date: Wed, 13 Apr 2022 08:02:11 -0400 Subject: [PATCH 584/591] Vestigial variable is removed --- lib/message_store/postgres/session.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index 79299bd..33c3de6 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -108,8 +108,6 @@ def execute(sql_command, params=nil) connect end - executed_time = nil - if params.nil? connection.exec(sql_command).tap do self.executed_time = clock.now From 34e7428f7978118993a39b98ddbbfcca1e2b12c6 Mon Sep 17 00:00:00 2001 From: Aaron Jensen Date: Mon, 14 Feb 2022 05:36:41 -0500 Subject: [PATCH 585/591] Get stream last supports an optional type parameter --- lib/message_store/postgres/controls/put.rb | 4 +- lib/message_store/postgres/get/stream/last.rb | 27 ++++++++------ .../automated/get/stream/get_last/category.rb | 23 ------------ .../automated/get/stream/get_last/get_last.rb | 19 ++++++++++ .../get/stream/get_last/stream_name.rb | 21 ----------- test/automated/get/stream/get_last/type.rb | 37 +++++++++++++++++++ 6 files changed, 74 insertions(+), 57 deletions(-) delete mode 100644 test/automated/get/stream/get_last/category.rb create mode 100644 test/automated/get/stream/get_last/get_last.rb delete mode 100644 test/automated/get/stream/get_last/stream_name.rb create mode 100644 test/automated/get/stream/get_last/type.rb diff --git a/lib/message_store/postgres/controls/put.rb b/lib/message_store/postgres/controls/put.rb index 13c5368..0a55b10 100644 --- a/lib/message_store/postgres/controls/put.rb +++ b/lib/message_store/postgres/controls/put.rb @@ -2,14 +2,14 @@ module MessageStore module Postgres module Controls module Put - def self.call(instances: nil, stream_name: nil, message_data: nil, message: nil, category: nil) + def self.call(instances: nil, stream_name: nil, message_data: nil, message: nil, category: nil, type: nil) instances ||= 1 stream_name ||= StreamName.example(category: category) message_data ||= message message_specified = !message_data.nil? - message_data ||= MessageData::Write.example + message_data ||= MessageData::Write.example(type: type) position = nil instances.times do diff --git a/lib/message_store/postgres/get/stream/last.rb b/lib/message_store/postgres/get/stream/last.rb index 865e7f8..21c20e9 100644 --- a/lib/message_store/postgres/get/stream/last.rb +++ b/lib/message_store/postgres/get/stream/last.rb @@ -11,10 +11,10 @@ def configure(session: nil) Session.configure(self, session: session) end - def call(stream_name) + def call(stream_name, type: nil) logger.trace(tag: :get) { "Getting last message data (Stream Name: #{stream_name})" } - result = get_result(stream_name) + result = get_result(stream_name, type) return nil if result.nil? @@ -26,14 +26,10 @@ def call(stream_name) message_data end - def get_result(stream_name) + def get_result(stream_name, type) logger.trace(tag: :get) { "Getting last record (Stream: #{stream_name})" } - sql_command = self.class.sql_command(stream_name) - - parameter_values = [ - stream_name - ] + parameter_values = parameter_values(stream_name, type) result = session.execute(sql_command, parameter_values) @@ -44,12 +40,21 @@ def get_result(stream_name) result end - def self.sql_command(stream_name) - parameters = '$1::varchar' - + def sql_command "SELECT * FROM get_last_stream_message(#{parameters});" end + def parameters + "$1::varchar, $2::varchar" + end + + def parameter_values(stream_name, type) + [ + stream_name, + type + ] + end + def convert(record) logger.trace(tag: :get) { "Converting record to message data" } diff --git a/test/automated/get/stream/get_last/category.rb b/test/automated/get/stream/get_last/category.rb deleted file mode 100644 index 8355cb9..0000000 --- a/test/automated/get/stream/get_last/category.rb +++ /dev/null @@ -1,23 +0,0 @@ -require_relative '../../../automated_init' - -context "Get" do - context "Stream" do - context "Last" do - context "Category" do - category = Controls::Category.example - - Controls::Put.(instances: 2, stream_name: category) - - write_message = Controls::MessageData::Write.example - - Put.(write_message, category) - - last_message = Get::Stream::Last.(category) - - test "Gets the last message in the stream" do - assert(last_message.data == write_message.data) - end - end - end - end -end diff --git a/test/automated/get/stream/get_last/get_last.rb b/test/automated/get/stream/get_last/get_last.rb new file mode 100644 index 0000000..fc7defc --- /dev/null +++ b/test/automated/get/stream/get_last/get_last.rb @@ -0,0 +1,19 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Last" do + stream_name, _ = Controls::Put.(instances: 2) + + write_message = Controls::MessageData::Write.example + + Put.(write_message, stream_name) + + last_message = Get::Stream::Last.(stream_name) + + test "Gets the last message in the stream" do + assert(last_message.data == write_message.data) + end + end + end +end diff --git a/test/automated/get/stream/get_last/stream_name.rb b/test/automated/get/stream/get_last/stream_name.rb deleted file mode 100644 index d62c3cd..0000000 --- a/test/automated/get/stream/get_last/stream_name.rb +++ /dev/null @@ -1,21 +0,0 @@ -require_relative '../../../automated_init' - -context "Get" do - context "Stream" do - context "Last" do - context "Stream Name" do - stream_name, _ = Controls::Put.(instances: 2) - - write_message = Controls::MessageData::Write.example - - Put.(write_message, stream_name) - - last_message = Get::Stream::Last.(stream_name) - - test "Gets the last message in the stream" do - assert(last_message.data == write_message.data) - end - end - end - end -end diff --git a/test/automated/get/stream/get_last/type.rb b/test/automated/get/stream/get_last/type.rb new file mode 100644 index 0000000..802ef39 --- /dev/null +++ b/test/automated/get/stream/get_last/type.rb @@ -0,0 +1,37 @@ +require_relative '../../../automated_init' + +context "Get" do + context "Stream" do + context "Last" do + context "Type" do + control_type = Controls::MessageData.type + other_type = "SomeOtherType" + + stream_name, _ = Controls::Put.(instances: 2, type: control_type) + + write_message = Controls::MessageData::Write.example(type: control_type) + + Put.(write_message, stream_name) + + Controls::Put.(instances: 2, stream_name: stream_name, type: other_type) + + last_message = Get::Stream::Last.(stream_name, type: control_type) + + test "Gets the last message in the stream" do + assert(last_message.data == write_message.data) + end + + context "Type" do + type = last_message.type + + comment "#{type}" + detail "Control: #{control_type}" + + test do + assert(type == control_type) + end + end + end + end + end +end From b4ac6b1115055c44822d6ea4b3d860c0edd754c0 Mon Sep 17 00:00:00 2001 From: Aaron Jensen Date: Tue, 12 Jul 2022 13:59:25 -0400 Subject: [PATCH 586/591] Session close is corrected --- lib/message_store/postgres/session.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb index e813f8b..8d89d6b 100644 --- a/lib/message_store/postgres/session.rb +++ b/lib/message_store/postgres/session.rb @@ -91,6 +91,10 @@ def connected? alias :open? :connected? def close + if connection.nil? + return + end + connection.close self.connection = nil end From 4c94f76b64fb9f322e8359f32d512f488761bdd1 Mon Sep 17 00:00:00 2001 From: Aaron Jensen Date: Tue, 12 Jul 2022 14:02:03 -0400 Subject: [PATCH 587/591] Get stream last is backwards compatible with versions of message-db that do not support type --- lib/message_store/postgres/get/stream/last.rb | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/lib/message_store/postgres/get/stream/last.rb b/lib/message_store/postgres/get/stream/last.rb index 21c20e9..86c2786 100644 --- a/lib/message_store/postgres/get/stream/last.rb +++ b/lib/message_store/postgres/get/stream/last.rb @@ -30,6 +30,7 @@ def get_result(stream_name, type) logger.trace(tag: :get) { "Getting last record (Stream: #{stream_name})" } parameter_values = parameter_values(stream_name, type) + sql_command = sql_command(type) result = session.execute(sql_command, parameter_values) @@ -40,19 +41,36 @@ def get_result(stream_name, type) result end - def sql_command + def sql_command(type) + parameters = parameters(type) + "SELECT * FROM get_last_stream_message(#{parameters});" end - def parameters - "$1::varchar, $2::varchar" + def parameters(type) + parameters = "$1::varchar" + + # Backwards compatibility with versions of message-db that do not + # support the type parameter - Aaron, Scott, Tue Jul 12 2022 + if not type.nil? + parameters << ", $2::varchar" + end + + parameters end def parameter_values(stream_name, type) - [ - stream_name, - type + parameter_values = [ + stream_name ] + + # Backwards compatibility with versions of message-db that do not + # support the type parameter - Aaron, Scott, Tue Jul 12 2022 + if not type.nil? + parameter_values << type + end + + parameter_values end def convert(record) From b69c780cc1dd70e9f51ef5ce7cfc1a0a7d05fc2a Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Wed, 12 Oct 2022 23:09:02 -0500 Subject: [PATCH 588/591] Package version is increased from 2.4.3.0 to 2.4.4.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 9918a45..2844931 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '2.4.3.0' + s.version = '2.4.4.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From be98c91d2ea57349e6103493e572ef9fd4c958c9 Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 14 Oct 2022 16:44:57 -0500 Subject: [PATCH 589/591] Get::Stream::Last Type parameter is positional, rather than named --- lib/message_store/postgres/get/stream/last.rb | 2 +- test/automated/get/stream/get_last/type.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/message_store/postgres/get/stream/last.rb b/lib/message_store/postgres/get/stream/last.rb index 86c2786..44414e0 100644 --- a/lib/message_store/postgres/get/stream/last.rb +++ b/lib/message_store/postgres/get/stream/last.rb @@ -11,7 +11,7 @@ def configure(session: nil) Session.configure(self, session: session) end - def call(stream_name, type: nil) + def call(stream_name, type=nil) logger.trace(tag: :get) { "Getting last message data (Stream Name: #{stream_name})" } result = get_result(stream_name, type) diff --git a/test/automated/get/stream/get_last/type.rb b/test/automated/get/stream/get_last/type.rb index 802ef39..97ba970 100644 --- a/test/automated/get/stream/get_last/type.rb +++ b/test/automated/get/stream/get_last/type.rb @@ -15,7 +15,7 @@ Controls::Put.(instances: 2, stream_name: stream_name, type: other_type) - last_message = Get::Stream::Last.(stream_name, type: control_type) + last_message = Get::Stream::Last.(stream_name, control_type) test "Gets the last message in the stream" do assert(last_message.data == write_message.data) From c05e88cb332500f2fa91c3882a33d8896f76a74a Mon Sep 17 00:00:00 2001 From: Scott Bellware Date: Fri, 14 Oct 2022 16:53:25 -0500 Subject: [PATCH 590/591] Package version is increased from 2.4.4.0 to 2.5.0.0 --- message_store-postgres.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec index 2844931..6c6376a 100644 --- a/message_store-postgres.gemspec +++ b/message_store-postgres.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = 'evt-message_store-postgres' - s.version = '2.4.4.0' + s.version = '2.5.0.0' s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' From 9b102d1392692a08f870d94bddd62bc790fbde8d Mon Sep 17 00:00:00 2001 From: Aaron Jensen Date: Sun, 22 Jan 2023 10:46:32 -0500 Subject: [PATCH 591/591] message-store and message-store-postgres are combined --- lib/message_store.rb | 20 +- lib/message_store/controls.rb | 8 +- lib/message_store/controls/get_last.rb | 33 ---- .../controls/message_data/write.rb | 20 ++ lib/message_store/controls/position.rb | 13 ++ lib/message_store/controls/put.rb | 26 +++ lib/message_store/controls/read.rb | 11 +- lib/message_store/controls/write.rb | 23 --- lib/message_store/get.rb | 151 ++++++++++++++- lib/message_store/get/category.rb | 79 ++++++++ .../get/category/consumer_group.rb | 18 ++ lib/message_store/get/category/correlation.rb | 15 ++ lib/message_store/get/condition.rb | 13 ++ lib/message_store/get/stream.rb | 65 +++++++ lib/message_store/get/stream/last.rb | 118 ++++++++---- .../get/stream/last/substitute.rb | 8 +- lib/message_store/get/substitute.rb | 1 + lib/message_store/postgres.rb | 24 --- lib/message_store/postgres/controls.rb | 8 - .../postgres/controls/category.rb | 7 - .../postgres/controls/message_data.rb | 31 --- .../postgres/controls/position.rb | 15 -- lib/message_store/postgres/controls/put.rb | 28 --- .../postgres/controls/stream_name.rb | 7 - lib/message_store/postgres/controls/time.rb | 7 - lib/message_store/postgres/get.rb | 165 ---------------- lib/message_store/postgres/get/category.rb | 81 -------- .../postgres/get/category/consumer_group.rb | 20 -- .../postgres/get/category/correlation.rb | 17 -- lib/message_store/postgres/get/condition.rb | 15 -- lib/message_store/postgres/get/stream.rb | 67 ------- lib/message_store/postgres/get/stream/last.rb | 89 --------- lib/message_store/postgres/log.rb | 9 - lib/message_store/postgres/put.rb | 146 --------------- lib/message_store/postgres/read.rb | 17 -- lib/message_store/postgres/session.rb | 176 ------------------ lib/message_store/postgres/settings.rb | 41 ---- lib/message_store/postgres/write.rb | 52 ------ lib/message_store/put.rb | 140 ++++++++++++++ lib/message_store/read.rb | 62 +++--- lib/message_store/read/iterator.rb | 11 +- lib/message_store/session.rb | 174 +++++++++++++++++ lib/message_store/settings.rb | 39 ++++ lib/message_store/write.rb | 92 +++++---- message_store-postgres.gemspec | 31 --- message_store.gemspec | 17 +- .../specialized/condition/condition.rb | 2 +- .../condition/not_activated_error.rb | 2 +- .../specialized/consumer_groups/error.rb | 10 +- .../category/specialized/correlation/error.rb | 2 +- .../stream/specialized/condition/condition.rb | 2 +- .../specialized/condition/not_activated.rb | 2 +- test/automated/get_last/call.rb | 38 ---- test/automated/get_last/configure.rb | 8 +- test/automated/get_last/substitute.rb | 4 +- test/automated/iterator_tests.rb | 6 - test/automated/read/condition.rb | 2 +- test/automated/read/default_batch_size.rb | 4 +- test/automated/session/build.rb | 4 +- test/automated/session/configure.rb | 2 +- test/automated/session/settings.rb | 4 +- test/automated/settings.rb | 4 +- test/automated/settings/default_path.rb | 2 +- test/automated/write/ids_are_assigned.rb | 5 +- .../concurrency/write_to_single_stream.rb | 6 +- test/test_init.rb | 16 +- tools/write_message.rb | 3 +- 67 files changed, 997 insertions(+), 1341 deletions(-) delete mode 100644 lib/message_store/controls/get_last.rb create mode 100644 lib/message_store/controls/position.rb create mode 100644 lib/message_store/controls/put.rb delete mode 100644 lib/message_store/controls/write.rb create mode 100644 lib/message_store/get/category.rb create mode 100644 lib/message_store/get/category/consumer_group.rb create mode 100644 lib/message_store/get/category/correlation.rb create mode 100644 lib/message_store/get/condition.rb create mode 100644 lib/message_store/get/stream.rb delete mode 100644 lib/message_store/postgres.rb delete mode 100644 lib/message_store/postgres/controls.rb delete mode 100644 lib/message_store/postgres/controls/category.rb delete mode 100644 lib/message_store/postgres/controls/message_data.rb delete mode 100644 lib/message_store/postgres/controls/position.rb delete mode 100644 lib/message_store/postgres/controls/put.rb delete mode 100644 lib/message_store/postgres/controls/stream_name.rb delete mode 100644 lib/message_store/postgres/controls/time.rb delete mode 100644 lib/message_store/postgres/get.rb delete mode 100644 lib/message_store/postgres/get/category.rb delete mode 100644 lib/message_store/postgres/get/category/consumer_group.rb delete mode 100644 lib/message_store/postgres/get/category/correlation.rb delete mode 100644 lib/message_store/postgres/get/condition.rb delete mode 100644 lib/message_store/postgres/get/stream.rb delete mode 100644 lib/message_store/postgres/get/stream/last.rb delete mode 100644 lib/message_store/postgres/log.rb delete mode 100644 lib/message_store/postgres/put.rb delete mode 100644 lib/message_store/postgres/read.rb delete mode 100644 lib/message_store/postgres/session.rb delete mode 100644 lib/message_store/postgres/settings.rb delete mode 100644 lib/message_store/postgres/write.rb create mode 100644 lib/message_store/put.rb create mode 100644 lib/message_store/session.rb create mode 100644 lib/message_store/settings.rb delete mode 100644 message_store-postgres.gemspec delete mode 100644 test/automated/get_last/call.rb delete mode 100644 test/automated/iterator_tests.rb diff --git a/lib/message_store.rb b/lib/message_store.rb index bfdfcba..73ec386 100644 --- a/lib/message_store.rb +++ b/lib/message_store.rb @@ -1,5 +1,4 @@ -require 'pp' -require 'json' +require 'pg' require 'casing' require 'identifier/uuid' @@ -9,9 +8,12 @@ require 'virtual' require 'async_invocation' +require 'log' +require 'settings' + require 'message_store/expected_version' -require 'message_store/no_stream' require 'message_store/id' +require 'message_store/no_stream' require 'message_store/stream_name' require 'message_store/message_data' @@ -21,10 +23,20 @@ require 'message_store/log' +require 'message_store/settings' +require 'message_store/session' + +require 'message_store/put' +require 'message_store/write' + require 'message_store/get' require 'message_store/get/substitute' +require 'message_store/get/condition' +require 'message_store/get/stream' require 'message_store/get/stream/last' require 'message_store/get/stream/last/substitute' +require 'message_store/get/category' +require 'message_store/get/category/correlation' +require 'message_store/get/category/consumer_group' require 'message_store/read/iterator' require 'message_store/read' -require 'message_store/write' diff --git a/lib/message_store/controls.rb b/lib/message_store/controls.rb index 8d8d60a..69acb78 100644 --- a/lib/message_store/controls.rb +++ b/lib/message_store/controls.rb @@ -9,11 +9,11 @@ require 'message_store/controls/category' require 'message_store/controls/stream_name' require 'message_store/controls/read' +require 'message_store/controls/position' require 'message_store/controls/message_data' -require 'message_store/controls/message_data/hash' require 'message_store/controls/message_data/metadata' -require 'message_store/controls/message_data/write' +require 'message_store/controls/message_data/hash' require 'message_store/controls/message_data/read' -require 'message_store/controls/write' +require 'message_store/controls/message_data/write' +require 'message_store/controls/put' require 'message_store/controls/get' -require 'message_store/controls/get_last' diff --git a/lib/message_store/controls/get_last.rb b/lib/message_store/controls/get_last.rb deleted file mode 100644 index fddc074..0000000 --- a/lib/message_store/controls/get_last.rb +++ /dev/null @@ -1,33 +0,0 @@ -module MessageStore - module Controls - module GetLast - def self.example - Example.build - end - - def self.default_session - :default_session - end - - class Example - include MessageStore::Get::Stream::Last - - attr_writer :session - def session - @session ||= GetLast.default_session - end - - def call(stream_name) - end - - def configure(session: nil) - self.session = session - end - - def session?(session) - self.session.equal?(session) - end - end - end - end -end diff --git a/lib/message_store/controls/message_data/write.rb b/lib/message_store/controls/message_data/write.rb index ff45a38..5bb3c86 100644 --- a/lib/message_store/controls/message_data/write.rb +++ b/lib/message_store/controls/message_data/write.rb @@ -48,6 +48,26 @@ def self.data def self.metadata MessageData::Metadata.data end + + module List + Entry = Struct.new(:stream_name, :category, :message_data) + + def self.get(instances: nil, stream_name: nil, category: nil) + instances ||= 1 + + list = [] + instances.times do + instance_stream_name = stream_name || StreamName.example(category: category) + instance_category = MessageStore::StreamName.get_category(instance_stream_name) + + write_message = Controls::MessageData::Write.example + + list << Entry.new(instance_stream_name, instance_category, write_message) + end + + list + end + end end end end diff --git a/lib/message_store/controls/position.rb b/lib/message_store/controls/position.rb new file mode 100644 index 0000000..9ddde03 --- /dev/null +++ b/lib/message_store/controls/position.rb @@ -0,0 +1,13 @@ +module MessageStore + module Controls + module Position + def self.example + 1 + end + + def self.max + (2 ** 63) - 1 + end + end + end +end diff --git a/lib/message_store/controls/put.rb b/lib/message_store/controls/put.rb new file mode 100644 index 0000000..348f9a7 --- /dev/null +++ b/lib/message_store/controls/put.rb @@ -0,0 +1,26 @@ +module MessageStore + module Controls + module Put + def self.call(instances: nil, stream_name: nil, message_data: nil, message: nil, category: nil, type: nil) + instances ||= 1 + stream_name ||= StreamName.example(category: category) + message_data ||= message + + message_specified = !message_data.nil? + + message_data ||= MessageData::Write.example(type: type) + + position = nil + instances.times do + position = MessageStore::Put.(message_data, stream_name) + + unless message_specified + message_data.id = MessageData::Write.id + end + end + + [stream_name, position] + end + end + end +end diff --git a/lib/message_store/controls/read.rb b/lib/message_store/controls/read.rb index 769a8c6..855ad23 100644 --- a/lib/message_store/controls/read.rb +++ b/lib/message_store/controls/read.rb @@ -4,19 +4,10 @@ module Read def self.example(stream_name: nil) stream_name ||= StreamName.example - read = Example.build(stream_name) - - get = Get.example(stream_name: stream_name) - read.iterator.get = get + read = MessageStore::Read.build(stream_name) read end - - class Example - include MessageStore::Read - - def configure(*); end - end end end end diff --git a/lib/message_store/controls/write.rb b/lib/message_store/controls/write.rb deleted file mode 100644 index 50296bb..0000000 --- a/lib/message_store/controls/write.rb +++ /dev/null @@ -1,23 +0,0 @@ -module MessageStore - module Controls - module Write - def self.example - Example.build - end - - class Example - include MessageStore::Write - - def configure(*) - end - - def write(batch, stream_name, expected_version: nil) - logger.trace { "Writing batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } - logger.debug { "Wrote batch (Stream Name: #{stream_name}, Number of Events: #{batch.length}, Expected Version: #{expected_version.inspect})" } - - nil - end - end - end - end -end diff --git a/lib/message_store/get.rb b/lib/message_store/get.rb index 54f93de..1169c7b 100644 --- a/lib/message_store/get.rb +++ b/lib/message_store/get.rb @@ -7,10 +7,157 @@ def self.included(cls) include Virtual include Log::Dependency - abstract :call + prepend BatchSize + + dependency :session, Session + abstract :stream_name - abstract :batch_size + abstract :sql_command + abstract :parameters + abstract :parameter_values abstract :last_position + abstract :log_text + + virtual :specialize_error + virtual :assure + end + end + + module BatchSize + def batch_size + @batch_size ||= Defaults.batch_size + end + end + + def self.build(stream_name, **args) + cls = specialization(stream_name) + cls.build(stream_name, **args) + end + + def self.configure(receiver, stream_name, **args) + attr_name = args.delete(:attr_name) + attr_name ||= :get + + instance = build(stream_name, **args) + receiver.public_send("#{attr_name}=", instance) + end + + ## Is there a path to removing this? - Aaron, Sun Jan 15 2023 + def configure(session: nil) + Session.configure(self, session: session) + end + + def self.call(stream_name, **args) + position = args.delete(:position) + instance = build(stream_name, **args) + instance.(position) + end + + def call(position=nil, stream_name: nil) + position ||= self.class::Defaults.position + + stream_name ||= self.stream_name + + assure + + logger.trace(tag: :get) { "Getting message data (#{log_text(stream_name, position)})" } + + result = get_result(stream_name, position) + + message_data = convert(result) + + logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, #{log_text(stream_name, position)})" } + logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } + + message_data + end + + def get_result(stream_name, position) + logger.trace(tag: :get) { "Getting result (#{log_text(stream_name, position)})" } + + parameter_values = parameter_values(stream_name, position) + + begin + result = session.execute(sql_command, parameter_values) + rescue PG::RaiseException => e + raise_error(e) + end + + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, #{log_text(stream_name, position)})" } + + result + end + + def convert(result) + logger.trace(tag: :get) { "Converting result to message data (Result Count: #{result.ntuples})" } + + message_data = result.map do |record| + Get.message_data(record) + end + + logger.debug(tag: :get) { "Converted result to message data (Message Data Count: #{message_data.length})" } + + message_data + end + + def self.message_data(record) + record['data'] = Get::Deserialize.data(record['data']) + record['metadata'] = Get::Deserialize.metadata(record['metadata']) + record['time'] = Get::Time.utc_coerced(record['time']) + + MessageData::Read.build(record) + end + + def raise_error(pg_error) + error_message = Get.error_message(pg_error) + + error = Condition.error(error_message) + + if error.nil? + error = specialize_error(error_message) + end + + if not error.nil? + logger.error { error_message } + raise error + end + + raise pg_error + end + + def self.error_message(pg_error) + pg_error.message.gsub('ERROR:', '').strip + end + + def self.specialization(stream_name) + if StreamName.category?(stream_name) + Category + else + Stream + end + end + + module Deserialize + def self.data(serialized_data) + return nil if serialized_data.nil? + Transform::Read.(serialized_data, :json, MessageData::Hash) + end + + def self.metadata(serialized_metadata) + return nil if serialized_metadata.nil? + Transform::Read.(serialized_metadata, :json, MessageData::Hash) + end + end + + module Time + def self.utc_coerced(local_time) + Clock::UTC.coerce(local_time) + end + end + + module Defaults + def self.batch_size + 1000 end end end diff --git a/lib/message_store/get/category.rb b/lib/message_store/get/category.rb new file mode 100644 index 0000000..7581bfe --- /dev/null +++ b/lib/message_store/get/category.rb @@ -0,0 +1,79 @@ +module MessageStore + module Get + class Category + Error = Class.new(RuntimeError) + + include Get + + initializer :category, na(:batch_size), :correlation, :consumer_group_member, :consumer_group_size, :condition + alias :stream_name :category + + def self.call(category, position: nil, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil) + instance = build(category, batch_size: batch_size, correlation: correlation, consumer_group_member: consumer_group_member, consumer_group_size: consumer_group_size, condition: condition, session: session) + instance.(position) + end + + def self.build(category, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil) + instance = new(category, batch_size, correlation, consumer_group_member, consumer_group_size, condition) + instance.configure(session: session) + instance + end + + def self.configure(receiver, category, attr_name: nil, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil) + attr_name ||= :get + instance = build(category, batch_size: batch_size, correlation: correlation, consumer_group_member: consumer_group_member, consumer_group_size: consumer_group_size, condition: condition, session: session) + receiver.public_send("#{attr_name}=", instance) + end + + def sql_command + "SELECT * FROM get_category_messages(#{parameters});" + end + + def parameters + '$1::varchar, $2::bigint, $3::bigint, $4::varchar, $5::bigint, $6::bigint, $7::varchar' + end + + def parameter_values(category, position) + [ + category, + position, + batch_size, + correlation, + consumer_group_member, + consumer_group_size, + condition + ] + end + + def last_position(batch) + batch.last.global_position + end + + def specialize_error(error_message) + error = Correlation.error(error_message) + + if error.nil? + error = ConsumerGroup.error(error_message) + end + + error + end + + def log_text(category, position) + "Category: #{category}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Consumer Group Member: #{consumer_group_member.inspect}, Consumer Group Size: #{consumer_group_size.inspect}, Condition: #{condition.inspect})" + end + + def assure + if not MessageStore::StreamName.category?(category) + raise Error, "Must be a category (Stream Name: #{category})" + end + end + + module Defaults + def self.position + 1 + end + end + end + end +end diff --git a/lib/message_store/get/category/consumer_group.rb b/lib/message_store/get/category/consumer_group.rb new file mode 100644 index 0000000..6b2c4e3 --- /dev/null +++ b/lib/message_store/get/category/consumer_group.rb @@ -0,0 +1,18 @@ +module MessageStore + module Get + class Category + module ConsumerGroup + Error = Class.new(RuntimeError) + + def self.error(error_message) + if error_message.start_with?('Consumer group size must not be less than 1') || + error_message.start_with?('Consumer group member must be less than the group size') || + error_message.start_with?('Consumer group member must not be less than 0') || + error_message.start_with?('Consumer group member and size must be specified') + Error.new(error_message) + end + end + end + end + end +end diff --git a/lib/message_store/get/category/correlation.rb b/lib/message_store/get/category/correlation.rb new file mode 100644 index 0000000..5211b3a --- /dev/null +++ b/lib/message_store/get/category/correlation.rb @@ -0,0 +1,15 @@ +module MessageStore + module Get + class Category + module Correlation + Error = Class.new(RuntimeError) + + def self.error(error_message) + if error_message.start_with?('Correlation must be a category') + Error.new(error_message) + end + end + end + end + end +end diff --git a/lib/message_store/get/condition.rb b/lib/message_store/get/condition.rb new file mode 100644 index 0000000..466a5fe --- /dev/null +++ b/lib/message_store/get/condition.rb @@ -0,0 +1,13 @@ +module MessageStore + module Get + module Condition + Error = Class.new(RuntimeError) + + def self.error(error_message) + if error_message.start_with?('Retrieval with SQL condition is not activated') + Get::Condition::Error.new(error_message) + end + end + end + end +end diff --git a/lib/message_store/get/stream.rb b/lib/message_store/get/stream.rb new file mode 100644 index 0000000..a84f762 --- /dev/null +++ b/lib/message_store/get/stream.rb @@ -0,0 +1,65 @@ +module MessageStore + module Get + class Stream + Error = Class.new(RuntimeError) + + include Get + + initializer :stream_name, na(:batch_size), :condition + + def self.call(stream_name, position: nil, batch_size: nil, condition: nil, session: nil) + instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) + instance.(position) + end + + def self.build(stream_name, batch_size: nil, condition: nil, session: nil) + instance = new(stream_name, batch_size, condition) + instance.configure(session: session) + instance + end + + def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, condition: nil, session: nil) + attr_name ||= :get + instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) + receiver.public_send("#{attr_name}=", instance) + end + + def sql_command + "SELECT * FROM get_stream_messages(#{parameters});" + end + + def parameters + '$1::varchar, $2::bigint, $3::bigint, $4::varchar' + end + + def parameter_values(stream_name, position) + [ + stream_name, + position, + batch_size, + condition + ] + end + + def last_position(batch) + batch.last.position + end + + def log_text(stream_name, position) + "Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition.inspect})" + end + + def assure + if MessageStore::StreamName.category?(stream_name) + raise Error, "Must be a stream name (Category: #{stream_name})" + end + end + + module Defaults + def self.position + 0 + end + end + end + end +end diff --git a/lib/message_store/get/stream/last.rb b/lib/message_store/get/stream/last.rb index 1d16c9d..6717b0a 100644 --- a/lib/message_store/get/stream/last.rb +++ b/lib/message_store/get/stream/last.rb @@ -1,62 +1,102 @@ module MessageStore module Get - module Stream - module Last - def self.included(cls) - cls.class_exec do - include Dependency - include Virtual - include Log::Dependency + class Stream + class Last + include Dependency + include Virtual + include Log::Dependency - extend Build - extend Call - extend Configure + dependency :session, Session - prepend InstanceActuator + def self.build(session: nil) + instance = new + Session.configure(instance, session: session) + instance + end - virtual :configure - abstract :call + def self.call(stream_name, type=nil, session: nil) + instance = build(session: session) + instance.(stream_name, type) + end - const_set :Substitute, Substitute - end + def self.configure(receiver, session: nil, attr_name: nil) + attr_name ||= :get_last + + instance = build(session: session) + receiver.public_send("#{attr_name}=", instance) + instance end - module InstanceActuator - def call(stream_name, type=nil) - logger.trace(tag: :get) { "Getting last message data (Stream Name: #{stream_name})" } + def call(stream_name, type=nil) + logger.trace(tag: :get) { "Getting last message data (Stream Name: #{stream_name})" } - message_data = super + result = get_result(stream_name, type) - logger.info(tag: :get) { "Finished getting message data (Stream Name: #{stream_name})" } - logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } + return nil if result.nil? - message_data - end + message_data = convert(result[0]) + + logger.info(tag: :get) { "Finished getting message data (Stream Name: #{stream_name})" } + logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } + + message_data end - module Build - def build(session: nil) - instance = new - instance.configure(session: session) - instance - end + def get_result(stream_name, type) + logger.trace(tag: :get) { "Getting last record (Stream: #{stream_name})" } + + parameter_values = parameter_values(stream_name, type) + sql_command = sql_command(type) + + result = session.execute(sql_command, parameter_values) + + logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" } + + return nil if result.ntuples == 0 + + result + end + + def sql_command(type) + parameters = parameters(type) + + "SELECT * FROM get_last_stream_message(#{parameters});" end - module Configure - def configure(receiver, session: nil, attr_name: nil) - attr_name ||= :get_last + def parameters(type) + parameters = "$1::varchar" - instance = build(session: session) - receiver.public_send("#{attr_name}=", instance) - instance + # Backwards compatibility with versions of message-db that do not + # support the type parameter - Aaron, Scott, Tue Jul 12 2022 + if not type.nil? + parameters << ", $2::varchar" end + + parameters end - module Call - def call(stream_name, type=nil, session: nil) - instance = build(session: session) - instance.(stream_name, type) + def parameter_values(stream_name, type) + parameter_values = [ + stream_name + ] + + # Backwards compatibility with versions of message-db that do not + # support the type parameter - Aaron, Scott, Tue Jul 12 2022 + if not type.nil? + parameter_values << type end + + parameter_values + end + + def convert(record) + logger.trace(tag: :get) { "Converting record to message data" } + + message_data = Get.message_data(record) + + logger.debug(tag: :get) { "Converted record to message data" } + + message_data end end end diff --git a/lib/message_store/get/stream/last/substitute.rb b/lib/message_store/get/stream/last/substitute.rb index b1e4de2..2627dfd 100644 --- a/lib/message_store/get/stream/last/substitute.rb +++ b/lib/message_store/get/stream/last/substitute.rb @@ -1,15 +1,13 @@ module MessageStore module Get - module Stream - module Last + class Stream + class Last module Substitute def self.build GetLast.new end - class GetLast - include Get::Stream::Last - + class GetLast < Stream::Last def call(stream_name, type=nil) streams[stream_name] end diff --git a/lib/message_store/get/substitute.rb b/lib/message_store/get/substitute.rb index 6fec04e..34d9f99 100644 --- a/lib/message_store/get/substitute.rb +++ b/lib/message_store/get/substitute.rb @@ -1,3 +1,4 @@ +## Add tests, this did not have tests in message-store either - Aaron, Sun Jan 15 2023 module MessageStore module Get class Substitute diff --git a/lib/message_store/postgres.rb b/lib/message_store/postgres.rb deleted file mode 100644 index fef1e16..0000000 --- a/lib/message_store/postgres.rb +++ /dev/null @@ -1,24 +0,0 @@ -require 'pg' - -require 'message_store' - -require 'log' -require 'telemetry' -require 'settings' - -require 'message_store/postgres/log' - -require 'message_store/postgres/settings' -require 'message_store/postgres/session' - -require 'message_store/postgres/put' -require 'message_store/postgres/write' - -require 'message_store/postgres/get' -require 'message_store/postgres/get/condition' -require 'message_store/postgres/get/stream' -require 'message_store/postgres/get/stream/last' -require 'message_store/postgres/get/category' -require 'message_store/postgres/get/category/correlation' -require 'message_store/postgres/get/category/consumer_group' -require 'message_store/postgres/read' diff --git a/lib/message_store/postgres/controls.rb b/lib/message_store/postgres/controls.rb deleted file mode 100644 index 3560cdc..0000000 --- a/lib/message_store/postgres/controls.rb +++ /dev/null @@ -1,8 +0,0 @@ -require 'message_store/controls' - -require 'message_store/postgres/controls/time' -require 'message_store/postgres/controls/position' -require 'message_store/postgres/controls/category' -require 'message_store/postgres/controls/stream_name' -require 'message_store/postgres/controls/message_data' -require 'message_store/postgres/controls/put' diff --git a/lib/message_store/postgres/controls/category.rb b/lib/message_store/postgres/controls/category.rb deleted file mode 100644 index 15f1667..0000000 --- a/lib/message_store/postgres/controls/category.rb +++ /dev/null @@ -1,7 +0,0 @@ -module MessageStore - module Postgres - module Controls - Category = MessageStore::Controls::Category - end - end -end diff --git a/lib/message_store/postgres/controls/message_data.rb b/lib/message_store/postgres/controls/message_data.rb deleted file mode 100644 index addfb6a..0000000 --- a/lib/message_store/postgres/controls/message_data.rb +++ /dev/null @@ -1,31 +0,0 @@ -module MessageStore - module Postgres - module Controls - MessageData = MessageStore::Controls::MessageData - - module MessageData - module Write - module List - Entry = Struct.new(:stream_name, :category, :message_data) - - def self.get(instances: nil, stream_name: nil, category: nil) - instances ||= 1 - - list = [] - instances.times do - instance_stream_name = stream_name || StreamName.example(category: category) - instance_category = MessageStore::StreamName.get_category(instance_stream_name) - - write_message = Controls::MessageData::Write.example - - list << Entry.new(instance_stream_name, instance_category, write_message) - end - - list - end - end - end - end - end - end -end diff --git a/lib/message_store/postgres/controls/position.rb b/lib/message_store/postgres/controls/position.rb deleted file mode 100644 index 5b5c5d0..0000000 --- a/lib/message_store/postgres/controls/position.rb +++ /dev/null @@ -1,15 +0,0 @@ -module MessageStore - module Postgres - module Controls - module Position - def self.example - 1 - end - - def self.max - (2 ** 63) - 1 - end - end - end - end -end diff --git a/lib/message_store/postgres/controls/put.rb b/lib/message_store/postgres/controls/put.rb deleted file mode 100644 index 0a55b10..0000000 --- a/lib/message_store/postgres/controls/put.rb +++ /dev/null @@ -1,28 +0,0 @@ -module MessageStore - module Postgres - module Controls - module Put - def self.call(instances: nil, stream_name: nil, message_data: nil, message: nil, category: nil, type: nil) - instances ||= 1 - stream_name ||= StreamName.example(category: category) - message_data ||= message - - message_specified = !message_data.nil? - - message_data ||= MessageData::Write.example(type: type) - - position = nil - instances.times do - position = MessageStore::Postgres::Put.(message_data, stream_name) - - unless message_specified - message_data.id = MessageData::Write.id - end - end - - [stream_name, position] - end - end - end - end -end diff --git a/lib/message_store/postgres/controls/stream_name.rb b/lib/message_store/postgres/controls/stream_name.rb deleted file mode 100644 index b35f009..0000000 --- a/lib/message_store/postgres/controls/stream_name.rb +++ /dev/null @@ -1,7 +0,0 @@ -module MessageStore - module Postgres - module Controls - StreamName = MessageStore::Controls::StreamName - end - end -end diff --git a/lib/message_store/postgres/controls/time.rb b/lib/message_store/postgres/controls/time.rb deleted file mode 100644 index 4d99aad..0000000 --- a/lib/message_store/postgres/controls/time.rb +++ /dev/null @@ -1,7 +0,0 @@ -module MessageStore - module Postgres - module Controls - Time = MessageStore::Controls::Time - end - end -end diff --git a/lib/message_store/postgres/get.rb b/lib/message_store/postgres/get.rb deleted file mode 100644 index ce440d8..0000000 --- a/lib/message_store/postgres/get.rb +++ /dev/null @@ -1,165 +0,0 @@ -module MessageStore - module Postgres - module Get - def self.included(cls) - cls.class_exec do - include MessageStore::Get - - prepend Call - prepend BatchSize - - dependency :session, Session - - abstract :stream_name - abstract :sql_command - abstract :parameters - abstract :parameter_values - abstract :last_position - abstract :log_text - - virtual :specialize_error - virtual :assure - end - end - - module BatchSize - def batch_size - @batch_size ||= Defaults.batch_size - end - end - - def self.build(stream_name, **args) - cls = specialization(stream_name) - cls.build(stream_name, **args) - end - - def self.configure(receiver, stream_name, **args) - attr_name = args.delete(:attr_name) - attr_name ||= :get - - instance = build(stream_name, **args) - receiver.public_send("#{attr_name}=", instance) - end - - def configure(session: nil) - Session.configure(self, session: session) - end - - def self.call(stream_name, **args) - position = args.delete(:position) - instance = build(stream_name, **args) - instance.(position) - end - - module Call - def call(position=nil, stream_name: nil) - position ||= self.class::Defaults.position - - stream_name ||= self.stream_name - - assure - - logger.trace(tag: :get) { "Getting message data (#{log_text(stream_name, position)})" } - - result = get_result(stream_name, position) - - message_data = convert(result) - - logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, #{log_text(stream_name, position)})" } - logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } - - message_data - end - end - - def get_result(stream_name, position) - logger.trace(tag: :get) { "Getting result (#{log_text(stream_name, position)})" } - - parameter_values = parameter_values(stream_name, position) - - begin - result = session.execute(sql_command, parameter_values) - rescue PG::RaiseException => e - raise_error(e) - end - - logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, #{log_text(stream_name, position)})" } - - result - end - - def convert(result) - logger.trace(tag: :get) { "Converting result to message data (Result Count: #{result.ntuples})" } - - message_data = result.map do |record| - Get.message_data(record) - end - - logger.debug(tag: :get) { "Converted result to message data (Message Data Count: #{message_data.length})" } - - message_data - end - - def self.message_data(record) - record['data'] = Get::Deserialize.data(record['data']) - record['metadata'] = Get::Deserialize.metadata(record['metadata']) - record['time'] = Get::Time.utc_coerced(record['time']) - - MessageData::Read.build(record) - end - - def raise_error(pg_error) - error_message = Get.error_message(pg_error) - - error = Condition.error(error_message) - - if error.nil? - error = specialize_error(error_message) - end - - if not error.nil? - logger.error { error_message } - raise error - end - - raise pg_error - end - - def self.error_message(pg_error) - pg_error.message.gsub('ERROR:', '').strip - end - - def self.specialization(stream_name) - if StreamName.category?(stream_name) - Category - else - Stream - end - end - - module Deserialize - def self.data(serialized_data) - return nil if serialized_data.nil? - Transform::Read.(serialized_data, :json, MessageData::Hash) - end - - def self.metadata(serialized_metadata) - return nil if serialized_metadata.nil? - Transform::Read.(serialized_metadata, :json, MessageData::Hash) - end - end - - module Time - def self.utc_coerced(local_time) - Clock::UTC.coerce(local_time) - end - end - - module Defaults - def self.batch_size - 1000 - end - end - end - end -end diff --git a/lib/message_store/postgres/get/category.rb b/lib/message_store/postgres/get/category.rb deleted file mode 100644 index d226735..0000000 --- a/lib/message_store/postgres/get/category.rb +++ /dev/null @@ -1,81 +0,0 @@ -module MessageStore - module Postgres - module Get - class Category - Error = Class.new(RuntimeError) - - include Get - - initializer :category, na(:batch_size), :correlation, :consumer_group_member, :consumer_group_size, :condition - alias :stream_name :category - - def self.call(category, position: nil, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil) - instance = build(category, batch_size: batch_size, correlation: correlation, consumer_group_member: consumer_group_member, consumer_group_size: consumer_group_size, condition: condition, session: session) - instance.(position) - end - - def self.build(category, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil) - instance = new(category, batch_size, correlation, consumer_group_member, consumer_group_size, condition) - instance.configure(session: session) - instance - end - - def self.configure(receiver, category, attr_name: nil, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil) - attr_name ||= :get - instance = build(category, batch_size: batch_size, correlation: correlation, consumer_group_member: consumer_group_member, consumer_group_size: consumer_group_size, condition: condition, session: session) - receiver.public_send("#{attr_name}=", instance) - end - - def sql_command - "SELECT * FROM get_category_messages(#{parameters});" - end - - def parameters - '$1::varchar, $2::bigint, $3::bigint, $4::varchar, $5::bigint, $6::bigint, $7::varchar' - end - - def parameter_values(category, position) - [ - category, - position, - batch_size, - correlation, - consumer_group_member, - consumer_group_size, - condition - ] - end - - def last_position(batch) - batch.last.global_position - end - - def specialize_error(error_message) - error = Correlation.error(error_message) - - if error.nil? - error = ConsumerGroup.error(error_message) - end - - error - end - - def log_text(category, position) - "Category: #{category}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Consumer Group Member: #{consumer_group_member.inspect}, Consumer Group Size: #{consumer_group_size.inspect}, Condition: #{condition.inspect})" - end - - def assure - if not MessageStore::StreamName.category?(category) - raise Error, "Must be a category (Stream Name: #{category})" - end - end - - module Defaults - def self.position - 1 - end - end - end - end - end -end diff --git a/lib/message_store/postgres/get/category/consumer_group.rb b/lib/message_store/postgres/get/category/consumer_group.rb deleted file mode 100644 index fc5bd0b..0000000 --- a/lib/message_store/postgres/get/category/consumer_group.rb +++ /dev/null @@ -1,20 +0,0 @@ -module MessageStore - module Postgres - module Get - class Category - module ConsumerGroup - Error = Class.new(RuntimeError) - - def self.error(error_message) - if error_message.start_with?('Consumer group size must not be less than 1') || - error_message.start_with?('Consumer group member must be less than the group size') || - error_message.start_with?('Consumer group member must not be less than 0') || - error_message.start_with?('Consumer group member and size must be specified') - Error.new(error_message) - end - end - end - end - end - end -end diff --git a/lib/message_store/postgres/get/category/correlation.rb b/lib/message_store/postgres/get/category/correlation.rb deleted file mode 100644 index c857077..0000000 --- a/lib/message_store/postgres/get/category/correlation.rb +++ /dev/null @@ -1,17 +0,0 @@ -module MessageStore - module Postgres - module Get - class Category - module Correlation - Error = Class.new(RuntimeError) - - def self.error(error_message) - if error_message.start_with?('Correlation must be a category') - Error.new(error_message) - end - end - end - end - end - end -end diff --git a/lib/message_store/postgres/get/condition.rb b/lib/message_store/postgres/get/condition.rb deleted file mode 100644 index 7e0cb99..0000000 --- a/lib/message_store/postgres/get/condition.rb +++ /dev/null @@ -1,15 +0,0 @@ -module MessageStore - module Postgres - module Get - module Condition - Error = Class.new(RuntimeError) - - def self.error(error_message) - if error_message.start_with?('Retrieval with SQL condition is not activated') - Get::Condition::Error.new(error_message) - end - end - end - end - end -end diff --git a/lib/message_store/postgres/get/stream.rb b/lib/message_store/postgres/get/stream.rb deleted file mode 100644 index f769e9b..0000000 --- a/lib/message_store/postgres/get/stream.rb +++ /dev/null @@ -1,67 +0,0 @@ -module MessageStore - module Postgres - module Get - class Stream - Error = Class.new(RuntimeError) - - include Get - - initializer :stream_name, na(:batch_size), :condition - - def self.call(stream_name, position: nil, batch_size: nil, condition: nil, session: nil) - instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) - instance.(position) - end - - def self.build(stream_name, batch_size: nil, condition: nil, session: nil) - instance = new(stream_name, batch_size, condition) - instance.configure(session: session) - instance - end - - def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, condition: nil, session: nil) - attr_name ||= :get - instance = build(stream_name, batch_size: batch_size, condition: condition, session: session) - receiver.public_send("#{attr_name}=", instance) - end - - def sql_command - "SELECT * FROM get_stream_messages(#{parameters});" - end - - def parameters - '$1::varchar, $2::bigint, $3::bigint, $4::varchar' - end - - def parameter_values(stream_name, position) - [ - stream_name, - position, - batch_size, - condition - ] - end - - def last_position(batch) - batch.last.position - end - - def log_text(stream_name, position) - "Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition.inspect})" - end - - def assure - if MessageStore::StreamName.category?(stream_name) - raise Error, "Must be a stream name (Category: #{stream_name})" - end - end - - module Defaults - def self.position - 0 - end - end - end - end - end -end diff --git a/lib/message_store/postgres/get/stream/last.rb b/lib/message_store/postgres/get/stream/last.rb deleted file mode 100644 index 44414e0..0000000 --- a/lib/message_store/postgres/get/stream/last.rb +++ /dev/null @@ -1,89 +0,0 @@ -module MessageStore - module Postgres - module Get - class Stream - class Last - include MessageStore::Get::Stream::Last - - dependency :session, Session - - def configure(session: nil) - Session.configure(self, session: session) - end - - def call(stream_name, type=nil) - logger.trace(tag: :get) { "Getting last message data (Stream Name: #{stream_name})" } - - result = get_result(stream_name, type) - - return nil if result.nil? - - message_data = convert(result[0]) - - logger.info(tag: :get) { "Finished getting message data (Stream Name: #{stream_name})" } - logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect } - - message_data - end - - def get_result(stream_name, type) - logger.trace(tag: :get) { "Getting last record (Stream: #{stream_name})" } - - parameter_values = parameter_values(stream_name, type) - sql_command = sql_command(type) - - result = session.execute(sql_command, parameter_values) - - logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" } - - return nil if result.ntuples == 0 - - result - end - - def sql_command(type) - parameters = parameters(type) - - "SELECT * FROM get_last_stream_message(#{parameters});" - end - - def parameters(type) - parameters = "$1::varchar" - - # Backwards compatibility with versions of message-db that do not - # support the type parameter - Aaron, Scott, Tue Jul 12 2022 - if not type.nil? - parameters << ", $2::varchar" - end - - parameters - end - - def parameter_values(stream_name, type) - parameter_values = [ - stream_name - ] - - # Backwards compatibility with versions of message-db that do not - # support the type parameter - Aaron, Scott, Tue Jul 12 2022 - if not type.nil? - parameter_values << type - end - - parameter_values - end - - def convert(record) - logger.trace(tag: :get) { "Converting record to message data" } - - message_data = Get.message_data(record) - - logger.debug(tag: :get) { "Converted record to message data" } - - message_data - end - end - end - end - end -end diff --git a/lib/message_store/postgres/log.rb b/lib/message_store/postgres/log.rb deleted file mode 100644 index 5f40928..0000000 --- a/lib/message_store/postgres/log.rb +++ /dev/null @@ -1,9 +0,0 @@ -module MessageStore - module Postgres - class Log < ::Log - def tag!(tags) - tags << :message_store - end - end - end -end diff --git a/lib/message_store/postgres/put.rb b/lib/message_store/postgres/put.rb deleted file mode 100644 index 55005f6..0000000 --- a/lib/message_store/postgres/put.rb +++ /dev/null @@ -1,146 +0,0 @@ -module MessageStore - module Postgres - class Put - include Dependency - include Log::Dependency - - dependency :session, Session - dependency :identifier, Identifier::UUID::Random - - def self.build(session: nil) - new.tap do |instance| - instance.configure(session: session) - end - end - - def configure(session: nil) - Session.configure(self, session: session) - Identifier::UUID::Random.configure(self) - end - - def self.configure(receiver, session: nil, attr_name: nil) - attr_name ||= :put - instance = build(session: session) - receiver.public_send "#{attr_name}=", instance - end - - def self.call(write_message, stream_name, expected_version: nil, session: nil) - instance = build(session: session) - instance.(write_message, stream_name, expected_version: expected_version) - end - - def call(write_message, stream_name, expected_version: nil) - logger.trace(tag: :put) { "Putting message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :message_data]) { write_message.pretty_inspect } - - write_message.id ||= identifier.get - - id, type, data, metadata = destructure_message(write_message) - expected_version = ExpectedVersion.canonize(expected_version) - - insert_message(id, stream_name, type, data, metadata, expected_version).tap do |position| - logger.info(tag: :put) { "Put message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect}, Position: #{position})" } - logger.info(tags: [:data, :message_data]) { write_message.pretty_inspect } - end - end - - def destructure_message(write_message) - id = write_message.id - type = write_message.type - data = write_message.data - metadata = write_message.metadata - - logger.debug(tags: [:data, :message_data]) { "ID: #{id.pretty_inspect}" } - logger.debug(tags: [:data, :message_data]) { "Type: #{type.pretty_inspect}" } - logger.debug(tags: [:data, :message_data]) { "Data: #{data.pretty_inspect}" } - logger.debug(tags: [:data, :message_data]) { "Metadata: #{metadata.pretty_inspect}" } - - return id, type, data, metadata - end - - def insert_message(id, stream_name, type, data, metadata, expected_version) - transformed_data = transformed_data(data) - transformed_metadata = transformed_metadata(metadata) - records = execute_query(id, stream_name, type, transformed_data, transformed_metadata, expected_version) - position(records) - end - - def execute_query(id, stream_name, type, transformed_data, transformed_metadata, expected_version) - logger.trace(tag: :put) { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } - - params = [ - id, - stream_name, - type, - transformed_data, - transformed_metadata, - expected_version - ] - - begin - records = session.execute(self.class.statement, params) - rescue PG::RaiseException => e - raise_error e - end - - logger.debug(tag: :put) { "Executed insert (Type: #{type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } - - records - end - - def self.statement - @statement ||= "SELECT write_message($1::varchar, $2::varchar, $3::varchar, $4::jsonb, $5::jsonb, $6::bigint);" - end - - def transformed_data(data) - transformed_data = nil - - if data.is_a?(Hash) && data.empty? - data = nil - end - - unless data.nil? - transformable_data = MessageData::Hash[data] - transformed_data = Transform::Write.(transformable_data, :json) - end - - logger.debug(tags: [:data, :serialize]) { "Transformed Data: #{transformed_data.inspect}" } - transformed_data - end - - def transformed_metadata(metadata) - transformed_metadata = nil - - if metadata.is_a?(Hash) && metadata.empty? - metadata = nil - end - - unless metadata.nil? - transformable_metadata = MessageData::Hash[metadata] - transformed_metadata = Transform::Write.(transformable_metadata, :json) - end - - logger.debug(tags: [:data, :serialize]) { "Transformed Metadata: #{transformed_metadata.inspect}" } - transformed_metadata - end - - def position(records) - position = nil - unless records[0].nil? - position = records[0].values[0] - end - position - end - - def raise_error(pg_error) - error_message = pg_error.message - if error_message.include? 'Wrong expected version' - error_message.gsub!('ERROR:', '').strip! - logger.error { error_message } - raise ExpectedVersion::Error, error_message - end - raise pg_error - end - end - end -end diff --git a/lib/message_store/postgres/read.rb b/lib/message_store/postgres/read.rb deleted file mode 100644 index 1c61e54..0000000 --- a/lib/message_store/postgres/read.rb +++ /dev/null @@ -1,17 +0,0 @@ -module MessageStore - module Postgres - class Read - include MessageStore::Read - - def configure(session: nil, condition: nil) - Get.configure(self.iterator, self.stream_name, batch_size: batch_size, condition: condition, session: session) - end - - module Defaults - def self.batch_size - Postgres::Get::Defaults.batch_size - end - end - end - end -end diff --git a/lib/message_store/postgres/session.rb b/lib/message_store/postgres/session.rb deleted file mode 100644 index 8d89d6b..0000000 --- a/lib/message_store/postgres/session.rb +++ /dev/null @@ -1,176 +0,0 @@ -module MessageStore - module Postgres - class Session - Error = Class.new(RuntimeError) - - include Dependency - include Settings::Setting - - include Log::Dependency - - dependency :clock, Clock::UTC - - def self.settings - Settings.names - end - - settings.each do |s| - setting s - end - - attr_accessor :connection - attr_accessor :executed_time - - def self.build(settings: nil) - instance = new - - settings ||= Settings.instance - settings.set(instance) - - Clock::UTC.configure(instance) - - instance - end - - def self.configure(receiver, session: nil, settings: nil, attr_name: nil) - attr_name ||= :session - - if session != nil && settings != nil - error_msg = "Session configured with both settings and session arguments. Use one or the other, but not both." - logger.error(tag: :session) { error_msg } - raise Error, error_msg - end - - instance = session || build(settings: settings) - receiver.public_send "#{attr_name}=", instance - instance - end - - def open - logger.trace(tag: :session) { "Connecting to database" } - - if connected? - logger.debug(tag: :session) { "Already connected. A new connection will not be built." } - return connection - end - - logger.debug(tag: :session) { "Not connected. A new connection will be built." } - connection = self.class.build_connection(self) - self.connection = connection - - logger.debug(tag: :session) { "Connected to database" } - - connection - end - alias :connect :open - - def self.build_connection(instance) - settings = instance.settings - logger.trace(tag: :session) { "Building new connection to database (Settings: #{LogText.settings(settings).inspect})" } - - connection = PG::Connection.open(settings) - connection.type_map_for_results = PG::BasicTypeMapForResults.new(connection) - - logger.debug(tag: :session) { "Built new connection to database (Settings: #{LogText.settings(settings).inspect})" } - - connection - end - - def connected? - return false if connection.nil? - - status = PG::CONNECTION_OK - begin - status = connection.status - rescue PG::ConnectionBad - status = nil - end - - status == PG::CONNECTION_OK - end - alias :open? :connected? - - def close - if connection.nil? - return - end - - connection.close - self.connection = nil - end - - def reset - connection.reset - end - - def execute(sql_command, params=nil) - logger.trace(tag: :session) { "Executing SQL command" } - logger.trace(tag: :sql) { sql_command } - logger.trace(tag: :data) { params.pretty_inspect } - - unless connected? - connect - end - - if params.nil? - connection.exec(sql_command).tap do - self.executed_time = clock.now - logger.debug(tag: :session) { "Executed SQL command (no params)" } - end - else - connection.exec_params(sql_command, params).tap do - self.executed_time = clock.now - logger.debug(tag: :session) { "Executed SQL command with params" } - end - end - end - - def executed_time_elapsed_milliseconds - return nil if executed_time.nil? - - (clock.now - executed_time) * 1000 - end - - def transaction(&blk) - unless connected? - connect - end - - connection.transaction(&blk) - end - - def escape(data) - connection = connect - - escaped_data = connection.escape(data) - - escaped_data - end - - def settings - settings = {} - self.class.settings.each do |s| - val = public_send(s) - settings[s] = val unless val.nil? - end - settings - end - - def self.logger - @logger ||= Log.get self - end - - module LogText - def self.settings(settings) - s = settings.dup - - if s.has_key?(:password) - s[:password] = '*' * 8 - end - - s - end - end - end - end -end diff --git a/lib/message_store/postgres/settings.rb b/lib/message_store/postgres/settings.rb deleted file mode 100644 index e45b6ea..0000000 --- a/lib/message_store/postgres/settings.rb +++ /dev/null @@ -1,41 +0,0 @@ -module MessageStore - module Postgres - class Settings < ::Settings - def self.instance - @instance ||= build - end - - def self.data_source - Defaults.data_source - end - - def self.names - [ - :dbname, - :host, - :hostaddr, - :port, - :user, - :password, - :connect_timeout, - :options, - :sslmode, - :krbsrvname, - :gsslib, - :service, - :keepalives, - :keepalives_idle, - :keepalives_interval, - :keepalives_count, - :tcp_user_timeout - ] - end - - class Defaults - def self.data_source - ENV['MESSAGE_STORE_SETTINGS_PATH'] || 'settings/message_store_postgres.json' - end - end - end - end -end diff --git a/lib/message_store/postgres/write.rb b/lib/message_store/postgres/write.rb deleted file mode 100644 index 7b707f6..0000000 --- a/lib/message_store/postgres/write.rb +++ /dev/null @@ -1,52 +0,0 @@ -module MessageStore - module Postgres - class Write - include MessageStore::Write - - dependency :put - - def configure(session: nil) - Put.configure(self, session: session) - end - - def write(batch, stream_name, expected_version: nil) - logger.trace(tag: :write) do - message_types = batch.map {|message_data| message_data.type }.uniq.join(', ') - "Writing batch (Stream Name: #{stream_name}, Types: #{message_types}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})" - end - - unless expected_version.nil? - expected_version = ExpectedVersion.canonize(expected_version) - end - - last_position = nil - put.session.transaction do - batch.each do |message_data| - last_position = write_message_data(message_data, stream_name, expected_version: expected_version) - - unless expected_version.nil? - expected_version += 1 - end - end - end - - logger.debug(tag: :write) do - message_types = batch.map {|message_data| message_data.type }.uniq.join(', ') - "Wrote batch (Stream Name: #{stream_name}, Types: #{message_types}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})" - end - - last_position - end - - def write_message_data(message_data, stream_name, expected_version: nil) - logger.trace(tag: :write) { "Writing message data (Stream Name: #{stream_name}, Type: #{message_data.type}, Expected Version: #{expected_version.inspect})" } - logger.trace(tags: [:data, :message_data]) { message_data.pretty_inspect } - - put.(message_data, stream_name, expected_version: expected_version).tap do - logger.debug(tag: :write) { "Wrote message data (Stream Name: #{stream_name}, Type: #{message_data.type}, Expected Version: #{expected_version.inspect})" } - logger.debug(tags: [:data, :message_data]) { message_data.pretty_inspect } - end - end - end - end -end diff --git a/lib/message_store/put.rb b/lib/message_store/put.rb new file mode 100644 index 0000000..e42acf9 --- /dev/null +++ b/lib/message_store/put.rb @@ -0,0 +1,140 @@ +module MessageStore + class Put + include Dependency + include Log::Dependency + + dependency :session, Session + dependency :identifier, Identifier::UUID::Random + + def self.build(session: nil) + instance = new + Session.configure(instance, session: session) + Identifier::UUID::Random.configure(instance) + instance + end + + def self.configure(receiver, session: nil, attr_name: nil) + attr_name ||= :put + instance = build(session: session) + receiver.public_send "#{attr_name}=", instance + end + + def self.call(write_message, stream_name, expected_version: nil, session: nil) + instance = build(session: session) + instance.(write_message, stream_name, expected_version: expected_version) + end + + def call(write_message, stream_name, expected_version: nil) + logger.trace(tag: :put) { "Putting message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" } + logger.trace(tags: [:data, :message_data]) { write_message.pretty_inspect } + + write_message.id ||= identifier.get + + id, type, data, metadata = destructure_message(write_message) + expected_version = ExpectedVersion.canonize(expected_version) + + insert_message(id, stream_name, type, data, metadata, expected_version).tap do |position| + logger.info(tag: :put) { "Put message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect}, Position: #{position})" } + logger.info(tags: [:data, :message_data]) { write_message.pretty_inspect } + end + end + + def destructure_message(write_message) + id = write_message.id + type = write_message.type + data = write_message.data + metadata = write_message.metadata + + logger.debug(tags: [:data, :message_data]) { "ID: #{id.pretty_inspect}" } + logger.debug(tags: [:data, :message_data]) { "Type: #{type.pretty_inspect}" } + logger.debug(tags: [:data, :message_data]) { "Data: #{data.pretty_inspect}" } + logger.debug(tags: [:data, :message_data]) { "Metadata: #{metadata.pretty_inspect}" } + + return id, type, data, metadata + end + + def insert_message(id, stream_name, type, data, metadata, expected_version) + transformed_data = transformed_data(data) + transformed_metadata = transformed_metadata(metadata) + records = execute_query(id, stream_name, type, transformed_data, transformed_metadata, expected_version) + position(records) + end + + def execute_query(id, stream_name, type, transformed_data, transformed_metadata, expected_version) + logger.trace(tag: :put) { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } + + params = [ + id, + stream_name, + type, + transformed_data, + transformed_metadata, + expected_version + ] + + begin + records = session.execute(self.class.statement, params) + rescue PG::RaiseException => e + raise_error e + end + + logger.debug(tag: :put) { "Executed insert (Type: #{type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" } + + records + end + + def self.statement + @statement ||= "SELECT write_message($1::varchar, $2::varchar, $3::varchar, $4::jsonb, $5::jsonb, $6::bigint);" + end + + def transformed_data(data) + transformed_data = nil + + if data.is_a?(Hash) && data.empty? + data = nil + end + + unless data.nil? + transformable_data = MessageData::Hash[data] + transformed_data = Transform::Write.(transformable_data, :json) + end + + logger.debug(tags: [:data, :serialize]) { "Transformed Data: #{transformed_data.inspect}" } + transformed_data + end + + def transformed_metadata(metadata) + transformed_metadata = nil + + if metadata.is_a?(Hash) && metadata.empty? + metadata = nil + end + + unless metadata.nil? + transformable_metadata = MessageData::Hash[metadata] + transformed_metadata = Transform::Write.(transformable_metadata, :json) + end + + logger.debug(tags: [:data, :serialize]) { "Transformed Metadata: #{transformed_metadata.inspect}" } + transformed_metadata + end + + def position(records) + position = nil + unless records[0].nil? + position = records[0].values[0] + end + position + end + + def raise_error(pg_error) + error_message = pg_error.message + if error_message.include? 'Wrong expected version' + error_message.gsub!('ERROR:', '').strip! + logger.error { error_message } + raise ExpectedVersion::Error, error_message + end + raise pg_error + end + end +end diff --git a/lib/message_store/read.rb b/lib/message_store/read.rb index bf3a501..e32b633 100644 --- a/lib/message_store/read.rb +++ b/lib/message_store/read.rb @@ -1,48 +1,36 @@ module MessageStore - module Read - def self.included(cls) - cls.class_exec do - include Dependency - include Initializer - include Virtual - include Log::Dependency + class Read + include Dependency + include Initializer + include Virtual + include Log::Dependency - extend Build - extend Call - extend Configure + Error = Class.new(RuntimeError) - dependency :iterator, Iterator + dependency :iterator, Iterator - initializer :stream_name, :position, :batch_size + initializer :stream_name, :position, :batch_size - abstract :configure - end - end + def self.build(stream_name, position: nil, batch_size: nil, session: nil, condition: nil, **arguments) + instance = new(stream_name, position, batch_size) - Error = Class.new(RuntimeError) + Iterator.configure(instance, position) - module Build - def build(stream_name, position: nil, batch_size: nil, session: nil, **arguments) - new(stream_name, position, batch_size).tap do |instance| - Iterator.configure(instance, position) - instance.configure(session: session, **arguments) - end - end + iterator = instance.iterator + Get.configure(iterator, stream_name, batch_size: batch_size, condition: condition, session: session) + + instance end - module Call - def call(stream_name, position: nil, batch_size: nil, session: nil, **arguments, &action) - instance = build(stream_name, position: position, batch_size: batch_size, session: session, **arguments) - instance.(&action) - end + def self.call(stream_name, position: nil, batch_size: nil, session: nil, **arguments, &action) + instance = build(stream_name, position: position, batch_size: batch_size, session: session, **arguments) + instance.(&action) end - module Configure - def configure(receiver, stream_name, attr_name: nil, position: nil, batch_size: nil, session: nil, **arguments) - attr_name ||= :read - instance = build(stream_name, position: position, batch_size: batch_size, session: session, **arguments) - receiver.public_send "#{attr_name}=", instance - end + def self.configure(receiver, stream_name, attr_name: nil, position: nil, batch_size: nil, session: nil, **arguments) + attr_name ||= :read + instance = build(stream_name, position: position, batch_size: batch_size, session: session, **arguments) + receiver.public_send "#{attr_name}=", instance end def call(&action) @@ -77,5 +65,11 @@ def enumerate_message_data(&action) logger.debug(tag: :read) { "Enumerated (Stream Name: #{stream_name})" } end + + module Defaults + def self.batch_size + Get::Defaults.batch_size + end + end end end diff --git a/lib/message_store/read/iterator.rb b/lib/message_store/read/iterator.rb index 22e3019..1774250 100644 --- a/lib/message_store/read/iterator.rb +++ b/lib/message_store/read/iterator.rb @@ -1,5 +1,6 @@ +## Add tests - Aaron, Sun Jan 15 2023 module MessageStore - module Read + class Read class Iterator include Dependency include Initializer @@ -25,10 +26,10 @@ def batch_size end def self.build(position=nil) - new.tap do |instance| - instance.starting_position = position - Log.get(self).debug { "Built Iterator (Starting Position: #{position.inspect})" } - end + instance = new + instance.starting_position = position + Log.get(self).debug { "Built Iterator (Starting Position: #{position.inspect})" } + instance end def self.configure(receiver, position=nil, attr_name: nil) diff --git a/lib/message_store/session.rb b/lib/message_store/session.rb new file mode 100644 index 0000000..c58fbff --- /dev/null +++ b/lib/message_store/session.rb @@ -0,0 +1,174 @@ +module MessageStore + class Session + Error = Class.new(RuntimeError) + + include Dependency + include Settings::Setting + + include Log::Dependency + + dependency :clock, Clock::UTC + + def self.settings + Settings.names + end + + settings.each do |s| + setting s + end + + attr_accessor :connection + attr_accessor :executed_time + + def self.build(settings: nil) + instance = new + + settings ||= Settings.instance + settings.set(instance) + + Clock::UTC.configure(instance) + + instance + end + + def self.configure(receiver, session: nil, settings: nil, attr_name: nil) + attr_name ||= :session + + if session != nil && settings != nil + error_msg = "Session configured with both settings and session arguments. Use one or the other, but not both." + logger.error(tag: :session) { error_msg } + raise Error, error_msg + end + + instance = session || build(settings: settings) + receiver.public_send "#{attr_name}=", instance + instance + end + + def open + logger.trace(tag: :session) { "Connecting to database" } + + if connected? + logger.debug(tag: :session) { "Already connected. A new connection will not be built." } + return connection + end + + logger.debug(tag: :session) { "Not connected. A new connection will be built." } + connection = self.class.build_connection(self) + self.connection = connection + + logger.debug(tag: :session) { "Connected to database" } + + connection + end + alias :connect :open + + def self.build_connection(instance) + settings = instance.settings + logger.trace(tag: :session) { "Building new connection to database (Settings: #{LogText.settings(settings).inspect})" } + + connection = PG::Connection.open(settings) + connection.type_map_for_results = PG::BasicTypeMapForResults.new(connection) + + logger.debug(tag: :session) { "Built new connection to database (Settings: #{LogText.settings(settings).inspect})" } + + connection + end + + def connected? + return false if connection.nil? + + status = PG::CONNECTION_OK + begin + status = connection.status + rescue PG::ConnectionBad + status = nil + end + + status == PG::CONNECTION_OK + end + alias :open? :connected? + + def close + if connection.nil? + return + end + + connection.close + self.connection = nil + end + + def reset + connection.reset + end + + def execute(sql_command, params=nil) + logger.trace(tag: :session) { "Executing SQL command" } + logger.trace(tag: :sql) { sql_command } + logger.trace(tag: :data) { params.pretty_inspect } + + unless connected? + connect + end + + if params.nil? + connection.exec(sql_command).tap do + self.executed_time = clock.now + logger.debug(tag: :session) { "Executed SQL command (no params)" } + end + else + connection.exec_params(sql_command, params).tap do + self.executed_time = clock.now + logger.debug(tag: :session) { "Executed SQL command with params" } + end + end + end + + def executed_time_elapsed_milliseconds + return nil if executed_time.nil? + + (clock.now - executed_time) * 1000 + end + + def transaction(&blk) + unless connected? + connect + end + + connection.transaction(&blk) + end + + def escape(data) + connection = connect + + escaped_data = connection.escape(data) + + escaped_data + end + + def settings + settings = {} + self.class.settings.each do |s| + val = public_send(s) + settings[s] = val unless val.nil? + end + settings + end + + def self.logger + @logger ||= Log.get self + end + + module LogText + def self.settings(settings) + s = settings.dup + + if s.has_key?(:password) + s[:password] = '*' * 8 + end + + s + end + end + end +end diff --git a/lib/message_store/settings.rb b/lib/message_store/settings.rb new file mode 100644 index 0000000..095ef45 --- /dev/null +++ b/lib/message_store/settings.rb @@ -0,0 +1,39 @@ +module MessageStore + class Settings < ::Settings + def self.instance + @instance ||= build + end + + def self.data_source + Defaults.data_source + end + + def self.names + [ + :dbname, + :host, + :hostaddr, + :port, + :user, + :password, + :connect_timeout, + :options, + :sslmode, + :krbsrvname, + :gsslib, + :service, + :keepalives, + :keepalives_idle, + :keepalives_interval, + :keepalives_count, + :tcp_user_timeout + ] + end + + class Defaults + def self.data_source + ENV['MESSAGE_STORE_SETTINGS_PATH'] || 'settings/message_store_postgres.json' + end + end + end +end diff --git a/lib/message_store/write.rb b/lib/message_store/write.rb index b5c1655..f407ed3 100644 --- a/lib/message_store/write.rb +++ b/lib/message_store/write.rb @@ -1,44 +1,28 @@ module MessageStore - module Write - def self.included(cls) - cls.class_exec do - include Dependency - include Virtual - include Log::Dependency + class Write + include Dependency + include Virtual + include Log::Dependency - extend Build - extend Call - extend Configure + dependency :identifier, Identifier::UUID::Random + dependency :put - dependency :identifier, Identifier::UUID::Random - - abstract :configure - abstract :write - end + def self.build(session: nil) + instance = new + Identifier::UUID::Random.configure(instance) + Put.configure(instance, session: session) + instance end - module Build - def build(session: nil) - instance = new - Identifier::UUID::Random.configure(instance) - instance.configure(session: session) - instance - end + def self.configure(receiver, session: nil, attr_name: nil) + attr_name ||= :write + instance = build(session: session) + receiver.public_send "#{attr_name}=", instance end - module Configure - def configure(receiver, session: nil, attr_name: nil) - attr_name ||= :write - instance = build(session: session) - receiver.public_send "#{attr_name}=", instance - end - end - - module Call - def call(message_data, stream_name, expected_version: nil, session: nil) - instance = build(session: session) - instance.(message_data, stream_name, expected_version: expected_version) - end + def self.call(message_data, stream_name, expected_version: nil, session: nil) + instance = build(session: session) + instance.(message_data, stream_name, expected_version: expected_version) end def call(message_data, stream_name, expected_version: nil) @@ -62,7 +46,45 @@ def call(message_data, stream_name, expected_version: nil) position end - alias :write :call + + def write(batch, stream_name, expected_version: nil) + logger.trace(tag: :write) do + message_types = batch.map {|message_data| message_data.type }.uniq.join(', ') + "Writing batch (Stream Name: #{stream_name}, Types: #{message_types}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})" + end + + unless expected_version.nil? + expected_version = ExpectedVersion.canonize(expected_version) + end + + last_position = nil + put.session.transaction do + batch.each do |message_data| + last_position = write_message_data(message_data, stream_name, expected_version: expected_version) + + unless expected_version.nil? + expected_version += 1 + end + end + end + + logger.debug(tag: :write) do + message_types = batch.map {|message_data| message_data.type }.uniq.join(', ') + "Wrote batch (Stream Name: #{stream_name}, Types: #{message_types}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})" + end + + last_position + end + + def write_message_data(message_data, stream_name, expected_version: nil) + logger.trace(tag: :write) { "Writing message data (Stream Name: #{stream_name}, Type: #{message_data.type}, Expected Version: #{expected_version.inspect})" } + logger.trace(tags: [:data, :message_data]) { message_data.pretty_inspect } + + put.(message_data, stream_name, expected_version: expected_version).tap do + logger.debug(tag: :write) { "Wrote message data (Stream Name: #{stream_name}, Type: #{message_data.type}, Expected Version: #{expected_version.inspect})" } + logger.debug(tags: [:data, :message_data]) { message_data.pretty_inspect } + end + end def set_ids(batch) batch.each do |message_data| diff --git a/message_store-postgres.gemspec b/message_store-postgres.gemspec deleted file mode 100644 index 6c6376a..0000000 --- a/message_store-postgres.gemspec +++ /dev/null @@ -1,31 +0,0 @@ -# -*- encoding: utf-8 -*- -Gem::Specification.new do |s| - s.name = 'evt-message_store-postgres' - s.version = '2.5.0.0' - s.summary = 'Message store implementation for PostgreSQL' - s.description = ' ' - - s.authors = ['The Eventide Project'] - s.email = 'opensource@eventide-project.org' - s.homepage = 'https://github.com/eventide-project/message-store-postgres' - s.licenses = ['MIT'] - - s.require_paths = ['lib'] - s.files = Dir.glob('{lib,database}/**/*') - s.platform = Gem::Platform::RUBY - s.required_ruby_version = '>= 2.4.0' - - s.executables = Dir.glob('scripts/evt-*').map(&File.method(:basename)) - s.bindir = 'scripts' - - s.add_runtime_dependency 'evt-message_store' - s.add_runtime_dependency 'evt-log' - s.add_runtime_dependency 'evt-settings' - - s.add_runtime_dependency 'message-db' - s.add_runtime_dependency 'pg' - - s.add_development_dependency 'test_bench' - s.add_development_dependency 'evt-diagnostics-sample' - s.add_development_dependency 'ntl-actor' -end diff --git a/message_store.gemspec b/message_store.gemspec index 0646371..09175a5 100644 --- a/message_store.gemspec +++ b/message_store.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |s| s.name = 'evt-message_store' s.version = '2.4.0.0' - s.summary = 'Common primitives for platform-specific message store implementations' + s.summary = 'Message store implementation for PostgreSQL' s.description = ' ' s.authors = ['The Eventide Project'] @@ -11,17 +11,26 @@ Gem::Specification.new do |s| s.licenses = ['MIT'] s.require_paths = ['lib'] - s.files = Dir.glob('{lib}/**/*') + s.files = Dir.glob('{lib,database}/**/*') s.platform = Gem::Platform::RUBY s.required_ruby_version = '>= 2.4.0' - s.add_runtime_dependency 'evt-casing' - s.add_runtime_dependency 'evt-schema' + s.executables = Dir.glob('scripts/evt-*').map(&File.method(:basename)) + s.bindir = 'scripts' + + s.add_runtime_dependency 'evt-log' + s.add_runtime_dependency 'evt-settings' s.add_runtime_dependency 'evt-initializer' s.add_runtime_dependency 'evt-identifier-uuid' s.add_runtime_dependency 'evt-transform' s.add_runtime_dependency 'evt-virtual' + s.add_runtime_dependency 'evt-casing' s.add_runtime_dependency 'evt-async_invocation' + s.add_runtime_dependency 'message-db' + s.add_runtime_dependency 'pg' + s.add_development_dependency 'test_bench' + s.add_development_dependency 'evt-diagnostics-sample' + s.add_development_dependency 'ntl-actor' end diff --git a/test/automated/get/category/specialized/condition/condition.rb b/test/automated/get/category/specialized/condition/condition.rb index b6664bc..7335174 100644 --- a/test/automated/get/category/specialized/condition/condition.rb +++ b/test/automated/get/category/specialized/condition/condition.rb @@ -10,7 +10,7 @@ condition = 'position = 0 OR position = 2' - settings = Postgres::Settings.build + settings = MessageStore::Settings.build session = Session.new settings.set(session) session.options = '-c message_store.sql_condition=on' diff --git a/test/automated/get/category/specialized/condition/not_activated_error.rb b/test/automated/get/category/specialized/condition/not_activated_error.rb index 41dd97b..1e6127a 100644 --- a/test/automated/get/category/specialized/condition/not_activated_error.rb +++ b/test/automated/get/category/specialized/condition/not_activated_error.rb @@ -9,7 +9,7 @@ condition = 'some condition' - settings = Postgres::Settings.build + settings = MessageStore::Settings.build session = Session.new settings.set(session) session.options = nil diff --git a/test/automated/get/category/specialized/consumer_groups/error.rb b/test/automated/get/category/specialized/consumer_groups/error.rb index 46ae2b2..5e15252 100644 --- a/test/automated/get/category/specialized/consumer_groups/error.rb +++ b/test/automated/get/category/specialized/consumer_groups/error.rb @@ -7,7 +7,7 @@ context "Error" do context "Consumer Group Size Is Less than 1" do test "Is an error" do - assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: 0, consumer_group_size: 0) end end @@ -15,7 +15,7 @@ context "Consumer Group Member Is Greater than the Consumer Group Size" do test "Is an error" do - assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: 2, consumer_group_size: 1) end end @@ -23,7 +23,7 @@ context "Consumer Group Member Is Less than 0" do test "Is an error" do - assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: -1, consumer_group_size: 1) end end @@ -31,7 +31,7 @@ context "Consumer Group Size is Missing" do test "Is an error" do - assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_member: 0) end end @@ -39,7 +39,7 @@ context "Consumer Group Member is Missing" do test "Is an error" do - assert_raises(MessageStore::Postgres::Get::Category::ConsumerGroup::Error) do + assert_raises(MessageStore::Get::Category::ConsumerGroup::Error) do Get::Category.('someCategory', consumer_group_size: 1) end end diff --git a/test/automated/get/category/specialized/correlation/error.rb b/test/automated/get/category/specialized/correlation/error.rb index 178adff..200f764 100644 --- a/test/automated/get/category/specialized/correlation/error.rb +++ b/test/automated/get/category/specialized/correlation/error.rb @@ -10,7 +10,7 @@ category = Controls::Category.example test "Is an error" do - assert_raises(MessageStore::Postgres::Get::Category::Correlation::Error) do + assert_raises(MessageStore::Get::Category::Correlation::Error) do Get::Category.(category, correlation: correlation) end end diff --git a/test/automated/get/stream/specialized/condition/condition.rb b/test/automated/get/stream/specialized/condition/condition.rb index e0f6be7..17630cb 100644 --- a/test/automated/get/stream/specialized/condition/condition.rb +++ b/test/automated/get/stream/specialized/condition/condition.rb @@ -8,7 +8,7 @@ condition = 'position = 0 OR position = 2' - settings = Postgres::Settings.build + settings = MessageStore::Settings.build session = Session.new settings.set(session) session.options = '-c message_store.sql_condition=on' diff --git a/test/automated/get/stream/specialized/condition/not_activated.rb b/test/automated/get/stream/specialized/condition/not_activated.rb index 0a34e4f..35f876b 100644 --- a/test/automated/get/stream/specialized/condition/not_activated.rb +++ b/test/automated/get/stream/specialized/condition/not_activated.rb @@ -9,7 +9,7 @@ condition = 'some condition' - settings = Postgres::Settings.build + settings = MessageStore::Settings.build session = Session.new settings.set(session) session.options = nil diff --git a/test/automated/get_last/call.rb b/test/automated/get_last/call.rb deleted file mode 100644 index 43cfcf9..0000000 --- a/test/automated/get_last/call.rb +++ /dev/null @@ -1,38 +0,0 @@ -require_relative '../automated_init' - -context "Get Last" do - context "Call" do - stream_name = Controls::StreamName.example - type = Controls::MessageData.type - - context "Not Specialized" do - cls = Class.new do - include MessageStore::Get::Stream::Last - end - - test "Raises virtual method error" do - assert_raises(Virtual::PureMethod::Error) do - cls.(stream_name) - end - end - end - - context "Specialized" do - specialized_method_executed = false - - cls = Class.new do - include MessageStore::Get::Stream::Last - - define_method(:call) do |_stream_name, type=nil| - specialized_method_executed = true if _stream_name == stream_name - end - end - - cls.(stream_name, type) - - test "Executes specialized method" do - assert(specialized_method_executed) - end - end - end -end diff --git a/test/automated/get_last/configure.rb b/test/automated/get_last/configure.rb index b943da5..fd0966c 100644 --- a/test/automated/get_last/configure.rb +++ b/test/automated/get_last/configure.rb @@ -7,7 +7,7 @@ receiver = OpenStruct.new - Controls::GetLast::Example.configure(receiver, session: session) + Get::Stream::Last.configure(receiver, session: session) get_last = receiver.get_last @@ -19,19 +19,19 @@ context "Session Not Given" do receiver = OpenStruct.new - Controls::GetLast::Example.configure(receiver) + Get::Stream::Last.configure(receiver) get_last = receiver.get_last test "Default session is used" do - assert(get_last.session == Controls::GetLast.default_session) + refute(get_last.session.nil?) end end context "Attribute Name Specified" do receiver = OpenStruct.new - Controls::GetLast::Example.configure(receiver, attr_name: :some_attr) + Get::Stream::Last.configure(receiver, attr_name: :some_attr) get_last = receiver.some_attr diff --git a/test/automated/get_last/substitute.rb b/test/automated/get_last/substitute.rb index 508a5f7..1eb598f 100644 --- a/test/automated/get_last/substitute.rb +++ b/test/automated/get_last/substitute.rb @@ -7,7 +7,7 @@ control_message_data = Controls::MessageData::Read.example context "Message Is Set" do - substitute = SubstAttr::Substitute.build(Controls::GetLast::Example) + substitute = SubstAttr::Substitute.build(Get::Stream::Last) substitute.set(stream_name, control_message_data) @@ -19,7 +19,7 @@ end context "Message Not Set" do - substitute = SubstAttr::Substitute.build(Controls::GetLast::Example) + substitute = SubstAttr::Substitute.build(Get::Stream::Last) message_data = substitute.(stream_name) diff --git a/test/automated/iterator_tests.rb b/test/automated/iterator_tests.rb deleted file mode 100644 index af5211b..0000000 --- a/test/automated/iterator_tests.rb +++ /dev/null @@ -1,6 +0,0 @@ -require_relative './automated_init' - -TestBench::CLI::Run.( - 'test/automated/iterator', - exclude_file_pattern: %r{/_|sketch|(_init\.rb|_tests\.rb)\z} -) diff --git a/test/automated/read/condition.rb b/test/automated/read/condition.rb index 9113e67..a3ecf75 100644 --- a/test/automated/read/condition.rb +++ b/test/automated/read/condition.rb @@ -8,7 +8,7 @@ message_count = 0 - settings = Postgres::Settings.build + settings = MessageStore::Settings.build session = Session.new settings.set(session) session.options = '-c message_store.sql_condition=on' diff --git a/test/automated/read/default_batch_size.rb b/test/automated/read/default_batch_size.rb index 821299f..507372a 100644 --- a/test/automated/read/default_batch_size.rb +++ b/test/automated/read/default_batch_size.rb @@ -2,10 +2,10 @@ context "Read" do context "Default batch Size" do - default_batch_size = MessageStore::Postgres::Read::Defaults.batch_size + default_batch_size = MessageStore::Read::Defaults.batch_size test "Is the Get implementation's default batch size" do - assert(default_batch_size == MessageStore::Postgres::Get::Defaults.batch_size) + assert(default_batch_size == MessageStore::Get::Defaults.batch_size) end end end diff --git a/test/automated/session/build.rb b/test/automated/session/build.rb index b9e837e..6f3e3b8 100644 --- a/test/automated/session/build.rb +++ b/test/automated/session/build.rb @@ -3,7 +3,7 @@ context "Session" do context "Build" do context "Settings is specified" do - settings = MessageStore::Postgres::Settings.build + settings = MessageStore::Settings.build session = Session.build(settings: settings) @@ -13,7 +13,7 @@ end context "Settings is not specified" do - settings = MessageStore::Postgres::Settings.build + settings = MessageStore::Settings.build session = Session.build diff --git a/test/automated/session/configure.rb b/test/automated/session/configure.rb index 1e748cf..12927b2 100644 --- a/test/automated/session/configure.rb +++ b/test/automated/session/configure.rb @@ -23,7 +23,7 @@ user: user } - settings = MessageStore::Postgres::Settings.build(settings_data) + settings = MessageStore::Settings.build(settings_data) Session.configure(receiver, settings: settings) diff --git a/test/automated/session/settings.rb b/test/automated/session/settings.rb index deb2fde..39eba74 100644 --- a/test/automated/session/settings.rb +++ b/test/automated/session/settings.rb @@ -4,10 +4,10 @@ context "Settings" do session = Session.build - settings = Postgres::Settings.build + settings = MessageStore::Settings.build settings_hash = settings.get.to_h - names = Postgres::Settings.names + names = MessageStore::Settings.names names.each do |name| test "#{name}" do diff --git a/test/automated/settings.rb b/test/automated/settings.rb index fa0a521..b7824df 100644 --- a/test/automated/settings.rb +++ b/test/automated/settings.rb @@ -1,12 +1,12 @@ require_relative 'automated_init' context "Settings" do - settings = Postgres::Settings.build + settings = MessageStore::Settings.build context "Names" do settings_hash = settings.get.to_h - names = Postgres::Settings.names + names = MessageStore::Settings.names names.each do |name| test "#{name}" do diff --git a/test/automated/settings/default_path.rb b/test/automated/settings/default_path.rb index 46ada36..9d282b1 100644 --- a/test/automated/settings/default_path.rb +++ b/test/automated/settings/default_path.rb @@ -7,7 +7,7 @@ overridden_path = 'some_path' ENV['MESSAGE_STORE_SETTINGS_PATH'] = overridden_path - settings_path = Postgres::Settings.data_source + settings_path = MessageStore::Settings.data_source test "Overridden by MESSAGE_STORE_SETTINGS_PATH environment variable" do assert(settings_path == overridden_path) diff --git a/test/automated/write/ids_are_assigned.rb b/test/automated/write/ids_are_assigned.rb index d75a3d2..f3893bb 100644 --- a/test/automated/write/ids_are_assigned.rb +++ b/test/automated/write/ids_are_assigned.rb @@ -11,7 +11,10 @@ batch = [write_message_1, write_message_2, write_message_3] - Controls::Write::Example.(batch, stream_name) + ## Review - Aaron, Sat Jan 21 2023 + write = Write.new + write.put = Put.new + write.(batch, stream_name) context "Missing IDs Are Assigned" do test "Message 1" do diff --git a/test/interactive/concurrency/write_to_single_stream.rb b/test/interactive/concurrency/write_to_single_stream.rb index 5f91f08..4ec3514 100644 --- a/test/interactive/concurrency/write_to_single_stream.rb +++ b/test/interactive/concurrency/write_to_single_stream.rb @@ -20,11 +20,11 @@ def initialize(stream_name) end handle :write_message do - message_data_1 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { actor: object_id }) - message_data_2 = MessageStore::Postgres::Controls::MessageData::Write.example(data: { actor: object_id }) + message_data_1 = MessageStore::Controls::MessageData::Write.example(data: { actor: object_id }) + message_data_2 = MessageStore::Controls::MessageData::Write.example(data: { actor: object_id }) batch = [message_data_1, message_data_2] - position = MessageStore::Postgres::Write.(batch, stream_name) + position = MessageStore::Write.(batch, stream_name) logger.info(tag: :actor) { "Wrote message data (Object ID: #{object_id}, Position: #{position}, Message Type: #{message_data_1.type.inspect}, Stream Name: #{stream_name.inspect})" } diff --git a/test/test_init.rb b/test/test_init.rb index a895fe5..55634d1 100644 --- a/test/test_init.rb +++ b/test/test_init.rb @@ -1,21 +1,11 @@ ENV['CONSOLE_DEVICE'] ||= 'stdout' -ENV['LOG_COLOR'] ||= 'on' - -if ENV['LOG_LEVEL'] - ENV['LOGGER'] ||= 'on' -else - ENV['LOG_LEVEL'] ||= 'trace' -end - -ENV['LOGGER'] ||= 'off' -ENV['LOG_OPTIONAL'] ||= 'on' +ENV['LOG_LEVEL'] ||= '_min' puts RUBY_DESCRIPTION -require_relative '../init.rb' +require_relative '../init' +require 'message_store/controls' require 'test_bench'; TestBench.activate -require 'message_store/controls' - include MessageStore diff --git a/tools/write_message.rb b/tools/write_message.rb index 4e3ce48..0c7d6ca 100755 --- a/tools/write_message.rb +++ b/tools/write_message.rb @@ -7,10 +7,9 @@ require_relative '../init' -require 'message_store/postgres/controls' +require 'message_store/controls' include MessageStore -include MessageStore::Postgres instances = Integer(ENV['INSTANCES'] || 1) stream_name = ENV['STREAM_NAME']