From fe0a6849b881d781ce17b5a8648b78c23cc96775 Mon Sep 17 00:00:00 2001 From: Shigeki Morimoto Date: Sat, 25 Jan 2014 15:27:08 +0900 Subject: [PATCH 01/13] Add ar:create task for padrino. --- lib/active_record/turntable.rb | 1 + lib/active_record/turntable/padrinotie.rb | 17 ++ .../turntable/padrinoties/databases.rb | 204 ++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100644 lib/active_record/turntable/padrinotie.rb create mode 100644 lib/active_record/turntable/padrinoties/databases.rb diff --git a/lib/active_record/turntable.rb b/lib/active_record/turntable.rb index 0d3814e1..e305fd92 100644 --- a/lib/active_record/turntable.rb +++ b/lib/active_record/turntable.rb @@ -69,4 +69,5 @@ def turntable_config end require "active_record/turntable/railtie" if defined?(Rails) + require "active_record/turntable/padrinotie" if defined?(Padrino) end diff --git a/lib/active_record/turntable/padrinotie.rb b/lib/active_record/turntable/padrinotie.rb new file mode 100644 index 00000000..f6a7b548 --- /dev/null +++ b/lib/active_record/turntable/padrinotie.rb @@ -0,0 +1,17 @@ +module ActiveRecord::Turntable + class Padrinotie + Padrino::Tasks.files << File.dirname(__FILE__) + "/padrinoties/databases.rb" + + # # rails loading hook + # ActiveSupport.on_load(:before_initialize) do + # ActiveSupport.on_load(:active_record) do + # ActiveRecord::Base.send(:include, ActiveRecord::Turntable) + # end + # end + + # # Swap QueryCache Middleware + # initializer "turntable.swap_query_cache_middleware" do |app| + # app.middleware.swap ActiveRecord::QueryCache, ActiveRecord::Turntable::Rack::QueryCache + # end + end +end diff --git a/lib/active_record/turntable/padrinoties/databases.rb b/lib/active_record/turntable/padrinoties/databases.rb new file mode 100644 index 00000000..d8b56fb8 --- /dev/null +++ b/lib/active_record/turntable/padrinoties/databases.rb @@ -0,0 +1,204 @@ +require 'active_record/turntable' +ActiveRecord::SchemaDumper.send(:include, ActiveRecord::Turntable::ActiveRecordExt::SchemaDumper) + +db_namespace = namespace :ar do + task :create do + if Padrino.env == "test" && ActiveRecord::Base.configurations["test"]["shards"] + dbs = ActiveRecord::Base.configurations["test"]["shards"].values + dbs += ActiveRecord::Base.configurations["test"]["seq"].values if ActiveRecord::Base.configurations["test"]["seq"] + dbs.each do |shard_config| + create_database(shard_config) + end + end + if shard_configs = ActiveRecord::Base.configurations[Padrino.env || 'development']["shards"] + dbs = shard_configs.values + dbs += ActiveRecord::Base.configurations[Padrino.env || 'development']["seq"].values if ActiveRecord::Base.configurations[Padrino.env || 'development']["seq"] + dbs.each do |shard_config| + create_database(shard_config) + end + end + config = ActiveRecord::Base.configurations[Padrino.env || 'development'] + ActiveRecord::Base.establish_connection(config) + end + + task :drop do + config = ActiveRecord::Base.configurations[Padrino.env || 'development'] + shard_configs = config["shards"] + if shard_configs + dbs = shard_configs.values + dbs += ActiveRecord::Base.configurations[Padrino.env || 'development']["seq"].values if ActiveRecord::Base.configurations[Padrino.env || 'development']["seq"] + dbs.each do |shard_config| + begin + drop_database(shard_config) + rescue Exception => e + $stderr.puts "Couldn't drop #{ config['database']} : #{e.inspect}" + end + end + end + ActiveRecord::Base.establish_connection(config) + end + + namespace :schema do + task :dump do + require 'active_record/schema_dumper' + config = ActiveRecord::Base.configurations[Padrino.env] + shard_configs = config["shards"] + shard_configs.merge!(config["seq"]) if config["seq"] + if shard_configs + shard_configs.each do |name, config| + filename = ENV['SCHEMA'] || Padrino.root("db", "schema-#{name}.rb") + File.open(filename, "w:utf-8") do |file| + ActiveRecord::Base.establish_connection(config) + ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, file) + end + end + end + ActiveRecord::Base.establish_connection(config) + db_namespace['schema:dump'].reenable + end + + desc 'Load a schema.rb file into the database' + task :load => :environment do + config = ActiveRecord::Base.configurations[Padrino.env] + shard_configs = config["shards"] + shard_configs.merge!(config["seq"]) if config["seq"] + if shard_configs + shard_configs.each do |name, config| + ActiveRecord::Base.establish_connection(config) + file = ENV['SCHEMA'] || Padrino.root("db", "schema-#{name}.rb") + if File.exists?(file) + load(file) + else + abort %{#{file} doesn't exist yet. Run "rake db:migrate" to create it then try again. If you do not intend to use a database, you should instead alter #{Padrino.root}/config/application.rb to limit the frameworks that will be loaded'} + end + end + end + ActiveRecord::Base.establish_connection(config) + end + end + + namespace :structure do + desc 'Dump the database structure to an SQL file' + task :dump => :environment do + config = ActiveRecord::Base.configurations[Padrino.env] + shard_configs = config["shards"] + shard_configs.merge!(config["seq"]) if config["seq"] + if shard_configs + shard_configs.each do |name, config| + case config['adapter'] + when /mysql/, 'oci', 'oracle' + ActiveRecord::Base.establish_connection(config) + File.open(Padrino.root("db", "#{Padrino.env}_#{name}_structure.sql"), "w+") { |f| f << ActiveRecord::Base.connection.structure_dump } + when /postgresql/ + ENV['PGHOST'] = config['host'] if config['host'] + ENV['PGPORT'] = config["port"].to_s if config['port'] + ENV['PGPASSWORD'] = config['password'].to_s if config['password'] + search_path = config['schema_search_path'] + unless search_path.blank? + search_path = search_path.split(",").map{|search_path| "--schema=#{search_path.strip}" }.join(" ") + end + `pg_dump -i -U "#{config['username']}" -s -x -O -f #{Padrino.root("db", "#{Padrino.env}_#{name}_structure.sql")} #{search_path} #{config['database']}` + raise 'Error dumping database' if $?.exitstatus == 1 + when /sqlite/ + dbfile = config['database'] || config['dbfile'] + `sqlite3 #{dbfile} .schema > db/#{Padrino.env}_#{name}_structure.sql` + when 'sqlserver' + `smoscript -s #{config['host']} -d #{config['database']} -u #{config['username']} -p #{config['password']} -f #{Padrino.root("db", "#{Padrino.env}_#{name}_structure.sql")} -A -U` + when "firebird" + set_firebird_env(config) + db_string = firebird_db_string(config) + sh "isql -a #{db_string} > #{Padrino.root("db", "#{Padrino.env}_#{name}_structure.sql")}" + else + raise "Task not supported by '#{config["adapter"]}'" + end + + if ActiveRecord::Base.connection.supports_migrations? + File.open(Padrino.root("db", "#{Padrino.env}_#{name}_structure.sql"), "a") { |f| f << ActiveRecord::Base.connection.dump_schema_information } + end + end + end + ActiveRecord::Base.establish_connection(config) + end + end + + + namespace :test do + # desc "Recreate the test databases from the development structure" + task :clone_structure do + config = ActiveRecord::Base.configurations[Padrino.env] + shard_configs = config["shards"] + shard_configs.merge!(config["seq"]) if config["seq"] + if shard_configs + shard_configs.each do |name, config| + case config['adapter'] + when /mysql/ + ActiveRecord::Base.establish_connection(config) + ActiveRecord::Base.connection.execute('SET foreign_key_checks = 0') + IO.readlines(Padrino.root("db", "#{Padrino.env}_#{name}_structure.sql")).join.split("\n\n").each do |table| + ActiveRecord::Base.connection.execute(table) + end + when /postgresql/ + ENV['PGHOST'] = config['host'] if config['host'] + ENV['PGPORT'] = config['port'].to_s if config['port'] + ENV['PGPASSWORD'] = config['password'].to_s if config['password'] + `psql -U "#{config['username']}" -f "#{Padrino.root("db","#{Padrino.env}#{name}_structure.sql")} #{config['database']} #{config['template']}` + when /sqlite/ + dbfile = config['database'] || config['dbfile'] + `sqlite3 #{dbfile} < "#{Padrino.root("db", "#{Padrino.env}#{name}_structure.sql")}` + when 'sqlserver' + `sqlcmd -S #{config['host']} -d #{config['database']} -U #{config['username']} -P #{config['password']} -i #{Padrino.root("db", "#{Padrino.env}#{name}_structure.sql")}` + when 'oci', 'oracle' + ActiveRecord::Base.establish_connection(config) + IO.readlines(Padrino.root("db", "#{Padrino.env}#{name}_structure.sql")).join.split(";\n\n").each do |ddl| + ActiveRecord::Base.connection.execute(ddl) + end + when 'firebird' + set_firebird_env(config) + db_string = firebird_db_string(config) + sh "isql -i #{Padrino.root("db","#{Padrino.env}#{name}_structure.sql")} #{db_string}" + else + raise "Task not supported by '#{config['adapter']}'" + end + end + end + ActiveRecord::Base.establish_connection(config) + end + + # desc "Empty the test database" + task :purge => :environment do + config = ActiveRecord::Base.configurations[Padrino.env] + shard_configs = config["shards"] + shard_configs.merge!(config["seq"]) if config["seq"] + if shard_configs + shard_configs.each do |name, config| + case config['adapter'] + when /mysql/ + ActiveRecord::Base.establish_connection(config) + ActiveRecord::Base.connection.recreate_database(config['database'], mysql_creation_options(config)) + when /postgresql/ + ActiveRecord::Base.clear_active_connections! + drop_database(config) + create_database(config) + when /sqlite/ + dbfile = config['database'] || config['dbfile'] + File.delete(dbfile) if File.exist?(dbfile) + when 'sqlserver' + # TODO + when "oci", "oracle" + ActiveRecord::Base.establish_connection(config) + ActiveRecord::Base.connection.structure_drop.split(";\n\n").each do |ddl| + ActiveRecord::Base.connection.execute(ddl) + end + when 'firebird' + ActiveRecord::Base.establish_connection(config) + ActiveRecord::Base.connection.recreate_database! + else + raise "Task not supported by '#{config['adapter']}'" + end + end + end + ActiveRecord::Base.establish_connection(config) + end + end +end + From 261a078a2070504e6012fc8ed5b69beb0b782265 Mon Sep 17 00:00:00 2001 From: Shigeki Morimoto Date: Sat, 25 Jan 2014 18:07:15 +0900 Subject: [PATCH 02/13] add generator for turntable with padrino. --- lib/active_record/turntable/padrinotie.rb | 13 +++++---- .../padrino/turntable/install_generators.rb | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 lib/generators/padrino/turntable/install_generators.rb diff --git a/lib/active_record/turntable/padrinotie.rb b/lib/active_record/turntable/padrinotie.rb index f6a7b548..8a623196 100644 --- a/lib/active_record/turntable/padrinotie.rb +++ b/lib/active_record/turntable/padrinotie.rb @@ -1,14 +1,15 @@ +# -*- coding: utf-8 -*- module ActiveRecord::Turntable class Padrinotie Padrino::Tasks.files << File.dirname(__FILE__) + "/padrinoties/databases.rb" - # # rails loading hook - # ActiveSupport.on_load(:before_initialize) do - # ActiveSupport.on_load(:active_record) do - # ActiveRecord::Base.send(:include, ActiveRecord::Turntable) - # end - # end + # padrino loading hook + Padrino.before_load do + ActiveRecord::Base.send(:include, ActiveRecord::Turntable) + Padrino::Generators.load_components! + require 'generators/padrino/turntable/install_generators' + end # # Swap QueryCache Middleware # initializer "turntable.swap_query_cache_middleware" do |app| # app.middleware.swap ActiveRecord::QueryCache, ActiveRecord::Turntable::Rack::QueryCache diff --git a/lib/generators/padrino/turntable/install_generators.rb b/lib/generators/padrino/turntable/install_generators.rb new file mode 100644 index 00000000..b132da68 --- /dev/null +++ b/lib/generators/padrino/turntable/install_generators.rb @@ -0,0 +1,29 @@ +require 'padrino-gen' + +module Padrino + module Generators + class Turntable < Thor::Group + Padrino::Generators.add_generator(:turntable, self) + + def self.source_root + File.expand_path("../../../templates", __FILE__) + end + + desc "Creates turntable configuration file (config/turntable.yml)" + + include Thor::Actions + include Padrino::Generators::Actions + include Padrino::Generators::Components::Actions + + desc "Creates turntable configuration file (config/turntable.yml)" + + def self.require_arguments? + false + end + + def create_turntable + copy_file "turntable.yml", "config/turntable.yml" + end + end + end +end From ee75ea08f27fac4155b697f7f6ac9795346bfdb3 Mon Sep 17 00:00:00 2001 From: Shigeki Morimoto Date: Sat, 25 Jan 2014 19:00:22 +0900 Subject: [PATCH 03/13] Add ActiveRecord::Turntable::RackupFramework to abstract rails, padrino or others. --- lib/active_record/turntable.rb | 7 +++++-- lib/active_record/turntable/base.rb | 1 - lib/active_record/turntable/cluster_helper_methods.rb | 2 +- lib/active_record/turntable/config.rb | 2 +- lib/active_record/turntable/padrinotie.rb | 1 - lib/active_record/turntable/seq_shard.rb | 2 +- lib/active_record/turntable/shard.rb | 2 +- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/active_record/turntable.rb b/lib/active_record/turntable.rb index e305fd92..146ba450 100644 --- a/lib/active_record/turntable.rb +++ b/lib/active_record/turntable.rb @@ -50,13 +50,16 @@ module ActiveRecord::Turntable include Base end + RackupFramework = Rails if defined?(Rails); + RackupFramework = Padrino if defined?(Padrino); + module ClassMethods DEFAULT_PATH = File.dirname(File.dirname(__FILE__)) def turntable_config_file @@turntable_config_file ||= - File.join(defined?(::Rails) ? - ::Rails.root.to_s : DEFAULT_PATH, 'config/turntable.yml') + File.join(defined?(ActiveRecord::Turntable::RackupFramework) ? + ActiveRecord::Turntable::RackupFramework.root.to_s : DEFAULT_PATH, 'config/turntable.yml') end def turntable_config_file=(filename) diff --git a/lib/active_record/turntable/base.rb b/lib/active_record/turntable/base.rb index ad66e17c..1db3dce8 100644 --- a/lib/active_record/turntable/base.rb +++ b/lib/active_record/turntable/base.rb @@ -41,7 +41,6 @@ def turntable(cluster_name, shard_key_name, options = {}) turntable_replace_connection_pool end - def turntable_replace_connection_pool ch = connection_handler cp = ConnectionProxy.new(self, turntable_cluster) diff --git a/lib/active_record/turntable/cluster_helper_methods.rb b/lib/active_record/turntable/cluster_helper_methods.rb index 5d6680fc..07243b0c 100644 --- a/lib/active_record/turntable/cluster_helper_methods.rb +++ b/lib/active_record/turntable/cluster_helper_methods.rb @@ -30,7 +30,7 @@ def recursive_transaction(pools, options, &block) end def force_connect_all_shards! - conf = configurations[Rails.env] + conf = configurations[ActiveRecord::Turntable::RackupFramework.env] shards = {} shards = shards.merge(conf["shards"]) if conf["shards"] shards = shards.merge(conf["seq"]) if conf["seq"] diff --git a/lib/active_record/turntable/config.rb b/lib/active_record/turntable/config.rb index 9910ddaf..19d5ed5c 100644 --- a/lib/active_record/turntable/config.rb +++ b/lib/active_record/turntable/config.rb @@ -14,7 +14,7 @@ def [](key) @config[key] end - def self.load!(config_file = ActiveRecord::Base.turntable_config_file, env = (defined?(Rails) ? Rails.env : 'development')) + def self.load!(config_file = ActiveRecord::Base.turntable_config_file, env = (defined?(ActiveRecord::Turntable::RackupFramework) ? ActiveRecord::Turntable::RackupFramework.env : 'development')) # FIXME instance.load!(config_file, env) end diff --git a/lib/active_record/turntable/padrinotie.rb b/lib/active_record/turntable/padrinotie.rb index 8a623196..208be85a 100644 --- a/lib/active_record/turntable/padrinotie.rb +++ b/lib/active_record/turntable/padrinotie.rb @@ -7,7 +7,6 @@ class Padrinotie Padrino.before_load do ActiveRecord::Base.send(:include, ActiveRecord::Turntable) - Padrino::Generators.load_components! require 'generators/padrino/turntable/install_generators' end # # Swap QueryCache Middleware diff --git a/lib/active_record/turntable/seq_shard.rb b/lib/active_record/turntable/seq_shard.rb index 00ac1616..2768ba00 100644 --- a/lib/active_record/turntable/seq_shard.rb +++ b/lib/active_record/turntable/seq_shard.rb @@ -12,7 +12,7 @@ def create_connection_class def retrieve_connection_pool ActiveRecord::Base.turntable_connections[name] ||= begin - config = ActiveRecord::Base.configurations[Rails.env]["seq"][name] + config = ActiveRecord::Base.configurations[ActiveRecord::Turntable::RackupFramework.env]["seq"][name] raise ArgumentError, "Unknown database config: #{name}, have #{ActiveRecord::Base.configurations.inspect}" unless config ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec_for(config)) end diff --git a/lib/active_record/turntable/shard.rb b/lib/active_record/turntable/shard.rb index f1b3db81..ea65d056 100644 --- a/lib/active_record/turntable/shard.rb +++ b/lib/active_record/turntable/shard.rb @@ -3,7 +3,7 @@ class Shard module Connections; end DEFAULT_CONFIG = { - "connection" => (defined?(Rails) ? Rails.env : "development") + "connection" => (defined?(ActiveRecord::Turntable::RackupFramework) ? ActiveRecord::Turntable::RackupFramework.env : "development") }.with_indifferent_access attr_reader :name From 4694191d304869c66af9062f55144a944011e7bd Mon Sep 17 00:00:00 2001 From: Shigeki Morimoto Date: Sat, 25 Jan 2014 19:00:40 +0900 Subject: [PATCH 04/13] fix generator --- lib/generators/padrino/turntable/install_generators.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/generators/padrino/turntable/install_generators.rb b/lib/generators/padrino/turntable/install_generators.rb index b132da68..fbe8927c 100644 --- a/lib/generators/padrino/turntable/install_generators.rb +++ b/lib/generators/padrino/turntable/install_generators.rb @@ -1,7 +1,9 @@ -require 'padrino-gen' +require 'thor/group' module Padrino module Generators + Padrino::Generators.load_components! + class Turntable < Thor::Group Padrino::Generators.add_generator(:turntable, self) From bee9980ad9e4409e29cdf2fa2b5c99b97f75a2fc Mon Sep 17 00:00:00 2001 From: MATSUBARA Nobutada Date: Fri, 6 Nov 2020 22:56:57 +0900 Subject: [PATCH 05/13] Fix: require active_record_ext/database_tasks on migration --- lib/active_record/turntable/migration.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/active_record/turntable/migration.rb b/lib/active_record/turntable/migration.rb index 76c4ac98..57bacc47 100644 --- a/lib/active_record/turntable/migration.rb +++ b/lib/active_record/turntable/migration.rb @@ -1,3 +1,5 @@ +require 'active_record/turntable/active_record_ext/database_tasks' + module ActiveRecord::Turntable::Migration extend ActiveSupport::Concern From 6172acff53e5b605e966e9db9f5ab39b8603f6e7 Mon Sep 17 00:00:00 2001 From: MATSUBARA Nobutada Date: Fri, 6 Nov 2020 22:57:41 +0900 Subject: [PATCH 06/13] Fix: DatabaseTasks.env is RackupFramework.env --- .../turntable/active_record_ext/database_tasks.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/active_record/turntable/active_record_ext/database_tasks.rb b/lib/active_record/turntable/active_record_ext/database_tasks.rb index dff045c4..c852adeb 100644 --- a/lib/active_record/turntable/active_record_ext/database_tasks.rb +++ b/lib/active_record/turntable/active_record_ext/database_tasks.rb @@ -3,6 +3,10 @@ module ActiveRecord module Tasks module DatabaseTasks + def env + @env ||= ActiveRecord::Turntable::RackupFramework.env + end + def create_all_turntable_cluster each_local_turntable_cluster_configuration { |name, configuration| puts "[turntable] *** executing to database: #{configuration['database']}" From 9694b3b9014e918c5a8b2b8d9f8599951d75d3c7 Mon Sep 17 00:00:00 2001 From: MATSUBARA Nobutada Date: Mon, 9 Nov 2020 22:37:32 +0900 Subject: [PATCH 07/13] Fix: target_shard? for cache_schema and no cluster table --- lib/active_record/turntable/migration.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/active_record/turntable/migration.rb b/lib/active_record/turntable/migration.rb index 57bacc47..762974b6 100644 --- a/lib/active_record/turntable/migration.rb +++ b/lib/active_record/turntable/migration.rb @@ -37,7 +37,8 @@ def shards(*connection_names) end def target_shard?(shard_name) - target_shards.blank? or target_shards.include?(shard_name) + return false if shard_name.present? && target_shards.blank? + shard_name.nil? or target_shards.blank? or target_shards.include?(shard_name) end def announce_with_turntable(message) From 25fc12ddaf690384f66febdfa11773e7680a3a60 Mon Sep 17 00:00:00 2001 From: Masahiro Tokioka Date: Mon, 4 Jan 2016 14:18:31 +0900 Subject: [PATCH 08/13] add offset option for next_sequence_value --- lib/active_record/turntable/sequencer/mysql.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/active_record/turntable/sequencer/mysql.rb b/lib/active_record/turntable/sequencer/mysql.rb index 82d0db3b..784f9b4c 100644 --- a/lib/active_record/turntable/sequencer/mysql.rb +++ b/lib/active_record/turntable/sequencer/mysql.rb @@ -11,9 +11,10 @@ def initialize(klass, options = {}) @options = options end - def next_sequence_value(sequence_name) + # mysql だけ offset を指定できるようにします + def next_sequence_value(sequence_name, offset = 1) conn = @klass.connection.seq.connection - conn.execute "UPDATE #{@klass.connection.quote_table_name(sequence_name)} SET id=LAST_INSERT_ID(id+1)" + conn.execute "UPDATE #{@klass.connection.quote_table_name(sequence_name)} SET id=LAST_INSERT_ID(id+#{offset})" res = conn.execute("SELECT LAST_INSERT_ID()") new_id = res.first.first.to_i raise SequenceNotFoundError if new_id.zero? From 5e90d11ae13305ea289f3e94367aad8e5cd56d23 Mon Sep 17 00:00:00 2001 From: MATSUBARA Nobutada Date: Fri, 13 Nov 2020 18:21:15 +0900 Subject: [PATCH 09/13] Fix: parse identifier with backquote --- lib/active_record/turntable/sql_tree_patch.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/active_record/turntable/sql_tree_patch.rb b/lib/active_record/turntable/sql_tree_patch.rb index c1754b67..eca8bddd 100644 --- a/lib/active_record/turntable/sql_tree_patch.rb +++ b/lib/active_record/turntable/sql_tree_patch.rb @@ -52,6 +52,7 @@ def each_token(&block) # :yields: SQLTree::Token when /\w/; tokenize_keyword(&block) when OPERATOR_CHARS; tokenize_operator(&block) when SQLTree.identifier_quote_char; tokenize_quoted_identifier(&block) + when SQLTree.identifier_quote_field_char; tokenize_identifier_with_quote(&block) end end @@ -73,6 +74,14 @@ def tokenize_possible_escaped_string(&block) tokenize_keyword(&block) end end + + def tokenize_identifier_with_quote(&block) + next_char # skip identifier_quote_field_char + literal = current_char + literal << next_char while SQLTree.identifier_quote_field_char != peek_char + next_char # skip identifier_quote_field_char + handle_token(SQLTree::Token::Identifier.new(literal), &block) + end end module SQLTree::Node From 3ce8d3648aa3cf5794b2c98c2160b743f431a7d4 Mon Sep 17 00:00:00 2001 From: MATSUBARA Nobutada Date: Tue, 24 Nov 2020 18:38:39 +0900 Subject: [PATCH 10/13] Fix order query in with_all (hanabokuro/activerecord-turntable#1) --- lib/active_record/turntable/mixer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/active_record/turntable/mixer.rb b/lib/active_record/turntable/mixer.rb index 069ff3ac..0c206852 100644 --- a/lib/active_record/turntable/mixer.rb +++ b/lib/active_record/turntable/mixer.rb @@ -144,7 +144,7 @@ def build_select_fader(tree, method, query, *args, &block) build_shards_with_same_query(@proxy.shards.values, query), method, query, *args, &block ) - elsif tree.group_by or tree.order_by or tree.limit.try(:value).to_i > 0 + elsif @proxy.current_shard.blank? and (tree.group_by or tree.order_by or tree.limit.try(:value).to_i > 0) raise CannotSpecifyShardError, "cannot specify shard for query: #{tree.to_sql}" elsif shard_keys.present? if SQLTree::Node::SelectDeclaration === tree.select.first and From 0b325269db380a82178877f6b1db69748ec7ec8e Mon Sep 17 00:00:00 2001 From: MATSUBARA Nobutada Date: Thu, 12 Nov 2020 20:06:47 +0900 Subject: [PATCH 11/13] revert migration structure to stable1 --- lib/active_record/turntable/migration.rb | 68 +++++++++++++++--------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/lib/active_record/turntable/migration.rb b/lib/active_record/turntable/migration.rb index 762974b6..2902f9fd 100644 --- a/lib/active_record/turntable/migration.rb +++ b/lib/active_record/turntable/migration.rb @@ -1,12 +1,11 @@ -require 'active_record/turntable/active_record_ext/database_tasks' - module ActiveRecord::Turntable::Migration extend ActiveSupport::Concern included do extend ShardDefinition - class_attribute :target_shards, :current_shard + class_attribute :target_shards, :current_shard, :target_seqs alias_method_chain :announce, :turntable + alias_method_chain :migrate, :turntable alias_method_chain :exec_migration, :turntable ::ActiveRecord::ConnectionAdapters::AbstractAdapter.send(:include, SchemaStatementsExt) ::ActiveRecord::Migration::CommandRecorder.send(:include, CommandRecorder) @@ -15,19 +14,17 @@ module ActiveRecord::Turntable::Migration module ShardDefinition def clusters(*cluster_names) - config = ActiveRecord::Base.turntable_config + config = ActiveRecord::Base.turntable_config["clusters"] + cluster_names = config.keys if cluster_names.first == :all (self.target_shards ||= []).concat( - if cluster_names.first == :all - config['clusters'].map do |name, cluster_conf| - cluster_conf["shards"].map {|shard| shard["connection"]} - end - else cluster_names.map do |cluster_name| - config['clusters'][cluster_name]["shards"].map do |shard| - shard["connection"] - end + config[cluster_name]["shards"].map { |shard| shard["connection"] } + end.flatten + ) + (self.target_seqs ||= []).concat( + cluster_names.map do |cluster_name| + config[cluster_name]["seq"].values.map { |seq| seq["connection"] } end.flatten - end ) end @@ -46,7 +43,26 @@ def announce_with_turntable(message) end def exec_migration_with_turntable(*args) - exec_migration_without_turntable(*args) if target_shard?(current_shard) + exec_migration_without_turntable(*args) # if target_shard?(current_shard) + end + + def migrate_with_turntable(*args) + return migrate_without_turntable(*args) if target_shards.blank? + + config = ActiveRecord::Base.configurations[ActiveRecord::Turntable::RackupFramework.env||"development"] + shard_conf = target_shards.map { |shard| [shard, config["shards"][shard]] }.to_h + seqs_conf = target_seqs.map { |seq| [seq, config["seq"][seq]] }.to_h + + # SHOW FULL FIELDS FROM `users` を実行してテーブルの情報を取得するためにデフォルトのデータベースも追加する + {"master": config}.merge(shard_conf).merge(seqs_conf).each do |connection_name, database_config| + next if database_config["database"].blank? + ActiveRecord::Base.clear_active_connections! + ActiveRecord::Base.establish_connection(database_config) + ActiveRecord::Migration.current_shard = connection_name + migrate_without_turntable(*args) + end + ActiveRecord::Base.establish_connection config + ActiveRecord::Base.clear_active_connections! end module SchemaStatementsExt @@ -117,28 +133,28 @@ module ClassMethods def up_with_turntable(migrations_paths, target_version = nil) up_without_turntable(migrations_paths, target_version) - ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration| - puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})" - _original_up(migrations_paths, target_version) - end + # ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration| + # puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})" + # _original_up(migrations_paths, target_version) + # end end def down_with_turntable(migrations_paths, target_version = nil, &block) down_without_turntable(migrations_paths, target_version, &block) - ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration| - puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})" - _original_down(migrations_paths, target_version, &block) - end + # ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration| + # puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})" + # _original_down(migrations_paths, target_version, &block) + # end end def run_with_turntable(*args) run_without_turntable(*args) - ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration| - puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})" - _original_run(*args) - end + # ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration| + # puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})" + # _original_run(*args) + # end end end end From 4cc8b6e6af49024c122012f4aceabdc8b18b9c86 Mon Sep 17 00:00:00 2001 From: "qi.wang" Date: Tue, 20 Aug 2019 17:24:06 +0900 Subject: [PATCH 12/13] use first shard as spec --- lib/active_record/turntable/connection_proxy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/active_record/turntable/connection_proxy.rb b/lib/active_record/turntable/connection_proxy.rb index d83fab01..16f43d3b 100644 --- a/lib/active_record/turntable/connection_proxy.rb +++ b/lib/active_record/turntable/connection_proxy.rb @@ -234,7 +234,7 @@ def supports_views?(*args) end def spec - @spec ||= master.connection_pool.spec + @spec ||= shards.values.first.connection_pool.spec end private From 37e5630881bf0e274022a335dc0e6eb33a85b2ff Mon Sep 17 00:00:00 2001 From: MATSUBARA Nobutada Date: Thu, 8 Apr 2021 18:19:24 +0900 Subject: [PATCH 13/13] Fix: use logger.debug --- .../turntable/active_record_ext/abstract_adapter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/active_record/turntable/active_record_ext/abstract_adapter.rb b/lib/active_record/turntable/active_record_ext/abstract_adapter.rb index 074a8169..327de89c 100644 --- a/lib/active_record/turntable/active_record_ext/abstract_adapter.rb +++ b/lib/active_record/turntable/active_record_ext/abstract_adapter.rb @@ -18,7 +18,7 @@ def log(sql, name = "SQL", binds = [], statement_name = nil) :turntable_shard_name => turntable_shard_name) { yield } rescue Exception => e message = "#{e.class.name}: #{e.message}: #{sql} : #{turntable_shard_name}" - @logger.error message if @logger + @logger.debug message if @logger exception = translate_exception(e, message) exception.set_backtrace e.backtrace raise exception