diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..f0c5c92 --- /dev/null +++ b/Gemfile @@ -0,0 +1,47 @@ +source 'https://rubygems.org' +git_source(:github) { |repo| "https://github.com/#{repo}.git" } + +ruby '2.6.1' + +gem 'rails', '~> 5.2.3' +gem 'pg', '>= 0.18', '< 2.0' +gem 'puma', '~> 3.11' +gem 'bootsnap', '>= 1.1.0', require: false +gem 'skylight' +gem 'pghero' +gem 'activerecord-import' +gem 'oj' +gem 'progress_bar' +gem 'strong_migrations' +gem 'newrelic_rpm' +gem 'piperator' +gem 'json-stream' +gem 'yajl-ruby', require: 'yajl' +gem 'yajl-ffi' + +group :development, :test do + # Call 'byebug' anywhere in the code to stop execution and get a debugger console + gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] + +end + +group :development do + # Access an interactive console on exception pages or by calling 'console' anywhere in the code. + gem 'web-console', '>= 3.3.0' + gem 'listen', '>= 3.0.5', '< 3.2' + gem 'bullet' +end + +group :test do + gem 'rspec' + gem 'rspec-rails' + gem 'rspec-sqlimit' + gem 'test-prof' + gem 'database_cleaner' + gem 'n_plus_one_control' + gem 'factory_bot_rails' + gem 'faker', :git => 'https://github.com/stympy/faker.git', :branch => 'master' +end + +# Windows does not include zoneinfo files, so bundle the tzinfo-data gem +gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..179db45 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,284 @@ +GIT + remote: https://github.com/stympy/faker.git + revision: 59a92644d5583828add1c68351074f6e52c6deb6 + branch: master + specs: + faker (1.9.3) + i18n (>= 0.7) + pastel (~> 0.7.2) + thor (~> 0.20.0) + tty-pager (~> 0.12.0) + tty-screen (~> 0.6.5) + tty-tree (~> 0.2.0) + +GEM + remote: https://rubygems.org/ + specs: + actioncable (5.2.3) + actionpack (= 5.2.3) + nio4r (~> 2.0) + websocket-driver (>= 0.6.1) + actionmailer (5.2.3) + actionpack (= 5.2.3) + actionview (= 5.2.3) + activejob (= 5.2.3) + mail (~> 2.5, >= 2.5.4) + rails-dom-testing (~> 2.0) + actionpack (5.2.3) + actionview (= 5.2.3) + activesupport (= 5.2.3) + rack (~> 2.0) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.0, >= 1.0.2) + actionview (5.2.3) + activesupport (= 5.2.3) + builder (~> 3.1) + erubi (~> 1.4) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.0, >= 1.0.3) + activejob (5.2.3) + activesupport (= 5.2.3) + globalid (>= 0.3.6) + activemodel (5.2.3) + activesupport (= 5.2.3) + activerecord (5.2.3) + activemodel (= 5.2.3) + activesupport (= 5.2.3) + arel (>= 9.0) + activerecord-import (1.0.1) + activerecord (>= 3.2) + activestorage (5.2.3) + actionpack (= 5.2.3) + activerecord (= 5.2.3) + marcel (~> 0.3.1) + activesupport (5.2.3) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + arel (9.0.0) + bindex (0.6.0) + binding_of_caller (0.8.0) + debug_inspector (>= 0.0.1) + bootsnap (1.4.2) + msgpack (~> 1.0) + builder (3.2.3) + bullet (5.9.0) + activesupport (>= 3.0.0) + uniform_notifier (~> 1.11) + byebug (11.0.1) + coderay (1.1.2) + concurrent-ruby (1.1.5) + crass (1.0.4) + database_cleaner (1.7.0) + debug_inspector (0.0.3) + diff-lcs (1.3) + equatable (0.5.0) + erubi (1.8.0) + factory_bot (5.0.2) + activesupport (>= 4.2.0) + factory_bot_rails (5.0.2) + factory_bot (~> 5.0.2) + railties (>= 4.2.0) + ffi (1.10.0) + globalid (0.4.2) + activesupport (>= 4.2.0) + highline (2.0.2) + i18n (1.6.0) + concurrent-ruby (~> 1.0) + interception (0.5) + json-stream (0.2.1) + listen (3.1.5) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + ruby_dep (~> 1.2) + loofah (2.2.3) + crass (~> 1.0.2) + nokogiri (>= 1.5.9) + mail (2.7.1) + mini_mime (>= 0.1.1) + marcel (0.3.3) + mimemagic (~> 0.3.2) + method_source (0.9.2) + mimemagic (0.3.3) + mini_mime (1.0.1) + mini_portile2 (2.4.0) + minitest (5.11.3) + msgpack (1.2.9) + n_plus_one_control (0.3.1) + newrelic_rpm (6.2.0.354) + nio4r (2.3.1) + nokogiri (1.10.2) + mini_portile2 (~> 2.4.0) + oj (3.7.12) + options (2.3.2) + pastel (0.7.2) + equatable (~> 0.5.0) + tty-color (~> 0.4.0) + pg (1.1.4) + pghero (2.2.0) + activerecord + piperator (0.3.0) + progress_bar (1.3.0) + highline (>= 1.6, < 3) + options (~> 2.3.0) + pry (0.12.2) + coderay (~> 1.1.0) + method_source (~> 0.9.0) + pry-byebug (3.7.0) + byebug (~> 11.0) + pry (~> 0.10) + pry-rails (0.3.9) + pry (>= 0.10.4) + pry-rescue (1.5.0) + interception (>= 0.5) + pry (>= 0.12.0) + pry-stack_explorer (0.4.9.3) + binding_of_caller (>= 0.7) + pry (>= 0.9.11) + puma (3.12.1) + rack (2.0.7) + rack-test (1.1.0) + rack (>= 1.0, < 3) + rails (5.2.3) + actioncable (= 5.2.3) + actionmailer (= 5.2.3) + actionpack (= 5.2.3) + actionview (= 5.2.3) + activejob (= 5.2.3) + activemodel (= 5.2.3) + activerecord (= 5.2.3) + activestorage (= 5.2.3) + activesupport (= 5.2.3) + bundler (>= 1.3.0) + railties (= 5.2.3) + sprockets-rails (>= 2.0.0) + rails-dom-testing (2.0.3) + activesupport (>= 4.2.0) + nokogiri (>= 1.6) + rails-html-sanitizer (1.0.4) + loofah (~> 2.2, >= 2.2.2) + railties (5.2.3) + actionpack (= 5.2.3) + activesupport (= 5.2.3) + method_source + rake (>= 0.8.7) + thor (>= 0.19.0, < 2.0) + rake (12.3.2) + rb-fsevent (0.10.3) + rb-inotify (0.10.0) + ffi (~> 1.0) + rspec (3.8.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-core (3.8.0) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-mocks (3.8.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-rails (3.8.2) + actionpack (>= 3.0) + activesupport (>= 3.0) + railties (>= 3.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-support (~> 3.8.0) + rspec-sqlimit (0.0.2) + rails (> 4.0, < 6.0) + rspec (~> 3.0) + rspec-support (3.8.0) + ruby_dep (1.5.0) + skylight (3.1.5) + skylight-core (= 3.1.5) + skylight-core (3.1.5) + activesupport (>= 4.2.0) + sprockets (3.7.2) + concurrent-ruby (~> 1.0) + rack (> 1, < 3) + sprockets-rails (3.2.1) + actionpack (>= 4.0) + activesupport (>= 4.0) + sprockets (>= 3.0.0) + strings (0.1.5) + strings-ansi (~> 0.1) + unicode-display_width (~> 1.5) + unicode_utils (~> 1.4) + strings-ansi (0.1.0) + strong_migrations (0.3.1) + activerecord (>= 3.2.0) + test-prof (0.8.0) + thor (0.20.3) + thread_safe (0.3.6) + tty-color (0.4.3) + tty-pager (0.12.1) + strings (~> 0.1.4) + tty-screen (~> 0.6) + tty-which (~> 0.4) + tty-screen (0.6.5) + tty-tree (0.2.0) + tty-which (0.4.0) + tzinfo (1.2.5) + thread_safe (~> 0.1) + unicode-display_width (1.5.0) + unicode_utils (1.4.0) + uniform_notifier (1.12.1) + web-console (3.7.0) + actionview (>= 5.0) + activemodel (>= 5.0) + bindex (>= 0.4.0) + railties (>= 5.0) + websocket-driver (0.7.0) + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.3) + yajl-ffi (0.1.2) + ffi (~> 1.9) + yajl-ruby (1.4.1) + +PLATFORMS + ruby + +DEPENDENCIES + activerecord-import + bootsnap (>= 1.1.0) + bullet + byebug + database_cleaner + factory_bot_rails + faker! + json-stream + listen (>= 3.0.5, < 3.2) + n_plus_one_control + newrelic_rpm + oj + pg (>= 0.18, < 2.0) + pghero + piperator + progress_bar + pry-byebug + pry-rails + pry-rescue + pry-stack_explorer + puma (~> 3.11) + rails (~> 5.2.3) + rspec + rspec-rails + rspec-sqlimit + skylight + strong_migrations + test-prof + tzinfo-data + web-console (>= 3.3.0) + yajl-ffi + yajl-ruby + +RUBY VERSION + ruby 2.6.1p33 + +BUNDLED WITH + 2.0.1 diff --git a/app/services/import_trips_service.rb b/app/services/import_trips_service.rb new file mode 100644 index 0000000..bb1b068 --- /dev/null +++ b/app/services/import_trips_service.rb @@ -0,0 +1,183 @@ +# frozen_string_literal: true + +require 'yajl/ffi' +require 'json/streamer' +require 'piperator' +require 'oj' + +class ImportTripsService < Oj::ScHandler + BATCH_SIZE = 1000 + TRIPS_COMMAND = "copy trips (from_id, to_id, start_time, duration_minutes, price_cents, bus_id) from stdin with csv delimiter ';'" + + attr_accessor :is_bus, :is_service, :nesting, :last_key, :buses, + :services, :buses_services, :cities, :conn, :trip, :bus, :service + + def self.load(file_name) + new(file_name).load + end + + def initialize(file_name) + @is_bus = false + @is_service = false + @nesting = 0 + + @trip = {} + @bus = {} + @service = [] + @cities = {} + @buses = {} + @services = {} + @buses_services = [] + + @file_name = file_name + @conn = ActiveRecord::Base.connection.raw_connection + end + + def hash_key(key) + @last_key = key + @is_bus = true if key == 'bus' + @is_service = true if key == 'services' + end + + def hash_start + @nesting += 1 + @service.clear + end + + def hash_set(_h, _key, value) + bus[@last_key] = value if is_bus + trip[@last_key] = value unless is_bus + end + + def hash_end + @nesting -= 1 + @is_bus = false if @is_bus + + push_to_db if @nesting.zero? + end + + def array_append(_a, value) + service << value if @is_service + end + + def array_end + @is_service = false if @is_service + end + + def load + clean_db! + load_services + load_trips + load_cities + load_buses + load_buses_services + end + + private + + def clean_db! + BusesService.delete_all + Trip.delete_all + City.delete_all + Bus.delete_all + Service.delete_all + end + + def push_to_db + from_id = fetch_city_id(trip['from']) + to_id = fetch_city_id(trip['to']) + bus_id = fetch_bus_id + + conn.put_copy_data("#{from_id};#{to_id};#{trip['start_time']};#{trip['duration_minutes']};#{trip['price_cents']};#{bus_id}\n") + end + + def load_trips + disable_indices_for(:trips) + conn.copy_data TRIPS_COMMAND do + io = File.open(@file_name, 'r') + Oj.sc_parse(self, io) + end + enable_indices_for(:trips) + end + + def disable_indices_for(table) + ActiveRecord::Base.connection.execute(<<-SQL.squish) + UPDATE pg_index + SET indisready=false + WHERE indrelid = ( + SELECT oid + FROM pg_class + WHERE relname='#{table}' + ); + SQL + end + + def enable_indices_for(table) + ActiveRecord::Base.connection.execute(<<-SQL.squish) + UPDATE pg_index + SET indisready=true + WHERE indrelid = ( + SELECT oid + FROM pg_class + WHERE relname='#{table}' + ); + SQL + + ActiveRecord::Base.connection.execute("REINDEX TABLE #{table}") + end + + def load_services + selection = Service.import(%i[name], Service::SERVICES.product, returning: :name) + ids = selection.ids + selection.results.each.with_index { |attr, i| services[attr] = ids[i].to_i } + end + + def load_cities + disable_indices_for :cities + City.import(cities.map { |city, id| { name: city, id: id } }, options) + enable_indices_for :cities + end + + def load_buses + disable_indices_for :buses + columns = %i[number model id].freeze + values = buses.map { |k, v| k << v } + Bus.import(columns, values, options) + enable_indices_for :buses + end + + def load_buses_services + disable_indices_for :buses_services + columns = %i[bus_id service_id].freeze + buses_services.uniq! + + BusesService.import(columns, buses_services, options) + enable_indices_for :buses_services + end + + def options + { + batch_size: BATCH_SIZE + } + end + + def fetch_city_id(city) + id = cities[city] + unless id + id = cities.size + 1 + cities[city] = id + end + id + end + + def fetch_bus_id + composite_key = [bus['number'], bus['model']] + bus_id = buses[composite_key] + unless bus_id + bus_id = buses.size + 1 + buses[composite_key] = bus_id + end + service.each { |s| buses_services << [bus_id, services[s]] } + bus_id + end +end