diff --git a/.rspec b/.rspec
new file mode 100644
index 00000000..c99d2e73
--- /dev/null
+++ b/.rspec
@@ -0,0 +1 @@
+--require spec_helper
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..8d64459e
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,34 @@
+FROM ruby:2.6.3-alpine
+
+ENV APP_PATH /var/www/schedule_app
+ENV TZ Europe/Moscow
+ENV LANG ru_RU.UTF-8
+ENV LANGUAGE ru_RU.UTF-8
+ENV LC_ALL ru_RU.UTF-8
+
+RUN apk update && apk add --no-cache yarn \
+ build-base curl-dev git postgresql-dev \
+ yaml-dev zlib-dev nodejs gcc g++ make busybox ctags \
+ tzdata
+
+RUN cp /usr/share/zoneinfo/Europe/Moscow /etc/localtime && \
+ echo "Europe/Moscow" > /etc/timezone
+
+RUN mkdir -p $APP_PATH
+
+WORKDIR $APP_PATH
+
+COPY Gemfile $APP_PATH/Gemfile
+COPY Gemfile.lock $APP_PATH/Gemfile.lock
+COPY package.json $APP_PATH/package.json
+
+RUN yarn global add node-sass
+RUN yarn
+RUN gem install bundler:2.0.2
+RUN bundle install
+
+COPY . $APP_PATH
+
+EXPOSE 3000/tcp
+#CMD sh -c "bin/setup"
+#CMD sh -c "bundle exec rails server -b 0.0.0.0 -p 3000"
diff --git a/Gemfile b/Gemfile
index e20b1260..8d6053be 100644
--- a/Gemfile
+++ b/Gemfile
@@ -8,18 +8,37 @@ gem 'pg', '>= 0.18', '< 2.0'
gem 'puma', '~> 3.11'
gem 'bootsnap', '>= 1.1.0', require: false
+
+# profilers
+gem 'rack-mini-profiler', require: false
+# For memory profiling
+gem 'memory_profiler'
+
+# For call-stack profiling flamegraphs
+gem 'flamegraph'
+gem 'stackprof'
+gem "bullet"
+gem 'meta_request'
+
+gem 'activerecord-import', require: false
+gem 'composite_primary_keys', '=11.2.0'
+gem 'oj'
+
group :development, :test do
# Call 'byebug' anywhere in the code to stop execution and get a debugger console
gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
end
group :development do
+ gem 'pry'
# Access an interactive console on exception pages or by calling 'console' anywhere in the code.
gem 'web-console', '>= 3.3.0'
gem 'listen', '>= 3.0.5', '< 3.2'
end
group :test do
+ gem 'rspec-rails', '~> 3.8'
+ gem 'rspec-benchmark'
end
# Windows does not include zoneinfo files, so bundle the tzinfo-data gem
diff --git a/Gemfile.lock b/Gemfile.lock
index fccf6f5f..50c8b2a5 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -33,6 +33,8 @@ GEM
activemodel (= 5.2.3)
activesupport (= 5.2.3)
arel (>= 9.0)
+ activerecord-import (1.0.4)
+ activerecord (>= 3.2)
activestorage (5.2.3)
actionpack (= 5.2.3)
activerecord (= 5.2.3)
@@ -43,15 +45,26 @@ GEM
minitest (~> 5.1)
tzinfo (~> 1.1)
arel (9.0.0)
+ benchmark-malloc (0.1.0)
+ benchmark-perf (0.5.0)
+ benchmark-trend (0.3.0)
bindex (0.6.0)
bootsnap (1.4.2)
msgpack (~> 1.0)
builder (3.2.3)
+ bullet (6.1.0)
+ activesupport (>= 3.0.0)
+ uniform_notifier (~> 1.11)
byebug (11.0.1)
+ coderay (1.1.2)
+ composite_primary_keys (11.2.0)
+ activerecord (~> 5.2.1)
concurrent-ruby (1.1.5)
crass (1.0.4)
+ diff-lcs (1.3)
erubi (1.8.0)
ffi (1.10.0)
+ flamegraph (0.9.5)
globalid (0.4.2)
activesupport (>= 4.2.0)
i18n (1.6.0)
@@ -67,6 +80,10 @@ GEM
mini_mime (>= 0.1.1)
marcel (0.3.3)
mimemagic (~> 0.3.2)
+ memory_profiler (0.9.14)
+ meta_request (0.7.2)
+ rack-contrib (>= 1.1, < 3)
+ railties (>= 3.0.0, < 7)
method_source (0.9.2)
mimemagic (0.3.3)
mini_mime (1.0.1)
@@ -76,9 +93,17 @@ GEM
nio4r (2.3.1)
nokogiri (1.10.2)
mini_portile2 (~> 2.4.0)
+ oj (3.10.5)
pg (1.1.4)
+ pry (0.12.2)
+ coderay (~> 1.1.0)
+ method_source (~> 0.9.0)
puma (3.12.1)
rack (2.0.6)
+ rack-contrib (2.1.0)
+ rack (~> 2.0)
+ rack-mini-profiler (1.1.6)
+ rack (>= 1.2.0)
rack-test (1.1.0)
rack (>= 1.0, < 3)
rails (5.2.3)
@@ -109,6 +134,32 @@ GEM
rb-fsevent (0.10.3)
rb-inotify (0.10.0)
ffi (~> 1.0)
+ rspec (3.9.0)
+ rspec-core (~> 3.9.0)
+ rspec-expectations (~> 3.9.0)
+ rspec-mocks (~> 3.9.0)
+ rspec-benchmark (0.5.1)
+ benchmark-malloc (~> 0.1.0)
+ benchmark-perf (~> 0.5.0)
+ benchmark-trend (~> 0.3.0)
+ rspec (>= 3.0.0, < 4.0.0)
+ rspec-core (3.9.1)
+ rspec-support (~> 3.9.1)
+ rspec-expectations (3.9.0)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.9.0)
+ rspec-mocks (3.9.1)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.9.0)
+ rspec-rails (3.9.0)
+ actionpack (>= 3.0)
+ activesupport (>= 3.0)
+ railties (>= 3.0)
+ rspec-core (~> 3.9.0)
+ rspec-expectations (~> 3.9.0)
+ rspec-mocks (~> 3.9.0)
+ rspec-support (~> 3.9.0)
+ rspec-support (3.9.2)
ruby_dep (1.5.0)
sprockets (3.7.2)
concurrent-ruby (~> 1.0)
@@ -117,10 +168,12 @@ GEM
actionpack (>= 4.0)
activesupport (>= 4.0)
sprockets (>= 3.0.0)
+ stackprof (0.2.15)
thor (0.20.3)
thread_safe (0.3.6)
tzinfo (1.2.5)
thread_safe (~> 0.1)
+ uniform_notifier (1.13.0)
web-console (3.7.0)
actionview (>= 5.0)
activemodel (>= 5.0)
@@ -134,12 +187,24 @@ PLATFORMS
ruby
DEPENDENCIES
+ activerecord-import
bootsnap (>= 1.1.0)
+ bullet
byebug
+ composite_primary_keys (= 11.2.0)
+ flamegraph
listen (>= 3.0.5, < 3.2)
+ memory_profiler
+ meta_request
+ oj
pg (>= 0.18, < 2.0)
+ pry
puma (~> 3.11)
+ rack-mini-profiler
rails (~> 5.2.3)
+ rspec-benchmark
+ rspec-rails (~> 3.8)
+ stackprof
tzinfo-data
web-console (>= 3.3.0)
diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb
index acb38be2..09514f6a 100644
--- a/app/controllers/trips_controller.rb
+++ b/app/controllers/trips_controller.rb
@@ -2,6 +2,6 @@ class TripsController < ApplicationController
def index
@from = City.find_by_name!(params[:from])
@to = City.find_by_name!(params[:to])
- @trips = Trip.where(from: @from, to: @to).order(:start_time)
+ @trips = Trip.where(from: @from, to: @to).order(:start_time).includes(:bus).includes(bus: :services)
end
end
diff --git a/app/models/bus.rb b/app/models/bus.rb
index 1dcc54cb..c306aa6a 100644
--- a/app/models/bus.rb
+++ b/app/models/bus.rb
@@ -12,8 +12,10 @@ class Bus < ApplicationRecord
'Газель',
].freeze
- has_many :trips
- has_and_belongs_to_many :services, join_table: :buses_services
+ self.primary_keys = :number, :model
+
+ has_many :trips, :foreign_key => [:number, :model]
+ has_and_belongs_to_many :services, join_table: :buses_services, :foreign_key => [:number, :model]
validates :number, presence: true, uniqueness: true
validates :model, inclusion: { in: MODELS }
diff --git a/app/models/service.rb b/app/models/service.rb
index 9cbb2a32..44e79470 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -12,7 +12,7 @@ class Service < ApplicationRecord
'Можно не печатать билет',
].freeze
- has_and_belongs_to_many :buses, join_table: :buses_services
+ has_and_belongs_to_many :buses, join_table: :buses_services, :foreign_key => :service_id, association_foreign_key: [:number, :model]
validates :name, presence: true
validates :name, inclusion: { in: SERVICES }
diff --git a/app/models/trip.rb b/app/models/trip.rb
index 9d63dfff..986616f9 100644
--- a/app/models/trip.rb
+++ b/app/models/trip.rb
@@ -3,7 +3,7 @@ class Trip < ApplicationRecord
belongs_to :from, class_name: 'City'
belongs_to :to, class_name: 'City'
- belongs_to :bus
+ belongs_to :bus, -> { includes :services }, :foreign_key => [:number, :model]
validates :from, presence: true
validates :to, presence: true
diff --git a/app/services/schedule_loader.rb b/app/services/schedule_loader.rb
new file mode 100644
index 00000000..fd86f0bc
--- /dev/null
+++ b/app/services/schedule_loader.rb
@@ -0,0 +1,105 @@
+require 'active_record'
+require 'activerecord-import'
+
+# frozen_string_literal: true
+STrip = Struct.new(:from, :to, :start_time, :duration_minutes, :price_cents, :number, :model)
+SBus = Struct.new(:number, :model, :services)
+
+class ScheduleLoader
+ def self.call(file_name)
+ @cities = {}
+ @services = {}
+ @buses = Set.new
+
+ @@services = {}
+ Service.pluck(:name, :id).map{|sr| @@services[sr[0].to_sym] = sr[1]}
+
+ # https://koshigoe.github.io/postgresql/ruby/2018/10/31/bulk-insert-vs-copy-in-postgres.html
+ @base_connection = ActiveRecord::Base.connection
+ @raw_conn = ActiveRecord::Base.connection.raw_connection
+ @encoder = PG::TextEncoder::CopyRow.new
+
+ @trips_command_raw = "copy trips (from_id, to_id, start_time, duration_minutes, price_cents, model, number) from stdin"
+ @buses_services_command_raw = "copy buses_services (service_id, model, number) from stdin"
+ @bus_command_raw = "copy buses (model, number) from stdin"
+
+ @nr2 = Array.new(2)
+ @nr3 = Array.new(3)
+ @nr7 = Array.new(7)
+
+ ActiveRecord::Base.transaction do
+ @raw_conn.exec('delete from buses_services;')
+ @raw_conn.exec('delete from trips;')
+ @raw_conn.exec('delete from cities;')
+ @raw_conn.exec('delete from buses;')
+ @base_connection.reset_pk_sequence!('cities')
+ @base_connection.reset_pk_sequence!('buses')
+
+ File.open(file_name) do |ff|
+ nesting = 0
+ str = +''
+
+ until ff.eof?
+ ch = ff.read(1) # читаем по одному символу
+ if ch == '{' # начинается объект, повышается вложенность
+ nesting += 1
+ str << ch
+ elsif ch == '}' # заканчивается объект, понижается вложенность
+ nesting -= 1
+ str << ch
+ if nesting == 0 # если закончился объкет уровня trip, парсим и импортируем его
+
+ trip = Oj.load(str)
+
+ import_trip(trip)
+
+ str = +''
+ end
+ elsif nesting >= 1
+ str << ch
+ end
+ end
+ end
+ City.import @cities.map {|cc| ({id: cc[1], name: cc[0]})}
+ end
+ end
+
+ def self.import_trip(trip)
+ from_id = @cities[trip['from']]
+ unless from_id
+ from_id = @cities.size + 1
+ @cities[trip['from']] = from_id
+ end
+
+ to_id = @cities[trip['to']]
+ unless to_id
+ to_id = @cities.size + 1
+ @cities[trip['to']] = to_id
+ end
+
+ bus_number = trip['bus']['number']
+ bus_model = trip['bus']['model']
+
+ bus_key = "#{bus_number}-#{bus_model}"
+
+ unless @buses.include?(bus_key)
+ @raw_conn.copy_data @bus_command_raw, @encoder do
+ @nr2 = bus_number, bus_model
+ @raw_conn.put_copy_data(@nr2)
+ end
+ @buses << bus_key
+
+ @raw_conn.copy_data @buses_services_command_raw, @encoder do
+ trip["bus"]["services"].map do |srv|
+ @nr3 = @@services[srv.to_sym], bus_number, bus_model
+ @raw_conn.put_copy_data(@nr3)
+ end
+ end
+ end
+
+ @raw_conn.copy_data @trips_command_raw, @encoder do
+ @nr7 = from_id,to_id, trip['start_time'],trip['duration_minutes'],trip['price_cents'],bus_number,bus_model
+ @raw_conn.put_copy_data(@nr7)
+ end
+ end
+end
diff --git a/app/views/trips/_delimiter.html.erb b/app/views/trips/_delimiter.html.erb
deleted file mode 100644
index 3f845ad0..00000000
--- a/app/views/trips/_delimiter.html.erb
+++ /dev/null
@@ -1 +0,0 @@
-====================================================
diff --git a/app/views/trips/_service.html.erb b/app/views/trips/_service.html.erb
deleted file mode 100644
index 178ea8c0..00000000
--- a/app/views/trips/_service.html.erb
+++ /dev/null
@@ -1 +0,0 @@
-<%= "#{service.name}" %>
diff --git a/app/views/trips/_services.html.erb b/app/views/trips/_services.html.erb
deleted file mode 100644
index 2de639fc..00000000
--- a/app/views/trips/_services.html.erb
+++ /dev/null
@@ -1,6 +0,0 @@
-Сервисы в автобусе:
-
- <% services.each do |service| %>
- <%= render "service", service: service %>
- <% end %>
-
diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb
index fa1de9aa..e8e76e98 100644
--- a/app/views/trips/_trip.html.erb
+++ b/app/views/trips/_trip.html.erb
@@ -3,3 +3,15 @@
<%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
<%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
<%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
+
+Сервисы в автобусе:
+
+ <% trip.bus.services.pluck(:name).map do |service| %>
+ <%#= render "service", service: service %>
+ - <%= service %>
+ <% end %>
+
+
+
+<%# render plain: '====================================================' %>
+====================================================
\ No newline at end of file
diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb
index a60bce41..44413751 100644
--- a/app/views/trips/index.html.erb
+++ b/app/views/trips/index.html.erb
@@ -2,15 +2,7 @@
<%= "Автобусы #{@from.name} – #{@to.name}" %>
- <%= "В расписании #{@trips.count} рейсов" %>
+ <%= "В расписании #{@trips.size} рейсов" %>
-
-<% @trips.each do |trip| %>
-
- <%= render "trip", trip: trip %>
- <% if trip.bus.services.present? %>
- <%= render "services", services: trip.bus.services %>
- <% end %>
-
- <%= render "delimiter" %>
-<% end %>
+<%# binding.pry %>
+<%= render partial: "trip", collection: @trips, cached: true %>
diff --git a/case-study.md b/case-study.md
new file mode 100644
index 00000000..0e1cdb9d
--- /dev/null
+++ b/case-study.md
@@ -0,0 +1,19 @@
+- [X] `rack-mini-profiler`
+- [X] `rails panel`
+- [X] `bullet`
+- [X] `explain` запросов
+
+реализовал composite primary key (model, number) через гем 'composite_primary_keys'. Нашел интересную статью с бенчмарками различных методик загрузки данных в Postgreql -
+`https://koshigoe.github.io/postgresql/ruby/2018/10/31/bulk-insert-vs-copy-in-postgres.html`. Прогнал локально тесты из статьи и реализовал самый быстрый, что дало прирост, но меньший чем ожидал. Посчитал что текущей скорости достаточно и перешел к оптимизации отображения расписания.
+- импорт данных из файла rake reload_json[fixtures/large.json] проходит за 40 секунд.
+- загрузка страницы с расписанием занимат 6-11 секунд
+- не удалось применить подсказку из
+````
+user: root
+USE eager loading detected
+ Bus::HABTM_Services => [:service]
+ Add to your query: .includes([:service])
+Call stack
+ /var/www/schedule_app/app/views/trips/_trip.html.erb:9:in `_app_views_trips__trip_html_erb___2552659478807376598_47345913761100'
+ /var/www/schedule_app/app/views/trips/index.html.erb:8:in `_app_views_trips_index_html_erb__2240206535625781100_47345814716480
+ ````
diff --git a/config/database.yml b/config/database.yml
index e116cfa6..94612a2c 100644
--- a/config/database.yml
+++ b/config/database.yml
@@ -23,7 +23,11 @@ default: &default
development:
<<: *default
- database: task-4_development
+ host: <%= ENV.fetch("DEV_SCHD_DATABASE_HOST", "localhost") %>
+ port: <%= ENV.fetch("DEV_SCHD_DATABASE_PORT", "5432") %>
+ username: <%= ENV.fetch("DEV_SCHD_DATABASE_USERNAME") %>
+ password: <%= ENV.fetch("DEV_SCHD_DATABASE_PASSWORD") %>
+ database: <%= "#{ENV.fetch("SCHD_DATABASE_NAME", "task-4")}_development" %>
# The specified database role being used to connect to postgres.
# To create additional roles in postgres see `$ createuser --help`.
@@ -57,7 +61,11 @@ development:
# Do not set this db to the same as development or production.
test:
<<: *default
- database: task-4_test
+ host: <%= ENV.fetch("DEV_SCHD_DATABASE_HOST", "localhost") %>
+ port: <%= ENV.fetch("DEV_SCHD_DATABASE_PORT", "5432") %>
+ username: <%= ENV.fetch("DEV_SCHD_DATABASE_USERNAME") %>
+ password: <%= ENV.fetch("DEV_SCHD_DATABASE_PASSWORD") %>
+ database: <%= "#{ENV.fetch("SCHD_DATABASE_NAME", "task-4")}_test" %>
# As with config/secrets.yml, you never want to store sensitive information,
# like your database password, in your source code. If your source code is
diff --git a/config/environments/development.rb b/config/environments/development.rb
index 1311e3e4..4e7a42e2 100644
--- a/config/environments/development.rb
+++ b/config/environments/development.rb
@@ -1,5 +1,13 @@
Rails.application.configure do
- # Settings specified here will take precedence over those in config/application.rb.
+ config.after_initialize do
+ Bullet.enable = true
+ Bullet.alert = true
+ Bullet.bullet_logger = true
+ Bullet.console = true
+ # Bullet.growl = true
+ Bullet.rails_logger = true
+ Bullet.add_footer = true
+ end # Settings specified here will take precedence over those in config/application.rb.
# In the development environment your application's code is reloaded on
# every request. This slows down response time but is perfect for development
diff --git a/config/environments/test.rb b/config/environments/test.rb
index 0a38fd3c..b31b09d3 100644
--- a/config/environments/test.rb
+++ b/config/environments/test.rb
@@ -1,5 +1,9 @@
Rails.application.configure do
- # Settings specified here will take precedence over those in config/application.rb.
+ config.after_initialize do
+ Bullet.enable = true
+ Bullet.bullet_logger = true
+ Bullet.raise = true # raise an error if n+1 query occurs
+ end # Settings specified here will take precedence over those in config/application.rb.
# The test environment is used exclusively to run your application's
# test suite. You never need to work with it otherwise. Remember that
diff --git a/config/initializers/rack_profiler.rb b/config/initializers/rack_profiler.rb
new file mode 100644
index 00000000..96b6cc98
--- /dev/null
+++ b/config/initializers/rack_profiler.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+if Rails.env.development?
+ require "rack-mini-profiler"
+
+ # initialization is skipped so trigger it
+ Rack::MiniProfilerRails.initialize!(Rails.application)
+end
diff --git a/db/migrate/20200307214701_change_bus_primary_key.rb b/db/migrate/20200307214701_change_bus_primary_key.rb
new file mode 100755
index 00000000..2a41e099
--- /dev/null
+++ b/db/migrate/20200307214701_change_bus_primary_key.rb
@@ -0,0 +1,14 @@
+class ChangeBusPrimaryKey < ActiveRecord::Migration[5.2]
+ def up
+ remove_column :buses, :id
+ execute 'ALTER TABLE buses ADD CONSTRAINT buses_pkey PRIMARY KEY ("number",model);'
+ add_index :buses, [:number, :model], unique: true
+ end
+
+ def down
+ execute 'ALTER TABLE buses DROP CONSTRAINT buses_pkey;'
+ add_column :buses, :id, :primary_key
+
+ remove_index :buses, [:number, :model]
+ end
+end
diff --git a/db/migrate/20200308003029_change_buses_services.rb b/db/migrate/20200308003029_change_buses_services.rb
new file mode 100755
index 00000000..70693844
--- /dev/null
+++ b/db/migrate/20200308003029_change_buses_services.rb
@@ -0,0 +1,15 @@
+class ChangeBusesServices < ActiveRecord::Migration[5.2]
+ def up
+ add_column :buses_services, :model, :string
+ add_column :buses_services, :number, :string
+ remove_column :buses_services, :bus_id
+ execute 'ALTER TABLE buses_services ADD CONSTRAINT buses_services_fk FOREIGN KEY (model,"number") REFERENCES buses(model,"number");'
+ end
+
+ def down
+ execute 'ALTER TABLE buses_services DROP CONSTRAINT buses_services_fk;'
+ remove_column :buses_services, :number
+ remove_column :buses_services, :model
+ add_column :trips, :bus_id, :integer
+ end
+end
diff --git a/db/migrate/20200308003123_change_trips_foreign_key.rb b/db/migrate/20200308003123_change_trips_foreign_key.rb
new file mode 100755
index 00000000..84585d69
--- /dev/null
+++ b/db/migrate/20200308003123_change_trips_foreign_key.rb
@@ -0,0 +1,15 @@
+class ChangeTripsForeignKey < ActiveRecord::Migration[5.2]
+ def up
+ add_column :trips, :model, :string
+ add_column :trips, :number, :string
+ remove_column :trips, :bus_id
+ execute 'ALTER TABLE trips ADD CONSTRAINT trips_fk FOREIGN KEY (model,"number") REFERENCES buses(model,"number");'
+ end
+
+ def down
+ execute 'ALTER TABLE trips DROP CONSTRAINT trips_fk;'
+ remove_column :trips, :model
+ remove_column :trips, :number
+ add_column :trips, :bus_id, :integer
+ end
+end
diff --git a/db/migrate/20200308114605_write_services.rb b/db/migrate/20200308114605_write_services.rb
new file mode 100755
index 00000000..711a247d
--- /dev/null
+++ b/db/migrate/20200308114605_write_services.rb
@@ -0,0 +1,26 @@
+class WriteServices < ActiveRecord::Migration[5.2]
+
+ def up
+ Service.delete_all
+ [
+ 'WiFi',
+ 'Туалет',
+ 'Работающий туалет',
+ 'Ремни безопасности',
+ 'Кондиционер общий',
+ 'Кондиционер Индивидуальный',
+ 'Телевизор общий',
+ 'Телевизор индивидуальный',
+ 'Стюардесса',
+ 'Можно не печатать билет'
+ ].each do |name|
+ ss = Service.new(name: name)
+ ss.save!
+ end
+
+ end
+
+ def down
+ Service.delete_all
+ end
+end
diff --git a/db/migrate/20200309155753_add_indices_to_bus_services.rb b/db/migrate/20200309155753_add_indices_to_bus_services.rb
new file mode 100755
index 00000000..4cfee4d6
--- /dev/null
+++ b/db/migrate/20200309155753_add_indices_to_bus_services.rb
@@ -0,0 +1,21 @@
+class AddIndicesToBusServices < ActiveRecord::Migration[5.2]
+ disable_ddl_transaction!
+
+ def up
+ add_index :buses_services, %i[model number service_id], algorithm: :concurrently
+ add_index :buses_services, %i[service_id model number], algorithm: :concurrently
+ add_index :cities, :name, algorithm: :concurrently
+ add_index :trips, %i[from_id to_id], algorithm: :concurrently
+ add_index :buses, :model, algorithm: :concurrently
+ add_index :buses, :number, algorithm: :concurrently
+ end
+
+ def down
+ remove_index :buses_services, %i[model number service_id]
+ remove_index :buses_services, %i[service_id model number]
+ remove_index :cities, :name
+ remove_index :trips, %i[from_id to_id]
+ remove_index :buses, :model
+ remove_index :buses, :number
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f6921e45..f86083d1 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,23 +10,31 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2019_03_30_193044) do
+ActiveRecord::Schema.define(version: 2020_03_09_155753) do
# These are extensions that must be enabled in order to support this database
+ enable_extension "pg_stat_statements"
enable_extension "plpgsql"
- create_table "buses", force: :cascade do |t|
- t.string "number"
- t.string "model"
+ create_table "buses", primary_key: ["number", "model"], force: :cascade do |t|
+ t.string "number", null: false
+ t.string "model", null: false
+ t.index ["model"], name: "index_buses_on_model"
+ t.index ["number", "model"], name: "index_buses_on_number_and_model", unique: true
+ t.index ["number"], name: "index_buses_on_number"
end
create_table "buses_services", force: :cascade do |t|
- t.integer "bus_id"
t.integer "service_id"
+ t.string "model"
+ t.string "number"
+ t.index ["model", "number", "service_id"], name: "index_buses_services_on_model_and_number_and_service_id"
+ t.index ["service_id", "model", "number"], name: "index_buses_services_on_service_id_and_model_and_number"
end
create_table "cities", force: :cascade do |t|
t.string "name"
+ t.index ["name"], name: "index_cities_on_name"
end
create_table "services", force: :cascade do |t|
@@ -39,7 +47,11 @@
t.string "start_time"
t.integer "duration_minutes"
t.integer "price_cents"
- t.integer "bus_id"
+ t.string "model"
+ t.string "number"
+ t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id"
end
+ add_foreign_key "buses_services", "buses", column: "model", primary_key: "model", name: "buses_services_fk"
+ add_foreign_key "trips", "buses", column: "model", primary_key: "model", name: "trips_fk"
end
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 00000000..a70cbc96
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,48 @@
+version: '3'
+services:
+ # PostgreSQL
+ db-postgresql:
+ image: postgres:9.6
+ volumes:
+ - "/opt/shared/postgresql-nfap/data:/var/lib/postgresql/data"
+ environment:
+ POSTGRES_USER: rails
+ POSTGRES_PASSWORD: password
+ tty: true
+ stdin_open: true
+ ports:
+ - "15432:5432"
+
+ pghero:
+ image: ankane/pghero
+ ports:
+ - "8080:8080"
+ environment:
+ DATABASE_URL: "postgres://rails:password@db-postgresql:5432/task-4_development"
+ tty: true
+ stdin_open: true
+ depends_on:
+ - db-postgresql
+
+ # schedule app
+ schedule_app:
+ build: .
+ environment:
+ - "PAGER='busybox more'"
+ - "SSH_AUTH_SOCK=/tmp/ssh-agent"
+ - "DISABLE_SPRING=1"
+ - "DEV_SCHD_DATABASE_PORT=5432"
+ - "DEV_SCHD_DATABASE_HOST=db-postgresql"
+ - "DEV_SCHD_DATABASE_USERNAME=rails"
+ - "DEV_SCHD_DATABASE_PASSWORD=password"
+ - "SCHD_DATABASE_NAME=task-4"
+ #- "DEV_REDIS_HOST=redis"
+ tty: true
+ stdin_open: true
+ volumes:
+ - "$PWD:/var/www/schedule_app:Z"
+ - $SSH_AUTH_SOCK:/tmp/ssh-agent:Z
+ ports:
+ - "3078:3000"
+ depends_on:
+ - db-postgresql
diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake
index 540fe871..9fc01666 100644
--- a/lib/tasks/utils.rake
+++ b/lib/tasks/utils.rake
@@ -1,34 +1,5 @@
# Наивная загрузка данных из json-файла в БД
# rake reload_json[fixtures/small.json]
task :reload_json, [:file_name] => :environment do |_task, args|
- json = JSON.parse(File.read(args.file_name))
-
- ActiveRecord::Base.transaction do
- City.delete_all
- Bus.delete_all
- Service.delete_all
- Trip.delete_all
- ActiveRecord::Base.connection.execute('delete from buses_services;')
-
- json.each do |trip|
- from = City.find_or_create_by(name: trip['from'])
- to = City.find_or_create_by(name: trip['to'])
- services = []
- trip['bus']['services'].each do |service|
- s = Service.find_or_create_by(name: service)
- services << s
- end
- bus = Bus.find_or_create_by(number: trip['bus']['number'])
- bus.update(model: trip['bus']['model'], services: services)
-
- Trip.create!(
- from: from,
- to: to,
- bus: bus,
- start_time: trip['start_time'],
- duration_minutes: trip['duration_minutes'],
- price_cents: trip['price_cents'],
- )
- end
- end
+ ScheduleLoader.call(args.file_name)
end
diff --git a/spec/bench_spec.rb b/spec/bench_spec.rb
new file mode 100644
index 00000000..41a57716
--- /dev/null
+++ b/spec/bench_spec.rb
@@ -0,0 +1,128 @@
+require 'active_record'
+require 'benchmark'
+require 'csv'
+
+def get_rss
+ `ps -o rss= -p #{Process.pid}`.to_i
+end
+
+COLUMN_LENGTH = 100
+COLUMN_SIZE = 10
+RECORD_SIZE = 100_000
+BATCH_SIZE = 500
+
+columns = Array.new(COLUMN_SIZE) { |i| ('a'.ord + (i % 26)).chr * COLUMN_LENGTH }
+records = Array.new(RECORD_SIZE, columns)
+
+# $ docker run --rm -p 5432:5432 -e POSTGRES_PASSWORD=password -d postgres:9.6
+# $ psql -h localhost -U postgres -d postgres
+ActiveRecord::Base.establish_connection(
+ adapter: 'postgresql',
+ encoding: 'unicode',
+ host: 'db-postgresql',
+ database: 'postgres',
+ username: 'rails',
+ password: 'password',
+)
+
+conn = ActiveRecord::Base.connection
+raw_conn = ActiveRecord::Base.connection.raw_connection
+
+%i(
+ benchmark_copy_format_text_1
+ benchmark_copy_format_text_2
+ benchmark_copy_format_csv_1
+ benchmark_copy_format_csv_2
+ benchmark_bulk_insert_1
+ benchmark_bulk_insert_2
+).each do |table_name|
+ conn.create_table table_name, force: true, id: false do |t|
+ COLUMN_SIZE.times { |i| t.string "col#{i}" }
+ end
+end
+
+rss_log = []
+
+Benchmark.bm(35) do |x|
+ x.report('COPY TEXT at once') do
+ get_rss.tap do |rss|
+ encoder = PG::TextEncoder::CopyRow.new
+ raw_conn.copy_data('copy benchmark_copy_format_text_1 from stdin', encoder) do
+ records.each do |row|
+ raw_conn.put_copy_data(row)
+ end
+ end
+ rss_log << get_rss - rss
+ end
+ end
+
+ x.report('COPY TEXT in batch') do
+ get_rss.tap do |rss|
+ encoder = PG::TextEncoder::CopyRow.new
+ records.each_slice(BATCH_SIZE) do |rows|
+ raw_conn.copy_data('copy benchmark_copy_format_text_2 from stdin', encoder) do
+ rows.each do |r|
+ raw_conn.put_copy_data(r)
+ end
+ end
+ end
+ rss_log << get_rss - rss
+ end
+ end
+
+ x.report('COPY CSV with CSV.generate_line') do
+ get_rss.tap do |rss|
+ records.each_slice(BATCH_SIZE) do |rows|
+ raw_conn.copy_data('copy benchmark_copy_format_csv_1 from stdin with (format csv)') do
+ rows.each do |row|
+ raw_conn.put_copy_data(CSV.generate_line(row, force_quotes: true))
+ end
+ end
+ end
+ rss_log << get_rss - rss
+ end
+ end
+
+ x.report('COPY CSV with Array#join') do
+ get_rss.tap do |rss|
+ records.each_slice(BATCH_SIZE) do |rows|
+ raw_conn.copy_data('copy benchmark_copy_format_csv_2 from stdin with (format csv)') do
+ rows.each do |row|
+ raw_conn.put_copy_data(row.map { |col| "\"#{col}\"" }.join(',') + "\n")
+ end
+ end
+ end
+ rss_log << get_rss - rss
+ end
+ end
+
+ x.report('BULK INSERT with generate SQLs') do
+ get_rss.tap do |rss|
+ records.each_slice(BATCH_SIZE) do |rows|
+ sql = 'insert into benchmark_bulk_insert_1 values '
+ sql << rows.map { |row| '(' + row.map { |col| "'#{col}'" }.join(',') + ')' }.join(',')
+ conn.execute(sql)
+ end
+ rss_log << get_rss - rss
+ end
+ end
+
+ sqls = []
+ records.each_slice(BATCH_SIZE) do |rows|
+ sql = 'insert into benchmark_bulk_insert_2 values '
+ sql << rows.map { |row| '(' + row.map { |col| "'#{col}'" }.join(',') + ')' }.join(',')
+ sqls << sql
+ end
+ x.report('BULK INSERT without generate SQLs') do
+ get_rss.tap do |rss|
+ sqls.each do |sql|
+ conn.execute(sql)
+ end
+ rss_log << get_rss - rss
+ end
+ end
+end
+
+
+puts "\n[RSS]\n"
+puts rss_log.map { |rss| format('+ %5d', rss) }.join("\n")
diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb
new file mode 100644
index 00000000..b06351ba
--- /dev/null
+++ b/spec/rails_helper.rb
@@ -0,0 +1,63 @@
+# This file is copied to spec/ when you run 'rails generate rspec:install'
+require 'spec_helper'
+ENV['RAILS_ENV'] ||= 'test'
+
+require File.expand_path('../config/environment', __dir__)
+
+# Prevent database truncation if the environment is production
+abort("The Rails environment is running in production mode!") if Rails.env.production?
+require 'rspec/rails'
+# Add additional requires below this line. Rails is not loaded until this point!
+
+# Requires supporting ruby files with custom matchers and macros, etc, in
+# spec/support/ and its subdirectories. Files matching `spec/**/*_spec.rb` are
+# run as spec files by default. This means that files in spec/support that end
+# in _spec.rb will both be required and run as specs, causing the specs to be
+# run twice. It is recommended that you do not name files matching this glob to
+# end with _spec.rb. You can configure this pattern with the --pattern
+# option on the command line or in ~/.rspec, .rspec or `.rspec-local`.
+#
+# The following line is provided for convenience purposes. It has the downside
+# of increasing the boot-up time by auto-requiring all files in the support
+# directory. Alternatively, in the individual `*_spec.rb` files, manually
+# require only the support files necessary.
+#
+# Dir[Rails.root.join('spec', 'support', '**', '*.rb')].each { |f| require f }
+
+# Checks for pending migrations and applies them before tests are run.
+# If you are not using ActiveRecord, you can remove these lines.
+begin
+ ActiveRecord::Migration.maintain_test_schema!
+rescue ActiveRecord::PendingMigrationError => e
+ puts e.to_s.strip
+ exit 1
+end
+RSpec.configure do |config|
+ # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures
+ config.fixture_path = "#{::Rails.root}/spec/fixtures"
+
+ # If you're not using ActiveRecord, or you'd prefer not to run each of your
+ # examples within a transaction, remove the following line or assign false
+ # instead of true.
+ config.use_transactional_fixtures = true
+
+ # RSpec Rails can automatically mix in different behaviours to your tests
+ # based on their file location, for example enabling you to call `get` and
+ # `post` in specs under `spec/controllers`.
+ #
+ # You can disable this behaviour by removing the line below, and instead
+ # explicitly tag your specs with their type, e.g.:
+ #
+ # RSpec.describe UsersController, :type => :controller do
+ # # ...
+ # end
+ #
+ # The different available types are documented in the features, such as in
+ # https://relishapp.com/rspec/rspec-rails/docs
+ config.infer_spec_type_from_file_location!
+
+ # Filter lines from Rails gems in backtraces.
+ config.filter_rails_from_backtrace!
+ # arbitrary gems may also be filtered via:
+ # config.filter_gems_from_backtrace("gem name")
+end
diff --git a/spec/schedule_loader_service_spec.rb b/spec/schedule_loader_service_spec.rb
new file mode 100644
index 00000000..7f168ff3
--- /dev/null
+++ b/spec/schedule_loader_service_spec.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe ScheduleLoader do
+ describe "loading small file" do
+ it { expect { ScheduleLoader.call("fixtures/small.json") }.to perform_under(2).sec.warmup(1).times.sample(2).times }
+ end
+
+ describe "loading medium file" do
+ it {expect { ScheduleLoader.call('fixtures/medium.json') }.to perform_under(10).sec.warmup(1).times.sample(2).times }
+ end
+
+ describe "loading large file" do
+ let(:file_name) { "fixtures/large.json" }
+
+ it {expect { ScheduleLoader.call('fixtures/large.json') }.to perform_under(60).sec.warmup(1).times.sample(2).times }
+ end
+end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
new file mode 100644
index 00000000..665ba9fa
--- /dev/null
+++ b/spec/spec_helper.rb
@@ -0,0 +1,98 @@
+require 'rspec-benchmark'
+# This file was generated by the `rails generate rspec:install` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+ include RSpec::Benchmark::Matchers
+ # rspec-expectations config goes here. You can use an alternate
+ # assertion/expectation library such as wrong or the stdlib/minitest
+ # assertions if you prefer.
+ config.expect_with :rspec do |expectations|
+ # This option will default to `true` in RSpec 4. It makes the `description`
+ # and `failure_message` of custom matchers include text for helper methods
+ # defined using `chain`, e.g.:
+ # be_bigger_than(2).and_smaller_than(4).description
+ # # => "be bigger than 2 and smaller than 4"
+ # ...rather than:
+ # # => "be bigger than 2"
+ expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+ end
+
+ # rspec-mocks config goes here. You can use an alternate test double
+ # library (such as bogus or mocha) by changing the `mock_with` option here.
+ config.mock_with :rspec do |mocks|
+ # Prevents you from mocking or stubbing a method that does not exist on
+ # a real object. This is generally recommended, and will default to
+ # `true` in RSpec 4.
+ mocks.verify_partial_doubles = true
+ end
+
+ # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+ # have no way to turn it off -- the option exists only for backwards
+ # compatibility in RSpec 3). It causes shared context metadata to be
+ # inherited by the metadata hash of host groups and examples, rather than
+ # triggering implicit auto-inclusion in groups with matching metadata.
+ config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+ # This allows you to limit a spec run to individual examples or groups
+ # you care about by tagging them with `:focus` metadata. When nothing
+ # is tagged with `:focus`, all examples get run. RSpec also provides
+ # aliases for `it`, `describe`, and `context` that include `:focus`
+ # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+ config.filter_run_when_matching :focus
+
+ # Allows RSpec to persist some state between runs in order to support
+ # the `--only-failures` and `--next-failure` CLI options. We recommend
+ # you configure your source control system to ignore this file.
+ config.example_status_persistence_file_path = "spec/examples.txt"
+
+ # Limits the available syntax to the non-monkey patched syntax that is
+ # recommended. For more details, see:
+ # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+ # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+ # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+ config.disable_monkey_patching!
+
+ # Many RSpec users commonly either run the entire suite or an individual
+ # file, and it's useful to allow more verbose output when running an
+ # individual spec file.
+ if config.files_to_run.one?
+ # Use the documentation formatter for detailed output,
+ # unless a formatter has already been configured
+ # (e.g. via a command-line flag).
+ config.default_formatter = "doc"
+ end
+
+ # Print the 10 slowest examples and example groups at the
+ # end of the spec run, to help surface which specs are running
+ # particularly slow.
+ config.profile_examples = 10
+
+ # Run specs in random order to surface order dependencies. If you find an
+ # order dependency and want to debug it, you can fix the order by providing
+ # the seed, which is printed after each run.
+ # --seed 1234
+ config.order = :random
+
+ # Seed global randomization in this process using the `--seed` CLI option.
+ # Setting this allows you to use `--seed` to deterministically reproduce
+ # test failures related to randomization by passing the same `--seed` value
+ # as the one that triggered the failure.
+ Kernel.srand config.seed
+=end
+end