From 984ad5f6fdbfe407d13c5d20e843a45f95e7f90a Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Sun, 1 Mar 2020 16:04:39 +0300 Subject: [PATCH 01/14] init commit. Docker-compose setup --- Dockerfile | 34 ++++++++++++++++++++++++++++++++++ config/database.yml | 12 ++++++++++-- docker-compose.yml | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..8d64459e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM ruby:2.6.3-alpine + +ENV APP_PATH /var/www/schedule_app +ENV TZ Europe/Moscow +ENV LANG ru_RU.UTF-8 +ENV LANGUAGE ru_RU.UTF-8 +ENV LC_ALL ru_RU.UTF-8 + +RUN apk update && apk add --no-cache yarn \ + build-base curl-dev git postgresql-dev \ + yaml-dev zlib-dev nodejs gcc g++ make busybox ctags \ + tzdata + +RUN cp /usr/share/zoneinfo/Europe/Moscow /etc/localtime && \ + echo "Europe/Moscow" > /etc/timezone + +RUN mkdir -p $APP_PATH + +WORKDIR $APP_PATH + +COPY Gemfile $APP_PATH/Gemfile +COPY Gemfile.lock $APP_PATH/Gemfile.lock +COPY package.json $APP_PATH/package.json + +RUN yarn global add node-sass +RUN yarn +RUN gem install bundler:2.0.2 +RUN bundle install + +COPY . $APP_PATH + +EXPOSE 3000/tcp +#CMD sh -c "bin/setup" +#CMD sh -c "bundle exec rails server -b 0.0.0.0 -p 3000" diff --git a/config/database.yml b/config/database.yml index e116cfa6..94612a2c 100644 --- a/config/database.yml +++ b/config/database.yml @@ -23,7 +23,11 @@ default: &default development: <<: *default - database: task-4_development + host: <%= ENV.fetch("DEV_SCHD_DATABASE_HOST", "localhost") %> + port: <%= ENV.fetch("DEV_SCHD_DATABASE_PORT", "5432") %> + username: <%= ENV.fetch("DEV_SCHD_DATABASE_USERNAME") %> + password: <%= ENV.fetch("DEV_SCHD_DATABASE_PASSWORD") %> + database: <%= "#{ENV.fetch("SCHD_DATABASE_NAME", "task-4")}_development" %> # The specified database role being used to connect to postgres. # To create additional roles in postgres see `$ createuser --help`. @@ -57,7 +61,11 @@ development: # Do not set this db to the same as development or production. test: <<: *default - database: task-4_test + host: <%= ENV.fetch("DEV_SCHD_DATABASE_HOST", "localhost") %> + port: <%= ENV.fetch("DEV_SCHD_DATABASE_PORT", "5432") %> + username: <%= ENV.fetch("DEV_SCHD_DATABASE_USERNAME") %> + password: <%= ENV.fetch("DEV_SCHD_DATABASE_PASSWORD") %> + database: <%= "#{ENV.fetch("SCHD_DATABASE_NAME", "task-4")}_test" %> # As with config/secrets.yml, you never want to store sensitive information, # like your database password, in your source code. If your source code is diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..cbb99765 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3' +services: + # PostgreSQL + db-postgresql: + image: postgres:9.6 + volumes: + - "/opt/shared/postgresql-nfap/data:/var/lib/postgresql/data" + environment: + POSTGRES_USER: rails + POSTGRES_PASSWORD: password + tty: true + stdin_open: true + ports: + - "15432:5432" + + # schedule app + schedule_app: + build: . + environment: + - "PAGER='busybox more'" + - "SSH_AUTH_SOCK=/tmp/ssh-agent" + - "DISABLE_SPRING=1" + - "DEV_SCHD_DATABASE_PORT=5432" + - "DEV_SCHD_DATABASE_HOST=db-postgresql" + - "DEV_SCHD_DATABASE_USERNAME=rails" + - "DEV_SCHD_DATABASE_PASSWORD=password" + - "SCHD_DATABASE_NAME=task-4" + #- "DEV_REDIS_HOST=redis" + tty: true + stdin_open: true + volumes: + - "$PWD:/var/www/schedule_app:Z" + - $SSH_AUTH_SOCK:/tmp/ssh-agent:Z + ports: + - "3078:3000" + depends_on: + - db-postgresql From ac385e69464755bfc39bddb774df7ab383c519ba Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Sun, 1 Mar 2020 22:03:39 +0300 Subject: [PATCH 02/14] rack-mini-profiler --- Gemfile | 8 ++++++++ Gemfile.lock | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/Gemfile b/Gemfile index e20b1260..eae657eb 100644 --- a/Gemfile +++ b/Gemfile @@ -17,6 +17,14 @@ group :development do # Access an interactive console on exception pages or by calling 'console' anywhere in the code. gem 'web-console', '>= 3.3.0' gem 'listen', '>= 3.0.5', '< 3.2' + # profilers + gem 'rack-mini-profiler', require: false + # For memory profiling + gem 'memory_profiler' + + # For call-stack profiling flamegraphs + gem 'flamegraph' + gem 'stackprof' end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index fccf6f5f..e196cf62 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -52,6 +52,7 @@ GEM crass (1.0.4) erubi (1.8.0) ffi (1.10.0) + flamegraph (0.9.5) globalid (0.4.2) activesupport (>= 4.2.0) i18n (1.6.0) @@ -67,6 +68,7 @@ GEM mini_mime (>= 0.1.1) marcel (0.3.3) mimemagic (~> 0.3.2) + memory_profiler (0.9.14) method_source (0.9.2) mimemagic (0.3.3) mini_mime (1.0.1) @@ -79,6 +81,8 @@ GEM pg (1.1.4) puma (3.12.1) rack (2.0.6) + rack-mini-profiler (1.1.6) + rack (>= 1.2.0) rack-test (1.1.0) rack (>= 1.0, < 3) rails (5.2.3) @@ -117,6 +121,7 @@ GEM actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) + stackprof (0.2.15) thor (0.20.3) thread_safe (0.3.6) tzinfo (1.2.5) @@ -136,10 +141,14 @@ PLATFORMS DEPENDENCIES bootsnap (>= 1.1.0) byebug + flamegraph listen (>= 3.0.5, < 3.2) + memory_profiler pg (>= 0.18, < 2.0) puma (~> 3.11) + rack-mini-profiler rails (~> 5.2.3) + stackprof tzinfo-data web-console (>= 3.3.0) From a1202fcfc22bb66b454a9e8cf09236e6214314cc Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Mon, 2 Mar 2020 02:03:59 +0300 Subject: [PATCH 03/14] rails panel --- Gemfile | 2 ++ Gemfile.lock | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/Gemfile b/Gemfile index eae657eb..898f4461 100644 --- a/Gemfile +++ b/Gemfile @@ -25,6 +25,8 @@ group :development do # For call-stack profiling flamegraphs gem 'flamegraph' gem 'stackprof' + # rails panel + gem 'meta_request' end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index e196cf62..79d94162 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -69,6 +69,9 @@ GEM marcel (0.3.3) mimemagic (~> 0.3.2) memory_profiler (0.9.14) + meta_request (0.7.2) + rack-contrib (>= 1.1, < 3) + railties (>= 3.0.0, < 7) method_source (0.9.2) mimemagic (0.3.3) mini_mime (1.0.1) @@ -81,6 +84,8 @@ GEM pg (1.1.4) puma (3.12.1) rack (2.0.6) + rack-contrib (2.1.0) + rack (~> 2.0) rack-mini-profiler (1.1.6) rack (>= 1.2.0) rack-test (1.1.0) @@ -144,6 +149,7 @@ DEPENDENCIES flamegraph listen (>= 3.0.5, < 3.2) memory_profiler + meta_request pg (>= 0.18, < 2.0) puma (~> 3.11) rack-mini-profiler From 09de722d6938b9615b635bab8f4c165e9dfc5c21 Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Wed, 4 Mar 2020 22:56:08 +0300 Subject: [PATCH 04/14] bullet && pg_hero --- Gemfile | 2 ++ Gemfile.lock | 5 +++++ config/environments/development.rb | 10 +++++++++- config/environments/test.rb | 6 +++++- docker-compose.yml | 12 ++++++++++++ 5 files changed, 33 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 898f4461..3e54b9ac 100644 --- a/Gemfile +++ b/Gemfile @@ -27,6 +27,8 @@ group :development do gem 'stackprof' # rails panel gem 'meta_request' + + gem "bullet" end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index 79d94162..50ba06c1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -47,6 +47,9 @@ GEM bootsnap (1.4.2) msgpack (~> 1.0) builder (3.2.3) + bullet (6.1.0) + activesupport (>= 3.0.0) + uniform_notifier (~> 1.11) byebug (11.0.1) concurrent-ruby (1.1.5) crass (1.0.4) @@ -131,6 +134,7 @@ GEM thread_safe (0.3.6) tzinfo (1.2.5) thread_safe (~> 0.1) + uniform_notifier (1.13.0) web-console (3.7.0) actionview (>= 5.0) activemodel (>= 5.0) @@ -145,6 +149,7 @@ PLATFORMS DEPENDENCIES bootsnap (>= 1.1.0) + bullet byebug flamegraph listen (>= 3.0.5, < 3.2) diff --git a/config/environments/development.rb b/config/environments/development.rb index 1311e3e4..4e7a42e2 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -1,5 +1,13 @@ Rails.application.configure do - # Settings specified here will take precedence over those in config/application.rb. + config.after_initialize do + Bullet.enable = true + Bullet.alert = true + Bullet.bullet_logger = true + Bullet.console = true + # Bullet.growl = true + Bullet.rails_logger = true + Bullet.add_footer = true + end # Settings specified here will take precedence over those in config/application.rb. # In the development environment your application's code is reloaded on # every request. This slows down response time but is perfect for development diff --git a/config/environments/test.rb b/config/environments/test.rb index 0a38fd3c..b31b09d3 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -1,5 +1,9 @@ Rails.application.configure do - # Settings specified here will take precedence over those in config/application.rb. + config.after_initialize do + Bullet.enable = true + Bullet.bullet_logger = true + Bullet.raise = true # raise an error if n+1 query occurs + end # Settings specified here will take precedence over those in config/application.rb. # The test environment is used exclusively to run your application's # test suite. You never need to work with it otherwise. Remember that diff --git a/docker-compose.yml b/docker-compose.yml index cbb99765..4c31c308 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,18 @@ services: ports: - "15432:5432" + pghero: + image: ankane/pghero + ports: + - "8080:8080" + environment: +# DATABASE_URL: "postgres://elreg:elreg@172.16.101.62:5432/fgiski_preproduction" + DATABASE_URL: "postgres://rails:password@db-postgresql:5432/task-4_development" + tty: true + stdin_open: true + depends_on: + - db-postgresql + # schedule app schedule_app: build: . From 8806e81e15a587a66b90e772632f6f885ba8cf1a Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Thu, 5 Mar 2020 00:48:20 +0300 Subject: [PATCH 05/14] load specs --- .rspec | 1 + Gemfile | 27 ++++---- Gemfile.lock | 37 +++++++++++ app/services/schedule_loader.rb | 36 ++++++++++ config/initializers/rack_profiler.rb | 8 +++ lib/tasks/utils.rake | 31 +-------- spec/rails_helper.rb | 63 ++++++++++++++++++ spec/schedule_loader_service_spec.rb | 19 ++++++ spec/spec_helper.rb | 98 ++++++++++++++++++++++++++++ 9 files changed, 278 insertions(+), 42 deletions(-) create mode 100644 .rspec create mode 100644 app/services/schedule_loader.rb create mode 100644 config/initializers/rack_profiler.rb create mode 100644 spec/rails_helper.rb create mode 100644 spec/schedule_loader_service_spec.rb create mode 100644 spec/spec_helper.rb diff --git a/.rspec b/.rspec new file mode 100644 index 00000000..c99d2e73 --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/Gemfile b/Gemfile index 3e54b9ac..91223b60 100644 --- a/Gemfile +++ b/Gemfile @@ -8,30 +8,33 @@ gem 'pg', '>= 0.18', '< 2.0' gem 'puma', '~> 3.11' gem 'bootsnap', '>= 1.1.0', require: false + +# profilers +gem 'rack-mini-profiler', require: false +# For memory profiling +gem 'memory_profiler' + +# For call-stack profiling flamegraphs +gem 'flamegraph' +gem 'stackprof' +gem "bullet" +gem 'meta_request' + group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] end group :development do + gem 'pry' # Access an interactive console on exception pages or by calling 'console' anywhere in the code. gem 'web-console', '>= 3.3.0' gem 'listen', '>= 3.0.5', '< 3.2' - # profilers - gem 'rack-mini-profiler', require: false - # For memory profiling - gem 'memory_profiler' - - # For call-stack profiling flamegraphs - gem 'flamegraph' - gem 'stackprof' - # rails panel - gem 'meta_request' - - gem "bullet" end group :test do + gem 'rspec-rails', '~> 3.8' + gem 'rspec-benchmark' end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem diff --git a/Gemfile.lock b/Gemfile.lock index 50ba06c1..64629a04 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -43,6 +43,9 @@ GEM minitest (~> 5.1) tzinfo (~> 1.1) arel (9.0.0) + benchmark-malloc (0.1.0) + benchmark-perf (0.5.0) + benchmark-trend (0.3.0) bindex (0.6.0) bootsnap (1.4.2) msgpack (~> 1.0) @@ -51,8 +54,10 @@ GEM activesupport (>= 3.0.0) uniform_notifier (~> 1.11) byebug (11.0.1) + coderay (1.1.2) concurrent-ruby (1.1.5) crass (1.0.4) + diff-lcs (1.3) erubi (1.8.0) ffi (1.10.0) flamegraph (0.9.5) @@ -85,6 +90,9 @@ GEM nokogiri (1.10.2) mini_portile2 (~> 2.4.0) pg (1.1.4) + pry (0.12.2) + coderay (~> 1.1.0) + method_source (~> 0.9.0) puma (3.12.1) rack (2.0.6) rack-contrib (2.1.0) @@ -121,6 +129,32 @@ GEM rb-fsevent (0.10.3) rb-inotify (0.10.0) ffi (~> 1.0) + rspec (3.9.0) + rspec-core (~> 3.9.0) + rspec-expectations (~> 3.9.0) + rspec-mocks (~> 3.9.0) + rspec-benchmark (0.5.1) + benchmark-malloc (~> 0.1.0) + benchmark-perf (~> 0.5.0) + benchmark-trend (~> 0.3.0) + rspec (>= 3.0.0, < 4.0.0) + rspec-core (3.9.1) + rspec-support (~> 3.9.1) + rspec-expectations (3.9.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.9.0) + rspec-mocks (3.9.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.9.0) + rspec-rails (3.9.0) + actionpack (>= 3.0) + activesupport (>= 3.0) + railties (>= 3.0) + rspec-core (~> 3.9.0) + rspec-expectations (~> 3.9.0) + rspec-mocks (~> 3.9.0) + rspec-support (~> 3.9.0) + rspec-support (3.9.2) ruby_dep (1.5.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) @@ -156,9 +190,12 @@ DEPENDENCIES memory_profiler meta_request pg (>= 0.18, < 2.0) + pry puma (~> 3.11) rack-mini-profiler rails (~> 5.2.3) + rspec-benchmark + rspec-rails (~> 3.8) stackprof tzinfo-data web-console (>= 3.3.0) diff --git a/app/services/schedule_loader.rb b/app/services/schedule_loader.rb new file mode 100644 index 00000000..a8444139 --- /dev/null +++ b/app/services/schedule_loader.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class ScheduleLoader + def self.call(file_name) + json = JSON.parse(File.read(file_name)) + + ActiveRecord::Base.transaction do + City.delete_all + Bus.delete_all + Service.delete_all + Trip.delete_all + ActiveRecord::Base.connection.execute('delete from buses_services;') + + json.each do |trip| + from = City.find_or_create_by(name: trip['from']) + to = City.find_or_create_by(name: trip['to']) + services = [] + trip['bus']['services'].each do |service| + s = Service.find_or_create_by(name: service) + services << s + end + bus = Bus.find_or_create_by(number: trip['bus']['number']) + bus.update(model: trip['bus']['model'], services: services) + + Trip.create!( + from: from, + to: to, + bus: bus, + start_time: trip['start_time'], + duration_minutes: trip['duration_minutes'], + price_cents: trip['price_cents'], + ) + end + end + end +end diff --git a/config/initializers/rack_profiler.rb b/config/initializers/rack_profiler.rb new file mode 100644 index 00000000..96b6cc98 --- /dev/null +++ b/config/initializers/rack_profiler.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +if Rails.env.development? + require "rack-mini-profiler" + + # initialization is skipped so trigger it + Rack::MiniProfilerRails.initialize!(Rails.application) +end diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 540fe871..2726a064 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,34 +1,5 @@ # Наивная загрузка данных из json-файла в БД # rake reload_json[fixtures/small.json] task :reload_json, [:file_name] => :environment do |_task, args| - json = JSON.parse(File.read(args.file_name)) - - ActiveRecord::Base.transaction do - City.delete_all - Bus.delete_all - Service.delete_all - Trip.delete_all - ActiveRecord::Base.connection.execute('delete from buses_services;') - - json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) - services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s - end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) - - Trip.create!( - from: from, - to: to, - bus: bus, - start_time: trip['start_time'], - duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], - ) - end - end + ScheduleLoader.call(file_name) end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb new file mode 100644 index 00000000..b06351ba --- /dev/null +++ b/spec/rails_helper.rb @@ -0,0 +1,63 @@ +# This file is copied to spec/ when you run 'rails generate rspec:install' +require 'spec_helper' +ENV['RAILS_ENV'] ||= 'test' + +require File.expand_path('../config/environment', __dir__) + +# Prevent database truncation if the environment is production +abort("The Rails environment is running in production mode!") if Rails.env.production? +require 'rspec/rails' +# Add additional requires below this line. Rails is not loaded until this point! + +# Requires supporting ruby files with custom matchers and macros, etc, in +# spec/support/ and its subdirectories. Files matching `spec/**/*_spec.rb` are +# run as spec files by default. This means that files in spec/support that end +# in _spec.rb will both be required and run as specs, causing the specs to be +# run twice. It is recommended that you do not name files matching this glob to +# end with _spec.rb. You can configure this pattern with the --pattern +# option on the command line or in ~/.rspec, .rspec or `.rspec-local`. +# +# The following line is provided for convenience purposes. It has the downside +# of increasing the boot-up time by auto-requiring all files in the support +# directory. Alternatively, in the individual `*_spec.rb` files, manually +# require only the support files necessary. +# +# Dir[Rails.root.join('spec', 'support', '**', '*.rb')].each { |f| require f } + +# Checks for pending migrations and applies them before tests are run. +# If you are not using ActiveRecord, you can remove these lines. +begin + ActiveRecord::Migration.maintain_test_schema! +rescue ActiveRecord::PendingMigrationError => e + puts e.to_s.strip + exit 1 +end +RSpec.configure do |config| + # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures + config.fixture_path = "#{::Rails.root}/spec/fixtures" + + # If you're not using ActiveRecord, or you'd prefer not to run each of your + # examples within a transaction, remove the following line or assign false + # instead of true. + config.use_transactional_fixtures = true + + # RSpec Rails can automatically mix in different behaviours to your tests + # based on their file location, for example enabling you to call `get` and + # `post` in specs under `spec/controllers`. + # + # You can disable this behaviour by removing the line below, and instead + # explicitly tag your specs with their type, e.g.: + # + # RSpec.describe UsersController, :type => :controller do + # # ... + # end + # + # The different available types are documented in the features, such as in + # https://relishapp.com/rspec/rspec-rails/docs + config.infer_spec_type_from_file_location! + + # Filter lines from Rails gems in backtraces. + config.filter_rails_from_backtrace! + # arbitrary gems may also be filtered via: + # config.filter_gems_from_backtrace("gem name") +end diff --git a/spec/schedule_loader_service_spec.rb b/spec/schedule_loader_service_spec.rb new file mode 100644 index 00000000..f8dcfdad --- /dev/null +++ b/spec/schedule_loader_service_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe ScheduleLoader do + describe "loading small file" do + it { expect { ScheduleLoader.call("fixtures/small.json") }.to perform_under(0.005).sec } + end + + # describe "loading medium file" do + # it {expect { ScheduleLoader.call('fixtures/medium.json') }.to perform_under(60).sec } + # end + + # describe "loading large file" do + # let(:file_name) { "fixtures/large.json" } + # + # it {expect { ScheduleLoader.call('fixtures/large.json') }.to perform_under(60).sec } + # end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 00000000..665ba9fa --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,98 @@ +require 'rspec-benchmark' +# This file was generated by the `rails generate rspec:install` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +RSpec.configure do |config| + include RSpec::Benchmark::Matchers + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + +# The settings below are suggested to provide a good initial experience +# with RSpec, but feel free to customize to your heart's content. +=begin + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "spec/examples.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ + # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ + # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode + config.disable_monkey_patching! + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + if config.files_to_run.one? + # Use the documentation formatter for detailed output, + # unless a formatter has already been configured + # (e.g. via a command-line flag). + config.default_formatter = "doc" + end + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed +=end +end From 0d1279f281d74e0b44da6d823f7d3fe57cc1995b Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Sun, 8 Mar 2020 04:42:07 +0300 Subject: [PATCH 06/14] composite foreign key (model, number) --- Gemfile | 3 +++ Gemfile.lock | 6 +++++ app/models/bus.rb | 6 +++-- app/models/service.rb | 2 +- app/models/trip.rb | 2 +- .../20200307214701_change_bus_primary_key.rb | 14 ++++++++++++ .../20200308003029_change_buses_services.rb | 13 +++++++++++ ...20200308003123_change_trips_foreign_key.rb | 13 +++++++++++ db/schema.rb | 15 +++++++++---- docker-compose.yml | 22 +++++++++---------- 10 files changed, 77 insertions(+), 19 deletions(-) create mode 100755 db/migrate/20200307214701_change_bus_primary_key.rb create mode 100755 db/migrate/20200308003029_change_buses_services.rb create mode 100755 db/migrate/20200308003123_change_trips_foreign_key.rb diff --git a/Gemfile b/Gemfile index 91223b60..d4fc989e 100644 --- a/Gemfile +++ b/Gemfile @@ -20,6 +20,9 @@ gem 'stackprof' gem "bullet" gem 'meta_request' +gem 'activerecord-import', require: false +gem 'composite_primary_keys', '=11.2.0' + group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] diff --git a/Gemfile.lock b/Gemfile.lock index 64629a04..9e4dd693 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -33,6 +33,8 @@ GEM activemodel (= 5.2.3) activesupport (= 5.2.3) arel (>= 9.0) + activerecord-import (1.0.4) + activerecord (>= 3.2) activestorage (5.2.3) actionpack (= 5.2.3) activerecord (= 5.2.3) @@ -55,6 +57,8 @@ GEM uniform_notifier (~> 1.11) byebug (11.0.1) coderay (1.1.2) + composite_primary_keys (11.2.0) + activerecord (~> 5.2.1) concurrent-ruby (1.1.5) crass (1.0.4) diff-lcs (1.3) @@ -182,9 +186,11 @@ PLATFORMS ruby DEPENDENCIES + activerecord-import bootsnap (>= 1.1.0) bullet byebug + composite_primary_keys (= 11.2.0) flamegraph listen (>= 3.0.5, < 3.2) memory_profiler diff --git a/app/models/bus.rb b/app/models/bus.rb index 1dcc54cb..c306aa6a 100644 --- a/app/models/bus.rb +++ b/app/models/bus.rb @@ -12,8 +12,10 @@ class Bus < ApplicationRecord 'Газель', ].freeze - has_many :trips - has_and_belongs_to_many :services, join_table: :buses_services + self.primary_keys = :number, :model + + has_many :trips, :foreign_key => [:number, :model] + has_and_belongs_to_many :services, join_table: :buses_services, :foreign_key => [:number, :model] validates :number, presence: true, uniqueness: true validates :model, inclusion: { in: MODELS } diff --git a/app/models/service.rb b/app/models/service.rb index 9cbb2a32..44e79470 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -12,7 +12,7 @@ class Service < ApplicationRecord 'Можно не печатать билет', ].freeze - has_and_belongs_to_many :buses, join_table: :buses_services + has_and_belongs_to_many :buses, join_table: :buses_services, :foreign_key => :service_id, association_foreign_key: [:number, :model] validates :name, presence: true validates :name, inclusion: { in: SERVICES } diff --git a/app/models/trip.rb b/app/models/trip.rb index 9d63dfff..675283d9 100644 --- a/app/models/trip.rb +++ b/app/models/trip.rb @@ -3,7 +3,7 @@ class Trip < ApplicationRecord belongs_to :from, class_name: 'City' belongs_to :to, class_name: 'City' - belongs_to :bus + belongs_to :bus, :foreign_key => [:number, :model] validates :from, presence: true validates :to, presence: true diff --git a/db/migrate/20200307214701_change_bus_primary_key.rb b/db/migrate/20200307214701_change_bus_primary_key.rb new file mode 100755 index 00000000..2a41e099 --- /dev/null +++ b/db/migrate/20200307214701_change_bus_primary_key.rb @@ -0,0 +1,14 @@ +class ChangeBusPrimaryKey < ActiveRecord::Migration[5.2] + def up + remove_column :buses, :id + execute 'ALTER TABLE buses ADD CONSTRAINT buses_pkey PRIMARY KEY ("number",model);' + add_index :buses, [:number, :model], unique: true + end + + def down + execute 'ALTER TABLE buses DROP CONSTRAINT buses_pkey;' + add_column :buses, :id, :primary_key + + remove_index :buses, [:number, :model] + end +end diff --git a/db/migrate/20200308003029_change_buses_services.rb b/db/migrate/20200308003029_change_buses_services.rb new file mode 100755 index 00000000..9e6162e7 --- /dev/null +++ b/db/migrate/20200308003029_change_buses_services.rb @@ -0,0 +1,13 @@ +class ChangeBusesServices < ActiveRecord::Migration[5.2] + def up + add_column :buses_services, :model, :string + add_column :buses_services, :number, :string + execute 'ALTER TABLE buses_services ADD CONSTRAINT buses_services_fk FOREIGN KEY (model,"number") REFERENCES buses(model,"number");' + end + + def down + execute 'ALTER TABLE buses_services DROP CONSTRAINT buses_services_fk;' + remove_column :buses_services, :number + remove_column :buses_services, :model + end +end diff --git a/db/migrate/20200308003123_change_trips_foreign_key.rb b/db/migrate/20200308003123_change_trips_foreign_key.rb new file mode 100755 index 00000000..2fe30afc --- /dev/null +++ b/db/migrate/20200308003123_change_trips_foreign_key.rb @@ -0,0 +1,13 @@ +class ChangeTripsForeignKey < ActiveRecord::Migration[5.2] + def up + add_column :trips, :model, :string + add_column :trips, :number, :string + execute 'ALTER TABLE trips ADD CONSTRAINT trips_fk FOREIGN KEY (model,"number") REFERENCES buses(model,"number");' + end + + def down + execute 'ALTER TABLE trips DROP CONSTRAINT trips_fk;' + remove_column :trips, :model + remove_column :trips, :number + end +end diff --git a/db/schema.rb b/db/schema.rb index f6921e45..059c41d8 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,19 +10,22 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_03_30_193044) do +ActiveRecord::Schema.define(version: 2020_03_08_003123) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" - create_table "buses", force: :cascade do |t| - t.string "number" - t.string "model" + create_table "buses", primary_key: ["number", "model"], force: :cascade do |t| + t.string "number", null: false + t.string "model", null: false + t.index ["number", "model"], name: "index_buses_on_number_and_model", unique: true end create_table "buses_services", force: :cascade do |t| t.integer "bus_id" t.integer "service_id" + t.string "model" + t.string "number" end create_table "cities", force: :cascade do |t| @@ -40,6 +43,10 @@ t.integer "duration_minutes" t.integer "price_cents" t.integer "bus_id" + t.string "model" + t.string "number" end + add_foreign_key "buses_services", "buses", column: "model", primary_key: "model", name: "buses_services_fk" + add_foreign_key "trips", "buses", column: "model", primary_key: "model", name: "trips_fk" end diff --git a/docker-compose.yml b/docker-compose.yml index 4c31c308..d7dee867 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,17 +13,17 @@ services: ports: - "15432:5432" - pghero: - image: ankane/pghero - ports: - - "8080:8080" - environment: -# DATABASE_URL: "postgres://elreg:elreg@172.16.101.62:5432/fgiski_preproduction" - DATABASE_URL: "postgres://rails:password@db-postgresql:5432/task-4_development" - tty: true - stdin_open: true - depends_on: - - db-postgresql +# pghero: +# image: ankane/pghero +# ports: +# - "8080:8080" +# environment: +## DATABASE_URL: "postgres://elreg:elreg@172.16.101.62:5432/fgiski_preproduction" +# DATABASE_URL: "postgres://rails:password@db-postgresql:5432/task-4_development" +# tty: true +# stdin_open: true +# depends_on: +# - db-postgresql # schedule app schedule_app: From dd6800d812e268508d67360f06ba81931b56eb3e Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Mon, 9 Mar 2020 15:35:05 +0300 Subject: [PATCH 07/14] put_copy_data large_file 53 sec --- Gemfile | 1 + Gemfile.lock | 2 + app/services/schedule_loader.rb | 95 +++++++++++++++---- .../20200308003029_change_buses_services.rb | 2 + ...20200308003123_change_trips_foreign_key.rb | 2 + db/migrate/20200308114605_write_services.rb | 26 +++++ db/schema.rb | 4 +- docker-compose.yml | 22 ++--- lib/tasks/utils.rake | 2 +- 9 files changed, 120 insertions(+), 36 deletions(-) create mode 100755 db/migrate/20200308114605_write_services.rb diff --git a/Gemfile b/Gemfile index d4fc989e..8d6053be 100644 --- a/Gemfile +++ b/Gemfile @@ -22,6 +22,7 @@ gem 'meta_request' gem 'activerecord-import', require: false gem 'composite_primary_keys', '=11.2.0' +gem 'oj' group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console diff --git a/Gemfile.lock b/Gemfile.lock index 9e4dd693..50c8b2a5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -93,6 +93,7 @@ GEM nio4r (2.3.1) nokogiri (1.10.2) mini_portile2 (~> 2.4.0) + oj (3.10.5) pg (1.1.4) pry (0.12.2) coderay (~> 1.1.0) @@ -195,6 +196,7 @@ DEPENDENCIES listen (>= 3.0.5, < 3.2) memory_profiler meta_request + oj pg (>= 0.18, < 2.0) pry puma (~> 3.11) diff --git a/app/services/schedule_loader.rb b/app/services/schedule_loader.rb index a8444139..b8ee4bad 100644 --- a/app/services/schedule_loader.rb +++ b/app/services/schedule_loader.rb @@ -1,36 +1,89 @@ +require 'active_record' +require 'activerecord-import' + # frozen_string_literal: true +STrip = Struct.new(:from, :to, :start_time, :duration_minutes, :price_cents, :number, :model) +SBus = Struct.new(:number, :model, :services) class ScheduleLoader def self.call(file_name) - json = JSON.parse(File.read(file_name)) + @cities = {} + @services = {} + @buses = Set.new + + https : // koshigoe.github.io / postgresql / ruby / 2018 / 10 / 31 / bulk - insert - vs - copy - in -postgres.html + @@services = Service.pluck(:name, :id).to_h ActiveRecord::Base.transaction do + trips_command = "copy trips (from_id, to_id, start_time, duration_minutes, price_cents, model, number) from stdin with csv delimiter ';'" + bus_command = "copy buses (model, number) from stdin with csv delimiter ';'" + buses_services_command = "copy buses_services (service_id, model, number) from stdin with csv delimiter ';'" + + ActiveRecord::Base.connection.execute('delete from buses_services;') + Trip.delete_all City.delete_all Bus.delete_all - Service.delete_all - Trip.delete_all ActiveRecord::Base.connection.execute('delete from buses_services;') + ActiveRecord::Base.connection.reset_pk_sequence!('cities') + ActiveRecord::Base.connection.reset_pk_sequence!('buses') + + File.open(file_name) do |ff| + nesting = 0 + str = +'' + + until ff.eof? + ch = ff.read(1) # читаем по одному символу + if ch == '{' # начинается объект, повышается вложенность + nesting += 1 + str << ch + elsif ch == '}' # заканчивается объект, понижается вложенность + nesting -= 1 + str << ch + if nesting == 0 # если закончился объкет уровня trip, парсим и импортируем его + + trip = Oj.load(str) - json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) - services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s + unless @buses.include?("#{trip['bus']['number']}-#{trip['bus']['model']}") + ActiveRecord::Base.connection.raw_connection.copy_data bus_command do + ActiveRecord::Base.connection.raw_connection.put_copy_data("#{trip['bus']['number']};#{trip['bus']['model']}\n") + end + ActiveRecord::Base.connection.raw_connection.copy_data buses_services_command do + trip["bus"]["services"].map do |srv| + ActiveRecord::Base.connection.raw_connection.put_copy_data("#{@@services[srv]};#{trip['bus']['number']};#{trip['bus']['model']}\n") + end + end + end + + ActiveRecord::Base.connection.raw_connection.copy_data trips_command do + import_trip(trip) + end + str = +'' + end + elsif nesting >= 1 + str << ch + end end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) - - Trip.create!( - from: from, - to: to, - bus: bus, - start_time: trip['start_time'], - duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], - ) end + City.import @cities.map {|cc| ({id: cc[1], name: cc[0]})} + end + end + + def self.import_trip(trip) + # binding.pry + from_id = @cities[trip['from']] + unless from_id + from_id = @cities.size + 1 + @cities[trip['from']] = from_id end + + to_id = @cities[trip['to']] + unless to_id + to_id = @cities.size + 1 + @cities[trip['to']] = to_id + end + + @buses << "#{trip['bus']['number']}-#{trip['bus']['model']}" + + ActiveRecord::Base.connection.raw_connection.put_copy_data("#{@cities[trip['from']]};#{@cities[trip['to']]};#{trip['start_time']};#{trip['duration_minutes']};#{trip['price_cents']};#{trip['bus']['number']};#{trip['bus']['model']}\n") end end diff --git a/db/migrate/20200308003029_change_buses_services.rb b/db/migrate/20200308003029_change_buses_services.rb index 9e6162e7..70693844 100755 --- a/db/migrate/20200308003029_change_buses_services.rb +++ b/db/migrate/20200308003029_change_buses_services.rb @@ -2,6 +2,7 @@ class ChangeBusesServices < ActiveRecord::Migration[5.2] def up add_column :buses_services, :model, :string add_column :buses_services, :number, :string + remove_column :buses_services, :bus_id execute 'ALTER TABLE buses_services ADD CONSTRAINT buses_services_fk FOREIGN KEY (model,"number") REFERENCES buses(model,"number");' end @@ -9,5 +10,6 @@ def down execute 'ALTER TABLE buses_services DROP CONSTRAINT buses_services_fk;' remove_column :buses_services, :number remove_column :buses_services, :model + add_column :trips, :bus_id, :integer end end diff --git a/db/migrate/20200308003123_change_trips_foreign_key.rb b/db/migrate/20200308003123_change_trips_foreign_key.rb index 2fe30afc..84585d69 100755 --- a/db/migrate/20200308003123_change_trips_foreign_key.rb +++ b/db/migrate/20200308003123_change_trips_foreign_key.rb @@ -2,6 +2,7 @@ class ChangeTripsForeignKey < ActiveRecord::Migration[5.2] def up add_column :trips, :model, :string add_column :trips, :number, :string + remove_column :trips, :bus_id execute 'ALTER TABLE trips ADD CONSTRAINT trips_fk FOREIGN KEY (model,"number") REFERENCES buses(model,"number");' end @@ -9,5 +10,6 @@ def down execute 'ALTER TABLE trips DROP CONSTRAINT trips_fk;' remove_column :trips, :model remove_column :trips, :number + add_column :trips, :bus_id, :integer end end diff --git a/db/migrate/20200308114605_write_services.rb b/db/migrate/20200308114605_write_services.rb new file mode 100755 index 00000000..711a247d --- /dev/null +++ b/db/migrate/20200308114605_write_services.rb @@ -0,0 +1,26 @@ +class WriteServices < ActiveRecord::Migration[5.2] + + def up + Service.delete_all + [ + 'WiFi', + 'Туалет', + 'Работающий туалет', + 'Ремни безопасности', + 'Кондиционер общий', + 'Кондиционер Индивидуальный', + 'Телевизор общий', + 'Телевизор индивидуальный', + 'Стюардесса', + 'Можно не печатать билет' + ].each do |name| + ss = Service.new(name: name) + ss.save! + end + + end + + def down + Service.delete_all + end +end diff --git a/db/schema.rb b/db/schema.rb index 059c41d8..e7f5f359 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_03_08_003123) do +ActiveRecord::Schema.define(version: 2020_03_08_114605) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -22,7 +22,6 @@ end create_table "buses_services", force: :cascade do |t| - t.integer "bus_id" t.integer "service_id" t.string "model" t.string "number" @@ -42,7 +41,6 @@ t.string "start_time" t.integer "duration_minutes" t.integer "price_cents" - t.integer "bus_id" t.string "model" t.string "number" end diff --git a/docker-compose.yml b/docker-compose.yml index d7dee867..4c31c308 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,17 +13,17 @@ services: ports: - "15432:5432" -# pghero: -# image: ankane/pghero -# ports: -# - "8080:8080" -# environment: -## DATABASE_URL: "postgres://elreg:elreg@172.16.101.62:5432/fgiski_preproduction" -# DATABASE_URL: "postgres://rails:password@db-postgresql:5432/task-4_development" -# tty: true -# stdin_open: true -# depends_on: -# - db-postgresql + pghero: + image: ankane/pghero + ports: + - "8080:8080" + environment: +# DATABASE_URL: "postgres://elreg:elreg@172.16.101.62:5432/fgiski_preproduction" + DATABASE_URL: "postgres://rails:password@db-postgresql:5432/task-4_development" + tty: true + stdin_open: true + depends_on: + - db-postgresql # schedule app schedule_app: diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 2726a064..9fc01666 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,5 +1,5 @@ # Наивная загрузка данных из json-файла в БД # rake reload_json[fixtures/small.json] task :reload_json, [:file_name] => :environment do |_task, args| - ScheduleLoader.call(file_name) + ScheduleLoader.call(args.file_name) end From 735a6ce067f53ace889258578f5698a8d27380f1 Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Mon, 9 Mar 2020 18:05:07 +0300 Subject: [PATCH 08/14] WIP --- app/services/schedule_loader.rb | 68 ++++++++++++++++++---------- spec/schedule_loader_service_spec.rb | 18 ++++---- 2 files changed, 52 insertions(+), 34 deletions(-) diff --git a/app/services/schedule_loader.rb b/app/services/schedule_loader.rb index b8ee4bad..632f370e 100644 --- a/app/services/schedule_loader.rb +++ b/app/services/schedule_loader.rb @@ -11,19 +11,31 @@ def self.call(file_name) @services = {} @buses = Set.new - https : // koshigoe.github.io / postgresql / ruby / 2018 / 10 / 31 / bulk - insert - vs - copy - in -postgres.html - @@services = Service.pluck(:name, :id).to_h + @trips_command = "copy trips (from_id, to_id, start_time, duration_minutes, price_cents, model, number) from stdin with csv delimiter ';'" + @trips_command_raw = "copy trips (from_id, to_id, start_time, duration_minutes, price_cents, model, number) from stdin" + + @bus_command = "copy buses (model, number) from stdin with csv delimiter ';'" + @buses_services_command = "copy buses_services (service_id, model, number) from stdin with csv delimiter ';'" + @buses_services_command_raw = "copy buses_services (service_id, model, number) from stdin" + + + @@services = {} + Service.pluck(:name, :id).map{|sr| @@services[sr[0].to_sym] = sr[1]} ActiveRecord::Base.transaction do - trips_command = "copy trips (from_id, to_id, start_time, duration_minutes, price_cents, model, number) from stdin with csv delimiter ';'" - bus_command = "copy buses (model, number) from stdin with csv delimiter ';'" - buses_services_command = "copy buses_services (service_id, model, number) from stdin with csv delimiter ';'" + + time_start = Time.now ActiveRecord::Base.connection.execute('delete from buses_services;') - Trip.delete_all - City.delete_all - Bus.delete_all + ActiveRecord::Base.connection.execute('delete from buses_services;') + ActiveRecord::Base.connection.execute('delete from trips;') + ActiveRecord::Base.connection.execute('delete from cities;') + ActiveRecord::Base.connection.execute('delete from buses;') + + # Trip.delete_all + # City.delete_all + # Bus.delete_all ActiveRecord::Base.connection.reset_pk_sequence!('cities') ActiveRecord::Base.connection.reset_pk_sequence!('buses') @@ -43,20 +55,8 @@ def self.call(file_name) trip = Oj.load(str) - unless @buses.include?("#{trip['bus']['number']}-#{trip['bus']['model']}") - ActiveRecord::Base.connection.raw_connection.copy_data bus_command do - ActiveRecord::Base.connection.raw_connection.put_copy_data("#{trip['bus']['number']};#{trip['bus']['model']}\n") - end - ActiveRecord::Base.connection.raw_connection.copy_data buses_services_command do - trip["bus"]["services"].map do |srv| - ActiveRecord::Base.connection.raw_connection.put_copy_data("#{@@services[srv]};#{trip['bus']['number']};#{trip['bus']['model']}\n") - end - end - end - - ActiveRecord::Base.connection.raw_connection.copy_data trips_command do - import_trip(trip) - end + import_trip(trip) + str = +'' end elsif nesting >= 1 @@ -65,11 +65,12 @@ def self.call(file_name) end end City.import @cities.map {|cc| ({id: cc[1], name: cc[0]})} + end_time = Time.now + pp (end_time - time_start) end end def self.import_trip(trip) - # binding.pry from_id = @cities[trip['from']] unless from_id from_id = @cities.size + 1 @@ -82,8 +83,25 @@ def self.import_trip(trip) @cities[trip['to']] = to_id end - @buses << "#{trip['bus']['number']}-#{trip['bus']['model']}" + bus_key = "#{trip['bus']['number']}-#{trip['bus']['model']}" - ActiveRecord::Base.connection.raw_connection.put_copy_data("#{@cities[trip['from']]};#{@cities[trip['to']]};#{trip['start_time']};#{trip['duration_minutes']};#{trip['price_cents']};#{trip['bus']['number']};#{trip['bus']['model']}\n") + bus_number = trip['bus']['number'] + bus_model = trip['bus']['model'] + + unless @buses.include?(bus_key) + ActiveRecord::Base.connection.raw_connection.copy_data @bus_command do + ActiveRecord::Base.connection.raw_connection.put_copy_data("#{bus_number};#{bus_model}\n") + end + @buses << bus_key + ActiveRecord::Base.connection.raw_connection.copy_data @buses_services_command do + trip["bus"]["services"].map do |srv| + ActiveRecord::Base.connection.raw_connection.put_copy_data("#{@@services[srv.to_sym]};#{bus_number};#{bus_model}\n") + end + end + end + + ActiveRecord::Base.connection.raw_connection.copy_data @trips_command do + ActiveRecord::Base.connection.raw_connection.put_copy_data("#{from_id};#{to_id};#{trip['start_time']};#{trip['duration_minutes']};#{trip['price_cents']};#{bus_number};#{bus_model}\n") + end end end diff --git a/spec/schedule_loader_service_spec.rb b/spec/schedule_loader_service_spec.rb index f8dcfdad..b422198d 100644 --- a/spec/schedule_loader_service_spec.rb +++ b/spec/schedule_loader_service_spec.rb @@ -4,16 +4,16 @@ RSpec.describe ScheduleLoader do describe "loading small file" do - it { expect { ScheduleLoader.call("fixtures/small.json") }.to perform_under(0.005).sec } + it { expect { ScheduleLoader.call("fixtures/small.json") }.to perform_under(2).sec } end - # describe "loading medium file" do - # it {expect { ScheduleLoader.call('fixtures/medium.json') }.to perform_under(60).sec } - # end + describe "loading medium file" do + it {expect { ScheduleLoader.call('fixtures/medium.json') }.to perform_under(60).sec.warmup(1).times.sample(1).times } + end + + describe "loading large file" do + let(:file_name) { "fixtures/large.json" } - # describe "loading large file" do - # let(:file_name) { "fixtures/large.json" } - # - # it {expect { ScheduleLoader.call('fixtures/large.json') }.to perform_under(60).sec } - # end + it {expect { ScheduleLoader.call('fixtures/large.json') }.to perform_under(60).sec.warmup(1).times.sample(1).times } + end end From 565449904209324d96b746dee797d060a45ea1a2 Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Mon, 9 Mar 2020 18:40:12 +0300 Subject: [PATCH 09/14] switch from csv to text encoder --- app/services/schedule_loader.rb | 60 ++++++++++++++-------------- docker-compose.yml | 22 +++++----- spec/schedule_loader_service_spec.rb | 6 +-- 3 files changed, 43 insertions(+), 45 deletions(-) diff --git a/app/services/schedule_loader.rb b/app/services/schedule_loader.rb index 632f370e..fd86f0bc 100644 --- a/app/services/schedule_loader.rb +++ b/app/services/schedule_loader.rb @@ -11,33 +11,29 @@ def self.call(file_name) @services = {} @buses = Set.new - @trips_command = "copy trips (from_id, to_id, start_time, duration_minutes, price_cents, model, number) from stdin with csv delimiter ';'" - @trips_command_raw = "copy trips (from_id, to_id, start_time, duration_minutes, price_cents, model, number) from stdin" - - @bus_command = "copy buses (model, number) from stdin with csv delimiter ';'" - @buses_services_command = "copy buses_services (service_id, model, number) from stdin with csv delimiter ';'" - @buses_services_command_raw = "copy buses_services (service_id, model, number) from stdin" - - @@services = {} Service.pluck(:name, :id).map{|sr| @@services[sr[0].to_sym] = sr[1]} - ActiveRecord::Base.transaction do - - time_start = Time.now + # https://koshigoe.github.io/postgresql/ruby/2018/10/31/bulk-insert-vs-copy-in-postgres.html + @base_connection = ActiveRecord::Base.connection + @raw_conn = ActiveRecord::Base.connection.raw_connection + @encoder = PG::TextEncoder::CopyRow.new - ActiveRecord::Base.connection.execute('delete from buses_services;') + @trips_command_raw = "copy trips (from_id, to_id, start_time, duration_minutes, price_cents, model, number) from stdin" + @buses_services_command_raw = "copy buses_services (service_id, model, number) from stdin" + @bus_command_raw = "copy buses (model, number) from stdin" - ActiveRecord::Base.connection.execute('delete from buses_services;') - ActiveRecord::Base.connection.execute('delete from trips;') - ActiveRecord::Base.connection.execute('delete from cities;') - ActiveRecord::Base.connection.execute('delete from buses;') + @nr2 = Array.new(2) + @nr3 = Array.new(3) + @nr7 = Array.new(7) - # Trip.delete_all - # City.delete_all - # Bus.delete_all - ActiveRecord::Base.connection.reset_pk_sequence!('cities') - ActiveRecord::Base.connection.reset_pk_sequence!('buses') + ActiveRecord::Base.transaction do + @raw_conn.exec('delete from buses_services;') + @raw_conn.exec('delete from trips;') + @raw_conn.exec('delete from cities;') + @raw_conn.exec('delete from buses;') + @base_connection.reset_pk_sequence!('cities') + @base_connection.reset_pk_sequence!('buses') File.open(file_name) do |ff| nesting = 0 @@ -65,8 +61,6 @@ def self.call(file_name) end end City.import @cities.map {|cc| ({id: cc[1], name: cc[0]})} - end_time = Time.now - pp (end_time - time_start) end end @@ -83,25 +77,29 @@ def self.import_trip(trip) @cities[trip['to']] = to_id end - bus_key = "#{trip['bus']['number']}-#{trip['bus']['model']}" - bus_number = trip['bus']['number'] bus_model = trip['bus']['model'] + bus_key = "#{bus_number}-#{bus_model}" + unless @buses.include?(bus_key) - ActiveRecord::Base.connection.raw_connection.copy_data @bus_command do - ActiveRecord::Base.connection.raw_connection.put_copy_data("#{bus_number};#{bus_model}\n") + @raw_conn.copy_data @bus_command_raw, @encoder do + @nr2 = bus_number, bus_model + @raw_conn.put_copy_data(@nr2) end @buses << bus_key - ActiveRecord::Base.connection.raw_connection.copy_data @buses_services_command do + + @raw_conn.copy_data @buses_services_command_raw, @encoder do trip["bus"]["services"].map do |srv| - ActiveRecord::Base.connection.raw_connection.put_copy_data("#{@@services[srv.to_sym]};#{bus_number};#{bus_model}\n") + @nr3 = @@services[srv.to_sym], bus_number, bus_model + @raw_conn.put_copy_data(@nr3) end end end - ActiveRecord::Base.connection.raw_connection.copy_data @trips_command do - ActiveRecord::Base.connection.raw_connection.put_copy_data("#{from_id};#{to_id};#{trip['start_time']};#{trip['duration_minutes']};#{trip['price_cents']};#{bus_number};#{bus_model}\n") + @raw_conn.copy_data @trips_command_raw, @encoder do + @nr7 = from_id,to_id, trip['start_time'],trip['duration_minutes'],trip['price_cents'],bus_number,bus_model + @raw_conn.put_copy_data(@nr7) end end end diff --git a/docker-compose.yml b/docker-compose.yml index 4c31c308..d7dee867 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,17 +13,17 @@ services: ports: - "15432:5432" - pghero: - image: ankane/pghero - ports: - - "8080:8080" - environment: -# DATABASE_URL: "postgres://elreg:elreg@172.16.101.62:5432/fgiski_preproduction" - DATABASE_URL: "postgres://rails:password@db-postgresql:5432/task-4_development" - tty: true - stdin_open: true - depends_on: - - db-postgresql +# pghero: +# image: ankane/pghero +# ports: +# - "8080:8080" +# environment: +## DATABASE_URL: "postgres://elreg:elreg@172.16.101.62:5432/fgiski_preproduction" +# DATABASE_URL: "postgres://rails:password@db-postgresql:5432/task-4_development" +# tty: true +# stdin_open: true +# depends_on: +# - db-postgresql # schedule app schedule_app: diff --git a/spec/schedule_loader_service_spec.rb b/spec/schedule_loader_service_spec.rb index b422198d..7f168ff3 100644 --- a/spec/schedule_loader_service_spec.rb +++ b/spec/schedule_loader_service_spec.rb @@ -4,16 +4,16 @@ RSpec.describe ScheduleLoader do describe "loading small file" do - it { expect { ScheduleLoader.call("fixtures/small.json") }.to perform_under(2).sec } + it { expect { ScheduleLoader.call("fixtures/small.json") }.to perform_under(2).sec.warmup(1).times.sample(2).times } end describe "loading medium file" do - it {expect { ScheduleLoader.call('fixtures/medium.json') }.to perform_under(60).sec.warmup(1).times.sample(1).times } + it {expect { ScheduleLoader.call('fixtures/medium.json') }.to perform_under(10).sec.warmup(1).times.sample(2).times } end describe "loading large file" do let(:file_name) { "fixtures/large.json" } - it {expect { ScheduleLoader.call('fixtures/large.json') }.to perform_under(60).sec.warmup(1).times.sample(1).times } + it {expect { ScheduleLoader.call('fixtures/large.json') }.to perform_under(60).sec.warmup(1).times.sample(2).times } end end From 242518ba8a4b697ee203f1858445563139623571 Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Mon, 9 Mar 2020 19:17:23 +0300 Subject: [PATCH 10/14] benchmark of different ways to load data into pg. https://koshigoe.github.io/postgresql/ruby/2018/10/31/bulk-insert-vs-copy-in-postgres.html --- spec/bench_spec.rb | 128 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 spec/bench_spec.rb diff --git a/spec/bench_spec.rb b/spec/bench_spec.rb new file mode 100644 index 00000000..41a57716 --- /dev/null +++ b/spec/bench_spec.rb @@ -0,0 +1,128 @@ +require 'active_record' +require 'benchmark' +require 'csv' + +def get_rss + `ps -o rss= -p #{Process.pid}`.to_i +end + +COLUMN_LENGTH = 100 +COLUMN_SIZE = 10 +RECORD_SIZE = 100_000 +BATCH_SIZE = 500 + +columns = Array.new(COLUMN_SIZE) { |i| ('a'.ord + (i % 26)).chr * COLUMN_LENGTH } +records = Array.new(RECORD_SIZE, columns) + +# $ docker run --rm -p 5432:5432 -e POSTGRES_PASSWORD=password -d postgres:9.6 +# $ psql -h localhost -U postgres -d postgres +ActiveRecord::Base.establish_connection( + adapter: 'postgresql', + encoding: 'unicode', + host: 'db-postgresql', + database: 'postgres', + username: 'rails', + password: 'password', +) + +conn = ActiveRecord::Base.connection +raw_conn = ActiveRecord::Base.connection.raw_connection + +%i( + benchmark_copy_format_text_1 + benchmark_copy_format_text_2 + benchmark_copy_format_csv_1 + benchmark_copy_format_csv_2 + benchmark_bulk_insert_1 + benchmark_bulk_insert_2 +).each do |table_name| + conn.create_table table_name, force: true, id: false do |t| + COLUMN_SIZE.times { |i| t.string "col#{i}" } + end +end + +rss_log = [] + +Benchmark.bm(35) do |x| + x.report('COPY TEXT at once') do + get_rss.tap do |rss| + encoder = PG::TextEncoder::CopyRow.new + raw_conn.copy_data('copy benchmark_copy_format_text_1 from stdin', encoder) do + records.each do |row| + raw_conn.put_copy_data(row) + end + end + rss_log << get_rss - rss + end + end + + x.report('COPY TEXT in batch') do + get_rss.tap do |rss| + encoder = PG::TextEncoder::CopyRow.new + records.each_slice(BATCH_SIZE) do |rows| + raw_conn.copy_data('copy benchmark_copy_format_text_2 from stdin', encoder) do + rows.each do |r| + raw_conn.put_copy_data(r) + end + end + end + rss_log << get_rss - rss + end + end + + x.report('COPY CSV with CSV.generate_line') do + get_rss.tap do |rss| + records.each_slice(BATCH_SIZE) do |rows| + raw_conn.copy_data('copy benchmark_copy_format_csv_1 from stdin with (format csv)') do + rows.each do |row| + raw_conn.put_copy_data(CSV.generate_line(row, force_quotes: true)) + end + end + end + rss_log << get_rss - rss + end + end + + x.report('COPY CSV with Array#join') do + get_rss.tap do |rss| + records.each_slice(BATCH_SIZE) do |rows| + raw_conn.copy_data('copy benchmark_copy_format_csv_2 from stdin with (format csv)') do + rows.each do |row| + raw_conn.put_copy_data(row.map { |col| "\"#{col}\"" }.join(',') + "\n") + end + end + end + rss_log << get_rss - rss + end + end + + x.report('BULK INSERT with generate SQLs') do + get_rss.tap do |rss| + records.each_slice(BATCH_SIZE) do |rows| + sql = 'insert into benchmark_bulk_insert_1 values ' + sql << rows.map { |row| '(' + row.map { |col| "'#{col}'" }.join(',') + ')' }.join(',') + conn.execute(sql) + end + rss_log << get_rss - rss + end + end + + sqls = [] + records.each_slice(BATCH_SIZE) do |rows| + sql = 'insert into benchmark_bulk_insert_2 values ' + sql << rows.map { |row| '(' + row.map { |col| "'#{col}'" }.join(',') + ')' }.join(',') + sqls << sql + end + x.report('BULK INSERT without generate SQLs') do + get_rss.tap do |rss| + sqls.each do |sql| + conn.execute(sql) + end + rss_log << get_rss - rss + end + end +end + + +puts "\n[RSS]\n" +puts rss_log.map { |rss| format('+ %5d', rss) }.join("\n") From dcf7a85f2ff820c6aa99cfdebbd792e6ad460a47 Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Mon, 9 Mar 2020 19:17:54 +0300 Subject: [PATCH 11/14] indicies --- db/migrate/20200309155753_add_indices_to_bus_services.rb | 9 +++++++++ db/schema.rb | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100755 db/migrate/20200309155753_add_indices_to_bus_services.rb diff --git a/db/migrate/20200309155753_add_indices_to_bus_services.rb b/db/migrate/20200309155753_add_indices_to_bus_services.rb new file mode 100755 index 00000000..9021d292 --- /dev/null +++ b/db/migrate/20200309155753_add_indices_to_bus_services.rb @@ -0,0 +1,9 @@ +class AddIndicesToBusServices < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + add_index :buses_services, %i[model number service_id], algorithm: :concurrently + add_index :buses_services, %i[service_id model number], algorithm: :concurrently + add_index :cities, :name, algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index e7f5f359..08910adb 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,9 +10,10 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_03_08_114605) do +ActiveRecord::Schema.define(version: 2020_03_09_155753) do # These are extensions that must be enabled in order to support this database + enable_extension "pg_stat_statements" enable_extension "plpgsql" create_table "buses", primary_key: ["number", "model"], force: :cascade do |t| @@ -25,10 +26,13 @@ t.integer "service_id" t.string "model" t.string "number" + t.index ["model", "number", "service_id"], name: "index_buses_services_on_model_and_number_and_service_id" + t.index ["service_id", "model", "number"], name: "index_buses_services_on_service_id_and_model_and_number" end create_table "cities", force: :cascade do |t| t.string "name" + t.index ["name"], name: "index_cities_on_name" end create_table "services", force: :cascade do |t| From b0a0ae8056bb4888cf67c01eec2ad373068fcaea Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Mon, 9 Mar 2020 19:28:45 +0300 Subject: [PATCH 12/14] indicies. Speeded up page load twice. from 25 second to 12 --- .../20200309155753_add_indices_to_bus_services.rb | 10 +++++++++- db/schema.rb | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/db/migrate/20200309155753_add_indices_to_bus_services.rb b/db/migrate/20200309155753_add_indices_to_bus_services.rb index 9021d292..557d34b7 100755 --- a/db/migrate/20200309155753_add_indices_to_bus_services.rb +++ b/db/migrate/20200309155753_add_indices_to_bus_services.rb @@ -1,9 +1,17 @@ class AddIndicesToBusServices < ActiveRecord::Migration[5.2] disable_ddl_transaction! - def change + def up add_index :buses_services, %i[model number service_id], algorithm: :concurrently add_index :buses_services, %i[service_id model number], algorithm: :concurrently add_index :cities, :name, algorithm: :concurrently + add_index :trips, %i[from_id to_id], algorithm: :concurrently + end + + def down + remove_index :buses_services, %i[model number service_id] + remove_index :buses_services, %i[service_id model number] + remove_index :cities, :name + remove_index :trips, %i[from_id to_id] end end diff --git a/db/schema.rb b/db/schema.rb index 08910adb..a7b1d878 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -47,6 +47,7 @@ t.integer "price_cents" t.string "model" t.string "number" + t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id" end add_foreign_key "buses_services", "buses", column: "model", primary_key: "model", name: "buses_services_fk" From 9e05daff6826b99dafceeef996dc20a0ded65972 Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Mon, 9 Mar 2020 22:11:54 +0300 Subject: [PATCH 13/14] rendering schedule of large file whithin 6-11 secunds --- app/controllers/trips_controller.rb | 2 +- app/models/trip.rb | 2 +- app/views/trips/_delimiter.html.erb | 1 - app/views/trips/_service.html.erb | 1 - app/views/trips/_services.html.erb | 6 ------ app/views/trips/_trip.html.erb | 12 +++++++++++ app/views/trips/index.html.erb | 14 +++---------- ...00309155753_add_indices_to_bus_services.rb | 4 ++++ db/schema.rb | 2 ++ docker-compose.yml | 21 +++++++++---------- 10 files changed, 33 insertions(+), 32 deletions(-) delete mode 100644 app/views/trips/_delimiter.html.erb delete mode 100644 app/views/trips/_service.html.erb delete mode 100644 app/views/trips/_services.html.erb diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index acb38be2..09514f6a 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -2,6 +2,6 @@ class TripsController < ApplicationController def index @from = City.find_by_name!(params[:from]) @to = City.find_by_name!(params[:to]) - @trips = Trip.where(from: @from, to: @to).order(:start_time) + @trips = Trip.where(from: @from, to: @to).order(:start_time).includes(:bus).includes(bus: :services) end end diff --git a/app/models/trip.rb b/app/models/trip.rb index 675283d9..986616f9 100644 --- a/app/models/trip.rb +++ b/app/models/trip.rb @@ -3,7 +3,7 @@ class Trip < ApplicationRecord belongs_to :from, class_name: 'City' belongs_to :to, class_name: 'City' - belongs_to :bus, :foreign_key => [:number, :model] + belongs_to :bus, -> { includes :services }, :foreign_key => [:number, :model] validates :from, presence: true validates :to, presence: true diff --git a/app/views/trips/_delimiter.html.erb b/app/views/trips/_delimiter.html.erb deleted file mode 100644 index 3f845ad0..00000000 --- a/app/views/trips/_delimiter.html.erb +++ /dev/null @@ -1 +0,0 @@ -==================================================== diff --git a/app/views/trips/_service.html.erb b/app/views/trips/_service.html.erb deleted file mode 100644 index 178ea8c0..00000000 --- a/app/views/trips/_service.html.erb +++ /dev/null @@ -1 +0,0 @@ -
  • <%= "#{service.name}" %>
    • diff --git a/app/views/trips/_services.html.erb b/app/views/trips/_services.html.erb deleted file mode 100644 index 2de639fc..00000000 --- a/app/views/trips/_services.html.erb +++ /dev/null @@ -1,6 +0,0 @@ -
  • Сервисы в автобусе:
    • -
      - <% services.each do |service| %> - <%= render "service", service: service %> - <% end %> -
    diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb index fa1de9aa..e8e76e98 100644 --- a/app/views/trips/_trip.html.erb +++ b/app/views/trips/_trip.html.erb @@ -3,3 +3,15 @@
  • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
  • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
  • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
    • + +
  • Сервисы в автобусе:
    • +
      + <% trip.bus.services.pluck(:name).map do |service| %> + <%#= render "service", service: service %> +
    • <%= service %>
      • + <% end %> +
    + + +<%# render plain: '====================================================' %> +==================================================== \ No newline at end of file diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce41..44413751 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -2,15 +2,7 @@ <%= "Автобусы #{@from.name} – #{@to.name}" %>

    - <%= "В расписании #{@trips.count} рейсов" %> + <%= "В расписании #{@trips.size} рейсов" %>

    - -<% @trips.each do |trip| %> -
      - <%= render "trip", trip: trip %> - <% if trip.bus.services.present? %> - <%= render "services", services: trip.bus.services %> - <% end %> -
    - <%= render "delimiter" %> -<% end %> +<%# binding.pry %> +<%= render partial: "trip", collection: @trips, cached: true %> diff --git a/db/migrate/20200309155753_add_indices_to_bus_services.rb b/db/migrate/20200309155753_add_indices_to_bus_services.rb index 557d34b7..4cfee4d6 100755 --- a/db/migrate/20200309155753_add_indices_to_bus_services.rb +++ b/db/migrate/20200309155753_add_indices_to_bus_services.rb @@ -6,6 +6,8 @@ def up add_index :buses_services, %i[service_id model number], algorithm: :concurrently add_index :cities, :name, algorithm: :concurrently add_index :trips, %i[from_id to_id], algorithm: :concurrently + add_index :buses, :model, algorithm: :concurrently + add_index :buses, :number, algorithm: :concurrently end def down @@ -13,5 +15,7 @@ def down remove_index :buses_services, %i[service_id model number] remove_index :cities, :name remove_index :trips, %i[from_id to_id] + remove_index :buses, :model + remove_index :buses, :number end end diff --git a/db/schema.rb b/db/schema.rb index a7b1d878..f86083d1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -19,7 +19,9 @@ create_table "buses", primary_key: ["number", "model"], force: :cascade do |t| t.string "number", null: false t.string "model", null: false + t.index ["model"], name: "index_buses_on_model" t.index ["number", "model"], name: "index_buses_on_number_and_model", unique: true + t.index ["number"], name: "index_buses_on_number" end create_table "buses_services", force: :cascade do |t| diff --git a/docker-compose.yml b/docker-compose.yml index d7dee867..a70cbc96 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,17 +13,16 @@ services: ports: - "15432:5432" -# pghero: -# image: ankane/pghero -# ports: -# - "8080:8080" -# environment: -## DATABASE_URL: "postgres://elreg:elreg@172.16.101.62:5432/fgiski_preproduction" -# DATABASE_URL: "postgres://rails:password@db-postgresql:5432/task-4_development" -# tty: true -# stdin_open: true -# depends_on: -# - db-postgresql + pghero: + image: ankane/pghero + ports: + - "8080:8080" + environment: + DATABASE_URL: "postgres://rails:password@db-postgresql:5432/task-4_development" + tty: true + stdin_open: true + depends_on: + - db-postgresql # schedule app schedule_app: From 12f4e16464cbe8cdf094ec9104b32d36f37590b6 Mon Sep 17 00:00:00 2001 From: Evgeny Gorbachev Date: Mon, 9 Mar 2020 22:22:41 +0300 Subject: [PATCH 14/14] case-study --- case-study.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 case-study.md diff --git a/case-study.md b/case-study.md new file mode 100644 index 00000000..0e1cdb9d --- /dev/null +++ b/case-study.md @@ -0,0 +1,19 @@ +- [X] `rack-mini-profiler` +- [X] `rails panel` +- [X] `bullet` +- [X] `explain` запросов + +реализовал composite primary key (model, number) через гем 'composite_primary_keys'. Нашел интересную статью с бенчмарками различных методик загрузки данных в Postgreql - +`https://koshigoe.github.io/postgresql/ruby/2018/10/31/bulk-insert-vs-copy-in-postgres.html`. Прогнал локально тесты из статьи и реализовал самый быстрый, что дало прирост, но меньший чем ожидал. Посчитал что текущей скорости достаточно и перешел к оптимизации отображения расписания. +- импорт данных из файла rake reload_json[fixtures/large.json] проходит за 40 секунд. +- загрузка страницы с расписанием занимат 6-11 секунд +- не удалось применить подсказку из +```` +user: root +USE eager loading detected + Bus::HABTM_Services => [:service] + Add to your query: .includes([:service]) +Call stack + /var/www/schedule_app/app/views/trips/_trip.html.erb:9:in `_app_views_trips__trip_html_erb___2552659478807376598_47345913761100' + /var/www/schedule_app/app/views/trips/index.html.erb:8:in `_app_views_trips_index_html_erb__2240206535625781100_47345814716480 + ````