From fa96034d8a90bb820c2d339ba2c0655e04d0a027 Mon Sep 17 00:00:00 2001 From: Sergey Kochubey Date: Sun, 19 May 2024 11:41:43 +0300 Subject: [PATCH 1/9] app init --- Gemfile.lock | 4 +++- bin/setup | 2 +- config/database.yml | 12 ++++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index fccf6f5f..ff36865c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -68,7 +68,9 @@ GEM marcel (0.3.3) mimemagic (~> 0.3.2) method_source (0.9.2) - mimemagic (0.3.3) + mimemagic (0.3.10) + nokogiri (~> 1) + rake mini_mime (1.0.1) mini_portile2 (2.4.0) minitest (5.11.3) diff --git a/bin/setup b/bin/setup index f294207b..3fa5f793 100755 --- a/bin/setup +++ b/bin/setup @@ -18,7 +18,7 @@ chdir APP_ROOT do system('bundle check') || system!('bundle install') # Install JavaScript dependencies if using Yarn - # system('bin/yarn') + system('bin/yarn') # puts "\n== Copying sample files ==" # unless File.exist?('config/database.yml') diff --git a/config/database.yml b/config/database.yml index e116cfa6..451421c4 100644 --- a/config/database.yml +++ b/config/database.yml @@ -20,6 +20,8 @@ default: &default # For details on connection pooling, see Rails configuration guide # http://guides.rubyonrails.org/configuring.html#database-pooling pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + host: localhost + port: 5432 development: <<: *default @@ -29,19 +31,19 @@ development: # To create additional roles in postgres see `$ createuser --help`. # When left blank, postgres will use the default role. This is # the same name as the operating system user that initialized the database. - #username: task-4 + username: bb_db_user # The password associated with the postgres role (username). - #password: + password: 340242 # Connect on a TCP socket. Omitted by default since the client uses a # domain socket that doesn't need configuration. Windows does not have # domain sockets, so uncomment these lines. - #host: localhost + # host: localhost # The TCP port the server listens on. Defaults to 5432. # If your server runs on a different port number, change accordingly. - #port: 5432 + # port: 5432 # Schema search path. The server defaults to $user,public #schema_search_path: myapp,sharedapp,public @@ -58,6 +60,8 @@ development: test: <<: *default database: task-4_test + username: bb_db_user + password: 340242 # As with config/secrets.yml, you never want to store sensitive information, # like your database password, in your source code. If your source code is From c1083ac23decc149d6a6da6622c93f40a94f3191 Mon Sep 17 00:00:00 2001 From: Sergey Kochubey Date: Sun, 19 May 2024 19:09:41 +0300 Subject: [PATCH 2/9] City and services in hash added --- Gemfile | 4 ++ Gemfile.lock | 7 +++ app/views/layouts/application.html.erb | 2 + config/routes.rb | 2 + lib/tasks/simple_task.rake | 68 ++++++++++++++++++++++++++ lib/tasks/utils.rake | 2 +- 6 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 lib/tasks/simple_task.rake diff --git a/Gemfile b/Gemfile index e20b1260..e9ace397 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,10 @@ gem 'pg', '>= 0.18', '< 2.0' gem 'puma', '~> 3.11' gem 'bootsnap', '>= 1.1.0', require: false +gem 'pghero' +# pg_query is used by pghero to provide suggestions to add indexes +gem 'pg_query', '>= 0.9.0' + group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] diff --git a/Gemfile.lock b/Gemfile.lock index ff36865c..99dae2d4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -54,6 +54,7 @@ GEM ffi (1.10.0) globalid (0.4.2) activesupport (>= 4.2.0) + google-protobuf (3.23.4) i18n (1.6.0) concurrent-ruby (~> 1.0) listen (3.1.5) @@ -79,6 +80,10 @@ GEM nokogiri (1.10.2) mini_portile2 (~> 2.4.0) pg (1.1.4) + pg_query (5.1.0) + google-protobuf (>= 3.22.3) + pghero (2.8.3) + activerecord (>= 5) puma (3.12.1) rack (2.0.6) rack-test (1.1.0) @@ -140,6 +145,8 @@ DEPENDENCIES byebug listen (>= 3.0.5, < 3.2) pg (>= 0.18, < 2.0) + pg_query (>= 0.9.0) + pghero puma (~> 3.11) rails (~> 5.2.3) tzinfo-data diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index e64170ee..bfc813b1 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -10,6 +10,8 @@ + <%= link_to "Database Analyzer", pg_hero_path, target: :blank %> +

<%= yield %> diff --git a/config/routes.rb b/config/routes.rb index a2da6a7b..c606f2b9 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -2,4 +2,6 @@ # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html get "/" => "statistics#index" get "автобусы/:from/:to" => "trips#index" + + mount PgHero::Engine, at: "pgdashboard" end diff --git a/lib/tasks/simple_task.rake b/lib/tasks/simple_task.rake new file mode 100644 index 00000000..55afc6bb --- /dev/null +++ b/lib/tasks/simple_task.rake @@ -0,0 +1,68 @@ +namespace :demo do + # Наивная загрузка данных из json-файла в БД + # rake reload_json[fixtures/small.json] + task :reload_json, [:file_name] => :environment do |_task, args| + + time = Benchmark.measure do + json = JSON.parse(File.read(args.file_name)) + + ActiveRecord::Base.transaction do + City.delete_all + Bus.delete_all + Service.delete_all + Trip.delete_all + ActiveRecord::Base.connection.execute('delete from buses_services;') + + cities_hash = {} + services_hash = {} + + fetch_city = lambda do |name| + return cities_hash[name] if cities_hash.key?(name) + + city = City.create(name: name) + cities_hash[name] = city + city + end + + fetch_service_id = lambda do |service| + #services_hash[service] ||= Service.find_or_create_by(name: service) + + return services_hash[service] if services_hash.key?(service) + + service_id = Service.create(name: service).id + services_hash[service] = service_id + service_id + end + + json.each do |trip| + from_city = fetch_city.call(trip['from']) + to_city = fetch_city.call(trip['to']) + + service_ids = [] + trip['bus']['services'].each do |service| + + service_ids << fetch_service_id.call(service) + end + + bus = Bus.find_or_create_by(number: trip['bus']['number']) + bus.service_ids = service_ids + #bus.update(model: trip['bus']['model'], services: services) + + + Trip.create!( + from: from_city, + to: to_city, + bus: bus, + start_time: trip['start_time'], + duration_minutes: trip['duration_minutes'], + price_cents: trip['price_cents'], + ) + end + + end + + end + + puts "Done in #{time.real} seconds" + end +end diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 540fe871..9f55d8bb 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,4 +1,4 @@ -# Наивная загрузка данных из json-файла в БД +# Наивная загрузка данных из json-файла в БД # rake reload_json[fixtures/small.json] task :reload_json, [:file_name] => :environment do |_task, args| json = JSON.parse(File.read(args.file_name)) From 27c833c6af8b32e8e38774c1cbb7d471ffa2dde6 Mon Sep 17 00:00:00 2001 From: Sergey Kochubey Date: Mon, 20 May 2024 18:55:52 +0300 Subject: [PATCH 3/9] bus cached --- lib/tasks/simple_task.rake | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/tasks/simple_task.rake b/lib/tasks/simple_task.rake index 55afc6bb..17716677 100644 --- a/lib/tasks/simple_task.rake +++ b/lib/tasks/simple_task.rake @@ -15,6 +15,7 @@ namespace :demo do cities_hash = {} services_hash = {} + buses_hash = {} fetch_city = lambda do |name| return cities_hash[name] if cities_hash.key?(name) @@ -25,8 +26,6 @@ namespace :demo do end fetch_service_id = lambda do |service| - #services_hash[service] ||= Service.find_or_create_by(name: service) - return services_hash[service] if services_hash.key?(service) service_id = Service.create(name: service).id @@ -34,20 +33,21 @@ namespace :demo do service_id end + fetch_bus = lambda do |number, model, bus_services| + return buses_hash["#{number}-#{model}"] if buses_hash.key?("#{number}-#{model}") + + service_ids = bus_services.map! { |service| fetch_service_id.call(service) } + bus = Bus.create(number: number, model: model) + bus.service_ids = service_ids + buses_hash["#{number}-#{model}"] = bus + bus + end + json.each do |trip| from_city = fetch_city.call(trip['from']) to_city = fetch_city.call(trip['to']) - service_ids = [] - trip['bus']['services'].each do |service| - - service_ids << fetch_service_id.call(service) - end - - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.service_ids = service_ids - #bus.update(model: trip['bus']['model'], services: services) - + bus = fetch_bus.call(trip['bus']['number'], trip['bus']['model'], trip['bus']['services']) Trip.create!( from: from_city, From cb266563570d2c400215b5a4a09cddc6f9c87fcd Mon Sep 17 00:00:00 2001 From: Sergey Kochubey Date: Mon, 20 May 2024 19:27:55 +0300 Subject: [PATCH 4/9] refactor --- lib/tasks/simple_task.rake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/tasks/simple_task.rake b/lib/tasks/simple_task.rake index 17716677..9b2139aa 100644 --- a/lib/tasks/simple_task.rake +++ b/lib/tasks/simple_task.rake @@ -37,8 +37,7 @@ namespace :demo do return buses_hash["#{number}-#{model}"] if buses_hash.key?("#{number}-#{model}") service_ids = bus_services.map! { |service| fetch_service_id.call(service) } - bus = Bus.create(number: number, model: model) - bus.service_ids = service_ids + bus = Bus.create(number: number, model: model, service_ids: service_ids) buses_hash["#{number}-#{model}"] = bus bus end From c417d696cf975e98e164264279ee3a6b95a59a53 Mon Sep 17 00:00:00 2001 From: Sergey Kochubey Date: Mon, 20 May 2024 22:40:10 +0300 Subject: [PATCH 5/9] test added --- .rspec | 1 + Gemfile | 4 + Gemfile.lock | 32 +++++++ ...0240520181028_add_index_to_buses_number.rb | 5 + db/schema.rb | 4 +- lib/tasks/simple_task.rake | 47 +++------- spec/rails_helper.rb | 64 +++++++++++++ spec/spec_helper.rb | 94 +++++++++++++++++++ 8 files changed, 214 insertions(+), 37 deletions(-) create mode 100644 .rspec create mode 100644 db/migrate/20240520181028_add_index_to_buses_number.rb create mode 100644 spec/rails_helper.rb create mode 100644 spec/spec_helper.rb diff --git a/.rspec b/.rspec new file mode 100644 index 00000000..c99d2e73 --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/Gemfile b/Gemfile index e9ace397..bf1b1cda 100644 --- a/Gemfile +++ b/Gemfile @@ -11,16 +11,20 @@ gem 'bootsnap', '>= 1.1.0', require: false gem 'pghero' # pg_query is used by pghero to provide suggestions to add indexes gem 'pg_query', '>= 0.9.0' +gem 'rack-mini-profiler' group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] + gem 'rspec-rails', '~> 4.0.0' end group :development do # Access an interactive console on exception pages or by calling 'console' anywhere in the code. gem 'web-console', '>= 3.3.0' gem 'listen', '>= 3.0.5', '< 3.2' + gem 'database_cleaner-active_record', '~> 2.0.1' + gem 'bullet' end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index 99dae2d4..6d1fe752 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -47,9 +47,17 @@ GEM bootsnap (1.4.2) msgpack (~> 1.0) builder (3.2.3) + bullet (7.1.6) + activesupport (>= 3.0.0) + uniform_notifier (~> 1.11) byebug (11.0.1) concurrent-ruby (1.1.5) crass (1.0.4) + database_cleaner-active_record (2.0.1) + activerecord (>= 5.a) + database_cleaner-core (~> 2.0.0) + database_cleaner-core (2.0.1) + diff-lcs (1.5.1) erubi (1.8.0) ffi (1.10.0) globalid (0.4.2) @@ -86,6 +94,8 @@ GEM activerecord (>= 5) puma (3.12.1) rack (2.0.6) + rack-mini-profiler (3.1.1) + rack (>= 1.2.0) rack-test (1.1.0) rack (>= 1.0, < 3) rails (5.2.3) @@ -116,6 +126,23 @@ GEM rb-fsevent (0.10.3) rb-inotify (0.10.0) ffi (~> 1.0) + rspec-core (3.13.0) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-rails (4.0.2) + actionpack (>= 4.2) + activesupport (>= 4.2) + railties (>= 4.2) + rspec-core (~> 3.10) + rspec-expectations (~> 3.10) + rspec-mocks (~> 3.10) + rspec-support (~> 3.10) + rspec-support (3.13.1) ruby_dep (1.5.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) @@ -128,6 +155,7 @@ GEM thread_safe (0.3.6) tzinfo (1.2.5) thread_safe (~> 0.1) + uniform_notifier (1.16.0) web-console (3.7.0) actionview (>= 5.0) activemodel (>= 5.0) @@ -142,13 +170,17 @@ PLATFORMS DEPENDENCIES bootsnap (>= 1.1.0) + bullet byebug + database_cleaner-active_record (~> 2.0.1) listen (>= 3.0.5, < 3.2) pg (>= 0.18, < 2.0) pg_query (>= 0.9.0) pghero puma (~> 3.11) + rack-mini-profiler rails (~> 5.2.3) + rspec-rails (~> 4.0.0) tzinfo-data web-console (>= 3.3.0) diff --git a/db/migrate/20240520181028_add_index_to_buses_number.rb b/db/migrate/20240520181028_add_index_to_buses_number.rb new file mode 100644 index 00000000..b1e57d2b --- /dev/null +++ b/db/migrate/20240520181028_add_index_to_buses_number.rb @@ -0,0 +1,5 @@ +class AddIndexToBusesNumber < ActiveRecord::Migration[5.2] + def change + add_index :buses, :number, unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index f6921e45..3a189617 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,14 +10,16 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_03_30_193044) do +ActiveRecord::Schema.define(version: 2024_05_20_181028) do # These are extensions that must be enabled in order to support this database + enable_extension "pg_stat_statements" enable_extension "plpgsql" create_table "buses", force: :cascade do |t| t.string "number" t.string "model" + t.index ["number"], name: "index_buses_on_number", unique: true end create_table "buses_services", force: :cascade do |t| diff --git a/lib/tasks/simple_task.rake b/lib/tasks/simple_task.rake index 9b2139aa..327e84bf 100644 --- a/lib/tasks/simple_task.rake +++ b/lib/tasks/simple_task.rake @@ -13,53 +13,28 @@ namespace :demo do Trip.delete_all ActiveRecord::Base.connection.execute('delete from buses_services;') - cities_hash = {} - services_hash = {} - buses_hash = {} - - fetch_city = lambda do |name| - return cities_hash[name] if cities_hash.key?(name) - - city = City.create(name: name) - cities_hash[name] = city - city - end - - fetch_service_id = lambda do |service| - return services_hash[service] if services_hash.key?(service) - - service_id = Service.create(name: service).id - services_hash[service] = service_id - service_id - end - - fetch_bus = lambda do |number, model, bus_services| - return buses_hash["#{number}-#{model}"] if buses_hash.key?("#{number}-#{model}") - - service_ids = bus_services.map! { |service| fetch_service_id.call(service) } - bus = Bus.create(number: number, model: model, service_ids: service_ids) - buses_hash["#{number}-#{model}"] = bus - bus - end + cities = {} + services = {} + buses = {} json.each do |trip| - from_city = fetch_city.call(trip['from']) - to_city = fetch_city.call(trip['to']) + from = cities[trip['from']] ||= City.create(name: trip['from']) + to = cities[trip['to']] ||= City.create(name: trip['to']) - bus = fetch_bus.call(trip['bus']['number'], trip['bus']['model'], trip['bus']['services']) + bus_services = trip['bus']['services'].map! { |s| services[s] ||= Service.create(name: s) } + + bus = buses[trip['bus']['number']] ||= Bus.create(number: trip['bus']['number'], model: trip['bus']['model'], services: bus_services) Trip.create!( - from: from_city, - to: to_city, + from: from, + to: to, bus: bus, start_time: trip['start_time'], duration_minutes: trip['duration_minutes'], price_cents: trip['price_cents'], ) - end - + end end - end puts "Done in #{time.real} seconds" diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb new file mode 100644 index 00000000..00345af7 --- /dev/null +++ b/spec/rails_helper.rb @@ -0,0 +1,64 @@ +# This file is copied to spec/ when you run 'rails generate rspec:install' +require 'spec_helper' +ENV['RAILS_ENV'] ||= 'test' +require File.expand_path('../config/environment', __dir__) +# Prevent database truncation if the environment is production +abort("The Rails environment is running in production mode!") if Rails.env.production? +require 'rspec/rails' +# Add additional requires below this line. Rails is not loaded until this point! + +# Requires supporting ruby files with custom matchers and macros, etc, in +# spec/support/ and its subdirectories. Files matching `spec/**/*_spec.rb` are +# run as spec files by default. This means that files in spec/support that end +# in _spec.rb will both be required and run as specs, causing the specs to be +# run twice. It is recommended that you do not name files matching this glob to +# end with _spec.rb. You can configure this pattern with the --pattern +# option on the command line or in ~/.rspec, .rspec or `.rspec-local`. +# +# The following line is provided for convenience purposes. It has the downside +# of increasing the boot-up time by auto-requiring all files in the support +# directory. Alternatively, in the individual `*_spec.rb` files, manually +# require only the support files necessary. +# +# Dir[Rails.root.join('spec', 'support', '**', '*.rb')].sort.each { |f| require f } + +# Checks for pending migrations and applies them before tests are run. +# If you are not using ActiveRecord, you can remove these lines. +begin + ActiveRecord::Migration.maintain_test_schema! +rescue ActiveRecord::PendingMigrationError => e + puts e.to_s.strip + exit 1 +end +RSpec.configure do |config| + # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures + config.fixture_path = "#{::Rails.root}/spec/fixtures" + + # If you're not using ActiveRecord, or you'd prefer not to run each of your + # examples within a transaction, remove the following line or assign false + # instead of true. + config.use_transactional_fixtures = true + + # You can uncomment this line to turn off ActiveRecord support entirely. + # config.use_active_record = false + + # RSpec Rails can automatically mix in different behaviours to your tests + # based on their file location, for example enabling you to call `get` and + # `post` in specs under `spec/controllers`. + # + # You can disable this behaviour by removing the line below, and instead + # explicitly tag your specs with their type, e.g.: + # + # RSpec.describe UsersController, type: :controller do + # # ... + # end + # + # The different available types are documented in the features, such as in + # https://relishapp.com/rspec/rspec-rails/docs + config.infer_spec_type_from_file_location! + + # Filter lines from Rails gems in backtraces. + config.filter_rails_from_backtrace! + # arbitrary gems may also be filtered via: + # config.filter_gems_from_backtrace("gem name") +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 00000000..327b58ea --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,94 @@ +# This file was generated by the `rails generate rspec:install` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See https://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +RSpec.configure do |config| + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + +# The settings below are suggested to provide a good initial experience +# with RSpec, but feel free to customize to your heart's content. +=begin + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "spec/examples.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # https://rspec.info/features/3-12/rspec-core/configuration/zero-monkey-patching-mode/ + config.disable_monkey_patching! + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + if config.files_to_run.one? + # Use the documentation formatter for detailed output, + # unless a formatter has already been configured + # (e.g. via a command-line flag). + config.default_formatter = "doc" + end + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed +=end +end From 7dbe89982da9a8c66b8e7a9a3c5c5efe8d1bd53b Mon Sep 17 00:00:00 2001 From: Sergey Kochubey Date: Tue, 21 May 2024 22:27:11 +0300 Subject: [PATCH 6/9] activerecord-import added --- Gemfile | 1 + Gemfile.lock | 3 +++ app/models/buses_service.rb | 4 ++++ lib/tasks/simple_task.rake | 29 ++++++++++++++++++++++------- 4 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 app/models/buses_service.rb diff --git a/Gemfile b/Gemfile index bf1b1cda..786fdd69 100644 --- a/Gemfile +++ b/Gemfile @@ -11,6 +11,7 @@ gem 'bootsnap', '>= 1.1.0', require: false gem 'pghero' # pg_query is used by pghero to provide suggestions to add indexes gem 'pg_query', '>= 0.9.0' +gem 'activerecord-import' gem 'rack-mini-profiler' group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index 6d1fe752..09c185d2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -33,6 +33,8 @@ GEM activemodel (= 5.2.3) activesupport (= 5.2.3) arel (>= 9.0) + activerecord-import (1.7.0) + activerecord (>= 4.2) activestorage (5.2.3) actionpack (= 5.2.3) activerecord (= 5.2.3) @@ -169,6 +171,7 @@ PLATFORMS ruby DEPENDENCIES + activerecord-import bootsnap (>= 1.1.0) bullet byebug diff --git a/app/models/buses_service.rb b/app/models/buses_service.rb new file mode 100644 index 00000000..6219d44e --- /dev/null +++ b/app/models/buses_service.rb @@ -0,0 +1,4 @@ +class BusesService < ApplicationRecord + belongs_to :bus + belongs_to :service +end diff --git a/lib/tasks/simple_task.rake b/lib/tasks/simple_task.rake index 327e84bf..93fb35f0 100644 --- a/lib/tasks/simple_task.rake +++ b/lib/tasks/simple_task.rake @@ -1,6 +1,6 @@ namespace :demo do # Наивная загрузка данных из json-файла в БД - # rake reload_json[fixtures/small.json] + # rake demo:reload_json[fixtures/small.json] task :reload_json, [:file_name] => :environment do |_task, args| time = Benchmark.measure do @@ -16,16 +16,22 @@ namespace :demo do cities = {} services = {} buses = {} + buses_services = {} + + trips = [] json.each do |trip| - from = cities[trip['from']] ||= City.create(name: trip['from']) - to = cities[trip['to']] ||= City.create(name: trip['to']) + from = cities[trip['from']] ||= City.new(name: trip['from']) + to = cities[trip['to']] ||= City.new(name: trip['to']) - bus_services = trip['bus']['services'].map! { |s| services[s] ||= Service.create(name: s) } + bus = buses[trip['bus']['number']] ||= Bus.new(number: trip['bus']['number'], model: trip['bus']['model']) - bus = buses[trip['bus']['number']] ||= Bus.create(number: trip['bus']['number'], model: trip['bus']['model'], services: bus_services) + trip['bus']['services'].each do |service| + bus_service = services[service] ||= Service.new(name: service) + buses_services[[bus, bus_service]] ||= BusesService.new(bus: bus, service: bus_service) + end - Trip.create!( + trips << Trip.new( from: from, to: to, bus: bus, @@ -33,10 +39,19 @@ namespace :demo do duration_minutes: trip['duration_minutes'], price_cents: trip['price_cents'], ) - end + end + + City.import cities.values + Bus.import buses.values + Service.import services.values + BusesService.import buses_services.values + Trip.import trips + + cities, services, buses, trips = nil end end puts "Done in #{time.real} seconds" + puts "MEMORY USAGE: %d MB" % (`ps -o rss= -p #{Process.pid}`.to_i / 1024) end end From df210be6e848390bfa192e21536aa6eacd14f522 Mon Sep 17 00:00:00 2001 From: Sergey Kochubey Date: Wed, 22 May 2024 00:15:02 +0300 Subject: [PATCH 7/9] optimize views --- Gemfile | 2 ++ Gemfile.lock | 2 ++ app/views/trips/index.html.erb | 18 ++++++++++++++---- ...20240521210145_create_pghero_query_stats.rb | 15 +++++++++++++++ db/schema.rb | 13 ++++++++++++- lib/tasks/simple_task.rake | 3 +-- 6 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 db/migrate/20240521210145_create_pghero_query_stats.rb diff --git a/Gemfile b/Gemfile index 786fdd69..4e6b69c8 100644 --- a/Gemfile +++ b/Gemfile @@ -14,6 +14,7 @@ gem 'pg_query', '>= 0.9.0' gem 'activerecord-import' gem 'rack-mini-profiler' + group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] @@ -26,6 +27,7 @@ group :development do gem 'listen', '>= 3.0.5', '< 3.2' gem 'database_cleaner-active_record', '~> 2.0.1' gem 'bullet' + gem 'stackprof' end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index 09c185d2..fe0c17ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -153,6 +153,7 @@ GEM actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) + stackprof (0.2.26) thor (0.20.3) thread_safe (0.3.6) tzinfo (1.2.5) @@ -184,6 +185,7 @@ DEPENDENCIES rack-mini-profiler rails (~> 5.2.3) rspec-rails (~> 4.0.0) + stackprof tzinfo-data web-console (>= 3.3.0) diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce41..b8c906ef 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -2,15 +2,25 @@ <%= "Автобусы #{@from.name} – #{@to.name}" %>

- <%= "В расписании #{@trips.count} рейсов" %> + <%= "В расписании #{@trips.size} рейсов" %>

<% @trips.each do |trip| %>
    - <%= render "trip", trip: trip %> +
  • <%= "Отправление: #{trip.start_time}" %>
    • +
  • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
    • +
  • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
    • +
  • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
    • +
  • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
    • + <% if trip.bus.services.present? %> - <%= render "services", services: trip.bus.services %> +
  • Сервисы в автобусе:
    • +
      + <% trip.bus.services.each do |service| %> +
    • <%= "#{service.name}" %>
      • + <% end %> +
    <% end %>
- <%= render "delimiter" %> + ==================================================== <% end %> diff --git a/db/migrate/20240521210145_create_pghero_query_stats.rb b/db/migrate/20240521210145_create_pghero_query_stats.rb new file mode 100644 index 00000000..fbf41263 --- /dev/null +++ b/db/migrate/20240521210145_create_pghero_query_stats.rb @@ -0,0 +1,15 @@ +class CreatePgheroQueryStats < ActiveRecord::Migration[5.2] + def change + create_table :pghero_query_stats do |t| + t.text :database + t.text :user + t.text :query + t.integer :query_hash, limit: 8 + t.float :total_time + t.integer :calls, limit: 8 + t.timestamp :captured_at + end + + add_index :pghero_query_stats, [:database, :captured_at] + end +end diff --git a/db/schema.rb b/db/schema.rb index 3a189617..a0a9b49b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2024_05_20_181028) do +ActiveRecord::Schema.define(version: 2024_05_21_210145) do # These are extensions that must be enabled in order to support this database enable_extension "pg_stat_statements" @@ -31,6 +31,17 @@ t.string "name" end + create_table "pghero_query_stats", force: :cascade do |t| + t.text "database" + t.text "user" + t.text "query" + t.bigint "query_hash" + t.float "total_time" + t.bigint "calls" + t.datetime "captured_at" + t.index ["database", "captured_at"], name: "index_pghero_query_stats_on_database_and_captured_at" + end + create_table "services", force: :cascade do |t| t.string "name" end diff --git a/lib/tasks/simple_task.rake b/lib/tasks/simple_task.rake index 93fb35f0..3c11793e 100644 --- a/lib/tasks/simple_task.rake +++ b/lib/tasks/simple_task.rake @@ -1,6 +1,5 @@ -namespace :demo do - # Наивная загрузка данных из json-файла в БД # rake demo:reload_json[fixtures/small.json] +namespace :demo do task :reload_json, [:file_name] => :environment do |_task, args| time = Benchmark.measure do From dc18d4b702c24a1c8c5797aa5b049be80b7f43c7 Mon Sep 17 00:00:00 2001 From: Sergey Kochubey Date: Wed, 22 May 2024 01:29:35 +0300 Subject: [PATCH 8/9] index added --- Gemfile | 3 +++ Gemfile.lock | 23 ++++++++++++++++ app/controllers/trips_controller.rb | 2 +- app/views/trips/_delimiter.html.erb | 1 - app/views/trips/_service.html.erb | 1 - app/views/trips/_services.html.erb | 6 ----- app/views/trips/_trip.html.erb | 5 ---- app/views/trips/index.html.erb | 2 +- config/environments/development.rb | 9 +++++++ config/initializers/strong_migrations.rb | 26 +++++++++++++++++++ ...0240521210145_create_pghero_query_stats.rb | 15 ----------- ...15_add_index_to_trips_from_id_and_to_id.rb | 7 +++++ ...20240521221903_add_index_to_cities_name.rb | 7 +++++ ...1222235_add_index_to_buses_services_bus.rb | 7 +++++ db/schema.rb | 16 +++--------- 15 files changed, 88 insertions(+), 42 deletions(-) delete mode 100644 app/views/trips/_delimiter.html.erb delete mode 100644 app/views/trips/_service.html.erb delete mode 100644 app/views/trips/_services.html.erb delete mode 100644 app/views/trips/_trip.html.erb create mode 100644 config/initializers/strong_migrations.rb delete mode 100644 db/migrate/20240521210145_create_pghero_query_stats.rb create mode 100644 db/migrate/20240521220915_add_index_to_trips_from_id_and_to_id.rb create mode 100644 db/migrate/20240521221903_add_index_to_cities_name.rb create mode 100644 db/migrate/20240521222235_add_index_to_buses_services_bus.rb diff --git a/Gemfile b/Gemfile index 4e6b69c8..c1e2713e 100644 --- a/Gemfile +++ b/Gemfile @@ -13,6 +13,9 @@ gem 'pghero' gem 'pg_query', '>= 0.9.0' gem 'activerecord-import' gem 'rack-mini-profiler' +gem "strong_migrations" +# gem 'meta_request' +# gem 'rails_performance' group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index fe0c17ca..6264b100 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -48,12 +48,14 @@ GEM bindex (0.6.0) bootsnap (1.4.2) msgpack (~> 1.0) + browser (5.3.1) builder (3.2.3) bullet (7.1.6) activesupport (>= 3.0.0) uniform_notifier (~> 1.11) byebug (11.0.1) concurrent-ruby (1.1.5) + connection_pool (2.4.1) crass (1.0.4) database_cleaner-active_record (2.0.1) activerecord (>= 5.a) @@ -78,6 +80,9 @@ GEM mini_mime (>= 0.1.1) marcel (0.3.3) mimemagic (~> 0.3.2) + meta_request (0.8.2) + rack-contrib (>= 1.1, < 3) + railties (>= 3.0.0, < 8) method_source (0.9.2) mimemagic (0.3.10) nokogiri (~> 1) @@ -96,6 +101,8 @@ GEM activerecord (>= 5) puma (3.12.1) rack (2.0.6) + rack-contrib (2.4.0) + rack (< 4) rack-mini-profiler (3.1.1) rack (>= 1.2.0) rack-test (1.1.0) @@ -118,6 +125,11 @@ GEM nokogiri (>= 1.6) rails-html-sanitizer (1.0.4) loofah (~> 2.2, >= 2.2.2) + rails_performance (1.2.2) + browser + railties + redis + redis-namespace railties (5.2.3) actionpack (= 5.2.3) activesupport (= 5.2.3) @@ -128,6 +140,12 @@ GEM rb-fsevent (0.10.3) rb-inotify (0.10.0) ffi (~> 1.0) + redis (5.2.0) + redis-client (>= 0.22.0) + redis-client (0.22.1) + connection_pool + redis-namespace (1.11.0) + redis (>= 4) rspec-core (3.13.0) rspec-support (~> 3.13.0) rspec-expectations (3.13.0) @@ -154,6 +172,8 @@ GEM activesupport (>= 4.0) sprockets (>= 3.0.0) stackprof (0.2.26) + strong_migrations (1.8.0) + activerecord (>= 5.2) thor (0.20.3) thread_safe (0.3.6) tzinfo (1.2.5) @@ -178,14 +198,17 @@ DEPENDENCIES byebug database_cleaner-active_record (~> 2.0.1) listen (>= 3.0.5, < 3.2) + meta_request pg (>= 0.18, < 2.0) pg_query (>= 0.9.0) pghero puma (~> 3.11) rack-mini-profiler rails (~> 5.2.3) + rails_performance rspec-rails (~> 4.0.0) stackprof + strong_migrations tzinfo-data web-console (>= 3.3.0) diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index acb38be2..f3afcb1d 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -2,6 +2,6 @@ class TripsController < ApplicationController def index @from = City.find_by_name!(params[:from]) @to = City.find_by_name!(params[:to]) - @trips = Trip.where(from: @from, to: @to).order(:start_time) + @trips = Trip.where(from: @from, to: @to).includes([bus: [:services]]).order(:start_time) end end diff --git a/app/views/trips/_delimiter.html.erb b/app/views/trips/_delimiter.html.erb deleted file mode 100644 index 3f845ad0..00000000 --- a/app/views/trips/_delimiter.html.erb +++ /dev/null @@ -1 +0,0 @@ -==================================================== diff --git a/app/views/trips/_service.html.erb b/app/views/trips/_service.html.erb deleted file mode 100644 index 178ea8c0..00000000 --- a/app/views/trips/_service.html.erb +++ /dev/null @@ -1 +0,0 @@ -
  • <%= "#{service.name}" %>
    • diff --git a/app/views/trips/_services.html.erb b/app/views/trips/_services.html.erb deleted file mode 100644 index 2de639fc..00000000 --- a/app/views/trips/_services.html.erb +++ /dev/null @@ -1,6 +0,0 @@ -
  • Сервисы в автобусе:
    • -
      - <% services.each do |service| %> - <%= render "service", service: service %> - <% end %> -
    diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb deleted file mode 100644 index fa1de9aa..00000000 --- a/app/views/trips/_trip.html.erb +++ /dev/null @@ -1,5 +0,0 @@ -
  • <%= "Отправление: #{trip.start_time}" %>
    • -
  • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
    • -
  • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
    • -
  • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
    • -
  • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
    • diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index b8c906ef..ad5d133a 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -2,7 +2,7 @@ <%= "Автобусы #{@from.name} – #{@to.name}" %>

    - <%= "В расписании #{@trips.size} рейсов" %> + <%= "В расписании #{@trips.length} рейсов" %>

    <% @trips.each do |trip| %> diff --git a/config/environments/development.rb b/config/environments/development.rb index 1311e3e4..8468e80a 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -1,4 +1,13 @@ Rails.application.configure do + # config.after_initialize do + # Bullet.enable = true + # Bullet.alert = true + # Bullet.bullet_logger = true + # Bullet.console = true + # Bullet.rails_logger = true + # Bullet.add_footer = true + # end + # Settings specified here will take precedence over those in config/application.rb. # In the development environment your application's code is reloaded on diff --git a/config/initializers/strong_migrations.rb b/config/initializers/strong_migrations.rb new file mode 100644 index 00000000..a79084a4 --- /dev/null +++ b/config/initializers/strong_migrations.rb @@ -0,0 +1,26 @@ +# Mark existing migrations as safe +StrongMigrations.start_after = 20240521214041 + +# Set timeouts for migrations +# If you use PgBouncer in transaction mode, delete these lines and set timeouts on the database user +StrongMigrations.lock_timeout = 10.seconds +StrongMigrations.statement_timeout = 1.hour + +# Analyze tables after indexes are added +# Outdated statistics can sometimes hurt performance +StrongMigrations.auto_analyze = true + +# Set the version of the production database +# so the right checks are run in development +# StrongMigrations.target_version = 10 + +# Add custom checks +# StrongMigrations.add_check do |method, args| +# if method == :add_index && args[0].to_s == "users" +# stop! "No more indexes on the users table" +# end +# end + +# Make some operations safe by default +# See https://github.com/ankane/strong_migrations#safe-by-default +# StrongMigrations.safe_by_default = true diff --git a/db/migrate/20240521210145_create_pghero_query_stats.rb b/db/migrate/20240521210145_create_pghero_query_stats.rb deleted file mode 100644 index fbf41263..00000000 --- a/db/migrate/20240521210145_create_pghero_query_stats.rb +++ /dev/null @@ -1,15 +0,0 @@ -class CreatePgheroQueryStats < ActiveRecord::Migration[5.2] - def change - create_table :pghero_query_stats do |t| - t.text :database - t.text :user - t.text :query - t.integer :query_hash, limit: 8 - t.float :total_time - t.integer :calls, limit: 8 - t.timestamp :captured_at - end - - add_index :pghero_query_stats, [:database, :captured_at] - end -end diff --git a/db/migrate/20240521220915_add_index_to_trips_from_id_and_to_id.rb b/db/migrate/20240521220915_add_index_to_trips_from_id_and_to_id.rb new file mode 100644 index 00000000..0e5f6898 --- /dev/null +++ b/db/migrate/20240521220915_add_index_to_trips_from_id_and_to_id.rb @@ -0,0 +1,7 @@ +class AddIndexToTripsFromIdAndToId < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + add_index :trips, [:from_id, :to_id], unique: false, algorithm: :concurrently + end +end diff --git a/db/migrate/20240521221903_add_index_to_cities_name.rb b/db/migrate/20240521221903_add_index_to_cities_name.rb new file mode 100644 index 00000000..0138da52 --- /dev/null +++ b/db/migrate/20240521221903_add_index_to_cities_name.rb @@ -0,0 +1,7 @@ +class AddIndexToCitiesName < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + add_index :cities, :name, algorithm: :concurrently + end +end diff --git a/db/migrate/20240521222235_add_index_to_buses_services_bus.rb b/db/migrate/20240521222235_add_index_to_buses_services_bus.rb new file mode 100644 index 00000000..d808482d --- /dev/null +++ b/db/migrate/20240521222235_add_index_to_buses_services_bus.rb @@ -0,0 +1,7 @@ +class AddIndexToBusesServicesBus < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + add_index :buses_services, :bus_id, algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index a0a9b49b..a5028685 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2024_05_21_210145) do +ActiveRecord::Schema.define(version: 2024_05_21_222235) do # These are extensions that must be enabled in order to support this database enable_extension "pg_stat_statements" @@ -25,21 +25,12 @@ create_table "buses_services", force: :cascade do |t| t.integer "bus_id" t.integer "service_id" + t.index ["bus_id"], name: "index_buses_services_on_bus_id" end create_table "cities", force: :cascade do |t| t.string "name" - end - - create_table "pghero_query_stats", force: :cascade do |t| - t.text "database" - t.text "user" - t.text "query" - t.bigint "query_hash" - t.float "total_time" - t.bigint "calls" - t.datetime "captured_at" - t.index ["database", "captured_at"], name: "index_pghero_query_stats_on_database_and_captured_at" + t.index ["name"], name: "index_cities_on_name" end create_table "services", force: :cascade do |t| @@ -53,6 +44,7 @@ t.integer "duration_minutes" t.integer "price_cents" t.integer "bus_id" + t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id" end end From ab50dc09e4dedda112867578edcae193612abb98 Mon Sep 17 00:00:00 2001 From: Sergey Kochubey Date: Wed, 22 May 2024 21:07:57 +0300 Subject: [PATCH 9/9] trips schedule optimization --- Gemfile | 1 + Gemfile.lock | 33 +++--- app/models/trip.rb | 18 +-- app/services/import_trips_service.rb | 49 ++++++++ case-study.md | 123 +++++++++++++++++++++ lib/tasks/utils.rake | 78 ++++++++----- spec/controllers/trips_controller_spec.rb | 12 ++ spec/services/import_trips_service_spec.rb | 21 ++++ spec/spec_helper.rb | 15 +-- 9 files changed, 279 insertions(+), 71 deletions(-) create mode 100644 app/services/import_trips_service.rb create mode 100644 case-study.md create mode 100644 spec/controllers/trips_controller_spec.rb create mode 100644 spec/services/import_trips_service_spec.rb diff --git a/Gemfile b/Gemfile index c1e2713e..5c711958 100644 --- a/Gemfile +++ b/Gemfile @@ -34,6 +34,7 @@ group :development do end group :test do + gem 'rspec-benchmark' end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem diff --git a/Gemfile.lock b/Gemfile.lock index 6264b100..4f0f1440 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -45,17 +45,18 @@ GEM minitest (~> 5.1) tzinfo (~> 1.1) arel (9.0.0) + benchmark-malloc (0.2.0) + benchmark-perf (0.6.0) + benchmark-trend (0.4.0) bindex (0.6.0) bootsnap (1.4.2) msgpack (~> 1.0) - browser (5.3.1) builder (3.2.3) bullet (7.1.6) activesupport (>= 3.0.0) uniform_notifier (~> 1.11) byebug (11.0.1) concurrent-ruby (1.1.5) - connection_pool (2.4.1) crass (1.0.4) database_cleaner-active_record (2.0.1) activerecord (>= 5.a) @@ -80,9 +81,6 @@ GEM mini_mime (>= 0.1.1) marcel (0.3.3) mimemagic (~> 0.3.2) - meta_request (0.8.2) - rack-contrib (>= 1.1, < 3) - railties (>= 3.0.0, < 8) method_source (0.9.2) mimemagic (0.3.10) nokogiri (~> 1) @@ -101,8 +99,6 @@ GEM activerecord (>= 5) puma (3.12.1) rack (2.0.6) - rack-contrib (2.4.0) - rack (< 4) rack-mini-profiler (3.1.1) rack (>= 1.2.0) rack-test (1.1.0) @@ -125,11 +121,6 @@ GEM nokogiri (>= 1.6) rails-html-sanitizer (1.0.4) loofah (~> 2.2, >= 2.2.2) - rails_performance (1.2.2) - browser - railties - redis - redis-namespace railties (5.2.3) actionpack (= 5.2.3) activesupport (= 5.2.3) @@ -140,12 +131,15 @@ GEM rb-fsevent (0.10.3) rb-inotify (0.10.0) ffi (~> 1.0) - redis (5.2.0) - redis-client (>= 0.22.0) - redis-client (0.22.1) - connection_pool - redis-namespace (1.11.0) - redis (>= 4) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-benchmark (0.6.0) + benchmark-malloc (~> 0.2) + benchmark-perf (~> 0.6) + benchmark-trend (~> 0.4) + rspec (>= 3.0) rspec-core (3.13.0) rspec-support (~> 3.13.0) rspec-expectations (3.13.0) @@ -198,14 +192,13 @@ DEPENDENCIES byebug database_cleaner-active_record (~> 2.0.1) listen (>= 3.0.5, < 3.2) - meta_request pg (>= 0.18, < 2.0) pg_query (>= 0.9.0) pghero puma (~> 3.11) rack-mini-profiler rails (~> 5.2.3) - rails_performance + rspec-benchmark rspec-rails (~> 4.0.0) stackprof strong_migrations diff --git a/app/models/trip.rb b/app/models/trip.rb index 9d63dfff..82920422 100644 --- a/app/models/trip.rb +++ b/app/models/trip.rb @@ -17,16 +17,16 @@ class Trip < ApplicationRecord def to_h { - from: from.name, - to: to.name, - start_time: start_time, - duration_minutes: duration_minutes, - price_cents: price_cents, - bus: { - number: bus.number, - model: bus.model, - services: bus.services.map(&:name), + 'bus' => { + 'model' => bus.model, + 'number' => bus.number, + 'services' => bus.services.map(&:name), }, + 'duration_minutes' => duration_minutes, + 'from' => from.name, + 'price_cents' => price_cents, + 'start_time' => start_time, + 'to' => to.name } end end diff --git a/app/services/import_trips_service.rb b/app/services/import_trips_service.rb new file mode 100644 index 00000000..511f438a --- /dev/null +++ b/app/services/import_trips_service.rb @@ -0,0 +1,49 @@ +class ImportTripsService + def self.call(filename) + json = JSON.parse(File.read(filename)) + + ActiveRecord::Base.transaction do + City.delete_all + Bus.delete_all + Service.delete_all + Trip.delete_all + ActiveRecord::Base.connection.execute('delete from buses_services;') + + cities = {} + services = {} + buses = {} + buses_services = {} + + trips = [] + + json.each do |trip| + from = cities[trip['from']] ||= City.new(name: trip['from']) + to = cities[trip['to']] ||= City.new(name: trip['to']) + + bus = buses[trip['bus']['number']] ||= Bus.new(number: trip['bus']['number'], model: trip['bus']['model']) + + trip['bus']['services'].each do |service| + bus_service = services[service] ||= Service.new(name: service) + buses_services[[bus, bus_service]] ||= BusesService.new(bus: bus, service: bus_service) + end + + trips << Trip.new( + from: from, + to: to, + bus: bus, + start_time: trip['start_time'], + duration_minutes: trip['duration_minutes'], + price_cents: trip['price_cents'], + ) + end + + City.import cities.values + Bus.import buses.values + Service.import services.values + BusesService.import buses_services.values + Trip.import trips + + cities, services, buses, trips = nil + end + end +end \ No newline at end of file diff --git a/case-study.md b/case-study.md new file mode 100644 index 00000000..929e04ec --- /dev/null +++ b/case-study.md @@ -0,0 +1,123 @@ +# Case-study оптимизации + +## Актуальная проблема: Импорт данных +К приложению поставляются файлы для импорта рейсов, но скрипт для их загрузки в БД работает очень долго для большого количества рейсов: +- `small.json` (1K рейсов) - 16,4 сек +- `medium.json` (10K рейсов) - 131 сек +- `large.json` (100K рейсов) - **не измерял**, но должны уложиться в бюджет в **пределах минуты** + +Будем использовать в качестве метрики загрузку файла с 1К рейсов - **16,4 сек** + +### Гарантия корректности работы оптимизированного скрипта +Перенёс работу скрипта в сервис и написал для него тест: +```ruby +context 'Correct trips import' do + let(:trip_from_file) { JSON.parse(File.read('fixtures/example.json')) } + + before { ImportTripsService.call('fixtures/example.json') } + + it 'trips from DB are equal to imported' do + expect(Trip.all.map(&:to_h).sort_by(&:to_s)).to eq trip_from_file.sort_by(&:to_s) + end +end +``` +Полученные из DB отсортированные рейсы должны быть эквивалентными рейсам из файла. Для проверки немного подкорректировал метод `Trip#to_f` + +### Feedback-Loop +Для эффективной работы по оптимизации выстроил `feedback-loop`, который позволил мне получать обратную связь по эффективности сделанных изменений. +- Поставил pgHero для изучения поведения запросов +- В самом скрипте провожу замер скорости выполнения +- В тесте сервиса слежу, чтобы проводимые изменения не ломали логику выполнения импорта рейсов + + +### Находка №1 - большое количество запросов SELECT +Изучаем статистику pgHero, обращаем внимание на некоторые нюансы которые видишь впервые: +- `pkeys` весят соизмеримо с самими талицами +- При импорте - сами запросы простые, но их количество слишком большое - **12861** запрос на **1000** рейсов +Сравниваю запросы с кодом скрипта - нахожу строки где вызывается их выполнение. Провожу рефакторинг - в основном добавил хранение City, Service и Bus во временных хэшах. +- Метрика изменилась: 1К рейсов - с **16,4 сек** до **7,5 сек**. Самих запросов SELECT стало значительно меньше - < 4000 + +### Находка №2 - добавление индекса к автобусам +Дальнейшее изучение pgHero показывает, что 44% тратится на INSERT INTO trips и 23% SELECT from buses. Сами рейсы пока трогать не будем, добавим индекс к автобусам. +*тогда ещё не был поставлен гем strong_migration, а так он бы поругался, что не хватает concurrency* +```ruby +add_index :buses, :number, unique: true +``` +- SELECT автобусов стал занимать 12%, а общая загрузка уменьшилась до **6,9 сек** + +### Находка 3 - activerecord-import +Становится понятно, что основное время тратится на обращение и подключение к БД при создании объектов, поэтому думаем как это обойти. Для этого будем использовать гем `activerecord-import`, который позволяет сохранить все объекты помещённые во временное хранилище за раз. Добавляем эту возможность в скрипт. Всё работает очень быстро, но тут падает тест на логику импорта - в базе не сохраняются сервисы автобусов. +Изучение этой проблемы показало что `activerecord-import` не поддерживает ассоциации (хотя в описании должен). Небольшое ознакомление с проблемой, показало, что не я один сталкивался с ней, но простого решения для неё нет. +Тогда добавим модель `BusesService` к соединительной таблице `buses_servises`, и собираем и сохраняем объекты тоже отдельно. +- Метрика изменилась: 1К рейсов - с **6,9 сек** до **1,5 сек**. Ура! + +### Попадание в метрику. Выводы. Тесты +Теперь можно проводить замер и на `large.json` - **55 сек** - как раз укладываемся в метрику! +Стало интересно сколько тратится памяти при таком импорте данных: +- small - 1.5 sec - 113 MB +- medium - 7 sec - 181 MB +- large - 55 sec - 732 MB + +Думал, что с увеличением количества рейсов памяти будет расходоваться больше. Наверное, из-за того что данные повторяются не происходит большой расход памяти. +Можно было попробовать обновить версию `rails` до 6 и использовать `insert_all`, или переписать импорт на потоковый режим, чтобы попытаться осилить файлы побольше, но так как имеется отставание по времени прохождения курса - переходим ко второй части ДЗ. +- Защитил скрипт тестом, время импорта 1000 рейсов не должно быть больше 2-х сек +```ruby +expect { ImportTripsService.call('fixtures/small.json') }.to perform_under(2).sec +``` + +## Актуальная проблема: Отображение расписаний +Когда рейсов становится слишком много страница с расписанием грузится очень медленно. +С файлом `small` время страницы `http://localhost:3000/автобусы/Самара/Москва` - 579ms + +### Находка 1 - чрезмерная загрузка паршиалов + COUNT +Первым делом изучаем логи сервера: +```log +Completed 200 OK in 579ms (Views: 475.8ms | ActiveRecord: 81.6ms) +``` +Большая часть времени - загрузка вьюх. И много времени уходит на запрос COUNT (@trips.count). Помним, что это всегда лишний запрос, поэтому меняем на #size. Эффекта нет - запрос COUNT продолжает отправляться. Пробуем #length - помогает. *Интересно, как в современных рельсах работает вопрос о размере* +Рефакторим вьюхи и меняем count на length: +```log +Completed 200 OK in 180ms (Views: 107.2ms | ActiveRecord: 70.1ms) +``` +Наблюдаем положительный эффект и там и там + +### Находка 2 Eager loading +Так как тема текущего урока - оптимизация БД, а не вьюх - увеличиваем количество рейсов до 10000 (medium.json). Время загрузки - 800 мс +Ставим `bullet` - он советует добавить: +- includes([:bus]) +- includes([:services]) +Получаем время загрузки вместо **800 мс - 155 мс**. Помним о `preload` - ставим его вместо `includes`, но разницы по времени вообще не получаем, поэтому оставляем `includes` - как и советовал изначально `bullet` + +### Находка 3 - индексы +Увеличиваем кол-во рейсов до 100к. Наш запрос занимает 600 мс. Из них 530 - вьюха, 48 - AR +Уже до этого ставили `rack-mini-profiler`, но знакомился с ним поверхностно - обращал внимание только на общее время загрузки страницы. Теперь решил изучить подробнее. Большое положительное удивление доставило количество опций - когда можно открывать отчёт с другими профилировщиками. Когда видишь это на уроке - не проникаешься его мощью. Но когда уже все запросы как родные - `rack-mini-profiler` может заменить почти всё остальное (наверное) + +Изучение `pgHero` и `rack-mini-profiler` показывает, что **41%** времени занимает занимает запрос рейсов где идёт поис `from_id` -> `to_id` - Значит, это первый претендент на индекс. + +Перед добавлением индексов ставим: +- strong_migrations +- rails panel +- rails_performance (вообще ничего интересного не показал) + +Добавляем индекс к +- cities :names +- trips [:from_id, :to_id] +`strong_migrations` предлагает использовать опции `disable_ddl_transaction!`, `algorithm: :concurrently` - окей, мы для этого его и ставили. +Получаем загрузку ActiveRecord: 18 ms. Загрузка вьюх остаётя прежней. + +Считаю, оптимизации загрузки страниц с рейсами мы добились. + +### Попадание в метрику. Выводы. Тесты +Итак страница с 1004 рейсами из 100_000 грузится за 485ms, из которых работа с БД - 20 ms. Возможно можно добавить пагинацию, или соорудить один мощный запрос, но уже поджимает время сдачи. Считаю что попадаем в метрику загрузки страницы с рейсами. + +Пишем тест для защиты полученных результатов в оптимизации. Он заключается в том, что экшен trips#index с параметрами Самара -> Москва должен отработать менее чем за 5 мс (100 000 записей в БД) +```ruby + describe 'GET #index' do + before { ImportTripsService.call('fixtures/medium.json') } + + it 'load 10 000 trips under 5 ms' do + expect { get :index, params: { from: 'Самара', to: 'Москва' } }.to perform_under(5).ms + end + end + ``` + diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 9f55d8bb..fc13eebc 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,34 +1,54 @@ -# Наивная загрузка данных из json-файла в БД # rake reload_json[fixtures/small.json] task :reload_json, [:file_name] => :environment do |_task, args| - json = JSON.parse(File.read(args.file_name)) - - ActiveRecord::Base.transaction do - City.delete_all - Bus.delete_all - Service.delete_all - Trip.delete_all - ActiveRecord::Base.connection.execute('delete from buses_services;') - - json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) - services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s + + time = Benchmark.measure do + json = JSON.parse(File.read(args.file_name)) + + ActiveRecord::Base.transaction do + City.delete_all + Bus.delete_all + Service.delete_all + Trip.delete_all + ActiveRecord::Base.connection.execute('delete from buses_services;') + + cities = {} + services = {} + buses = {} + buses_services = {} + + trips = [] + + json.each do |trip| + from = cities[trip['from']] ||= City.new(name: trip['from']) + to = cities[trip['to']] ||= City.new(name: trip['to']) + + bus = buses[trip['bus']['number']] ||= Bus.new(number: trip['bus']['number'], model: trip['bus']['model']) + + trip['bus']['services'].each do |service| + bus_service = services[service] ||= Service.new(name: service) + buses_services[[bus, bus_service]] ||= BusesService.new(bus: bus, service: bus_service) + end + + trips << Trip.new( + from: from, + to: to, + bus: bus, + start_time: trip['start_time'], + duration_minutes: trip['duration_minutes'], + price_cents: trip['price_cents'], + ) end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) - - Trip.create!( - from: from, - to: to, - bus: bus, - start_time: trip['start_time'], - duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], - ) - end + + City.import cities.values + Bus.import buses.values + Service.import services.values + BusesService.import buses_services.values + Trip.import trips + + cities, services, buses, trips = nil + end end + + puts "Done in #{time.real} seconds" + puts "MEMORY USAGE: %d MB" % (`ps -o rss= -p #{Process.pid}`.to_i / 1024) end diff --git a/spec/controllers/trips_controller_spec.rb b/spec/controllers/trips_controller_spec.rb new file mode 100644 index 00000000..e4f8708d --- /dev/null +++ b/spec/controllers/trips_controller_spec.rb @@ -0,0 +1,12 @@ +require 'rails_helper' + +RSpec.describe TripsController, type: :controller do + + describe 'GET #index' do + before { ImportTripsService.call('fixtures/medium.json') } + + it 'load 10 000 trips under 5 ms' do + expect { get :index, params: { from: 'Самара', to: 'Москва' } }.to perform_under(5).ms + end + end +end diff --git a/spec/services/import_trips_service_spec.rb b/spec/services/import_trips_service_spec.rb new file mode 100644 index 00000000..945b4d72 --- /dev/null +++ b/spec/services/import_trips_service_spec.rb @@ -0,0 +1,21 @@ +require 'rails_helper' + +describe "ImportTripsService" do + context 'Correct trips import' do + let(:trip_from_file) { JSON.parse(File.read('fixtures/example.json')) } + + before { ImportTripsService.call('fixtures/example.json') } + + it 'trips from DB are equal to imported' do + expect(Trip.all.map(&:to_h).sort_by(&:to_s)).to eq trip_from_file.sort_by(&:to_s) + end + end + + it 'save trips in DB' do + expect { ImportTripsService.call('fixtures/example.json') }.to change { Trip.count }.by(10) + end + + it '1000 trips perform under 2 sec' do + expect { ImportTripsService.call('fixtures/small.json') }.to perform_under(2).sec + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 327b58ea..9fe9c528 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,16 +1,4 @@ -# This file was generated by the `rails generate rspec:install` command. Conventionally, all -# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. -# The generated `.rspec` file contains `--require spec_helper` which will cause -# this file to always be loaded, without a need to explicitly require it in any -# files. -# -# Given that it is always loaded, you are encouraged to keep this file as -# light-weight as possible. Requiring heavyweight dependencies from this file -# will add to the boot time of your test suite on EVERY test run, even for an -# individual file that may not need all of that loaded. Instead, consider making -# a separate helper file that requires the additional dependencies and performs -# the additional setup, and require it from the spec files that actually need -# it. +require 'rspec-benchmark' # # See https://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration RSpec.configure do |config| @@ -91,4 +79,5 @@ # as the one that triggered the failure. Kernel.srand config.seed =end + config.include RSpec::Benchmark::Matchers end