From 43f7116358ad31742d09f60aab670c3a476d4a3c Mon Sep 17 00:00:00 2001 From: Dmitry Badichan Date: Mon, 24 Feb 2020 10:35:28 +0100 Subject: [PATCH 1/5] Change rake import --- .gitignore | 1 + .rspec | 2 + Gemfile | 16 ++++++ Gemfile.lock | 41 +++++++++++++++ app/models/bus.rb | 12 ++++- app/models/bus_service.rb | 14 +++++ app/models/city.rb | 7 +++ app/models/service.rb | 12 ++++- app/models/trip.rb | 12 +++++ app/services/import_data.rb | 100 ++++++++++++++++++++++++++++++++++++ lib/tasks/utils.rake | 37 ++++--------- ruby_prof_reports/.keep | 0 spec/data.json | 38 ++++++++++++++ spec/import_data_spec.rb | 33 ++++++++++++ spec/rails_helper.rb | 21 ++++++++ spec/spec_helper.rb | 12 +++++ 16 files changed, 329 insertions(+), 29 deletions(-) create mode 100644 .rspec create mode 100644 app/models/bus_service.rb create mode 100644 app/services/import_data.rb create mode 100644 ruby_prof_reports/.keep create mode 100644 spec/data.json create mode 100644 spec/import_data_spec.rb create mode 100644 spec/rails_helper.rb create mode 100644 spec/spec_helper.rb diff --git a/.gitignore b/.gitignore index 59c74047..1c98b2c9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /tmp /log /public +/coverage diff --git a/.rspec b/.rspec new file mode 100644 index 00000000..210a1d6b --- /dev/null +++ b/.rspec @@ -0,0 +1,2 @@ +--format documentation +--color \ No newline at end of file diff --git a/Gemfile b/Gemfile index e20b1260..0cb6ba00 100644 --- a/Gemfile +++ b/Gemfile @@ -14,13 +14,29 @@ group :development, :test do end group :development do + gem "annotate", "~> 3.1" + # Access an interactive console on exception pages or by calling 'console' anywhere in the code. gem 'web-console', '>= 3.3.0' gem 'listen', '>= 3.0.5', '< 3.2' + gem 'rspec-rails' end group :test do + gem 'rspec-rails' + gem 'simplecov' end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] + +gem "rack-mini-profiler", "~> 1.1" + +gem "stackprof", "~> 0.2.15" + +gem "oj", "~> 3.10" + +gem "ruby-prof", "~> 1.2" + +gem "activerecord-import", "~> 1.0" + diff --git a/Gemfile.lock b/Gemfile.lock index fccf6f5f..c6f03dff 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -33,6 +33,8 @@ GEM activemodel (= 5.2.3) activesupport (= 5.2.3) arel (>= 9.0) + activerecord-import (1.0.4) + activerecord (>= 3.2) activestorage (5.2.3) actionpack (= 5.2.3) activerecord (= 5.2.3) @@ -42,6 +44,9 @@ GEM i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) + annotate (3.1.0) + activerecord (>= 3.2, < 7.0) + rake (>= 10.4, < 14.0) arel (9.0.0) bindex (0.6.0) bootsnap (1.4.2) @@ -50,6 +55,8 @@ GEM byebug (11.0.1) concurrent-ruby (1.1.5) crass (1.0.4) + diff-lcs (1.3) + docile (1.3.2) erubi (1.8.0) ffi (1.10.0) globalid (0.4.2) @@ -76,9 +83,12 @@ GEM nio4r (2.3.1) nokogiri (1.10.2) mini_portile2 (~> 2.4.0) + oj (3.10.2) pg (1.1.4) puma (3.12.1) rack (2.0.6) + rack-mini-profiler (1.1.6) + rack (>= 1.2.0) rack-test (1.1.0) rack (>= 1.0, < 3) rails (5.2.3) @@ -109,7 +119,29 @@ GEM rb-fsevent (0.10.3) rb-inotify (0.10.0) ffi (~> 1.0) + rspec-core (3.9.1) + rspec-support (~> 3.9.1) + rspec-expectations (3.9.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.9.0) + rspec-mocks (3.9.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.9.0) + rspec-rails (3.9.0) + actionpack (>= 3.0) + activesupport (>= 3.0) + railties (>= 3.0) + rspec-core (~> 3.9.0) + rspec-expectations (~> 3.9.0) + rspec-mocks (~> 3.9.0) + rspec-support (~> 3.9.0) + rspec-support (3.9.2) + ruby-prof (1.2.0) ruby_dep (1.5.0) + simplecov (0.18.2) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov-html (0.12.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -117,6 +149,7 @@ GEM actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) + stackprof (0.2.15) thor (0.20.3) thread_safe (0.3.6) tzinfo (1.2.5) @@ -134,12 +167,20 @@ PLATFORMS ruby DEPENDENCIES + activerecord-import (~> 1.0) + annotate (~> 3.1) bootsnap (>= 1.1.0) byebug listen (>= 3.0.5, < 3.2) + oj (~> 3.10) pg (>= 0.18, < 2.0) puma (~> 3.11) + rack-mini-profiler (~> 1.1) rails (~> 5.2.3) + rspec-rails + ruby-prof (~> 1.2) + simplecov + stackprof (~> 0.2.15) tzinfo-data web-console (>= 3.3.0) diff --git a/app/models/bus.rb b/app/models/bus.rb index 1dcc54cb..4fc641c7 100644 --- a/app/models/bus.rb +++ b/app/models/bus.rb @@ -1,3 +1,11 @@ +# == Schema Information +# +# Table name: buses +# +# id :bigint not null, primary key +# number :string +# model :string +# class Bus < ApplicationRecord MODELS = [ 'Икарус', @@ -13,7 +21,9 @@ class Bus < ApplicationRecord ].freeze has_many :trips - has_and_belongs_to_many :services, join_table: :buses_services + + has_many :buses_services, class_name: 'BusService' + has_many :services, through: :buses_services validates :number, presence: true, uniqueness: true validates :model, inclusion: { in: MODELS } diff --git a/app/models/bus_service.rb b/app/models/bus_service.rb new file mode 100644 index 00000000..41ee0234 --- /dev/null +++ b/app/models/bus_service.rb @@ -0,0 +1,14 @@ +# == Schema Information +# +# Table name: buses_services +# +# id :bigint not null, primary key +# bus_id :integer +# service_id :integer +# +class BusService < ApplicationRecord + self.table_name = 'buses_services' + + belongs_to :bus + belongs_to :service +end diff --git a/app/models/city.rb b/app/models/city.rb index 19ec7f36..957290d8 100644 --- a/app/models/city.rb +++ b/app/models/city.rb @@ -1,3 +1,10 @@ +# == Schema Information +# +# Table name: cities +# +# id :bigint not null, primary key +# name :string +# class City < ApplicationRecord validates :name, presence: true, uniqueness: true validate :name_has_no_spaces diff --git a/app/models/service.rb b/app/models/service.rb index 9cbb2a32..f006da7b 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -1,3 +1,10 @@ +# == Schema Information +# +# Table name: services +# +# id :bigint not null, primary key +# name :string +# class Service < ApplicationRecord SERVICES = [ 'WiFi', @@ -12,7 +19,10 @@ class Service < ApplicationRecord 'Можно не печатать билет', ].freeze - has_and_belongs_to_many :buses, join_table: :buses_services + has_many :buses_services, + class_name: 'BusService' + + has_many :buses, through: :buses_services validates :name, presence: true validates :name, inclusion: { in: SERVICES } diff --git a/app/models/trip.rb b/app/models/trip.rb index 9d63dfff..f610f267 100644 --- a/app/models/trip.rb +++ b/app/models/trip.rb @@ -1,3 +1,15 @@ +# == Schema Information +# +# Table name: trips +# +# id :bigint not null, primary key +# from_id :integer +# to_id :integer +# start_time :string +# duration_minutes :integer +# price_cents :integer +# bus_id :integer +# class Trip < ApplicationRecord HHMM_REGEXP = /([0-1][0-9]|[2][0-3]):[0-5][0-9]/ diff --git a/app/services/import_data.rb b/app/services/import_data.rb new file mode 100644 index 00000000..a0ed7473 --- /dev/null +++ b/app/services/import_data.rb @@ -0,0 +1,100 @@ +class ImportData + attr_reader :file_name, :cities, :buses, :services, :buses_services + + def initialize(file_name) + @file_name = file_name + @cities = {} + @buses = {} + @services = {} + @buses_services = {} + end + + def exec + ActiveRecord::Base.transaction do + trips_command = + "copy trips (from_id, to_id, start_time, duration_minutes, price_cents, bus_id) from stdin with csv delimiter ';'" + + City.delete_all + Bus.delete_all + Service.delete_all + Trip.delete_all + ActiveRecord::Base.connection.execute("delete from buses_services;") + ActiveRecord::Base.connection.reset_pk_sequence!("cities") + ActiveRecord::Base.connection.reset_pk_sequence!("buses") + ActiveRecord::Base.connection.reset_pk_sequence!("services") + + connection.copy_data trips_command do + File.open(file_name) do |ff| + nesting = 0 + str = +"" + + while !ff.eof? + ch = ff.read(1) # читаем по одному символу + case + when ch == "{" # начинается объект, повышается вложенность + nesting += 1 + str << ch + when ch == "}" # заканчивается объект, понижается вложенность + nesting -= 1 + str << ch + if nesting == 0 # если закончился объкет уровня trip, парсим и импортируем его + trip = Oj.load(str) + import(trip) + # progress_bar.increment + str = +"" + end + when nesting >= 1 + str << ch + end + end + end + end + + City.import @cities.map { |el| { name: el[0] } } + Service.import @services.map { |el| { name: el[0] } } + Bus.import @buses.map { |hash_key, el| { number: hash_key, model: el[:model] } } + BusService.import @buses_services.map { |el| { service_id: el[0], bus_id: el[1] } } + end + end + + private + + def parse_id(dict, hash_key) + to_id = send(dict)[hash_key] + if !to_id + to_id = send(dict).size + 1 + send(dict)[hash_key] = to_id + end + to_id + end + + def bus_number(trip) + find_bus = buses[trip["number"]] + if !find_bus + bus_id = buses.size + 1 + find_bus = buses[trip["number"]] = { + id: bus_id, + model: trip["model"], + } + trip["services"].each do |service| + service_id = buses_services.size + 1 + buses_services[service_id] = bus_id + parse_id :services, service + end + end + find_bus + end + + def import(trip) + from_id = parse_id(:cities, trip["from"]) + to_id = parse_id(:cities, trip["to"]) + bus = bus_number(trip["bus"]) + + # стримим подготовленный чанк данных в postgres + connection.put_copy_data("#{from_id};#{to_id};#{trip["start_time"]};#{trip["duration_minutes"]};#{trip["price_cents"]};#{bus[:id]}\n") + end + + def connection + @connection ||= ActiveRecord::Base.connection.raw_connection + end +end diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 540fe871..c6ccca2a 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,34 +1,17 @@ +require 'ruby-prof' + # Наивная загрузка данных из json-файла в БД # rake reload_json[fixtures/small.json] task :reload_json, [:file_name] => :environment do |_task, args| - json = JSON.parse(File.read(args.file_name)) + profile = RubyProf.profile do + ImportData.new(JSON.parse(File.read(args.file_name), symbolize_names: true)).exec + end - ActiveRecord::Base.transaction do - City.delete_all - Bus.delete_all - Service.delete_all - Trip.delete_all - ActiveRecord::Base.connection.execute('delete from buses_services;') - json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) - services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s - end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) + printer1 = RubyProf::GraphHtmlPrinter.new(result) + printer1.print(File.open("ruby_prof_reports/graph.html", "w+")) - Trip.create!( - from: from, - to: to, - bus: bus, - start_time: trip['start_time'], - duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], - ) - end - end + printer = RubyProf::FlatPrinter.new(result) + printer.print(STDOUT) end +# data = JSON.parse(File.read('spec/data.json'), symbolize_names: true) diff --git a/ruby_prof_reports/.keep b/ruby_prof_reports/.keep new file mode 100644 index 00000000..e69de29b diff --git a/spec/data.json b/spec/data.json new file mode 100644 index 00000000..7796f861 --- /dev/null +++ b/spec/data.json @@ -0,0 +1,38 @@ +[ + { + "bus": { + "model": "Икарус", + "number": "123", + "services": ["Туалет", "WiFi"] + }, + "duration_minutes": 168, + "from": "Москва", + "price_cents": 474, + "start_time": "11:00", + "to": "Самара" + }, + { + "bus": { + "model": "Икарус", + "number": "123", + "services": ["Туалет", "WiFi"] + }, + "duration_minutes": 37, + "from": "Самара", + "price_cents": 173, + "start_time": "17:30", + "to": "Москва" + }, + { + "bus": { + "model": "Икарус", + "number": "123", + "services": ["Туалет", "WiFi"] + }, + "duration_minutes": 323, + "from": "Москва", + "price_cents": 672, + "start_time": "12:00", + "to": "Самара" + } +] diff --git a/spec/import_data_spec.rb b/spec/import_data_spec.rb new file mode 100644 index 00000000..0ef6248d --- /dev/null +++ b/spec/import_data_spec.rb @@ -0,0 +1,33 @@ +require 'rails_helper' + +RSpec.describe ImportData, type: :service do + describe "populate db" do + data = JSON.parse(File.read('spec/data.json'), symbolize_names: true) + + ImportData.new('spec/data.json').exec + + cities = data.map{|el| [el[:to], el[:from]]}.flatten.uniq + bus_services = data.map{|el| el[:bus][:services]}.flatten.uniq + buses = data.map{|el| el[:bus][:number]}.flatten.uniq + + models = data.inject({}) do |acc,val| + acc[val[:bus][:number]] = { + model: val[:bus][:model], + services: val[:bus][:services], + } + acc + end + + it { expect(City.count).to eq(cities.size) } + it { expect(Service.count).to eq bus_services.size } + it { expect(Bus.count).to eq buses.size } + it { expect(Trip.count).to eq data.size } + + Bus.find_each do |bus| + it { expect(bus.model).to eq models[bus.number][:model] } + it { + expect(bus.services.map(&:name)).to eq models[bus.number][:services] + } + end + end +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb new file mode 100644 index 00000000..f28ab225 --- /dev/null +++ b/spec/rails_helper.rb @@ -0,0 +1,21 @@ +require 'spec_helper' +ENV['RAILS_ENV'] ||= 'test' + +require File.expand_path('../config/environment', __dir__) + +abort("The Rails environment is running in production mode!") if Rails.env.production? +require 'rspec/rails' + +begin + ActiveRecord::Migration.maintain_test_schema! +rescue ActiveRecord::PendingMigrationError => e + puts e.to_s.strip + exit 1 +end + +RSpec.configure do |config| + config.fixture_path = "#{::Rails.root}/spec/fixtures" + config.use_transactional_fixtures = true + config.infer_spec_type_from_file_location! + config.filter_rails_from_backtrace! +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 00000000..711f9d61 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,12 @@ +require 'simplecov' +SimpleCov.start + +RSpec.configure do |config| + config.expect_with :rspec do |expectations| + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + config.mock_with :rspec do |mocks| + mocks.verify_partial_doubles = true + end + config.shared_context_metadata_behavior = :apply_to_host_groups +end From 037467757ab74077015bfd190ce5628fb591e06c Mon Sep 17 00:00:00 2001 From: Dmitry Badichan Date: Mon, 24 Feb 2020 10:40:57 +0100 Subject: [PATCH 2/5] Change rake import --- lib/tasks/utils.rake | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index c6ccca2a..8de6e573 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,17 +1,11 @@ -require 'ruby-prof' +require 'benchmark' # Наивная загрузка данных из json-файла в БД # rake reload_json[fixtures/small.json] task :reload_json, [:file_name] => :environment do |_task, args| - profile = RubyProf.profile do - ImportData.new(JSON.parse(File.read(args.file_name), symbolize_names: true)).exec + time = Benchmark.realtime do + ImportData.new(args.file_name).exec end - - printer1 = RubyProf::GraphHtmlPrinter.new(result) - printer1.print(File.open("ruby_prof_reports/graph.html", "w+")) - - printer = RubyProf::FlatPrinter.new(result) - printer.print(STDOUT) + puts "Finish in #{time.round(2)}" end -# data = JSON.parse(File.read('spec/data.json'), symbolize_names: true) From c879738acad058f15411f397254092a2447069b8 Mon Sep 17 00:00:00 2001 From: Dmitry Badichan Date: Mon, 24 Feb 2020 11:41:48 +0100 Subject: [PATCH 3/5] Change rake import --- spec/import_data_spec.rb | 51 +++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/spec/import_data_spec.rb b/spec/import_data_spec.rb index 0ef6248d..db20fb5c 100644 --- a/spec/import_data_spec.rb +++ b/spec/import_data_spec.rb @@ -1,33 +1,40 @@ -require 'rails_helper' +require "rails_helper" RSpec.describe ImportData, type: :service do describe "populate db" do - data = JSON.parse(File.read('spec/data.json'), symbolize_names: true) + data = JSON.parse(File.read("spec/data.json"), symbolize_names: true) - ImportData.new('spec/data.json').exec + ImportData.new("spec/data.json").exec - cities = data.map{|el| [el[:to], el[:from]]}.flatten.uniq - bus_services = data.map{|el| el[:bus][:services]}.flatten.uniq - buses = data.map{|el| el[:bus][:number]}.flatten.uniq + data.each do |trip| + from = City.where(name: trip[:from]) + to = City.where(name: trip[:to]) - models = data.inject({}) do |acc,val| - acc[val[:bus][:number]] = { - model: val[:bus][:model], - services: val[:bus][:services], - } - acc - end + it { expect(from).to exist } + it { expect(to).to exist } + + trip[:bus][:services].each do |service| + it { expect(Service.where(name: service)).to exist } + end + + buses = Bus.where(number: trip[:bus][:number]) + bus = buses.first + + it { expect(buses).to exist } + it { expect(bus.try(:model)).to eq trip[:bus][:model] } + + it { expect(bus&.services.map(&:name)).to match_array trip[:bus][:services] } - it { expect(City.count).to eq(cities.size) } - it { expect(Service.count).to eq bus_services.size } - it { expect(Bus.count).to eq buses.size } - it { expect(Trip.count).to eq data.size } + trips = Trip.where( + from_id: from.first.id, + to_id: to.first.id, + bus_id: bus.id, + start_time: trip[:start_time], + duration_minutes: trip[:duration_minutes], + price_cents: trip[:price_cents], + ) - Bus.find_each do |bus| - it { expect(bus.model).to eq models[bus.number][:model] } - it { - expect(bus.services.map(&:name)).to eq models[bus.number][:services] - } + it { expect(trips).to exist } end end end From 5644a4d8db8be507d537298408082ba5ab6884dd Mon Sep 17 00:00:00 2001 From: Dmitry Badichan Date: Mon, 24 Feb 2020 19:44:27 +0100 Subject: [PATCH 4/5] Change render methods --- Gemfile | 8 ++++++- Gemfile.lock | 22 ++++++++++++++++++- app/controllers/trips_controller.rb | 2 +- app/views/trips/_services.html.erb | 4 +--- app/views/trips/_trip.html.erb | 17 +++++++++----- app/views/trips/index.html.erb | 11 +--------- bin/setup | 4 ++-- config/environments/development.rb | 6 +++++ config/initializers/rack_profiler.rb | 8 +++++++ config/routes.rb | 1 + ...55_index_foreign_keys_in_buses_services.rb | 5 +++++ ...00224192256_index_foreign_keys_in_trips.rb | 5 +++++ ...0200224192257_create_pghero_query_stats.rb | 15 +++++++++++++ db/schema.rb | 16 +++++++++++++- 14 files changed, 100 insertions(+), 24 deletions(-) create mode 100644 config/initializers/rack_profiler.rb create mode 100644 db/migrate/20200224192255_index_foreign_keys_in_buses_services.rb create mode 100644 db/migrate/20200224192256_index_foreign_keys_in_trips.rb create mode 100644 db/migrate/20200224192257_create_pghero_query_stats.rb diff --git a/Gemfile b/Gemfile index 0cb6ba00..9a9cc9c7 100644 --- a/Gemfile +++ b/Gemfile @@ -15,6 +15,9 @@ end group :development do gem "annotate", "~> 3.1" + gem 'meta_request' + gem 'bullet' + gem 'active_record_doctor' # Access an interactive console on exception pages or by calling 'console' anywhere in the code. gem 'web-console', '>= 3.3.0' @@ -30,7 +33,7 @@ end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] -gem "rack-mini-profiler", "~> 1.1" +gem 'rack-mini-profiler', require: false gem "stackprof", "~> 0.2.15" @@ -40,3 +43,6 @@ gem "ruby-prof", "~> 1.2" gem "activerecord-import", "~> 1.0" +gem "pghero", "~> 2.4" + +gem "pg_query", "~> 1.2" diff --git a/Gemfile.lock b/Gemfile.lock index c6f03dff..c663c377 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -24,6 +24,9 @@ GEM erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) + active_record_doctor (1.7.1) + activerecord (>= 4.2) + railties (>= 4.2) activejob (5.2.3) activesupport (= 5.2.3) globalid (>= 0.3.6) @@ -52,6 +55,9 @@ GEM bootsnap (1.4.2) msgpack (~> 1.0) builder (3.2.3) + bullet (6.1.0) + activesupport (>= 3.0.0) + uniform_notifier (~> 1.11) byebug (11.0.1) concurrent-ruby (1.1.5) crass (1.0.4) @@ -74,6 +80,9 @@ GEM mini_mime (>= 0.1.1) marcel (0.3.3) mimemagic (~> 0.3.2) + meta_request (0.7.2) + rack-contrib (>= 1.1, < 3) + railties (>= 3.0.0, < 7) method_source (0.9.2) mimemagic (0.3.3) mini_mime (1.0.1) @@ -85,8 +94,13 @@ GEM mini_portile2 (~> 2.4.0) oj (3.10.2) pg (1.1.4) + pg_query (1.2.0) + pghero (2.4.1) + activerecord (>= 5) puma (3.12.1) rack (2.0.6) + rack-contrib (2.1.0) + rack (~> 2.0) rack-mini-profiler (1.1.6) rack (>= 1.2.0) rack-test (1.1.0) @@ -154,6 +168,7 @@ GEM thread_safe (0.3.6) tzinfo (1.2.5) thread_safe (~> 0.1) + uniform_notifier (1.13.0) web-console (3.7.0) actionview (>= 5.0) activemodel (>= 5.0) @@ -167,15 +182,20 @@ PLATFORMS ruby DEPENDENCIES + active_record_doctor activerecord-import (~> 1.0) annotate (~> 3.1) bootsnap (>= 1.1.0) + bullet byebug listen (>= 3.0.5, < 3.2) + meta_request oj (~> 3.10) pg (>= 0.18, < 2.0) + pg_query (~> 1.2) + pghero (~> 2.4) puma (~> 3.11) - rack-mini-profiler (~> 1.1) + rack-mini-profiler rails (~> 5.2.3) rspec-rails ruby-prof (~> 1.2) diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index acb38be2..9af920c4 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -2,6 +2,6 @@ class TripsController < ApplicationController def index @from = City.find_by_name!(params[:from]) @to = City.find_by_name!(params[:to]) - @trips = Trip.where(from: @from, to: @to).order(:start_time) + @trips = Trip.includes(bus: [:services, :buses_services]).references(:bus).where(from: @from, to: @to).order(:start_time) end end diff --git a/app/views/trips/_services.html.erb b/app/views/trips/_services.html.erb index 2de639fc..e7d8f27d 100644 --- a/app/views/trips/_services.html.erb +++ b/app/views/trips/_services.html.erb @@ -1,6 +1,4 @@
  • Сервисы в автобусе:
      • - <% services.each do |service| %> - <%= render "service", service: service %> - <% end %> + <%= render partial: "service", collection: services %>
    diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb index fa1de9aa..1d06e7ca 100644 --- a/app/views/trips/_trip.html.erb +++ b/app/views/trips/_trip.html.erb @@ -1,5 +1,12 @@ -
  • <%= "Отправление: #{trip.start_time}" %>
    • -
  • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
    • -
  • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
    • -
  • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
    • -
  • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
    • +<% cache trip do %> +
      +
    • <%= "Отправление: #{trip.start_time}" %>
      • +
    • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
      • +
    • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
      • +
    • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
      • +
    • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
      • + <% if trip.bus.services.present? %> + <%= render "services", services: trip.bus.services %> + <% end %> +
    +<% end %> diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce41..599b716c 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -4,13 +4,4 @@

    <%= "В расписании #{@trips.count} рейсов" %>

    - -<% @trips.each do |trip| %> -
      - <%= render "trip", trip: trip %> - <% if trip.bus.services.present? %> - <%= render "services", services: trip.bus.services %> - <% end %> -
    - <%= render "delimiter" %> -<% end %> +<%= render partial: "trip", collection: @trips, spacer_template: 'delimiter' %> diff --git a/bin/setup b/bin/setup index f294207b..5534a287 100755 --- a/bin/setup +++ b/bin/setup @@ -28,8 +28,8 @@ chdir APP_ROOT do puts "\n== Preparing database ==" system! 'bin/rails db:setup' - puts "\n== Loading data from fixtures/small.json ==" - system! 'bin/rake reload_json[fixtures/small.json]' + puts "\n== Loading data from fixtures/large.json ==" + system! 'bin/rake reload_json[fixtures/large.json]' puts "\n== Removing old logs and tempfiles ==" system! 'bin/rails log:clear tmp:clear' diff --git a/config/environments/development.rb b/config/environments/development.rb index 1311e3e4..2b5bb30e 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -58,4 +58,10 @@ # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. config.file_watcher = ActiveSupport::EventedFileUpdateChecker + + config.after_initialize do + Bullet.enable = true + Bullet.alert = true + Bullet.rails_logger = true + end end diff --git a/config/initializers/rack_profiler.rb b/config/initializers/rack_profiler.rb new file mode 100644 index 00000000..96b6cc98 --- /dev/null +++ b/config/initializers/rack_profiler.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +if Rails.env.development? + require "rack-mini-profiler" + + # initialization is skipped so trigger it + Rack::MiniProfilerRails.initialize!(Rails.application) +end diff --git a/config/routes.rb b/config/routes.rb index a2da6a7b..7f620876 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -2,4 +2,5 @@ # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html get "/" => "statistics#index" get "автобусы/:from/:to" => "trips#index" + mount PgHero::Engine, at: "pghero" end diff --git a/db/migrate/20200224192255_index_foreign_keys_in_buses_services.rb b/db/migrate/20200224192255_index_foreign_keys_in_buses_services.rb new file mode 100644 index 00000000..b5b667a5 --- /dev/null +++ b/db/migrate/20200224192255_index_foreign_keys_in_buses_services.rb @@ -0,0 +1,5 @@ +class IndexForeignKeysInBusesServices < ActiveRecord::Migration[5.2] + def change + add_index :buses_services, [:bus_id, :service_id] + end +end diff --git a/db/migrate/20200224192256_index_foreign_keys_in_trips.rb b/db/migrate/20200224192256_index_foreign_keys_in_trips.rb new file mode 100644 index 00000000..bd61700c --- /dev/null +++ b/db/migrate/20200224192256_index_foreign_keys_in_trips.rb @@ -0,0 +1,5 @@ +class IndexForeignKeysInTrips < ActiveRecord::Migration[5.2] + def change + add_index :trips, [:bus_id, :from_id, :to_id] + end +end diff --git a/db/migrate/20200224192257_create_pghero_query_stats.rb b/db/migrate/20200224192257_create_pghero_query_stats.rb new file mode 100644 index 00000000..fbf41263 --- /dev/null +++ b/db/migrate/20200224192257_create_pghero_query_stats.rb @@ -0,0 +1,15 @@ +class CreatePgheroQueryStats < ActiveRecord::Migration[5.2] + def change + create_table :pghero_query_stats do |t| + t.text :database + t.text :user + t.text :query + t.integer :query_hash, limit: 8 + t.float :total_time + t.integer :calls, limit: 8 + t.timestamp :captured_at + end + + add_index :pghero_query_stats, [:database, :captured_at] + end +end diff --git a/db/schema.rb b/db/schema.rb index f6921e45..2acef74d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,9 +10,10 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_03_30_193044) do +ActiveRecord::Schema.define(version: 2020_02_24_192257) do # These are extensions that must be enabled in order to support this database + enable_extension "pg_stat_statements" enable_extension "plpgsql" create_table "buses", force: :cascade do |t| @@ -23,12 +24,24 @@ create_table "buses_services", force: :cascade do |t| t.integer "bus_id" t.integer "service_id" + t.index ["bus_id", "service_id"], name: "index_buses_services_on_bus_id_and_service_id" end create_table "cities", force: :cascade do |t| t.string "name" end + create_table "pghero_query_stats", force: :cascade do |t| + t.text "database" + t.text "user" + t.text "query" + t.bigint "query_hash" + t.float "total_time" + t.bigint "calls" + t.datetime "captured_at" + t.index ["database", "captured_at"], name: "index_pghero_query_stats_on_database_and_captured_at" + end + create_table "services", force: :cascade do |t| t.string "name" end @@ -40,6 +53,7 @@ t.integer "duration_minutes" t.integer "price_cents" t.integer "bus_id" + t.index ["bus_id", "from_id", "to_id"], name: "index_trips_on_bus_id_and_from_id_and_to_id" end end From 9bda54f5380605d52822fc10216623e52c602008 Mon Sep 17 00:00:00 2001 From: Dmitry Badichan Date: Wed, 26 Feb 2020 18:26:06 +0100 Subject: [PATCH 5/5] Clean up --- app/services/import_data.rb | 12 ++++++------ app/views/trips/_trip.html.erb | 22 ++++++++++------------ 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/app/services/import_data.rb b/app/services/import_data.rb index a0ed7473..725905a5 100644 --- a/app/services/import_data.rb +++ b/app/services/import_data.rb @@ -60,10 +60,10 @@ def exec private def parse_id(dict, hash_key) - to_id = send(dict)[hash_key] + to_id = dict[hash_key] if !to_id - to_id = send(dict).size + 1 - send(dict)[hash_key] = to_id + to_id = dict.size + 1 + dict[hash_key] = to_id end to_id end @@ -79,15 +79,15 @@ def bus_number(trip) trip["services"].each do |service| service_id = buses_services.size + 1 buses_services[service_id] = bus_id - parse_id :services, service + parse_id services, service end end find_bus end def import(trip) - from_id = parse_id(:cities, trip["from"]) - to_id = parse_id(:cities, trip["to"]) + from_id = parse_id(cities, trip["from"]) + to_id = parse_id(cities, trip["to"]) bus = bus_number(trip["bus"]) # стримим подготовленный чанк данных в postgres diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb index 1d06e7ca..21072a33 100644 --- a/app/views/trips/_trip.html.erb +++ b/app/views/trips/_trip.html.erb @@ -1,12 +1,10 @@ -<% cache trip do %> -
      -
    • <%= "Отправление: #{trip.start_time}" %>
      • -
    • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
      • -
    • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
      • -
    • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
      • -
    • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
      • - <% if trip.bus.services.present? %> - <%= render "services", services: trip.bus.services %> - <% end %> -
    -<% end %> +
      +
    • <%= "Отправление: #{trip.start_time}" %>
      • +
    • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
      • +
    • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
      • +
    • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
      • +
    • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
      • + <% if trip.bus.services.present? %> + <%= render "services", services: trip.bus.services %> + <% end %> +