diff --git a/.dev_to/compose.yml b/.dev_to/compose.yml index 6db3e51f..a86d1574 100644 --- a/.dev_to/compose.yml +++ b/.dev_to/compose.yml @@ -45,6 +45,11 @@ x-backend: &backend PSQL_HISTFILE: /usr/local/hist/.psql_history IRB_HISTFILE: /usr/local/hist/.irb_history EDITOR: vi + NEW_RELIC_LICENSE_KEY: ${NR_LICENSE_KEY} + NEW_RELIC_APP_NAME: "dev_to" + NEW_RELIC_LOG: "stdout" + NEW_RELIC_AGENT_ENABLED: "true" + NEW_RELIC_ENV: "development" depends_on: &backend_depends_on postgres: condition: service_healthy diff --git a/.gitignore b/.gitignore index 53c5b710..e961c3b7 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,4 @@ package-lock.json #sitemap /public/sitemap.xml.gz +.env \ No newline at end of file diff --git a/.ruby-version b/.ruby-version index bea438e9..47b322c9 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.3.1 +3.4.1 diff --git a/Gemfile b/Gemfile index b107eb92..4107a058 100644 --- a/Gemfile +++ b/Gemfile @@ -40,6 +40,7 @@ gem "dalli", "~> 2.7" gem "delayed_job_active_record", "~> 4.1" gem "delayed_job_web", "~> 1.4" gem "devise", "~> 4.6" +gem 'dotenv-rails' gem "draper", "~> 3.0" gem "email_validator", "~> 1.6" gem "emoji_regex", "~> 1.0" @@ -105,6 +106,11 @@ gem "validate_url", "~> 1.0" gem "webpacker", "~> 3.6" gem "webpush", "~> 0.3" +# Monitoring +gem "newrelic_rpm" +gem "rack-mini-profiler" +gem "pghero" + group :development do gem "better_errors", "~> 2.5" gem "binding_of_caller", "~> 0.8" @@ -119,7 +125,7 @@ group :development do gem "web-console", "~> 3.7" end -group :development, :test do +group :development, :test, :profile do gem "capybara", "~> 3.13" gem "derailed", "~> 0.1" gem "erb_lint", "~> 0.0", require: false @@ -132,6 +138,9 @@ group :development, :test do gem "rspec-retry", "~> 0.6" gem "rubocop", "~> 0.63", require: false gem "rubocop-rspec", "~> 1.31" + gem "stackprof", "~> 0.2" + gem "ruby-prof", "~> 0.17" + gem "foreman" # gem "spring", "~> 2.0" # gem "spring-commands-rspec", "~> 1.0" gem "vcr", "~> 4.0" @@ -146,12 +155,10 @@ group :test do gem "launchy", "~> 2.4" gem "pundit-matchers", "~> 1.6" gem "rails-controller-testing", "~> 1.0" - gem "ruby-prof", "~> 0.17", require: false gem "selenium-webdriver", "~> 3.141" gem "shoulda-matchers", "4.0.0.rc1", require: false gem "simplecov", "~> 0.16", require: false gem "sinatra", "~> 2.0" - gem "stackprof", "~> 0.2", require: false, platforms: :ruby gem "stripe-ruby-mock", "~> 2.5", require: "stripe_mock" gem "test-prof", "~> 0.7" gem "timecop", "~> 0.9" diff --git a/Gemfile.lock b/Gemfile.lock index 130d7472..ffc3e496 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -254,6 +254,10 @@ GEM diff-lcs (1.6.0) docile (1.4.1) domain_name (0.6.20240107) + dotenv (2.8.1) + dotenv-rails (2.8.1) + dotenv (= 2.8.1) + railties (>= 3.2) draper (3.1.0) actionpack (>= 5.0) activemodel (>= 5.0) @@ -472,6 +476,7 @@ GEM fog-xml (0.1.5) fog-core nokogiri (>= 1.5.11, < 2.0.0) + foreman (0.88.1) formatador (0.3.0) front_matter_parser (0.2.1) fugit (1.11.1) @@ -646,6 +651,7 @@ GEM net-smtp (0.5.1) net-protocol netrc (0.11.0) + newrelic_rpm (9.17.0) nio4r (2.7.4) nokogiri (1.15.7-aarch64-linux) racc (~> 1.4) @@ -697,6 +703,8 @@ GEM ast (~> 2.4.1) racc pg (1.5.9) + pghero (2.8.3) + activerecord (>= 5) pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) @@ -726,6 +734,8 @@ GEM rack (2.2.11) rack-host-redirect (1.3.0) rack + rack-mini-profiler (3.3.1) + rack (>= 1.2.0) rack-protection (2.2.4) rack rack-proxy (0.7.7) @@ -1048,6 +1058,7 @@ DEPENDENCIES derailed (~> 0.1) derailed_benchmarks (~> 1.3) devise (~> 4.6) + dotenv-rails draper (~> 3.0) email_validator (~> 1.6) emoji_regex (~> 1.0) @@ -1062,6 +1073,7 @@ DEPENDENCIES figaro (~> 1.1) fix-db-schema-conflicts fog (~> 1.41) + foreman front_matter_parser (~> 0.2) gemoji (~> 3.0.0) gibbon (~> 2.2) @@ -1081,6 +1093,7 @@ DEPENDENCIES liquid (~> 4.0) memory_profiler (~> 0.9) nakayoshi_fork + newrelic_rpm nokogiri (~> 1.10) octokit (~> 4.13) omniauth (~> 1.9) @@ -1088,6 +1101,7 @@ DEPENDENCIES omniauth-twitter (~> 1.4) parallel_tests (~> 2.27) pg (~> 1.1) + pghero pry (~> 0.12) pry-byebug (~> 3.7) pry-rails (~> 0.3) @@ -1098,6 +1112,7 @@ DEPENDENCIES pusher (~> 1.3) pusher-push-notifications (~> 1.0) rack-host-redirect (~> 1.3) + rack-mini-profiler rack-timeout (~> 0.5) rails (~> 5.1.6) rails-assets-airbrake-js-client (~> 1.5)! diff --git a/Procfile.prof b/Procfile.prof new file mode 100644 index 00000000..f6093523 --- /dev/null +++ b/Procfile.prof @@ -0,0 +1,3 @@ +web: bin/rails s -p 3000 +webpacker: ./bin/webpack-dev-server +job: bin/rake jobs:work diff --git a/app/controllers/stories_controller.rb b/app/controllers/stories_controller.rb index df60029f..69cf94f4 100644 --- a/app/controllers/stories_controller.rb +++ b/app/controllers/stories_controller.rb @@ -1,6 +1,13 @@ class StoriesController < ApplicationController before_action :authenticate_user!, except: %i[index search show feed new] before_action :set_cache_control_headers, only: %i[index search show] + around_action :profile_with_stackprof, only: :index + + def profile_with_stackprof + StackProf.run(mode: :wall, raw: true, out: "tmp/stackprof-wall.dump", interval: 50) do + yield + end + end def index add_param_context(:username, :tag) diff --git a/app/views/stories/_main_stories_feed.html.erb b/app/views/stories/_main_stories_feed.html.erb index b6cd0a65..15bd910c 100644 --- a/app/views/stories/_main_stories_feed.html.erb +++ b/app/views/stories/_main_stories_feed.html.erb @@ -55,7 +55,9 @@ <% if !user_signed_in? && i == 4 %> <%= render "stories/sign_in_invitation" %> <% end %> - <%= render "articles/single_story", story: story %> + <% cache ["single_story", story] do %> + <%= render "articles/single_story", story: story %> + <% end %> <% end %> <% end %> <% if @stories.size > 1 %> diff --git a/bin/startup b/bin/startup index c7e84efb..8e55c856 100755 --- a/bin/startup +++ b/bin/startup @@ -12,5 +12,10 @@ end chdir APP_ROOT do puts "== STARTING UP ==" - system! "foreman start -f Procfile.dev" + if ENV["RAILS_ENV"] == "profile" + system! "bundle exec rake assets:precompile" + system! "foreman start -f Procfile.prof" + else + system! "foreman start -f Procfile.dev" + end end diff --git a/case-stady.md b/case-stady.md new file mode 100644 index 00000000..33920d00 --- /dev/null +++ b/case-stady.md @@ -0,0 +1,154 @@ +## Актуальная проблема +Cтраница `StoriesController#index` загружается очень медленно. Необходимо исследовать проблему на возможность оптимизации. + +## Формирование метрики +В качестве метрики я использовал данные из `ab` бенмарка, а именно среднее значение времени затраченного на выполнение запроса +на тесте `ab -n 100 -c 5 http://localhost:3000/` + +## Статистика до оптимизации +Выполнение бенчмарка `ab -n 100 -c 5 http://localhost:3000/` показало значение `Time per request: 4792.132 [ms]` +``` +Concurrency Level: 5 +Time taken for tests: 95.843 seconds +Complete requests: 100 +Failed requests: 62 + (Connect: 0, Receive: 0, Length: 62, Exceptions: 0) +Total transferred: 16150036 bytes +HTML transferred: 16048916 bytes +Requests per second: 1.04 [#/sec] (mean) +Time per request: 4792.132 [ms] (mean) +Time per request: 958.426 [ms] (mean, across all concurrent requests) +Transfer rate: 164.56 [Kbytes/sec] received + +Connection Times (ms) + min mean[+/-sd] median max +Connect: 0 0 0.2 0 2 +Processing: 2766 4644 285.4 4675 4950 +Waiting: 2765 4643 285.4 4674 4950 +Total: 2766 4644 285.4 4675 4950 + +Percentage of the requests served within a certain time (ms) + 50% 4675 + 66% 4712 + 75% 4757 + 80% 4797 + 90% 4852 + 95% 4915 + 98% 4947 + 99% 4950 + 100% 4950 (longest request) +``` + +## Поиск точек роста +Для поиска точек роста я воспользовался инструментами rack-mini-profiler и Newrelic. + +Newrelic подтверждает значения полученные в ab: + + +rack-mini-profiler показал большое количество рендеринга `_single_story.html.erb` + + +## Оптимизация + +Кэшируем проблемный паршиал: +``` +<% cache ["single_story", story] do %> + <%= render "articles/single_story", story: story %> +<% end %> +``` +Проверяем `ab` бенчмарком насколько изменилась метрика: +``` +Server Software: +Server Hostname: localhost +Server Port: 3000 + +Document Path: / +Document Length: 158982 bytes + +Concurrency Level: 5 +Time taken for tests: 20.597 seconds +Complete requests: 100 +Failed requests: 15 + (Connect: 0, Receive: 0, Length: 15, Exceptions: 0) +Total transferred: 15999369 bytes +HTML transferred: 15898184 bytes +Requests per second: 4.86 [#/sec] (mean) +Time per request: 1029.829 [ms] (mean) +Time per request: 205.966 [ms] (mean, across all concurrent requests) +Transfer rate: 758.59 [Kbytes/sec] received + +Connection Times (ms) + min mean[+/-sd] median max +Connect: 0 1 2.0 0 13 +Processing: 240 1010 111.1 1018 1266 +Waiting: 240 1009 111.1 1015 1262 +Total: 241 1011 110.9 1018 1266 + +Percentage of the requests served within a certain time (ms) + 50% 1018 + 66% 1041 + 75% 1060 + 80% 1073 + 90% 1125 + 95% 1158 + 98% 1229 + 99% 1266 + 100% 1266 (longest request) +``` +Получаем значение Time per request: 1029.829 [ms], что почти в 5 раз быстрее первоначальных цифр. +rack-mini-profiler теперь показывает что данные кэшируется и лишний раз паршиал `_single_story.html.erb` не рендерится. +При этом показатели количества комментариев и количества лайков показывают актальное значение, тк сам объект story не закэширован. + +Попробовал профилирование с помощью Stackprof в `around_action` режиме и Ruby-prof с `callgrind`. +Удобно для профилирования отдельных методов. Из минусов, в трейсе много лишней рельсовой информации. + +## Local_Production env +Создал новый энв `profile`, который максимально приближен к production. +`ab` бенчмарк показал Time per request: 92.812 [ms], что в 10 раз быстрее последнего оптимизированного варианта. +``` +ab -n 100 -c 5 http://localhost:3000/ +This is ApacheBench, Version 2.3 <$Revision: 1913912 $> +Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ +Licensed to The Apache Software Foundation, http://www.apache.org/ + +Benchmarking localhost (be patient).....done + + +Server Software: +Server Hostname: localhost +Server Port: 3000 + +Document Path: / +Document Length: 136409 bytes + +Concurrency Level: 5 +Time taken for tests: 1.856 seconds +Complete requests: 100 +Failed requests: 97 + (Connect: 0, Receive: 0, Length: 97, Exceptions: 0) +Non-2xx responses: 97 +Total transferred: 602228 bytes +HTML transferred: 584118 bytes +Requests per second: 53.87 [#/sec] (mean) +Time per request: 92.812 [ms] (mean) +Time per request: 18.562 [ms] (mean, across all concurrent requests) +Transfer rate: 316.83 [Kbytes/sec] received + +Connection Times (ms) + min mean[+/-sd] median max +Connect: 0 0 0.9 0 7 +Processing: 6 63 123.9 18 760 +Waiting: 6 62 123.8 17 760 +Total: 6 64 123.9 18 760 + +Percentage of the requests served within a certain time (ms) + 50% 18 + 66% 24 + 75% 41 + 80% 53 + 90% 207 + 95% 272 + 98% 759 + 99% 760 + 100% 760 (longest request) +``` diff --git a/config/application.rb b/config/application.rb index 1b4c391f..bce9d1d7 100644 --- a/config/application.rb +++ b/config/application.rb @@ -35,6 +35,7 @@ class Application < Rails::Application config.active_job.queue_adapter = :delayed_job config.middleware.use Rack::Deflater + config.middleware.use(Rack::RubyProf, enabled: true, path: "tmp/profile") # Globally handle Pundit::NotAuthorizedError by serving 404 config.action_dispatch.rescue_responses["Pundit::NotAuthorizedError"] = :not_found diff --git a/config/environments/profile.rb b/config/environments/profile.rb new file mode 100644 index 00000000..ee352c01 --- /dev/null +++ b/config/environments/profile.rb @@ -0,0 +1,126 @@ +# config/environments/profile.rb +Rails.application.configure do + # config.middleware.use(Rack::RubyProf, :path => 'ruby-prof-results') + # config.middleware.use ProfileMiddleware + + # Verifies that versions and hashed value of the package contents in the project's package.json + config.webpacker.check_yarn_integrity = false + + # Settings specified here will take precedence over those in config/application.rb. + + # Code is not reloaded between requests. + config.cache_classes = true + + # Eager load code on boot. This eager loads most of Rails and + # your application in memory, allowing both threaded web servers + # and those relying on copy on write to perform better. + # Rake tasks automatically ignore this option for performance. + config.eager_load = true + + # Full error reports are disabled and caching is turned on. + config.consider_all_requests_local = false + config.action_controller.perform_caching = true + + # Enable Rack::Cache to put a simple HTTP cache in front of your application + # Add `rack-cache` to your Gemfile before enabling this. + # For large-scale production use, consider using a caching reverse proxy like + # NGINX, varnish or squid. + # config.action_dispatch.rack_cache = true + config.read_encrypted_secrets = true + + # Disable serving static files from the `/public` folder by default since + # Apache or NGINX already handles this. + config.public_file_server.enabled = ENV["RAILS_SERVE_STATIC_FILES"].present? + config.public_file_server.headers = { + "Cache-Control" => "public, s-maxage=2592000, max-age=86400" + } + + # Compress JavaScripts and CSS. + config.assets.js_compressor = Uglifier.new(harmony: true) + # config.assets.css_compressor = :sass + + # Do not fallback to assets pipeline if a precompiled asset is missed. + config.assets.compile = true + config.assets.debug = false + + # Asset digests allow you to set far-future HTTP expiration dates on all assets, + # yet still be able to expire them through the digest params. + config.assets.digest = true + + # `config.assets.precompile` and `config.assets.version` + # have moved to config/initializers/assets.rb + + # Specifies the header that your server uses for sending files. + # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache + # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX + + # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. + # config.force_ssl = true + + # Use the lowest log level to ensure availability of diagnostic information + # when problems arise. + config.log_level = :debug + + # Prepend all log lines with the following tags. + config.log_tags = [:request_id] + + # Use a different logger for distributed setups. + # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) + + # Use a different cache store in production. + # config.cache_store = :mem_cache_store + config.cache_store = :memory_store + config.public_file_server.headers = { + "Cache-Control" => "public, max-age=172800" + } + + config.action_mailer.perform_caching = false + + # Ignore bad email addresses and do not raise email delivery errors. + # Set this to true and configure the email server for immediate delivery to raise delivery errors. + config.action_mailer.raise_delivery_errors = false + + # Enable locale fallbacks for I18n (makes lookups for any locale fall back to + # the I18n.default_locale when a translation cannot be found). + config.i18n.fallbacks = [I18n.default_locale] + + # Send deprecation notices to registered listeners. + config.active_support.deprecation = :notify + + # Use default logging formatter so that PID and timestamp are not suppressed. + # config.log_formatter = ::Logger::Formatter.new + config.log_formatter = ::Logger::Formatter.new + if ENV["RAILS_LOG_TO_STDOUT"].present? + logger = ActiveSupport::Logger.new(STDOUT) + logger.formatter = config.log_formatter + config.logger = ActiveSupport::TaggedLogging.new(logger) + end + + # Timber.io logger + send_logs_to_timber = ENV["SEND_LOGS_TO_TIMBER"] || "false" # <---- set to false to stop sending dev logs to Timber.io + log_device = send_logs_to_timber == "true" ? Timber::LogDevices::HTTP.new(ENV["TIMBER"]) : STDOUT + logger = Timber::Logger.new(log_device) + logger.level = config.log_level + config.logger = ActiveSupport::TaggedLogging.new(logger) + + # Do not dump schema after migrations. + config.active_record.dump_schema_after_migration = false + + config.app_domain = "localhost:3000" + + config.action_mailer.delivery_method = :smtp + config.action_mailer.perform_deliveries = true + config.action_mailer.default_url_options = { host: config.app_domain } + config.action_mailer.smtp_settings = { + address: "smtp.gmail.com", + port: "587", + enable_starttls_auto: true, + user_name: '<%= ENV["DEVELOPMENT_EMAIL_USERNAME"] %>', + password: '<%= ENV["DEVELOPMENT_EMAIL_PASSWORD"] %>', + authentication: :plain, + domain: "localhost:3000" + } + + config.action_mailer.preview_path = "#{Rails.root}/spec/mailers/previews" +end + diff --git a/config/initializers/carrierwave.rb b/config/initializers/carrierwave.rb index 9e191ac1..30e87b7f 100644 --- a/config/initializers/carrierwave.rb +++ b/config/initializers/carrierwave.rb @@ -3,7 +3,7 @@ require "carrierwave/storage/fog" CarrierWave.configure do |config| - if Rails.env.development? || Rails.env.test? + if Rails.env.development? || Rails.env.test? || Rails.env.profile? config.storage = :file else # config.fog_provider = 'fog-aws' diff --git a/config/newrelic.yml b/config/newrelic.yml new file mode 100644 index 00000000..a4c5a300 --- /dev/null +++ b/config/newrelic.yml @@ -0,0 +1,66 @@ +# +# This file configures the New Relic Agent. New Relic monitors Ruby, Java, +# .NET, PHP, Python, Node, and Go applications with deep visibility and low +# overhead. For more information, visit www.newrelic.com. +# +# Generated October 28, 2022 +# +# This configuration file is custom generated for NewRelic Administration +# +# For full documentation of agent configuration options, please refer to +# https://docs.newrelic.com/docs/agents/ruby-agent/installation-configuration/ruby-agent-configuration + +common: &default_settings + # Required license key associated with your New Relic account. + license_key: <%= ENV['NR_LICENSE_KEY'] %> + + # Your application name. Renaming here affects where data displays in New + # Relic. For more details, see https://docs.newrelic.com/docs/apm/new-relic-apm/maintenance/renaming-applications + app_name: 'dev_to' + + distributed_tracing: + enabled: true + + # To disable the agent regardless of other settings, uncomment the following: + + # agent_enabled: false + + # Logging level for log/newrelic_agent.log + log_level: info + + application_logging: + # If `true`, all logging-related features for the agent can be enabled or disabled + # independently. If `false`, all logging-related features are disabled. + enabled: true + forwarding: + # If `true`, the agent captures log records emitted by this application. + enabled: true + # Defines the maximum number of log records to buffer in memory at a time. + max_samples_stored: 10000 + metrics: + # If `true`, the agent captures metrics related to logging for this application. + enabled: true + local_decorating: + # If `true`, the agent decorates logs with metadata to link to entities, hosts, traces, and spans. + # This requires a log forwarder to send your log files to New Relic. + # This should not be used when forwarding is enabled. + enabled: false + +# Environment-specific settings are in this section. +# RAILS_ENV or RACK_ENV (as appropriate) is used to determine the environment. +# If your application has other named environments, configure them here. +development: + <<: *default_settings + app_name: 'dev_to (Development)' + +test: + <<: *default_settings + # It doesn't make sense to report to New Relic from automated test runs. + monitor_mode: false + +staging: + <<: *default_settings + app_name: 'dev_to (Staging)' + +production: + <<: *default_settings diff --git a/config/routes.rb b/config/routes.rb index 8ba426bb..2f7b28b4 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -7,6 +7,8 @@ registrations: "registrations" } + mount PgHero::Engine, at: "pghero" + authenticated :user, ->(user) { user.admin? } do mount DelayedJobWeb, at: "/delayed_job" end diff --git a/config/secrets.yml b/config/secrets.yml index 73f5e05c..80f824b2 100644 --- a/config/secrets.yml +++ b/config/secrets.yml @@ -16,6 +16,8 @@ development: test: secret_key_base: 42dd7834039ebbea271af22635a6782ee15e519b14629c5276bfcdd4cff841e9926994784bb43a335a8f8c9739bb254ea3afe831839d4dc65654ec7516ec25f0 +profile: + secret_key_base: a60edc976c913b19fd9fc8118936fbe1df2b07f4eecc5ad32f975e33cd4ea36b150c1ce933b681b90874a46568041629003dcbfc07238f7dca91741bcd1ec870 # Do not keep production secrets in the repository, # instead read values from the environment. diff --git a/config/webpack/profile.js b/config/webpack/profile.js new file mode 100644 index 00000000..d7346478 --- /dev/null +++ b/config/webpack/profile.js @@ -0,0 +1,3 @@ +const environment = require('./environment'); + +module.exports = environment.toWebpackConfig(); diff --git a/config/webpacker.yml b/config/webpacker.yml index 2dfcd170..27f41c40 100644 --- a/config/webpacker.yml +++ b/config/webpacker.yml @@ -46,6 +46,18 @@ test: # Compile test packs to a separate directory public_output_path: packs-test +profile: + <<: *default + compile: true + + dev_server: + host: 0.0.0.0 + port: 3035 + hmr: false + https: false + + cache_manifest: true + production: <<: *default diff --git a/public/newrelic.png b/public/newrelic.png new file mode 100644 index 00000000..2fd4f717 Binary files /dev/null and b/public/newrelic.png differ diff --git a/public/single_story.png b/public/single_story.png new file mode 100644 index 00000000..010e91e1 Binary files /dev/null and b/public/single_story.png differ