Optimization task 4

.dev_to/compose.yml

@@ -8,7 +8,7 @@ x-app: &app
   image: optimize-dev-to:1.0.0
   environment: &env
     NODE_ENV: ${NODE_ENV:-development}
-    RAILS_ENV: ${RAILS_ENV:-development}
+    RAILS_ENV: ${RAILS_ENV:-local_production}
   tmpfs:
     - /tmp
     - /app/tmp/pids
@@ -33,6 +33,9 @@ x-backend: &backend
     ALGOLIASEARCH_APPLICATION_ID: ${ALGOLIASEARCH_APPLICATION_ID}
     ALGOLIASEARCH_API_KEY: ${ALGOLIASEARCH_API_KEY}
     ALGOLIASEARCH_SEARCH_ONLY_KEY: ${ALGOLIASEARCH_SEARCH_ONLY_KEY}
+    NEW_RELIC_KEY: ${NEW_RELIC_KEY}
+    SCOUT_KEY: ${SCOUT_KEY}
+    RAILS_ENV: local_production
     REDIS_URL: redis://redis:6379/
     DATABASE_URL: postgres://postgres:postgres@postgres:5432
     WEBPACKER_DEV_SERVER_HOST: webpacker

Gemfile

@@ -104,8 +104,11 @@ gem "uglifier", "~> 4.1"
 gem "validate_url", "~> 1.0"
 gem "webpacker", "~> 3.6"
 gem "webpush", "~> 0.3"
+gem 'newrelic_rpm'
+gem 'scout_apm'
+gem 'rack-mini-profiler'
 
-group :development do
+group :development, :local_production do
   gem "better_errors", "~> 2.5"
   gem "binding_of_caller", "~> 0.8"
   gem "brakeman", "~> 4.4", require: false
@@ -119,7 +122,7 @@ group :development do
   gem "web-console", "~> 3.7"
 end
 
-group :development, :test do
+group :development, :local_production, :test do
   gem "capybara", "~> 3.13"
   gem "derailed", "~> 0.1"
   gem "erb_lint", "~> 0.0", require: false

Gemfile.lock

@@ -646,6 +646,7 @@ GEM
     net-smtp (0.5.1)
       net-protocol
     netrc (0.11.0)
+    newrelic_rpm (9.17.0)
     nio4r (2.7.4)
     nokogiri (1.15.7-aarch64-linux)
       racc (~> 1.4)
@@ -726,6 +727,8 @@ GEM
     rack (2.2.11)
     rack-host-redirect (1.3.0)
       rack
+    rack-mini-profiler (3.3.1)
+      rack (>= 1.2.0)
     rack-protection (2.2.4)
       rack
     rack-proxy (0.7.7)
@@ -877,6 +880,8 @@ GEM
       addressable (>= 2.3.5)
       faraday (> 0.8, < 2.0)
     sax-machine (1.3.2)
+    scout_apm (5.6.1)
+      parser
     sdoc (1.1.0)
       rdoc (>= 5.0)
     selectize-rails (0.12.6)
@@ -1081,6 +1086,7 @@ DEPENDENCIES
   liquid (~> 4.0)
   memory_profiler (~> 0.9)
   nakayoshi_fork
+  newrelic_rpm
   nokogiri (~> 1.10)
   octokit (~> 4.13)
   omniauth (~> 1.9)
@@ -1098,6 +1104,7 @@ DEPENDENCIES
   pusher (~> 1.3)
   pusher-push-notifications (~> 1.0)
   rack-host-redirect (~> 1.3)
+  rack-mini-profiler
   rack-timeout (~> 0.5)
   rails (~> 5.1.6)
   rails-assets-airbrake-js-client (~> 1.5)!
@@ -1118,6 +1125,7 @@ DEPENDENCIES
   s3_direct_upload (~> 0.1)
   sail (~> 1.5)
   sass-rails (~> 5.0)
+  scout_apm
   sdoc (~> 1.0)
   selenium-webdriver (~> 3.141)
   serviceworker-rails (~> 0.5)

app/helpers/application_helper.rb

@@ -83,7 +83,7 @@ def icon_url(name)
   end
 
   def cloudinary(url, width = nil, _quality = 80, _format = "jpg")
-    return url if Rails.env.development? && (url.blank? || url.exclude?("http"))
+    return url if (Rails.env.development? || Rails.env.local_production?) && (url.blank? || url.exclude?("http"))
 
     service_path = "https://res.cloudinary.com/practicaldev/image/fetch"
 
@@ -101,7 +101,7 @@ def cloudinary(url, width = nil, _quality = 80, _format = "jpg")
   def cloud_cover_url(url)
     return if url.blank?
     return asset_path("triple-unicorn") if Rails.env.test?
-    return url if Rails.env.development?
+    return url if Rails.env.development? || Rails.env.local_production?
 
     width = 1000
     height = 420

app/observers/article_observer.rb

@@ -1,6 +1,6 @@
 class ArticleObserver < ApplicationObserver
   def after_save(article)
-    return if Rails.env.development?
+    return if Rails.env.development? || Rails.env.local_production?
 
     if article.published && article.published_at > 30.seconds.ago
       SlackBot.delay.ping "New Article Published: #{article.title}\nhttps://dev.to#{article.path}",

app/observers/comment_observer.rb

@@ -1,6 +1,6 @@
 class CommentObserver < ApplicationObserver
   def after_save(comment)
-    return if Rails.env.development?
+    return if Rails.env.development? || Rails.env.local_production?
 
     warned_user_ping(comment)
   rescue StandardError

app/observers/organization_observer.rb

@@ -1,6 +1,6 @@
 class OrganizationObserver < ActiveRecord::Observer
   def after_create(organization)
-    return if Rails.env.development?
+    return if Rails.env.development? || Rails.env.local_production?
 
     SlackBot.delay.ping(
       "New Org Created: #{organization.name}\nhttps://dev.to/#{organization.username}",

app/views/stories/_main_stories_feed.html.erb

@@ -55,7 +55,9 @@
     <% if !user_signed_in? && i == 4 %>
       <%= render "stories/sign_in_invitation" %>
     <% end %>
-    <%= render "articles/single_story", story: story %>
+    <% cache ["v1", story, story.updated_at, story.comments_count, story.positive_reactions_count] do %>
+      <%= render "articles/single_story", story: story %>
+    <% end %>
   <% end %>
 <% end %>
 <% if @stories.size > 1 %>

case-study.md

@@ -0,0 +1,85 @@
+# Задание 4
+
+## Актуальная проблема
+В проекте dev.to выявлена проблема производительности главной страницы:
+- Медленный рендеринг главной страницы (StoriesController#index) (Особенно затратный рендеринг partial-ов _single_story.html.erb)
+- Отсутствие кэширования страниц
+
+## Формирование метрик
+Для оценки эффективности оптимизации определены следующие метрики:
+
+- Время полной загрузки главной страницы
+- Время рендеринга partial _single_story.html.erb
+- Количество запросов к БД при рендеринге страницы
+- Использование CPU и памяти
+- Включил кеширование на локальном окружении
+- Использование `benchmark` с помощью `ab` (`ab -n 100 -c 5 http://localhost:3000/`)
+- Добавил local_production окружение
+
+## Feedback-Loop
+Построен быстрый цикл обратной связи:
+
+- NewRelic APM для мониторинга метрик
+- rack-mini-profiler для профилирования рендеринга
+- Поиск точек роста
+
+## Использованы инструменты профилирования:
+
+- NewRelic для анализа узких мест
+- rack-mini-profiler для детального профилирования рендеринга
+- Логи Rails для анализа SQL-запросов
+
+## Результаты оптимизации:
+
+### 1. Отсутствует local_production окружение
+#### Произвел замеры с использованием `ab` до добавления local_production:
+```
+Concurrency Level:      5
+Time taken for tests:   61.726 seconds
+Complete requests:      100
+Failed requests:        99
+   (Connect: 0, Receive: 0, Length: 99, Exceptions: 0)
+Total transferred:      16131253 bytes
+HTML transferred:       16030175 bytes
+Requests per second:    1.62 [#/sec] (mean)
+Time per request:       3086.288 [ms] (mean)
+Time per request:       617.258 [ms] (mean, across all concurrent requests)
+Transfer rate:          255.21 [Kbytes/sec] received
+```
+
+#### После добавления:
+```
+Concurrency Level:      5
+Time taken for tests:   14.988 seconds
+Complete requests:      100
+Failed requests:        0
+Total transferred:      13161800 bytes
+HTML transferred:       13115100 bytes
+Requests per second:    6.67 [#/sec] (mean)
+Time per request:       749.387 [ms] (mean)
+Time per request:       149.877 [ms] (mean, across all concurrent requests)
+Transfer rate:          857.59 [Kbytes/sec] received
+```
+- Время обработки всех запросов сократилось в 4 раза с 60 до 15 секунд. Также количество Failed сократилось с 99 до 0.
+
+### 2. Отсутствие кеширования partial-ов _single_story.html.erb
+- Readme задания и rack-mini-profiler
+- Добавил кеширование partial'а, учел, что в него входят счётчики лайков и комментариев.
+
+#### Замеры после добавления кеширования:
+```
+Concurrency Level:      5
+Time taken for tests:   7.179 seconds
+Complete requests:      100
+Failed requests:        0
+Total transferred:      13028000 bytes
+HTML transferred:       12981300 bytes
+Requests per second:    13.93 [#/sec] (mean)
+Time per request:       358.953 [ms] (mean)
+Time per request:       71.791 [ms] (mean, across all concurrent requests)
+Transfer rate:          1772.19 [Kbytes/sec] received
+```
+- Время обработки всех запросов сократилось в два раза с 14 до 7 секунд.
+- Время выполнения отдельного запроса составило 360ms
+
+

config/environments/local_production.rb

@@ -0,0 +1,109 @@
+# rubocop:disable Metrics/BlockLength
+#
+def yarn_integrity_enabled?
+  ENV.fetch("YARN_INTEGRITY_ENABLED", "true") == "true"
+end
+
+Rails.application.configure do
+  # Verifies that versions and hashed value of the package contents in the project's package.json
+  config.webpacker.check_yarn_integrity = yarn_integrity_enabled?
+
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # In the development environment your application's code is reloaded on
+  # every request. This slows down response time but is perfect for development
+  # since you don't have to restart the web server when you make code changes.
+  config.cache_classes = true
+
+  # Do not eager load code on boot.
+  config.eager_load = true
+
+  # Show full error reports and disable caching.
+  config.consider_all_requests_local = true
+
+  # Enable/disable caching. By default caching is disabled.
+  if Rails.root.join("tmp/caching-dev.txt").exist?
+    config.action_controller.perform_caching = true
+
+    config.cache_store = :memory_store
+    config.public_file_server.headers = {
+      "Cache-Control" => "public, max-age=172800"
+    }
+  else
+    config.action_controller.perform_caching = false
+
+    config.cache_store = :null_store
+  end
+
+  config.assets_debug = false
+  config.assets_compile = false
+
+  config.web_console.development_only = false
+
+  # Don't care if the mailer can't send.
+  config.action_mailer.raise_delivery_errors = false
+
+  # Print deprecation notices to the Rails logger.
+  config.active_support.deprecation = :log
+
+  # Raise an error on page load if there are pending migrations.
+  config.active_record.migration_error = :page_load
+
+  # Debug mode disables concatenation and preprocessing of assets.
+  # This option may cause significant delays in view rendering with a large
+  # number of complex assets.
+  config.assets.debug = false
+
+  # Asset digests allow you to set far-future HTTP expiration dates on all assets,
+  # yet still be able to expire them through the digest params.
+  config.assets.digest = false
+
+  # Supress logger output for asset requests.
+  config.assets.quiet = true
+
+  # Adds additional error checking when serving assets at runtime.
+  # Checks for improperly declared sprockets dependencies.
+  # Raises helpful error messages.
+  config.assets.raise_runtime_errors = true
+
+  config.action_mailer.perform_caching = false
+
+  config.app_domain = "localhost:3000"
+
+  config.action_mailer.default_url_options = { host: "localhost:3000" }
+  config.action_mailer.delivery_method = :smtp
+  config.action_mailer.perform_deliveries = true
+  config.action_mailer.default_url_options = { host: config.app_domain }
+  config.action_mailer.smtp_settings = {
+    address: "smtp.gmail.com",
+    port: "587",
+    enable_starttls_auto: true,
+    user_name: '<%= ENV["DEVELOPMENT_EMAIL_USERNAME"] %>',
+    password: '<%= ENV["DEVELOPMENT_EMAIL_PASSWORD"] %>',
+    authentication: :plain,
+    domain: "localhost:3000"
+  }
+
+  config.action_mailer.preview_path = "#{Rails.root}/spec/mailers/previews"
+
+  # Raises error for missing translations
+  # config.action_view.raise_on_missing_translations = true
+
+  config.public_file_server.enabled = true
+
+  config.file_watcher = ActiveSupport::EventedFileUpdateChecker
+
+  # Install the Timber.io logger
+  send_logs_to_timber = ENV["SEND_LOGS_TO_TIMBER"] || "false" # <---- set to false to stop sending dev logs to Timber.io
+  log_device = send_logs_to_timber == "true" ? Timber::LogDevices::HTTP.new(ENV["TIMBER"]) : STDOUT
+  logger = Timber::Logger.new(log_device)
+  logger.level = config.log_level
+  config.logger = ActiveSupport::TaggedLogging.new(logger)
+
+  config.after_initialize do
+    Bullet.enable = true
+    Bullet.console = true
+  end
+end
+
+# rubocop:enable Metrics/BlockLength

config/initializers/airbrake.rb

@@ -41,7 +41,7 @@
   # environments.
   # NOTE: This option *does not* work if you don't set the 'environment' option.
   # https://github.com/airbrake/airbrake-ruby#ignore_environments
-  c.ignore_environments = %w[test development]
+  c.ignore_environments = %w[test development local_production]
 
   # A list of parameters that should be filtered out of what is sent to
   # Airbrake. By default, all "password" attributes will have their contents

config/initializers/carrierwave.rb

@@ -3,7 +3,7 @@
 require "carrierwave/storage/fog"
 
 CarrierWave.configure do |config|
-  if Rails.env.development? || Rails.env.test?
+  if Rails.env.development? || Rails.env.test? || Rails.env.local_production?
     config.storage = :file
   else
     # config.fog_provider = 'fog-aws'

config/initializers/honeycomb.rb

@@ -3,7 +3,7 @@
 key = ApplicationConfig["HONEYCOMB_API_KEY"]
 dataset = "dev.to-#{Rails.env}"
 
-$libhoney = if Rails.env.development? || Rails.env.test?
+$libhoney = if Rails.env.development? || Rails.env.test? || Rails.env.local_production?
               Libhoney::NullClient.new
             else
               Libhoney::Client.new(

config/initializers/reverse_markdown.rb

@@ -4,7 +4,7 @@
 # Because files are eagerloaded in production, this fix is only
 # applicable in development (and test, when needed)
 
-if Rails.env.development? || Rails.env.test?
+if Rails.env.development? || Rails.env.test? || Rails.env.local_production?
   Rails.application.config.to_prepare do
     Dir.glob(Rails.root.join("app/lib/reverse_markdown/converters/*.rb")).sort.each do |filename|
       require_dependency filename