diff --git a/.envrc.template b/.envrc.template index 99a1ddaaf9..557c0005b0 100644 --- a/.envrc.template +++ b/.envrc.template @@ -30,8 +30,12 @@ export WIREMOCK_TRIP_PLAN_PROXY_URL=http://otp-local.mbtace.com # export DOCKER_USERNAME= # export DOCKER_PASSWORD= -# You can optionally set a Redis host. It will default to 127.0.0.1 +# You can optionally set a Redis host and port. +# The default host is 127.0.0.1 +# The default port is 6379 +# Because we run Redis Cluster, you might have to set the port to 30001 # export REDIS_HOST= +# export REDIS_PORT= # These credentials control access to resetting cache entries for the CMS. # You can set them to be whatever you want, but they'll need to match those on the Drupal side. diff --git a/config/deps/logger.exs b/config/deps/logger.exs index 5296d1ad88..8358067d09 100644 --- a/config/deps/logger.exs +++ b/config/deps/logger.exs @@ -26,7 +26,7 @@ if config_env() == :dev do config :logger, :console, format: "[$level] $message\n" config :logger, - level: :warning, + level: :notice, colors: [enabled: true] end diff --git a/config/dotcom/cms.exs b/config/dotcom/cms.exs index 10cf0c2112..7c70bd9d49 100644 --- a/config/dotcom/cms.exs +++ b/config/dotcom/cms.exs @@ -5,10 +5,14 @@ config :dotcom, config :dotcom, :cms_api, CMS.API.HTTPClient +config :dotcom, :cms_cache, CMS.Cache + if config_env() == :test do config :dotcom, :drupal, cms_root: "http://cms.test", cms_static_path: "/sites/default/files" config :dotcom, :cms_api, CMS.API.Static + + config :dotcom, :cms_cache, CMS.TestCache end diff --git a/config/runtime.exs b/config/runtime.exs index a337ba53ae..fb26bb6b70 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -25,12 +25,18 @@ if System.get_env("PHX_SERVER") do end redis_host_env = System.get_env("REDIS_HOST", "127.0.0.1") +redis_port_env = System.get_env("REDIS_PORT", "6379") redis_host = if redis_host_env == "", do: "127.0.0.1", else: redis_host_env +redis_port = + if redis_port_env == "", + do: 6379, + else: String.to_integer(redis_port_env) + if config_env() == :dev do # For development, we disable any cache and enable # debugging and code reloading. @@ -58,28 +64,18 @@ if config_env() == :dev do end end -if config_env() == :prod do - config :dotcom, CMS.Cache, - mode: :redis_cluster, - redis_cluster: [ - configuration_endpoints: [ - conn_opts: [ - host: redis_host, - port: 6379 - ] +config :dotcom, CMS.Cache, + mode: :redis_cluster, + redis_cluster: [ + configuration_endpoints: [ + conn_opts: [ + host: redis_host, + port: redis_port ] - ], - stats: false, - telemetry: false -else - config :dotcom, CMS.Cache, - conn_opts: [ - host: redis_host, - port: 6379 - ], - stats: false, - telemetry: false -end + ] + ], + stats: true, + telemetry: true if config_env() == :test do config :dotcom, DotcomWeb.Router, diff --git a/lib/cms/cache.ex b/lib/cms/cache.ex index 8cef81055e..adafd44460 100644 --- a/lib/cms/cache.ex +++ b/lib/cms/cache.ex @@ -1,5 +1,7 @@ defmodule CMS.Cache do - use Nebulex.Cache, - otp_app: :dotcom, - adapter: NebulexRedisAdapter + @moduledoc """ + A standard implementation of Nebulex. + """ + + use Nebulex.Cache, otp_app: :dotcom, adapter: NebulexRedisAdapter end diff --git a/lib/cms/repo.ex b/lib/cms/repo.ex index 3b14e3d599..07b46dc798 100644 --- a/lib/cms/repo.ex +++ b/lib/cms/repo.ex @@ -1,14 +1,16 @@ defmodule CMS.Repo do - require Logger - @moduledoc """ + Interface for the content CMS. + Returns a variety of content related structs, like %Event{} or %Basic{}. - Interface for the content CMS. Returns a variety of content - related structs, like %Event{} or %Basic{} - + The repo relies heavily on `CMS.Cache` which implements the Nebulex Redis Adapter. + The cache is set with `@cache Application.get_env(:cms, :cache)` so that we can easily swap out a local cache during tests. + The base ttl for the repo is one hour. """ - use RepoCache, ttl: :timer.minutes(1) + use Nebulex.Caching.Decorators + + require Logger import CMS.Helpers, only: [preview_opts: 1] @@ -27,8 +29,12 @@ defmodule CMS.Repo do alias Routes.Route + @cache Application.compile_env!(:dotcom, :cms_cache) + @cms_api Application.compile_env!(:dotcom, :cms_api) + @ttl :timer.hours(1) + @spec get_page(String.t(), map) :: Page.t() | {:error, API.error()} def get_page(path, query_params \\ %{}) do case view_or_preview(path, query_params) do @@ -50,21 +56,29 @@ defmodule CMS.Repo do @spec news_entry_by(Keyword.t()) :: NewsEntry.t() | :not_found def news_entry_by(opts) do - news = - cache(opts, fn _ -> - case @cms_api.view("/cms/news", opts) do - {:ok, api_data} -> Enum.map(api_data, &NewsEntry.from_api/1) - _ -> [] - end - end) - - case news do + case do_news_entry_by(opts) do [record | _] -> record [] -> :not_found end end - @spec events(Keyword.t()) :: [Event.t()] + @decorate cacheable( + cache: @cache, + on_error: :nothing, + opts: [ttl: 60_000] + ) + def do_news_entry_by(opts) do + case @cms_api.view("/cms/news", opts) do + {:ok, api_data} -> Enum.map(api_data, &NewsEntry.from_api/1) + _ -> [] + end + end + + @decorate cacheable( + cache: @cache, + on_error: :nothing, + opts: [ttl: 60_000] + ) def events(opts \\ []) do case @cms_api.view("/cms/events", opts) do {:ok, api_data} -> Enum.map(api_data, &Event.from_api/1) @@ -88,31 +102,40 @@ defmodule CMS.Repo do end end - @spec whats_happening() :: [WhatsHappeningItem.t()] - def whats_happening do - cache([], fn _ -> - case @cms_api.view("/cms/whats-happening", []) do - {:ok, api_data} -> Enum.map(api_data, &WhatsHappeningItem.from_api/1) - _ -> [] - end - end) + @decorate cacheable( + cache: @cache, + key: "/cms/whats-happening", + on_error: :nothing, + opts: [ttl: 60_000] + ) + def whats_happening() do + case @cms_api.view("/cms/whats-happening", []) do + {:ok, api_data} -> Enum.map(api_data, &WhatsHappeningItem.from_api/1) + _ -> [] + end end @spec banner() :: Banner.t() | nil def banner do - cached_value = - cache([], fn _ -> - # Banners were previously called Important Notices - case @cms_api.view("/cms/important-notices", []) do - {:ok, [api_data | _]} -> Banner.from_api(api_data) - {:ok, _} -> :empty - {:error, _} -> :error - end - end) + cached_value = do_banner() if cached_value == :empty || cached_value == :error, do: nil, else: cached_value end + @decorate cacheable( + cache: @cache, + key: "/cms/important-notices", + on_error: :nothing, + opts: [ttl: 60_000] + ) + def do_banner() do + case @cms_api.view("/cms/important-notices", []) do + {:ok, [api_data | _]} -> Banner.from_api(api_data) + {:ok, _} -> :empty + {:error, _} -> :error + end + end + @spec search(String.t(), integer, [String.t()]) :: any def search(query, offset, content_types) do params = [q: query, page: offset] ++ Enum.map(content_types, &{:"type[]", &1}) @@ -124,7 +147,7 @@ defmodule CMS.Repo do @spec get_schedule_pdfs(Route.id_t()) :: [RoutePdf.t()] def get_schedule_pdfs(route_id) do - case cache(route_id, &do_get_schedule_pdfs/1, timeout: :timer.hours(6)) do + case do_get_schedule_pdfs(route_id) do {:ok, pdfs} -> pdfs @@ -138,6 +161,12 @@ defmodule CMS.Repo do end end + @decorate cacheable( + cache: @cache, + key: "/cms/schedules/#{route_id}", + on_error: :nothing, + opts: [ttl: @ttl] + ) defp do_get_schedule_pdfs(route_id) do case @cms_api.view("/cms/schedules/#{route_id}", []) do {:ok, pdfs} -> @@ -150,7 +179,7 @@ defmodule CMS.Repo do @spec get_route_pdfs(Route.id_t()) :: [RoutePdf.t()] def get_route_pdfs(route_id) do - case cache(route_id, &do_get_route_pdfs/1, timeout: :timer.hours(6)) do + case do_get_route_pdfs(route_id) do {:ok, pdfs} -> pdfs @@ -164,6 +193,12 @@ defmodule CMS.Repo do end end + @decorate cacheable( + cache: @cache, + key: "/cms/route_pdfs/#{route_id}", + on_error: :nothing, + opts: [ttl: @ttl] + ) defp do_get_route_pdfs(route_id) do case @cms_api.view("/cms/route-pdfs/#{route_id}", []) do {:ok, []} -> @@ -182,6 +217,29 @@ defmodule CMS.Repo do end end + # BEGIN PAGE CACHING # + + @behaviour Nebulex.Caching.KeyGenerator + + @impl true + def generate(_, _, [path, %Plug.Conn.Unfetched{aspect: :query_params}]) do + "/cms/#{String.trim(path, "/")}" + end + + def generate(_, _, [path, params]) do + "/cms/#{String.trim(path, "/")}" <> params_to_string(params) + end + + defp params_to_string(params) when params == %{}, do: "" + + defp params_to_string(params) when is_map(params) do + [head | tail] = Enum.map(params, fn {k, v} -> "#{k}=#{v}" end) + + ["?#{head}", "#{Enum.join(tail, "&")}"] + |> Enum.reject(&(&1 == "")) + |> Enum.join("&") + end + @spec view_or_preview(String.t(), map) :: {:ok, map} | {:error, API.error()} defp view_or_preview(path, %{"preview" => _, "vid" => "latest"} = params) do # "preview" value is deprecated. Use empty string or nil to get latest revision. @@ -201,10 +259,18 @@ defmodule CMS.Repo do end end + @decorate cacheable( + cache: @cache, + key_generator: __MODULE__, + on_error: :nothing, + opts: [ttl: @ttl] + ) defp view_or_preview(path, params) do - cache([path: path, params: params], fn _ -> @cms_api.view(path, params) end) + @cms_api.view(path, params) end + # END PAGE CACHING # + @spec handle_revision({:error, any} | {:ok, [map]}) :: {:error, String.t()} | {:ok, map} defp handle_revision({:error, err}), do: {:error, err} @@ -344,16 +410,15 @@ defmodule CMS.Repo do @doc "Get all the events, paginating through results if needed, and caches the result" @spec events_for_year(Calendar.year()) :: [%Teaser{}] def events_for_year(year) do - range = [ + do_events_for_range( min: Timex.beginning_of_year(year) |> Util.convert_to_iso_format(), max: Timex.end_of_year(year) |> Timex.shift(days: 1) |> Util.convert_to_iso_format() - ] - - cache([range: range], fn _ -> do_events_for_range(range) end) + ) end @spec do_events_for_range([min: String.t(), max: String.t()], non_neg_integer(), [%Teaser{}]) :: [%Teaser{}] + @decorate cacheable(cache: @cache, on_error: :nothing, opts: [ttl: @ttl]) defp do_events_for_range(range, offset \\ 0, all_events \\ []) do per_page = 50 diff --git a/lib/cms/telemetry.ex b/lib/cms/telemetry.ex new file mode 100644 index 0000000000..3a0b708e76 --- /dev/null +++ b/lib/cms/telemetry.ex @@ -0,0 +1,52 @@ +defmodule CMS.Telemetry do + @moduledoc """ + This supervisor establishes a connection between the telemetry_poller and our telemetry reporters. + Cache stats are emitted by the Nebulex Redis Adapter. + We poll for them every minute. + + Currently, they are passed to two reporters: + + The Statsd reporter will eventually be hooked up to Splunk metrics. + For now, it does no harm to emit them even though nothing is listening. + + The custom reporter logs in a format that can be picked up in Splunk logs. + Eventually, this should be removed. + """ + + use Supervisor + + alias Telemetry.Metrics + + def start_link(arg) do + Supervisor.start_link(__MODULE__, arg, name: __MODULE__) + end + + def init(_arg) do + children = [ + {:telemetry_poller, measurements: periodic_measurements(), period: 60_000}, + {CMS.Telemetry.Reporter, metrics: reporter_metrics()}, + {TelemetryMetricsStatsd, metrics: statsd_metrics()} + ] + + Supervisor.init(children, strategy: :one_for_one) + end + + defp reporter_metrics do + [ + Metrics.last_value("cms.cache.stats.updates") + ] + end + + defp statsd_metrics do + [ + Metrics.last_value("cms.cache.stats.hits"), + Metrics.last_value("cms.cache.stats.misses") + ] + end + + defp periodic_measurements do + [ + {Application.get_env(:dotcom, :cms_cache, CMS.Cache), :dispatch_stats, []} + ] + end +end diff --git a/lib/cms/telemetry/reporter.ex b/lib/cms/telemetry/reporter.ex new file mode 100644 index 0000000000..4ec655afba --- /dev/null +++ b/lib/cms/telemetry/reporter.ex @@ -0,0 +1,87 @@ +defmodule CMS.Telemetry.Reporter do + @moduledoc """ + This custom Telemetry Reporter logs hit rate information for the `CMS.Cache`. + + See https://blog.miguelcoba.com/telemetry-and-metrics-in-elixir#heading-customreporter for more on writing custom reporters. + """ + + use GenServer + + require Logger + + alias Telemetry.Metrics + + def start_link(metrics: metrics) do + GenServer.start_link(__MODULE__, metrics) + end + + @impl true + def init(metrics) do + Process.flag(:trap_exit, true) + + groups = Enum.group_by(metrics, & &1.event_name) + + for {event, metrics} <- groups do + :telemetry.attach({__MODULE__, event, self()}, event, &__MODULE__.handle_event/4, metrics) + end + + :telemetry.attach( + "cache-command-exception", + [:cms, :cache, :command, :exception], + &__MODULE__.handle_exception/4, + nil + ) + + {:ok, Map.keys(groups)} + end + + @impl true + def terminate(_, events) do + for event <- events do + :telemetry.detach({__MODULE__, event, self()}) + end + + :ok + end + + def handle_event(_event_name, measurements, metadata, metrics) do + metrics + |> Enum.map(&handle_metric(&1, measurements, metadata)) + end + + def handle_exception( + event_name, + _measurements, + %{kind: kind, reason: %Redix.ConnectionError{reason: reason}}, + _config + ) do + key = event_name |> Enum.map(&Atom.to_string/1) |> Enum.join(".") + + Logger.warning("#{key} kind=#{kind} reason=#{reason}") + end + + def handle_exception( + event_name, + _measurements, + %{kind: kind, reason: %Redix.Error{message: message}}, + _config + ) do + key = event_name |> Enum.map(&Atom.to_string/1) |> Enum.join(".") + + Logger.warning("#{key} kind=#{kind} reason=#{message}") + end + + defp handle_metric(%Metrics.LastValue{}, %{hits: hits, misses: misses}, _metadata) do + total = hits + misses + + if total > 0 do + Logger.notice( + "cms.cache.stats hits=#{hits} misses=#{misses} total=#{total} hit_rate=#{hits / total}" + ) + end + end + + defp handle_metric(metric, _measurements, _metadata) do + Logger.warning("cms.cache.unsupported_metric metric=#{metric.__struct__}") + end +end diff --git a/lib/dotcom/application.ex b/lib/dotcom/application.ex index b79b49093b..8fc65fc399 100644 --- a/lib/dotcom/application.ex +++ b/lib/dotcom/application.ex @@ -41,8 +41,8 @@ defmodule Dotcom.Application do ]} }, RepoCache.Log, - CMS.Cache, - CMS.Repo, + {Application.get_env(:dotcom, :cms_cache, CMS.Cache), []}, + CMS.Telemetry, V3Api.Cache, Schedules.Repo, Schedules.RepoCondensed, diff --git a/lib/dotcom_web/controllers/cms_controller.ex b/lib/dotcom_web/controllers/cms_controller.ex index 9fe6fdf5e5..20f5b8c2c6 100644 --- a/lib/dotcom_web/controllers/cms_controller.ex +++ b/lib/dotcom_web/controllers/cms_controller.ex @@ -34,6 +34,8 @@ defmodule DotcomWeb.CMSController do Page.ProjectUpdate ] + @cache Application.compile_env!(:dotcom, :cms_cache) + @spec page(Conn.t(), map) :: Conn.t() def page(%Conn{request_path: path, query_params: query_params} = conn, _params) do conn = Conn.assign(conn, :try_encoded_on_404?, Map.has_key?(query_params, "id")) @@ -43,29 +45,32 @@ defmodule DotcomWeb.CMSController do |> handle_page_response(conn) end + @doc """ + Resets a cache key based on the URL params. + PATCH /cms/foo/bar will reset the cache key /cms/foo/bar. + This corresponds to the CMS page /foo/bar. + """ def reset_cache_key(conn, %{"object" => object, "id" => id}) do - Logger.notice("cms.cache.delete redis_host=#{System.get_env("REDIS_HOST")}") - try do - CMS.Cache.delete("/cms/#{object}/#{id}") + @cache.delete("/cms/#{object}/#{id}") Logger.notice("cms.cache.delete path=/cms/#{object}/#{id}") rescue e in Redix.ConnectionError -> Logger.warning("cms.cache.delete error=redis-#{e.reason}") + e in Redix.Error -> Logger.warning("cms.cache.delete error=redis-#{e.message}") end send_resp(conn, 202, "") |> halt() end def reset_cache_key(conn, %{"id" => id}) do - Logger.notice("cms.cache.delete redis_host=#{System.get_env("REDIS_HOST")}") - try do - CMS.Cache.delete("/cms/#{id}") + @cache.delete("/cms/#{id}") Logger.notice("cms.cache.delete path=/cms/#{id}") rescue e in Redix.ConnectionError -> Logger.warning("cms.cache.delete error=redis-#{e.reason}") + e in Redix.Error -> Logger.warning("cms.cache.delete error=redis-#{e.message}") end send_resp(conn, 202, "") |> halt() diff --git a/lib/dotcom_web/templates/mode/index.html.eex b/lib/dotcom_web/templates/mode/index.html.eex index 86e17fefe8..9d951074bc 100644 --- a/lib/dotcom_web/templates/mode/index.html.eex +++ b/lib/dotcom_web/templates/mode/index.html.eex @@ -44,8 +44,8 @@
<%= link("View fares overview", to: cms_static_page_path(@conn, "/fares"), class: "c-call-to-action") %> diff --git a/lib/routes/route.ex b/lib/routes/route.ex index 360d7fa972..751ef3a057 100644 --- a/lib/routes/route.ex +++ b/lib/routes/route.ex @@ -117,6 +117,8 @@ defmodule Routes.Route do def icon_atom(%__MODULE__{} = route), do: type_atom(route.type) + def icon_atom(nil), do: nil + @spec path_atom(t) :: gtfs_route_type def path_atom(%__MODULE__{type: 2}), do: :"commuter-rail" def path_atom(%__MODULE__{type: type}), do: type_atom(type) diff --git a/mix.exs b/mix.exs index f7bf3622ac..650a308bed 100644 --- a/mix.exs +++ b/mix.exs @@ -75,8 +75,10 @@ defmodule DotCom.Mixfile do {:bypass, "~> 1.0", [only: :test]}, {:castore, "~> 0.1.11"}, {:con_cache, "~> 0.12.0"}, + {:crc, "0.10.5"}, {:credo, "~> 1.5", only: [:dev, :test]}, {:csv, "~> 3.0.5"}, + {:decorator, "1.4.0"}, {:dialyxir, ">= 1.0.0-rc.4", [only: [:test, :dev], runtime: false]}, {:diskusage_logger, "~> 0.2.0"}, {:eflame, "~> 1.0", only: :dev}, @@ -127,8 +129,10 @@ defmodule DotCom.Mixfile do {:server_sent_event_stage, "~> 1.0"}, {:sizeable, "~> 0.1.5"}, {:sweet_xml, "~> 0.7.1", only: [:prod, :dev]}, - {:telemetry_metrics, "~> 0.6"}, - {:telemetry_poller, "~> 0.5"}, + {:telemetry, "0.4.3"}, + {:telemetry_metrics, "0.6.1"}, + {:telemetry_metrics_statsd, "0.7.0"}, + {:telemetry_poller, "0.5.1"}, {:timex, ">= 2.0.0"}, {:unrooted_polytree, "~> 0.1.1"}, {:wallaby, "~> 0.30", [runtime: false, only: [:test, :dev]]} diff --git a/mix.lock b/mix.lock index a3e962cc86..79bd6c3ea9 100644 --- a/mix.lock +++ b/mix.lock @@ -11,13 +11,16 @@ "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"}, "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, + "crc": {:hex, :crc, "0.10.5", "ee12a7c056ac498ef2ea985ecdc9fa53c1bfb4e53a484d9f17ff94803707dfd8", [:mix, :rebar3], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "3e673b6495a9525c5c641585af1accba59a1eb33de697bedf341e247012c2c7f"}, "credo": {:hex, :credo, "1.5.5", "e8f422026f553bc3bebb81c8e8bf1932f498ca03339856c7fec63d3faac8424b", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "dd8623ab7091956a855dc9f3062486add9c52d310dfd62748779c4315d8247de"}, "csv": {:hex, :csv, "3.0.5", "3c1455127e92de8845806db89554ad7d45e0212974be41dd9c38a5c881861713", [:mix], [], "hexpm", "cbbe5455c93df5f3f2943e995e28b7a8808361ba34cf3e44267d77a01eaf1609"}, + "decorator": {:hex, :decorator, "1.4.0", "a57ac32c823ea7e4e67f5af56412d12b33274661bb7640ec7fc882f8d23ac419", [:mix], [], "hexpm", "0a07cedd9083da875c7418dea95b78361197cf2bf3211d743f6f7ce39656597f"}, "dialyxir": {:hex, :dialyxir, "1.0.0-rc.4", "71b42f5ee1b7628f3e3a6565f4617dfb02d127a0499ab3e72750455e986df001", [:mix], [{:erlex, "~> 0.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "4bba10c6f267a0dd127d687d1295f6a11af6a7f160cc0e261c46f1962a98d7d8"}, "diskusage_logger": {:hex, :diskusage_logger, "0.2.0", "04fc48b538fe4de43153542a71ea94f623d54707d85844123baacfceedf625c3", [:mix], [], "hexpm", "e3f2aed1b0fc4590931c089a6453a4c4eb4c945912aa97bcabcc0cff7851f34d"}, "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "eflame": {:hex, :eflame, "1.0.1", "0664d287e39eef3c413749254b3af5f4f8b00be71c1af67d325331c4890be0fc", [:mix], [], "hexpm", "e0b08854a66f9013129de0b008488f3411ae9b69b902187837f994d7a99cf04e"}, "ehmon": {:git, "https://github.com/mbta/ehmon.git", "1fb603262bd02d74a16183bd8f344dcace9d7561", []}, + "elixir_make": {:hex, :elixir_make, "0.7.8", "505026f266552ee5aabca0b9f9c229cbb496c689537c9f922f3eb5431157efc7", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "7a71945b913d37ea89b06966e1342c85cfe549b15e6d6d081e8081c493062c07"}, "erlex": {:hex, :erlex, "0.1.6", "c01c889363168d3fdd23f4211647d8a34c0f9a21ec726762312e08e083f3d47e", [:mix], [], "hexpm", "f9388f7d1a668bee6ebddc040422ed6340af74aced153e492330da4c39516d92"}, "ex_aws": {:hex, :ex_aws, "2.5.0", "1785e69350b16514c1049330537c7da10039b1a53e1d253bbd703b135174aec3", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "971b86e5495fc0ae1c318e35e23f389e74cf322f2c02d34037c6fc6d405006f1"}, "ex_aws_s3": {:hex, :ex_aws_s3, "2.4.0", "ce8decb6b523381812798396bc0e3aaa62282e1b40520125d1f4eff4abdff0f4", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "85dda6e27754d94582869d39cba3241d9ea60b6aa4167f9c88e309dc687e56bb"}, @@ -92,6 +95,7 @@ "sweet_xml": {:hex, :sweet_xml, "0.7.3", "debb256781c75ff6a8c5cbf7981146312b66f044a2898f453709a53e5031b45b", [:mix], [], "hexpm", "e110c867a1b3fe74bfc7dd9893aa851f0eed5518d0d7cad76d7baafd30e4f5ba"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, + "telemetry_metrics_statsd": {:hex, :telemetry_metrics_statsd, "0.7.0", "92732fae63db31ef2508df6faee7d81401883e33f2976715a82f296a33a45cee", [:mix], [{:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "797e34a856376dfd4e96347da0f747fcff4e0cadf6e6f0f989598f563cad05ff"}, "telemetry_poller": {:hex, :telemetry_poller, "0.5.1", "21071cc2e536810bac5628b935521ff3e28f0303e770951158c73eaaa01e962a", [:rebar3], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4cab72069210bc6e7a080cec9afffad1b33370149ed5d379b81c7c5f0c663fd4"}, "tesla": {:hex, :tesla, "1.5.0", "7ee3616be87024a2b7231ae14474310c9b999c3abb1f4f8dbc70f86bd9678eef", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "1d0385e41fbd76af3961809088aef15dec4c2fdaab97b1c93c6484cb3695a122"}, "timex": {:hex, :timex, "3.1.24", "d198ae9783ac807721cca0c5535384ebdf99da4976be8cefb9665a9262a1e9e3", [:mix], [{:combine, "~> 0.7", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "ca852258d788542c263b12dbf55375fe2ccf5674e7b20995e3d84d2d4412bc0f"}, diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000000..900a3226dd --- /dev/null +++ b/package-lock.json @@ -0,0 +1,11 @@ +{ + "name": "dotcom-scripts", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "dotcom-scripts", + "hasInstallScript": true + } + } +} diff --git a/test/cms/repo_test.exs b/test/cms/repo_test.exs index e56a85ca2e..abf15d89c2 100644 --- a/test/cms/repo_test.exs +++ b/test/cms/repo_test.exs @@ -31,6 +31,13 @@ defmodule CMS.RepoTest do WhatsHappeningItem } + setup do + cache = Application.get_env(:dotcom, :cms_cache) + cache.flush() + + %{cache: cache} + end + describe "news_entry_by/1" do test "returns the news entry for the given id" do assert %NewsEntry{id: 3519} = Repo.news_entry_by(id: 3519) @@ -60,31 +67,64 @@ defmodule CMS.RepoTest do end describe "get_page/1" do - test "caches views" do + test "generates the correct key for /*" do + path = "/foo" + + assert Repo.generate(nil, nil, [path, %{}]) == "/cms" <> path + end + + test "generates the correct key for /**/*" do + path = "/foo/bar" + + assert Repo.generate(nil, nil, [path, %{}]) == "/cms" <> path + end + + test "generates the correct key for /**/*?*=*" do + path = "/foo/bar" + params = %{"baz" => "bop"} + + assert Repo.generate(nil, nil, [path, params]) == "/cms" <> path <> "?baz=bop" + end + + test "generates the correct key for /**/*?*=*&*=*" do + path = "/foo/bar" + params = %{"bam" => "bop", "baz" => "qux"} + + assert Repo.generate(nil, nil, [path, params]) == "/cms" <> path <> "?bam=bop&baz=qux" + end + + test "caches views", %{cache: cache} do path = "/news/2018/news-entry" params = %{} - cache_key = {:view_or_preview, path: path, params: params} + key = "/cms" <> path - # ensure cache is empty - case ConCache.get(Repo, cache_key) do - nil -> - :ok + assert cache.get(key) == nil - {:ok, %{"type" => [%{"target_id" => "news_entry"}]}} -> - ConCache.dirty_delete(Repo, cache_key) - end + Repo.get_page(path, params) - assert %NewsEntry{} = Repo.get_page(path, params) - assert {:ok, %{"type" => [%{"target_id" => "news_entry"}]}} = ConCache.get(Repo, cache_key) + assert cache.get(key) != nil end - test "does not cache previews" do - path = "/basic_page_no_sidebar" + test "sets the ttl to < :infinity", %{cache: cache} do + path = "/news/2018/news-entry" + params = %{} + key = "/cms" <> path + + Repo.get_page(path, params) + + assert cache.ttl(key) != :infinity + end + + test "does not cache previews", %{cache: cache} do + path = "/news/2018/news-entry" params = %{"preview" => "", "vid" => "112", "nid" => "6"} - cache_key = {:view_or_preview, path: path, params: params} - assert ConCache.get(Repo, cache_key) == nil - assert %Basic{} = Repo.get_page(path, params) - assert ConCache.get(Repo, cache_key) == nil + key = "/cms" <> path + + assert cache.get(key) == nil + + Repo.get_page(path, params) + + assert cache.get(key) == nil end test "given the path for a Basic page" do @@ -550,11 +590,11 @@ defmodule CMS.RepoTest do } end - year = 2022 - mock_2022_opts = opts.(year) + year = 2018 + mock_2018_opts = opts.(year) - with_mock Static, view: fn "/cms/teasers", ^mock_2022_opts -> {:ok, []} end do - _events = Repo.events_for_year(year) + with_mock Static, view: fn "/cms/teasers", ^mock_2018_opts -> {:ok, []} end do + Repo.events_for_year(year) Static.view("/cms/teasers", opts.(year)) |> assert_called() diff --git a/test/cms/telemetry/reporter_test.exs b/test/cms/telemetry/reporter_test.exs new file mode 100644 index 0000000000..553c100ea8 --- /dev/null +++ b/test/cms/telemetry/reporter_test.exs @@ -0,0 +1,26 @@ +defmodule CMS.Telemetry.ReporterTest do + use ExUnit.Case, async: true + + import ExUnit.CaptureLog + + test "the hit rate gets logged" do + CMS.Telemetry.Reporter.start_link( + metrics: [Telemetry.Metrics.last_value("cms.cache.stats.updates")] + ) + + assert capture_log(fn -> + :telemetry.execute([:cms, :cache, :stats], %{hits: 99, misses: 1}) + end) =~ "hit_rate=0.99" + end + + test "cache command exceptions get logged" do + CMS.Telemetry.Reporter.start_link(metrics: []) + + assert capture_log(fn -> + :telemetry.execute([:cms, :cache, :command, :exception], %{duration: 0}, %{ + kind: :error, + reason: %Redix.ConnectionError{reason: :closed} + }) + end) =~ "cms.cache.command.exception kind=error reason=closed" + end +end diff --git a/test/dotcom_web/controllers/cms_controller_test.exs b/test/dotcom_web/controllers/cms_controller_test.exs index d58b8f7f7f..4a0a925e4d 100644 --- a/test/dotcom_web/controllers/cms_controller_test.exs +++ b/test/dotcom_web/controllers/cms_controller_test.exs @@ -1,10 +1,10 @@ defmodule DotcomWeb.CMSControllerTest do use DotcomWeb.ConnCase, async: false - import ExUnit.CaptureLog - alias Plug.Conn + @cache Application.compile_env!(:dotcom, :cms_cache) + describe "GET - page" do test "renders a basic page when the CMS returns a CMS.Page.Basic", %{conn: conn} do conn = get(conn, "/basic_page_no_sidebar") @@ -186,13 +186,40 @@ defmodule DotcomWeb.CMSControllerTest do end describe "PATCH /cms/*" do - test "it logs that no redis connection was made", %{conn: conn} do - assert capture_log(fn -> - conn - |> put_req_header("content-type", "application/json") - |> put_req_header("authorization", "Basic " <> Base.encode64("username:password")) - |> patch("/cms/foo/bar") - end) =~ "cms.cache.delete error=redis-closed" + test "it removes an entry from the cache", %{conn: conn} do + path = "/cms/foo" + + @cache.put(path, "bar") + + assert @cache.get(path) != nil + + conn = + conn + |> put_req_header("content-type", "application/json") + |> put_req_header("authorization", "Basic " <> Base.encode64("username:password")) + |> patch(path) + + assert @cache.get(path) == nil + assert conn.status == 202 + end + end + + describe "PATCH /cms/**/*" do + test "it removes an entry from the cache", %{conn: conn} do + path = "/cms/foo/bar" + + @cache.put(path, "baz") + + assert @cache.get(path) != nil + + conn = + conn + |> put_req_header("content-type", "application/json") + |> put_req_header("authorization", "Basic " <> Base.encode64("username:password")) + |> patch(path) + + assert @cache.get(path) == nil + assert conn.status == 202 end end end diff --git a/test/support/test_cache.ex b/test/support/test_cache.ex new file mode 100644 index 0000000000..f1996cf389 --- /dev/null +++ b/test/support/test_cache.ex @@ -0,0 +1,7 @@ +defmodule CMS.TestCache do + @moduledoc """ + This local Nebulex instance allows us to run tests w/out the need for Redis. + """ + + use Nebulex.Cache, otp_app: :cms, adapter: Nebulex.Adapters.Local +end