Prune resolved errors

guides/Getting Started.md

@@ -138,3 +138,11 @@ We currently do not support notifications out of the box.
 However, we provideo some detailed Telemetry events that you may use to implement your own notifications following your custom rules and notification channels.
 
 If you want to take a look at the events you can attach to, take a look at `ErrorTracker.Telemetry` module documentation.
+
+## Pruning resolved errors
+
+By default errors are kept in the database indefinitely. This is not ideal for production
+environments where you may want to prune old errors that have been resolved.
+
+The `ErrorTracker.Plugins.Pruner` module provides automatic pruning functionality with a configurable
+interval and error age.

lib/error_tracker/application.ex

@@ -4,7 +4,7 @@ defmodule ErrorTracker.Application do
   use Application
 
   def start(_type, _args) do
-    children = []
+    children = Application.get_env(:error_tracker, :plugins, [])
 
     attach_handlers()
 

lib/error_tracker/plugins/pruner.ex

@@ -0,0 +1,142 @@
+defmodule ErrorTracker.Plugins.Pruner do
+  @moduledoc """
+  Periodically delete resolved errors based on their age.
+
+  Pruning allows you to keep your database size under control by removing old errors that are not
+  needed anymore.
+
+  ## Using the pruner
+
+  To enable the pruner you must register the plugin in the ErrorTracker configuration. This will use
+  the default options, which is to prune errors resolved after 5 minutes.
+
+      config :error_tracker,
+        plugins: [ErrorTracker.Plugins.Pruner]
+
+  You can override the default options by passing them as an argument when registering the plugin.
+
+      config :error_tracker,
+        plugins: [{ErrorTracker.Plugins.Pruner, max_age: :timer.minutes(30)}]
+
+  ## Options
+
+  - `:limit`  - the maximum number of errors to prune on each execution. Occurrences are removed
+    along the errors. The default is 200 to prevent timeouts and unnecesary database load.
+
+  - `:max_age` - the number of milliseconds after a resolved error may be pruned. The default is 24
+    hours.
+
+  - `:interval` - the interval in milliseconds between pruning runs. The default is 30 minutes.
+
+  You may find the `:timer` module functions useful to pass readable values to the `:max_age` and
+  `:interval` options.
+
+  ## Manual pruning
+
+  In certain cases you may prefer to run the pruner manually. This can be done by calling the
+  `prune_errors/2` function from your application code. This function supports the `:limit` and
+  `:max_age` options as described above.
+
+  For example, you may call this function from an Oban worker so you can leverage Oban's cron
+  capabilities and have a more granular control over when pruning is run.
+
+      defmodule MyApp.ErrorPruner do
+        use Oban.Job
+
+        def perform(%Job{}) do
+          ErrorTracker.Plugins.Pruner.prune_errors(limit: 10_000, max_age: :timer.minutes(60))
+        end
+      end
+  """
+  use GenServer
+
+  import Ecto.Query
+
+  alias ErrorTracker.Error
+  alias ErrorTracker.Occurrence
+  alias ErrorTracker.Repo
+
+  @doc """
+  Prunes resolved errors.
+
+  You do not need to use this function if you activate the Pruner plugin. This function is exposed
+  only for advanced use cases and Oban integration.
+
+  ## Options
+
+  - `:limit`  - the maximum number of errors to prune on each execution. Occurrences are removed
+    along the errors. The default is 200 to prevent timeouts and unnecesary database load.
+
+  - `:max_age` - the number of milliseconds after a resolved error may be pruned. The default is 24
+    hours. You may find the `:timer` module functions useful to pass readable values to this option.
+  """
+  @spec prune_errors(keyword()) :: {:ok, list(Error.t())}
+  def prune_errors(opts \\ []) do
+    limit = opts[:limit] || raise ":limit option is required"
+    max_age = opts[:max_age] || raise ":max_age option is required"
+    time = DateTime.add(DateTime.utc_now(), max_age, :millisecond)
+
+    errors =
+      Repo.all(
+        from error in Error,
+          select: [:id, :kind, :source_line, :source_function],
+          where: error.status == :resolved,
+          where: error.last_occurrence_at < ^time,
+          limit: ^limit
+      )
+
+    if Enum.any?(errors) do
+      :ok =
+        errors
+        |> Ecto.assoc(:occurrences)
+        |> limit(1000)
+        |> prune_occurrences()
+        |> Stream.run()
+
+      Repo.delete_all(from error in Error, where: error.id in ^Enum.map(errors, & &1.id))
+    end
+
+    {:ok, errors}
+  end
+
+  defp prune_occurrences(occurrences_query) do
+    Stream.unfold(occurrences_query, fn occurrences_query ->
+      occurrences_ids = Repo.all(from occurrence in occurrences_query, select: occurrence.id)
+
+      case Repo.delete_all(from o in Occurrence, where: o.id in ^occurrences_ids) do
+        {0, _} -> nil
+        {deleted, _} -> {deleted, occurrences_query}
+      end
+    end)
+  end
+
+  def start_link(state \\ []) do
+    GenServer.start_link(__MODULE__, state, name: __MODULE__)
+  end
+
+  @impl GenServer
+  @doc false
+  def init(state \\ []) do
+    state = %{
+      limit: state[:limit] || 200,
+      max_age: state[:max_age] || :timer.hours(24),
+      interval: state[:interval] || :timer.minutes(30)
+    }
+
+    {:ok, schedule_prune(state)}
+  end
+
+  @impl GenServer
+  @doc false
+  def handle_info(:prune, state) do
+    {:ok, _pruned} = prune_errors(state)
+
+    {:noreply, schedule_prune(state)}
+  end
+
+  defp schedule_prune(state = %{interval: interval}) do
+    Process.send_after(self(), :prune, interval)
+
+    state
+  end
+end

lib/error_tracker/repo.ex

@@ -25,6 +25,10 @@ defmodule ErrorTracker.Repo do
     dispatch(:all, [queryable], opts)
   end
 
+  def delete_all(queryable, opts \\ []) do
+    dispatch(:delete_all, [queryable], opts)
+  end
+
   def aggregate(queryable, aggregate, opts \\ []) do
     dispatch(:aggregate, [queryable, aggregate], opts)
   end

mix.exs

@@ -65,6 +65,9 @@ defmodule ErrorTracker.MixProject do
         ErrorTracker.Integrations.Phoenix,
         ErrorTracker.Integrations.Plug
       ],
+      Plugins: [
+        ErrorTracker.Plugins.Pruner
+      ],
       Schemas: [
         ErrorTracker.Error,
         ErrorTracker.Occurrence,

priv/repo/seeds.exs

@@ -0,0 +1,45 @@
+adapter =
+  case Application.get_env(:error_tracker, :ecto_adapter) do
+    :postgres -> Ecto.Adapters.Postgres
+    :sqlite3 -> Ecto.Adapters.SQLite3
+  end
+
+defmodule ErrorTrackerDev.Repo do
+  use Ecto.Repo, otp_app: :error_tracker, adapter: adapter
+end
+
+ErrorTrackerDev.Repo.start_link()
+
+ErrorTrackerDev.Repo.delete_all(ErrorTracker.Error)
+
+errors =
+  for i <- 1..100 do
+    %{
+      kind: "Error #{i}",
+      reason: "Reason #{i}",
+      source_line: "line",
+      source_function: "function",
+      status: :unresolved,
+      fingerprint: "#{i}",
+      last_occurrence_at: DateTime.utc_now(),
+      inserted_at: DateTime.utc_now(),
+      updated_at: DateTime.utc_now()
+    }
+  end
+
+{_, errors} = dbg(ErrorTrackerDev.Repo.insert_all(ErrorTracker.Error, errors, returning: [:id]))
+
+for error <- errors do
+  occurrences =
+    for _i <- 1..200 do
+      %{
+        context: %{},
+        reason: "REASON",
+        stacktrace: %ErrorTracker.Stacktrace{},
+        error_id: error.id,
+        inserted_at: DateTime.utc_now()
+      }
+    end
+
+  ErrorTrackerDev.Repo.insert_all(ErrorTracker.Occurrence, occurrences)
+end