diff --git a/assets/js/dashboard/extra/exploration.js b/assets/js/dashboard/extra/exploration.js index e6bd9f19752e..09aecc2467e3 100644 --- a/assets/js/dashboard/extra/exploration.js +++ b/assets/js/dashboard/extra/exploration.js @@ -77,12 +77,15 @@ function dashboardStateForQuery(dashboardState, steps) { // Serialize steps into the wire format expected by the API. function stepsToJourneyParam(steps) { return JSON.stringify( - steps.map(({ name, pathname, includes_subpaths, subpaths_count }) => ({ - name, - pathname, - includes_subpaths, - subpaths_count - })) + steps.map( + ({ name, pathname, includes_subpaths, subpaths_count, is_goal }) => ({ + name, + pathname, + includes_subpaths, + subpaths_count, + is_goal + }) + ) ) } diff --git a/extra/lib/plausible/stats/exploration.ex b/extra/lib/plausible/stats/exploration.ex index 6eaa5f77f292..1bf0a5437349 100644 --- a/extra/lib/plausible/stats/exploration.ex +++ b/extra/lib/plausible/stats/exploration.ex @@ -8,16 +8,22 @@ defmodule Plausible.Stats.Exploration do @type t() :: %__MODULE__{} - @derive {Jason.Encoder, only: [:name, :pathname, :label, :includes_subpaths, :subpaths_count]} - defstruct name: nil, pathname: "", label: nil, includes_subpaths: false, subpaths_count: 0 + @derive {Jason.Encoder, + only: [:name, :pathname, :label, :includes_subpaths, :subpaths_count, :is_goal]} + defstruct name: nil, + pathname: "", + label: nil, + includes_subpaths: false, + subpaths_count: 0, + is_goal: false @spec from(map()) :: t() def from(step) do - new(step.name, step.pathname, step.includes_subpaths, step.subpaths_count) + new(step.name, step.pathname, step.includes_subpaths, step.subpaths_count, step.is_goal) end - @spec new(String.t(), String.t(), boolean(), non_neg_integer()) :: t() - def new(name, pathname, includes_subpaths \\ false, subpaths_count \\ 0) + @spec new(String.t(), String.t(), boolean(), non_neg_integer(), boolean()) :: t() + def new(name, pathname, includes_subpaths \\ false, subpaths_count \\ 0, is_goal \\ false) when is_boolean(includes_subpaths) and is_integer(subpaths_count) do label = if name != "pageview" do @@ -31,7 +37,8 @@ defmodule Plausible.Stats.Exploration do name: name, pathname: pathname, includes_subpaths: includes_subpaths, - subpaths_count: subpaths_count + subpaths_count: subpaths_count, + is_goal: is_goal } end end @@ -42,6 +49,7 @@ defmodule Plausible.Stats.Exploration do alias Plausible.ClickhouseRepo alias Plausible.Stats.Base + alias Plausible.Stats.Filters alias Plausible.Stats.Query @type journey() :: [Journey.Step.t()] @@ -76,24 +84,29 @@ defmodule Plausible.Stats.Exploration do @spec max_steps() :: pos_integer() def max_steps, do: @max_steps - @spec next_steps(Query.t(), journey(), keyword()) :: + @spec next_steps(Plausible.Site.t(), Query.t(), journey(), keyword()) :: {:ok, [next_step()]} | {:error, :journey_too_long} - def next_steps(query, journey, opts \\ []) + def next_steps(site, query, journey, opts \\ []) - def next_steps(_query, journey, _opts) when length(journey) >= @max_steps do + def next_steps(_site, _query, journey, _opts) when length(journey) >= @max_steps do {:error, :journey_too_long} end - def next_steps(query, journey, opts) do + def next_steps(site, query, journey, opts) do opts = Keyword.merge(@next_steps_defaults, opts) direction = Keyword.fetch!(opts, :direction) search_term = Keyword.fetch!(opts, :search_term) max_candidates = min(Keyword.fetch!(opts, :max_candidates), @max_candidates) include_wilcard? = Keyword.fetch!(opts, :include_wildcard?) + goals = + site + |> Plausible.Goals.for_site(include_goals_with_custom_props?: false) + |> filter_eligible_goals() + query |> Base.base_event_query() - |> next_steps_query(journey, search_term, direction, max_candidates, include_wilcard?) + |> next_steps_query(journey, search_term, direction, max_candidates, include_wilcard?, goals) # We pass the query struct to record query metadata for # the CH debug console. |> ClickhouseRepo.all(query: query) @@ -143,9 +156,9 @@ defmodule Plausible.Stats.Exploration do to include implicit wildcard pathnames in suggestions or not (default: true) """ - @spec interesting_funnel(Query.t(), keyword()) :: + @spec interesting_funnel(Plausible.Site.t(), Query.t(), keyword()) :: {:ok, %{funnel: [funnel_step()], candidates: [next_step()]}} | {:error, :not_found} - def interesting_funnel(query, opts \\ []) do + def interesting_funnel(site, query, opts \\ []) do max_steps = min(Keyword.get(opts, :max_steps, 6), @max_steps) max_candidates = min(Keyword.get(opts, :max_candidates, 10), @max_candidates) @@ -157,14 +170,15 @@ defmodule Plausible.Stats.Exploration do ) with {:ok, result} <- - build_interesting_journey(query, max_steps, max_candidates, include_wildcard?), + build_interesting_journey(site, query, max_steps, max_candidates, include_wildcard?), {:ok, funnel} <- journey_funnel(query, result.journey) do {:ok, %{funnel: funnel, candidates: result.candidates}} end end - defp build_interesting_journey(query, max_steps, max_candidates, include_wildcard?) do + defp build_interesting_journey(site, query, max_steps, max_candidates, include_wildcard?) do case do_build_journey( + site, query, [], [], @@ -179,6 +193,7 @@ defmodule Plausible.Stats.Exploration do end defp do_build_journey( + _site, _query, journey, step_candidates, @@ -192,6 +207,7 @@ defmodule Plausible.Stats.Exploration do end defp do_build_journey( + site, query, journey, step_candidates, @@ -201,7 +217,7 @@ defmodule Plausible.Stats.Exploration do include_wildcard? ) do {:ok, candidates} = - next_steps(query, journey, + next_steps(site, query, journey, max_candidates: max_candidates, include_wildcard?: include_wildcard? ) @@ -214,6 +230,7 @@ defmodule Plausible.Stats.Exploration do new_seen = MapSet.put(seen, normalize_step_key(step)) do_build_journey( + site, query, journey ++ [step], step_candidates ++ [candidates], @@ -238,7 +255,22 @@ defmodule Plausible.Stats.Exploration do defp normalize_pathname("/"), do: "/" defp normalize_pathname(pathname), do: String.trim_trailing(pathname, "/") - defp next_steps_query(query, steps, search_term, direction, max_candidates, include_wildcard?) + defp filter_eligible_goals(goals) do + Enum.reject(goals, fn g -> + Plausible.Goal.Revenue.revenue?(g) or g.scroll_threshold > -1 or + Plausible.Goal.has_custom_props?(g) + end) + end + + defp next_steps_query( + query, + steps, + search_term, + direction, + max_candidates, + include_wildcard?, + goals + ) when is_direction(direction) do next_step_idx = length(steps) + 1 q_steps = steps_query(query, next_step_idx, direction) @@ -250,9 +282,12 @@ defmodule Plausible.Stats.Exploration do from(s in subquery(q_steps), where: selected_as(:name) != "", select: %{ + user_id: s.user_id, name: selected_as(field(s, ^next_name), :name), - pathname: selected_as(field(s, ^next_pathname), :pathname) - } + pathname: selected_as(field(s, ^next_pathname), :pathname), + _sample_factor: fragment("any(?)", s._sample_factor) + }, + group_by: [selected_as(:name), selected_as(:pathname), s.user_id] ) q_matches = @@ -264,14 +299,6 @@ defmodule Plausible.Stats.Exploration do from(s in q, where: ^step_condition) end) - q_per_user_matches = - from(m in q_matches, - select_merge: %{user_id: m.user_id, _sample_factor: fragment("any(?)", m._sample_factor)}, - group_by: [selected_as(:name), selected_as(:pathname), m.user_id] - ) - - q_combined = combined_query(q_per_user_matches, include_wildcard?) - # Fan out each q_combined row into up to two output rows (exact + wildcard) # using ARRAY JOIN over a small boolean array. # @@ -280,55 +307,29 @@ defmodule Plausible.Stats.Exploration do # subpath, or same visitor count as exact). ARRAY JOIN then emits one or more # rows per group. The joined boolean `is_wildcard` selects which values to # use for visitors / includes_subpaths / subpaths_count. - q_all_matches = - from(m in subquery(q_combined), - join: - is_wildcard in fragment( - """ - arrayFilter( - x -> x = false OR (? = 'pageview' AND ? != '/' AND ? > 1 AND ? != ?), - [false, true] - ) - """, - m.name, - m.pathname, - m.subpaths_count, - m.wildcard_visitors, - m.exact_visitors - ), - on: true, - hints: "ARRAY", - where: selected_as(:visitors) > 0, - select: %{ - name: m.name, - pathname: m.pathname, - visitors: - selected_as( - fragment("if(?, ?, ?)", is_wildcard, m.wildcard_visitors, m.exact_visitors), - :visitors - ), - includes_subpaths: fragment("CAST(?, 'Bool')", is_wildcard), - subpaths_count: fragment("if(?, ?, 0)", is_wildcard, m.subpaths_count) - } - ) - - from(m in subquery(q_all_matches), + q_wildcard_combined_matches = + q_matches + |> combined_wildcard_query(include_wildcard?) + |> combined_wildcard_matches_query() + + q_all_combined_matches = + if q_goal_matches = goals_query(q_matches, goals) do + q_wildcard_combined_matches + |> exclude_goal_matches(goals) + |> union_all(^q_goal_matches) + else + q_wildcard_combined_matches + end + + from(m in subquery(q_all_combined_matches), select: %{ step: %Journey.Step{ - label: - selected_as( - fragment( - "if(? != 'pageview', ?, ?)", - m.name, - m.name, - m.pathname - ), - :label - ), + label: selected_as(m.label, :label), name: m.name, pathname: m.pathname, includes_subpaths: m.includes_subpaths, - subpaths_count: m.subpaths_count + subpaths_count: m.subpaths_count, + is_goal: m.is_goal }, visitors: m.visitors }, @@ -342,6 +343,81 @@ defmodule Plausible.Stats.Exploration do |> maybe_search(search_term) end + defp goals_query(_, []), do: nil + + defp goals_query(q_matches, goals) do + values = + Enum.map(goals, fn g -> + pathname = g.page_path || "" + + regex_pathname = + if String.contains?(pathname, "*") do + Filters.Utils.page_regex(pathname) + else + "" + end + + %{ + label: g.display_name, + name: g.event_name || "pageview", + pathname: pathname, + regex_pathname: regex_pathname + } + end) + + types = %{label: :string, name: :string, pathname: :string, regex_pathname: :string} + + query = + from(m in subquery(q_matches), + inner_join: g in values(values, types), + on: + g.name == m.name and + (g.name != "pageview" or + (g.name == "pageview" and + fragment( + "if(? != '', match(?, ?), ? = ?)", + g.regex_pathname, + m.pathname, + g.regex_pathname, + m.pathname, + g.pathname + ))), + select: %{ + label: selected_as(g.label, :label), + name: selected_as(g.name, :name), + pathname: selected_as(g.pathname, :pathname), + visitors: scale_sample(fragment("uniq(?)", m.user_id)), + includes_subpaths: fragment("CAST(?, 'Bool')", false), + subpaths_count: 0, + is_goal: fragment("CAST(?, 'Bool')", true) + }, + group_by: [selected_as(:label), selected_as(:name), selected_as(:pathname)] + ) + + from(m in subquery(query), + where: m.visitors > 0 + ) + end + + defp exclude_goal_matches(query, goals) do + to_exclude = + goals + |> Enum.filter(fn g -> is_nil(g.page_path) or not String.contains?(g.page_path, "*") end) + |> Enum.map(fn g -> + %{ + name: g.event_name || "pageview", + pathname: g.page_path || "" + } + end) + + types = %{name: :string, pathname: :string} + + from m in subquery(query), + left_join: g in values(to_exclude, types), + on: g.name == m.name and g.pathname == m.pathname, + where: g.name == "" or m.includes_subpaths + end + # Expand each (name, pathname, user_id) row into all prefix paths via # ARRAY JOIN, then aggregate once to get both exact and wildcard visitor # counts in a single scan of events_v2. @@ -361,7 +437,7 @@ defmodule Plausible.Stats.Exploration do arraySlice(split_pathname, 1, 1)), [?]) """ - defp combined_query(q_matches, true = _include_wildcard?) do + defp combined_wildcard_query(q_matches, true = _include_wildcard?) do from(em in subquery(q_matches), join: pname in fragment(@wildcard_array_join, em.name, em.pathname, em.pathname), on: true, @@ -380,7 +456,7 @@ defmodule Plausible.Stats.Exploration do ) end - defp combined_query(q_matches, false = _include_wildcard?) do + defp combined_wildcard_query(q_matches, false = _include_wildcard?) do from(em in subquery(q_matches), where: em.name != "pageview" or selected_as(:pathname) != "", select: %{ @@ -395,6 +471,51 @@ defmodule Plausible.Stats.Exploration do ) end + defp combined_wildcard_matches_query(q_wildcard_combined) do + from(m in subquery(q_wildcard_combined), + join: + is_wildcard in fragment( + """ + arrayFilter( + x -> x = false OR (? = 'pageview' AND ? != '/' AND ? > 1 AND ? != ?), + [false, true] + ) + """, + m.name, + m.pathname, + m.subpaths_count, + m.wildcard_visitors, + m.exact_visitors + ), + on: true, + hints: "ARRAY", + where: selected_as(:visitors) > 0, + select: %{ + label: + selected_as( + fragment( + "if(? != 'pageview', ?, ?)", + m.name, + m.name, + m.pathname + ), + :label + ), + name: selected_as(m.name, :name), + pathname: selected_as(m.pathname, :pathname), + visitors: + selected_as( + fragment("if(?, ?, ?)", is_wildcard, m.wildcard_visitors, m.exact_visitors), + :visitors + ), + includes_subpaths: + selected_as(fragment("CAST(?, 'Bool')", is_wildcard), :includes_subpaths), + subpaths_count: fragment("if(?, ?, 0)", is_wildcard, m.subpaths_count), + is_goal: fragment("CAST(?, 'Bool')", false) + } + ) + end + defp journey_funnel_query(query, steps, direction) do q_steps = steps_query(query, length(steps), direction) @@ -524,22 +645,33 @@ defmodule Plausible.Stats.Exploration do end defp step_condition(step, count) when count <= @max_steps do - if step.includes_subpaths do - escaped = Regex.escape(step.pathname) + cond do + step.includes_subpaths -> + escaped = Regex.escape(step.pathname) - pattern = "^#{escaped}(/.+)?$" + pattern = "^#{escaped}(/.+)?$" - dynamic( - [s], - field(s, ^:"name#{count}") == ^step.name and - fragment("match(?, ?)", field(s, ^:"pathname#{count}"), ^pattern) - ) - else - dynamic( - [s], - field(s, ^:"name#{count}") == ^step.name and - field(s, ^:"pathname#{count}") == ^step.pathname - ) + dynamic( + [s], + field(s, ^:"name#{count}") == ^step.name and + fragment("match(?, ?)", field(s, ^:"pathname#{count}"), ^pattern) + ) + + step.is_goal and step.name == "pageview" and String.contains?(step.pathname, "*") -> + pattern = Filters.Utils.page_regex(step.pathname) + + dynamic( + [s], + field(s, ^:"name#{count}") == ^step.name and + fragment("match(?, ?)", field(s, ^:"pathname#{count}"), ^pattern) + ) + + true -> + dynamic( + [s], + field(s, ^:"name#{count}") == ^step.name and + field(s, ^:"pathname#{count}") == ^step.pathname + ) end end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 3aeca5b42093..8d723d33e733 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -151,7 +151,7 @@ defmodule PlausibleWeb.Api.StatsController do include_wildcard? = not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site), {:ok, next_steps} <- - Exploration.next_steps(query, journey, + Exploration.next_steps(site, query, journey, search_term: search_term, direction: direction, include_wildcard?: include_wildcard? @@ -188,7 +188,7 @@ defmodule PlausibleWeb.Api.StatsController do include_wildcard? = not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site) - case Exploration.interesting_funnel(query, + case Exploration.interesting_funnel(site, query, max_steps: params["max_steps"], max_candidates: params["max_candidates"], include_wildcard?: include_wildcard? @@ -209,7 +209,7 @@ defmodule PlausibleWeb.Api.StatsController do include_wildcard? = not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site), {:ok, next_steps} <- - Exploration.next_steps(query, journey, + Exploration.next_steps(site, query, journey, search_term: search_term, direction: direction, include_wildcard?: include_wildcard? @@ -242,13 +242,15 @@ defmodule PlausibleWeb.Api.StatsController do "name" => name, "pathname" => pathname, "includes_subpaths" => includes_subpaths, - "subpaths_count" => subpaths_count + "subpaths_count" => subpaths_count, + "is_goal" => is_goal }) do Exploration.Journey.Step.new( name, pathname, includes_subpaths, - subpaths_count + subpaths_count, + is_goal ) end diff --git a/test/plausible/stats/exploration_test.exs b/test/plausible/stats/exploration_test.exs index e2deaf5dbcea..4cb916f594f3 100644 --- a/test/plausible/stats/exploration_test.exs +++ b/test/plausible/stats/exploration_test.exs @@ -238,6 +238,51 @@ defmodule Plausible.Stats.ExplorationTest do assert step3.conversion_rate == "50" assert step3.conversion_rate_step == "100" end + + test "handles goal with a pattern" do + journey = [ + %Exploration.Journey.Step{name: "pageview", pathname: "/site*", is_goal: true}, + %Exploration.Journey.Step{name: "pageview", pathname: "/dashboard"} + ] + + site = new_site() + now = DateTime.utc_now() + + populate_stats(site, [ + build(:pageview, + user_id: 123, + pathname: "/sites", + timestamp: DateTime.shift(now, minute: -50) + ), + build(:pageview, + user_id: 123, + pathname: "/dashboard", + timestamp: DateTime.shift(now, minute: -40) + ), + build(:pageview, + user_id: 124, + pathname: "/sites/settings", + timestamp: DateTime.shift(now, minute: -50) + ) + ]) + + query = QueryBuilder.build!(site, input_date_range: :all) + + assert {:ok, [step1, step2]} = Exploration.journey_funnel(query, journey) + + assert step1.step.pathname == "/site*" + assert step1.visitors == 2 + assert step1.dropoff == 0 + assert step1.dropoff_percentage == "0" + assert step1.conversion_rate == "100" + assert step1.conversion_rate_step == "0" + assert step2.step.pathname == "/dashboard" + assert step2.visitors == 1 + assert step2.dropoff == 1 + assert step2.dropoff_percentage == "50" + assert step2.conversion_rate == "50" + assert step2.conversion_rate_step == "50" + end end describe "interesting_funnel" do @@ -245,7 +290,7 @@ defmodule Plausible.Stats.ExplorationTest do query = QueryBuilder.build!(site, input_date_range: :all) assert {:ok, %{funnel: [step1, step2, step3, step4]}} = - Exploration.interesting_funnel(query) + Exploration.interesting_funnel(site, query) assert step1.step.pathname == "/home" assert step1.visitors == 2 @@ -264,7 +309,7 @@ defmodule Plausible.Stats.ExplorationTest do query = QueryBuilder.build!(site, input_date_range: :all) assert {:ok, %{funnel: [step1, step2]}} = - Exploration.interesting_funnel(query, max_steps: 2) + Exploration.interesting_funnel(site, query, max_steps: 2) assert step1.step.pathname == "/home" assert step2.step.pathname == "/login" @@ -274,7 +319,7 @@ defmodule Plausible.Stats.ExplorationTest do empty_site = new_site() query = QueryBuilder.build!(empty_site, input_date_range: :all) - assert {:error, :not_found} = Exploration.interesting_funnel(query) + assert {:error, :not_found} = Exploration.interesting_funnel(empty_site, query) end test "stops when no more unseen steps are available" do @@ -291,7 +336,8 @@ defmodule Plausible.Stats.ExplorationTest do query = QueryBuilder.build!(site, input_date_range: :all) - assert {:ok, %{funnel: [step1]}} = Exploration.interesting_funnel(query, max_steps: 6) + assert {:ok, %{funnel: [step1]}} = + Exploration.interesting_funnel(site, query, max_steps: 6) assert step1.step.pathname == "/only-page" assert step1.visitors == 1 @@ -304,7 +350,7 @@ defmodule Plausible.Stats.ExplorationTest do filters: [[:is, "visit:browser", ["Firefox"]]] ) - assert {:ok, result} = Exploration.interesting_funnel(query) + assert {:ok, result} = Exploration.interesting_funnel(site, query) pathnames = Enum.map(result.funnel, & &1.step.pathname) assert pathnames == ["/docs", "/logout"] @@ -361,7 +407,7 @@ defmodule Plausible.Stats.ExplorationTest do query = QueryBuilder.build!(site, input_date_range: :all) - assert {:ok, result} = Exploration.interesting_funnel(query) + assert {:ok, result} = Exploration.interesting_funnel(site, query) pathnames = Enum.map(result.funnel, & &1.step.pathname) assert pathnames == ["/a", "/b", "/c"] @@ -378,7 +424,7 @@ defmodule Plausible.Stats.ExplorationTest do ] assert {:ok, [next_step1, next_step2, next_step3]} = - Exploration.next_steps(query, journey) + Exploration.next_steps(site, query, journey) assert next_step1.step.label == "/docs" assert next_step1.step.pathname == "/docs" @@ -400,7 +446,7 @@ defmodule Plausible.Stats.ExplorationTest do ] assert {:ok, [%{step: %{pathname: "/docs"}}]} = - Exploration.next_steps(query, journey, max_candidates: 1) + Exploration.next_steps(site, query, journey, max_candidates: 1) end test "returns error on too long journey", %{site: site} do @@ -411,14 +457,14 @@ defmodule Plausible.Stats.ExplorationTest do %Exploration.Journey.Step{name: "pageview", pathname: "/page#{idx}"} end) - assert {:error, :journey_too_long} = Exploration.next_steps(query, journey) + assert {:error, :journey_too_long} = Exploration.next_steps(site, query, journey) end test "suggests the first step in the journey", %{site: site} do query = QueryBuilder.build!(site, input_date_range: :all) assert {:ok, [next_step1, next_step2, next_step3, next_step4]} = - Exploration.next_steps(query, []) + Exploration.next_steps(site, query, []) assert next_step1.step.pathname == "/home" assert next_step1.visitors == 2 @@ -438,7 +484,7 @@ defmodule Plausible.Stats.ExplorationTest do ) assert {:ok, [next_step1, next_step2, next_step3, next_step4]} = - Exploration.next_steps(query, []) + Exploration.next_steps(site, query, []) assert next_step1.step.pathname == "/docs" assert next_step1.visitors == 1 @@ -458,7 +504,8 @@ defmodule Plausible.Stats.ExplorationTest do %Exploration.Journey.Step{name: "pageview", pathname: "/login"} ] - assert {:ok, [next_step]} = Exploration.next_steps(query, journey, search_term: "doc") + assert {:ok, [next_step]} = + Exploration.next_steps(site, query, journey, search_term: "doc") assert next_step.step.pathname == "/docs" assert next_step.visitors == 1 @@ -490,7 +537,7 @@ defmodule Plausible.Stats.ExplorationTest do query = QueryBuilder.build!(site, input_date_range: :all) - assert {:ok, [next_step1, next_step2]} = Exploration.next_steps(query, []) + assert {:ok, [next_step1, next_step2]} = Exploration.next_steps(site, query, []) assert next_step1.step.label == "/" assert next_step1.visitors == 2 @@ -530,7 +577,7 @@ defmodule Plausible.Stats.ExplorationTest do ] assert {:ok, [next_step]} = - Exploration.next_steps(query, journey, search_term: "up") + Exploration.next_steps(site, query, journey, search_term: "up") assert next_step.step.label == "Signup" assert next_step.step.name == "Signup" @@ -546,7 +593,7 @@ defmodule Plausible.Stats.ExplorationTest do ] assert {:ok, [next_step1, next_step2]} = - Exploration.next_steps(query, journey, direction: :backward) + Exploration.next_steps(site, query, journey, direction: :backward) assert next_step1.visitors == 1 assert next_step2.step.pathname == "/login" @@ -605,13 +652,19 @@ defmodule Plausible.Stats.ExplorationTest do [ %{step: %{pathname: "/:dashboard"}} ]} = - Exploration.next_steps(query, journey, search_term: "", direction: :forward) + Exploration.next_steps(site, query, journey, + search_term: "", + direction: :forward + ) assert {:ok, [ %{step: %{pathname: "/:dashboard"}} ]} = - Exploration.next_steps(query, journey, search_term: "", direction: :backward) + Exploration.next_steps(site, query, journey, + search_term: "", + direction: :backward + ) end test "treats identical sequence of events as a single step" do @@ -690,7 +743,7 @@ defmodule Plausible.Stats.ExplorationTest do ] assert {:ok, [next_step1, next_step2, next_step3]} = - Exploration.next_steps(query, journey) + Exploration.next_steps(site, query, journey) assert next_step1.step.pathname == "/docs" assert next_step1.visitors == 1 @@ -705,7 +758,7 @@ defmodule Plausible.Stats.ExplorationTest do %Exploration.Journey.Step{name: "pageview", pathname: "/docs"} ] - assert {:ok, [next_step]} = Exploration.next_steps(query, journey) + assert {:ok, [next_step]} = Exploration.next_steps(site, query, journey) assert next_step.step.pathname == "/logout" assert next_step.visitors == 1 @@ -760,7 +813,8 @@ defmodule Plausible.Stats.ExplorationTest do %Exploration.Journey.Step{name: "pageview", pathname: "/logout"} ] - assert {:ok, [next_step]} = Exploration.next_steps(query, journey, direction: :backward) + assert {:ok, [next_step]} = + Exploration.next_steps(site, query, journey, direction: :backward) assert next_step.step.pathname == "/login" assert next_step.visitors == 1 @@ -811,7 +865,8 @@ defmodule Plausible.Stats.ExplorationTest do query = QueryBuilder.build!(site, input_date_range: :all) - assert {:ok, [next_step1, next_step2, next_step3]} = Exploration.next_steps(query, []) + assert {:ok, [next_step1, next_step2, next_step3]} = + Exploration.next_steps(site, query, []) assert next_step1.step.pathname == "/home" assert next_step1.visitors == 2 @@ -824,7 +879,7 @@ defmodule Plausible.Stats.ExplorationTest do %Exploration.Journey.Step{name: "pageview", pathname: "/home"} ] - assert {:ok, [next_step]} = Exploration.next_steps(query, journey) + assert {:ok, [next_step]} = Exploration.next_steps(site, query, journey) assert next_step.step.pathname == "/login" assert next_step.visitors == 2 @@ -834,11 +889,173 @@ defmodule Plausible.Stats.ExplorationTest do %Exploration.Journey.Step{name: "pageview", pathname: "/login"} ] - assert {:ok, [next_step]} = Exploration.next_steps(query, journey) + assert {:ok, [next_step]} = Exploration.next_steps(site, query, journey) assert next_step.step.pathname == "/logout" assert next_step.visitors == 2 end + + test "considers existing goals in the listing" do + now = DateTime.utc_now() + site = new_site() + + Plausible.Goals.create(site, %{"page_path" => "/home"}) + Plausible.Goals.create(site, %{"event_name" => "Signup"}) + + Plausible.Goals.create(site, %{ + "page_path" => "/sites/new", + "display_name" => "Create a site" + }) + + Plausible.Goals.create(site, %{"page_path" => "/site*"}) + + populate_stats(site, [ + build(:pageview, + user_id: 123, + pathname: "/home", + timestamp: DateTime.shift(now, minute: -300) + ), + build(:pageview, + user_id: 123, + pathname: "/register", + timestamp: DateTime.shift(now, minute: -290) + ), + build(:event, + user_id: 123, + name: "Signup", + pathname: "/register", + timestamp: DateTime.shift(now, minute: -280) + ), + build(:pageview, + user_id: 123, + pathname: "/activate", + timestamp: DateTime.shift(now, minute: -270) + ), + build(:pageview, + user_id: 123, + pathname: "/sites/new", + timestamp: DateTime.shift(now, minute: -260) + ), + build(:pageview, + user_id: 123, + pathname: "/sites", + timestamp: DateTime.shift(now, minute: -250) + ), + build(:pageview, + user_id: 124, + pathname: "/home", + timestamp: DateTime.shift(now, minute: -300) + ), + build(:pageview, + user_id: 124, + pathname: "/register", + timestamp: DateTime.shift(now, minute: -290) + ), + build(:event, + user_id: 124, + name: "Signup", + pathname: "/register", + timestamp: DateTime.shift(now, minute: -280) + ), + build(:pageview, + user_id: 124, + pathname: "/activate", + timestamp: DateTime.shift(now, minute: -270) + ), + build(:pageview, + user_id: 124, + pathname: "/sites", + timestamp: DateTime.shift(now, minute: -250) + ), + build(:pageview, + user_id: 125, + pathname: "/home", + timestamp: DateTime.shift(now, minute: -300) + ), + build(:pageview, + user_id: 125, + pathname: "/register", + timestamp: DateTime.shift(now, minute: -290) + ), + build(:event, + user_id: 125, + name: "Signup", + pathname: "/register", + timestamp: DateTime.shift(now, minute: -280) + ), + build(:pageview, + user_id: 126, + pathname: "/home", + timestamp: DateTime.shift(now, minute: -300) + ), + build(:pageview, + user_id: 126, + pathname: "/register", + timestamp: DateTime.shift(now, minute: -290) + ), + build(:pageview, + user_id: 127, + pathname: "/home", + timestamp: DateTime.shift(now, minute: -300) + ) + ]) + + query = QueryBuilder.build!(site, input_date_range: :all) + + assert {:ok, + [ + next_step1, + next_step2, + next_step3, + next_step4, + next_step5, + next_step6, + next_step7 + ]} = + Exploration.next_steps(site, query, []) + + assert next_step1.step.label == "Visit /home" + assert next_step1.step.name == "pageview" + assert next_step1.step.pathname == "/home" + assert next_step1.step.is_goal + assert next_step1.visitors == 5 + + assert next_step2.step.label == "/register" + assert next_step2.step.name == "pageview" + assert next_step2.step.pathname == "/register" + refute next_step2.step.is_goal + assert next_step2.visitors == 4 + + assert next_step3.step.label == "Signup" + assert next_step3.step.name == "Signup" + assert next_step3.step.pathname == "" + assert next_step3.step.is_goal + assert next_step3.visitors == 3 + + assert next_step4.step.label == "/activate" + assert next_step4.step.name == "pageview" + assert next_step4.step.pathname == "/activate" + refute next_step4.step.is_goal + assert next_step4.visitors == 2 + + assert next_step5.step.label == "Visit /site*" + assert next_step5.step.name == "pageview" + assert next_step5.step.pathname == "/site*" + assert next_step5.step.is_goal + assert next_step5.visitors == 2 + + assert next_step6.step.label == "/sites" + assert next_step6.step.name == "pageview" + assert next_step6.step.pathname == "/sites" + refute next_step6.step.is_goal + assert next_step6.visitors == 2 + + assert next_step7.step.label == "Create a site" + assert next_step7.step.name == "pageview" + assert next_step7.step.pathname == "/sites/new" + assert next_step7.step.is_goal + assert next_step7.visitors == 1 + end end describe "implicit wildcard pathnames" do @@ -916,7 +1133,7 @@ defmodule Plausible.Stats.ExplorationTest do %{site: site} do query = QueryBuilder.build!(site, input_date_range: :all) - result = Exploration.next_steps(query, []) + result = Exploration.next_steps(site, query, []) assert {:ok, [ @@ -956,7 +1173,7 @@ defmodule Plausible.Stats.ExplorationTest do } do query = QueryBuilder.build!(site, input_date_range: :all) - result = Exploration.next_steps(query, [], include_wildcard?: false) + result = Exploration.next_steps(site, query, [], include_wildcard?: false) assert {:ok, [