From db1642ef655b0fb893082631bf807d5e56081655 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Thu, 30 Oct 2025 15:31:45 +0200 Subject: [PATCH 1/2] Make the category/group name unique index case insensitive --- lib/sanbase/metric/category/metric_group.ex | 2 +- .../metric/category/scripts/copy_categories.ex | 1 + ...017_improve_categorization_unique_index.exs | 18 ++++++++++++++++++ priv/repo/structure.sql | 5 +++-- 4 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 priv/repo/migrations/20251030132017_improve_categorization_unique_index.exs diff --git a/lib/sanbase/metric/category/metric_group.ex b/lib/sanbase/metric/category/metric_group.ex index 560a8b290a..4b4564bb2c 100644 --- a/lib/sanbase/metric/category/metric_group.ex +++ b/lib/sanbase/metric/category/metric_group.ex @@ -62,7 +62,7 @@ defmodule Sanbase.Metric.Category.MetricGroup do def create(attrs) do %__MODULE__{} |> changeset(attrs) - |> Repo.insert() + |> Repo.insert(on_conflict: :nothing) end @doc """ diff --git a/lib/sanbase/metric/category/scripts/copy_categories.ex b/lib/sanbase/metric/category/scripts/copy_categories.ex index 49b4a50052..d9fcbf6c15 100644 --- a/lib/sanbase/metric/category/scripts/copy_categories.ex +++ b/lib/sanbase/metric/category/scripts/copy_categories.ex @@ -34,6 +34,7 @@ defmodule Sanbase.Metric.Category.Scripts.CopyCategories do case map do %{metric_registry_id: metric_registry_id} when is_integer(metric_registry_id) -> Sanbase.Metric.Category.create_mapping(%{ + # metric: map.metric, metric_registry_id: metric_registry_id, category_id: category_id, group_id: group_id, diff --git a/priv/repo/migrations/20251030132017_improve_categorization_unique_index.exs b/priv/repo/migrations/20251030132017_improve_categorization_unique_index.exs new file mode 100644 index 0000000000..3653743edb --- /dev/null +++ b/priv/repo/migrations/20251030132017_improve_categorization_unique_index.exs @@ -0,0 +1,18 @@ +defmodule Sanbase.Repo.Migrations.ImproveCategorizationUniqueIndex do + use Ecto.Migration + + def change do + drop(unique_index(:metric_categories, [:name])) + drop(unique_index(:metric_groups, [:name, :category_id])) + + create( + unique_index(:metric_categories, ["(lower(name))"], name: :metric_categories_name_index) + ) + + create( + unique_index(:metric_groups, ["(lower(name))", :category_id], + name: :metric_groups_name_category_id_index + ) + ) + end +end diff --git a/priv/repo/structure.sql b/priv/repo/structure.sql index 2b6c5b4235..cccd397b58 100644 --- a/priv/repo/structure.sql +++ b/priv/repo/structure.sql @@ -8343,7 +8343,7 @@ CREATE INDEX menus_user_id_index ON public.menus USING btree (user_id); -- Name: metric_categories_name_index; Type: INDEX; Schema: public; Owner: - -- -CREATE UNIQUE INDEX metric_categories_name_index ON public.metric_categories USING btree (name); +CREATE UNIQUE INDEX metric_categories_name_index ON public.metric_categories USING btree (lower((name)::text)); -- @@ -8371,7 +8371,7 @@ CREATE UNIQUE INDEX metric_category_mappings_module_metric_category_id_group_id_ -- Name: metric_groups_name_category_id_index; Type: INDEX; Schema: public; Owner: - -- -CREATE UNIQUE INDEX metric_groups_name_category_id_index ON public.metric_groups USING btree (name, category_id); +CREATE UNIQUE INDEX metric_groups_name_category_id_index ON public.metric_groups USING btree (lower((name)::text), category_id); -- @@ -11147,3 +11147,4 @@ INSERT INTO public."schema_migrations" (version) VALUES (20251023083446); INSERT INTO public."schema_migrations" (version) VALUES (20251023114153); INSERT INTO public."schema_migrations" (version) VALUES (20251027142731); INSERT INTO public."schema_migrations" (version) VALUES (20251027154645); +INSERT INTO public."schema_migrations" (version) VALUES (20251030132017); From 881de03674f03034ec6a724986ea96b58f90591c Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Mon, 17 Nov 2025 17:10:23 +0200 Subject: [PATCH 2/2] backup --- lib/sanbase/metric/category/display_order.ex | 88 +++++++--- .../category/metric_category_mapping.ex | 2 +- .../metric/category/metric_ui_metadata.ex | 2 +- .../auto_categorize_social_metrics.livemd | 159 ++++++++++++++++++ 4 files changed, 226 insertions(+), 25 deletions(-) create mode 100644 livebooks/auto_categorize_social_metrics.livemd diff --git a/lib/sanbase/metric/category/display_order.ex b/lib/sanbase/metric/category/display_order.ex index 85f1b9fa39..4356e77642 100644 --- a/lib/sanbase/metric/category/display_order.ex +++ b/lib/sanbase/metric/category/display_order.ex @@ -16,56 +16,92 @@ defmodule Sanbase.Metric.Category.DisplayOrder do Returns the same shape as DisplayOrder.get_ordered_metrics for compatibility. The ordering follows: category order → ungrouped mappings → groups (by order) → mappings within groups → ui_metadata within mappings. + Pass `true` as the second argument to include metrics without UI metadata. """ - @spec get_ordered_metrics([MetricCategory.t()]) :: %{metrics: [map()], categories: [map()]} - def get_ordered_metrics(categories) do + @spec get_ordered_metrics([MetricCategory.t()], boolean()) :: %{ + metrics: [map()], + categories: [map()] + } + def get_ordered_metrics(categories \\ nil, include_without_ui_metadata \\ false) do + # The returned metrics are taken from the mappings preloads in the categories. + # Make sure to preload the necessary associations when calling this function. + categories = categories || MetricCategory.list_ordered() + categories = Enum.sort_by(categories, & &1.display_order, :asc) + + raise_if_no_preloads(categories) + ordered_category_data = Enum.map(categories, fn cat -> %{id: cat.id, name: cat.name} end) + # This map is used to enrich the output with registry metric data + # It it is not used to generate the list of metrics. + # The list of metrics registry_metrics = build_registry_map() metrics = categories - |> Enum.flat_map(&flatten_category/1) + |> Enum.flat_map(&flatten_category(&1, include_without_ui_metadata)) |> Enum.map(&transform_to_output_format(&1, registry_metrics)) - %{ - metrics: metrics, - categories: ordered_category_data - } + %{metrics: metrics, categories: ordered_category_data} end - defp flatten_category(category) do - ungrouped_metrics = flatten_mappings(category.mappings, category, nil) + defp raise_if_no_preloads(categories) do + if Enum.any?(categories, fn cat -> not Ecto.assoc_loaded?(cat.mappings) end), + do: raise("Mappings must be preloaded for all categories") + + if Enum.any?(categories, fn cat -> + Enum.any?(cat.mappings, fn mapping -> + not Ecto.assoc_loaded?(mapping.metric_registry) or + not Ecto.assoc_loaded?(mapping.ui_metadata_list) + end) + end), + do: raise("Groups must be preloaded for all categories") + end + + defp flatten_category(category, include_without_ui_metadata) do + ungrouped_metrics = + flatten_mappings(category.mappings, category, _group = nil, include_without_ui_metadata) grouped_metrics = category.groups |> Enum.flat_map(fn group -> - flatten_mappings(group.mappings, category, group) + flatten_mappings(group.mappings, category, group, include_without_ui_metadata) end) ungrouped_metrics ++ grouped_metrics end - defp flatten_mappings(mappings, category, group) do + defp flatten_mappings(mappings, category, group, include_without_ui_metadata) do mappings |> Enum.filter(&mapping_belongs_to_group?(&1, group)) - |> Enum.flat_map(&expand_mapping_with_ui_metadata(&1, category, group)) + |> Enum.flat_map(&expand_mapping(&1, category, group, include_without_ui_metadata)) end - defp mapping_belongs_to_group?(mapping, nil), do: is_nil(mapping.group_id) + defp mapping_belongs_to_group?(mapping, _group = nil), do: is_nil(mapping.group_id) defp mapping_belongs_to_group?(mapping, group), do: mapping.group_id == group.id - defp expand_mapping_with_ui_metadata(mapping, category, group) do - if mapping.ui_metadata_list == [] do - [{mapping, nil, category, group}] - else - Enum.map(mapping.ui_metadata_list, fn ui_metadata -> - {mapping, ui_metadata, category, group} - end) + defp expand_mapping(mapping, category, group, include_without_ui_metadata) do + cond do + mapping.ui_metadata_list == [] and false == include_without_ui_metadata -> + [] + + mapping.ui_metadata_list == [] and true == include_without_ui_metadata -> + [{mapping, nil, category, group}] + + true -> + Enum.map(mapping.ui_metadata_list, fn ui_metadata -> + {mapping, ui_metadata, category, group} + end) end end defp transform_to_output_format({mapping, ui_metadata, category, group}, registry_metrics) do + # Determine the metric name (not the human readable name) of the metric. + # In case of mapping with ui_metadata, the metric name comes from it. + # The mapping can be linked to a registry metric, or to a code metric. + # In case of include_without_ui_metadata=true, there is no ui_metadata, + # so the metric name comes from the mapping. + # Such records without ui_metadata will be used only locally for tests metric_name = determine_metric_name(mapping, ui_metadata) registry_metric = Map.get(registry_metrics, metric_name) @@ -110,10 +146,16 @@ defmodule Sanbase.Metric.Category.DisplayOrder do } end - defp get_ui_human_readable_name(nil, metric_name), do: metric_name - defp get_ui_human_readable_name(ui_metadata, metric_name) do - ui_metadata.ui_human_readable_name || metric_name + if ui_metadata && ui_metadata.ui_human_readable_name do + ui_metadata.ui_human_readable_name + else + with {:ok, human_readable_name} <- Sanbase.Metric.human_readable_name(metric_name) do + human_readable_name + else + _ -> metric_name + end + end end defp ui_metadata_or_mapping_field(nil, mapping, field), do: Map.get(mapping, field) diff --git a/lib/sanbase/metric/category/metric_category_mapping.ex b/lib/sanbase/metric/category/metric_category_mapping.ex index 17d0c940b2..22686246e5 100644 --- a/lib/sanbase/metric/category/metric_category_mapping.ex +++ b/lib/sanbase/metric/category/metric_category_mapping.ex @@ -87,7 +87,7 @@ defmodule Sanbase.Metric.Category.MetricCategoryMapping do def create(attrs) do %__MODULE__{} |> changeset(attrs) - |> Repo.insert() + |> Repo.insert(on_conflict: :nothing, returning: true) end @doc """ diff --git a/lib/sanbase/metric/category/metric_ui_metadata.ex b/lib/sanbase/metric/category/metric_ui_metadata.ex index b8bf8a19e8..9380e0692a 100644 --- a/lib/sanbase/metric/category/metric_ui_metadata.ex +++ b/lib/sanbase/metric/category/metric_ui_metadata.ex @@ -78,7 +78,7 @@ defmodule Sanbase.Metric.UIMetadata do def create(attrs) do %__MODULE__{} |> changeset(attrs) - |> Repo.insert() + |> Repo.insert(on_conflict: :nothing) end @doc """ diff --git a/livebooks/auto_categorize_social_metrics.livemd b/livebooks/auto_categorize_social_metrics.livemd new file mode 100644 index 0000000000..5dc3e8d332 --- /dev/null +++ b/livebooks/auto_categorize_social_metrics.livemd @@ -0,0 +1,159 @@ +# Auto-categorize Social Registry metrics + +## Section + +```elixir +drop_existing_categories_and_groups = true +add_social_metrics_new_categories = true +add_social_metrics_ui_metadata = false +``` + +```elixir +all_metrics = Sanbase.Metric.available_metrics() +length(all_metrics) +``` + +```elixir +categorized_metrics = + Sanbase.Metric.Category.MetricCategoryMapping.list_all() + |> Enum.flat_map(fn + %{metric_registry: %{} = registry} -> + [registry] |> Sanbase.Metric.Registry.resolve() |> Enum.map(& &1.metric) + + %{metric: metric} when is_binary(metric) -> + [metric] + end) + +not_categorized_metrics = all_metrics -- categorized_metrics + +IO.puts(""" +Categorized: #{length(categorized_metrics)} +Not Categorized: #{length(not_categorized_metrics)} +""") +``` + +```elixir +category_name_to_id = Sanbase.Metric.Category.list_categories() |> Map.new(&{&1.name, &1.id}) +``` + +```elixir +# Social groups +social_groups = Sanbase.Metric.Category.list_groups_by_category(category_name_to_id["Social"]) + +max_display_order = + case social_groups do + [] -> + 1 + + [_ | _] = list -> + Enum.max_by(list, & &1.display_order).display_order + end + +groups = ["Sentiment", "Social Volume", "Social Dominance"] + +Enum.with_index(groups, max_display_order) +|> Enum.map(fn {group_name, display_order} -> + category_id = category_name_to_id["Social"] + group = Sanbase.Metric.Category.MetricGroup.get_by_name_and_category(group_name, category_id) + group && Sanbase.Metric.Category.delete_group(group) + + Sanbase.Metric.Category.create_group(%{ + name: group_name, + category_id: category_id, + display_order: display_order + }) +end) + +group_name_to_id = + Sanbase.Metric.Category.list_groups_with_category() |> Map.new(&{&1.name, &1.id}) +``` + +```elixir +group_name_to_id = + Sanbase.Metric.Category.list_groups_with_category() |> Map.new(&{&1.name, &1.id}) +``` + +```elixir +deduce_category = fn m -> + cond do + String.contains?(m, [ + "_change_", + "_1h", + "_v2", + "_1d", + "_5m", + "moving_average", + "volume_consumed" + ]) -> + nil + + # Sentiment + m =~ "sentiment" and + String.contains?(m, ["positive", "negative", "neutral", "bullish", "bearish"]) -> + {"Social", "Sentiment"} + + # Social Volume + m =~ "social_volume" -> + {"Social", "Social Volume"} + + # Social Dominance + m =~ "social_dominance" -> + {"Social", "Social Dominance"} + + true -> + nil + end +end +``` + +```elixir +metric_registry_name_to_id = + Sanbase.Metric.Registry.all() + |> Sanbase.Metric.Registry.resolve() + |> Map.new(&{&1.metric, &1.id}) + +all_metrics +|> Enum.filter(&String.contains?(&1, "social_dominance")) +|> Enum.filter(&Map.has_key?(metric_registry_name_to_id, &1)) +|> IO.inspect() +|> Enum.map(fn m -> + registry_id = metric_registry_name_to_id[m] + + case deduce_category.(m) do + nil -> + :ok + + {category, group} -> + # Drop existing mapping + case Sanbase.Metric.Category.get_mapping_by_metric_registry_id(registry_id) do + {:ok, [existing_mapping]} -> + Sanbase.Metric.Category.delete_mapping(existing_mapping) + + _ -> + :ok + end + + # Create new mapping + {:ok, mapping} = + Sanbase.Metric.Category.create_mapping(%{ + metric_registry_id: registry_id, + category_id: category_name_to_id[category], + group_id: group_name_to_id[group], + display_order: 1 + }) + + # Create UI Metadata + {:ok, _} = + Sanbase.Metric.Category.create_ui_metadata(%{ + metric: m, + display_order_in_mapping: 1, + metric_category_mapping_id: mapping.id, + chart_style: "line" + }) + end +end) +``` + +```elixir +group_name_to_id["Social Dominance"] +```