Remove retired earmark and unused phoenix_storybook dependencies

.circleci/config.yml

@@ -188,11 +188,8 @@ jobs:
           command: |
             # GHSA-g2wm-735q-3f56: cowlib cookie encoder CRLF injection (low);
             #   no patched release available yet.
-            # GHSA-55hg-8qxv-qj4p / GHSA-833p-95jq-929q / GHSA-mrhx-6pw9-q5fh:
-            #   phoenix_storybook advisories. Dev-only dep, not compiled into
-            #   prod and storybook routes are dev-only. Tracked for removal.
             sudo -u lightning mix deps.audit \
-              --ignore-advisory-ids GHSA-g2wm-735q-3f56,GHSA-55hg-8qxv-qj4p,GHSA-833p-95jq-929q,GHSA-mrhx-6pw9-q5fh \
+              --ignore-advisory-ids GHSA-g2wm-735q-3f56 \
               || echo "deps.audit" >> /tmp/lint_failed
       - run:
           name: "Check for retired Hex packages"

.formatter.exs

@@ -4,8 +4,7 @@
   inputs: [
     "*.{heex,ex,exs}",
     "priv/*/seeds.exs",
-    "{config,lib,test}/**/*.{heex,ex,exs}",
-    "storybook/**/*.exs"
+    "{config,lib,test}/**/*.{heex,ex,exs}"
   ],
   subdirectories: ["priv/*/migrations"],
   line_length: 81

CHANGELOG.md

@@ -25,6 +25,11 @@ and this project adheres to
 
 ### Changed
 
+- Migrated off the retired `earmark` markdown dependency in favour of `mdex`.
+  [#4878](https://github.com/OpenFn/lightning/issues/4878)
+- Removed the unused dev-only `phoenix_storybook` dependency, clearing its
+  advisories from the `mix deps.audit` ignore list.
+  [#4846](https://github.com/OpenFn/lightning/issues/4846)
 - Bump worker to 1.27.0
 
 ### Fixed

RUNNINGLOCAL.md

@@ -183,8 +183,8 @@ LOCAL_ADAPTORS=true mix phx.server
 ```
 
 Each path in `OPENFN_ADAPTORS_REPO` must contain a `packages` subdirectory.
-Paths that are missing or unreadable are logged and skipped, so the rest of
-the list still loads.
+Paths that are missing or unreadable are logged and skipped, so the rest of the
+list still loads.
 
 ### Problems with Apple Silicon
 

codecov.yml

@@ -17,7 +17,6 @@ ignore:
   - "lib/lightning/release.ex"
   - "lib/lightning/sentry_event_filter.ex"
   - "test/support/**"
-  - "storybook/**"
 
   # Phoenix declarative scaffolding — boot-only or pure DSL
   - "lib/lightning_web.ex"

config/config.exs

@@ -102,7 +102,6 @@ config :esbuild,
          --external:/fonts/*
          --external:/images/*
          js/app.js
-         js/storybook.js
          js/editor/Editor.tsx
          js/react/components/DataclipViewer.tsx
          js/react/components/CollectionPreviewViewer.tsx
@@ -145,13 +144,6 @@ config :tailwind,
       --output=priv/static/assets/app.css
     ),
     cd: Path.expand("..", __DIR__)
-  ],
-  storybook: [
-    args: ~w(
-      --input=assets/css/storybook.css
-      --output=priv/static/assets/storybook.css
-    ),
-    cd: Path.expand("..", __DIR__)
   ]
 
 # Configures Elixir's Logger

config/dev.exs

@@ -33,8 +33,7 @@ config :lightning, LightningWeb.Endpoint,
     # Start the esbuild watcher by calling Esbuild.install_and_run(:default, args)
     esbuild:
       {Esbuild, :install_and_run, [:default, ~w(--sourcemap=inline --watch)]},
-    tailwind: {Tailwind, :install_and_run, [:default, ~w(--watch)]},
-    storybook_tailwind: {Tailwind, :install_and_run, [:storybook, ~w(--watch)]}
+    tailwind: {Tailwind, :install_and_run, [:default, ~w(--watch)]}
   ]
 
 config :lightning,
@@ -112,8 +111,7 @@ config :lightning, LightningWeb.Endpoint,
       ~r"priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$",
       ~r"priv/gettext/.*(po)$",
       ~r"lib/lightning_web/(live|components|views)/.*(ex|heex)$",
-      ~r"lib/lightning_web/templates/.*(eex)$",
-      ~r"storybook/.*(exs)$"
+      ~r"lib/lightning_web/templates/.*(eex)$"
     ]
   ]
 

coveralls.json

@@ -4,7 +4,6 @@
     "lib/lightning/release.ex",
     "lib/lightning/sentry_event_filter.ex",
     "lib/mix/tasks/install_runtime.ex",
-    "test/support/*",
-    "storybook/*"
+    "test/support/*"
   ]
 }

lib/lightning_web/live/ai_assistant/component.ex

@@ -18,8 +18,6 @@ defmodule LightningWeb.AiAssistant.Component do
 
   require Logger
 
-  @dialyzer {:nowarn_function, process_ast: 2}
-
   @default_page_size 20
   @message_preview_length 50
   @typing_animation_delay_ms 100
@@ -1558,78 +1556,82 @@ defmodule LightningWeb.AiAssistant.Component do
     Calendar.strftime(datetime, "%I:%M %p")
   end
 
+  # Default per-element HTML attributes injected into rendered markdown so
+  # assistant messages pick up Tailwind styling. Keyed by HTML tag name.
+  @assistant_messages_attributes %{
+    "a" => %{
+      class: "text-primary-400 hover:text-primary-600",
+      target: "_blank"
+    },
+    "h1" => %{class: "text-2xl font-bold mb-6"},
+    "h2" => %{class: "text-xl font-semibold mb-4 mt-8"},
+    "ol" => %{class: "list-decimal pl-8 space-y-1"},
+    "ul" => %{class: "list-disc pl-8 space-y-1"},
+    "li" => %{class: "text-gray-800"},
+    "p" => %{class: "mt-1 mb-2 text-gray-800"},
+    "pre" => %{
+      class:
+        "rounded-md font-mono bg-slate-100 border-2 border-slate-200 text-slate-800 my-4 p-2 overflow-auto"
+    }
+  }
+
+  # Match Earmark's default of passing raw HTML through untouched. The content
+  # is AI-generated markdown rendered for the same user who prompted it.
+  #
+  # Earmark defaulted to `gfm: true`, so enable the GFM extensions (tables,
+  # strikethrough, bare-URL autolinks, task lists) to keep assistant replies —
+  # which frequently contain tables — rendering as they did before.
+  @mdex_options [
+    extension: [
+      table: true,
+      strikethrough: true,
+      autolink: true,
+      tasklist: true
+    ],
+    render: [unsafe: true]
+  ]
+
   attr :id, :string, required: true
   attr :content, :string, required: true
   attr :attributes, :map, default: %{}
 
   def formatted_content(assigns) do
-    assistant_messages_attributes = %{
-      "a" => %{
-        class: "text-primary-400 hover:text-primary-600",
-        target: "_blank"
-      },
-      "h1" => %{class: "text-2xl font-bold mb-6"},
-      "h2" => %{class: "text-xl font-semibold mb-4 mt-8"},
-      "ol" => %{class: "list-decimal pl-8 space-y-1"},
-      "ul" => %{class: "list-disc pl-8 space-y-1"},
-      "li" => %{class: "text-gray-800"},
-      "p" => %{class: "mt-1 mb-2 text-gray-800"},
-      "pre" => %{
-        class:
-          "rounded-md font-mono bg-slate-100 border-2 border-slate-200 text-slate-800 my-4 p-2 overflow-auto"
-      }
-    }
-
     merged_attributes =
-      Map.merge(assistant_messages_attributes, assigns.attributes)
+      Map.merge(@assistant_messages_attributes, assigns.attributes)
 
-    assigns =
-      case Earmark.Parser.as_ast(assigns.content) do
-        {:ok, ast, _} ->
-          process_ast(ast, merged_attributes) |> raw()
+    rendered =
+      case MDEx.to_html(assigns.content, @mdex_options) do
+        {:ok, html} ->
+          html |> inject_attributes(merged_attributes) |> raw()
 
-        _ ->
+        {:error, _reason} ->
           assigns.content
       end
-      |> then(&assign(assigns, :content, &1))
+
+    assigns = assign(assigns, :content, rendered)
 
     ~H"""
     <article id={@id}>{@content}</article>
     """
   end
 
-  defp process_ast(ast, attributes) do
-    ast
-    |> Earmark.Transform.map_ast(&process_node(&1, attributes))
-    |> Earmark.Transform.transform()
-  end
-
-  defp process_node({element_type, attrs, _content, _meta} = node, attributes) do
-    case Map.get(attributes, element_type) do
-      nil -> node
-      attribute_map -> apply_attributes(node, element_type, attrs, attribute_map)
-    end
-  end
-
-  defp process_node(other, _attributes), do: other
-
-  defp apply_attributes(node, "code", attrs, attribute_map) do
-    case find_class_attr(attrs) do
-      {_, [lang]} ->
-        trimmed_lang = String.trim(lang)
-        Earmark.AstTools.merge_atts_in_node(node, class: trimmed_lang)
-
-      _ ->
-        Earmark.AstTools.merge_atts_in_node(node, attribute_map)
-    end
+  # MDEx node structs cannot carry arbitrary HTML attributes, so inject the
+  # per-element classes into the rendered HTML instead.
+  defp inject_attributes(html, attributes) do
+    Enum.reduce(attributes, html, fn {tag, attrs}, acc ->
+      inject_tag_attributes(acc, tag, attrs)
+    end)
   end
 
-  defp apply_attributes(node, _element_type, _attrs, attribute_map) do
-    Earmark.AstTools.merge_atts_in_node(node, attribute_map)
-  end
+  defp inject_tag_attributes(html, tag, attrs) do
+    attrs_string =
+      Enum.map_join(attrs, " ", fn {key, value} -> ~s(#{key}="#{value}") end)
 
-  defp find_class_attr(attrs) do
-    Enum.find(attrs, fn {attr, _} -> attr == "class" end)
+    String.replace(
+      html,
+      ~r/<#{Regex.escape(tag)}(?=[\s>])/,
+      ~s(<#{tag} #{attrs_string})
+    )
   end
 
   attr :user, Lightning.Accounts.User, default: nil

lib/lightning_web/router.ex

@@ -306,18 +306,6 @@ defmodule LightningWeb.Router do
   end
 
   do_in(:dev) do
-    import PhoenixStorybook.Router
-
-    scope "/" do
-      storybook_assets()
-    end
-
-    scope "/" do
-      pipe_through :browser
-
-      live_storybook("/storybook", backend_module: LightningWeb.Storybook)
-    end
-
     scope "/dev" do
       pipe_through :browser
 

mix.exs

@@ -120,7 +120,6 @@ defmodule Lightning.MixProject do
       {:phoenix_live_dashboard, "~> 0.8"},
       {:phoenix_live_reload, "~> 1.5", only: :dev},
       {:phoenix_live_view, "~> 1.0.17"},
-      {:phoenix_storybook, "~> 0.9.2", only: :dev},
       {:cors_plug, "~> 3.0"},
       {:plug_cowboy, "~> 2.5"},
       {:postgrex, ">= 0.0.0"},
@@ -152,7 +151,7 @@ defmodule Lightning.MixProject do
       {:eqrcode, "~> 0.2"},
       # Github API Secret Encoding
       {:enacl, github: "aeternity/enacl", branch: "master"},
-      {:earmark, "~> 1.4"},
+      {:mdex, "~> 0.13"},
       {:eventually, "~> 1.1", only: [:test]},
       {:benchee, "~> 1.5.0", only: :dev},
       {:statistics, "~> 0.6", only: :dev},