Slim large notebooks with deferred notes

emanote/CHANGELOG.md

@@ -29,6 +29,10 @@
 - TOC sidebar: tightened entry padding and styled the overflow scrollbar (Firefox `scrollbar-width: thin` + WebKit pseudo-element) so long tables of contents no longer surface the chunky OS-default bar (closes [#668](https://github.com/srid/emanote/issues/668)).
 - Live server: assets bundled under `_emanote-static/` (skylighting CSS, self-hosted fonts, inverted-tree CSS, emanote-logo, Stork CSS+JS) now cache-bust with `?t=<mtime>` instead of being served bare. Edits to any of these files in `emanote run` invalidate the browser cache without a manual restart. Templates use a new `<emanoteStaticUrl path="…">${url}</emanoteStaticUrl>` splice; the older `${ema:emanoteStaticLayerUrl}` continues to work for third-party templates but skips the cache buster (closes [#666](https://github.com/srid/emanote/issues/666)).
 
+**Performance improvements**
+
+- Large Markdown notebooks use substantially less live-server memory by storing simple notes in a deferred form and compacting repeated startup wikilink relations while preserving backlink contexts on demand (closes [#66](https://github.com/srid/emanote/issues/66)).
+
 ## 1.4.0.0 (2025-08-18)
 
 **Notable features**

emanote/emanote.cabal

@@ -108,6 +108,7 @@ common library-common
     , commonmark-wikilink    >=0.2
     , containers
     , data-default
+    , deepseq
     , deriving-aeson
     , directory
     , ema                    >=0.10.1
@@ -254,6 +255,7 @@ test-suite test
     build-depends: emanote
     other-modules:
       Emanote.Model.Link.RelSpec
+      Emanote.Model.NoteSpec
       Emanote.Model.QuerySpec
       Emanote.Model.TocSpec
       Emanote.Pandoc.ExternalLinkSpec

emanote/src/Emanote/Model/Graph.hs

@@ -12,7 +12,7 @@ import Emanote.Model.Link.Resolve qualified as Resolve
 import Emanote.Model.Meta (lookupRouteMeta)
 import Emanote.Model.Note qualified as MN
 import Emanote.Model.Note qualified as N
-import Emanote.Model.Type (Model, modelIndexRoute, modelNotes, modelRels, parentLmlRoute)
+import Emanote.Model.Type (Model, modelIndexRoute, modelLookupNoteByRoute', modelNotes, modelRels, parentLmlRoute)
 import Emanote.Route qualified as R
 import Emanote.Route.SiteRoute qualified as SR
 import Optics.Operators as Lens ((^.))
@@ -143,6 +143,7 @@ folgezettelTreesFrom model fromRoute =
     loop fromRoute allRoutes
   where
     allRoutes = Set.fromList $ fmap N._noteRoute $ Ix.toList $ model ^. modelNotes
+    childRoutesByParent = folgezettelChildMap model
     -- Run in a state monad of unvisited routes, taking visited routes as argument.
     go :: (MonadState (Set R.LMLRoute) m) => Set R.LMLRoute -> R.LMLRoute -> m (Tree R.LMLRoute)
     go visitedRoutes route
@@ -151,10 +152,35 @@ folgezettelTreesFrom model fromRoute =
           pure $ Node route []
       | otherwise = do
           modify $ Set.delete route
-          let children = folgezettelChildrenFor model route
+          let children = ordNub $ Map.findWithDefault mempty route childRoutesByParent
           cs <- go (Set.insert route visitedRoutes) `traverse` children
           pure $ Node route cs
 
+folgezettelChildMap :: Model -> Map R.LMLRoute [R.LMLRoute]
+folgezettelChildMap model =
+  Map.fromListWith (<>)
+    $ relationEdges
+    <> folderEdges
+  where
+    relationEdges = do
+      rel <- Ix.toList $ model ^. modelRels
+      let source = rel ^. Rel.relFrom
+      case rel ^. Rel.relTo of
+        Rel.URTWikiLink (WL.WikiLinkBranch, wl) -> do
+          child <- maybeToList $ lookupNoteByWikiLink model source wl
+          pure (source, one child)
+        Rel.URTWikiLink (WL.WikiLinkTag, wl) -> do
+          parent <- maybeToList $ lookupNoteByWikiLink model source wl
+          pure (parent, one source)
+        _ ->
+          mempty
+    folderEdges = do
+      note <- Ix.toList $ model ^. modelNotes
+      parentFolder <- maybeToList $ N.noteParent note
+      let parentRoute = R.defaultLmlRoute parentFolder
+      guard $ folderFolgezettelEnabledFor model parentRoute
+      pure (parentRoute, one $ note ^. MN.noteRoute)
+
 folderFolgezettelEnabledFor :: Model -> R.LMLRoute -> Bool
 folderFolgezettelEnabledFor model r =
   lookupRouteMeta defaultValue ("emanote" :| ["folder-folgezettel"]) r model
@@ -178,9 +204,13 @@ modelLookupBacklinks r model =
   sortOn (Calendar.backlinkSortKey model . fst)
     $ groupNE
     $ backlinkRels r model
-    <&> \rel ->
-      (rel ^. Rel.relFrom, rel ^. Rel.relCtx)
+    >>= backlinkContexts
   where
+    backlinkContexts rel =
+      expandedBacklinkContexts r model rel
+        <&> \rel' ->
+          (rel' ^. Rel.relFrom, rel' ^. Rel.relCtx)
+
     groupNE :: forall a b. (Ord a) => [(a, b)] -> [(a, NonEmpty b)]
     groupNE =
       Map.toList . foldl' f Map.empty
@@ -191,20 +221,33 @@ modelLookupBacklinks r model =
             Nothing -> Map.insert x (one y) m
             Just ys -> Map.insert x (ys <> one y) m
 
+expandedBacklinkContexts :: R.LMLRoute -> Model -> Rel.Rel -> [Rel.Rel]
+expandedBacklinkContexts targetR model rel =
+  case modelLookupNoteByRoute' (rel ^. Rel.relFrom) model of
+    Just sourceNote
+      | Just deferred <- MN.noteDeferred sourceNote ->
+          filter (isBacklinkTo targetR model)
+            . Ix.toList
+            $ Rel.noteTextRels sourceNote (deferred ^. MN.deferredNoteText)
+    _ ->
+      [rel]
+
 -- | Rels pointing *to* this route
 backlinkRels :: R.LMLRoute -> Model -> [Rel.Rel]
 backlinkRels r model =
   let allPossibleLinks = Rel.unresolvedRelsTo $ toModelRoute r
       rels = Ix.toList $ (model ^. modelRels) @+ allPossibleLinks
-   in filter isUnambiguousBacklink rels
+   in filter (isBacklinkTo r model) rels
   where
     toModelRoute = R.ModelRoute_LML R.LMLView_Html
-    -- Check that 'rel' points to 'r' even after resolving any ambiguous wiki links
-    isUnambiguousBacklink rel = isJust $ do
-      SR.SiteRoute_ResourceRoute (SR.ResourceRoute_LML _ r') <-
-        Rel.getResolved $ Resolve.resolveRel model rel
-      guard $ r == r'
-      pass
+
+-- Check that 'rel' points to 'r' even after resolving any ambiguous wiki links.
+isBacklinkTo :: R.LMLRoute -> Model -> Rel.Rel -> Bool
+isBacklinkTo r model rel = isJust $ do
+  SR.SiteRoute_ResourceRoute (SR.ResourceRoute_LML _ r') <-
+    Rel.getResolved $ Resolve.resolveRel model rel
+  guard $ r == r'
+  pass
 
 -- | Rels pointing *from* this route
 frontlinkRels :: R.LMLRoute -> Model -> [Rel.Rel]

emanote/src/Emanote/Model/Link/Rel.hs

@@ -10,7 +10,7 @@ import Data.IxSet.Typed qualified as Ix
 import Data.List.NonEmpty qualified as NEL
 import Data.Map.Strict qualified as Map
 import Data.Text qualified as T
-import Emanote.Model.Note (Note, noteDoc, noteResolveLinkBase, noteRoute)
+import Emanote.Model.Note (Note, deferredNoteText, noteDeferred, noteDoc, noteResolveLinkBase, noteRoute)
 import Emanote.Route (LMLRoute, ModelRoute)
 import Emanote.Route qualified as R
 import Emanote.Route.SiteRoute.Type qualified as SR
@@ -67,17 +67,74 @@ makeLenses ''Rel
 
 noteRels :: Note -> IxRel
 noteRels note =
-  extractLinks . LC.queryLinksWithContext $ note ^. noteDoc
+  case noteDeferred note of
+    Just deferred ->
+      deduplicateRelTargets $ noteTextRels note (deferred ^. deferredNoteText)
+    Nothing ->
+      extractLinks . LC.queryLinksWithContext $ note ^. noteDoc
   where
+    parentR = noteResolveLinkBase note
+    noteR = note ^. noteRoute
+
     extractLinks :: Map Text (NonEmpty ([(Text, Text)], [B.Block])) -> IxRel
     extractLinks m =
       Ix.fromList
         $ flip concatMap (Map.toList m)
         $ \(url, instances) -> do
           flip mapMaybe (toList instances) $ \(attrs, ctx) -> do
-            let parentR = noteResolveLinkBase note
             (target, _manchor) <- parseUnresolvedRelTarget parentR attrs url
-            pure $ Rel (note ^. noteRoute) target ctx
+            pure $ Rel noteR target ctx
+
+-- | Keep one representative relation per unresolved target for the startup model.
+deduplicateRelTargets :: IxRel -> IxRel
+deduplicateRelTargets =
+  Ix.fromList . Map.elems . foldl' insertRel Map.empty . Ix.toList
+  where
+    insertRel rels rel =
+      Map.insertWith keepOld (rel ^. relTo) rel rels
+    keepOld _new old =
+      old
+
+-- | Extract every wikilink relation from simple Markdown source text.
+noteTextRels :: Note -> Text -> IxRel
+noteTextRels note =
+  Ix.fromList . concatMap lineRels . lines
+  where
+    parentR = noteResolveLinkBase note
+    noteR = note ^. noteRoute
+
+    lineRels line =
+      case wikiLinksInLine line of
+        [] ->
+          []
+        links ->
+          let ctx = [B.Para [B.Str $ T.strip line]]
+           in flip mapMaybe links $ \(attrs, url) -> do
+                (target, _manchor) <- parseUnresolvedRelTarget parentR attrs url
+                pure $ Rel noteR target ctx
+
+wikiLinksInLine :: Text -> [([(Text, Text)], Text)]
+wikiLinksInLine =
+  go
+  where
+    go line =
+      case T.breakOn "[[" line of
+        (_, "") -> []
+        (before, rest) ->
+          case T.breakOn "]]" (T.drop 2 rest) of
+            (_, "") -> []
+            (target0, after) ->
+              let target = T.takeWhile (/= '|') target0
+                  attrs = [("data-wikilink-type", wikiLinkType before after)]
+               in (attrs, target) : go (T.drop 2 after)
+
+    wikiLinkType before after
+      | T.isPrefixOf "]]#" after = "WikiLinkBranch"
+      | otherwise =
+          case snd <$> T.unsnoc before of
+            Just '!' -> "WikiLinkEmbed"
+            Just '#' -> "WikiLinkTag"
+            _ -> "WikiLinkNormal"
 
 {- | All `UnresolvedRelTarget`s that could resolve to the given
 `ModelRoute`. The `URTResource` form is built from the canonical URL