diff --git a/.taplo.toml b/.taplo.toml index aea4ee9bd99d2..464186ec0267c 100644 --- a/.taplo.toml +++ b/.taplo.toml @@ -2,6 +2,7 @@ exclude = [ "node_modules/**/*.toml", "target/**/*.toml", "packages/frontend/apps/ios/App/Packages/AffineGraphQL/**/*.toml", + "packages/frontend/apps/ios/App/Packages/Intelligents/**/*.toml", ] # https://taplo.tamasfe.dev/configuration/formatter-options.html diff --git a/Cargo.lock b/Cargo.lock index 5d63b12f4d126..ffa99c539473b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,23 +64,46 @@ dependencies = [ name = "affine_common" version = "0.1.0" dependencies = [ - "assert-json-diff", "chrono", "criterion", "hex", - "nanoid", "napi", - "pulldown-cmark", "rand 0.9.4", "rayon", - "serde", - "serde_json", "sha3", "tempfile", +] + +[[package]] +name = "affine_doc_loader" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af9fa0b8359900e4c150b717e837eec813f29ee6981fa72d83e671974c0f78f" +dependencies = [ + "nanoid 0.5.0", + "pulldown-cmark", + "serde", + "serde_json", "thiserror 2.0.18", "y-octo", ] +[[package]] +name = "affine_importer" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a698069c4f0ae26bbc3c07deaf84bb0d55b0ed80dae44d6879ca1bf63eee8e9" +dependencies = [ + "affine_doc_loader", + "chrono", + "nanoid 0.5.0", + "serde", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.18", + "zip", +] + [[package]] name = "affine_media_capture" version = "0.0.0" @@ -121,7 +144,7 @@ dependencies = [ "base64-simd", "chrono", "homedir", - "lru", + "lru 0.16.3", "mermaid-rs-renderer", "objc2", "objc2-foundation", @@ -138,24 +161,32 @@ name = "affine_native" version = "0.0.0" dependencies = [ "affine_common", + "affine_doc_loader", + "affine_importer", "affine_media_capture", "affine_nbstore", "affine_sqlite_v1", "chrono", + "hex", + "infer", "mermaid-rs-renderer", "mimalloc", "napi", "napi-build", "napi-derive", "once_cell", + "onenote_parser", "serde_json", + "sha2 0.11.0", "sqlx", "thiserror 2.0.18", "tokio", + "typed-path", "typst", "typst-as-lib", "typst-svg", "uuid", + "zip", ] [[package]] @@ -163,6 +194,7 @@ name = "affine_nbstore" version = "0.0.0" dependencies = [ "affine_common", + "affine_doc_loader", "affine_schema", "anyhow", "bincode 2.0.1", @@ -196,6 +228,7 @@ version = "1.0.0" dependencies = [ "aes-gcm", "affine_common", + "affine_doc_loader", "anyhow", "assetpack-core", "assetpack-transform-precomp2", @@ -235,7 +268,7 @@ dependencies = [ "sqlx", "tempfile", "thiserror 2.0.18", - "tiktoken-rs", + "tiktoken-rs 0.7.0", "tokio", "url", "uuid", @@ -483,16 +516,6 @@ dependencies = [ "winnow 0.7.15", ] -[[package]] -name = "assert-json-diff" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "assetpack-core" version = "0.1.0" @@ -678,7 +701,7 @@ checksum = "53d0c374feba1b9a59042a7c1cf00ce7c34b977b9134fe7c42b08e5183729f66" dependencies = [ "paste", "roman-numerals-rs", - "strum", + "strum 0.27.2", "unic-langid", "unicode-normalization", "unscanny", @@ -847,6 +870,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ "hybrid-array", + "zeroize", ] [[package]] @@ -985,6 +1009,18 @@ dependencies = [ "libbz2-rs-sys", ] +[[package]] +name = "cab" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171228650e6721d5acc0868a462cd864f49ac5f64e4a42cde270406e64e404d2" +dependencies = [ + "byteorder", + "flate2", + "lzxd", + "time", +] + [[package]] name = "cabac" version = "0.15.0" @@ -1357,6 +1393,12 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "convert_case" version = "0.11.0" @@ -1850,6 +1892,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "deflate64" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" + [[package]] name = "der" version = "0.7.10" @@ -1914,6 +1962,7 @@ dependencies = [ "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", + "zeroize", ] [[package]] @@ -1968,9 +2017,9 @@ dependencies = [ [[package]] name = "doc_extractor" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a05485e5262ab131ff0137a369a13cd2e3c5dd2e5c1e57a5c387998cc6fd182f" +checksum = "f43cff8b35cc5673becb65d2417917f540a3e544655071f891d8c564cd48d1c2" dependencies = [ "infer", "office_oxide", @@ -1979,10 +2028,10 @@ dependencies = [ "readability", "serde", "serde_json", - "strum_macros", + "strum_macros 0.28.0", "text-splitter", "thiserror 2.0.18", - "tiktoken-rs", + "tiktoken-rs 0.12.0", "tree-sitter", "tree-sitter-c", "tree-sitter-cpp", @@ -2052,9 +2101,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" dependencies = [ "serde", ] @@ -2129,6 +2178,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "enum-primitive-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba7795da175654fe16979af73f81f26a8ea27638d8d9823d317016888a63dc4c" +dependencies = [ + "num-traits", + "quote", + "syn 2.0.117", +] + [[package]] name = "env_filter" version = "1.0.1" @@ -2165,7 +2225,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2829,6 +2889,17 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + [[package]] name = "hashlink" version = "0.10.0" @@ -3213,7 +3284,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.61.2", ] [[package]] @@ -3714,7 +3785,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4268,6 +4339,15 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "lru" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -4283,6 +4363,12 @@ dependencies = [ "sha2 0.11.0", ] +[[package]] +name = "lzxd" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17f346186eccb574ba5581acefc514f0c70a642db4f96e245034a0a158a7168" + [[package]] name = "mac" version = "0.1.1" @@ -4374,9 +4460,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" @@ -4529,6 +4615,15 @@ dependencies = [ "rand 0.8.6", ] +[[package]] +name = "nanoid" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628de41fe064cc3f0cf07f3d299ee3e73521adaff72278731d5c8cae3797873" +dependencies = [ + "rand 0.9.4", +] + [[package]] name = "napi" version = "3.8.3" @@ -4899,16 +4994,16 @@ dependencies = [ [[package]] name = "office_oxide" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "866a8fa5e1a756f8cc9f9f614031ad7e33adc7f6972eab74a35b0c799fe08da5" +checksum = "8900703278aea7d2329f5c490b62653306b1083efc7993ee18e3e86dcfd5afeb" dependencies = [ "atoi_simd", "encoding_rs", "fast-float2", "libc", "log", - "quick-xml 0.40.1", + "quick-xml 0.41.0", "serde", "serde_json", "thiserror 2.0.18", @@ -4940,6 +5035,38 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "onenote_parser" +version = "1.1.1" +source = "git+https://github.com/msiemens/onenote.rs?rev=5138a39a3f4e72b840932f9872fecde52fa9da60#5138a39a3f4e72b840932f9872fecde52fa9da60" +dependencies = [ + "bytes", + "cab", + "encoding_rs", + "enum-primitive-derive", + "itertools 0.14.0", + "log", + "lru 0.18.0", + "num-traits", + "onenote_parser-macros", + "sanitize-filename", + "thiserror 2.0.18", + "time", + "typed-path", + "uuid", + "widestring", +] + +[[package]] +name = "onenote_parser-macros" +version = "1.1.1" +source = "git+https://github.com/msiemens/onenote.rs?rev=5138a39a3f4e72b840932f9872fecde52fa9da60#5138a39a3f4e72b840932f9872fecde52fa9da60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "oorandom" version = "11.1.5" @@ -5080,6 +5207,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" +[[package]] +name = "pbkdf2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +dependencies = [ + "digest 0.11.3", + "hmac 0.13.0", +] + [[package]] name = "pdf_oxide" version = "0.3.65" @@ -5456,6 +5593,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppmd-rust" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -5660,6 +5803,15 @@ name = "quick-xml" version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -5718,7 +5870,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6160,7 +6312,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6219,7 +6371,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6323,6 +6475,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sanitize-filename" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc984f4f9ceb736a7bb755c3e3bd17dc56370af2600c9780dcc48c66453da34d" +dependencies = [ + "regex", +] + [[package]] name = "schannel" version = "0.1.29" @@ -6551,6 +6712,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -6871,7 +7043,7 @@ dependencies = [ "rand 0.8.6", "rsa", "serde", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "smallvec", "sqlx-core", @@ -6960,6 +7132,7 @@ dependencies = [ "cfg-if", "libc", "psm", + "windows-sys 0.52.0", "windows-sys 0.59.0", ] @@ -7038,7 +7211,16 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros", + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", ] [[package]] @@ -7053,6 +7235,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "subsetter" version = "0.2.6" @@ -7389,7 +7583,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7405,9 +7599,9 @@ dependencies = [ [[package]] name = "text-splitter" -version = "0.27.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7e97f5863248f7d07896a1816bd4110cb1b0b122741f157d702121a270bf33" +checksum = "3f3eeec76988617ff1434d754d7e8e197be2cb8981ea77c17e32f3f7a2c6f95e" dependencies = [ "ahash", "auto_enums", @@ -7417,9 +7611,9 @@ dependencies = [ "itertools 0.14.0", "memchr", "pulldown-cmark", - "strum", + "strum 0.28.0", "thiserror 2.0.18", - "tiktoken-rs", + "tiktoken-rs 0.12.0", ] [[package]] @@ -7529,6 +7723,21 @@ dependencies = [ "rustc-hash 1.1.0", ] +[[package]] +name = "tiktoken-rs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2" +dependencies = [ + "anyhow", + "base64", + "bstr", + "fancy-regex 0.17.0", + "lazy_static", + "regex", + "rustc-hash 2.1.1", +] + [[package]] name = "time" version = "0.3.47" @@ -7537,6 +7746,7 @@ checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", + "js-sys", "num-conv", "powerfmt", "serde_core", @@ -7879,9 +8089,9 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.25.10" +version = "0.26.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87" +checksum = "3c343ed63e3f5c64d1acdecb5d2c13d4e169cb5fde0052106ebaa6c6f27f9e55" dependencies = [ "cc", "regex", @@ -7893,9 +8103,9 @@ dependencies = [ [[package]] name = "tree-sitter-c" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3aad8f0129083a59fe8596157552d2bb7148c492d44c21558d68ca1c722707" +checksum = "a9b2eb57a55fed6b00812912e730b7a275cf4fe98bfd6a5d76263d4438371728" dependencies = [ "cc", "tree-sitter-language", @@ -7913,9 +8123,9 @@ dependencies = [ [[package]] name = "tree-sitter-go" -version = "0.23.4" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b13d476345220dbe600147dd444165c5791bf85ef53e28acbedd46112ee18431" +checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea" dependencies = [ "cc", "tree-sitter-language", @@ -7923,9 +8133,9 @@ dependencies = [ [[package]] name = "tree-sitter-javascript" -version = "0.23.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf40bf599e0416c16c125c3cec10ee5ddc7d1bb8b0c60fa5c4de249ad34dc1b1" +checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5" dependencies = [ "cc", "tree-sitter-language", @@ -7939,9 +8149,9 @@ checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" [[package]] name = "tree-sitter-python" -version = "0.23.6" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d065aaa27f3aaceaf60c1f0e0ac09e1cb9eb8ed28e7bcdaa52129cffc7f4b04" +checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c" dependencies = [ "cc", "tree-sitter-language", @@ -7949,9 +8159,9 @@ dependencies = [ [[package]] name = "tree-sitter-rust" -version = "0.24.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b9b18034c684a2420722be8b2a91c9c44f2546b631c039edf575ccba8c61be1" +checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45" dependencies = [ "cc", "tree-sitter-language", @@ -9090,7 +9300,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -9154,19 +9364,6 @@ dependencies = [ "windows-strings 0.4.2", ] -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - [[package]] name = "windows-future" version = "0.2.1" @@ -9686,7 +9883,7 @@ dependencies = [ "byteorder", "log", "loom", - "nanoid", + "nanoid 0.4.0", "nom 8.0.0", "ordered-float", "rand 0.9.4", @@ -9876,12 +10073,25 @@ version = "8.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ + "aes 0.9.1", + "bzip2", + "constant_time_eq", "crc32fast", + "deflate64", "flate2", + "getrandom 0.4.2", + "hmac 0.13.0", "indexmap", + "lzma-rust2", "memchr", + "pbkdf2", + "ppmd-rust", + "sha1 0.11.0", + "time", "typed-path", + "zeroize", "zopfli", + "zstd", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e96fcec7a2b89..000420ccaff9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,8 @@ resolver = "3" [workspace.dependencies] aes-gcm = "0.10" affine_common = { path = "./packages/common/native" } + affine_doc_loader = "0.1.2" + affine_importer = "0.1.1" affine_nbstore = { path = "./packages/frontend/native/nbstore" } anyhow = "1" assert-json-diff = "2.0" @@ -30,7 +32,7 @@ resolver = "3" criterion2 = { version = "3", default-features = false } crossbeam-channel = "0.5" dispatch2 = "0.3" - doc_extractor = "0.1.0" + doc_extractor = "0.1.1" dotenvy = "0.15" file-format = { version = "0.28", features = ["reader"] } hex = "0.4" @@ -67,6 +69,9 @@ resolver = "3" objc2-foundation = "0.3" ogg = "0.9" once_cell = "1" + onenote_parser = { git = "https://github.com/msiemens/onenote.rs", rev = "5138a39a3f4e72b840932f9872fecde52fa9da60", package = "onenote_parser", features = [ + "onepkg", + ] } p256 = { version = "0.13", features = ["ecdsa", "pem"] } pulldown-cmark = "0.13" rand = "0.9" @@ -90,6 +95,7 @@ resolver = "3" thiserror = "2" tiktoken-rs = "0.7" tokio = "1.45" + typed-path = "0.12.3" typst = "0.14.2" typst-as-lib = { version = "0.15.4", default-features = false, features = [ "packages", @@ -116,6 +122,7 @@ resolver = "3" ] } windows-core = { version = "0.61" } y-octo = "0.0.3" + zip = "8.6" [profile.dev.package.sqlx-macros] opt-level = 3 diff --git a/blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts b/blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts index 528ad86d9359b..882291b1050ab 100644 --- a/blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts +++ b/blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts @@ -2,7 +2,11 @@ import { readFileSync } from 'node:fs'; import { basename, resolve } from 'node:path'; import { + BearTransformer, + commitImportBatchToWorkspace, + type ImportBatch, MarkdownTransformer, + NotionHtmlTransformer, ObsidianTransformer, } from '@blocksuite/affine/widgets/linked-doc'; import { @@ -84,6 +88,23 @@ function zipFixture(entries: Record) { return new Blob([buffer], { type: 'application/zip' }); } +async function commitPlannedImport( + collection: TestWorkspace, + schema: Schema, + planned: T +) { + const committed = await commitImportBatchToWorkspace( + collection, + schema, + planned.batch + ); + return { + ...planned, + ...committed, + docIds: committed.docIds, + }; +} + function exportSnapshot(doc: Store): DocSnapshot { const job = doc.getTransformer([ docLinkBaseURLMiddleware(doc.workspace.id), @@ -203,6 +224,12 @@ function snapshotDocByTitle( return simplifyBlockForSnapshot(exportSnapshot(doc!).blocks, titleById); } +function titleMap(collection: TestWorkspace) { + return new Map( + collection.meta.docMetas.map(meta => [meta.id, meta.title ?? '']) + ); +} + function collectSimplifiedDeltas( block: Record ): Record[] { @@ -218,6 +245,27 @@ function collectSimplifiedDeltas( return [...deltas, ...childDeltas]; } +function collectSnapshotDeltas( + block: BlockSnapshot +): DeltaInsert[] { + const text = block.props.text as + | { delta?: DeltaInsert[] } + | undefined; + return [ + ...(text?.delta ?? []), + ...(block.children ?? []).flatMap(child => collectSnapshotDeltas(child)), + ]; +} + +function folderChild( + folder: { children: Map } | undefined, + name: string +) { + return folder?.children.get(name) as + | { children: Map; pageId?: string; icon?: unknown } + | undefined; +} + describe('snapshot to markdown', () => { test('code', async () => { const blockSnapshot: BlockSnapshot = { @@ -377,13 +425,16 @@ Hello world '# Nested Page\nNested body', }); - const { docIds, folderHierarchy } = - await MarkdownTransformer.importNotionMarkdownZip({ + const { docIds, folderHierarchy } = await commitPlannedImport( + collection, + schema, + await MarkdownTransformer.planNotionMarkdownZip({ collection, schema, imported, extensions: testStoreExtensions, - }); + }) + ); expect(docIds).toHaveLength(2); expect( @@ -431,13 +482,16 @@ Hello world '# SDK架构\nNested body', }); - const { folderHierarchy } = - await MarkdownTransformer.importNotionMarkdownZip({ + const { folderHierarchy } = await commitPlannedImport( + collection, + schema, + await MarkdownTransformer.planNotionMarkdownZip({ collection, schema, imported, extensions: testStoreExtensions, - }); + }) + ); const [rootFolder] = [...(folderHierarchy?.children.values() ?? [])]; expect(rootFolder?.name).toBe('Export'); @@ -463,12 +517,16 @@ Hello world '---\ntitle: Frontmatter Title\n---\nBody', }); - const { docIds } = await MarkdownTransformer.importNotionMarkdownZip({ + const { docIds } = await commitPlannedImport( collection, schema, - imported, - extensions: testStoreExtensions, - }); + await MarkdownTransformer.planNotionMarkdownZip({ + collection, + schema, + imported, + extensions: testStoreExtensions, + }) + ); expect(docIds).toHaveLength(1); expect(collection.meta.getDocMeta(docIds[0])?.title).toBe( @@ -491,12 +549,16 @@ Hello world 'test/2.md': 'target page', }); - const { docIds } = await MarkdownTransformer.importMarkdownZip({ + const { docIds } = await commitPlannedImport( collection, schema, - imported, - extensions: testStoreExtensions, - }); + await MarkdownTransformer.planMarkdownZip({ + collection, + schema, + imported, + extensions: testStoreExtensions, + }) + ); expect(docIds).toHaveLength(2); const titleById = new Map( @@ -527,6 +589,88 @@ Hello world }); }); + test('imports markdown zip assets, nested zip, CJK paths, and duplicate names', async () => { + const schema = new Schema().register(AffineSchemas); + const collection = new TestWorkspace(); + collection.storeExtensions = testStoreExtensions; + collection.meta.initialize(); + + const imported = zipFixture({ + '入口.md': + '![logo](./assets/logo.png)\n[同名](./folder/duplicate.md)\n![archive](./nested.zip)', + 'assets/logo.png': new Uint8Array([137, 80, 78, 71]), + 'folder/duplicate.md': 'folder duplicate', + 'other/duplicate.md': 'other duplicate', + 'nested.zip': zipBytes({ + 'ignored.md': 'nested markdown should stay an attachment', + }), + }); + + const { docIds, folderHierarchy } = await commitPlannedImport( + collection, + schema, + await MarkdownTransformer.planMarkdownZip({ + collection, + schema, + imported, + extensions: testStoreExtensions, + }) + ); + + expect(docIds).toHaveLength(3); + expect( + collection.meta.docMetas + .map(meta => meta.title) + .sort((a, b) => (a ?? '').localeCompare(b ?? '')) + ).toEqual(['duplicate', 'duplicate', '入口']); + + const titles = titleMap(collection); + const entry = snapshotDocByTitle(collection, '入口', titles); + expect(JSON.stringify(entry)).toContain('"sourceId":""'); + const entrySnapshot = exportSnapshot( + collection + .getDoc( + collection.meta.docMetas.find(meta => meta.title === '入口')!.id + )! + .getStore({ + id: collection.meta.docMetas.find(meta => meta.title === '入口')!.id, + }) + ); + const linkedPageDelta = collectSnapshotDeltas(entrySnapshot.blocks).find( + delta => delta.attributes?.reference?.type === 'LinkedPage' + ); + const linkedPageId = + linkedPageDelta?.attributes?.reference?.type === 'LinkedPage' + ? linkedPageDelta.attributes.reference.pageId + : undefined; + expect(linkedPageId).toBeTruthy(); + expect( + JSON.stringify( + snapshotDocByTitle( + collection, + 'duplicate', + new Map([[linkedPageId!, 'duplicate']]) + ) + ) + ).toContain('folder duplicate'); + expect(collectSimplifiedDeltas(entry)).toContainEqual({ + insert: ' ', + reference: { + type: 'LinkedPage', + page: 'duplicate', + title: '同名', + }, + }); + expect(JSON.stringify(entry).match(/"sourceId":""/g)).toHaveLength( + 2 + ); + expect(folderHierarchy?.children.has('folder')).toBe(true); + expect(folderHierarchy?.children.has('other')).toBe(true); + expect( + collection.meta.docMetas.some(meta => meta.title === 'ignored') + ).toBe(false); + }); + test('imports notion markdown zip relative doc links as linked pages', async () => { const schema = new Schema().register(AffineSchemas); const collection = new TestWorkspace(); @@ -540,12 +684,16 @@ Hello world '# Target\ntarget page', }); - const { docIds } = await MarkdownTransformer.importNotionMarkdownZip({ + const { docIds } = await commitPlannedImport( collection, schema, - imported, - extensions: testStoreExtensions, - }); + await MarkdownTransformer.planNotionMarkdownZip({ + collection, + schema, + imported, + extensions: testStoreExtensions, + }) + ); expect(docIds).toHaveLength(2); const titleById = new Map( @@ -587,13 +735,16 @@ Hello world }), }); - const { docIds, folderHierarchy } = - await MarkdownTransformer.importNotionMarkdownZip({ + const { docIds, folderHierarchy } = await commitPlannedImport( + collection, + schema, + await MarkdownTransformer.planNotionMarkdownZip({ collection, schema, imported, extensions: testStoreExtensions, - }); + }) + ); expect(docIds).toHaveLength(4); const titleById = new Map( @@ -646,16 +797,20 @@ Hello world 'vault/archive.zip' ); - const { docIds } = await ObsidianTransformer.importObsidianVault({ + const { docIds } = await commitPlannedImport( collection, schema, - importedFiles: [ - markdownFixture('entry.md'), - markdownFixture('linked.md'), - attachment, - ], - extensions: testStoreExtensions, - }); + await ObsidianTransformer.planObsidianVault({ + collection, + schema, + importedFiles: [ + markdownFixture('entry.md'), + markdownFixture('linked.md'), + attachment, + ], + extensions: testStoreExtensions, + }) + ); expect(docIds).toHaveLength(2); const titleById = new Map( @@ -673,6 +828,147 @@ Hello world }).toMatchSnapshot(); }); + test('imports notion html zip golden baseline', async () => { + const schema = new Schema().register(AffineSchemas); + const collection = new TestWorkspace(); + collection.storeExtensions = testStoreExtensions; + collection.meta.initialize(); + + const imported = zipFixture({ + 'Export/index.html': 'workspace index', + 'Export/Project.html': ` + + +
+
+

Project body

+ +
+ + + `, + 'Export/Project/Nested.html': ` + + +

Nested body

+ + + `, + 'Export/assets/logo.png': new Uint8Array([137, 80, 78, 71]), + }); + + const result = await commitPlannedImport( + collection, + schema, + await NotionHtmlTransformer.planNotionHtmlZip({ + collection, + schema, + imported, + extensions: testStoreExtensions, + }) + ); + + expect(result.isWorkspaceFile).toBe(true); + expect(result.hasMarkdown).toBe(false); + expect(result.pageIds).toHaveLength(2); + expect(collection.meta.docMetas.map(meta => meta.title)).toEqual(['', '']); + + const titles = titleMap(collection); + const importedSnapshots = result.pageIds.map(pageId => + simplifyBlockForSnapshot( + exportSnapshot(collection.getDoc(pageId)!.getStore({ id: pageId })) + .blocks, + titles + ) + ); + const projectSnapshot = importedSnapshots.find(snapshot => + JSON.stringify(snapshot).includes('Project body') + ); + expect(projectSnapshot).toBeTruthy(); + expect(JSON.stringify(projectSnapshot)).toContain('Project body'); + expect(JSON.stringify(projectSnapshot)).toContain('"sourceId":""'); + + const exportFolder = folderChild(result.folderHierarchy, 'Export'); + const projectNode = folderChild(exportFolder, 'Project'); + expect(result.pageIds).toContain(projectNode?.pageId); + expect(projectNode?.icon).toEqual({ type: 'emoji', content: '✅' }); + expect(result.pageIds).toContain( + folderChild(projectNode, 'Nested')?.pageId + ); + }); + + test('imports bear backup golden baseline', async () => { + const schema = new Schema().register(AffineSchemas); + const collection = new TestWorkspace(); + collection.storeExtensions = testStoreExtensions; + collection.meta.initialize(); + + const imported = zipFixture({ + 'Notes/Idea.textbundle/text.md': [ + '# Bear Title', + '', + '![photo](assets/photo.png)', + '', + '==🟢green highlight==', + '', + '#work/project', + '#Blue Tag#', + ].join('\n'), + 'Notes/Idea.textbundle/info.json': JSON.stringify({ + 'net.shinyfrog.bear': { + creationDate: '2024-01-02T03:04:05.000Z', + modificationDate: '2024-01-03T03:04:05.000Z', + }, + }), + 'Notes/Idea.textbundle/assets/photo.png': new Uint8Array([ + 137, 80, 78, 71, + ]), + }); + + const { docIds, tags, folderHierarchy } = await commitPlannedImport( + collection, + schema, + await BearTransformer.planBearBackup({ + collection, + schema, + imported, + extensions: testStoreExtensions, + }) + ); + + expect(docIds).toHaveLength(1); + const meta = collection.meta.getDocMeta(docIds[0]); + expect(meta?.title).toBe('Bear Title'); + expect(meta?.createDate).toBe(Date.parse('2024-01-02T03:04:05.000Z')); + expect(meta?.updatedDate).toBe(Date.parse('2024-01-03T03:04:05.000Z')); + expect([...tags.keys()]).toEqual(['Blue Tag', 'work/project']); + + const titles = titleMap(collection); + const snapshot = snapshotDocByTitle(collection, 'Bear Title', titles); + expect(JSON.stringify(snapshot)).toContain('"sourceId":""'); + expect(JSON.stringify(snapshot)).toContain('green highlight'); + + const blueTag = folderChild(folderHierarchy, 'Blue Tag'); + expect([ + ...( + (blueTag?.children as Map | undefined) ?? new Map() + ).values(), + ]).toEqual( + expect.arrayContaining([expect.objectContaining({ pageId: docIds[0] })]) + ); + const project = folderChild( + folderChild(folderHierarchy, 'work'), + 'project' + ); + expect([ + ...( + (project?.children as Map | undefined) ?? new Map() + ).values(), + ]).toEqual( + expect.arrayContaining([expect.objectContaining({ pageId: docIds[0] })]) + ); + }); + test('paragraph', async () => { const blockSnapshot: BlockSnapshot = { type: 'block', diff --git a/blocksuite/affine/shared/src/utils/file/filesys.ts b/blocksuite/affine/shared/src/utils/file/filesys.ts index 65454a7958d8e..b1a025a238dde 100644 --- a/blocksuite/affine/shared/src/utils/file/filesys.ts +++ b/blocksuite/affine/shared/src/utils/file/filesys.ts @@ -120,6 +120,12 @@ const FileTypes: NonNullable = [ ['.docx'], }, }, + { + description: 'OneNote', + accept: { + 'application/onenote': ['.one', '.onetoc2', '.onepkg'], + }, + }, { description: 'MindMap', accept: { @@ -140,8 +146,58 @@ type AcceptTypes = | 'Html' | 'Zip' | 'Docx' + | 'OneNote' | 'MindMap'; +type OpenFileOptions = { + fileSystemAccess?: boolean; + snapshot?: boolean; +}; + +function copyToArrayBuffer(bytes: Uint8Array) { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + +function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +export async function snapshotFile(file: File, relativePath?: string) { + let bytes: Uint8Array | null = null; + for (let attempt = 0; attempt < 3; attempt++) { + try { + bytes = new Uint8Array(await file.arrayBuffer()); + break; + } catch (error) { + if (!isNotReadableError(error) || attempt === 2) { + throw error; + } + await sleep(100 * (attempt + 1)); + } + } + if (!bytes) { + throw new DOMException('File could not be read', 'NotReadableError'); + } + const snapshot = new File([copyToArrayBuffer(bytes)], file.name, { + type: file.type, + lastModified: file.lastModified, + }); + const path = relativePath ?? file.webkitRelativePath; + if (path) { + Object.defineProperty(snapshot, 'webkitRelativePath', { + value: path, + writable: false, + }); + } + return snapshot; +} + +export async function snapshotFiles(files: File[]) { + return Promise.all(files.map(file => snapshotFile(file))); +} + function canUseFileSystemAccessAPI( api: 'showOpenFilePicker' | 'showDirectoryPicker' ) { @@ -157,11 +213,21 @@ function canUseFileSystemAccessAPI( ); } +function isAbortError(error: unknown) { + return error instanceof Error && error.name === 'AbortError'; +} + +function isNotReadableError(error: unknown) { + return error instanceof Error && error.name === 'NotReadableError'; +} + export async function openFilesWith( acceptType: AcceptTypes = 'Any', - multiple: boolean = true + multiple: boolean = true, + options?: OpenFileOptions ): Promise { const supportsFileSystemAccess = + options?.fileSystemAccess !== false && canUseFileSystemAccessAPI('showOpenFilePicker'); // If the File System Access API is supported… @@ -180,15 +246,19 @@ export async function openFilesWith( // Show the file picker, optionally allowing multiple files. const handles = await window.showOpenFilePicker(pickerOpts); - return await Promise.all(handles.map(handle => handle.getFile())); + const files = await Promise.all(handles.map(handle => handle.getFile())); + return options?.snapshot ? await snapshotFiles(files) : files; } catch (err) { + if (options?.snapshot && !isAbortError(err)) { + throw err; + } console.error(err); return null; } } // Fallback if the File System Access API is not supported. - return new Promise(resolve => { + return new Promise((resolve, reject) => { // Append a new `` and hide it. const input = document.createElement('input'); input.classList.add('affine-upload-input'); @@ -205,13 +275,27 @@ export async function openFilesWith( document.body.append(input); // The `change` event fires when the user interacts with the dialog. input.addEventListener('change', () => { - // Remove the `` again from the DOM. - input.remove(); + const files = input.files ? Array.from(input.files) : null; + const finish = (result: File[] | null) => { + // Remove the `` again from the DOM. + input.remove(); + resolve(result); + }; - resolve(input.files ? Array.from(input.files) : null); + if (files && options?.snapshot) { + snapshotFiles(files).then(finish, error => { + input.remove(); + reject(error); + }); + } else { + finish(files); + } }); // The `cancel` event fires when the user cancels the dialog. - input.addEventListener('cancel', () => resolve(null)); + input.addEventListener('cancel', () => { + input.remove(); + resolve(null); + }); // Show the picker. if ('showPicker' in HTMLInputElement.prototype) { input.showPicker(); @@ -221,12 +305,18 @@ export async function openFilesWith( }); } -export async function openDirectory(): Promise { +export async function openDirectory( + options?: OpenFileOptions +): Promise { const supportsFileSystemAccess = canUseFileSystemAccessAPI( 'showDirectoryPicker' ); - if (supportsFileSystemAccess && window.showDirectoryPicker) { + if ( + options?.fileSystemAccess !== false && + supportsFileSystemAccess && + window.showDirectoryPicker + ) { try { const dirHandle = await window.showDirectoryPicker(); const files: File[] = []; @@ -241,11 +331,15 @@ export async function openDirectory(): Promise { const fileHandle = handle as FileSystemFileHandle; if (fileHandle.getFile) { const file = await fileHandle.getFile(); - Object.defineProperty(file, 'webkitRelativePath', { - value: relativePath, - writable: false, - }); - files.push(file); + if (options?.snapshot) { + files.push(await snapshotFile(file, relativePath)); + } else { + Object.defineProperty(file, 'webkitRelativePath', { + value: relativePath, + writable: false, + }); + files.push(file); + } } } else if (handle.kind === 'directory') { await readDirectory( @@ -259,12 +353,15 @@ export async function openDirectory(): Promise { await readDirectory(dirHandle, ''); return files; } catch (err) { + if (options?.snapshot && !isAbortError(err)) { + throw err; + } console.error(err); return null; } } - return new Promise(resolve => { + return new Promise((resolve, reject) => { const input = document.createElement('input'); input.classList.add('affine-upload-input'); input.style.display = 'none'; @@ -276,11 +373,26 @@ export async function openDirectory(): Promise { document.body.append(input); input.addEventListener('change', () => { - input.remove(); - resolve(input.files ? Array.from(input.files) : null); + const files = input.files ? Array.from(input.files) : null; + const finish = (result: File[] | null) => { + input.remove(); + resolve(result); + }; + + if (files && options?.snapshot) { + snapshotFiles(files).then(finish, error => { + input.remove(); + reject(error); + }); + } else { + finish(files); + } }); - input.addEventListener('cancel', () => resolve(null)); + input.addEventListener('cancel', () => { + input.remove(); + resolve(null); + }); if ('showPicker' in HTMLInputElement.prototype) { input.showPicker(); diff --git a/blocksuite/affine/widgets/linked-doc/src/import-doc/import-doc.ts b/blocksuite/affine/widgets/linked-doc/src/import-doc/import-doc.ts index f15c39434636c..26da0c71a01dd 100644 --- a/blocksuite/affine/widgets/linked-doc/src/import-doc/import-doc.ts +++ b/blocksuite/affine/widgets/linked-doc/src/import-doc/import-doc.ts @@ -2,15 +2,9 @@ import { CloseIcon, ExportToHTMLIcon, ExportToMarkdownIcon, - HelpIcon, NewIcon, - NotionIcon, } from '@blocksuite/affine-components/icons'; -import { - openDirectory, - openFilesWith, - openSingleFileWith, -} from '@blocksuite/affine-shared/utils'; +import { openFilesWith } from '@blocksuite/affine-shared/utils'; import { WithDisposable } from '@blocksuite/global/lit'; import type { ExtensionType, Schema, Workspace } from '@blocksuite/store'; import { html, LitElement, type PropertyValues } from 'lit'; @@ -18,8 +12,6 @@ import { query, state } from 'lit/decorators.js'; import { HtmlTransformer } from '../transformers/html.js'; import { MarkdownTransformer } from '../transformers/markdown.js'; -import { NotionHtmlTransformer } from '../transformers/notion-html.js'; -import { ObsidianTransformer } from '../transformers/obsidian.js'; import { styles } from './styles.js'; export type OnSuccessHandler = ( @@ -59,114 +51,69 @@ export class ImportDoc extends WithDisposable(LitElement) { } private async _importHtml() { - const files = await openFilesWith('Html'); - if (!files) return; - const pageIds: string[] = []; - for (const file of files) { - const text = await file.text(); - const needLoading = file.size > SHOW_LOADING_SIZE; - const fileName = file.name.split('.').slice(0, -1).join('.'); - if (needLoading) { - this.hidden = false; - this._loading = true; - } else { - this.abortController.abort(); - } - const pageId = await HtmlTransformer.importHTMLToDoc({ - collection: this.collection, - schema: this.schema, - extensions: this.extensions, - html: text, - fileName, - }); - needLoading && this.abortController.abort(); - if (pageId) { - pageIds.push(pageId); + try { + const files = await openFilesWith('Html'); + if (!files) return; + const pageIds: string[] = []; + for (const file of files) { + const text = await file.text(); + const needLoading = file.size > SHOW_LOADING_SIZE; + const fileName = file.name.split('.').slice(0, -1).join('.'); + if (needLoading) { + this.hidden = false; + this._loading = true; + } else { + this.abortController.abort(); + } + const pageId = await HtmlTransformer.importHTMLToDoc({ + collection: this.collection, + schema: this.schema, + extensions: this.extensions, + html: text, + fileName, + }); + needLoading && this.abortController.abort(); + if (pageId) { + pageIds.push(pageId); + } } + this._onImportSuccess(pageIds); + } catch (error) { + this._onFail(error); } - this._onImportSuccess(pageIds); } private async _importMarkDown() { - const files = await openFilesWith('Markdown'); - if (!files) return; - const pageIds: string[] = []; - for (const file of files) { - const text = await file.text(); - const fileName = file.name.split('.').slice(0, -1).join('.'); - const needLoading = file.size > SHOW_LOADING_SIZE; - if (needLoading) { - this.hidden = false; - this._loading = true; - } else { - this.abortController.abort(); + try { + const files = await openFilesWith('Markdown'); + if (!files) return; + const pageIds: string[] = []; + for (const file of files) { + const text = await file.text(); + const fileName = file.name.split('.').slice(0, -1).join('.'); + const needLoading = file.size > SHOW_LOADING_SIZE; + if (needLoading) { + this.hidden = false; + this._loading = true; + } else { + this.abortController.abort(); + } + const pageId = await MarkdownTransformer.importMarkdownToDoc({ + collection: this.collection, + schema: this.schema, + markdown: text, + fileName, + extensions: this.extensions, + }); + needLoading && this.abortController.abort(); + if (pageId) { + pageIds.push(pageId); + } } - const pageId = await MarkdownTransformer.importMarkdownToDoc({ - collection: this.collection, - schema: this.schema, - markdown: text, - fileName, - extensions: this.extensions, - }); - needLoading && this.abortController.abort(); - if (pageId) { - pageIds.push(pageId); - } - } - this._onImportSuccess(pageIds); - } - - private async _importNotion() { - const file = await openSingleFileWith('Zip'); - if (!file) return; - const needLoading = file.size > SHOW_LOADING_SIZE; - if (needLoading) { - this.hidden = false; - this._loading = true; - } else { - this.abortController.abort(); - } - const { entryId, pageIds, isWorkspaceFile, hasMarkdown } = - await NotionHtmlTransformer.importNotionZip({ - collection: this.collection, - schema: this.schema, - imported: file, - extensions: this.extensions, - }); - needLoading && this.abortController.abort(); - if (hasMarkdown) { - this._onFail( - 'Importing markdown files from Notion is deprecated. Please export your Notion pages as HTML.' - ); - return; + this._onImportSuccess(pageIds); + } catch (error) { + this._onFail(error); } - this._onImportSuccess(entryId ? [entryId] : [], { - isWorkspaceFile, - importedCount: pageIds.length, - }); - } - - private async _importObsidian() { - const files = await openDirectory(); - if (!files || files.length === 0) return; - const needLoading = - files.reduce((acc, f) => acc + f.size, 0) > SHOW_LOADING_SIZE; - if (needLoading) { - this.hidden = false; - this._loading = true; - } else { - this.abortController.abort(); - } - const { docIds, docEmojis } = await ObsidianTransformer.importObsidianVault( - { - collection: this.collection, - schema: this.schema, - importedFiles: files, - extensions: this.extensions, - } - ); - needLoading && this.abortController.abort(); - this._onImportSuccess(docIds, { docEmojis }); } private _onCloseClick(event: MouseEvent) { @@ -174,8 +121,8 @@ export class ImportDoc extends WithDisposable(LitElement) { this.abortController.abort(); } - private _onFail(message: string) { - this.onFail?.(message); + private _onFail(error: unknown) { + this.onFail?.(error instanceof Error ? error.message : String(error)); } private _onImportSuccess( @@ -213,14 +160,6 @@ export class ImportDoc extends WithDisposable(LitElement) { window.removeEventListener('mousemove', this._onMouseMove); } - private _openLearnImportLink(event: MouseEvent) { - event.stopPropagation(); - window.open( - 'https://affine.pro/blog/import-your-data-from-notion-into-affine', - '_blank' - ); - } - override render() { if (this._loading) { return html` @@ -276,30 +215,6 @@ export class ImportDoc extends WithDisposable(LitElement) {
- - ${NotionIcon} -
- ${HelpIcon} - - Learn how to Import your Notion pages into AFFiNE. - -
-
- - ${ExportToMarkdownIcon} - ${NewIcon} diff --git a/blocksuite/affine/widgets/linked-doc/src/transformers/bear.ts b/blocksuite/affine/widgets/linked-doc/src/transformers/bear.ts index 0ca9c8d984672..476b63e363b1f 100644 --- a/blocksuite/affine/widgets/linked-doc/src/transformers/bear.ts +++ b/blocksuite/affine/widgets/linked-doc/src/transformers/bear.ts @@ -7,17 +7,30 @@ import { } from '@blocksuite/affine-shared/adapters'; import { Container } from '@blocksuite/global/di'; import { sha } from '@blocksuite/global/utils'; -import type { ExtensionType, Schema, Workspace } from '@blocksuite/store'; +import type { + DocSnapshot, + ExtensionType, + Schema, + Workspace, +} from '@blocksuite/store'; import { extMimeMap, Transformer } from '@blocksuite/store'; import JSZip from 'jszip'; +import { + blobsFromAssets, + type ImportBatch, + type ImportDoc, + type ImportFolder, + type ImportTag, + type ImportWarning, +} from './import-batch.js'; import { createCollectionDocCRUD } from './markdown.js'; /** Recursive tree node representing a tag-based folder hierarchy. */ -type FolderHierarchy = { +export type BearFolderHierarchy = { name: string; path: string; - children: Map; + children: Map; pageId?: string; parentPath?: string; }; @@ -29,10 +42,11 @@ type BearImportOptions = { extensions: ExtensionType[]; }; -type BearImportResult = { +export type PlanBearBackupResult = { docIds: string[]; tags: Map; - folderHierarchy: FolderHierarchy; + folderHierarchy: BearFolderHierarchy; + batch: ImportBatch; }; type BundleEntry = { @@ -182,8 +196,8 @@ function deduplicateTags(tags: string[]): string[] { */ function buildFolderHierarchyFromTags( tagDocMap: Map -): FolderHierarchy { - const root: FolderHierarchy = { +): BearFolderHierarchy { + const root: BearFolderHierarchy = { name: '', path: '', children: new Map(), @@ -226,6 +240,34 @@ function buildFolderHierarchyFromTags( return root; } +function flattenFolderHierarchy(root: BearFolderHierarchy): ImportFolder[] { + const folders: ImportFolder[] = []; + + const visit = (node: BearFolderHierarchy) => { + if (node.name) { + folders.push({ + path: node.path, + name: node.name, + parentPath: node.parentPath, + pageId: node.pageId, + }); + } + for (const child of node.children.values()) { + visit(child); + } + }; + + for (const child of root.children.values()) { + visit(child); + } + + return folders; +} + +function tagsToBatchTags(tags: Map): ImportTag[] { + return Array.from(tags, ([name, docIds]) => ({ name, docIds })); +} + const GFM_CALLOUT_MAP: Record = { IMPORTANT: '\u26A0', NOTE: '\uD83D\uDCDD', @@ -250,9 +292,9 @@ function convertGfmCallouts(markdown: string): string { if (!inCodeBlock) { lines[i] = lines[i].replace( /^(>\s*)\[!(\w+)\]/, - (_match, prefix: string, type: string) => { + (match, prefix: string, type: string) => { const emoji = GFM_CALLOUT_MAP[type.toUpperCase()]; - return emoji ? `${prefix}[!${emoji}]` : _match; + return emoji ? `${prefix}[!${emoji}]` : match; } ); } @@ -306,7 +348,7 @@ function convertHighlights(markdown: string): string { if (!inCodeBlock) { lines[i] = lines[i].replace( /==(\S(?:[^=]|=[^=])*?)==/g, - (_match, content: string) => { + (...[, content]) => { const firstChar = String.fromCodePoint(content.codePointAt(0)!); const color = HIGHLIGHT_COLOR_MAP[firstChar]; if (color) { @@ -340,16 +382,24 @@ function extractTitle(markdown: string, bundleName: string): string { return bundleName.replace(/\.textbundle$/i, '') || 'Untitled'; } +function applySnapshotTitle(snapshot: DocSnapshot, title: string) { + snapshot.meta.title = title; + snapshot.blocks.props.title = { + '$blocksuite:internal:text$': true, + delta: [{ insert: title }], + }; +} + /** * Import a Bear .bear2bk backup file. * Uses JSZip for lazy/streaming decompression to handle large backups. */ -async function importBearBackup({ +async function planBearBackup({ collection, schema, imported, extensions, -}: BearImportOptions): Promise { +}: BearImportOptions): Promise { const provider = getProvider(extensions); // JSZip reads the zip directory without decompressing all entries @@ -358,7 +408,7 @@ async function importBearBackup({ // Scan entries and group by textbundle const bundleMap = new Map(); - zip.forEach((path, _entry) => { + zip.forEach(path => { if (path.includes('__MACOSX') || path.includes('.DS_Store')) return; const tbMatch = path.match(/^(.+?\.textbundle)\/(.*)/i); @@ -421,6 +471,12 @@ async function importBearBackup({ } const docIds: string[] = []; + const docs: ImportDoc[] = []; + const importBlobs = new Map< + string, + Awaited>[0] + >(); + const warnings: ImportWarning[] = []; const tagDocMap = new Map(); // Process bundles sequentially to limit memory. @@ -497,44 +553,70 @@ async function importBearBackup({ } const mdAdapter = new MarkdownAdapter(job, provider); - const doc = await mdAdapter.toDoc({ + const snapshot = await mdAdapter.toDocSnapshot({ file: markdown, assets: job.assetsManager, }); - if (doc) { - docIds.push(doc.id); + const docId = collection.idGenerator(); + snapshot.meta.id = docId; + applySnapshotTitle(snapshot, title); + const meta: ImportDoc['meta'] = { title }; + if (bearMeta?.creationDate) { + const ts = Date.parse(String(bearMeta.creationDate)); + if (!isNaN(ts)) meta.createDate = ts; + } + if (bearMeta?.modificationDate) { + const ts = Date.parse(String(bearMeta.modificationDate)); + if (!isNaN(ts)) meta.updatedDate = ts; + } - const metaPatch: Record = {}; - if (bearMeta?.creationDate) { - const ts = Date.parse(String(bearMeta.creationDate)); - if (!isNaN(ts)) metaPatch.createDate = ts; - } - if (bearMeta?.modificationDate) { - const ts = Date.parse(String(bearMeta.modificationDate)); - if (!isNaN(ts)) metaPatch.updatedDate = ts; - } - if (Object.keys(metaPatch).length) { - collection.meta.setDocMeta(doc.id, metaPatch); - } + docs.push({ + id: docId, + snapshot, + meta, + }); + docIds.push(docId); - for (const tag of tags) { - if (!tagDocMap.has(tag)) { - tagDocMap.set(tag, []); - } - tagDocMap.get(tag)!.push(doc.id); + for (const tag of tags) { + if (!tagDocMap.has(tag)) { + tagDocMap.set(tag, []); } + tagDocMap.get(tag)!.push(docId); + } + + for (const blob of await blobsFromAssets( + pendingAssets, + pendingPathBlobIdMap + )) { + importBlobs.set(blob.blobId, blob); } - } catch (err) { - console.warn(`Failed to import bundle: ${entry.bundlePath}`, err); + } catch { + warnings.push({ + code: 'bear-bundle-import-failed', + message: `Failed to import bundle: ${entry.bundlePath}`, + sourcePath: entry.bundlePath, + }); } } const folderHierarchy = buildFolderHierarchyFromTags(tagDocMap); - return { docIds, tags: tagDocMap, folderHierarchy }; + return { + docIds, + tags: tagDocMap, + folderHierarchy, + batch: { + docs, + blobs: Array.from(importBlobs.values()), + folders: flattenFolderHierarchy(folderHierarchy), + tags: tagsToBatchTags(tagDocMap), + warnings, + done: true, + }, + }; } /** Public API for importing Bear .bear2bk backup archives. */ export const BearTransformer = { - importBearBackup, + planBearBackup, }; diff --git a/blocksuite/affine/widgets/linked-doc/src/transformers/import-batch.ts b/blocksuite/affine/widgets/linked-doc/src/transformers/import-batch.ts new file mode 100644 index 0000000000000..d8b37f39d44ed --- /dev/null +++ b/blocksuite/affine/widgets/linked-doc/src/transformers/import-batch.ts @@ -0,0 +1,157 @@ +import { + type DocMeta, + type DocSnapshot, + type Schema, + Transformer, + type Workspace, +} from '@blocksuite/store'; + +export type ImportBlob = { + blobId: string; + sourcePath: string; + fileName: string; + mime: string; + bytes: Uint8Array; +}; + +export type ImportIconData = + | { + type: 'emoji'; + unicode: string; + } + | { + type: 'image'; + content: string; + }; + +export type ImportFolder = { + path: string; + name: string; + parentPath?: string; + pageId?: string; + icon?: ImportIconData; +}; + +export type ImportDoc = { + id: string; + sourcePath?: string; + snapshot: DocSnapshot; + meta?: Partial< + Pick< + DocMeta, + 'title' | 'createDate' | 'updatedDate' | 'tags' | 'favorite' | 'trash' + > + >; +}; + +export type ImportTag = { + name: string; + docIds: string[]; +}; + +export type ImportIcon = { + docId: string; + icon: ImportIconData; +}; + +export type ImportWarning = { + code: string; + message: string; + sourcePath?: string; +}; + +export type ImportBatch = { + docs: ImportDoc[]; + blobs: ImportBlob[]; + folders?: ImportFolder[]; + tags?: ImportTag[]; + icons?: ImportIcon[]; + warnings?: ImportWarning[]; + progress?: { + completed: number; + total: number; + }; + entryId?: string; + isWorkspaceFile?: boolean; + done: boolean; +}; + +export type ImportCommitResult = { + docIds: string[]; + entryId?: string; + isWorkspaceFile?: boolean; + rootFolderId?: string; + warnings: ImportWarning[]; +}; + +export async function blobsFromAssets( + assets: ReadonlyMap, + pathBlobIdMap: ReadonlyMap = new Map() +) { + const sourcePathByBlobId = new Map(); + for (const [path, blobId] of pathBlobIdMap) { + sourcePathByBlobId.set(blobId, path); + } + + return Promise.all( + Array.from( + assets, + async ([blobId, file]): Promise => ({ + blobId, + sourcePath: sourcePathByBlobId.get(blobId) ?? file.name, + fileName: file.name, + mime: file.type, + bytes: new Uint8Array(await file.arrayBuffer()), + }) + ) + ); +} + +function copyToArrayBuffer(bytes: Uint8Array) { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + +export async function commitImportBatchToWorkspace( + collection: Workspace, + schema: Schema, + batch: ImportBatch +): Promise { + for (const blob of batch.blobs) { + await collection.blobSync.set( + blob.blobId, + new File([copyToArrayBuffer(blob.bytes)], blob.fileName, { + type: blob.mime, + }) + ); + } + + const transformer = new Transformer({ + schema, + blobCRUD: collection.blobSync, + docCRUD: { + create: (id: string) => collection.createDoc(id).getStore({ id }), + get: (id: string) => collection.getDoc(id)?.getStore({ id }) ?? null, + delete: (id: string) => collection.removeDoc(id), + }, + middlewares: [], + }); + + const docIds: string[] = []; + for (const doc of batch.docs) { + const store = await transformer.snapshotToDoc(doc.snapshot); + if (!store) continue; + docIds.push(store.id); + if (doc.meta && Object.keys(doc.meta).length) { + collection.meta.setDocMeta(store.id, doc.meta); + } + } + + return { + docIds, + entryId: batch.entryId, + isWorkspaceFile: batch.isWorkspaceFile, + warnings: batch.warnings ?? [], + }; +} diff --git a/blocksuite/affine/widgets/linked-doc/src/transformers/index.ts b/blocksuite/affine/widgets/linked-doc/src/transformers/index.ts index 46152a77ace49..3c7c070fe6cd1 100644 --- a/blocksuite/affine/widgets/linked-doc/src/transformers/index.ts +++ b/blocksuite/affine/widgets/linked-doc/src/transformers/index.ts @@ -1,9 +1,22 @@ export { BearTransformer } from './bear.js'; export { DocxTransformer } from './docx.js'; export { HtmlTransformer } from './html.js'; +export { + blobsFromAssets, + commitImportBatchToWorkspace, + type ImportBatch, + type ImportBlob, + type ImportCommitResult, + type ImportDoc, + type ImportFolder, + type ImportIcon, + type ImportIconData, + type ImportTag, + type ImportWarning, +} from './import-batch.js'; export { MarkdownTransformer } from './markdown.js'; export { NotionHtmlTransformer } from './notion-html.js'; export { ObsidianTransformer } from './obsidian.js'; export { PdfTransformer } from './pdf.js'; -export { createAssetsArchive, download } from './utils.js'; +export { createAssetsArchive, download, Unzip } from './utils.js'; export { ZipTransformer } from './zip.js'; diff --git a/blocksuite/affine/widgets/linked-doc/src/transformers/markdown.ts b/blocksuite/affine/widgets/linked-doc/src/transformers/markdown.ts index c96044d14d58e..1450bb3b67927 100644 --- a/blocksuite/affine/widgets/linked-doc/src/transformers/markdown.ts +++ b/blocksuite/affine/widgets/linked-doc/src/transformers/markdown.ts @@ -16,6 +16,7 @@ import { BlockSuiteError, ErrorCode } from '@blocksuite/global/exceptions'; import { sha } from '@blocksuite/global/utils'; import type { DocMeta, + DocSnapshot, ExtensionType, Schema, Store, @@ -23,6 +24,12 @@ import type { } from '@blocksuite/store'; import { extMimeMap, Transformer } from '@blocksuite/store'; +import { + blobsFromAssets, + type ImportBatch, + type ImportDoc, + type ImportFolder, +} from './import-batch.js'; import type { AssetMap, ImportedFileEntry, PathBlobIdMap } from './type.js'; import { createAssetsArchive, download, parseMatter, Unzip } from './utils.js'; @@ -438,6 +445,14 @@ function prepareNotionMarkdownFile({ }; } +function applySnapshotTitle(snapshot: DocSnapshot, title: string) { + snapshot.meta.title = title; + snapshot.blocks.props.title = { + '$blocksuite:internal:text$': true, + delta: [{ insert: title }], + }; +} + /** * Filters hidden/system entries that should never participate in imports. */ @@ -685,7 +700,7 @@ async function importMarkdownToDoc({ * @param options.imported The zip file as a Blob * @returns A Promise that resolves to an array of IDs of the newly created docs */ -type FolderHierarchy = { +export type FolderHierarchy = { name: string; path: string; children: Map; @@ -693,18 +708,19 @@ type FolderHierarchy = { parentPath?: string; }; -export type ImportMarkdownZipResult = { +export type PlanMarkdownZipResult = { docIds: string[]; folderHierarchy?: FolderHierarchy; + batch: ImportBatch; }; -async function importMarkdownZip({ +async function planMarkdownZip({ collection, schema, imported, extensions, -}: ImportMarkdownZipOptions): Promise { - return importMarkdownZipInternal({ +}: ImportMarkdownZipOptions): Promise { + return planMarkdownZipInternal({ collection, schema, imported, @@ -712,13 +728,13 @@ async function importMarkdownZip({ }); } -async function importNotionMarkdownZip({ +async function planNotionMarkdownZip({ collection, schema, imported, extensions, -}: ImportMarkdownZipOptions): Promise { - return importMarkdownZipInternal({ +}: ImportMarkdownZipOptions): Promise { + return planMarkdownZipInternal({ collection, schema, imported, @@ -731,7 +747,7 @@ async function importNotionMarkdownZip({ }); } -async function importMarkdownZipInternal({ +async function planMarkdownZipInternal({ collection, schema, imported, @@ -741,12 +757,13 @@ async function importMarkdownZipInternal({ prepareMarkdownFile = prepareDefaultMarkdownFile, preserveCommonRoot = false, recursiveZip = false, -}: ImportMarkdownZipInternalOptions): Promise { +}: ImportMarkdownZipInternalOptions): Promise { const provider = getProvider([ markdownZipDocLinkToDeltaMatcher, ...extensions, ]); const docIds: string[] = []; + const docs: ImportDoc[] = []; const pendingAssets: AssetMap = new Map(); const pendingPathBlobIdMap: PathBlobIdMap = new Map(); const docPathMap: Array<{ fullPath: string; docId: string }> = []; @@ -812,16 +829,20 @@ async function importMarkdownZipInternal({ assets: job.assetsManager, }); snapshot.meta.id = pageId; - const doc = await job.snapshotToDoc(snapshot); - if (doc) { - applyMetaPatch(collection, doc.id, meta); - docIds.push(doc.id); - docPathMap.push({ fullPath, docId: doc.id }); - } + applySnapshotTitle(snapshot, preferredTitle); + docs.push({ + id: pageId, + snapshot, + meta: { + ...meta, + title: preferredTitle, + }, + }); + docIds.push(pageId); + docPathMap.push({ fullPath, docId: pageId }); }) ); - // Build folder hierarchy from zip paths const folderHierarchy = buildMarkdownZipFolderHierarchy( docPathMap, normalizeFolderName, @@ -829,7 +850,18 @@ async function importMarkdownZipInternal({ createRootFolderForTopLevelDocs ); - return { docIds, folderHierarchy }; + return { + docIds, + folderHierarchy, + batch: { + docs, + blobs: await blobsFromAssets(pendingAssets, pendingPathBlobIdMap), + folders: folderHierarchy + ? flattenFolderHierarchy(folderHierarchy) + : undefined, + done: true, + }, + }; } /** @@ -921,10 +953,34 @@ function buildMarkdownZipFolderHierarchy( return root.children.size > 0 ? root : undefined; } +function flattenFolderHierarchy(root: FolderHierarchy): ImportFolder[] { + const folders: ImportFolder[] = []; + + const visit = (node: FolderHierarchy) => { + if (node.name) { + folders.push({ + path: node.path, + name: node.name, + parentPath: node.parentPath, + pageId: node.pageId, + }); + } + for (const child of node.children.values()) { + visit(child); + } + }; + + for (const child of root.children.values()) { + visit(child); + } + + return folders; +} + export const MarkdownTransformer = { exportDoc, importMarkdownToBlock, importMarkdownToDoc, - importMarkdownZip, - importNotionMarkdownZip, + planMarkdownZip, + planNotionMarkdownZip, }; diff --git a/blocksuite/affine/widgets/linked-doc/src/transformers/notion-html.ts b/blocksuite/affine/widgets/linked-doc/src/transformers/notion-html.ts index 1febb99812ba7..277b97349a307 100644 --- a/blocksuite/affine/widgets/linked-doc/src/transformers/notion-html.ts +++ b/blocksuite/affine/widgets/linked-doc/src/transformers/notion-html.ts @@ -12,6 +12,14 @@ import { type Workspace, } from '@blocksuite/store'; +import { + blobsFromAssets, + type ImportBatch, + type ImportDoc, + type ImportFolder, + type ImportIconData, + type ImportWarning, +} from './import-batch.js'; import { Unzip } from './utils.js'; type ImportNotionZipOptions = { @@ -21,12 +29,12 @@ type ImportNotionZipOptions = { extensions: ExtensionType[]; }; -type PageIcon = { +export type PageIcon = { type: 'emoji' | 'image'; - content: string; // emoji unicode or image URL/data + content: string; }; -type FolderHierarchy = { +export type FolderHierarchy = { name: string; path: string; children: Map; @@ -35,12 +43,13 @@ type FolderHierarchy = { icon?: PageIcon; }; -type ImportNotionZipResult = { +export type PlanNotionHtmlZipResult = { entryId: string | undefined; pageIds: string[]; isWorkspaceFile: boolean; hasMarkdown: boolean; folderHierarchy?: FolderHierarchy; + batch: ImportBatch; }; function getProvider(extensions: ExtensionType[]) { @@ -61,22 +70,9 @@ function parseFolderPath(filePath: string): { } function extractPageIcon(doc: Document): PageIcon | undefined { - // Look for Notion page icon in the HTML - // Notion export format:
- - console.log('=== Extracting page icon ==='); - - // Check if there's a head section with title for debugging - const headTitle = doc.querySelector('head title'); - if (headTitle) { - console.log('Page title from head:', headTitle.textContent); - } - - // Look for the exact Notion export structure: .page-header-icon .icon const notionIconSpan = doc.querySelector('.page-header-icon .icon'); if (notionIconSpan && notionIconSpan.textContent) { const iconContent = notionIconSpan.textContent.trim(); - console.log('Found Notion icon (.page-header-icon .icon):', iconContent); if (/\p{Emoji}/u.test(iconContent)) { return { type: 'emoji', @@ -85,50 +81,29 @@ function extractPageIcon(doc: Document): PageIcon | undefined { } } - // Look for page header area for debugging - const pageHeader = doc.querySelector('.page-header-icon'); - if (pageHeader) { - console.log( - 'Found .page-header-icon:', - pageHeader.outerHTML.substring(0, 300) + '...' - ); - } - - // Fallback: try to find emoji icons with older selectors const emojiIcon = doc.querySelector('.page-header-icon .notion-emoji'); if (emojiIcon && emojiIcon.textContent) { - console.log( - 'Found emoji icon (.page-header-icon .notion-emoji):', - emojiIcon.textContent - ); return { type: 'emoji', content: emojiIcon.textContent.trim(), }; } - // Try alternative emoji selectors const altEmojiIcon = doc.querySelector('[role="img"][aria-label]'); if ( altEmojiIcon && altEmojiIcon.textContent && /\p{Emoji}/u.test(altEmojiIcon.textContent) ) { - console.log( - 'Found emoji icon ([role="img"][aria-label]):', - altEmojiIcon.textContent - ); return { type: 'emoji', content: altEmojiIcon.textContent.trim(), }; } - // Look for image icons in the page header const imageIcon = doc.querySelector('.page-header-icon img'); if (imageIcon) { const src = imageIcon.getAttribute('src'); - console.log('Found image icon (.page-header-icon img):', src); if (src) { return { type: 'image', @@ -137,27 +112,15 @@ function extractPageIcon(doc: Document): PageIcon | undefined { } } - // Fallback: Look for any span with emoji class "icon" in page header area const iconSpans = doc.querySelectorAll('span.icon'); for (const span of iconSpans) { if (span.textContent && /\p{Emoji}/u.test(span.textContent.trim())) { const parent = span.parentElement; - console.log( - 'Found emoji in span.icon:', - span.textContent, - 'parent classes:', - parent?.className - ); - // Check if this is in a page header context if ( parent && (parent.classList.contains('page-header-icon') || parent.closest('.page-header-icon')) ) { - console.log( - 'Using emoji from span.icon in page header:', - span.textContent - ); return { type: 'emoji', content: span.textContent.trim(), @@ -166,15 +129,11 @@ function extractPageIcon(doc: Document): PageIcon | undefined { } } - // Fallback: Try to find icons in the page title area that might contain emoji const pageTitle = doc.querySelector('.page-title, h1'); if (pageTitle && pageTitle.textContent) { - console.log('Page title element found:', pageTitle.textContent); const text = pageTitle.textContent.trim(); - // Check if the title starts with an emoji const emojiMatch = text.match(/^(\p{Emoji}+)/u); if (emojiMatch) { - console.log('Found emoji in title:', emojiMatch[1]); return { type: 'emoji', content: emojiMatch[1], @@ -182,7 +141,6 @@ function extractPageIcon(doc: Document): PageIcon | undefined { } } - console.log('No page icon found'); return undefined; } @@ -200,7 +158,6 @@ function buildFolderHierarchy( let current = root; let currentPath = ''; - // Navigate/create folder structure for (const folderName of folderParts) { const parentPath = currentPath; currentPath = currentPath ? `${currentPath}/${folderName}` : folderName; @@ -216,7 +173,6 @@ function buildFolderHierarchy( current = current.children.get(folderName)!; } - // If this is a page file, associate it with the current folder if (fileName.endsWith('.html') && !fileName.startsWith('index.html')) { const pageName = fileName.replace(/\.html$/, ''); if (!current.children.has(pageName)) { @@ -229,7 +185,6 @@ function buildFolderHierarchy( icon: icon, }); } else { - // Update existing entry with pageId and icon const existingPage = current.children.get(pageName)!; existingPage.pageId = pageId; if (icon) { @@ -242,29 +197,59 @@ function buildFolderHierarchy( return root; } -/** - * Imports a Notion zip file into the BlockSuite collection. - * - * @param options - The options for importing. - * @param options.collection - The BlockSuite document collection. - * @param options.schema - The schema of the BlockSuite document collection. - * @param options.imported - The imported zip file as a Blob. - * - * @returns A promise that resolves to an object containing: - * - entryId: The ID of the entry page (if any). - * - pageIds: An array of imported page IDs. - * - isWorkspaceFile: Whether the imported file is a workspace file. - * - hasMarkdown: Whether the zip contains markdown files. - * - folderHierarchy: The parsed folder hierarchy from the Notion export. - */ -async function importNotionZip({ +function toImportIconData(icon?: PageIcon): ImportIconData | undefined { + if (!icon) return undefined; + if (icon.type === 'emoji') { + return { + type: 'emoji', + unicode: icon.content, + }; + } + return { + type: 'image', + content: icon.content, + }; +} + +function flattenFolderHierarchy(root: FolderHierarchy): ImportFolder[] { + const folders: ImportFolder[] = []; + + const visit = (node: FolderHierarchy) => { + if (node.name) { + folders.push({ + path: node.path, + name: node.name, + parentPath: node.parentPath, + pageId: node.pageId, + icon: toImportIconData(node.icon), + }); + } + for (const child of node.children.values()) { + visit(child); + } + }; + + for (const child of root.children.values()) { + visit(child); + } + + return folders; +} + +async function planNotionHtmlZip({ collection, schema, imported, extensions, -}: ImportNotionZipOptions): Promise { +}: ImportNotionZipOptions): Promise { const provider = getProvider(extensions); const pageIds: string[] = []; + const docs: ImportDoc[] = []; + const blobs = new Map< + string, + Awaited>[0] + >(); + const warnings: ImportWarning[] = []; let isWorkspaceFile = false; let hasMarkdown = false; let entryId: string | undefined; @@ -280,7 +265,7 @@ async function importNotionZip({ const pageMap = new Map(); const pagePaths: string[] = []; const promises: Promise[] = []; - const pendingAssets = new Map(); + const pendingAssets = new Map(); const pendingPathBlobIdMap = new Map(); for (const { path, content, index } of unzip) { if (path.startsWith('__MACOSX/')) continue; @@ -305,11 +290,7 @@ async function importNotionZip({ const text = await content.text(); const doc = new DOMParser().parseFromString(text, 'text/html'); const pageBody = doc.querySelector('.page-body'); - if (pageBody && pageBody.children.length === 0) { - // Skip empty pages - continue; - } - // Extract page icon from the HTML + if (pageBody && pageBody.children.length === 0) continue; pageIcon = extractPageIcon(doc); } @@ -327,10 +308,12 @@ async function importNotionZip({ continue; } if (index === 0 && fileName.endsWith('.csv')) { - window.open( - 'https://affine.pro/blog/import-your-data-from-notion-into-affine', - '_blank' - ); + warnings.push({ + code: 'notion-csv-export', + message: + 'The imported Notion export appears to be CSV instead of HTML.', + sourcePath: path, + }); continue; } if (fileName.endsWith('.zip')) { @@ -342,7 +325,7 @@ async function importNotionZip({ } const blob = content; const ext = path.split('.').at(-1) ?? ''; - const mime = extMimeMap.get(ext) ?? ''; + const mime = extMimeMap.get(ext.toLowerCase()) ?? ''; const key = await sha(await blob.arrayBuffer()); const filePathSplit = path.split('/'); while (filePathSplit.length > 1) { @@ -375,32 +358,57 @@ async function importNotionZip({ pathBlobIdMap.set(key, value); } } - const page = await htmlAdapter.toDoc({ + const snapshot = await htmlAdapter.toDocSnapshot({ file: await zipFile.get(path)!.text(), pageId: pageMap.get(path), pageMap, assets: job.assetsManager, }); - if (page) { - pageIds.push(page.id); - } + docs.push({ + id: snapshot.meta.id, + snapshot, + }); + pageIds.push(snapshot.meta.id); }); promises.push(...pagePromises); + promises.push( + blobsFromAssets(pendingAssets, pendingPathBlobIdMap).then(importBlobs => { + for (const blob of importBlobs) { + blobs.set(blob.blobId, blob); + } + }) + ); return promises; }; const allPromises = await parseZipFile(imported); await Promise.all(allPromises.flat()); entryId = entryId ?? pageIds[0]; - // Build folder hierarchy from collected paths const folderHierarchy = pagePathsWithIds.length > 0 ? buildFolderHierarchy(pagePathsWithIds) : undefined; - return { entryId, pageIds, isWorkspaceFile, hasMarkdown, folderHierarchy }; + return { + entryId, + pageIds, + isWorkspaceFile, + hasMarkdown, + folderHierarchy, + batch: { + docs, + blobs: Array.from(blobs.values()), + folders: folderHierarchy + ? flattenFolderHierarchy(folderHierarchy) + : undefined, + warnings, + entryId, + isWorkspaceFile, + done: true, + }, + }; } export const NotionHtmlTransformer = { - importNotionZip, + planNotionHtmlZip, }; diff --git a/blocksuite/affine/widgets/linked-doc/src/transformers/obsidian.ts b/blocksuite/affine/widgets/linked-doc/src/transformers/obsidian.ts index c5688caa70d7e..1ce78a48082dd 100644 --- a/blocksuite/affine/widgets/linked-doc/src/transformers/obsidian.ts +++ b/blocksuite/affine/widgets/linked-doc/src/transformers/obsidian.ts @@ -20,7 +20,11 @@ import { extMimeMap, nanoid } from '@blocksuite/store'; import type { Html, Text } from 'mdast'; import { - applyMetaPatch, + blobsFromAssets, + type ImportBatch, + type ImportDoc, +} from './import-batch.js'; +import { bindImportedAssetsToJob, createMarkdownImportJob, getProvider, @@ -129,8 +133,8 @@ function preprocessTitleHeader(markdown: string): string { function preprocessObsidianCallouts(markdown: string): string { return markdown.replace( - /^(> *)\[!([^\]\n]+)\]([+-]?)([^\n]*)/gm, - (_, prefix, type, _fold, rest) => { + /^(> *)\[!([^\]\n]+)\](?:[+-]?)([^\n]*)/gm, + (_, prefix, type, rest) => { const calloutToken = CALLOUT_TYPE_MAP[type.trim().toLowerCase()] ?? DEFAULT_CALLOUT_EMOJI; const title = rest.trim(); @@ -688,12 +692,16 @@ export type ImportObsidianVaultResult = { docEmojis: Map; }; -export async function importObsidianVault({ +export type PlanObsidianVaultResult = ImportObsidianVaultResult & { + batch: ImportBatch; +}; + +export async function planObsidianVault({ collection, schema, importedFiles, extensions, -}: ImportObsidianVaultOptions): Promise { +}: ImportObsidianVaultOptions): Promise { const provider = getProvider([ obsidianWikilinkToDeltaMatcher, obsidianAttachmentEmbedMarkdownAdapterMatcher, @@ -701,6 +709,7 @@ export async function importObsidianVault({ ]); const docIds: string[] = []; + const docs: ImportDoc[] = []; const docEmojis = new Map(); const pendingAssets: AssetMap = new Map(); const pendingPathBlobIdMap: PathBlobIdMap = new Map(); @@ -813,22 +822,31 @@ export async function importObsidianVault({ if (snapshot) { snapshot.meta.id = predefinedId; - const doc = await job.snapshotToDoc(snapshot); - if (doc) { - applyMetaPatch(collection, doc.id, { - ...meta, - title: preferredTitle, - trash: false, - }); - docIds.push(doc.id); - } + docs.push({ + id: predefinedId, + snapshot, + meta: { ...meta, title: preferredTitle, trash: false }, + }); + docIds.push(predefinedId); } }) ); - return { docIds, docEmojis }; + return { + docIds, + docEmojis, + batch: { + docs, + blobs: await blobsFromAssets(pendingAssets, pendingPathBlobIdMap), + icons: Array.from(docEmojis, ([docId, emoji]) => ({ + docId, + icon: { type: 'emoji', unicode: emoji }, + })), + done: true, + }, + }; } export const ObsidianTransformer = { - importObsidianVault, + planObsidianVault, }; diff --git a/blocksuite/playground/apps/_common/components/starter-debug-menu.ts b/blocksuite/playground/apps/_common/components/starter-debug-menu.ts index 186b17a81ee06..4f00ceb729f5e 100644 --- a/blocksuite/playground/apps/_common/components/starter-debug-menu.ts +++ b/blocksuite/playground/apps/_common/components/starter-debug-menu.ts @@ -54,6 +54,7 @@ import { type Workspace, } from '@blocksuite/affine/store'; import { + commitImportBatchToWorkspace, createAssetsArchive, download, HtmlTransformer, @@ -436,12 +437,17 @@ export class StarterDebugMenu extends ShadowlessElement { try { const file = await openSingleFileWith('Zip'); if (!file) return; - const { docIds } = await MarkdownTransformer.importMarkdownZip({ + const { batch } = await MarkdownTransformer.planMarkdownZip({ collection: this.collection, schema: this.editor.doc.schema, imported: file, extensions: this._getStoreManager().get('store'), }); + const { docIds } = await commitImportBatchToWorkspace( + this.collection, + this.editor.doc.schema, + batch + ); if (!this.editor.host) return; toast( this.editor.host, @@ -473,16 +479,21 @@ export class StarterDebugMenu extends ShadowlessElement { try { const file = await openSingleFileWith('Zip'); if (!file) return; - const result = await NotionHtmlTransformer.importNotionZip({ + const { batch } = await NotionHtmlTransformer.planNotionHtmlZip({ collection: this.collection, schema: this.editor.doc.schema, imported: file, extensions: this._getStoreManager().get('store'), }); + const result = await commitImportBatchToWorkspace( + this.collection, + this.editor.doc.schema, + batch + ); if (!this.editor.host) return; toast( this.editor.host, - `Successfully imported ${result.pageIds.length} Notion HTML pages.` + `Successfully imported ${result.docIds.length} Notion HTML pages.` ); } catch (error) { console.error('Failed to import Notion HTML Zip:', error); diff --git a/packages/backend/native/Cargo.toml b/packages/backend/native/Cargo.toml index 40fa7f95028a3..0e8e087452dc6 100644 --- a/packages/backend/native/Cargo.toml +++ b/packages/backend/native/Cargo.toml @@ -10,11 +10,8 @@ crate-type = ["cdylib"] [dependencies] aes-gcm = { workspace = true } -affine_common = { workspace = true, features = [ - "hashcash", - "napi", - "ydoc-loader", -] } +affine_common = { workspace = true, features = ["hashcash", "napi"] } +affine_doc_loader = { workspace = true } anyhow = { workspace = true } assetpack-core = "0.1.0" assetpack-transform-precomp2 = "0.1.0" diff --git a/packages/backend/native/src/doc.rs b/packages/backend/native/src/doc.rs index 6dca32cc90c8b..22e8b7f73ffa1 100644 --- a/packages/backend/native/src/doc.rs +++ b/packages/backend/native/src/doc.rs @@ -1,6 +1,6 @@ -use affine_common::{ - doc_parser::{self, BlockInfo, CrawlResult, MarkdownResult, PageDocContent, WorkspaceDocContent}, - napi_utils::map_napi_err, +use affine_common::napi_utils::map_napi_err; +use affine_doc_loader::{ + self as doc_loader, BlockInfo, CrawlResult, MarkdownResult, PageDocContent, WorkspaceDocContent, }; use napi::bindgen_prelude::*; use napi_derive::napi; @@ -109,7 +109,7 @@ impl From for NativeCrawlResult { #[napi] pub fn parse_doc_from_binary(doc_bin: Buffer, doc_id: String) -> Result { let result = map_napi_err( - doc_parser::parse_doc_from_binary(doc_bin.into(), doc_id), + doc_loader::parse_doc_from_binary(doc_bin.into(), doc_id), Status::GenericFailure, )?; Ok(result.into()) @@ -118,7 +118,7 @@ pub fn parse_doc_from_binary(doc_bin: Buffer, doc_id: String) -> Result) -> Result> { let result = map_napi_err( - doc_parser::parse_page_doc(doc_bin.into(), max_summary_length.map(|v| v as isize)), + doc_loader::parse_page_doc(doc_bin.into(), max_summary_length.map(|v| v as isize)), Status::GenericFailure, )?; Ok(result.map(Into::into)) @@ -126,7 +126,7 @@ pub fn parse_page_doc(doc_bin: Buffer, max_summary_length: Option) -> Resul #[napi] pub fn parse_workspace_doc(doc_bin: Buffer) -> Result> { - let result = map_napi_err(doc_parser::parse_workspace_doc(doc_bin.into()), Status::GenericFailure)?; + let result = map_napi_err(doc_loader::parse_workspace_doc(doc_bin.into()), Status::GenericFailure)?; Ok(result.map(Into::into)) } @@ -138,7 +138,7 @@ pub fn parse_doc_to_markdown( doc_url_prefix: Option, ) -> Result { let result = map_napi_err( - doc_parser::parse_doc_to_markdown(doc_bin.into(), doc_id, ai_editable.unwrap_or(false), doc_url_prefix), + doc_loader::parse_doc_to_markdown(doc_bin.into(), doc_id, ai_editable.unwrap_or(false), doc_url_prefix), Status::GenericFailure, )?; Ok(result.into()) @@ -147,7 +147,7 @@ pub fn parse_doc_to_markdown( #[napi] pub fn read_all_doc_ids_from_root_doc(doc_bin: Buffer, include_trash: Option) -> Result> { let result = map_napi_err( - doc_parser::get_doc_ids_from_binary(doc_bin.into(), include_trash.unwrap_or(false)), + doc_loader::get_doc_ids_from_binary(doc_bin.into(), include_trash.unwrap_or(false)), Status::GenericFailure, )?; Ok(result) @@ -165,7 +165,7 @@ pub fn read_all_doc_ids_from_root_doc(doc_bin: Buffer, include_trash: Option Result { let result = map_napi_err( - doc_parser::build_full_doc(&title, &markdown, &doc_id), + doc_loader::build_full_doc(&title, &markdown, &doc_id), Status::GenericFailure, )?; Ok(Buffer::from(result)) @@ -184,7 +184,7 @@ pub fn create_doc_with_markdown(title: String, markdown: String, doc_id: String) #[napi] pub fn update_doc_with_markdown(existing_binary: Buffer, new_markdown: String, doc_id: String) -> Result { let result = map_napi_err( - doc_parser::update_doc(&existing_binary, &new_markdown, &doc_id), + doc_loader::update_doc(&existing_binary, &new_markdown, &doc_id), Status::GenericFailure, )?; Ok(Buffer::from(result)) @@ -202,7 +202,7 @@ pub fn update_doc_with_markdown(existing_binary: Buffer, new_markdown: String, d #[napi] pub fn update_doc_title(existing_binary: Buffer, title: String, doc_id: String) -> Result { let result = map_napi_err( - doc_parser::update_doc_title(&existing_binary, &doc_id, &title), + doc_loader::update_doc_title(&existing_binary, &doc_id, &title), Status::GenericFailure, )?; Ok(Buffer::from(result)) @@ -229,7 +229,7 @@ pub fn update_doc_properties( updated_by: Option, ) -> Result { let result = map_napi_err( - doc_parser::update_doc_properties( + doc_loader::update_doc_properties( &existing_binary, &properties_doc_id, &target_doc_id, @@ -254,7 +254,7 @@ pub fn update_doc_properties( #[napi] pub fn add_doc_to_root_doc(root_doc_bin: Buffer, doc_id: String, title: Option) -> Result { let result = map_napi_err( - doc_parser::add_doc_to_root_doc(root_doc_bin.into(), &doc_id, title.as_deref()), + doc_loader::add_doc_to_root_doc(root_doc_bin.into(), &doc_id, title.as_deref()), Status::GenericFailure, )?; Ok(Buffer::from(result)) @@ -267,7 +267,7 @@ pub fn build_public_root_doc(root_doc_bin: Buffer, doc_metas: Vec>(); let result = map_napi_err( - doc_parser::build_public_root_doc(&root_doc_bin, &metas), + doc_loader::build_public_root_doc(&root_doc_bin, &metas), Status::GenericFailure, )?; Ok(Buffer::from(result)) @@ -285,7 +285,7 @@ pub fn build_public_root_doc(root_doc_bin: Buffer, doc_metas: Vec Result { let result = map_napi_err( - doc_parser::update_root_doc_meta_title(&root_doc_bin, &doc_id, &title), + doc_loader::update_root_doc_meta_title(&root_doc_bin, &doc_id, &title), Status::GenericFailure, )?; Ok(Buffer::from(result)) diff --git a/packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs b/packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs index 2b223c12709c5..fee91800552ab 100644 --- a/packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs +++ b/packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs @@ -1,4 +1,4 @@ -use affine_common::doc_parser; +use affine_doc_loader as doc_loader; use chrono::{DateTime, Utc}; use sqlx::{FromRow, PgPool}; use y_octo::Doc; @@ -102,7 +102,7 @@ async fn load_workspace_doc_ids(pool: &PgPool, workspace_id: &str) -> RuntimeRes let Some(root) = load_current_doc(pool, workspace_id, workspace_id).await? else { return Ok(Vec::new()); }; - let ids = doc_parser::get_doc_ids_from_binary(root.blob, false) + let ids = doc_loader::get_doc_ids_from_binary(root.blob, false) .map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs root doc parse failed: {err}")))?; let mut ids = ids; ids.sort(); @@ -206,7 +206,7 @@ async fn purge_removed_doc_refs(pool: &PgPool, workspace_id: &str, current_doc_i } fn extract_refs(snapshot: &SnapshotRow) -> RuntimeResult> { - let parsed = doc_parser::parse_doc_from_binary(snapshot.blob.clone(), snapshot.doc_id.clone()) + let parsed = doc_loader::parse_doc_from_binary(snapshot.blob.clone(), snapshot.doc_id.clone()) .map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs parse failed: {err}")))?; let mut refs = Vec::new(); for block in parsed.blocks { @@ -232,7 +232,7 @@ mod tests { fn doc_blob_refs_extracts_image_refs() { let doc_id = "doc-blob-ref-test".to_string(); let blob = - doc_parser::build_full_doc("Doc", "![Alt](blob://image-blob-key)", &doc_id).expect("doc fixture should build"); + doc_loader::build_full_doc("Doc", "![Alt](blob://image-blob-key)", &doc_id).expect("doc fixture should build"); let snapshot = SnapshotRow { workspace_id: "workspace".to_string(), doc_id, diff --git a/packages/backend/server/src/__tests__/models/user.spec.ts b/packages/backend/server/src/__tests__/models/user.spec.ts index 1f8707d5d6bc2..c126db0fe146f 100644 --- a/packages/backend/server/src/__tests__/models/user.spec.ts +++ b/packages/backend/server/src/__tests__/models/user.spec.ts @@ -1,4 +1,5 @@ -import ava, { TestFn } from 'ava'; +import { PrismaClient } from '@prisma/client'; +import ava, { type TestFn } from 'ava'; import Sinon from 'sinon'; import { EmailAlreadyUsed, EventBus } from '../../base'; @@ -9,6 +10,7 @@ import { createTestingModule, sleep, type TestingModule } from '../utils'; interface Context { module: TestingModule; models: Models; + db: PrismaClient; user: UserModel; } @@ -19,6 +21,7 @@ test.before(async t => { t.context.user = module.get(UserModel); t.context.models = module.get(Models); + t.context.db = module.get(PrismaClient); t.context.module = module; }); @@ -262,6 +265,38 @@ test('should delete user', async t => { t.is(user2, null); }); +test('should delete user with pending invitation missing normalized email', async t => { + const owner = await t.context.user.create({ + email: 'owner@affine.pro', + }); + const invitee = await t.context.user.create({ + email: 'invitee@affine.pro', + registered: false, + }); + const workspace = await t.context.models.workspace.create(owner.id); + + const invitation = await t.context.db.workspaceInvitation.create({ + data: { + workspaceId: workspace.id, + inviteeUserId: invitee.id, + inviterUserId: owner.id, + requestedRole: 'member', + status: 'pending', + kind: 'email', + }, + }); + + await t.context.user.delete(invitee.id); + + t.is( + await t.context.db.workspaceInvitation.findUnique({ + where: { id: invitation.id }, + }), + null + ); + t.is(await t.context.user.get(invitee.id), null); +}); + test('should trigger user.deleted event', async t => { const event = t.context.module.get(EventBus); const spy = Sinon.spy(); diff --git a/packages/backend/server/src/core/mail/__tests__/job.spec.ts b/packages/backend/server/src/core/mail/__tests__/job.spec.ts index 9e2283f1b769a..5bad0da05d733 100644 --- a/packages/backend/server/src/core/mail/__tests__/job.spec.ts +++ b/packages/backend/server/src/core/mail/__tests__/job.spec.ts @@ -38,6 +38,7 @@ test('should clear pending mail records when user is deleted', async t => { const userKey = `${sendMailKey}:SignIn:${user.email}`; const userRetryKey = `${sendMailKey}:VerifyEmail:${user.email}`; const anotherKey = `${sendMailKey}:SignIn:${another.email}`; + const senderRetryKey = `${retryMailKey}:MemberInvitation:invited@affine.pro`; await cache.mapSet(sendMailKey, userKey, 1); await cache.mapSet(sendMailKey, anotherKey, 1); @@ -51,6 +52,20 @@ test('should clear pending mail records when user is deleted', async t => { props: { url: 'https://affine.pro/verify' }, }) ); + await cache.mapSet( + retryMailKey, + senderRetryKey, + JSON.stringify({ + startTime: Date.now(), + name: 'MemberInvitation', + to: 'invited@affine.pro', + props: { + user: { $$userId: user.id }, + workspace: { $$workspaceId: 'workspace-id' }, + url: 'https://affine.pro/invite', + }, + }) + ); await mailJob.onUserDeleted({ ...user, ownedWorkspaces: [] }); @@ -62,6 +77,17 @@ test('should clear pending mail records when user is deleted', async t => { to: user.email, } as Jobs['notification.sendMail']) ); + t.true( + await shouldRemove({ + name: 'MemberInvitation', + to: 'invited@affine.pro', + props: { + user: { $$userId: user.id }, + workspace: { $$workspaceId: 'workspace-id' }, + url: 'https://affine.pro/invite', + }, + } as Jobs['notification.sendMail']) + ); t.false( await shouldRemove({ to: another.email, @@ -69,6 +95,7 @@ test('should clear pending mail records when user is deleted', async t => { ); t.is(await cache.mapGet(sendMailKey, userKey), undefined); t.is(await cache.mapGet(retryMailKey, userRetryKey), undefined); + t.is(await cache.mapGet(retryMailKey, senderRetryKey), undefined); t.is(await cache.mapGet(sendMailKey, anotherKey), 1); }); diff --git a/packages/backend/server/src/core/mail/job.ts b/packages/backend/server/src/core/mail/job.ts index 92dd2e5377625..4b21d00f75e04 100644 --- a/packages/backend/server/src/core/mail/job.ts +++ b/packages/backend/server/src/core/mail/job.ts @@ -126,6 +126,38 @@ export class MailJob { ); } + private isInvitationMailSentByUser( + job: Jobs['notification.sendMail'], + userId: string + ) { + return ( + job.name === 'MemberInvitation' && + 'user' in job.props && + typeof job.props.user === 'object' && + job.props.user !== null && + '$$userId' in job.props.user && + job.props.user.$$userId === userId + ); + } + + private async deleteInvitationMailCacheBySender(userId: string) { + const keys = await this.cache.mapKeys(retryMailKey); + await Promise.all( + keys.map(async key => { + const job = await this.cache.mapGet< + Jobs['notification.sendMail'] | string + >(retryMailKey, key); + const jobData = + typeof job === 'string' + ? (JSON.parse(job) as Jobs['notification.sendMail']) + : job; + if (jobData && this.isInvitationMailSentByUser(jobData, userId)) { + await this.cache.mapDelete(retryMailKey, key); + } + }) + ); + } + private async sendMailInternal({ startTime, name, @@ -294,9 +326,11 @@ export class MailJob { async onUserDeleted(user: Events['user.deleted']) { await Promise.all([ this.deleteRecipientMailCache(user.email), + this.deleteInvitationMailCacheBySender(user.id), this.queue.removeWhere( 'notification.sendMail', - job => job.to === user.email + job => + job.to === user.email || this.isInvitationMailSentByUser(job, user.id) ), ]); } diff --git a/packages/backend/server/src/models/user.ts b/packages/backend/server/src/models/user.ts index bf21d107ab7a3..7e7b8fe0cd6dc 100644 --- a/packages/backend/server/src/models/user.ts +++ b/packages/backend/server/src/models/user.ts @@ -284,6 +284,10 @@ export class UserModel extends BaseModel { } } + await this.db.workspaceInvitation.deleteMany({ + where: { inviteeUserId: id }, + }); + const user = await this.db.user.delete({ where: { id } }); this.event.emit('user.deleted', { diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index 897526ae2de53..01d3e34a9e406 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -2959,4 +2959,4 @@ type tokenType { refresh: String! sessionToken: String token: String! -} +} \ No newline at end of file diff --git a/packages/common/native/Cargo.toml b/packages/common/native/Cargo.toml index b4f52125c14a2..bb04923aee29b 100644 --- a/packages/common/native/Cargo.toml +++ b/packages/common/native/Cargo.toml @@ -6,32 +6,16 @@ publish = false version = "0.1.0" [features] -default = [] +default = [] hashcash = ["chrono", "hex", "sha3", "rand"] -napi = ["dep:napi"] -ydoc-loader = [ - "assert-json-diff", - "nanoid", - "pulldown-cmark", - "serde", - "serde_json", - "thiserror", - "y-octo", -] +napi = ["dep:napi"] [dependencies] -assert-json-diff = { workspace = true, optional = true } -chrono = { workspace = true, optional = true } -hex = { workspace = true, optional = true } -nanoid = { workspace = true, optional = true } -napi = { workspace = true, optional = true } -pulldown-cmark = { workspace = true, optional = true } -rand = { workspace = true, optional = true } -serde = { workspace = true, optional = true, features = ["derive"] } -serde_json = { workspace = true, optional = true } -sha3 = { workspace = true, optional = true } -thiserror = { workspace = true, optional = true } -y-octo = { workspace = true, optional = true } +chrono = { workspace = true, optional = true } +hex = { workspace = true, optional = true } +napi = { workspace = true, optional = true } +rand = { workspace = true, optional = true } +sha3 = { workspace = true, optional = true } [dev-dependencies] criterion = { workspace = true } diff --git a/packages/common/native/src/doc_parser/block_spec.rs b/packages/common/native/src/doc_parser/block_spec.rs deleted file mode 100644 index e0195bd689cfd..0000000000000 --- a/packages/common/native/src/doc_parser/block_spec.rs +++ /dev/null @@ -1,601 +0,0 @@ -use y_octo::{Any, Map, TextAttributes, TextDeltaOp, TextInsert}; - -use super::{ - ParseError, - blocksuite::get_string, - schema::{ - PROP_CAPTION, PROP_CHECKED, PROP_COLUMN_ID_SUFFIX, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER, - PROP_ORDER_SUFFIX, PROP_ROW_ID_SUFFIX, PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL, - PROP_VIDEO_ID, PROP_WIDTH, SYS_FLAVOUR, table_cell_text_key, - }, - table::{MarkdownTableOptions, render_markdown_table}, - value::{value_to_f64, value_to_string}, -}; - -/// Block flavours used in AFFiNE documents. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum BlockFlavour { - Paragraph, - List, - Code, - Divider, - Image, - Table, - Bookmark, - EmbedYoutube, - EmbedIframe, - Callout, -} - -impl BlockFlavour { - pub fn as_str(&self) -> &'static str { - match self { - BlockFlavour::Paragraph => "affine:paragraph", - BlockFlavour::List => "affine:list", - BlockFlavour::Code => "affine:code", - BlockFlavour::Divider => "affine:divider", - BlockFlavour::Image => "affine:image", - BlockFlavour::Table => "affine:table", - BlockFlavour::Bookmark => "affine:bookmark", - BlockFlavour::EmbedYoutube => "affine:embed-youtube", - BlockFlavour::EmbedIframe => "affine:embed-iframe", - BlockFlavour::Callout => "affine:callout", - } - } - - pub fn from_str(value: &str) -> Option { - match value { - "affine:paragraph" => Some(BlockFlavour::Paragraph), - "affine:list" => Some(BlockFlavour::List), - "affine:code" => Some(BlockFlavour::Code), - "affine:divider" => Some(BlockFlavour::Divider), - "affine:image" => Some(BlockFlavour::Image), - "affine:table" => Some(BlockFlavour::Table), - "affine:bookmark" => Some(BlockFlavour::Bookmark), - "affine:embed-youtube" => Some(BlockFlavour::EmbedYoutube), - "affine:embed-iframe" => Some(BlockFlavour::EmbedIframe), - "affine:callout" => Some(BlockFlavour::Callout), - _ => None, - } - } -} - -/// Block types for paragraphs and lists. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum BlockType { - // Paragraph types - Text, - H1, - H2, - H3, - H4, - H5, - H6, - Quote, - // List types - Bulleted, - Numbered, - Todo, - // Preserve unknown types when loading from ydoc. - Unknown(String), -} - -impl BlockType { - pub fn as_str(&self) -> &str { - match self { - BlockType::Text => "text", - BlockType::H1 => "h1", - BlockType::H2 => "h2", - BlockType::H3 => "h3", - BlockType::H4 => "h4", - BlockType::H5 => "h5", - BlockType::H6 => "h6", - BlockType::Quote => "quote", - BlockType::Bulleted => "bulleted", - BlockType::Numbered => "numbered", - BlockType::Todo => "todo", - BlockType::Unknown(value) => value.as_str(), - } - } - - pub fn from_str(value: &str) -> Option { - match value { - "text" => Some(BlockType::Text), - "h1" => Some(BlockType::H1), - "h2" => Some(BlockType::H2), - "h3" => Some(BlockType::H3), - "h4" => Some(BlockType::H4), - "h5" => Some(BlockType::H5), - "h6" => Some(BlockType::H6), - "quote" => Some(BlockType::Quote), - "bulleted" => Some(BlockType::Bulleted), - "numbered" => Some(BlockType::Numbered), - "todo" => Some(BlockType::Todo), - _ => None, - } - } - - pub fn from_str_lossy(value: String) -> Self { - Self::from_str(&value).unwrap_or(BlockType::Unknown(value)) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub(super) struct ImageSpec { - pub(super) source_id: String, - pub(super) caption: Option, - pub(super) width: Option, - pub(super) height: Option, -} - -impl ImageSpec { - pub(super) fn render_markdown(&self) -> String { - let blob_url = format!("blob://{}", self.source_id); - let caption = self.caption.as_deref().unwrap_or(""); - - if self.width.is_some() || self.height.is_some() || !caption.is_empty() { - let width_text = self - .width - .map(|value| value.to_string()) - .unwrap_or_else(|| "auto".into()); - let height_text = self - .height - .map(|value| value.to_string()) - .unwrap_or_else(|| "auto".into()); - return format!( - "\n\n" - ); - } - - let alt = if caption.is_empty() { - self.source_id.as_str() - } else { - caption - }; - format!("\n![{alt}]({blob_url})\n\n") - } - - pub(super) fn from_block_map(block: &Map) -> Self { - let source_id = get_string(block, PROP_SOURCE_ID).unwrap_or_default(); - let caption = get_string(block, PROP_CAPTION); - let width = block.get(PROP_WIDTH).and_then(value_to_f64); - let height = block.get(PROP_HEIGHT).and_then(value_to_f64); - ImageSpec { - source_id, - caption, - width, - height, - } - } - - pub(super) fn normalize_source(src: &str) -> Result { - let trimmed = src.trim(); - if trimmed.is_empty() { - return Err(ParseError::ParserError("invalid_image_source".into())); - } - if let Some(rest) = trimmed.strip_prefix("blob://") { - if rest.is_empty() { - return Err(ParseError::ParserError("invalid_image_source".into())); - } - return Ok(rest.to_string()); - } - if !trimmed.contains('/') && !trimmed.contains(':') { - return Ok(trimmed.to_string()); - } - if let Some(pos) = trimmed.rfind("/blobs/") { - let id = &trimmed[pos + "/blobs/".len()..]; - let id = id.split(['?', '#']).next().unwrap_or(""); - if !id.is_empty() { - return Ok(id.to_string()); - } - } - Err(ParseError::ParserError("unsupported_image_source".into())) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub(super) struct TableSpec { - pub(super) rows: Vec>, -} - -impl TableSpec { - pub(super) fn from_block_map(block: &Map) -> Self { - let mut row_entries: Vec<(String, String)> = Vec::new(); - let mut column_entries: Vec<(String, String)> = Vec::new(); - - for key in block.keys() { - if key.starts_with(PROP_ROWS_PREFIX) - && key.ends_with(PROP_ROW_ID_SUFFIX) - && let Some(row_id) = block.get(key).and_then(|value| value_to_string(&value)) - { - let base = key.trim_end_matches(PROP_ROW_ID_SUFFIX); - let order_key = format!("{base}{PROP_ORDER_SUFFIX}"); - let order = block - .get(&order_key) - .and_then(|value| value_to_string(&value)) - .unwrap_or_default(); - row_entries.push((order, row_id)); - } - if key.starts_with(PROP_COLUMNS_PREFIX) - && key.ends_with(PROP_COLUMN_ID_SUFFIX) - && let Some(column_id) = block.get(key).and_then(|value| value_to_string(&value)) - { - let base = key.trim_end_matches(PROP_COLUMN_ID_SUFFIX); - let order_key = format!("{base}{PROP_ORDER_SUFFIX}"); - let order = block - .get(&order_key) - .and_then(|value| value_to_string(&value)) - .unwrap_or_default(); - column_entries.push((order, column_id)); - } - } - - row_entries.sort_by(|a, b| a.0.cmp(&b.0)); - column_entries.sort_by(|a, b| a.0.cmp(&b.0)); - - let mut rows = Vec::new(); - for (_, row_id) in row_entries { - let mut row = Vec::new(); - for (_, column_id) in &column_entries { - let cell_key = table_cell_text_key(&row_id, column_id); - let cell_text = block - .get(&cell_key) - .and_then(|value| value_to_string(&value)) - .unwrap_or_default(); - row.push(cell_text); - } - rows.push(row); - } - - Self { rows } - } - - pub(super) fn render_markdown(&self) -> Option { - let options = MarkdownTableOptions::new(false, "
", true); - render_markdown_table(&self.rows, options) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub(super) struct BookmarkSpec { - pub(super) url: String, - pub(super) caption: Option, -} - -#[derive(Debug, Clone, PartialEq)] -pub(super) struct EmbedYoutubeSpec { - pub(super) video_id: String, -} - -#[derive(Debug, Clone, PartialEq)] -pub(super) struct EmbedIframeSpec { - pub(super) url: String, -} - -#[derive(Debug, Clone)] -pub(super) struct BlockSpec { - pub(super) flavour: BlockFlavour, - pub(super) block_type: Option, - pub(super) text: Vec, - pub(super) checked: Option, - pub(super) language: Option, - pub(super) order: Option, - pub(super) image: Option, - pub(super) table: Option, - pub(super) bookmark: Option, - pub(super) embed_youtube: Option, - pub(super) embed_iframe: Option, -} - -impl BlockSpec { - pub(super) fn is_exact(&self, other: &BlockSpec) -> bool { - self.flavour == other.flavour - && self.block_type == other.block_type - && self.checked == other.checked - && self.language == other.language - && self.image == other.image - && self.table == other.table - && self.bookmark == other.bookmark - && self.embed_youtube == other.embed_youtube - && self.embed_iframe == other.embed_iframe - && text_delta_eq(&self.text, &other.text) - } - - pub(super) fn is_similar(&self, other: &BlockSpec) -> bool { - self.flavour == other.flavour && self.block_type == other.block_type - } - - pub(super) fn block_type_str(&self) -> Option<&str> { - self.block_type.as_ref().map(BlockType::as_str) - } - - pub(super) fn from_block_map(block: &Map) -> Result { - let flavour_value = get_string(block, SYS_FLAVOUR).unwrap_or_default(); - let Some(flavour) = BlockFlavour::from_str(&flavour_value) else { - return Err(ParseError::ParserError(format!( - "unsupported block flavour: {flavour_value}" - ))); - }; - Ok(Self::from_block_map_with_flavour(block, flavour)) - } - - pub(super) fn from_block_map_with_flavour(block: &Map, flavour: BlockFlavour) -> Self { - if flavour == BlockFlavour::Image { - return BlockSpec { - flavour, - block_type: None, - text: Vec::new(), - checked: None, - language: None, - order: None, - image: Some(ImageSpec::from_block_map(block)), - table: None, - bookmark: None, - embed_youtube: None, - embed_iframe: None, - }; - } - - if flavour == BlockFlavour::Table { - return BlockSpec { - flavour, - block_type: None, - text: Vec::new(), - checked: None, - language: None, - order: None, - image: None, - table: Some(TableSpec::from_block_map(block)), - bookmark: None, - embed_youtube: None, - embed_iframe: None, - }; - } - - if flavour == BlockFlavour::Bookmark { - return BlockSpec { - flavour, - block_type: None, - text: Vec::new(), - checked: None, - language: None, - order: None, - image: None, - table: None, - bookmark: Some(BookmarkSpec { - url: get_string(block, PROP_URL).unwrap_or_default(), - caption: get_string(block, PROP_CAPTION), - }), - embed_youtube: None, - embed_iframe: None, - }; - } - - if flavour == BlockFlavour::EmbedYoutube { - return BlockSpec { - flavour, - block_type: None, - text: Vec::new(), - checked: None, - language: None, - order: None, - image: None, - table: None, - bookmark: None, - embed_youtube: Some(EmbedYoutubeSpec { - video_id: get_string(block, PROP_VIDEO_ID).unwrap_or_default(), - }), - embed_iframe: None, - }; - } - - if flavour == BlockFlavour::EmbedIframe { - return BlockSpec { - flavour, - block_type: None, - text: Vec::new(), - checked: None, - language: None, - order: None, - image: None, - table: None, - bookmark: None, - embed_youtube: None, - embed_iframe: Some(EmbedIframeSpec { - url: get_string(block, PROP_URL).unwrap_or_default(), - }), - }; - } - - let block_type = match get_string(block, PROP_TYPE) { - Some(value) => Some(BlockType::from_str_lossy(value)), - None => match flavour { - BlockFlavour::Paragraph => Some(BlockType::Text), - BlockFlavour::List => Some(BlockType::Bulleted), - _ => None, - }, - }; - - let text = block - .get(PROP_TEXT) - .and_then(|v| v.to_text()) - .map(|text| text.to_delta()) - .unwrap_or_default(); - - let checked = block.get(PROP_CHECKED).and_then(|v| v.to_any()).and_then(|a| match a { - Any::True => Some(true), - Any::False => Some(false), - _ => None, - }); - let language = get_string(block, PROP_LANGUAGE); - let order = block.get(PROP_ORDER).and_then(|v| v.to_any()).and_then(|a| match a { - Any::Integer(value) => Some(value as i64), - Any::BigInt64(value) => Some(value), - Any::Float32(value) => Some(value.0 as i64), - Any::Float64(value) => Some(value.0 as i64), - _ => None, - }); - - BlockSpec { - flavour, - block_type, - text, - checked, - language, - order, - image: None, - table: None, - bookmark: None, - embed_youtube: None, - embed_iframe: None, - } - } -} - -#[derive(Debug, Clone)] -pub(super) struct BlockNode { - pub(super) spec: BlockSpec, - pub(super) children: Vec, -} - -pub(super) trait TreeNode { - fn children(&self) -> &[Self] - where - Self: Sized; -} - -impl TreeNode for BlockNode { - fn children(&self) -> &[BlockNode] { - &self.children - } -} - -pub(super) fn count_tree_nodes(nodes: &[T]) -> usize { - nodes.iter().map(|node| 1 + count_tree_nodes(node.children())).sum() -} - -pub(super) fn text_delta_eq(a: &[TextDeltaOp], b: &[TextDeltaOp]) -> bool { - if a.len() != b.len() { - return false; - } - - for (left, right) in a.iter().zip(b.iter()) { - match (left, right) { - ( - TextDeltaOp::Insert { - insert: TextInsert::Text(text_a), - format: format_a, - }, - TextDeltaOp::Insert { - insert: TextInsert::Text(text_b), - format: format_b, - }, - ) => { - if text_a != text_b { - return false; - } - if !attrs_eq(format_a.as_ref(), format_b.as_ref()) { - return false; - } - } - _ => return false, - } - } - - true -} - -fn attrs_eq(a: Option<&TextAttributes>, b: Option<&TextAttributes>) -> bool { - match (a, b) { - (None, None) => true, - (Some(left), Some(right)) => { - if left.len() != right.len() { - return false; - } - left.iter().all(|(key, value)| right.get(key) == Some(value)) - } - _ => false, - } -} - -#[cfg(test)] -mod tests { - use y_octo::DocOptions; - - use super::{super::write::builder::text_ops_from_plain, *}; - use crate::doc_parser::build_full_doc; - - fn spec_from_markdown(markdown: &str, doc_id: &str, flavour: &str) -> BlockSpec { - let bin = build_full_doc("Title", markdown, doc_id).expect("create doc"); - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc.apply_update_from_binary_v1(&bin).expect("apply update"); - let blocks_map = doc.get_map("blocks").expect("blocks map"); - - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() - && get_string(&block_map, SYS_FLAVOUR).as_deref() == Some(flavour) - { - return BlockSpec::from_block_map(&block_map).expect("spec"); - } - } - - panic!("block not found: {flavour}"); - } - - #[test] - fn test_from_block_map_paragraph() { - let spec = spec_from_markdown("Plain paragraph.", "block-spec-paragraph", "affine:paragraph"); - assert_eq!(spec.flavour, BlockFlavour::Paragraph); - assert_eq!(spec.block_type, Some(BlockType::Text)); - assert_eq!(spec.text, text_ops_from_plain("Plain paragraph.")); - } - - #[test] - fn test_from_block_map_list_checked() { - let spec = spec_from_markdown("- [x] Done", "block-spec-list", "affine:list"); - assert_eq!(spec.flavour, BlockFlavour::List); - assert_eq!(spec.block_type, Some(BlockType::Todo)); - assert_eq!(spec.checked, Some(true)); - assert_eq!(spec.text, text_ops_from_plain("Done")); - } - - #[test] - fn test_from_block_map_image() { - let spec = spec_from_markdown("![Alt](blob://image-id)", "block-spec-image", "affine:image"); - assert_eq!(spec.flavour, BlockFlavour::Image); - let image = spec.image.expect("image spec"); - assert_eq!(image.source_id, "image-id"); - assert_eq!(image.caption.as_deref(), Some("Alt")); - assert_eq!(image.width, None); - assert_eq!(image.height, None); - } - - #[test] - fn test_from_block_map_table() { - let spec = spec_from_markdown( - "| A | B |\n| --- | --- |\n| 1 | 2 |", - "block-spec-table", - "affine:table", - ); - assert_eq!(spec.flavour, BlockFlavour::Table); - let table = spec.table.expect("table spec"); - assert_eq!( - table.rows, - vec![ - vec!["A".to_string(), "B".to_string()], - vec!["1".to_string(), "2".to_string()] - ] - ); - } - - #[test] - fn test_from_block_map_embed_iframe() { - let spec = spec_from_markdown( - r#""#, - "block-spec-embed-iframe", - "affine:embed-iframe", - ); - assert_eq!(spec.flavour, BlockFlavour::EmbedIframe); - assert_eq!(spec.embed_iframe.as_ref().unwrap().url, "https://example.com/embed"); - } -} diff --git a/packages/common/native/src/doc_parser/blocksuite.rs b/packages/common/native/src/doc_parser/blocksuite.rs deleted file mode 100644 index ad266b0b10cb0..0000000000000 --- a/packages/common/native/src/doc_parser/blocksuite.rs +++ /dev/null @@ -1,176 +0,0 @@ -use std::collections::{HashMap, HashSet}; - -use y_octo::Map; - -use super::{ - schema::{SYS_CHILDREN, SYS_FLAVOUR, SYS_ID}, - value::value_to_string, -}; - -pub(super) struct BlockIndex { - pub(super) block_pool: HashMap, - pub(super) parent_lookup: HashMap, -} - -pub(super) struct DocContext { - pub(super) block_pool: HashMap, - pub(super) parent_lookup: HashMap, - pub(super) root_block_id: String, -} - -pub(super) struct BlockWalker { - queue: Vec<(Option, String)>, - visited: HashSet, -} - -pub(super) fn build_block_index(blocks_map: &Map) -> BlockIndex { - let mut block_pool: HashMap = HashMap::new(); - let mut parent_lookup: HashMap = HashMap::new(); - - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() - && let Some(block_id) = get_block_id(&block_map) - { - for child_id in collect_child_ids(&block_map) { - parent_lookup.insert(child_id, block_id.clone()); - } - block_pool.insert(block_id, block_map); - } - } - - BlockIndex { - block_pool, - parent_lookup, - } -} - -impl DocContext { - pub(super) fn from_blocks_map(blocks_map: &Map, root_flavour: &str) -> Option { - let BlockIndex { - block_pool, - parent_lookup, - } = build_block_index(blocks_map); - - let root_block_id = find_block_id_by_flavour(&block_pool, root_flavour)?; - Some(Self { - block_pool, - parent_lookup, - root_block_id, - }) - } - - pub(super) fn walker(&self) -> BlockWalker { - BlockWalker::new(&self.root_block_id) - } -} - -impl BlockWalker { - fn new(root_block_id: &str) -> Self { - let mut visited = HashSet::new(); - visited.insert(root_block_id.to_string()); - - Self { - queue: vec![(None, root_block_id.to_string())], - visited, - } - } - - pub(super) fn next(&mut self) -> Option<(Option, String)> { - self.queue.pop() - } - - pub(super) fn enqueue_children(&mut self, parent_block_id: &str, block: &Map) { - let mut child_ids = collect_child_ids(block); - for child_id in child_ids.drain(..).rev() { - if self.visited.insert(child_id.clone()) { - self.queue.push((Some(parent_block_id.to_string()), child_id)); - } - } - } -} - -pub(super) fn find_block_id_by_flavour(block_pool: &HashMap, flavour: &str) -> Option { - block_pool.iter().find_map(|(id, block)| { - get_flavour(block) - .filter(|block_flavour| block_flavour == flavour) - .map(|_| id.clone()) - }) -} - -pub(super) fn collect_child_ids(block: &Map) -> Vec { - block - .get(SYS_CHILDREN) - .and_then(|value| value.to_array()) - .map(|array| { - array - .iter() - .filter_map(|value| value_to_string(&value)) - .collect::>() - }) - .unwrap_or_default() -} - -pub(super) fn get_block_id(block: &Map) -> Option { - get_string(block, SYS_ID) -} - -pub(super) fn get_flavour(block: &Map) -> Option { - get_string(block, SYS_FLAVOUR) -} - -pub(super) fn get_string(block: &Map, key: &str) -> Option { - block.get(key).and_then(|value| value_to_string(&value)) -} - -pub(super) fn get_list_depth( - block_id: &str, - parent_lookup: &HashMap, - blocks: &HashMap, -) -> usize { - let mut depth = 0; - let mut current_id = block_id.to_string(); - - while let Some(parent_id) = parent_lookup.get(¤t_id) { - if let Some(parent_block) = blocks.get(parent_id) - && get_flavour(parent_block).as_deref() == Some("affine:list") - { - depth += 1; - current_id = parent_id.clone(); - continue; - } - break; - } - depth -} - -pub(super) fn nearest_by_flavour( - start: &str, - flavour: &str, - parent_lookup: &HashMap, - blocks: &HashMap, -) -> Option { - let mut cursor = Some(start.to_string()); - while let Some(node) = cursor { - if let Some(block) = blocks.get(&node) - && get_flavour(block).as_deref() == Some(flavour) - { - return Some(block.clone()); - } - cursor = parent_lookup.get(&node).cloned(); - } - None -} - -pub(super) fn find_child_id_by_flavour( - parent: &Map, - block_pool: &HashMap, - flavour: &str, -) -> Option { - collect_child_ids(parent).into_iter().find(|id| { - block_pool - .get(id) - .and_then(get_flavour) - .as_deref() - .is_some_and(|value| value == flavour) - }) -} diff --git a/packages/common/native/src/doc_parser/doc_loader.rs b/packages/common/native/src/doc_parser/doc_loader.rs deleted file mode 100644 index 10e2f210d7261..0000000000000 --- a/packages/common/native/src/doc_parser/doc_loader.rs +++ /dev/null @@ -1,39 +0,0 @@ -use y_octo::{Doc, DocOptions}; - -use super::ParseError; - -pub(super) fn is_empty_doc(binary: &[u8]) -> bool { - binary.is_empty() || binary == [0, 0] -} - -pub(super) fn load_doc(binary: &[u8], doc_id: Option<&str>) -> Result { - if is_empty_doc(binary) { - return Err(ParseError::InvalidBinary); - } - - let mut doc = build_doc(doc_id); - doc - .apply_update_from_binary_v1(binary) - .map_err(|_| ParseError::InvalidBinary)?; - Ok(doc) -} - -pub(super) fn load_doc_or_new(binary: &[u8]) -> Result { - if is_empty_doc(binary) { - return Ok(DocOptions::new().build()); - } - - let mut doc = DocOptions::new().build(); - doc - .apply_update_from_binary_v1(binary) - .map_err(|_| ParseError::InvalidBinary)?; - Ok(doc) -} - -fn build_doc(doc_id: Option<&str>) -> Doc { - let options = DocOptions::new(); - match doc_id { - Some(doc_id) => options.with_guid(doc_id.to_string()).build(), - None => options.build(), - } -} diff --git a/packages/common/native/src/doc_parser/error.rs b/packages/common/native/src/doc_parser/error.rs deleted file mode 100644 index cf9aed4374f9b..0000000000000 --- a/packages/common/native/src/doc_parser/error.rs +++ /dev/null @@ -1,38 +0,0 @@ -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use y_octo::JwstCodecError; - -#[derive(Error, Debug, Serialize, Deserialize)] -pub enum ParseError { - #[error("doc_not_found")] - DocNotFound, - #[error("invalid_binary")] - InvalidBinary, - #[error("sqlite_error: {0}")] - SqliteError(String), - #[error("parser_error: {0}")] - ParserError(String), - #[error("unknown: {0}")] - Unknown(String), -} - -impl From for ParseError { - fn from(value: JwstCodecError) -> Self { - if matches!( - value, - JwstCodecError::DamagedDocumentJson - | JwstCodecError::IncompleteDocument(_) - | JwstCodecError::InvalidWriteBuffer(_) - | JwstCodecError::UpdateInvalid(_) - | JwstCodecError::StructClockInvalid { expect: _, actually: _ } - | JwstCodecError::StructSequenceInvalid { client_id: _, clock: _ } - | JwstCodecError::StructSequenceNotExists(_) - | JwstCodecError::RootStructNotFound(_) - | JwstCodecError::ParentNotFound - | JwstCodecError::IndexOutOfBound(_) - ) { - return ParseError::InvalidBinary; - } - Self::ParserError(value.to_string()) - } -} diff --git a/packages/common/native/src/doc_parser/markdown/delta.rs b/packages/common/native/src/doc_parser/markdown/delta.rs deleted file mode 100644 index 29a1d96f09745..0000000000000 --- a/packages/common/native/src/doc_parser/markdown/delta.rs +++ /dev/null @@ -1,842 +0,0 @@ -use std::{ - cell::RefCell, - collections::{HashMap, HashSet}, - rc::{Rc, Weak}, -}; - -use y_octo::{AHashMap, Any, Map, Text, TextAttributes, TextDeltaOp, TextInsert, Value}; - -use super::{ - super::value::{ - any_as_string, any_as_u64, any_truthy, build_reference_payload, params_any_map_to_json, value_to_any, - }, - inline::InlineStyle, -}; - -#[derive(Debug, Clone)] -struct InlineReference { - ref_type: Option, - page_id: String, - title: Option, - params: Option>, - mode: Option, -} - -#[derive(Debug, Clone)] -pub(crate) struct InlineReferencePayload { - pub(crate) doc_id: String, - pub(crate) payload: String, -} - -#[derive(Debug, Clone)] -pub(crate) struct DeltaToMdOptions { - doc_url_prefix: Option, -} - -impl DeltaToMdOptions { - pub(crate) fn new(doc_url_prefix: Option) -> Self { - Self { doc_url_prefix } - } - - fn build_reference_link(&self, reference: &InlineReference) -> (String, String) { - let title = reference.title.clone().unwrap_or_default(); - - if let Some(prefix) = self.doc_url_prefix.as_deref() { - let prefix = prefix.trim_end_matches('/'); - return (title, format!("{}/{}", prefix, reference.page_id)); - } - - let mut parts = Vec::new(); - parts.push(reference.ref_type.clone().unwrap_or_else(|| "LinkedPage".into())); - parts.push(reference.page_id.clone()); - if let Some(mode) = reference.mode.as_ref() { - parts.push(mode.clone()); - } - - (title, parts.join(":")) - } -} - -pub(crate) fn text_to_inline_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option { - block - .get(key) - .and_then(|value| value.to_text()) - .map(|text| delta_to_inline_markdown(&text, options)) -} - -pub(crate) fn delta_ops_to_markdown(ops: &[TextDeltaOp], options: &DeltaToMdOptions) -> String { - delta_to_markdown_with_options(ops, options, true) -} - -pub(crate) fn delta_ops_to_plain_text(ops: &[TextDeltaOp]) -> String { - let mut out = String::new(); - for op in ops { - if let TextDeltaOp::Insert { - insert: TextInsert::Text(text), - .. - } = op - { - out.push_str(text); - } - } - out -} - -pub(crate) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec { - let mut refs = Vec::new(); - let mut seen: HashSet<(String, String)> = HashSet::new(); - - for op in delta { - let attrs = match op { - TextDeltaOp::Insert { - format: Some(format), .. - } => format, - _ => continue, - }; - - let reference = match attrs.get(InlineStyle::Reference.key()).and_then(parse_inline_reference) { - Some(reference) => reference, - None => continue, - }; - - let payload = match inline_reference_payload(&reference) { - Some(payload) => payload, - None => continue, - }; - - let key = (reference.page_id.clone(), payload.clone()); - if seen.insert(key.clone()) { - refs.push(InlineReferencePayload { - doc_id: key.0, - payload: key.1, - }); - } - } - - refs -} - -pub(crate) fn extract_inline_references_from_value(value: &Value) -> Vec { - if let Some(text) = value.to_text() { - return extract_inline_references(&text.to_delta()); - } - - let Some(any) = value_to_any(value) else { - return Vec::new(); - }; - - extract_inline_references_from_any(&any) -} - -fn extract_inline_references_from_any(value: &Any) -> Vec { - let Some(ops) = delta_ops_from_any(value) else { - return Vec::new(); - }; - - let mut refs = Vec::new(); - let mut seen: HashSet<(String, String)> = HashSet::new(); - - for op in ops { - let reference = match op - .attributes - .get(InlineStyle::Reference.key()) - .and_then(parse_inline_reference) - { - Some(reference) => reference, - None => continue, - }; - - let payload = match inline_reference_payload(&reference) { - Some(payload) => payload, - None => continue, - }; - - let key = (reference.page_id.clone(), payload.clone()); - if seen.insert(key.clone()) { - refs.push(InlineReferencePayload { - doc_id: key.0, - payload: key.1, - }); - } - } - - refs -} - -fn parse_inline_reference(value: &Any) -> Option { - let map = match value { - Any::Object(map) => map, - _ => return None, - }; - - let page_id = map.get("pageId").and_then(any_as_string).map(str::to_string)?; - let title = map.get("title").and_then(any_as_string).map(str::to_string); - let ref_type = map.get("type").and_then(any_as_string).map(str::to_string); - let params = map.get("params").and_then(|value| match value { - Any::Object(map) => Some(map.clone()), - _ => None, - }); - let mode = params - .as_ref() - .and_then(|params| params.get("mode")) - .and_then(any_as_string) - .map(str::to_string); - - Some(InlineReference { - ref_type, - page_id, - title, - params, - mode, - }) -} - -fn inline_reference_payload(reference: &InlineReference) -> Option { - let params = reference.params.as_ref().map(params_any_map_to_json); - Some(build_reference_payload(&reference.page_id, params)) -} - -fn delta_to_inline_markdown(text: &Text, options: &DeltaToMdOptions) -> String { - delta_to_markdown_with_options(&text.to_delta(), options, false) -} - -fn delta_to_markdown_with_options(delta: &[TextDeltaOp], options: &DeltaToMdOptions, trailing_newline: bool) -> String { - let ops = build_delta_ops(delta); - delta_ops_to_markdown_with_options(&ops, options, trailing_newline) -} - -fn delta_ops_to_markdown_with_options(ops: &[DeltaOp], options: &DeltaToMdOptions, trailing_newline: bool) -> String { - let root = convert_delta_ops(ops, options); - let mut rendered = render_node(&root); - rendered = rendered.trim_end().to_string(); - if trailing_newline { - rendered.push('\n'); - } - rendered -} - -#[derive(Debug, Clone)] -struct DeltaOp { - insert: DeltaInsert, - attributes: TextAttributes, -} - -#[derive(Debug, Clone)] -enum DeltaInsert { - Text(String), - Embed(Vec), -} - -fn delta_ops_from_any(value: &Any) -> Option> { - let map = match value { - Any::Object(map) => map, - _ => return None, - }; - match map.get("$blocksuite:internal:text$") { - Some(Any::True) => {} - _ => return None, - } - - let delta = map.get("delta")?; - let entries = match delta { - Any::Array(entries) => entries, - _ => return None, - }; - - let mut ops = Vec::new(); - for entry in entries { - if let Some(op) = delta_op_from_any(entry) { - ops.push(op); - } - } - - Some(ops) -} - -fn delta_op_from_any(value: &Any) -> Option { - let map = match value { - Any::Object(map) => map, - _ => return None, - }; - - let insert_value = map.get("insert")?; - let insert = match insert_value { - Any::String(text) => DeltaInsert::Text(text.clone()), - Any::Array(values) => DeltaInsert::Embed(values.clone()), - _ => DeltaInsert::Embed(vec![insert_value.clone()]), - }; - - let attributes = map.get("attributes").and_then(any_to_attributes).unwrap_or_default(); - - Some(DeltaOp { insert, attributes }) -} - -fn any_to_attributes(value: &Any) -> Option { - let map = match value { - Any::Object(map) => map, - _ => return None, - }; - - let mut attrs = TextAttributes::new(); - for (key, value) in map.iter() { - attrs.insert(key.clone(), value.clone()); - } - Some(attrs) -} - -fn delta_any_to_inline_markdown(value: &Any, options: &DeltaToMdOptions) -> Option { - delta_ops_from_any(value).map(|ops| delta_ops_to_markdown_with_options(&ops, options, false)) -} - -pub(crate) fn delta_value_to_inline_markdown(value: &Value, options: &DeltaToMdOptions) -> Option { - if let Some(text) = value.to_text() { - return Some(delta_to_inline_markdown(&text, options)); - } - - let any = value_to_any(value)?; - delta_any_to_inline_markdown(&any, options) -} - -fn build_delta_ops(delta: &[TextDeltaOp]) -> Vec { - let mut ops = Vec::new(); - - for op in delta { - let (insert, attrs) = match op { - TextDeltaOp::Insert { insert, format } => (insert, format.clone().unwrap_or_default()), - _ => continue, - }; - - match insert { - TextInsert::Text(text) => ops.push(DeltaOp { - insert: DeltaInsert::Text(text.clone()), - attributes: attrs, - }), - TextInsert::Embed(values) => ops.push(DeltaOp { - insert: DeltaInsert::Embed(values.clone()), - attributes: attrs, - }), - } - } - - ops -} - -#[derive(Debug)] -struct Group { - node: Rc>, - kind: String, - distance: usize, - count: usize, -} - -fn convert_delta_ops(ops: &[DeltaOp], options: &DeltaToMdOptions) -> Rc> { - let root = Node::new_root(); - let mut group: Option = None; - let mut active_inline: HashMap = HashMap::new(); - let mut beginning_of_line = false; - - let mut line = Node::new_line(); - let mut el = line.clone(); - Node::append(&root, line.clone()); - - for index in 0..ops.len() { - let op = &ops[index]; - let next_attrs = ops.get(index + 1).map(|next| &next.attributes); - - match &op.insert { - DeltaInsert::Embed(values) => { - apply_inline_attributes(&mut el, &op.attributes, None, &mut active_inline, options); - for value in values { - match value { - Any::Object(map) => { - for (key, value) in map.iter() { - match key.as_str() { - "image" => { - if let Some(src) = any_as_string(value) { - let url = encode_link(src); - Node::append(&el, Node::new_text(&format!("![]({url})"))); - } - } - "thematic_break" => { - let current_open = el.borrow().open.clone(); - el.borrow_mut().open = format!("\n---\n{current_open}"); - } - _ => {} - } - } - } - Any::String(value) => { - Node::append(&el, Node::new_text(value)); - } - _ => {} - } - } - } - DeltaInsert::Text(text) => { - let lines: Vec<&str> = text.split('\n').collect(); - if has_block_level_attribute(&op.attributes) { - for _ in 1..lines.len() { - for (attr, value) in op.attributes.iter() { - match attr.as_str() { - "header" => { - if let Some(level) = any_as_u64(value) { - let prefix = "#".repeat(level as usize); - let current_open = line.borrow().open.clone(); - line.borrow_mut().open = format!("{prefix} {current_open}"); - new_line(&root, &mut line, &mut el, &mut active_inline); - break; - } - } - "blockquote" => { - let current_open = line.borrow().open.clone(); - line.borrow_mut().open = format!("> {current_open}"); - new_line(&root, &mut line, &mut el, &mut active_inline); - break; - } - "list" => { - if group.as_ref().is_some_and(|g| g.kind != attr.as_str()) { - group = None; - } - if group.is_none() { - let group_node = Node::new_line(); - Node::append(&root, group_node.clone()); - group = Some(Group { - node: group_node, - kind: attr.to_string(), - distance: 0, - count: 0, - }); - } - - if let Some(group) = group.as_mut() { - Node::append(&group.node, line.clone()); - group.distance = 0; - match any_as_string(value) { - Some("bullet") => { - let current_open = line.borrow().open.clone(); - line.borrow_mut().open = format!("- {current_open}"); - } - Some("checked") => { - let current_open = line.borrow().open.clone(); - line.borrow_mut().open = format!("- [x] {current_open}"); - } - Some("unchecked") => { - let current_open = line.borrow().open.clone(); - line.borrow_mut().open = format!("- [ ] {current_open}"); - } - Some("ordered") => { - group.count += 1; - let current_open = line.borrow().open.clone(); - line.borrow_mut().open = format!("{}. {}", group.count, current_open); - } - _ => {} - } - } - - new_line(&root, &mut line, &mut el, &mut active_inline); - break; - } - _ => {} - } - } - } - beginning_of_line = true; - } else { - for (line_index, segment) in lines.iter().enumerate() { - if (line_index > 0 || beginning_of_line) && group.is_some() { - let reset_group = group.as_mut().map(|group| { - group.distance += 1; - group.distance >= 2 - }); - if reset_group == Some(true) { - group = None; - } - } - - apply_inline_attributes(&mut el, &op.attributes, next_attrs, &mut active_inline, options); - Node::append(&el, Node::new_text(segment)); - if line_index + 1 < lines.len() { - new_line(&root, &mut line, &mut el, &mut active_inline); - } - } - beginning_of_line = false; - } - } - } - } - - root -} - -fn apply_inline_attributes( - el: &mut Rc>, - attrs: &TextAttributes, - next: Option<&TextAttributes>, - active_inline: &mut HashMap, - options: &DeltaToMdOptions, -) { - let mut first = Vec::new(); - let mut then = Vec::new(); - let mut seen: HashSet = HashSet::new(); - - let mut tag = el.clone(); - loop { - let format = match tag.borrow().format.clone() { - Some(format) => format, - None => break, - }; - seen.insert(format.clone()); - - let should_close = match attrs.get(&format) { - Some(value) => !any_truthy(value) || tag.borrow().open != tag.borrow().close, - None => true, - }; - - if should_close { - for key in seen.iter() { - active_inline.remove(key); - } - let parent = { - let tag_ref = tag.borrow(); - tag_ref.parent.as_ref().and_then(|p| p.upgrade()) - }; - if let Some(parent) = parent { - *el = parent.clone(); - tag = parent; - continue; - } - break; - } - - let parent = { - let tag_ref = tag.borrow(); - tag_ref.parent.as_ref().and_then(|p| p.upgrade()) - }; - if let Some(parent) = parent { - tag = parent; - } else { - break; - } - } - - for (attr, value) in attrs.iter() { - if !is_inline_attribute(attr) || !any_truthy(value) { - continue; - } - if let Some(active) = active_inline.get(attr) - && active == value - { - continue; - } - - let next_matches = next - .and_then(|next_attrs| next_attrs.get(attr)) - .map(|next_value| next_value == value) - .unwrap_or(false); - - if next_matches { - first.push(attr.clone()); - } else { - then.push(attr.clone()); - } - active_inline.insert(attr.clone(), value.clone()); - } - - for attr in first.into_iter().chain(then) { - if let Some(node) = inline_node_for_attr(&attr, attrs, options) { - node.borrow_mut().format = Some(attr.clone()); - Node::append(el, node.clone()); - *el = node; - } - } -} - -fn inline_node_for_attr(attr: &str, attrs: &TextAttributes, options: &DeltaToMdOptions) -> Option>> { - let style = InlineStyle::from_key(attr)?; - if let Some(delimiter) = style.delimiter() { - return Some(Node::new_inline(delimiter.open, delimiter.close)); - } - - match style { - InlineStyle::Underline => Some(Node::new_inline("", "")), - InlineStyle::Color => { - let color = attrs.get(attr).and_then(any_as_string)?.trim(); - if color.is_empty() { - None - } else { - Some(Node::new_inline(&format!(""), "")) - } - } - InlineStyle::Link => attrs - .get(attr) - .and_then(any_as_string) - .map(|url| Node::new_inline("[", &format!("]({url})"))), - InlineStyle::Reference => attrs.get(attr).and_then(parse_inline_reference).map(|reference| { - let (title, link) = options.build_reference_link(&reference); - Node::new_inline("[", &format!("{title}]({link})")) - }), - _ => None, - } -} - -fn has_block_level_attribute(attrs: &TextAttributes) -> bool { - attrs.contains_key("header") || attrs.contains_key("blockquote") || attrs.contains_key("list") -} - -fn is_inline_attribute(attr: &str) -> bool { - InlineStyle::from_key(attr).is_some() -} - -fn encode_link(link: &str) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - - #[inline] - fn push_pct(out: &mut String, b: u8) { - out.push('%'); - out.push(HEX[(b >> 4) as usize] as char); - out.push(HEX[(b & 0x0f) as usize] as char); - } - - #[inline] - fn is_allowed(b: u8) -> bool { - matches!( - b, - b'A'..=b'Z' - | b'a'..=b'z' - | b'0'..=b'9' - | b'-' - | b'_' - | b'.' - | b'!' - | b'~' - | b'*' - | b'\'' - | b';' - | b',' - | b'/' - | b'?' - | b':' - | b'@' - | b'&' - | b'=' - | b'+' - | b'$' - | b'#' - ) - } - - let mut out = String::with_capacity(link.len()); - - for &b in link.as_bytes() { - match b { - b'(' | b')' => push_pct(&mut out, b), - b if is_allowed(b) => out.push(b as char), - b => push_pct(&mut out, b), - } - } - - if let Some(i) = out.find("?response-content-disposition=attachment") { - out.truncate(i); - } else if let Some(i) = out.find("&response-content-disposition=attachment") { - out.truncate(i); - } - - out -} - -#[derive(Debug)] -struct Node { - open: String, - close: String, - text: String, - children: Vec>>, - parent: Option>>, - format: Option, -} - -impl Node { - fn new_root() -> Rc> { - Rc::new(RefCell::new(Node { - open: String::new(), - close: String::new(), - text: String::new(), - children: Vec::new(), - parent: None, - format: None, - })) - } - - fn new_inline(open: &str, close: &str) -> Rc> { - Rc::new(RefCell::new(Node { - open: open.to_string(), - close: close.to_string(), - text: String::new(), - children: Vec::new(), - parent: None, - format: None, - })) - } - - fn new_text(text: &str) -> Rc> { - Rc::new(RefCell::new(Node { - open: String::new(), - close: String::new(), - text: text.to_string(), - children: Vec::new(), - parent: None, - format: None, - })) - } - - fn new_line() -> Rc> { - Rc::new(RefCell::new(Node { - open: String::new(), - close: "\n".to_string(), - text: String::new(), - children: Vec::new(), - parent: None, - format: None, - })) - } - - fn append(parent: &Rc>, child: Rc>) { - if let Some(old_parent) = child.borrow().parent.as_ref().and_then(|p| p.upgrade()) { - let mut old_parent = old_parent.borrow_mut(); - old_parent.children.retain(|existing| !Rc::ptr_eq(existing, &child)); - } - - child.borrow_mut().parent = Some(Rc::downgrade(parent)); - parent.borrow_mut().children.push(child); - } -} - -fn render_node(node: &Rc>) -> String { - let node_ref = node.borrow(); - let mut inner = node_ref.text.clone(); - for child in node_ref.children.iter() { - inner.push_str(&render_node(child)); - } - - if inner.trim().is_empty() - && node_ref.open == node_ref.close - && !node_ref.open.is_empty() - && !node_ref.close.is_empty() - { - return String::new(); - } - - let wrapped = !node_ref.open.is_empty() && !node_ref.close.is_empty(); - let empty_inner = inner.trim().is_empty(); - let mut fragments = Vec::new(); - - if inner.starts_with(' ') && !empty_inner && wrapped { - fragments.push(" ".to_string()); - } - if !node_ref.open.is_empty() { - fragments.push(node_ref.open.clone()); - } - fragments.push(if wrapped { - inner.trim().to_string() - } else { - inner.clone() - }); - if !node_ref.close.is_empty() { - fragments.push(node_ref.close.clone()); - } - if inner.ends_with(' ') && !empty_inner && wrapped { - fragments.push(" ".to_string()); - } - - fragments.join("") -} - -fn new_line( - root: &Rc>, - line: &mut Rc>, - el: &mut Rc>, - active_inline: &mut HashMap, -) { - *line = Node::new_line(); - *el = line.clone(); - Node::append(root, line.clone()); - active_inline.clear(); -} - -#[cfg(test)] -mod tests { - use serde_json::Value; - use y_octo::{Any, TextAttributes, TextDeltaOp, TextInsert}; - - use super::*; - - #[test] - fn test_delta_to_inline_markdown_link() { - let mut attrs = TextAttributes::new(); - attrs.insert( - InlineStyle::Link.key().into(), - Any::String("https://example.com".into()), - ); - - let delta = vec![TextDeltaOp::Insert { - insert: TextInsert::Text("AFFiNE".into()), - format: Some(attrs), - }]; - - let options = DeltaToMdOptions::new(None); - let rendered = delta_to_markdown_with_options(&delta, &options, false); - assert_eq!(rendered, "[AFFiNE](https://example.com)"); - } - - #[test] - fn test_extract_inline_references_payload() { - let mut ref_map = AHashMap::default(); - ref_map.insert("pageId".into(), Any::String("doc123".into())); - ref_map.insert("title".into(), Any::String("Doc Title".into())); - ref_map.insert("type".into(), Any::String("LinkedPage".into())); - - let mut attrs = TextAttributes::new(); - attrs.insert(InlineStyle::Reference.key().into(), Any::Object(ref_map)); - - let delta = vec![TextDeltaOp::Insert { - insert: TextInsert::Text("Doc Title".into()), - format: Some(attrs), - }]; - - let refs = extract_inline_references(&delta); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].doc_id, "doc123"); - - let payload: Value = serde_json::from_str(&refs[0].payload).unwrap(); - assert_eq!(payload, serde_json::json!({ "docId": "doc123" })); - } - - #[test] - fn test_delta_to_inline_markdown_underline() { - let mut attrs = TextAttributes::new(); - attrs.insert(InlineStyle::Underline.key().into(), Any::True); - - let delta = vec![TextDeltaOp::Insert { - insert: TextInsert::Text("Under".into()), - format: Some(attrs), - }]; - - let options = DeltaToMdOptions::new(None); - let rendered = delta_to_markdown_with_options(&delta, &options, false); - assert_eq!(rendered, "Under"); - } - - #[test] - fn test_delta_to_inline_markdown_color() { - let mut attrs = TextAttributes::new(); - attrs.insert(InlineStyle::Color.key().into(), Any::String("red".into())); - - let delta = vec![TextDeltaOp::Insert { - insert: TextInsert::Text("Red".into()), - format: Some(attrs), - }]; - - let options = DeltaToMdOptions::new(None); - let rendered = delta_to_markdown_with_options(&delta, &options, false); - assert_eq!(rendered, "Red"); - } -} diff --git a/packages/common/native/src/doc_parser/markdown/inline.rs b/packages/common/native/src/doc_parser/markdown/inline.rs deleted file mode 100644 index aa8d9b31d09f7..0000000000000 --- a/packages/common/native/src/doc_parser/markdown/inline.rs +++ /dev/null @@ -1,71 +0,0 @@ -const INLINE_ATTR_BOLD: &str = "bold"; -const INLINE_ATTR_ITALIC: &str = "italic"; -const INLINE_ATTR_UNDERLINE: &str = "underline"; -const INLINE_ATTR_STRIKE: &str = "strike"; -const INLINE_ATTR_CODE: &str = "code"; -const INLINE_ATTR_LINK: &str = "link"; -const INLINE_ATTR_REFERENCE: &str = "reference"; -const INLINE_ATTR_COLOR: &str = "color"; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum InlineStyle { - Bold, - Italic, - Underline, - Strike, - Code, - Link, - Reference, - Color, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) struct InlineDelimiter { - pub(super) open: &'static str, - pub(super) close: &'static str, -} - -impl InlineStyle { - pub(super) fn key(self) -> &'static str { - match self { - InlineStyle::Bold => INLINE_ATTR_BOLD, - InlineStyle::Italic => INLINE_ATTR_ITALIC, - InlineStyle::Underline => INLINE_ATTR_UNDERLINE, - InlineStyle::Strike => INLINE_ATTR_STRIKE, - InlineStyle::Code => INLINE_ATTR_CODE, - InlineStyle::Link => INLINE_ATTR_LINK, - InlineStyle::Reference => INLINE_ATTR_REFERENCE, - InlineStyle::Color => INLINE_ATTR_COLOR, - } - } - - pub(super) fn from_key(key: &str) -> Option { - match key { - INLINE_ATTR_BOLD => Some(InlineStyle::Bold), - INLINE_ATTR_ITALIC => Some(InlineStyle::Italic), - INLINE_ATTR_UNDERLINE => Some(InlineStyle::Underline), - INLINE_ATTR_STRIKE => Some(InlineStyle::Strike), - INLINE_ATTR_CODE => Some(InlineStyle::Code), - INLINE_ATTR_LINK => Some(InlineStyle::Link), - INLINE_ATTR_REFERENCE => Some(InlineStyle::Reference), - INLINE_ATTR_COLOR => Some(InlineStyle::Color), - _ => None, - } - } - - pub(super) fn delimiter(self) -> Option { - match self { - InlineStyle::Italic => Some(InlineDelimiter { open: "_", close: "_" }), - InlineStyle::Bold => Some(InlineDelimiter { - open: "**", - close: "**", - }), - InlineStyle::Strike => Some(InlineDelimiter { - open: "~~", - close: "~~", - }), - InlineStyle::Code => Some(InlineDelimiter { open: "`", close: "`" }), - InlineStyle::Link | InlineStyle::Reference | InlineStyle::Underline | InlineStyle::Color => None, - } - } -} diff --git a/packages/common/native/src/doc_parser/markdown/mod.rs b/packages/common/native/src/doc_parser/markdown/mod.rs deleted file mode 100644 index 0a036bc7c7f5f..0000000000000 --- a/packages/common/native/src/doc_parser/markdown/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -mod delta; -mod inline; -mod parser; -mod render; - -pub(crate) use delta::{ - DeltaToMdOptions, InlineReferencePayload, delta_value_to_inline_markdown, extract_inline_references, - extract_inline_references_from_value, text_to_inline_markdown, -}; -#[cfg(test)] -pub(crate) use parser::MAX_MARKDOWN_CHARS; -pub(crate) use parser::{MAX_BLOCKS, parse_markdown_blocks}; -pub(crate) use render::{MarkdownRenderer, MarkdownWriter}; diff --git a/packages/common/native/src/doc_parser/markdown/parser.rs b/packages/common/native/src/doc_parser/markdown/parser.rs deleted file mode 100644 index efe6db07615fc..0000000000000 --- a/packages/common/native/src/doc_parser/markdown/parser.rs +++ /dev/null @@ -1,1723 +0,0 @@ -//! Shared markdown utilities for the doc_parser module - -use std::collections::HashMap; - -use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, Options, Parser, Tag, TagEnd}; -use y_octo::{Any, TextAttributes, TextDeltaOp, TextInsert}; - -use super::{ - super::{ - ParseError, - block_spec::{ - BlockFlavour, BlockNode, BlockSpec, BlockType, BookmarkSpec, EmbedIframeSpec, EmbedYoutubeSpec, ImageSpec, - TableSpec, count_tree_nodes, - }, - }, - inline::InlineStyle, -}; - -const DEFAULT_CODE_LANG: &str = "plain text"; -pub(crate) const MAX_MARKDOWN_CHARS: usize = 200_000; -pub(crate) const MAX_BLOCKS: usize = 2_000; - -fn markdown_options() -> Options { - Options::ENABLE_STRIKETHROUGH - | Options::ENABLE_TABLES - | Options::ENABLE_TASKLISTS - | Options::ENABLE_HEADING_ATTRIBUTES -} - -impl BlockType { - pub fn from_heading_level(level: HeadingLevel) -> Self { - match level { - HeadingLevel::H1 => BlockType::H1, - HeadingLevel::H2 => BlockType::H2, - HeadingLevel::H3 => BlockType::H3, - HeadingLevel::H4 => BlockType::H4, - HeadingLevel::H5 => BlockType::H5, - HeadingLevel::H6 => BlockType::H6, - } - } -} - -#[derive(Debug, Clone)] -pub(super) struct MarkdownDocument { - pub blocks: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct InlineAttr { - style: InlineStyle, - value: Option, -} - -impl InlineAttr { - fn new(style: InlineStyle) -> Self { - Self { style, value: None } - } - - fn color(value: String) -> Self { - Self { - style: InlineStyle::Color, - value: Some(value), - } - } - - fn link(url: String) -> Self { - Self { - style: InlineStyle::Link, - value: Some(url), - } - } - - fn key(&self) -> &'static str { - self.style.key() - } -} - -#[derive(Debug, Default)] -struct InlineState { - stack: Vec, -} - -impl InlineState { - fn push(&mut self, attr: InlineAttr) { - self.stack.push(attr); - } - - fn pop(&mut self, attr: InlineAttr) { - if let Some(pos) = self.stack.iter().rposition(|item| item.key() == attr.key()) { - self.stack.remove(pos); - } - } - - fn attrs(&self) -> Option { - if self.stack.is_empty() { - return None; - } - - let mut attrs = TextAttributes::new(); - for attr in &self.stack { - match attr.style { - InlineStyle::Link => { - if let Some(url) = attr.value.as_ref() { - attrs.insert(attr.key().into(), Any::String(url.clone())); - } - } - InlineStyle::Color => { - if let Some(color) = attr.value.as_ref() { - attrs.insert(attr.key().into(), Any::String(color.clone())); - } - } - _ => { - attrs.insert(attr.key().into(), Any::True); - } - } - } - - Some(attrs) - } - - fn attrs_with(&self, extra: InlineAttr) -> Option { - let mut attrs = self.attrs().unwrap_or_default(); - let key = extra.key(); - match extra.style { - InlineStyle::Link => { - if let Some(url) = extra.value.as_ref() { - attrs.insert(key.into(), Any::String(url.clone())); - } - } - InlineStyle::Color => { - if let Some(color) = extra.value.as_ref() { - attrs.insert(key.into(), Any::String(color.clone())); - } - } - _ => { - attrs.insert(key.into(), Any::True); - } - } - Some(attrs) - } -} - -#[derive(Debug)] -struct BlockDraft { - flavour: BlockFlavour, - block_type: Option, - checked: Option, - language: Option, - order: Option, - text: Vec, - children: Vec, -} - -impl BlockDraft { - fn new(flavour: BlockFlavour, block_type: Option) -> Self { - let block_type = if flavour == BlockFlavour::Paragraph { - block_type.or(Some(BlockType::Text)) - } else { - block_type - }; - - Self { - flavour, - block_type, - checked: None, - language: None, - order: None, - text: Vec::new(), - children: Vec::new(), - } - } - - fn push_text(&mut self, text: &str, attrs: Option) { - if text.is_empty() { - return; - } - - self.text.push(TextDeltaOp::Insert { - insert: TextInsert::Text(text.to_string()), - format: attrs, - }); - } - - fn is_empty(&self) -> bool { - self.text.is_empty() && self.children.is_empty() - } - - fn finish(self) -> BlockNode { - BlockNode { - spec: BlockSpec { - flavour: self.flavour, - block_type: self.block_type, - text: self.text, - checked: self.checked, - language: self.language, - order: self.order, - image: None, - table: None, - bookmark: None, - embed_youtube: None, - embed_iframe: None, - }, - children: self.children, - } - } -} - -#[derive(Debug)] -struct ListContext { - ordered: bool, - next_index: i64, -} - -#[derive(Debug)] -struct ImageDraft { - source: String, - caption: String, - width: Option, - height: Option, -} - -impl ImageDraft { - fn finish(self) -> Result { - let caption = if self.caption.trim().is_empty() { - None - } else { - Some(self.caption) - }; - let source_id = ImageSpec::normalize_source(&self.source)?; - Ok(ImageSpec { - source_id, - caption, - width: self.width, - height: self.height, - }) - } -} - -#[derive(Debug, Default)] -struct TableState { - rows: Vec>, - current_row: Vec, - current_cell: String, - pending_link: Option, - pending_image: Option, - row_in_progress: bool, - cell_in_progress: bool, - in_head: bool, -} - -impl TableState { - fn start_row(&mut self) { - self.current_row.clear(); - self.row_in_progress = true; - } - - fn finish_row(&mut self) { - if self.row_in_progress { - self.rows.push(std::mem::take(&mut self.current_row)); - } - self.row_in_progress = false; - } - - fn start_cell(&mut self) { - self.current_cell.clear(); - self.cell_in_progress = true; - } - - fn finish_cell(&mut self) { - if self.cell_in_progress { - let cell = self.current_cell.trim().to_string(); - self.current_row.push(cell); - } - self.current_cell.clear(); - self.cell_in_progress = false; - } - - fn push_text(&mut self, text: &str) { - self.current_cell.push_str(text); - } - - fn push_marker(&mut self, marker: &str) { - self.current_cell.push_str(marker); - } -} - -pub(crate) fn parse_markdown_blocks(markdown: &str) -> Result, ParseError> { - let normalized = normalize_markdown(markdown); - if normalized.len() > MAX_MARKDOWN_CHARS { - return Err(ParseError::ParserError("markdown_too_large".into())); - } - - validate_markdown_inner(&normalized)?; - let parsed = parse_markdown_inner(&normalized)?; - if count_tree_nodes(&parsed.blocks) > MAX_BLOCKS { - return Err(ParseError::ParserError("block_count_too_large".into())); - } - Ok(parsed.blocks) -} - -/// Parses markdown content into blocks suitable for building a ydoc. -/// -/// The first H1 can be skipped to act as the document title. -fn parse_markdown_inner(markdown: &str) -> Result { - let parser = Parser::new_ext(markdown, markdown_options()); - - let mut blocks: Vec = Vec::new(); - - let mut inline = InlineState::default(); - let mut list_stack: Vec = Vec::new(); - let mut list_items: Vec = Vec::new(); - let mut active: Option = None; - let mut in_blockquote = false; - let mut pending_image: Option = None; - let mut pending_bookmark: Option = None; - let mut table_state: Option = None; - let mut span_stack: Vec = Vec::new(); - - for event in parser { - let mut table_completed: Option>> = None; - let mut table_handled = false; - if let Some(state) = table_state.as_mut() { - match &event { - Event::Start(Tag::TableHead) => { - state.in_head = true; - table_handled = true; - } - Event::End(TagEnd::TableHead) => { - if state.cell_in_progress { - state.finish_cell(); - } - state.finish_row(); - state.in_head = false; - table_handled = true; - } - Event::Start(Tag::TableRow) => { - state.start_row(); - table_handled = true; - } - Event::End(TagEnd::TableRow) => { - if state.cell_in_progress { - state.finish_cell(); - } - state.finish_row(); - table_handled = true; - } - Event::Start(Tag::TableCell) => { - if state.in_head && !state.row_in_progress { - state.start_row(); - } - state.start_cell(); - table_handled = true; - } - Event::End(TagEnd::TableCell) => { - state.finish_cell(); - table_handled = true; - } - Event::Start(Tag::Image { dest_url, .. }) => { - state.pending_image = Some(ImageDraft { - source: dest_url.to_string(), - caption: String::new(), - width: None, - height: None, - }); - table_handled = true; - } - Event::End(TagEnd::Image) => { - if let Some(image) = state.pending_image.take() { - let alt = image.caption.trim(); - let src = image.source; - let fragment = if alt.is_empty() { - format!("![]({src})") - } else { - format!("![{alt}]({src})") - }; - state.push_text(&fragment); - } - table_handled = true; - } - Event::Text(text) => { - if let Some(image) = state.pending_image.as_mut() { - image.caption.push_str(text); - } else { - state.push_text(text); - } - table_handled = true; - } - Event::Code(code) => { - let fragment = format!("`{code}`"); - state.push_text(&fragment); - table_handled = true; - } - Event::SoftBreak | Event::HardBreak => { - state.push_text(" "); - table_handled = true; - } - Event::Start(Tag::Strong) => { - state.push_marker("**"); - table_handled = true; - } - Event::End(TagEnd::Strong) => { - state.push_marker("**"); - table_handled = true; - } - Event::Start(Tag::Emphasis) => { - state.push_marker("_"); - table_handled = true; - } - Event::End(TagEnd::Emphasis) => { - state.push_marker("_"); - table_handled = true; - } - Event::Start(Tag::Strikethrough) => { - state.push_marker("~~"); - table_handled = true; - } - Event::End(TagEnd::Strikethrough) => { - state.push_marker("~~"); - table_handled = true; - } - Event::Start(Tag::Link { dest_url, .. }) => { - state.push_marker("["); - state.pending_link = Some(dest_url.to_string()); - table_handled = true; - } - Event::End(TagEnd::Link) => { - if let Some(url) = state.pending_link.take() { - state.push_marker(&format!("]({url})")); - } - table_handled = true; - } - Event::Html(html) | Event::InlineHtml(html) => { - if let Some(text) = extract_wrapped_html_text(html) { - state.push_text(&text); - } else if is_html_line_break(html) { - state.push_text("\n"); - } else if let Some(tag) = parse_html_tag(html) - && matches!(tag.name.as_str(), "u" | "span") - { - // Ignore inline formatting tags inside table cells. - } else if !html.trim().is_empty() { - state.push_text(html); - } - table_handled = true; - } - Event::End(TagEnd::Table) => { - if state.cell_in_progress { - state.finish_cell(); - } - state.finish_row(); - table_completed = Some(std::mem::take(&mut state.rows)); - table_handled = true; - } - _ => {} - } - } - if let Some(rows) = table_completed { - table_state = None; - attach_block(table_block(rows), &mut list_items, &mut blocks); - continue; - } - if table_handled { - continue; - } - if matches!(event, Event::Start(Tag::Table(_))) { - if let Some(block) = active.take() { - attach_block(block.finish(), &mut list_items, &mut blocks); - } - table_state = Some(TableState::default()); - continue; - } - - match event { - Event::Start(Tag::Heading { level, .. }) => { - active = Some(BlockDraft::new( - BlockFlavour::Paragraph, - Some(BlockType::from_heading_level(level)), - )); - } - Event::End(TagEnd::Heading(_)) => { - if let Some(block) = active.take() { - attach_block(block.finish(), &mut list_items, &mut blocks); - } - } - Event::Start(Tag::Paragraph) => { - if in_blockquote { - if active.is_none() { - active = Some(BlockDraft::new(BlockFlavour::Paragraph, Some(BlockType::Quote))); - } - } else if list_items.is_empty() { - active = Some(BlockDraft::new(BlockFlavour::Paragraph, Some(BlockType::Text))); - } - } - Event::End(TagEnd::Paragraph) => { - if let Some(url) = pending_bookmark.take() - && active.as_ref().is_some_and(|block| block.is_empty()) - { - active = None; - attach_block(bookmark_block(url), &mut list_items, &mut blocks); - continue; - } - if in_blockquote { - if let Some(block) = active.as_mut() { - block.push_text("\n", None); - } - } else if let Some(block) = active.take() { - attach_block(block.finish(), &mut list_items, &mut blocks); - } - } - Event::Start(Tag::BlockQuote(_)) => { - in_blockquote = true; - } - Event::End(TagEnd::BlockQuote(_)) => { - in_blockquote = false; - if let Some(block) = active.take() { - attach_block(block.finish(), &mut list_items, &mut blocks); - } - } - Event::Start(Tag::List(start_num)) => { - let start = start_num.unwrap_or(1) as i64; - list_stack.push(ListContext { - ordered: start_num.is_some(), - next_index: start, - }); - } - Event::End(TagEnd::List(_)) => { - list_stack.pop(); - } - Event::Start(Tag::Item) => { - let Some(context) = list_stack.last_mut() else { - continue; - }; - let order = if context.ordered { - let order = context.next_index; - context.next_index += 1; - Some(order) - } else { - None - }; - - let block_type = if context.ordered { - BlockType::Numbered - } else { - BlockType::Bulleted - }; - - let mut draft = BlockDraft::new(BlockFlavour::List, Some(block_type)); - draft.checked = None; - draft.order = order; - list_items.push(draft); - } - Event::End(TagEnd::Item) => { - if let Some(block) = list_items.pop() { - let finished = block.finish(); - if let Some(parent) = list_items.last_mut() { - parent.children.push(finished); - } else { - blocks.push(finished); - } - } - } - Event::TaskListMarker(checked) => { - if let Some(item) = list_items.last_mut() { - item.checked = Some(checked); - item.block_type = Some(BlockType::Todo); - item.order = None; - } - } - Event::Start(Tag::CodeBlock(kind)) => { - let mut draft = BlockDraft::new(BlockFlavour::Code, None); - match kind { - CodeBlockKind::Fenced(lang) => { - if !lang.is_empty() { - draft.language = Some(lang.to_string()); - } else { - draft.language = Some(DEFAULT_CODE_LANG.to_string()); - } - } - CodeBlockKind::Indented => { - draft.language = Some(DEFAULT_CODE_LANG.to_string()); - } - } - active = Some(draft); - } - Event::End(TagEnd::CodeBlock) => { - if let Some(block) = active.take() { - attach_block(block.finish(), &mut list_items, &mut blocks); - } - } - Event::Start(Tag::Image { dest_url, .. }) => { - if let Some(block) = active.take() - && !block.is_empty() - { - attach_block(block.finish(), &mut list_items, &mut blocks); - } - pending_image = Some(ImageDraft { - source: dest_url.to_string(), - caption: String::new(), - width: None, - height: None, - }); - } - Event::End(TagEnd::Image) => { - if let Some(image) = pending_image.take() { - let image = image.finish()?; - attach_block(image_block(image), &mut list_items, &mut blocks); - } - } - Event::Text(text) => { - if pending_bookmark.is_some() && !text.trim().is_empty() { - pending_bookmark = None; - } - if let Some(image) = pending_image.as_mut() { - image.caption.push_str(&text); - } else if let Some(block) = active.as_mut() { - let attrs = inline.attrs(); - block.push_text(&text, attrs); - } else if let Some(item) = list_items.last_mut() { - let attrs = inline.attrs(); - item.push_text(&text, attrs); - } - } - Event::Html(html) | Event::InlineHtml(html) => { - if is_ai_editable_comment(&html) { - continue; - } - if let Some((text, attrs)) = parse_wrapped_inline_html(&html) { - if let Some(image) = pending_image.as_mut() { - image.caption.push_str(&text); - } else if let Some(block) = active.as_mut() { - block.push_text(&text, attrs); - } else if let Some(item) = list_items.last_mut() { - item.push_text(&text, attrs); - } - continue; - } - if handle_inline_html_tag(&html, &mut inline, &mut span_stack) { - continue; - } - if let Some(image) = parse_img_tag(&html) { - if let Some(block) = active.take() - && !block.is_empty() - { - attach_block(block.finish(), &mut list_items, &mut blocks); - } - let image = image.finish()?; - attach_block(image_block(image), &mut list_items, &mut blocks); - } else if let Some(embed) = parse_iframe_tag(&html) { - if let Some(block) = active.take() - && !block.is_empty() - { - attach_block(block.finish(), &mut list_items, &mut blocks); - } - match embed { - IframeEmbed::Youtube(video_id) => { - attach_block(embed_youtube_block(video_id), &mut list_items, &mut blocks); - } - IframeEmbed::Iframe(url) => { - attach_block(embed_iframe_block(url), &mut list_items, &mut blocks); - } - } - } else if is_html_line_break(&html) { - if let Some(image) = pending_image.as_mut() { - image.caption.push(' '); - } else if let Some(block) = active.as_mut() { - let attrs = inline.attrs(); - block.push_text("\n", attrs); - } else if let Some(item) = list_items.last_mut() { - let attrs = inline.attrs(); - item.push_text("\n", attrs); - } - } else if let Some(block) = active.as_mut() { - let attrs = inline.attrs(); - block.push_text(&html, attrs); - } else if let Some(item) = list_items.last_mut() { - let attrs = inline.attrs(); - item.push_text(&html, attrs); - } else if !html.trim().is_empty() { - let mut draft = BlockDraft::new(BlockFlavour::Code, None); - draft.language = Some("html".to_string()); - draft.push_text(&html, None); - attach_block(draft.finish(), &mut list_items, &mut blocks); - } - } - Event::Code(code) => { - if pending_bookmark.is_some() && !code.trim().is_empty() { - pending_bookmark = None; - } - if let Some(image) = pending_image.as_mut() { - image.caption.push_str(&code); - } else { - let attrs = inline.attrs_with(InlineAttr::new(InlineStyle::Code)); - if let Some(block) = active.as_mut() { - block.push_text(&code, attrs); - } else if let Some(item) = list_items.last_mut() { - item.push_text(&code, attrs); - } - } - } - Event::SoftBreak | Event::HardBreak => { - if pending_bookmark.is_some() { - pending_bookmark = None; - } - if let Some(image) = pending_image.as_mut() { - image.caption.push(' '); - } else { - let break_text = if matches!(active.as_ref().map(|b| b.flavour), Some(BlockFlavour::Code)) { - "\n" - } else { - " " - }; - if let Some(block) = active.as_mut() { - let attrs = inline.attrs(); - block.push_text(break_text, attrs); - } else if let Some(item) = list_items.last_mut() { - let attrs = inline.attrs(); - item.push_text(break_text, attrs); - } - } - } - Event::Rule => { - let divider = BlockDraft::new(BlockFlavour::Divider, None).finish(); - attach_block(divider, &mut list_items, &mut blocks); - } - Event::Start(Tag::Strong) => inline.push(InlineAttr::new(InlineStyle::Bold)), - Event::End(TagEnd::Strong) => inline.pop(InlineAttr::new(InlineStyle::Bold)), - Event::Start(Tag::Emphasis) => inline.push(InlineAttr::new(InlineStyle::Italic)), - Event::End(TagEnd::Emphasis) => inline.pop(InlineAttr::new(InlineStyle::Italic)), - Event::Start(Tag::Strikethrough) => inline.push(InlineAttr::new(InlineStyle::Strike)), - Event::End(TagEnd::Strikethrough) => inline.pop(InlineAttr::new(InlineStyle::Strike)), - Event::Start(Tag::Link { dest_url, .. }) => { - if let Some(url) = parse_bookmark_url(&dest_url) - && active - .as_ref() - .is_some_and(|block| block.flavour == BlockFlavour::Paragraph && block.is_empty()) - && list_items.is_empty() - { - pending_bookmark = Some(url); - } else { - inline.push(InlineAttr::link(dest_url.to_string())); - } - } - Event::End(TagEnd::Link) if pending_bookmark.is_none() => { - inline.pop(InlineAttr::new(InlineStyle::Link)); - } - _ => {} - } - } - - if let Some(block) = active.take() { - attach_block(block.finish(), &mut list_items, &mut blocks); - } - if let Some(image) = pending_image.take() { - let image = image.finish()?; - attach_block(image_block(image), &mut list_items, &mut blocks); - } - if let Some(mut state) = table_state.take() { - state.finish_row(); - if !state.rows.is_empty() { - attach_block(table_block(state.rows), &mut list_items, &mut blocks); - } - } - - Ok(MarkdownDocument { blocks }) -} - -fn validate_markdown_inner(markdown: &str) -> Result<(), ParseError> { - let parser = Parser::new_ext(markdown, markdown_options()); - - for event in parser { - match event { - Event::Start(tag) => ensure_supported_tag(&tag)?, - Event::Html(html) | Event::InlineHtml(html) => { - if is_ai_editable_comment(&html) { - continue; - } - if parse_img_tag(&html).is_some() { - continue; - } - if parse_iframe_tag(&html).is_some() { - continue; - } - if is_html_line_break(&html) { - continue; - } - if is_supported_inline_html(&html) { - continue; - } - return Err(ParseError::ParserError("unsupported_markdown:html".into())); - } - Event::FootnoteReference(_) => { - return Err(ParseError::ParserError("unsupported_markdown:footnote".into())); - } - Event::InlineMath(_) | Event::DisplayMath(_) => { - return Err(ParseError::ParserError("unsupported_markdown:math".into())); - } - _ => {} - } - } - - Ok(()) -} - -fn ensure_supported_tag(tag: &Tag) -> Result<(), ParseError> { - match tag { - Tag::Paragraph - | Tag::Heading { .. } - | Tag::BlockQuote(_) - | Tag::CodeBlock(_) - | Tag::List(_) - | Tag::Item - | Tag::Emphasis - | Tag::Strong - | Tag::Strikethrough - | Tag::Link { .. } - | Tag::Image { .. } - | Tag::Table(_) - | Tag::TableHead - | Tag::TableRow - | Tag::TableCell => Ok(()), - Tag::HtmlBlock => Ok(()), - Tag::FootnoteDefinition(_) => Err(ParseError::ParserError("unsupported_markdown:footnote".into())), - Tag::DefinitionList | Tag::DefinitionListTitle | Tag::DefinitionListDefinition => { - Err(ParseError::ParserError("unsupported_markdown:definition_list".into())) - } - Tag::Superscript => Err(ParseError::ParserError("unsupported_markdown:superscript".into())), - Tag::Subscript => Err(ParseError::ParserError("unsupported_markdown:subscript".into())), - Tag::MetadataBlock(_) => Err(ParseError::ParserError("unsupported_markdown:metadata".into())), - } -} - -fn parse_img_tag(html: &str) -> Option { - let tag = html.trim(); - if !tag.starts_with("().ok()); - let height = attrs.get("height").and_then(|value| value.parse::().ok()); - Some(ImageDraft { - source, - caption: caption.unwrap_or_default(), - width, - height, - }) -} - -enum IframeEmbed { - Youtube(String), - Iframe(String), -} - -fn parse_iframe_tag(html: &str) -> Option { - let tag = html.trim(); - if !tag.to_ascii_lowercase().starts_with(" Option { - let src = src.trim(); - let prefix = "https://www.youtube.com/embed/"; - if !src.starts_with(prefix) { - return None; - } - let id = &src[prefix.len()..]; - let id = id.split(['?', '#', '/']).next().unwrap_or(""); - if id.is_empty() { None } else { Some(id.to_string()) } -} - -const AFFINE_DOMAINS: [&str; 6] = [ - "affine.pro", - "app.affine.pro", - "insider.affine.pro", - "affine.fail", - "toeverything.app", - "apple.getaffineapp.com", -]; - -fn is_valid_generic_embed_url(url: &str) -> bool { - let Some(host) = parse_https_host(url) else { - return false; - }; - !AFFINE_DOMAINS - .iter() - .any(|domain| host == *domain || host.ends_with(&format!(".{domain}"))) -} - -fn parse_https_host(url: &str) -> Option { - let trimmed = url.trim(); - if trimmed.is_empty() { - return None; - } - let lower = trimmed.to_ascii_lowercase(); - let (scheme, rest) = lower.split_once("://")?; - if scheme != "https" { - return None; - } - let host_port = rest.split(&['/', '?', '#'][..]).next().unwrap_or(""); - if host_port.is_empty() { - return None; - } - let host = host_port.split('@').next_back().unwrap_or(""); - let host = host.split(':').next().unwrap_or(""); - if host.is_empty() { - return None; - } - Some(host.to_string()) -} - -fn is_ai_editable_comment(html: &str) -> bool { - let trimmed = html.trim(); - if !trimmed.starts_with("") { - return false; - } - let body = trimmed.trim_start_matches("").trim(); - body.contains("block_id=") && body.contains("flavour=") -} - -fn is_html_line_break(html: &str) -> bool { - let trimmed = html.trim(); - if !trimmed.starts_with('<') || !trimmed.ends_with('>') { - return false; - } - let inner = trimmed.trim_start_matches('<').trim_end_matches('>').trim(); - let inner = inner.trim_end_matches('/').trim(); - inner.eq_ignore_ascii_case("br") -} - -fn parse_bookmark_url(dest_url: &str) -> Option { - let (prefix, url) = dest_url.split_once(',')?; - if !prefix.trim().eq_ignore_ascii_case("bookmark") { - return None; - } - let url = url.trim(); - if url.is_empty() { None } else { Some(url.to_string()) } -} - -fn normalize_markdown(markdown: &str) -> String { - if !markdown.contains('<') { - return markdown.to_string(); - } - normalize_html_lists(markdown) -} - -#[derive(Debug, Clone, Copy)] -enum ListKind { - Unordered, - Ordered, -} - -#[derive(Debug, Clone, Copy)] -struct ListState { - kind: ListKind, - counter: usize, -} - -fn normalize_html_lists(markdown: &str) -> String { - if !markdown.contains(" = None; - let mut list_stack: Vec = Vec::new(); - - for chunk in markdown.split_inclusive('\n') { - let line = chunk.strip_suffix('\n').unwrap_or(chunk); - let newline = if chunk.ends_with('\n') { "\n" } else { "" }; - let trimmed = line.trim_start(); - - if let Some(marker) = fence_marker_start(trimmed) { - if !in_fence { - in_fence = true; - fence_marker = Some(marker.to_string()); - } else if fence_marker.as_deref() == Some(marker) { - in_fence = false; - fence_marker = None; - } - out.push_str(line); - out.push_str(newline); - continue; - } - - if in_fence { - out.push_str(line); - out.push_str(newline); - continue; - } - - let normalized_line = normalize_html_lists_line(line, &mut list_stack); - out.push_str(&normalized_line); - out.push_str(newline); - } - - out -} - -fn fence_marker_start(line: &str) -> Option<&'static str> { - if line.starts_with("```") { - Some("```") - } else if line.starts_with("~~~") { - Some("~~~") - } else { - None - } -} - -fn normalize_html_lists_line(line: &str, list_stack: &mut Vec) -> String { - let mut out = String::with_capacity(line.len()); - let bytes = line.as_bytes(); - let mut i = 0; - - while i < line.len() { - if bytes[i] == b'<' - && let Some(rel_end) = line[i..].find('>') - { - let end = i + rel_end; - let tag = &line[i..=end]; - if let Some(tag_info) = parse_html_tag(tag) { - match tag_info.name.as_str() { - "ul" => { - if tag_info.closing { - list_stack.pop(); - } else if !tag_info.self_closing { - list_stack.push(ListState { - kind: ListKind::Unordered, - counter: 0, - }); - } - } - "ol" => { - if tag_info.closing { - list_stack.pop(); - } else if !tag_info.self_closing { - list_stack.push(ListState { - kind: ListKind::Ordered, - counter: 0, - }); - } - } - "li" => { - if tag_info.closing { - if !out.ends_with('\n') { - out.push('\n'); - } - } else { - if !out.is_empty() && !out.ends_with('\n') { - out.push('\n'); - } - let depth = list_stack.len().saturating_sub(1); - if depth > 0 { - out.push_str(&" ".repeat(depth)); - } - let prefix = match list_stack.last_mut() { - Some(state) => match state.kind { - ListKind::Unordered => "- ".to_string(), - ListKind::Ordered => { - state.counter += 1; - format!("{}. ", state.counter) - } - }, - None => "- ".to_string(), - }; - out.push_str(&prefix); - } - } - _ => out.push_str(tag), - } - i = end + 1; - continue; - } - } - - let ch = line[i..].chars().next().unwrap(); - out.push(ch); - i += ch.len_utf8(); - } - - out -} - -#[derive(Debug, Clone)] -struct HtmlTag { - name: String, - closing: bool, - self_closing: bool, - attrs: HashMap, -} - -fn parse_html_tag(html: &str) -> Option { - let trimmed = html.trim(); - if !trimmed.starts_with('<') || !trimmed.ends_with('>') { - return None; - } - if trimmed.starts_with("\n")); - } - markdown.push_str(&block_markdown); - } - - Ok(MarkdownResult { - title: doc_title, - markdown, - known_unsupported_blocks, - unknown_blocks, - }) -} - -pub fn parse_doc_from_binary(doc_bin: Vec, doc_id: String) -> Result { - let doc = load_doc(&doc_bin, Some(doc_id.as_str()))?; - - let blocks_map = doc.get_map("blocks")?; - if blocks_map.is_empty() { - return Err(ParseError::ParserError("blocks map is empty".into())); - } - - let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR) - .ok_or_else(|| ParseError::ParserError("root block not found".into()))?; - let mut walker = context.walker(); - let mut blocks: Vec = Vec::with_capacity(context.block_pool.len()); - let mut doc_title = String::new(); - let mut summary = SummaryBuilder::new(SUMMARY_LIMIT as isize); - - while let Some((parent_block_id, block_id)) = walker.next() { - let block = match context.block_pool.get(&block_id) { - Some(block) => block, - None => continue, - }; - - let flavour = match get_flavour(block) { - Some(flavour) => flavour, - None => continue, - }; - - let parent_block = parent_block_id.as_ref().and_then(|id| context.block_pool.get(id)); - let parent_flavour = parent_block.and_then(get_flavour); - - let note_block = nearest_by_flavour(&block_id, NOTE_FLAVOUR, &context.parent_lookup, &context.block_pool); - let note_block_id = note_block.as_ref().and_then(get_block_id); - let display_mode = determine_display_mode(note_block.as_ref()); - - // enqueue children first to keep traversal order similar to JS implementation - walker.enqueue_children(&block_id, block); - - let build_block = |database_name: Option<&String>| { - BlockInfo::base( - &block_id, - &flavour, - parent_flavour.as_ref(), - parent_block_id.as_ref(), - compose_additional(&display_mode, note_block_id.as_ref(), database_name), - ) - }; - - if flavour == PAGE_FLAVOUR { - let title = get_string(block, "prop:title").unwrap_or_default(); - doc_title = title.clone(); - let mut info = build_block(None); - info.content = Some(vec![title]); - blocks.push(info); - continue; - } - - if matches!(flavour.as_str(), "affine:paragraph" | "affine:list" | "affine:code") { - if let Some(text) = block.get("prop:text").and_then(|value| value.to_text()) { - let database_name = if flavour == "affine:paragraph" && parent_flavour.as_deref() == Some("affine:database") { - parent_block.and_then(|map| get_string(map, "prop:title")) - } else { - None - }; - - let content = text.to_string(); - let text_len = text.len() as usize; - let refs = extract_inline_references(&text.to_delta()); - - let mut info = build_block(database_name.as_ref()); - info.content = Some(vec![content.clone()]); - if !refs.is_empty() { - info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect()); - info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect()); - } - blocks.push(info); - summary.push_text(&content, text_len); - } - continue; - } - - if matches!(flavour.as_str(), "affine:embed-linked-doc" | "affine:embed-synced-doc") { - if let Some(page_id) = get_string(block, "prop:pageId") { - let mut info = build_block(None); - let payload = embed_ref_payload(block, &page_id); - apply_doc_ref(&mut info, page_id, payload); - blocks.push(info); - } - continue; - } - - if flavour == "affine:attachment" { - if let Some(blob_id) = get_string(block, "prop:sourceId") { - let mut info = build_block(None); - let name = get_string(block, "prop:name").unwrap_or_default(); - apply_blob_info(&mut info, blob_id, name); - blocks.push(info); - } - continue; - } - - if flavour == "affine:image" { - let image = ImageSpec::from_block_map(block); - if !image.source_id.is_empty() { - let mut info = build_block(None); - let caption = image.caption.unwrap_or_default(); - apply_blob_info(&mut info, image.source_id, caption); - blocks.push(info); - } - continue; - } - - if flavour == "affine:surface" { - let texts = gather_surface_texts(block); - let mut info = build_block(None); - info.content = Some(texts); - blocks.push(info); - continue; - } - - if flavour == "affine:database" { - let (texts, database_name) = gather_database_texts(block); - let mut info = BlockInfo::base( - &block_id, - &flavour, - parent_flavour.as_ref(), - parent_block_id.as_ref(), - compose_additional(&display_mode, note_block_id.as_ref(), database_name.as_ref()), - ); - info.content = Some(texts); - let refs = collect_database_cell_references(block); - if !refs.is_empty() { - info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect()); - info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect()); - } - blocks.push(info); - continue; - } - - if flavour == "affine:latex" { - if let Some(content) = get_string(block, "prop:latex") { - let mut info = build_block(None); - info.content = Some(vec![content]); - blocks.push(info); - } - continue; - } - - if flavour == "affine:table" { - let contents = table_cell_texts(block); - let mut info = build_block(None); - info.content = Some(contents); - blocks.push(info); - continue; - } - - if BOOKMARK_FLAVOURS.contains(&flavour.as_str()) { - blocks.push(build_block(None)); - } - } - - if doc_title.is_empty() { - doc_title = DEFAULT_PAGE_TITLE.into(); - } - - Ok(CrawlResult { - blocks, - title: doc_title, - summary: summary.into_string(), - }) -} - -pub fn get_doc_ids_from_binary(doc_bin: Vec, include_trash: bool) -> Result, ParseError> { - let doc = load_doc(&doc_bin, None)?; - - let mut doc_ids = Vec::new(); - let meta = doc.get_map("meta")?; - let pages_value = meta.get("pages"); - if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) { - for page_val in pages.iter() { - if let Some(page) = page_val.to_map() { - let id = get_string(&page, "id"); - if let Some(id) = id { - let trash = page - .get("trash") - .and_then(|v| match v.to_any() { - Some(Any::True) => Some(true), - Some(Any::False) => Some(false), - _ => None, - }) - .unwrap_or(false); - - if include_trash || !trash { - doc_ids.push(id); - } - } - } - } - return Ok(doc_ids); - } - - if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) { - for entry in entries { - let Any::Object(map) = entry else { - continue; - }; - let id = map.get("id").and_then(any_as_string).map(str::to_string); - if let Some(id) = id { - let trash = map.get("trash").map(any_truthy).unwrap_or(false); - if include_trash || !trash { - doc_ids.push(id); - } - } - } - } - - Ok(doc_ids) -} - -fn block_level(block_id: &str, root_id: &str, parent_lookup: &HashMap) -> usize { - let mut level = 0; - let mut cursor = block_id; - while let Some(parent) = parent_lookup.get(cursor) { - level += 1; - if parent == root_id { - break; - } - cursor = parent; - } - level -} - -pub(super) fn text_content(block: &Map, key: &str) -> Option<(String, usize)> { - block.get(key).and_then(|value| { - value.to_text().map(|text| { - let content = text.to_string(); - let len = text.len() as usize; - (content, len) - }) - }) -} - -fn determine_display_mode(note_block: Option<&Map>) -> String { - match note_block.and_then(|block| get_string(block, "prop:displayMode")) { - Some(mode) if mode == "both" => "page".into(), - Some(mode) => mode, - None => "edgeless".into(), - } -} - -fn compose_additional( - display_mode: &str, - note_block_id: Option<&String>, - database_name: Option<&String>, -) -> Option { - let mut payload = JsonMap::new(); - payload.insert("displayMode".into(), JsonValue::String(display_mode.to_string())); - if let Some(note_id) = note_block_id { - payload.insert("noteBlockId".into(), JsonValue::String(note_id.clone())); - } - if let Some(name) = database_name { - payload.insert("databaseName".into(), JsonValue::String(name.clone())); - } - Some(JsonValue::Object(payload).to_string()) -} - -fn apply_blob_info(info: &mut BlockInfo, blob_id: String, content: String) { - info.blob = Some(vec![blob_id]); - info.content = Some(vec![content]); -} - -fn apply_doc_ref(info: &mut BlockInfo, page_id: String, payload: Option) { - info.ref_doc_id = Some(vec![page_id]); - if let Some(payload) = payload { - info.ref_info = Some(vec![payload]); - } -} - -fn embed_ref_payload(block: &Map, page_id: &str) -> Option { - let params = block.get("prop:params").as_ref().and_then(params_value_to_json); - Some(build_reference_payload(page_id, params)) -} - -fn gather_surface_texts(block: &Map) -> Vec { - let mut texts = Vec::new(); - let elements = match block.get("prop:elements").and_then(|value| value.to_map()) { - Some(map) => map, - None => return texts, - }; - - if elements - .get("type") - .and_then(|value| value_to_string(&value)) - .as_deref() - != Some("$blocksuite:internal:native$") - { - return texts; - } - - if let Some(value_map) = elements.get("value").and_then(|value| value.to_map()) { - for value in value_map.values() { - if let Some(element) = value.to_map() - && let Some(text) = element.get("text").and_then(|value| value.to_text()) - { - texts.push(text.to_string()); - } - } - } - - texts.sort(); - texts -} - -fn table_cell_texts(block: &Map) -> Vec { - let mut contents = Vec::new(); - for key in block.keys() { - if key.starts_with("prop:cells.") - && key.ends_with(".text") - && let Some(value) = block.get(key).and_then(|value| value_to_string(&value)) - && !value.is_empty() - { - contents.push(value); - } - } - contents -} - -pub(super) fn text_content_for_summary(block: &Map, key: &str) -> Option<(String, usize)> { - if let Some((text, len)) = text_content(block, key) { - return Some((text, len)); - } - - block.get(key).and_then(|value| { - value_to_string(&value).map(|text| { - let len = text.chars().count(); - (text, len) - }) - }) -} - -#[cfg(test)] -mod tests { - use serde_json::json; - use y_octo::{AHashMap, Any, DocOptions, TextAttributes, TextDeltaOp, TextInsert, Value}; - - use super::*; - use crate::doc_parser::build_full_doc; - - #[test] - fn test_parse_doc_from_binary() { - let json = include_bytes!("../../../fixtures/demo.ydoc.json"); - let doc_bin = include_bytes!("../../../fixtures/demo.ydoc").to_vec(); - let doc_id = "dYpV7PPhk8amRkY5IAcVO".to_string(); - - let result = parse_doc_from_binary(doc_bin, doc_id).unwrap(); - let config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict) - .numeric_mode(assert_json_diff::NumericMode::AssumeFloat); - assert_json_diff::assert_json_matches!( - serde_json::from_slice::(json).unwrap(), - serde_json::json!(result), - config - ); - } - - #[test] - fn test_database_cell_references() { - let doc_id = "doc-with-db".to_string(); - let doc = DocOptions::new().with_guid(doc_id.clone()).build(); - let mut blocks = doc.get_or_create_map("blocks").unwrap(); - - let mut page = doc.create_map().unwrap(); - page.insert("sys:id".into(), "page").unwrap(); - page.insert("sys:flavour".into(), "affine:page").unwrap(); - let mut page_children = doc.create_array().unwrap(); - page_children.push("note").unwrap(); - page.insert("sys:children".into(), Value::Array(page_children)).unwrap(); - let mut page_title = doc.create_text().unwrap(); - page_title.insert(0, "Page").unwrap(); - page.insert("prop:title".into(), Value::Text(page_title)).unwrap(); - blocks.insert("page".into(), Value::Map(page)).unwrap(); - - let mut note = doc.create_map().unwrap(); - note.insert("sys:id".into(), "note").unwrap(); - note.insert("sys:flavour".into(), "affine:note").unwrap(); - let mut note_children = doc.create_array().unwrap(); - note_children.push("db").unwrap(); - note.insert("sys:children".into(), Value::Array(note_children)).unwrap(); - note.insert("prop:displayMode".into(), "page").unwrap(); - blocks.insert("note".into(), Value::Map(note)).unwrap(); - - let mut db = doc.create_map().unwrap(); - db.insert("sys:id".into(), "db").unwrap(); - db.insert("sys:flavour".into(), "affine:database").unwrap(); - db.insert("sys:children".into(), Value::Array(doc.create_array().unwrap())) - .unwrap(); - let mut db_title = doc.create_text().unwrap(); - db_title.insert(0, "Database").unwrap(); - db.insert("prop:title".into(), Value::Text(db_title)).unwrap(); - - let mut columns = doc.create_array().unwrap(); - let mut column = doc.create_map().unwrap(); - column.insert("id".into(), "col1").unwrap(); - column.insert("name".into(), "Text").unwrap(); - column.insert("type".into(), "rich-text").unwrap(); - column - .insert("data".into(), Value::Map(doc.create_map().unwrap())) - .unwrap(); - columns.push(Value::Map(column)).unwrap(); - db.insert("prop:columns".into(), Value::Array(columns)).unwrap(); - - let mut cell_text = doc.create_text().unwrap(); - let mut reference = AHashMap::default(); - reference.insert("pageId".into(), Any::String("target-doc".into())); - let mut params = AHashMap::default(); - params.insert("mode".into(), Any::String("page".into())); - reference.insert("params".into(), Any::Object(params)); - let mut attrs = TextAttributes::new(); - attrs.insert("reference".into(), Any::Object(reference)); - cell_text - .apply_delta(&[ - TextDeltaOp::Insert { - insert: TextInsert::Text("See ".into()), - format: None, - }, - TextDeltaOp::Insert { - insert: TextInsert::Text("Target".into()), - format: Some(attrs), - }, - ]) - .unwrap(); - - let mut cell = doc.create_map().unwrap(); - cell.insert("columnId".into(), "col1").unwrap(); - cell.insert("value".into(), Value::Text(cell_text)).unwrap(); - let mut row = doc.create_map().unwrap(); - row.insert("col1".into(), Value::Map(cell)).unwrap(); - let mut cells = doc.create_map().unwrap(); - cells.insert("row1".into(), Value::Map(row)).unwrap(); - db.insert("prop:cells".into(), Value::Map(cells)).unwrap(); - - blocks.insert("db".into(), Value::Map(db)).unwrap(); - - let doc_bin = doc.encode_update_v1().unwrap(); - let result = parse_doc_from_binary(doc_bin, doc_id).unwrap(); - let db_block = result.blocks.iter().find(|block| block.block_id == "db").unwrap(); - assert_eq!(db_block.ref_doc_id, Some(vec!["target-doc".to_string()])); - assert_eq!( - db_block.ref_info, - Some(vec![build_reference_payload( - "target-doc", - Some(json!({"mode": "page"})) - )]) - ); - } - - #[test] - fn test_parse_doc_to_markdown_ai_editable_image_table() { - let doc_id = "ai-editable-doc"; - let markdown = "![Alt](blob://image-id)\n\n| A | B |\n| --- | --- |\n| 1 | 2 |"; - let doc_bin = build_full_doc("Title", markdown, doc_id).expect("create doc"); - - let result = parse_doc_to_markdown(doc_bin, doc_id.to_string(), true, None).expect("parse doc"); - let md = result.markdown; - - assert!(md.contains("flavour=affine:image")); - assert!(md.contains("blob://image-id")); - assert!(md.contains("|A|B|")); - assert!(md.contains("|---|---|")); - assert!( - result - .known_unsupported_blocks - .iter() - .any(|block| block.ends_with(":affine:note")) - ); - assert!(result.unknown_blocks.is_empty()); - } - - #[test] - fn test_parse_doc_to_markdown_with_edgeless_text_container() { - let doc_id = "edgeless-text-doc".to_string(); - let doc = DocOptions::new().with_guid(doc_id.clone()).build(); - let mut blocks = doc.get_or_create_map("blocks").unwrap(); - - let mut page = doc.create_map().unwrap(); - page.insert("sys:id".into(), "page").unwrap(); - page.insert("sys:flavour".into(), "affine:page").unwrap(); - let mut page_children = doc.create_array().unwrap(); - page_children.push("surface").unwrap(); - page.insert("sys:children".into(), Value::Array(page_children)).unwrap(); - let mut page_title = doc.create_text().unwrap(); - page_title.insert(0, "Page").unwrap(); - page.insert("prop:title".into(), Value::Text(page_title)).unwrap(); - blocks.insert("page".into(), Value::Map(page)).unwrap(); - - let mut surface = doc.create_map().unwrap(); - surface.insert("sys:id".into(), "surface").unwrap(); - surface.insert("sys:flavour".into(), "affine:surface").unwrap(); - let mut surface_children = doc.create_array().unwrap(); - surface_children.push("edgeless-text").unwrap(); - surface - .insert("sys:children".into(), Value::Array(surface_children)) - .unwrap(); - blocks.insert("surface".into(), Value::Map(surface)).unwrap(); - - let mut edgeless_text = doc.create_map().unwrap(); - edgeless_text.insert("sys:id".into(), "edgeless-text").unwrap(); - edgeless_text - .insert("sys:flavour".into(), "affine:edgeless-text") - .unwrap(); - let mut edgeless_text_children = doc.create_array().unwrap(); - edgeless_text_children.push("paragraph").unwrap(); - edgeless_text - .insert("sys:children".into(), Value::Array(edgeless_text_children)) - .unwrap(); - blocks - .insert("edgeless-text".into(), Value::Map(edgeless_text)) - .unwrap(); - - let mut paragraph = doc.create_map().unwrap(); - paragraph.insert("sys:id".into(), "paragraph").unwrap(); - paragraph.insert("sys:flavour".into(), "affine:paragraph").unwrap(); - paragraph - .insert("sys:children".into(), Value::Array(doc.create_array().unwrap())) - .unwrap(); - paragraph.insert("prop:type".into(), "text").unwrap(); - let mut paragraph_text = doc.create_text().unwrap(); - paragraph_text.insert(0, "hello from edgeless").unwrap(); - paragraph - .insert("prop:text".into(), Value::Text(paragraph_text)) - .unwrap(); - blocks.insert("paragraph".into(), Value::Map(paragraph)).unwrap(); - - let doc_bin = doc.encode_update_v1().unwrap(); - let result = parse_doc_to_markdown(doc_bin, doc_id, false, None).expect("parse doc"); - - assert!(result.markdown.is_empty()); - assert!( - result - .known_unsupported_blocks - .contains(&"surface:affine:surface".to_string()) - ); - assert!( - result - .known_unsupported_blocks - .contains(&"edgeless-text:affine:edgeless-text".to_string()) - ); - assert!(result.unknown_blocks.is_empty()); - } - - #[test] - fn test_parse_doc_to_markdown_collects_unknown_blocks() { - let doc_id = "unknown-block-doc".to_string(); - let doc = DocOptions::new().with_guid(doc_id.clone()).build(); - let mut blocks = doc.get_or_create_map("blocks").unwrap(); - - let mut page = doc.create_map().unwrap(); - page.insert("sys:id".into(), "page").unwrap(); - page.insert("sys:flavour".into(), "affine:page").unwrap(); - let mut page_children = doc.create_array().unwrap(); - page_children.push("mystery").unwrap(); - page.insert("sys:children".into(), Value::Array(page_children)).unwrap(); - let mut page_title = doc.create_text().unwrap(); - page_title.insert(0, "Page").unwrap(); - page.insert("prop:title".into(), Value::Text(page_title)).unwrap(); - blocks.insert("page".into(), Value::Map(page)).unwrap(); - - let mut mystery = doc.create_map().unwrap(); - mystery.insert("sys:id".into(), "mystery").unwrap(); - mystery.insert("sys:flavour".into(), "affine:custom-unknown").unwrap(); - let mut mystery_children = doc.create_array().unwrap(); - mystery_children.push("paragraph").unwrap(); - mystery - .insert("sys:children".into(), Value::Array(mystery_children)) - .unwrap(); - blocks.insert("mystery".into(), Value::Map(mystery)).unwrap(); - - let mut paragraph = doc.create_map().unwrap(); - paragraph.insert("sys:id".into(), "paragraph").unwrap(); - paragraph.insert("sys:flavour".into(), "affine:paragraph").unwrap(); - paragraph - .insert("sys:children".into(), Value::Array(doc.create_array().unwrap())) - .unwrap(); - paragraph.insert("prop:type".into(), "text").unwrap(); - let mut paragraph_text = doc.create_text().unwrap(); - paragraph_text.insert(0, "child of unknown").unwrap(); - paragraph - .insert("prop:text".into(), Value::Text(paragraph_text)) - .unwrap(); - blocks.insert("paragraph".into(), Value::Map(paragraph)).unwrap(); - - let doc_bin = doc.encode_update_v1().unwrap(); - let result = parse_doc_to_markdown(doc_bin, doc_id, false, None).expect("parse doc"); - - assert!(result.markdown.is_empty()); - assert!(result.known_unsupported_blocks.is_empty()); - assert_eq!(result.unknown_blocks, vec!["mystery:affine:custom-unknown".to_string()]); - } -} diff --git a/packages/common/native/src/doc_parser/roundtrip_tests.rs b/packages/common/native/src/doc_parser/roundtrip_tests.rs deleted file mode 100644 index 0418507f594ca..0000000000000 --- a/packages/common/native/src/doc_parser/roundtrip_tests.rs +++ /dev/null @@ -1,56 +0,0 @@ -use super::{build_full_doc, parse_doc_to_markdown}; - -fn assert_markdown_roundtrip(markdown: &str, expected: &str) { - let doc_id = "roundtrip-doc"; - let title = "Roundtrip Title"; - let bin = build_full_doc(title, markdown, doc_id).expect("create doc"); - let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc"); - assert_eq!(result.title, title); - assert_eq!(result.markdown, expected); -} - -#[test] -fn test_roundtrip_inline_styles() { - let markdown = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com)."; - let expected = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com).\n\n"; - assert_markdown_roundtrip(markdown, expected); -} - -#[test] -fn test_roundtrip_list_items() { - let markdown = "- Item 1\n- Item 2\n- [ ] Task\n- [x] Done"; - let expected = "* Item 1\n* Item 2\n- [ ] Task\n- [x] Done\n"; - assert_markdown_roundtrip(markdown, expected); -} - -#[test] -fn test_roundtrip_code_block() { - let markdown = "```rust\nfn main() {}\n```"; - let expected = "```rust\nfn main() {}\n\n```\n\n"; - assert_markdown_roundtrip(markdown, expected); -} - -#[test] -fn test_roundtrip_code_block_indentation() { - let markdown = "```python\n def indented():\n return \"ok\"\n```"; - let doc_id = "roundtrip-indent"; - let title = "Roundtrip Title"; - let bin = build_full_doc(title, markdown, doc_id).expect("create doc"); - let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc"); - assert!(result.markdown.contains("\n def indented():")); - assert!(result.markdown.contains("\n return \"ok\"")); -} - -#[test] -fn test_roundtrip_table() { - let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |"; - let expected = "|A|B|\n|---|---|\n|1|2|\n\n"; - assert_markdown_roundtrip(markdown, expected); -} - -#[test] -fn test_roundtrip_image_with_caption() { - let markdown = "![Alt](blob://image-id)"; - let expected = "\n\n"; - assert_markdown_roundtrip(markdown, expected); -} diff --git a/packages/common/native/src/doc_parser/schema.rs b/packages/common/native/src/doc_parser/schema.rs deleted file mode 100644 index f14b4fd993e89..0000000000000 --- a/packages/common/native/src/doc_parser/schema.rs +++ /dev/null @@ -1,57 +0,0 @@ -pub(super) const PAGE_FLAVOUR: &str = "affine:page"; -pub(super) const NOTE_FLAVOUR: &str = "affine:note"; -pub(super) const SURFACE_FLAVOUR: &str = "affine:surface"; - -pub(super) const SYS_ID: &str = "sys:id"; -pub(super) const SYS_FLAVOUR: &str = "sys:flavour"; -pub(super) const SYS_VERSION: &str = "sys:version"; -pub(super) const SYS_CHILDREN: &str = "sys:children"; - -pub(super) const PROP_TITLE: &str = "prop:title"; -pub(super) const PROP_TEXT: &str = "prop:text"; -pub(super) const PROP_TYPE: &str = "prop:type"; -pub(super) const PROP_CHECKED: &str = "prop:checked"; -pub(super) const PROP_LANGUAGE: &str = "prop:language"; -pub(super) const PROP_ORDER: &str = "prop:order"; - -pub(super) const PROP_ELEMENTS: &str = "prop:elements"; -pub(super) const PROP_BACKGROUND: &str = "prop:background"; -pub(super) const PROP_XYWH: &str = "prop:xywh"; -pub(super) const PROP_INDEX: &str = "prop:index"; -pub(super) const PROP_HIDDEN: &str = "prop:hidden"; -pub(super) const PROP_DISPLAY_MODE: &str = "prop:displayMode"; - -pub(super) const PROP_SOURCE_ID: &str = "prop:sourceId"; -pub(super) const PROP_CAPTION: &str = "prop:caption"; -pub(super) const PROP_WIDTH: &str = "prop:width"; -pub(super) const PROP_HEIGHT: &str = "prop:height"; -pub(super) const PROP_URL: &str = "prop:url"; -pub(super) const PROP_VIDEO_ID: &str = "prop:videoId"; - -pub(super) const PROP_ROWS_PREFIX: &str = "prop:rows."; -pub(super) const PROP_COLUMNS_PREFIX: &str = "prop:columns."; -pub(super) const PROP_CELLS_PREFIX: &str = "prop:cells."; -pub(super) const PROP_ROW_ID_SUFFIX: &str = ".rowId"; -pub(super) const PROP_COLUMN_ID_SUFFIX: &str = ".columnId"; -pub(super) const PROP_ORDER_SUFFIX: &str = ".order"; -pub(super) const PROP_TEXT_SUFFIX: &str = ".text"; - -pub(super) fn table_row_id_key(row_id: &str) -> String { - format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ROW_ID_SUFFIX}") -} - -pub(super) fn table_row_order_key(row_id: &str) -> String { - format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ORDER_SUFFIX}") -} - -pub(super) fn table_column_id_key(column_id: &str) -> String { - format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_COLUMN_ID_SUFFIX}") -} - -pub(super) fn table_column_order_key(column_id: &str) -> String { - format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_ORDER_SUFFIX}") -} - -pub(super) fn table_cell_text_key(row_id: &str, column_id: &str) -> String { - format!("{PROP_CELLS_PREFIX}{row_id}:{column_id}{PROP_TEXT_SUFFIX}") -} diff --git a/packages/common/native/src/doc_parser/table.rs b/packages/common/native/src/doc_parser/table.rs deleted file mode 100644 index 6b8241426acb4..0000000000000 --- a/packages/common/native/src/doc_parser/table.rs +++ /dev/null @@ -1,71 +0,0 @@ -#[derive(Clone, Copy)] -pub(super) struct MarkdownTableOptions { - pub(super) escape_pipes: bool, - pub(super) newline_replacement: &'static str, - pub(super) trim: bool, -} - -impl MarkdownTableOptions { - pub(super) const fn new(escape_pipes: bool, newline_replacement: &'static str, trim: bool) -> Self { - Self { - escape_pipes, - newline_replacement, - trim, - } - } -} - -pub(super) fn render_markdown_table(rows: &[Vec], options: MarkdownTableOptions) -> Option { - let (header, body) = rows.split_first()?; - let header_line = format_table_row(header, options); - let separator_line = format_table_row(&vec!["---".to_string(); header.len()], options); - let mut lines = vec![header_line, separator_line]; - for row in body { - lines.push(format_table_row(row, options)); - } - Some(lines.join("\n")) -} - -fn format_table_row(row: &[String], options: MarkdownTableOptions) -> String { - let cells = row - .iter() - .map(|cell| format_table_cell(cell, options)) - .collect::>(); - format!("|{}|", cells.join("|")) -} - -fn format_table_cell(cell: &str, options: MarkdownTableOptions) -> String { - let mut value = if options.trim { - cell.trim().to_string() - } else { - cell.to_string() - }; - - if options.escape_pipes { - value = value.replace('|', "\\|"); - } - if !options.newline_replacement.is_empty() { - value = collapse_newlines(&value, options.newline_replacement); - } - value -} - -fn collapse_newlines(value: &str, replacement: &str) -> String { - if replacement.is_empty() { - return value.to_string(); - } - let mut out = String::with_capacity(value.len()); - let mut in_newline = false; - for ch in value.chars() { - if ch == '\n' { - if !in_newline { - out.push_str(replacement); - in_newline = true; - } - } else { - in_newline = false; - out.push(ch); - } - } - out -} diff --git a/packages/common/native/src/doc_parser/value.rs b/packages/common/native/src/doc_parser/value.rs deleted file mode 100644 index 8526f56ac95ab..0000000000000 --- a/packages/common/native/src/doc_parser/value.rs +++ /dev/null @@ -1,131 +0,0 @@ -use serde_json::{Map as JsonMap, Value as JsonValue}; -use y_octo::{AHashMap, Any, Value}; - -pub(super) fn any_truthy(value: &Any) -> bool { - match value { - Any::True => true, - Any::False | Any::Null | Any::Undefined => false, - Any::String(value) => !value.is_empty(), - Any::Integer(value) => *value != 0, - Any::Float32(value) => value.0 != 0.0, - Any::Float64(value) => value.0 != 0.0, - Any::BigInt64(value) => *value != 0, - Any::Object(_) | Any::Array(_) | Any::Binary(_) => true, - } -} - -pub(super) fn any_as_string(value: &Any) -> Option<&str> { - match value { - Any::String(value) => Some(value), - _ => None, - } -} - -pub(super) fn any_as_u64(value: &Any) -> Option { - match value { - Any::Integer(value) if *value >= 0 => Some(*value as u64), - Any::Float32(value) if value.0 >= 0.0 => Some(value.0 as u64), - Any::Float64(value) if value.0 >= 0.0 => Some(value.0 as u64), - Any::BigInt64(value) if *value >= 0 => Some(*value as u64), - _ => None, - } -} - -pub(super) fn value_to_string(value: &Value) -> Option { - if let Some(text) = value.to_text() { - return Some(text.to_string()); - } - - if let Some(any) = value.to_any() { - return any_to_string(&any); - } - - None -} - -pub(super) fn value_to_any(value: &Value) -> Option { - if let Some(any) = value.to_any() { - return Some(any); - } - - if let Some(text) = value.to_text() { - return Some(Any::String(text.to_string())); - } - - if let Some(array) = value.to_array() { - let mut values = Vec::new(); - for item in array.iter() { - if let Some(any) = value_to_any(&item) { - values.push(any); - } else if let Some(text) = value_to_string(&item) { - values.push(Any::String(text)); - } - } - return Some(Any::Array(values)); - } - - if let Some(map) = value.to_map() { - let mut values = AHashMap::default(); - for key in map.keys() { - if let Some(entry) = map.get(key) { - if let Some(any) = value_to_any(&entry) { - values.insert(key.to_string(), any); - } else if let Some(text) = value_to_string(&entry) { - values.insert(key.to_string(), Any::String(text)); - } - } - } - return Some(Any::Object(values)); - } - - None -} - -pub(super) fn value_to_f64(value: Value) -> Option { - value.to_any().and_then(|any| match any { - Any::Integer(v) => Some(v as f64), - Any::BigInt64(v) => Some(v as f64), - Any::Float32(v) => Some(v.0 as f64), - Any::Float64(v) => Some(v.0), - _ => None, - }) -} - -pub(super) fn any_to_string(any: &Any) -> Option { - match any { - Any::String(value) => Some(value.to_string()), - Any::Integer(value) => Some(value.to_string()), - Any::Float32(value) => Some(value.0.to_string()), - Any::Float64(value) => Some(value.0.to_string()), - Any::BigInt64(value) => Some(value.to_string()), - Any::True => Some("true".into()), - Any::False => Some("false".into()), - Any::Null | Any::Undefined => None, - Any::Array(_) | Any::Object(_) | Any::Binary(_) => serde_json::to_string(any).ok(), - } -} - -pub(super) fn params_any_map_to_json(params: &AHashMap) -> JsonValue { - let mut values = JsonMap::new(); - for (key, value) in params.iter() { - if let Ok(value) = serde_json::to_value(value) { - values.insert(key.clone(), value); - } - } - JsonValue::Object(values) -} - -pub(super) fn params_value_to_json(params: &Value) -> Option { - serde_json::to_value(params).ok() -} - -pub(super) fn build_reference_payload(doc_id: &str, params: Option) -> String { - let mut payload = JsonMap::new(); - payload.insert("docId".into(), JsonValue::String(doc_id.to_string())); - if let Some(JsonValue::Object(params)) = params { - for (key, value) in params.into_iter() { - payload.insert(key, value); - } - } - JsonValue::Object(payload).to_string() -} diff --git a/packages/common/native/src/doc_parser/write/builder.rs b/packages/common/native/src/doc_parser/write/builder.rs deleted file mode 100644 index 37b962ceaa325..0000000000000 --- a/packages/common/native/src/doc_parser/write/builder.rs +++ /dev/null @@ -1,426 +0,0 @@ -use y_octo::{TextDeltaOp, TextInsert}; - -use super::{ - super::schema::{ - PROP_CAPTION, PROP_CELLS_PREFIX, PROP_CHECKED, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER, - PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL, PROP_VIDEO_ID, PROP_WIDTH, SYS_CHILDREN, - SYS_FLAVOUR, SYS_ID, SYS_VERSION, table_cell_text_key, table_column_id_key, table_column_order_key, - table_row_id_key, table_row_order_key, - }, - *, -}; - -pub(super) const BOXED_NATIVE_TYPE: &str = "$blocksuite:internal:native$"; -pub(super) const NOTE_BG_LIGHT: &str = "#ffffff"; -pub(super) const NOTE_BG_DARK: &str = "#252525"; -const TABLE_ORDER_WIDTH: usize = 6; - -pub(super) fn block_version(flavour: &str) -> i32 { - match flavour { - "affine:page" => 2, - "affine:surface" => 5, - "affine:note" => 1, - "affine:paragraph" => 1, - "affine:list" => 1, - "affine:code" => 1, - "affine:divider" => 1, - "affine:image" => 1, - "affine:table" => 1, - "affine:bookmark" => 1, - "affine:embed-youtube" => 1, - "affine:embed-iframe" => 1, - "affine:callout" => 1, - _ => 1, - } -} - -pub(super) struct TextBlockProps<'a> { - pub block_type: Option<&'a str>, - pub checked: Option, - pub language: Option<&'a str>, - pub order: Option, - pub text: &'a [TextDeltaOp], -} - -pub(super) struct ImageBlockProps<'a> { - pub source_id: &'a str, - pub caption: Option<&'a str>, - pub width: Option, - pub height: Option, -} - -pub(super) struct BookmarkBlockProps<'a> { - pub url: &'a str, - pub caption: Option<&'a str>, -} - -pub(super) struct EmbedYoutubeBlockProps<'a> { - pub video_id: &'a str, -} - -pub(super) struct EmbedIframeBlockProps<'a> { - pub url: &'a str, -} - -pub(super) fn insert_text(doc: &Doc, block: &mut Map, key: &str, ops: &[TextDeltaOp]) -> Result<(), ParseError> { - let mut text = doc.create_text()?; - // Attach first so updates encode parent types before their contents. - block.insert(key.to_string(), Value::Text(text.clone()))?; - if !ops.is_empty() { - text.apply_delta(ops)?; - } - Ok(()) -} - -pub(crate) fn text_ops_from_plain(text: &str) -> Vec { - if text.is_empty() { - Vec::new() - } else { - vec![TextDeltaOp::Insert { - insert: TextInsert::Text(text.to_string()), - format: None, - }] - } -} - -pub(super) fn insert_children(doc: &Doc, block: &mut Map, children: &[String]) -> Result<(), ParseError> { - let mut array = doc.create_array()?; - // Attach first so updates encode parent types before their contents. - block.insert(SYS_CHILDREN.to_string(), Value::Array(array.clone()))?; - for child_id in children { - array.push(child_id.to_string())?; - } - Ok(()) -} - -pub(super) fn insert_block_map(doc: &Doc, blocks_map: &mut Map, block_id: &str) -> Result { - let empty_map = doc.create_map()?; - blocks_map.insert(block_id.to_string(), Value::Map(empty_map))?; - - blocks_map - .get(block_id) - .and_then(|value| value.to_map()) - .ok_or_else(|| ParseError::ParserError("Failed to retrieve inserted block map".into())) -} - -pub(super) fn insert_sys_fields(block: &mut Map, block_id: &str, flavour: &str) -> Result<(), ParseError> { - block.insert(SYS_ID.to_string(), Any::String(block_id.to_string()))?; - block.insert(SYS_FLAVOUR.to_string(), Any::String(flavour.to_string()))?; - block.insert(SYS_VERSION.to_string(), Any::Integer(block_version(flavour)))?; - Ok(()) -} - -pub(super) fn apply_text_block_props( - doc: &Doc, - block: &mut Map, - props: &TextBlockProps<'_>, - preserve_text: bool, - clear_missing: bool, -) -> Result<(), ParseError> { - match props.block_type { - Some(block_type) => { - block.insert(PROP_TYPE.to_string(), Any::String(block_type.to_string()))?; - } - None => { - if clear_missing && block.get(PROP_TYPE).is_some() { - block.remove(PROP_TYPE); - } - } - } - - if !preserve_text && !props.text.is_empty() { - insert_text(doc, block, PROP_TEXT, props.text)?; - } else if !preserve_text && clear_missing && block.get(PROP_TEXT).is_some() { - block.remove(PROP_TEXT); - } - - match props.checked { - Some(checked) => { - block.insert(PROP_CHECKED.to_string(), if checked { Any::True } else { Any::False })?; - } - None => { - if clear_missing && block.get(PROP_CHECKED).is_some() { - block.remove(PROP_CHECKED); - } - } - } - - match props.language { - Some(language) => { - block.insert(PROP_LANGUAGE.to_string(), Any::String(language.to_string()))?; - } - None => { - if clear_missing && block.get(PROP_LANGUAGE).is_some() { - block.remove(PROP_LANGUAGE); - } - } - } - - match props.order { - Some(order) => { - block.insert(PROP_ORDER.to_string(), Any::Float64((order as f64).into()))?; - } - None => { - if clear_missing && block.get(PROP_ORDER).is_some() { - block.remove(PROP_ORDER); - } - } - } - - Ok(()) -} - -pub(super) fn apply_image_block_props( - block: &mut Map, - props: &ImageBlockProps<'_>, - clear_missing: bool, -) -> Result<(), ParseError> { - block.insert(PROP_SOURCE_ID.to_string(), Any::String(props.source_id.to_string()))?; - - match props.caption { - Some(caption) => { - block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?; - } - None => { - if clear_missing && block.get(PROP_CAPTION).is_some() { - block.remove(PROP_CAPTION); - } - } - } - - match props.width { - Some(width) => { - block.insert(PROP_WIDTH.to_string(), Any::Float64(width.into()))?; - } - None => { - if clear_missing && block.get(PROP_WIDTH).is_some() { - block.remove(PROP_WIDTH); - } - } - } - - match props.height { - Some(height) => { - block.insert(PROP_HEIGHT.to_string(), Any::Float64(height.into()))?; - } - None => { - if clear_missing && block.get(PROP_HEIGHT).is_some() { - block.remove(PROP_HEIGHT); - } - } - } - - Ok(()) -} - -pub(super) fn apply_bookmark_block_props( - block: &mut Map, - props: &BookmarkBlockProps<'_>, - clear_missing: bool, -) -> Result<(), ParseError> { - block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?; - - match props.caption { - Some(caption) => { - block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?; - } - None => { - if clear_missing && block.get(PROP_CAPTION).is_some() { - block.remove(PROP_CAPTION); - } - } - } - - Ok(()) -} - -pub(super) fn apply_embed_youtube_block_props( - block: &mut Map, - props: &EmbedYoutubeBlockProps<'_>, -) -> Result<(), ParseError> { - block.insert(PROP_VIDEO_ID.to_string(), Any::String(props.video_id.to_string()))?; - Ok(()) -} - -pub(super) fn apply_embed_iframe_block_props( - block: &mut Map, - props: &EmbedIframeBlockProps<'_>, -) -> Result<(), ParseError> { - block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?; - Ok(()) -} - -pub(super) fn apply_table_block_props(block: &mut Map, rows: &[Vec]) -> Result<(), ParseError> { - clear_table_props(block); - - if rows.is_empty() { - return Ok(()); - } - - let column_count = rows.iter().map(|row| row.len()).max().unwrap_or(0); - let column_ids: Vec = (0..column_count).map(|_| nanoid::nanoid!()).collect(); - - for (col_idx, column_id) in column_ids.iter().enumerate() { - let order = format_table_order(col_idx); - block.insert(table_column_id_key(column_id), Any::String(column_id.to_string()))?; - block.insert(table_column_order_key(column_id), Any::String(order))?; - } - - for (row_idx, row) in rows.iter().enumerate() { - let row_id = nanoid::nanoid!(); - let order = format_table_order(row_idx); - block.insert(table_row_id_key(&row_id), Any::String(row_id.to_string()))?; - block.insert(table_row_order_key(&row_id), Any::String(order))?; - - for (col_idx, column_id) in column_ids.iter().enumerate() { - let cell_text = row.get(col_idx).cloned().unwrap_or_default(); - block.insert(table_cell_text_key(&row_id, column_id), Any::String(cell_text))?; - } - } - - Ok(()) -} - -pub(super) struct ApplyBlockOptions { - pub preserve_text: bool, - pub clear_missing: bool, -} - -pub(super) fn apply_block_spec( - doc: &Doc, - block: &mut Map, - spec: &BlockSpec, - options: ApplyBlockOptions, -) -> Result<(), ParseError> { - match spec.flavour { - BlockFlavour::Image => { - if options.preserve_text { - return Ok(()); - } - let image = spec - .image - .as_ref() - .ok_or_else(|| ParseError::ParserError("image spec missing".into()))?; - let props = ImageBlockProps { - source_id: &image.source_id, - caption: image.caption.as_deref(), - width: image.width, - height: image.height, - }; - apply_image_block_props(block, &props, options.clear_missing)?; - } - BlockFlavour::Bookmark => { - if options.preserve_text { - return Ok(()); - } - let bookmark = spec - .bookmark - .as_ref() - .ok_or_else(|| ParseError::ParserError("bookmark spec missing".into()))?; - let props = BookmarkBlockProps { - url: &bookmark.url, - caption: bookmark.caption.as_deref(), - }; - apply_bookmark_block_props(block, &props, options.clear_missing)?; - } - BlockFlavour::EmbedYoutube => { - if options.preserve_text { - return Ok(()); - } - let embed = spec - .embed_youtube - .as_ref() - .ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?; - let props = EmbedYoutubeBlockProps { - video_id: &embed.video_id, - }; - apply_embed_youtube_block_props(block, &props)?; - } - BlockFlavour::EmbedIframe => { - if options.preserve_text { - return Ok(()); - } - let embed = spec - .embed_iframe - .as_ref() - .ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?; - let props = EmbedIframeBlockProps { url: &embed.url }; - apply_embed_iframe_block_props(block, &props)?; - } - BlockFlavour::Callout => { - return Ok(()); - } - BlockFlavour::Table => { - if options.preserve_text { - return Ok(()); - } - let table = spec - .table - .as_ref() - .ok_or_else(|| ParseError::ParserError("table spec missing".into()))?; - apply_table_block_props(block, &table.rows)?; - } - _ => { - let props = TextBlockProps { - block_type: spec.block_type_str(), - checked: spec.checked, - language: spec.language.as_deref(), - order: spec.order, - text: &spec.text, - }; - apply_text_block_props(doc, block, &props, options.preserve_text, options.clear_missing)?; - } - } - - Ok(()) -} - -pub(super) fn insert_block_tree(doc: &Doc, blocks_map: &mut Map, node: &BlockNode) -> Result { - let block_id = nanoid::nanoid!(); - let mut block_map = insert_block_map(doc, blocks_map, &block_id)?; - - insert_sys_fields(&mut block_map, &block_id, node.spec.flavour.as_str())?; - apply_block_spec( - doc, - &mut block_map, - &node.spec, - ApplyBlockOptions { - preserve_text: false, - clear_missing: false, - }, - )?; - - let child_ids = node - .children - .iter() - .map(|child| insert_block_tree(doc, blocks_map, child)) - .collect::, _>>()?; - insert_children(doc, &mut block_map, &child_ids)?; - - Ok(block_id) -} - -fn clear_table_props(block: &mut Map) { - let keys = block - .keys() - .filter(|key| { - key.starts_with(PROP_ROWS_PREFIX) || key.starts_with(PROP_COLUMNS_PREFIX) || key.starts_with(PROP_CELLS_PREFIX) - }) - .map(|s| s.to_string()) - .collect::>(); - for key in keys { - block.remove(&key); - } -} - -fn format_table_order(index: usize) -> String { - format!("{index:0width$}", width = TABLE_ORDER_WIDTH) -} - -pub(super) fn boxed_empty_map(doc: &Doc) -> Result { - doc.create_map().map_err(ParseError::from) -} - -pub(super) fn note_background_map(doc: &Doc) -> Result { - doc.create_map().map_err(ParseError::from) -} diff --git a/packages/common/native/src/doc_parser/write/create.rs b/packages/common/native/src/doc_parser/write/create.rs deleted file mode 100644 index c7d7022f6e325..0000000000000 --- a/packages/common/native/src/doc_parser/write/create.rs +++ /dev/null @@ -1,281 +0,0 @@ -//! Markdown to YDoc conversion module -//! -//! Converts markdown content into AFFiNE-compatible y-octo document binary -//! format. - -use y_octo::DocOptions; - -use super::{ - super::{ - markdown::parse_markdown_blocks, - schema::{PROP_BACKGROUND, PROP_DISPLAY_MODE, PROP_ELEMENTS, PROP_HIDDEN, PROP_INDEX, PROP_XYWH, SURFACE_FLAVOUR}, - }, - builder::{ - BOXED_NATIVE_TYPE, NOTE_BG_DARK, NOTE_BG_LIGHT, boxed_empty_map, insert_block_map, insert_block_tree, - insert_children, insert_sys_fields, insert_text, note_background_map, text_ops_from_plain, - }, - *, -}; - -/// Converts markdown into an AFFiNE-compatible y-octo document binary. -/// -/// # Arguments -/// * `title` - The document title -/// * `markdown` - The markdown content to convert -/// * `doc_id` - The document ID to use -/// -/// # Returns -/// A binary vector containing the y-octo encoded document update -pub fn build_full_doc(title: &str, markdown: &str, doc_id: &str) -> Result, ParseError> { - let nodes = parse_markdown_blocks(markdown)?; - build_doc_update(doc_id, title, &nodes) -} - -fn build_doc_update(doc_id: &str, title: &str, blocks: &[BlockNode]) -> Result, ParseError> { - let doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - let mut blocks_map = doc.get_or_create_map("blocks")?; - - let page_id = nanoid::nanoid!(); - let surface_id = nanoid::nanoid!(); - let note_id = nanoid::nanoid!(); - - // Insert root blocks first to establish stable IDs. - let mut page_map = insert_block_map(&doc, &mut blocks_map, &page_id)?; - let mut surface_map = insert_block_map(&doc, &mut blocks_map, &surface_id)?; - let mut note_map = insert_block_map(&doc, &mut blocks_map, ¬e_id)?; - - // Create content blocks under note. - let content_ids = insert_block_trees(&doc, &mut blocks_map, blocks)?; - - // Page block - insert_sys_fields(&mut page_map, &page_id, PAGE_FLAVOUR)?; - insert_children(&doc, &mut page_map, &[surface_id.clone(), note_id.clone()])?; - insert_text(&doc, &mut page_map, PROP_TITLE, &text_ops_from_plain(title))?; - - // Surface block - insert_sys_fields(&mut surface_map, &surface_id, SURFACE_FLAVOUR)?; - insert_children(&doc, &mut surface_map, &[])?; - let mut boxed = boxed_empty_map(&doc)?; - surface_map.insert(PROP_ELEMENTS.to_string(), Value::Map(boxed.clone()))?; - boxed.insert("type".to_string(), Any::String(BOXED_NATIVE_TYPE.to_string()))?; - let value = doc.create_map()?; - boxed.insert("value".to_string(), Value::Map(value))?; - - // Note block - insert_sys_fields(&mut note_map, ¬e_id, NOTE_FLAVOUR)?; - insert_children(&doc, &mut note_map, &content_ids)?; - let mut background = note_background_map(&doc)?; - note_map.insert(PROP_BACKGROUND.to_string(), Value::Map(background.clone()))?; - background.insert("light".to_string(), Any::String(NOTE_BG_LIGHT.to_string()))?; - background.insert("dark".to_string(), Any::String(NOTE_BG_DARK.to_string()))?; - note_map.insert(PROP_XYWH.to_string(), Any::String("[0,0,800,95]".to_string()))?; - note_map.insert(PROP_INDEX.to_string(), Any::String("a0".to_string()))?; - note_map.insert(PROP_HIDDEN.to_string(), Any::False)?; - note_map.insert(PROP_DISPLAY_MODE.to_string(), Any::String("both".to_string()))?; - - Ok(doc.encode_update_v1()?) -} - -fn insert_block_trees(doc: &Doc, blocks_map: &mut Map, blocks: &[BlockNode]) -> Result, ParseError> { - let mut ids = Vec::with_capacity(blocks.len()); - for block in blocks { - let id = insert_block_tree(doc, blocks_map, block)?; - ids.push(id); - } - Ok(ids) -} - -#[cfg(test)] -mod tests { - use y_octo::{Any, DocOptions}; - - use super::{ - super::super::{ - blocksuite::get_string, - markdown::{MAX_BLOCKS, MAX_MARKDOWN_CHARS}, - schema::PAGE_FLAVOUR, - }, - *, - }; - - #[test] - fn test_simple_markdown() { - let markdown = "# Hello World\n\nThis is a test paragraph."; - let result = build_full_doc("Hello World", markdown, "test-doc-id"); - assert!(result.is_ok()); - let bin = result.unwrap(); - assert!(!bin.is_empty()); - } - - #[test] - fn test_title_from_param() { - let markdown = "# Markdown Title\n\nContent."; - let doc_id = "title-param-test"; - let bin = build_full_doc("External Title", markdown, doc_id).expect("create doc"); - - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc.apply_update_from_binary_v1(&bin).expect("apply update"); - - let blocks_map = doc.get_map("blocks").expect("blocks map"); - let mut title = None; - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() - && get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR) - { - title = get_string(&block_map, "prop:title"); - break; - } - } - - assert_eq!(title.as_deref(), Some("External Title")); - } - - #[test] - fn test_markdown_with_list() { - let markdown = "# Test List\n\n- Item 1\n- Item 2\n- Item 3"; - let result = build_full_doc("Test List", markdown, "test-doc-id"); - assert!(result.is_ok()); - } - - #[test] - fn test_markdown_with_code() { - let markdown = "# Code Example\n\n```rust\nfn main() {\n println!(\"Hello\");\n}\n```"; - let result = build_full_doc("Code Example", markdown, "test-doc-id"); - assert!(result.is_ok()); - } - - #[test] - fn test_markdown_with_headings() { - let markdown = "# H1\n\n## H2\n\n### H3\n\nParagraph text."; - let result = build_full_doc("H1", markdown, "test-doc-id"); - assert!(result.is_ok()); - } - - #[test] - fn test_empty_markdown() { - let result = build_full_doc("Untitled", "", "test-doc-id"); - assert!(result.is_ok()); - let bin = result.unwrap(); - assert!(!bin.is_empty()); - } - - #[test] - fn test_whitespace_only_markdown() { - let result = build_full_doc("Untitled", " \n\n\t\n ", "test-doc-id"); - assert!(result.is_ok()); - let bin = result.unwrap(); - assert!(!bin.is_empty()); - } - - #[test] - fn test_markdown_without_h1() { - let markdown = "## Secondary Heading\n\nSome content without H1."; - let result = build_full_doc("Title", markdown, "test-doc-id"); - assert!(result.is_ok()); - } - - #[test] - fn test_nested_lists() { - let markdown = "# Nested Lists\n\n- Item 1\n - Nested 1.1\n - Nested 1.2\n- Item 2\n - Nested 2.1"; - let result = build_full_doc("Nested Lists", markdown, "test-doc-id"); - assert!(result.is_ok()); - } - - #[test] - fn test_blockquote() { - let markdown = "# Title\n\n> A blockquote"; - let result = build_full_doc("Title", markdown, "test-doc-id"); - assert!(result.is_ok()); - } - - #[test] - fn test_divider() { - let markdown = "# Title\n\nBefore divider\n\n---\n\nAfter divider"; - let result = build_full_doc("Title", markdown, "test-doc-id"); - assert!(result.is_ok()); - } - - #[test] - fn test_numbered_list() { - let markdown = "# Title\n\n1. First item\n2. Second item"; - let result = build_full_doc("Title", markdown, "test-doc-id"); - assert!(result.is_ok()); - } - - #[test] - fn test_markdown_too_large() { - let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1); - let result = build_full_doc("Title", &markdown, "test-doc-id"); - assert!(result.is_err()); - } - - #[test] - fn test_markdown_block_limit() { - let mut markdown = String::from("# Title\n\n"); - for i in 0..=MAX_BLOCKS { - markdown.push_str(&format!("Paragraph {i}\n\n")); - } - let result = build_full_doc("Title", &markdown, "test-doc-id"); - assert!(result.is_err()); - } - - #[test] - fn test_markdown_with_image() { - let markdown = "![Alt](blob://image-id)"; - let doc_id = "image-doc"; - let bin = build_full_doc("Title", markdown, doc_id).expect("create doc"); - - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc.apply_update_from_binary_v1(&bin).expect("apply update"); - - let blocks_map = doc.get_map("blocks").expect("blocks map"); - let mut found = false; - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() - && get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image") - { - let source_id = get_string(&block_map, "prop:sourceId"); - assert_eq!(source_id.as_deref(), Some("image-id")); - found = true; - break; - } - } - - assert!(found); - } - - #[test] - fn test_markdown_with_table() { - let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |"; - let doc_id = "table-doc"; - let bin = build_full_doc("Title", markdown, doc_id).expect("create doc"); - - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc.apply_update_from_binary_v1(&bin).expect("apply update"); - - let blocks_map = doc.get_map("blocks").expect("blocks map"); - let mut found_cell = false; - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() - && get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table") - { - for key in block_map.keys() { - if key.starts_with("prop:cells.") && key.ends_with(".text") { - let value = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a { - Any::String(value) => Some(value), - _ => None, - }); - if let Some(value) = value - && (value == "A" || value == "1") - { - found_cell = true; - break; - } - } - } - } - } - - assert!(found_cell); - } -} diff --git a/packages/common/native/src/doc_parser/write/doc_meta.rs b/packages/common/native/src/doc_parser/write/doc_meta.rs deleted file mode 100644 index f988f01be7e5a..0000000000000 --- a/packages/common/native/src/doc_parser/write/doc_meta.rs +++ /dev/null @@ -1,132 +0,0 @@ -use super::{ - builder::{insert_text, text_ops_from_plain}, - root_doc::ensure_pages_array, - *, -}; - -pub fn update_doc_title(existing_binary: &[u8], doc_id: &str, title: &str) -> Result, ParseError> { - let doc = load_doc(existing_binary, Some(doc_id))?; - - let state_before = doc.get_state_vector(); - let blocks_map = doc.get_map("blocks")?; - if blocks_map.is_empty() { - return Err(ParseError::ParserError("blocks map is empty".into())); - } - - let mut page_block = find_page_block(&blocks_map)?; - let current = get_string(&page_block, PROP_TITLE).unwrap_or_default(); - if current != title { - insert_text(&doc, &mut page_block, PROP_TITLE, &text_ops_from_plain(title))?; - } - - Ok(doc.encode_state_as_update_v1(&state_before)?) -} - -pub fn update_root_doc_meta_title(root_doc_bin: &[u8], doc_id: &str, title: &str) -> Result, ParseError> { - let doc = load_doc_or_new(root_doc_bin)?; - - let state_before = doc.get_state_vector(); - let mut meta = doc.get_or_create_map("meta")?; - let mut pages = ensure_pages_array(&doc, &mut meta)?; - - let mut found = false; - for idx in 0..pages.len() { - let Some(mut page) = pages.get(idx).and_then(|v| v.to_map()) else { - continue; - }; - if get_string(&page, "id").as_deref() == Some(doc_id) { - page.insert("title".to_string(), Any::String(title.to_string()))?; - found = true; - break; - } - } - - if !found { - let page_map = doc.create_map()?; - - let idx = pages.len(); - pages.insert(idx, Value::Map(page_map))?; - - if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) { - inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?; - inserted_page.insert("title".to_string(), Any::String(title.to_string()))?; - - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as i64) - .unwrap_or(0); - inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?; - - let tags = doc.create_array()?; - inserted_page.insert("tags".to_string(), Value::Array(tags))?; - } - } - - Ok(doc.encode_state_as_update_v1(&state_before)?) -} - -fn find_page_block(blocks_map: &Map) -> Result { - let index = build_block_index(blocks_map); - let page_id = find_block_id_by_flavour(&index.block_pool, PAGE_FLAVOUR) - .ok_or_else(|| ParseError::ParserError("page block not found".into()))?; - blocks_map - .get(&page_id) - .and_then(|value| value.to_map()) - .ok_or_else(|| ParseError::ParserError("page block not found".into())) -} - -#[cfg(test)] -mod tests { - use y_octo::DocOptions; - - use super::*; - use crate::doc_parser::{add_doc_to_root_doc, build_full_doc}; - - #[test] - fn test_update_doc_title() { - let doc_id = "doc-meta-title-test"; - let initial = build_full_doc("Old Title", "Content.", doc_id).expect("create doc"); - let delta = update_doc_title(&initial, doc_id, "New Title").expect("update title"); - - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc.apply_update_from_binary_v1(&initial).expect("apply initial"); - doc.apply_update_from_binary_v1(&delta).expect("apply delta"); - - let blocks_map = doc.get_map("blocks").expect("blocks map"); - let mut title = None; - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() - && get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR) - { - title = get_string(&block_map, "prop:title"); - break; - } - } - - assert_eq!(title.as_deref(), Some("New Title")); - } - - #[test] - fn test_update_root_doc_meta_title() { - let doc_id = "root-meta-title-test"; - let root_bin = add_doc_to_root_doc(Vec::new(), doc_id, Some("Old Title")).expect("create root meta"); - let delta = update_root_doc_meta_title(&root_bin, doc_id, "New Title").expect("update meta"); - - let mut doc = DocOptions::new().build(); - doc.apply_update_from_binary_v1(&root_bin).expect("apply root"); - doc.apply_update_from_binary_v1(&delta).expect("apply delta"); - - let meta = doc.get_map("meta").expect("meta map"); - let pages = meta.get("pages").and_then(|v| v.to_array()).expect("pages array"); - let mut title = None; - for page in pages.iter() { - if let Some(page_map) = page.to_map() - && get_string(&page_map, "id").as_deref() == Some(doc_id) - { - title = get_string(&page_map, "title"); - break; - } - } - assert_eq!(title.as_deref(), Some("New Title")); - } -} diff --git a/packages/common/native/src/doc_parser/write/doc_properties.rs b/packages/common/native/src/doc_parser/write/doc_properties.rs deleted file mode 100644 index 53f67dd043f1e..0000000000000 --- a/packages/common/native/src/doc_parser/write/doc_properties.rs +++ /dev/null @@ -1,95 +0,0 @@ -use y_octo::{Any, DocOptions, Map}; - -use super::{ - super::{doc_loader::is_empty_doc, value::value_to_string}, - ParseError, -}; - -pub fn update_doc_properties( - existing_binary: &[u8], - properties_doc_id: &str, - target_doc_id: &str, - created_by: Option<&str>, - updated_by: Option<&str>, -) -> Result, ParseError> { - let doc = if is_empty_doc(existing_binary) { - DocOptions::new().with_guid(properties_doc_id.to_string()).build() - } else { - super::load_doc(existing_binary, Some(properties_doc_id))? - }; - - let state_before = doc.get_state_vector(); - let mut record = doc.get_or_create_map(target_doc_id)?; - let mut changed = false; - - if record.get("id").is_none() { - record.insert("id".to_string(), Any::String(target_doc_id.to_string()))?; - changed = true; - } - - if let Some(created_by) = created_by - && get_record_string(&record, "createdBy").as_deref() != Some(created_by) - { - record.insert("createdBy".to_string(), Any::String(created_by.to_string()))?; - changed = true; - } - - if let Some(updated_by) = updated_by - && get_record_string(&record, "updatedBy").as_deref() != Some(updated_by) - { - record.insert("updatedBy".to_string(), Any::String(updated_by.to_string()))?; - changed = true; - } - - if !changed { - return Ok(Vec::new()); - } - - Ok(doc.encode_state_as_update_v1(&state_before)?) -} - -fn get_record_string(record: &Map, key: &str) -> Option { - record.get(key).and_then(|value| value_to_string(&value)) -} - -#[cfg(test)] -mod tests { - use y_octo::DocOptions; - - use super::*; - - #[test] - fn test_update_doc_properties_creates_record() { - let properties_doc_id = "doc-properties"; - let target_doc_id = "doc-1"; - let update = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-1")) - .expect("update properties"); - - let mut doc = DocOptions::new().with_guid(properties_doc_id.to_string()).build(); - doc.apply_update_from_binary_v1(&update).expect("apply"); - - let record = doc.get_map(target_doc_id).expect("record map"); - assert_eq!(get_record_string(&record, "id").as_deref(), Some(target_doc_id)); - assert_eq!(get_record_string(&record, "createdBy").as_deref(), Some("user-1")); - assert_eq!(get_record_string(&record, "updatedBy").as_deref(), Some("user-1")); - } - - #[test] - fn test_update_doc_properties_no_change() { - let properties_doc_id = "doc-properties-no-change"; - let target_doc_id = "doc-2"; - let initial = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-2")) - .expect("initial update"); - - let delta = update_doc_properties( - &initial, - properties_doc_id, - target_doc_id, - Some("user-1"), - Some("user-2"), - ) - .expect("no change update"); - - assert!(delta.is_empty()); - } -} diff --git a/packages/common/native/src/doc_parser/write/mod.rs b/packages/common/native/src/doc_parser/write/mod.rs deleted file mode 100644 index 17fe2ae00f301..0000000000000 --- a/packages/common/native/src/doc_parser/write/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -pub mod builder; -mod create; -mod doc_meta; -mod doc_properties; -mod root_doc; -mod update; - -pub use create::build_full_doc; -pub use doc_meta::{update_doc_title, update_root_doc_meta_title}; -pub use doc_properties::update_doc_properties; -pub use root_doc::{add_doc_to_root_doc, build_public_root_doc}; -pub use update::update_doc; -use y_octo::{Any, Doc, Map, Value}; - -use super::{ - ParseError, - block_spec::{BlockFlavour, BlockNode, BlockSpec}, - blocksuite::{build_block_index, find_block_id_by_flavour, get_string}, - doc_loader::{load_doc, load_doc_or_new}, - schema::{NOTE_FLAVOUR, PAGE_FLAVOUR, PROP_TITLE}, -}; diff --git a/packages/common/native/src/doc_parser/write/root_doc.rs b/packages/common/native/src/doc_parser/write/root_doc.rs deleted file mode 100644 index cf94c3582ba01..0000000000000 --- a/packages/common/native/src/doc_parser/write/root_doc.rs +++ /dev/null @@ -1,291 +0,0 @@ -use std::collections::HashSet; - -use y_octo::Array; - -use super::*; - -const DEFAULT_DOC_TITLE: &str = "Untitled"; -const SIMPLE_PAGE_META_KEYS: &[&str] = &[ - "id", - "title", - "createDate", - "updatedDate", - "trash", - "trashDate", - "headerImage", -]; - -fn any_to_value(doc: &Doc, any: Any) -> Result { - match any { - Any::Array(values) => { - let mut array = doc.create_array()?; - for value in values { - let item = any_to_value(doc, value)?; - array.push(item)?; - } - Ok(Value::Array(array)) - } - Any::Object(values) => { - let mut map = doc.create_map()?; - for (key, value) in values { - let item = any_to_value(doc, value)?; - map.insert(key, item)?; - } - Ok(Value::Map(map)) - } - _ => Ok(Value::Any(any)), - } -} - -pub(super) fn ensure_pages_array(doc: &Doc, meta: &mut Map) -> Result { - let pages_value = meta.get("pages"); - if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) { - return Ok(pages); - } - - if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) { - let mut pages = doc.create_array()?; - for entry in entries { - let value = any_to_value(doc, entry)?; - pages.push(value)?; - } - meta.insert("pages".to_string(), Value::Array(pages.clone()))?; - return Ok(pages); - } - - let pages = doc.create_array()?; - meta.insert("pages".to_string(), Value::Array(pages.clone()))?; - Ok(pages) -} - -/// Adds a document ID to the root doc's meta.pages array. -/// Returns a binary update that can be applied to the root doc. -/// -/// # Arguments -/// * `root_doc_bin` - The current root doc binary -/// * `doc_id` - The document ID to add -/// * `title` - Optional title for the document -/// -/// # Returns -/// A Vec containing the y-octo update binary to add the doc -pub fn add_doc_to_root_doc(root_doc_bin: Vec, doc_id: &str, title: Option<&str>) -> Result, ParseError> { - // Handle empty or minimal root doc - create a new one - let doc = load_doc_or_new(&root_doc_bin)?; - - // Capture state before modifications to encode only the delta - let state_before = doc.get_state_vector(); - - // Get or create the meta map - let mut meta = doc.get_or_create_map("meta")?; - - let mut pages = ensure_pages_array(&doc, &mut meta)?; - - // Check if doc already exists - let doc_exists = pages.iter().any(|page_val| { - page_val - .to_map() - .and_then(|page| get_string(&page, "id")) - .map(|id| id == doc_id) - .unwrap_or(false) - }); - - if !doc_exists { - let page_map = doc.create_map()?; - - let idx = pages.len(); - pages.insert(idx, Value::Map(page_map))?; - - if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) { - inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?; - - let page_title = title.unwrap_or(DEFAULT_DOC_TITLE); - inserted_page.insert("title".to_string(), Any::String(page_title.to_string()))?; - - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as i64) - .unwrap_or(0); - inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?; - - let tags = doc.create_array()?; - inserted_page.insert("tags".to_string(), Value::Array(tags))?; - } - } - - // Encode only the changes (delta) since state_before - Ok(doc.encode_state_as_update_v1(&state_before)?) -} - -fn insert_page_stub(doc: &Doc, pages: &mut Array, doc_id: &str, title: Option<&str>) -> Result<(), ParseError> { - let page_map = doc.create_map()?; - let idx = pages.len(); - pages.insert(idx, Value::Map(page_map))?; - - if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) { - inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?; - inserted_page.insert( - "title".to_string(), - Any::String(title.unwrap_or(DEFAULT_DOC_TITLE).to_string()), - )?; - - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as i64) - .unwrap_or(0); - inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?; - - let tags = doc.create_array()?; - inserted_page.insert("tags".to_string(), Value::Array(tags))?; - } - - Ok(()) -} - -fn insert_simple_array(doc: &Doc, values: &[Any]) -> Result { - let mut array = doc.create_array()?; - for value in values { - match value { - Any::Array(_) | Any::Object(_) => continue, - _ => array.push(Value::Any(value.clone()))?, - } - } - Ok(Value::Array(array)) -} - -fn insert_page_from_any(doc: &Doc, pages: &mut Array, page: Any) -> Result, ParseError> { - let Any::Object(page) = page else { - return Ok(None); - }; - - let Some(page_id) = page.get("id").and_then(|value| match value { - Any::String(value) => Some(value.clone()), - _ => None, - }) else { - return Ok(None); - }; - - let page_map = doc.create_map()?; - let idx = pages.len(); - pages.insert(idx, Value::Map(page_map))?; - - let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) else { - return Ok(None); - }; - - for key in SIMPLE_PAGE_META_KEYS { - let Some(value) = page.get(*key) else { - continue; - }; - match value { - Any::Array(values) => { - inserted_page.insert((*key).to_string(), insert_simple_array(doc, values)?)?; - } - Any::Object(_) => continue, - _ => { - inserted_page.insert((*key).to_string(), Value::Any(value.clone()))?; - } - } - } - - if let Some(Any::Array(tags)) = page.get("tags") { - inserted_page.insert("tags".to_string(), insert_simple_array(doc, tags)?)?; - } - - Ok(Some(page_id)) -} - -pub fn build_public_root_doc(root_doc_bin: &[u8], doc_metas: &[(&str, Option<&str>)]) -> Result, ParseError> { - let source = load_doc_or_new(root_doc_bin)?; - let public_doc_ids = doc_metas - .iter() - .map(|(doc_id, _title)| (*doc_id).to_string()) - .collect::>(); - - let doc = Doc::default(); - let mut meta = doc.get_or_create_map("meta")?; - let mut pages = ensure_pages_array(&doc, &mut meta)?; - let mut copied = HashSet::new(); - - if let Ok(source_meta) = source.get_map("meta") { - let source_pages_value = source_meta.get("pages"); - - if let Some(source_pages) = source_pages_value.as_ref().and_then(|value| value.to_array()) { - for page_val in source_pages.iter() { - let Some(page) = page_val.to_map() else { - continue; - }; - let Some(page_id) = get_string(&page, "id") else { - continue; - }; - if !public_doc_ids.contains(&page_id) { - continue; - } - - let page_object = Any::Object( - page - .iter() - .filter_map(|(key, value)| value.to_any().map(|any| (key.to_string(), any))) - .collect(), - ); - if let Some(inserted_page_id) = insert_page_from_any(&doc, &mut pages, page_object)? { - copied.insert(inserted_page_id); - } - } - } else if let Some(Any::Array(entries)) = source_pages_value.and_then(|value| value.to_any()) { - for entry in entries { - let Any::Object(page) = entry.clone() else { - continue; - }; - let Some(Any::String(page_id)) = page.get("id") else { - continue; - }; - if !public_doc_ids.contains(page_id) { - continue; - } - - if let Some(inserted_page_id) = insert_page_from_any(&doc, &mut pages, entry)? { - copied.insert(inserted_page_id); - } - } - } - } - - for (doc_id, title) in doc_metas { - if copied.contains(*doc_id) { - continue; - } - insert_page_stub(&doc, &mut pages, doc_id, *title)?; - } - - Ok(doc.encode_update_v1()?) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::doc_parser::get_doc_ids_from_binary; - - #[test] - fn test_build_public_root_doc_filters_private_pages() { - let root = add_doc_to_root_doc(Vec::new(), "public-doc", Some("Public")).expect("create public entry"); - let update = add_doc_to_root_doc(root.clone(), "private-doc", Some("Private")).expect("create private entry"); - - let mut merged = load_doc_or_new(&root).expect("load root"); - merged - .apply_update_from_binary_v1(&update) - .expect("apply second update"); - let merged_bin = merged.encode_update_v1().expect("encode merged"); - - let public_root = build_public_root_doc( - &merged_bin, - &[("public-doc", Some("Public")), ("missing-public-doc", Some("Fallback"))], - ) - .expect("build public root"); - - let doc_ids = get_doc_ids_from_binary(public_root, false).expect("read public root doc ids"); - assert_eq!( - doc_ids, - vec!["public-doc".to_string(), "missing-public-doc".to_string()] - ); - } -} diff --git a/packages/common/native/src/doc_parser/write/update.rs b/packages/common/native/src/doc_parser/write/update.rs deleted file mode 100644 index ddf47e21eeea4..0000000000000 --- a/packages/common/native/src/doc_parser/write/update.rs +++ /dev/null @@ -1,671 +0,0 @@ -//! Update YDoc module -//! -//! Provides functionality to update existing AFFiNE documents by applying -//! surgical y-octo operations based on content differences. - -use std::collections::HashMap; - -use super::{ - super::{ - block_spec::{TreeNode, count_tree_nodes, text_delta_eq}, - blocksuite::{collect_child_ids, find_child_id_by_flavour}, - markdown::{MAX_BLOCKS, parse_markdown_blocks}, - }, - builder::{ApplyBlockOptions, apply_block_spec, insert_block_tree, insert_children}, - *, -}; - -const MAX_LCS_CELLS: usize = 2_000_000; - -#[derive(Debug, Clone)] -struct StoredNode { - id: String, - spec: BlockSpec, - children: Vec, -} - -impl TreeNode for StoredNode { - fn children(&self) -> &[StoredNode] { - &self.children - } -} - -struct DocState { - doc: Doc, - note_id: String, - blocks: Vec, -} - -#[derive(Debug)] -enum PatchOp { - Keep(usize, usize), - Delete(usize), - Insert(usize), - Update(usize, usize), -} - -/// Updates an existing document with new markdown content. -/// -/// This function performs structural diffing between the existing document -/// and the new markdown content, then applies block-level replacements -/// for changed blocks. This enables proper CRDT merging with concurrent -/// edits from other clients. -/// -/// # Arguments -/// * `existing_binary` - The current document binary -/// * `new_markdown` - The new markdown content (document title is not updated) -/// * `doc_id` - The document ID -/// -/// # Returns -/// A binary vector representing only the delta (changes) to apply -pub fn update_doc(existing_binary: &[u8], new_markdown: &str, doc_id: &str) -> Result, ParseError> { - let mut new_nodes = parse_markdown_blocks(new_markdown)?; - let state = load_doc_state(existing_binary, doc_id)?; - - check_limits(&state.blocks, &new_nodes)?; - - let state_before = state.doc.get_state_vector(); - - let mut blocks_map = state.doc.get_map("blocks")?; - - let new_children = sync_nodes(&state.doc, &mut blocks_map, &state.blocks, &mut new_nodes)?; - sync_children(&state.doc, &mut blocks_map, &state.note_id, &new_children)?; - - Ok(state.doc.encode_state_as_update_v1(&state_before)?) -} - -fn load_doc_state(binary: &[u8], doc_id: &str) -> Result { - let doc = load_doc(binary, Some(doc_id))?; - - let blocks_map = doc.get_map("blocks")?; - if blocks_map.is_empty() { - return Err(ParseError::ParserError("blocks map is empty".into())); - } - - let block_index = build_block_index(&blocks_map); - let page_id = find_block_id_by_flavour(&block_index.block_pool, PAGE_FLAVOUR) - .ok_or_else(|| ParseError::ParserError("page block not found".into()))?; - let page_block = block_index - .block_pool - .get(&page_id) - .ok_or_else(|| ParseError::ParserError("page block not found".into()))?; - let note_id = find_child_id_by_flavour(page_block, &block_index.block_pool, NOTE_FLAVOUR) - .ok_or_else(|| ParseError::ParserError("note block not found".into()))?; - let note_block = block_index - .block_pool - .get(¬e_id) - .ok_or_else(|| ParseError::ParserError("note block not found".into()))?; - let content_ids = collect_child_ids(note_block); - - let mut blocks = Vec::new(); - for block_id in content_ids { - let block = block_index - .block_pool - .get(&block_id) - .ok_or_else(|| ParseError::ParserError("content block not found".into()))?; - blocks.push(build_stored_tree(&block_id, block, &block_index.block_pool)?); - } - - Ok(DocState { doc, note_id, blocks }) -} - -fn build_stored_tree(block_id: &str, block: &Map, pool: &HashMap) -> Result { - let spec = BlockSpec::from_block_map(block)?; - - let child_ids = collect_child_ids(block); - if !child_ids.is_empty() && !matches!(spec.flavour, BlockFlavour::List | BlockFlavour::Callout) { - return Err(ParseError::ParserError(format!( - "unsupported children on block: {block_id}" - ))); - } - let mut children = Vec::new(); - for child_id in child_ids { - let child_block = pool - .get(&child_id) - .ok_or_else(|| ParseError::ParserError("child block not found".into()))?; - children.push(build_stored_tree(&child_id, child_block, pool)?); - } - - Ok(StoredNode { - id: block_id.to_string(), - spec, - children, - }) -} - -fn sync_nodes( - doc: &Doc, - blocks_map: &mut Map, - current: &[StoredNode], - target: &mut [BlockNode], -) -> Result, ParseError> { - let ops = diff_blocks(current, target); - let mut new_children = Vec::new(); - let mut to_remove = Vec::new(); - - for op in ops { - match op { - PatchOp::Keep(old_idx, new_idx) => { - let old_node = ¤t[old_idx]; - let new_node = &target[new_idx]; - update_block_props(doc, blocks_map, old_node, &new_node.spec, true)?; - let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?; - sync_children(doc, blocks_map, &old_node.id, &child_ids)?; - new_children.push(old_node.id.clone()); - } - PatchOp::Update(old_idx, new_idx) => { - let old_node = ¤t[old_idx]; - let new_node = &target[new_idx]; - update_block_props(doc, blocks_map, old_node, &new_node.spec, false)?; - let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?; - sync_children(doc, blocks_map, &old_node.id, &child_ids)?; - new_children.push(old_node.id.clone()); - } - PatchOp::Insert(new_idx) => { - let new_id = insert_block_tree(doc, blocks_map, &target[new_idx])?; - new_children.push(new_id); - } - PatchOp::Delete(old_idx) => { - let node = ¤t[old_idx]; - if node.spec.flavour == BlockFlavour::Callout { - new_children.push(node.id.clone()); - } else { - collect_tree_ids(node, &mut to_remove); - } - } - } - } - - for id in to_remove { - blocks_map.remove(&id); - } - - Ok(new_children) -} - -fn diff_blocks(current: &[StoredNode], target: &[BlockNode]) -> Vec { - let old_len = current.len(); - let new_len = target.len(); - - if old_len == 0 { - return (0..new_len).map(PatchOp::Insert).collect(); - } - if new_len == 0 { - return (0..old_len).map(PatchOp::Delete).collect(); - } - - let mut lcs = vec![vec![0usize; new_len + 1]; old_len + 1]; - - for i in 1..=old_len { - for j in 1..=new_len { - let old_spec = ¤t[i - 1].spec; - let new_spec = &target[j - 1].spec; - - if old_spec.is_exact(new_spec) { - lcs[i][j] = lcs[i - 1][j - 1] + 1; - } else { - lcs[i][j] = std::cmp::max(lcs[i - 1][j], lcs[i][j - 1]); - } - } - } - - let mut ops = Vec::new(); - let mut i = old_len; - let mut j = new_len; - - while i > 0 || j > 0 { - if i > 0 && j > 0 { - let old_spec = ¤t[i - 1].spec; - let new_spec = &target[j - 1].spec; - - if old_spec.is_exact(new_spec) { - ops.push(PatchOp::Keep(i - 1, j - 1)); - i -= 1; - j -= 1; - } else if old_spec.is_similar(new_spec) - && lcs[i - 1][j - 1] >= lcs[i - 1][j] - && lcs[i - 1][j - 1] >= lcs[i][j - 1] - { - ops.push(PatchOp::Update(i - 1, j - 1)); - i -= 1; - j -= 1; - } else if lcs[i][j - 1] >= lcs[i - 1][j] { - ops.push(PatchOp::Insert(j - 1)); - j -= 1; - } else { - ops.push(PatchOp::Delete(i - 1)); - i -= 1; - } - } else if j > 0 { - ops.push(PatchOp::Insert(j - 1)); - j -= 1; - } else { - ops.push(PatchOp::Delete(i - 1)); - i -= 1; - } - } - - ops.reverse(); - ops -} - -fn update_block_props( - doc: &Doc, - blocks_map: &mut Map, - node: &StoredNode, - target: &BlockSpec, - preserve_text: bool, -) -> Result<(), ParseError> { - let Some(mut block) = blocks_map.get(&node.id).and_then(|v| v.to_map()) else { - return Err(ParseError::ParserError(format!("Block {} not found", node.id))); - }; - - let preserve = match target.flavour { - BlockFlavour::Image - | BlockFlavour::Table - | BlockFlavour::Bookmark - | BlockFlavour::EmbedYoutube - | BlockFlavour::EmbedIframe => preserve_text, - _ => preserve_text || text_delta_eq(&node.spec.text, &target.text), - }; - - apply_block_spec( - doc, - &mut block, - target, - ApplyBlockOptions { - preserve_text: preserve, - clear_missing: true, - }, - )?; - - Ok(()) -} - -fn sync_children(doc: &Doc, blocks_map: &mut Map, block_id: &str, children: &[String]) -> Result<(), ParseError> { - let Some(mut block) = blocks_map.get(block_id).and_then(|v| v.to_map()) else { - return Err(ParseError::ParserError("Block not found".into())); - }; - - let current_children = collect_child_ids(&block); - if current_children != children { - insert_children(doc, &mut block, children)?; - } - - Ok(()) -} - -fn collect_tree_ids(node: &StoredNode, output: &mut Vec) { - output.push(node.id.clone()); - for child in &node.children { - collect_tree_ids(child, output); - } -} - -fn check_limits(current: &[StoredNode], target: &[BlockNode]) -> Result<(), ParseError> { - let current_count = count_tree_nodes(current); - let target_count = count_tree_nodes(target); - - if current_count > MAX_BLOCKS || target_count > MAX_BLOCKS { - return Err(ParseError::ParserError("block_count_too_large".into())); - } - - if current_count.saturating_mul(target_count) > MAX_LCS_CELLS { - return Err(ParseError::ParserError("diff_matrix_too_large".into())); - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use y_octo::{Any, DocOptions, TextDeltaOp, TextInsert}; - - use super::{super::builder::text_ops_from_plain, *}; - use crate::doc_parser::{ - block_spec::BlockType, blocksuite::get_string, build_full_doc, markdown::MAX_MARKDOWN_CHARS, parse_doc_to_markdown, - }; - - #[test] - fn test_compute_text_diff_simple() { - let ops = text_ops_from_plain("hello world"); - assert_eq!(ops.len(), 1); - match &ops[0] { - TextDeltaOp::Insert { - insert: TextInsert::Text(text), - format: None, - } => { - assert_eq!(text, "hello world"); - } - _ => panic!("unexpected delta op"), - } - } - - #[test] - fn test_content_block_similarity() { - let b1 = BlockSpec { - flavour: BlockFlavour::Paragraph, - block_type: Some(BlockType::H1), - text: text_ops_from_plain("Hello"), - checked: None, - language: None, - order: None, - image: None, - table: None, - bookmark: None, - embed_youtube: None, - embed_iframe: None, - }; - let b2 = BlockSpec { - flavour: BlockFlavour::Paragraph, - block_type: Some(BlockType::H1), - text: text_ops_from_plain("World"), - checked: None, - language: None, - order: None, - image: None, - table: None, - bookmark: None, - embed_youtube: None, - embed_iframe: None, - }; - let b3 = BlockSpec { - flavour: BlockFlavour::Paragraph, - block_type: Some(BlockType::H2), - text: text_ops_from_plain("Hello"), - checked: None, - language: None, - order: None, - image: None, - table: None, - bookmark: None, - embed_youtube: None, - embed_iframe: None, - }; - - assert!(b1.is_similar(&b2)); - assert!(!b1.is_similar(&b3)); - } - - #[test] - fn test_update_ydoc_roundtrip() { - let initial_md = "# Test Document\n\nFirst paragraph.\n\nSecond paragraph."; - let doc_id = "update-test"; - - let initial_bin = build_full_doc("Test Document", initial_md, doc_id).expect("Should create initial doc"); - - let updated_md = "# Test Document\n\nFirst paragraph.\n\nModified second paragraph.\n\nNew third paragraph."; - - let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); - assert!(!delta.is_empty(), "Delta should contain changes"); - } - - #[test] - fn test_update_ydoc_does_not_update_page_title() { - let initial_md = "# Original Title\n\nContent here."; - let doc_id = "title-test"; - - let initial_bin = build_full_doc("Original Title", initial_md, doc_id).expect("Should create initial doc"); - - let updated_md = "# New Title\n\nContent here."; - let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); - - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - doc.apply_update_from_binary_v1(&delta).expect("Should apply delta"); - - let blocks_map = doc.get_map("blocks").expect("blocks map exists"); - let mut title = None; - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() - && get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR) - { - title = get_string(&block_map, "prop:title"); - break; - } - } - - assert_eq!(title.as_deref(), Some("Original Title")); - } - - #[test] - fn test_update_ydoc_no_changes() { - let markdown = "# Same Title\n\nSame content."; - let doc_id = "no-change-test"; - - let initial_bin = build_full_doc("Same Title", markdown, doc_id).expect("Should create initial doc"); - let delta = update_doc(&initial_bin, markdown, doc_id).expect("Should compute delta"); - - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - doc - .apply_update_from_binary_v1(&delta) - .expect("Should apply delta even with no changes"); - } - - #[test] - fn test_update_ydoc_ignores_ai_editable_comments() { - let markdown = "Plain paragraph."; - let doc_id = "ai-comment-test"; - - let initial_bin = build_full_doc("Title", markdown, doc_id).expect("Should create initial doc"); - - let ai_markdown = parse_doc_to_markdown(initial_bin.clone(), doc_id.to_string(), true, None) - .expect("parse doc") - .markdown; - assert!(ai_markdown.contains("block_id=")); - - let delta = update_doc(&initial_bin, &ai_markdown, doc_id).expect("Should compute delta"); - - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - doc.apply_update_from_binary_v1(&delta).expect("Should apply delta"); - - let before = parse_doc_to_markdown(initial_bin, doc_id.to_string(), false, None) - .expect("parse before") - .markdown; - let after = parse_doc_to_markdown(doc.encode_update_v1().unwrap(), doc_id.to_string(), false, None) - .expect("parse after") - .markdown; - - assert_eq!(after, before); - } - - #[test] - fn test_update_ydoc_add_block() { - let initial_md = "# Add Block Test\n\nOriginal paragraph."; - let doc_id = "add-block-test"; - - let initial_bin = build_full_doc("Add Block Test", initial_md, doc_id).expect("Should create initial doc"); - - let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - initial_doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len(); - - let updated_md = "# Add Block Test\n\nOriginal paragraph.\n\nNew paragraph added."; - let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); - assert!(!delta.is_empty(), "Delta should contain changes"); - - let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - updated_doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - updated_doc - .apply_update_from_binary_v1(&delta) - .expect("Should apply delta with new block"); - - let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len(); - assert!( - updated_count > initial_count, - "Expected more blocks after insert, got {updated_count} vs {initial_count}" - ); - } - - #[test] - fn test_update_ydoc_delete_block() { - let initial_md = "# Delete Block Test\n\nFirst paragraph.\n\nSecond paragraph to delete."; - let doc_id = "delete-block-test"; - - let initial_bin = build_full_doc("Delete Block Test", initial_md, doc_id).expect("Should create initial doc"); - - let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - initial_doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len(); - - let updated_md = "# Delete Block Test\n\nFirst paragraph."; - let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); - assert!(!delta.is_empty(), "Delta should contain changes"); - - let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - updated_doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - updated_doc - .apply_update_from_binary_v1(&delta) - .expect("Should apply delta with block deletion"); - - let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len(); - assert!( - updated_count < initial_count, - "Expected fewer blocks after deletion, got {updated_count} vs {initial_count}" - ); - } - - #[test] - fn test_update_ydoc_update_image_caption() { - let initial_md = "![Alt](blob://image-id)"; - let doc_id = "image-update-test"; - let initial_bin = build_full_doc("Image", initial_md, doc_id).expect("create doc"); - - let updated_md = "![New Caption](blob://image-id)"; - let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta"); - - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial"); - doc.apply_update_from_binary_v1(&delta).expect("apply delta"); - - let blocks_map = doc.get_map("blocks").expect("blocks map"); - let mut caption = None; - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() - && get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image") - { - caption = get_string(&block_map, "prop:caption"); - break; - } - } - - assert_eq!(caption.as_deref(), Some("New Caption")); - } - - #[test] - fn test_update_ydoc_update_table_cell() { - let initial_md = "| A | B |\n| --- | --- |\n| 1 | 2 |"; - let doc_id = "table-update-test"; - let initial_bin = build_full_doc("Table", initial_md, doc_id).expect("create doc"); - - let updated_md = "| A | B |\n| --- | --- |\n| 1 | 9 |"; - let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta"); - - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial"); - doc.apply_update_from_binary_v1(&delta).expect("apply delta"); - - let blocks_map = doc.get_map("blocks").expect("blocks map"); - let mut found = false; - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() - && get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table") - { - for key in block_map.keys() { - if key.starts_with("prop:cells.") - && key.ends_with(".text") - && let Some(value) = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a { - Any::String(value) => Some(value), - _ => None, - }) - && value == "9" - { - found = true; - break; - } - } - } - } - - assert!(found); - } - - #[test] - fn test_update_ydoc_concurrent_merge_simulation() { - let base_md = "# Concurrent Test\n\nBase paragraph."; - let doc_id = "concurrent-test"; - - let base_bin = build_full_doc("Concurrent Test", base_md, doc_id).expect("Should create base doc"); - - let mut base_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - base_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base"); - let base_count = base_doc.get_map("blocks").expect("blocks map exists").len(); - - let client_a_md = "# Concurrent Test\n\nModified by client A."; - let delta_a = update_doc(&base_bin, client_a_md, doc_id).expect("Delta A"); - - let client_b_md = "# Concurrent Test\n\nBase paragraph.\n\nAdded by client B."; - let delta_b = update_doc(&base_bin, client_b_md, doc_id).expect("Delta B"); - - let mut final_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - final_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base"); - final_doc.apply_update_from_binary_v1(&delta_a).expect("Apply delta A"); - final_doc.apply_update_from_binary_v1(&delta_b).expect("Apply delta B"); - - let final_count = final_doc.get_map("blocks").expect("blocks map exists").len(); - assert!( - final_count > base_count, - "Expected merged blocks after concurrent updates, got {final_count} vs {base_count}" - ); - } - - #[test] - fn test_update_ydoc_empty_binary_errors() { - let markdown = "# New Document\n\nCreated from empty binary."; - let doc_id = "empty-fallback-test"; - - let result = update_doc(&[], markdown, doc_id); - assert!(result.is_err()); - - let result = update_doc(&[0, 0], markdown, doc_id); - assert!(result.is_err()); - } - - #[test] - fn test_update_ydoc_markdown_too_large() { - let initial_md = "# Title\n\nContent."; - let doc_id = "size-limit-test"; - let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc"); - - let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1); - let result = update_doc(&initial_bin, &markdown, doc_id); - assert!(result.is_err()); - } - - #[test] - fn test_update_ydoc_rejects_unsupported_markdown() { - let initial_md = "# Title\n\nContent."; - let doc_id = "unsupported-test"; - let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc"); - - let markdown = "# Title\n\n
HTML
"; - let result = update_doc(&initial_bin, markdown, doc_id); - assert!(result.is_err()); - } -} diff --git a/packages/common/native/src/lib.rs b/packages/common/native/src/lib.rs index 7185a108eda2f..3f8fe10377c5b 100644 --- a/packages/common/native/src/lib.rs +++ b/packages/common/native/src/lib.rs @@ -1,5 +1,3 @@ -#[cfg(feature = "ydoc-loader")] -pub mod doc_parser; #[cfg(feature = "hashcash")] pub mod hashcash; #[cfg(feature = "napi")] diff --git a/packages/frontend/apps/electron-renderer/src/app/native-import.ts b/packages/frontend/apps/electron-renderer/src/app/native-import.ts new file mode 100644 index 0000000000000..9282b66564ba7 --- /dev/null +++ b/packages/frontend/apps/electron-renderer/src/app/native-import.ts @@ -0,0 +1,15 @@ +import { registerNativeImportSessionHandlers } from '@affine/core/modules/import'; +import { apis } from '@affine/electron-api'; + +const importApis = apis?.import; + +if (importApis) { + registerNativeImportSessionHandlers({ + createImportSession: options => + importApis.createImportSessionFromSource(options), + nextImportBatch: sessionId => importApis.nextImportBatch(sessionId), + cancelImportSession: sessionId => importApis.cancelImportSession(sessionId), + disposeImportSession: sessionId => + importApis.disposeImportSession(sessionId), + }); +} diff --git a/packages/frontend/apps/electron-renderer/src/app/setup.ts b/packages/frontend/apps/electron-renderer/src/app/setup.ts index 78fb623056537..1502bd95e11c3 100644 --- a/packages/frontend/apps/electron-renderer/src/app/setup.ts +++ b/packages/frontend/apps/electron-renderer/src/app/setup.ts @@ -2,6 +2,7 @@ import '@affine/core/bootstrap/electron'; import '@affine/core/bootstrap/cleanup'; import '@affine/component/theme'; import './global.css'; +import './native-import'; import { apis } from '@affine/electron-api'; import { bindNativeDBApis } from '@affine/nbstore/sqlite'; diff --git a/packages/frontend/apps/electron/package.json b/packages/frontend/apps/electron/package.json index a0d158345e9bb..605ab4002d75b 100644 --- a/packages/frontend/apps/electron/package.json +++ b/packages/frontend/apps/electron/package.json @@ -15,6 +15,7 @@ "./main/exposed": "./src/main/exposed.ts", "./preload/electron-api": "./src/preload/electron-api.ts", "./preload/shared-storage": "./src/preload/shared-storage.ts", + "./shared/import": "./src/shared/import.ts", "./main/shared-state-schema": "./src/main/shared-state-schema.ts", "./main/updater/event": "./src/main/updater/event.ts", "./main/windows-manager": "./src/main/windows-manager/index.ts" diff --git a/packages/frontend/apps/electron/src/main/handlers.ts b/packages/frontend/apps/electron/src/main/handlers.ts index b11ffc592b32e..e9f999bee3b4d 100644 --- a/packages/frontend/apps/electron/src/main/handlers.ts +++ b/packages/frontend/apps/electron/src/main/handlers.ts @@ -7,6 +7,7 @@ import { byokStorageHandlers } from './byok-storage/handlers'; import { clipboardHandlers } from './clipboard'; import { configStorageHandlers } from './config-storage'; import { findInPageHandlers } from './find-in-page'; +import { importHandlers } from './import'; import { getLogFilePath, logger, revealLogFile } from './logger'; import { recordingHandlers } from './recording'; import { checkSource } from './security-restrictions'; @@ -39,6 +40,7 @@ export const allHandlers = { updater: updaterHandlers, configStorage: configStorageHandlers, findInPage: findInPageHandlers, + import: importHandlers, sharedStorage: sharedStorageHandlers, worker: workerHandlers, recording: recordingHandlers, diff --git a/packages/frontend/apps/electron/src/main/import/handlers.ts b/packages/frontend/apps/electron/src/main/import/handlers.ts new file mode 100644 index 0000000000000..5a8efb6a9c649 --- /dev/null +++ b/packages/frontend/apps/electron/src/main/import/handlers.ts @@ -0,0 +1,39 @@ +import type { CreateImportSessionOptions } from '@affine/native'; +import { + cancelImportSession, + createImportSession, + disposeImportSession, + nextImportBatch, +} from '@affine/native'; + +export const importHandlers = { + createImportSession: ( + event: Electron.IpcMainInvokeEvent, + options: CreateImportSessionOptions + ) => { + void event; + return createImportSession({ + format: options.format, + source: options.source, + batchLimits: options.batchLimits, + }); + }, + nextImportBatch: (event: Electron.IpcMainInvokeEvent, sessionId: string) => { + void event; + return nextImportBatch(sessionId); + }, + cancelImportSession: ( + event: Electron.IpcMainInvokeEvent, + sessionId: string + ) => { + void event; + return cancelImportSession(sessionId); + }, + disposeImportSession: ( + event: Electron.IpcMainInvokeEvent, + sessionId: string + ) => { + void event; + return disposeImportSession(sessionId); + }, +}; diff --git a/packages/frontend/apps/electron/src/main/import/index.ts b/packages/frontend/apps/electron/src/main/import/index.ts new file mode 100644 index 0000000000000..ebe9f43ed47be --- /dev/null +++ b/packages/frontend/apps/electron/src/main/import/index.ts @@ -0,0 +1 @@ +export { importHandlers } from './handlers'; diff --git a/packages/frontend/apps/electron/src/preload/electron-api.ts b/packages/frontend/apps/electron/src/preload/electron-api.ts index 4429699b433b9..f82dcaf1e0166 100644 --- a/packages/frontend/apps/electron/src/preload/electron-api.ts +++ b/packages/frontend/apps/electron/src/preload/electron-api.ts @@ -3,10 +3,14 @@ import type { MessagePort } from 'node:worker_threads'; import type { EventBasedChannel } from 'async-call-rpc'; import { AsyncCall } from 'async-call-rpc'; -import { ipcRenderer } from 'electron'; +import { ipcRenderer, webUtils } from 'electron'; import { Subject } from 'rxjs'; import { z } from 'zod'; +import type { + CreateImportSessionFromSourceOptions, + NativeImportBrowserSource, +} from '../shared/import'; import { AFFINE_API_CHANNEL_NAME, AFFINE_EVENT_CHANNEL_NAME, @@ -275,9 +279,61 @@ function getHelperAPIs() { const mainAPIs = getMainAPIs(); const helperAPIs = getHelperAPIs(); +type DirectoryImportFile = File & { webkitRelativePath?: string }; + +function filePathFromFile(file: File) { + return webUtils.getPathForFile(file); +} + +function directoryPathFromFiles(files: File[]) { + const first = files.find( + (file): file is DirectoryImportFile => + !!(file as DirectoryImportFile).webkitRelativePath + ); + if (!first) return null; + const filePath = filePathFromFile(first); + if (!filePath) return null; + const relativePath = first.webkitRelativePath; + if (!relativePath) return null; + const relativeParts = relativePath.split('/'); + let rootPath = filePath.replaceAll('\\', '/'); + for (let i = relativeParts.length - 1; i > 0; i--) { + const part = relativeParts[i]; + if (part && rootPath.endsWith(`/${part}`)) { + rootPath = rootPath.slice(0, -part.length - 1); + } + } + return rootPath || null; +} + +function resolveNativeImportSource(source: NativeImportBrowserSource) { + if (source.kind === 'file') { + const path = filePathFromFile(source.file); + return path ? { kind: 'filePath', path } : null; + } + const path = directoryPathFromFiles(source.files); + return path ? { kind: 'directoryPath', path } : null; +} + export const apis = { ...mainAPIs.apis, ...helperAPIs.apis, + import: { + ...mainAPIs.apis.import, + createImportSessionFromSource( + options: CreateImportSessionFromSourceOptions + ) { + const source = resolveNativeImportSource(options.source); + if (!source) { + throw new Error('Native import requires a local file source'); + } + return mainAPIs.apis.import.createImportSession({ + format: options.format, + source, + batchLimits: options.batchLimits, + }); + }, + }, }; export const events = { diff --git a/packages/frontend/apps/electron/src/shared/import.ts b/packages/frontend/apps/electron/src/shared/import.ts new file mode 100644 index 0000000000000..a9bdd32975956 --- /dev/null +++ b/packages/frontend/apps/electron/src/shared/import.ts @@ -0,0 +1,29 @@ +export type NativeImportFormat = + | 'markdownZip' + | 'notionZip' + | 'obsidian' + | 'bearZip' + | 'oneNote'; + +export type NativeImportBrowserSource = + | { kind: 'file'; file: File } + | { kind: 'directory'; files: File[] }; + +export type CreateImportSessionFromSourceOptions = { + format: NativeImportFormat; + source: NativeImportBrowserSource; + batchLimits?: { + maxDocs?: number; + maxBlobs?: number; + maxBlobBytes?: number; + }; +}; + +export type NativeImportSessionHandlers = { + createImportSession( + options: CreateImportSessionFromSourceOptions + ): Promise | string; + nextImportBatch(sessionId: string): Promise | string | null; + cancelImportSession(sessionId: string): Promise | void; + disposeImportSession(sessionId: string): Promise | void; +}; diff --git a/packages/frontend/core/src/desktop/dialogs/import/commit-service.spec.ts b/packages/frontend/core/src/desktop/dialogs/import/commit-service.spec.ts new file mode 100644 index 0000000000000..6dedd277d237e --- /dev/null +++ b/packages/frontend/core/src/desktop/dialogs/import/commit-service.spec.ts @@ -0,0 +1,547 @@ +import 'fake-indexeddb/auto'; + +import { getStoreManager } from '@affine/core/blocksuite/manager/store'; +import { getAFFiNEWorkspaceSchema } from '@affine/core/modules/workspace'; +import type { DocSnapshot } from '@blocksuite/affine/store'; +import { TestWorkspace } from '@blocksuite/affine/store/test'; +import type { ImportBatch } from '@blocksuite/affine/widgets/linked-doc'; +import { describe, expect, test, vi } from 'vitest'; + +import { ImportCommitService } from './commit-service'; + +function docSnapshot(id: string, title: string): DocSnapshot { + return { + type: 'page', + meta: { + id, + title, + createDate: 0, + tags: [], + }, + blocks: { + type: 'block', + id: `block:${id}`, + flavour: 'affine:page', + props: { + title: { + '$blocksuite:internal:text$': true, + delta: [{ insert: title }], + }, + }, + children: [ + { + type: 'block', + id: `block:${id}:note`, + flavour: 'affine:note', + props: {}, + children: [ + { + type: 'block', + id: `block:${id}:paragraph`, + flavour: 'affine:paragraph', + props: { + type: 'text', + text: { + '$blocksuite:internal:text$': true, + delta: [{ insert: title }], + }, + }, + children: [], + }, + ], + }, + ], + }, + }; +} + +function createFolderTree() { + const folders = new Map(); + const links: { parentId: string; docId: string }[] = []; + let nextId = 0; + + class FolderNode { + readonly children: string[] = []; + + constructor(readonly id: string) { + folders.set(id, this); + } + + createFolder() { + const id = `folder-${++nextId}`; + this.children.push(id); + new FolderNode(id); + return id; + } + + createLink(...[, docId]: ['doc', string]) { + links.push({ parentId: this.id, docId }); + } + + indexAt() { + return 'after'; + } + } + + const rootFolder = new FolderNode('root'); + return { + links, + service: { + folderTree: { + rootFolder, + ['folderNode$']: (id: string) => ({ value: folders.get(id) }), + }, + }, + }; +} + +function createCommitService( + collection: TestWorkspace, + options: { + organizeService?: unknown; + explorerIconService?: unknown; + tagService?: unknown; + } = {} +) { + return new ImportCommitService({ + collection, + schema: getAFFiNEWorkspaceSchema(), + extensions: getStoreManager().config.init().value.get('store'), + organizeService: options.organizeService as never, + explorerIconService: options.explorerIconService as never, + tagService: options.tagService as never, + logger: { + warn: vi.fn(), + }, + }); +} + +describe('ImportCommitService', () => { + test('commits native batch blobs, docs, folders, icons, and warnings', async () => { + const collection = new TestWorkspace({ id: 'test' }); + collection.meta.initialize(); + const folderTree = createFolderTree(); + const setIcon = vi.fn(); + const service = createCommitService(collection, { + organizeService: folderTree.service, + explorerIconService: { setIcon }, + }); + + const batch: ImportBatch = { + blobs: [ + { + blobId: 'blob-1', + sourcePath: 'assets/image.png', + fileName: 'image.png', + mime: 'image/png', + bytes: new Uint8Array([1, 2, 3]), + }, + ], + docs: [ + { + id: 'doc-1', + snapshot: docSnapshot('doc-1', 'Imported'), + meta: { title: 'Committed title', favorite: true }, + }, + ], + folders: [ + { path: 'root-folder', name: 'Root folder' }, + { + path: 'root-folder/doc-1', + name: 'Imported', + parentPath: 'root-folder', + pageId: 'doc-1', + icon: { type: 'emoji', unicode: '✅' }, + }, + ], + warnings: [{ code: 'lossy', message: 'Dropped unsupported block' }], + done: true, + }; + + const result = await service.commitBatch(batch); + + expect(result).toEqual({ + docIds: ['doc-1'], + rootFolderId: 'folder-1', + warnings: batch.warnings, + }); + await expect(collection.blobSync.get('blob-1')).resolves.toBeInstanceOf( + File + ); + expect(collection.getDoc('doc-1')).not.toBeNull(); + expect(collection.meta.getDocMeta('doc-1')).toMatchObject({ + title: 'Committed title', + favorite: true, + }); + expect(folderTree.links).toEqual([ + { parentId: 'folder-1', docId: 'doc-1' }, + ]); + expect(setIcon).toHaveBeenCalledWith({ + where: 'doc', + id: 'doc-1', + icon: { type: 'emoji', unicode: '✅' }, + }); + }); + + test('records doc commit failures as warnings and continues remaining docs', async () => { + const collection = new TestWorkspace({ id: 'test' }); + collection.meta.initialize(); + const service = createCommitService(collection); + const result = await service.commitBatch({ + blobs: [], + docs: [ + { + id: 'doc-skip', + sourcePath: 'docs/skip.md', + snapshot: { ...docSnapshot('doc-skip', 'Skip'), blocks: null }, + } as never, + { + id: 'doc-ok', + sourcePath: 'docs/ok.md', + snapshot: docSnapshot('doc-ok', 'Ok'), + meta: { title: 'Ok' }, + }, + ], + done: true, + }); + + expect(result.docIds).toEqual(['doc-ok']); + expect(result.warnings).toEqual([ + { + code: 'skipped_doc', + sourcePath: 'docs/skip.md', + message: + 'Skipped docs/skip.md: document snapshot could not be committed', + }, + ]); + expect(collection.getDoc('doc-ok')).not.toBeNull(); + }); + + test('records doc meta failures as warnings without dropping committed docs', async () => { + const collection = new TestWorkspace({ id: 'test' }); + collection.meta.initialize(); + const service = createCommitService(collection); + const originalSetDocMeta = collection.meta.setDocMeta.bind(collection.meta); + const setDocMeta = vi.spyOn(collection.meta, 'setDocMeta'); + setDocMeta.mockImplementation((id, meta) => { + if (id === 'doc-meta') { + throw new Error('meta failed'); + } + return originalSetDocMeta(id, meta); + }); + + try { + const result = await service.commitBatch({ + blobs: [], + docs: [ + { + id: 'doc-meta', + sourcePath: 'docs/meta.md', + snapshot: docSnapshot('doc-meta', 'Meta'), + meta: { title: 'Meta' }, + }, + { + id: 'doc-ok', + sourcePath: 'docs/ok.md', + snapshot: docSnapshot('doc-ok', 'Ok'), + meta: { title: 'Ok' }, + }, + ], + done: true, + }); + + expect(result.docIds).toEqual(['doc-meta', 'doc-ok']); + expect(result.warnings).toEqual([ + { + code: 'doc_meta_failed', + sourcePath: 'docs/meta.md', + message: 'Failed to apply metadata for docs/meta.md: meta failed', + }, + ]); + expect(collection.getDoc('doc-meta')).not.toBeNull(); + expect(collection.getDoc('doc-ok')).not.toBeNull(); + expect(collection.meta.getDocMeta('doc-ok')).toMatchObject({ + title: 'Ok', + }); + } finally { + setDocMeta.mockRestore(); + } + }); + + test('commits native tag names as workspace tags', async () => { + const collection = new TestWorkspace({ id: 'test' }); + collection.meta.initialize(); + const tags = new Map(); + const service = createCommitService(collection, { + tagService: { + randomTagColor: () => 'red', + tagList: { + ['tags$']: { + value: [], + }, + createTag: (value: string) => { + const tag = { id: `tag-${tags.size + 1}`, value }; + tags.set(value, tag); + return tag; + }, + }, + }, + }); + + await service.commitBatch({ + blobs: [], + docs: [ + { + id: 'doc-1', + snapshot: docSnapshot('doc-1', 'Tagged'), + meta: { tags: ['Blue Tag'], title: 'Tagged' }, + }, + ], + tags: [{ name: 'work/project', docIds: ['doc-1'] }], + done: true, + }); + + expect([...tags.values()]).toEqual([ + { id: 'tag-1', value: 'Blue Tag' }, + { id: 'tag-2', value: 'work' }, + ]); + expect(collection.meta.getDocMeta('doc-1')?.tags).toEqual([ + 'tag-1', + 'tag-2', + ]); + }); + + test('commits batch folders, icons, and collapsed Bear root tags', async () => { + const collection = new TestWorkspace({ id: 'test' }); + collection.meta.initialize(); + collection.createDoc('doc-1'); + collection.createDoc('doc-2'); + const folderTree = createFolderTree(); + const setIcon = vi.fn(); + const tags = new Map(); + const service = createCommitService(collection, { + organizeService: folderTree.service, + explorerIconService: { setIcon }, + tagService: { + randomTagColor: () => 'red', + tagList: { + ['tags$']: { + value: [], + }, + createTag: (value: string) => { + const tag = { id: `tag-${tags.size + 1}`, value }; + tags.set(value, tag); + return tag; + }, + }, + }, + }); + + const result = await service.commitBatch({ + docs: [], + blobs: [], + icons: [{ docId: 'doc-1', icon: { type: 'emoji', unicode: '📘' } }], + tags: [ + { name: 'work/project', docIds: ['doc-1'] }, + { name: 'work/research', docIds: ['doc-2'] }, + ], + folders: [ + { path: 'Bear', name: 'Bear' }, + { + path: 'Bear/Idea', + name: 'Idea', + parentPath: 'Bear', + pageId: 'doc-1', + }, + ], + done: true, + }); + + expect(result).toEqual({ + docIds: [], + rootFolderId: 'folder-1', + warnings: [], + }); + expect(folderTree.links).toEqual([ + { parentId: 'folder-1', docId: 'doc-1' }, + ]); + expect(setIcon).toHaveBeenCalledWith({ + where: 'doc', + id: 'doc-1', + icon: { type: 'emoji', unicode: '📘' }, + }); + expect([...tags.values()]).toEqual([{ id: 'tag-1', value: 'work' }]); + expect(collection.meta.getDocMeta('doc-1')?.tags).toEqual(['tag-1']); + expect(collection.meta.getDocMeta('doc-2')?.tags).toEqual(['tag-1']); + }); + + test('keeps folder and doc links idempotent across repeated commits', async () => { + const collection = new TestWorkspace({ id: 'test' }); + collection.meta.initialize(); + collection.createDoc('doc-1'); + const folderTree = createFolderTree(); + const service = createCommitService(collection, { + organizeService: folderTree.service, + }); + const batch: ImportBatch = { + docs: [], + blobs: [], + folders: [ + { path: 'Root', name: 'Root' }, + { + path: 'Root/Doc', + name: 'Doc', + parentPath: 'Root', + pageId: 'doc-1', + }, + ], + done: true, + }; + + const first = await service.commitBatch(batch); + const second = await service.commitBatch(batch); + + expect(first.rootFolderId).toBe('folder-1'); + expect(second.rootFolderId).toBe('folder-1'); + expect(folderTree.links).toEqual([ + { parentId: 'folder-1', docId: 'doc-1' }, + ]); + }); + + test('resolves partial batch folder links when parent arrives later', async () => { + const collection = new TestWorkspace({ id: 'test' }); + collection.meta.initialize(); + collection.createDoc('doc-1'); + const folderTree = createFolderTree(); + const service = createCommitService(collection, { + organizeService: folderTree.service, + }); + + await service.commitBatch({ + docs: [], + blobs: [], + folders: [ + { + path: 'Root/Doc', + name: 'Doc', + parentPath: 'Root', + pageId: 'doc-1', + }, + ], + done: false, + }); + await service.commitBatch({ + docs: [], + blobs: [], + folders: [{ path: 'Root', name: 'Root' }], + done: true, + }); + + expect(folderTree.links).toEqual([ + { parentId: 'folder-1', docId: 'doc-1' }, + ]); + }); + + test('warns and clears unresolved folders on final batch', async () => { + const collection = new TestWorkspace({ id: 'test' }); + collection.meta.initialize(); + collection.createDoc('doc-1'); + const folderTree = createFolderTree(); + const service = createCommitService(collection, { + organizeService: folderTree.service, + }); + + const result = await service.commitBatch({ + docs: [], + blobs: [], + folders: [ + { + path: 'Missing/Doc', + name: 'Doc', + parentPath: 'Missing', + pageId: 'doc-1', + }, + ], + done: true, + }); + + expect(folderTree.links).toEqual([]); + expect(result.warnings).toEqual([ + { + code: 'unresolved_folder', + sourcePath: 'Missing/Doc', + message: + 'Skipped folder placement for Missing/Doc: parent folder was not found', + }, + ]); + }); + + test('applies icons for root-level imported pages', async () => { + const collection = new TestWorkspace({ id: 'test' }); + collection.meta.initialize(); + collection.createDoc('doc-1'); + const folderTree = createFolderTree(); + const setIcon = vi.fn(); + const service = createCommitService(collection, { + organizeService: folderTree.service, + explorerIconService: { setIcon }, + }); + + await service.commitBatch({ + docs: [], + blobs: [], + folders: [ + { + path: 'Doc', + name: 'Doc', + pageId: 'doc-1', + icon: { type: 'emoji', unicode: '📌' }, + }, + ], + done: true, + }); + + expect(setIcon).toHaveBeenCalledWith({ + where: 'doc', + id: 'doc-1', + icon: { type: 'emoji', unicode: '📌' }, + }); + }); + + test('keeps Notion page nodes usable as child containers', async () => { + const collection = new TestWorkspace({ id: 'test' }); + collection.meta.initialize(); + collection.createDoc('project'); + collection.createDoc('nested'); + const folderTree = createFolderTree(); + const service = createCommitService(collection, { + organizeService: folderTree.service, + }); + + await service.commitBatch({ + docs: [], + blobs: [], + folders: [ + { path: 'Export', name: 'Export' }, + { + path: 'Export/Project', + name: 'Project', + parentPath: 'Export', + pageId: 'project', + }, + { + path: 'Export/Project/Nested', + name: 'Nested', + parentPath: 'Export/Project', + pageId: 'nested', + }, + ], + done: true, + }); + + expect(folderTree.links).toEqual([ + { parentId: 'folder-1', docId: 'project' }, + { parentId: 'folder-2', docId: 'nested' }, + ]); + }); +}); diff --git a/packages/frontend/core/src/desktop/dialogs/import/commit-service.ts b/packages/frontend/core/src/desktop/dialogs/import/commit-service.ts new file mode 100644 index 0000000000000..ab5a870c68fbd --- /dev/null +++ b/packages/frontend/core/src/desktop/dialogs/import/commit-service.ts @@ -0,0 +1,333 @@ +import type { IconData } from '@affine/component'; +import type { ExplorerIconService } from '@affine/core/modules/explorer-icon/services/explorer-icon'; +import type { OrganizeService } from '@affine/core/modules/organize'; +import type { TagService } from '@affine/core/modules/tag'; +import { + type ExtensionType, + type Schema, + Transformer, + type Workspace, +} from '@blocksuite/affine/store'; +import type { + ImportBatch, + ImportCommitResult, + ImportFolder, + ImportIconData, +} from '@blocksuite/affine/widgets/linked-doc'; + +type Logger = { + warn: (message: string, ...args: unknown[]) => void; +}; + +type CommitServiceOptions = { + collection: Workspace; + schema: Schema; + extensions: ExtensionType[]; + organizeService?: OrganizeService; + explorerIconService?: ExplorerIconService; + tagService?: TagService; + logger: Logger; +}; + +type EmojiIconData = Extract; +const EMOJI_ICON_TYPE = 'emoji' as EmojiIconData['type']; + +function copyToArrayBuffer(bytes: Uint8Array) { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + +function errorMessage(error: unknown) { + return error instanceof Error + ? error.message || error.name + : 'Unknown error occurred'; +} + +export class ImportCommitService { + private readonly folderIdByPath = new Map(); + private readonly linkedDocsByFolder = new Set(); + private readonly pendingFolders: ImportFolder[] = []; + + constructor(private readonly options: CommitServiceOptions) {} + + async commitBatch(batch: ImportBatch): Promise { + const warnings = [...(batch.warnings ?? [])]; + for (const blob of batch.blobs) { + const bytes = new Uint8Array(blob.bytes); + await this.options.collection.blobSync.set( + blob.blobId, + new File([copyToArrayBuffer(bytes)], blob.fileName, { + type: blob.mime, + }) + ); + } + + const transformer = new Transformer({ + schema: this.options.schema, + blobCRUD: this.options.collection.blobSync, + docCRUD: { + create: (id: string) => + this.options.collection.createDoc(id).getStore({ id }), + get: (id: string) => + this.options.collection.getDoc(id)?.getStore({ id }) ?? null, + delete: (id: string) => this.options.collection.removeDoc(id), + }, + middlewares: [], + }); + + const docIds: string[] = []; + const tags = new Map(); + for (const doc of batch.docs) { + let store: Awaited>; + try { + store = await transformer.snapshotToDoc(doc.snapshot); + } catch (error) { + warnings.push({ + code: 'skipped_doc', + sourcePath: doc.sourcePath, + message: `Skipped ${doc.sourcePath ?? doc.id}: ${errorMessage(error)}`, + }); + continue; + } + if (!store) { + warnings.push({ + code: 'skipped_doc', + sourcePath: doc.sourcePath, + message: `Skipped ${doc.sourcePath ?? doc.id}: document snapshot could not be committed`, + }); + continue; + } + docIds.push(store.id); + if (doc.meta && Object.keys(doc.meta).length) { + try { + const { tags: docTags, ...meta } = doc.meta; + if (Object.keys(meta).length) { + this.options.collection.meta.setDocMeta(store.id, meta); + } + for (const tag of docTags ?? []) { + const docIds = tags.get(tag) ?? []; + docIds.push(store.id); + tags.set(tag, docIds); + } + } catch (error) { + warnings.push({ + code: 'doc_meta_failed', + sourcePath: doc.sourcePath, + message: `Failed to apply metadata for ${doc.sourcePath ?? doc.id}: ${errorMessage(error)}`, + }); + } + } + } + for (const tag of batch.tags ?? []) { + const docIds = tags.get(tag.name) ?? []; + docIds.push(...tag.docIds); + tags.set(tag.name, docIds); + } + + const rootFolderId = this.applyNativeFolders( + batch.folders ?? [], + warnings, + batch.done + ); + this.applyNativeTags(tags); + this.applyNativeIcons(batch.icons); + return { + docIds, + entryId: batch.entryId, + isWorkspaceFile: batch.isWorkspaceFile, + rootFolderId, + warnings, + }; + } + + private applyNativeFolders( + folders: ImportFolder[], + warnings: ImportCommitResult['warnings'], + batchDone: boolean + ): string | undefined { + const { organizeService } = this.options; + if (folders.length === 0) return undefined; + if (!organizeService) { + for (const folder of folders) { + if (folder.pageId && !folder.parentPath) { + this.applyIcon(folder.pageId, folder.icon); + } + } + return undefined; + } + + try { + let rootFolderId: string | undefined; + this.pendingFolders.push(...folders); + let progressed = true; + + while (progressed && this.pendingFolders.length) { + progressed = false; + const nextPending: ImportFolder[] = []; + const pending = this.pendingFolders.splice(0); + const childParentPaths = new Set( + pending + .map(folder => folder.parentPath) + .filter((path): path is string => !!path) + ); + for (const folder of pending) { + const needsContainer = + !folder.pageId || childParentPaths.has(folder.path); + if (needsContainer && !this.folderIdByPath.has(folder.path)) { + const parent = folder.parentPath + ? this.folderIdByPath.has(folder.parentPath) + ? organizeService.folderTree.folderNode$( + this.folderIdByPath.get(folder.parentPath) ?? '' + ).value + : null + : organizeService.folderTree.rootFolder; + if (!parent) { + nextPending.push(folder); + continue; + } + const folderId = parent.createFolder( + folder.name, + parent.indexAt('after') + ); + this.folderIdByPath.set(folder.path, folderId); + rootFolderId ??= folderId; + progressed = true; + } else if (needsContainer) { + rootFolderId ??= this.folderIdByPath.get(folder.path); + } + + if (folder.pageId) { + if (!this.applyFolderDocLink(folder)) { + nextPending.push(folder); + continue; + } + progressed = true; + } + } + if (progressed && nextPending.length) { + this.pendingFolders.push(...nextPending); + } else if (batchDone) { + for (const folder of nextPending) { + warnings.push({ + code: 'unresolved_folder', + sourcePath: folder.path, + message: `Skipped folder placement for ${folder.path}: parent folder was not found`, + }); + } + break; + } else { + this.pendingFolders.push(...nextPending); + break; + } + } + + return rootFolderId; + } catch (error) { + this.options.logger.warn('Failed to commit import folders:', error); + return undefined; + } + } + + private applyFolderDocLink(folder: ImportFolder): boolean { + const { organizeService } = this.options; + if (!folder.pageId) return true; + if (!folder.parentPath) { + this.applyIcon(folder.pageId, folder.icon); + return true; + } + if (!organizeService) return true; + const parentFolderId = this.folderIdByPath.get(folder.parentPath); + if (!parentFolderId) return false; + const linkKey = `${parentFolderId}:${folder.pageId}`; + if (!this.linkedDocsByFolder.has(linkKey)) { + const parent = + organizeService.folderTree.folderNode$(parentFolderId).value; + if (!parent) return false; + parent.createLink('doc', folder.pageId, parent.indexAt('after')); + this.linkedDocsByFolder.add(linkKey); + } + this.applyIcon(folder.pageId, folder.icon); + return true; + } + + private applyNativeIcons(icons?: ImportBatch['icons']) { + for (const icon of icons ?? []) { + this.applyIcon(icon.docId, icon.icon); + } + } + + private applyIcon(id: string, icon?: ImportIconData) { + if (!icon || !this.options.explorerIconService) return; + const iconData = toIconData(icon); + if (!iconData) return; + this.options.explorerIconService.setIcon({ + where: 'doc', + id, + icon: iconData, + }); + } + + private applyNativeTags(tags?: Map) { + const { tagService, collection } = this.options; + if (!tagService || !tags?.size) return; + + try { + const existingTagMap = new Map(); + for (const tag of tagService.tagList.tags$.value) { + existingTagMap.set(tag.value$.value.toLowerCase(), tag.id); + } + + const rootTagDocMap = new Map< + string, + { displayName: string; docs: Set } + >(); + for (const [tagName, tagDocIds] of tags) { + const originalRoot = tagName.split('/')[0]; + const key = originalRoot.toLowerCase(); + let entry = rootTagDocMap.get(key); + if (!entry) { + entry = { displayName: originalRoot, docs: new Set() }; + rootTagDocMap.set(key, entry); + } + for (const docId of tagDocIds) { + entry.docs.add(docId); + } + } + + for (const [rootTagKey, { displayName, docs }] of rootTagDocMap) { + let tagId = existingTagMap.get(rootTagKey); + if (!tagId) { + const newTag = tagService.tagList.createTag( + displayName, + tagService.randomTagColor() + ); + tagId = newTag.id; + existingTagMap.set(rootTagKey, tagId); + } + + for (const docId of docs) { + const doc = collection.getDoc(docId); + const currentTags = doc?.meta?.tags ?? []; + if (!currentTags.includes(tagId)) { + collection.meta.setDocMeta(docId, { + tags: [...currentTags, tagId], + }); + } + } + } + } catch (error) { + this.options.logger.warn('Failed to commit import tags:', error); + } + } +} + +function toIconData(icon: ImportIconData): IconData | undefined { + if (icon.type === 'emoji') { + return { + type: EMOJI_ICON_TYPE, + unicode: icon.unicode, + }; + } + return undefined; +} diff --git a/packages/frontend/core/src/desktop/dialogs/import/index.tsx b/packages/frontend/core/src/desktop/dialogs/import/index.tsx index 7331e42297de3..2bc4f2fd4f8c1 100644 --- a/packages/frontend/core/src/desktop/dialogs/import/index.tsx +++ b/packages/frontend/core/src/desktop/dialogs/import/index.tsx @@ -1,10 +1,4 @@ -import { - Button, - IconButton, - type IconData, - IconType, - Modal, -} from '@affine/component'; +import { Button, IconButton, Modal } from '@affine/component'; import { getStoreManager } from '@affine/core/blocksuite/manager/store'; import { useAsyncCallback } from '@affine/core/components/hooks/affine-async-hooks'; import { useNavigateHelper } from '@affine/core/components/hooks/use-navigate-helper'; @@ -13,9 +7,10 @@ import { GlobalDialogService, type WORKSPACE_DIALOG_SCHEMA, } from '@affine/core/modules/dialogs'; -import { ExplorerIconService } from '@affine/core/modules/explorer-icon/services/explorer-icon'; -import { OrganizeService } from '@affine/core/modules/organize'; -import { TagService } from '@affine/core/modules/tag'; +import { + type ImportRunContext, + ImportService, +} from '@affine/core/modules/import'; import { UrlService } from '@affine/core/modules/url'; import { getAFFiNEWorkspaceSchema, @@ -28,12 +23,10 @@ import track from '@affine/track'; import { openDirectory, openFilesWith } from '@blocksuite/affine/shared/utils'; import type { Workspace } from '@blocksuite/affine/store'; import { - BearTransformer, DocxTransformer, HtmlTransformer, + type ImportWarning, MarkdownTransformer, - NotionHtmlTransformer, - ObsidianTransformer, ZipTransformer, } from '@blocksuite/affine/widgets/linked-doc'; import { @@ -54,6 +47,7 @@ import { type SVGAttributes, useCallback, useMemo, + useRef, useState, } from 'react'; @@ -61,207 +55,97 @@ import * as style from './styles.css'; const logger = new DebugLogger('import'); -type NotionPageIcon = { - type: 'emoji' | 'image'; - content: string; // emoji unicode or image URL/data -}; - -type FolderHierarchy = { - name: string; - path: string; - children: Map; - pageId?: string; - parentPath?: string; - icon?: NotionPageIcon; -}; - -// Helper function to create folder structure using OrganizeService -function createFolderStructure( - organizeService: OrganizeService, - hierarchy: FolderHierarchy, - parentFolderId: string | null = null, - explorerIconService?: ExplorerIconService -): { - folderId: string | null; - docLinks: Array<{ folderId: string; docId: string }>; -} { - const docLinks: Array<{ folderId: string; docId: string }> = []; - const rootFolder = organizeService.folderTree.rootFolder; - - function processHierarchyNode( - node: FolderHierarchy, - currentParentId: string | null - ): string | null { - let currentFolderId = currentParentId; - - // If this node represents a folder (has children but no pageId), create it - if (node.children.size > 0 && !node.pageId && node.name) { - const parent = currentParentId - ? organizeService.folderTree.folderNode$(currentParentId).value - : rootFolder; - - if (parent) { - const index = parent.indexAt('after'); - currentFolderId = parent.createFolder(node.name, index); - } - } - - // Process all children - for (const child of node.children.values()) { - if (child.pageId) { - // This is a document, link it to the current folder - if (currentFolderId) { - docLinks.push({ folderId: currentFolderId, docId: child.pageId }); - } - - // Set icon for the document if available - if (child.icon && explorerIconService) { - logger.debug('=== Setting icon for document ==='); - logger.debug('Document ID:', child.pageId); - logger.debug('Icon data:', child.icon); - - try { - let iconData: IconData | undefined; - if (child.icon.type === 'emoji') { - iconData = { - type: IconType.Emoji, - unicode: child.icon.content, - }; - logger.debug('Created emoji icon data:', iconData); - } else if (child.icon.type === 'image') { - // For image icons, we'd need to handle blob conversion - // For now, let's skip image icons or convert them to default - // This could be enhanced later to download and convert images to blobs - logger.debug( - 'Skipping image icon (not implemented):', - child.icon.content - ); - iconData = undefined; - } - - if (iconData) { - logger.debug('Calling explorerIconService.setIcon with:', { - where: 'doc', - id: child.pageId, - icon: iconData, - }); - explorerIconService.setIcon({ - where: 'doc', - id: child.pageId, - icon: iconData, - }); - logger.debug('Icon set successfully for document:', child.pageId); - } else { - logger.debug('No valid icon data to set'); - } - } catch (error) { - logger.error( - 'Error setting icon for document:', - child.pageId, - error - ); - logger.warn( - 'Failed to set icon for document:', - child.pageId, - error - ); - } - } else { - if (!child.icon) { - logger.debug('No icon found for document:', child.pageId); - } - if (!explorerIconService) { - logger.debug( - 'ExplorerIconService not available for document:', - child.pageId - ); - } - } - } else if (child.children.size > 0) { - // This is a subfolder, process it recursively - processHierarchyNode(child, currentFolderId); - } - } - - return currentFolderId; - } - - const rootFolderId = processHierarchyNode(hierarchy, parentFolderId); - return { folderId: rootFolderId, docLinks }; -} - -/** - * Creates the folder tree described by {@link folderHierarchy} via - * {@link OrganizeService} and links every document into its folder. - * Returns the root folder ID on success, or `undefined` if the - * hierarchy is empty or an error occurs. - * - * When {@link explorerIconService} is provided, document icons from the - * hierarchy (e.g. Notion page emojis) are applied. Callers that do not - * need icon support can omit it safely. - */ -function applyFolderHierarchy( - organizeService: OrganizeService, - folderHierarchy: FolderHierarchy, - explorerIconService?: ExplorerIconService -): string | undefined { - if (folderHierarchy.children.size === 0) return undefined; - try { - const { folderId, docLinks } = createFolderStructure( - organizeService, - folderHierarchy, - null, - explorerIconService - ); - for (const { folderId, docId } of docLinks) { - const folder = organizeService.folderTree.folderNode$(folderId).value; - if (folder) { - const index = folder.indexAt('after'); - folder.createLink('doc', docId, index); - } - } - return folderId || undefined; - } catch (error) { - logger.warn('Failed to create folder structure:', error); - return undefined; - } +function shouldSnapshotPickedFiles(type: ImportType, acceptType: AcceptType) { + if (acceptType === 'Directory' || acceptType === 'Skip') return false; + return !['markdownZip', 'notion', 'bear', 'oneNote'].includes(type); } type ImportType = | 'markdown' | 'markdownZip' | 'notion' - | 'notionMarkdown' | 'obsidian' | 'bear' + | 'oneNote' | 'snapshot' | 'html' | 'docx' | 'dotaffinefile'; -type AcceptType = 'Markdown' | 'Zip' | 'Html' | 'Docx' | 'Directory' | 'Skip'; // Skip is used for dotaffinefile +type AcceptType = + | 'Markdown' + | 'Zip' + | 'Html' + | 'Docx' + | 'OneNote' + | 'Directory' + | 'Skip'; // Skip is used for dotaffinefile type Status = 'idle' | 'importing' | 'success' | 'error'; +type ImportErrorState = { + code: string; + message: string; + sourcePath?: string; +}; type ImportResult = { docIds: string[]; entryId?: string; isWorkspaceFile?: boolean; rootFolderId?: string; importedWorkspace?: WorkspaceMetadata; + warnings?: ImportWarning[]; }; type ImportedWorkspacePayload = { workspace: WorkspaceMetadata; }; +type ImportFunctionArgs = { + docCollection: Workspace; + files: File[]; + importAffineFile: () => Promise; + importService?: ImportService; + context: ImportRunContext; +}; + +function toImportErrorState(error: unknown): ImportErrorState { + const sourcePath = + typeof error === 'object' && + error !== null && + 'sourcePath' in error && + typeof error.sourcePath === 'string' + ? error.sourcePath + : undefined; + if (error instanceof DOMException && error.name === 'AbortError') { + return { + code: 'cancelled', + message: 'Import cancelled', + sourcePath, + }; + } + if (error instanceof Error) { + return { + code: error.name || 'import-error', + message: error.message || 'Unknown error occurred', + sourcePath, + }; + } + return { + code: 'unknown', + message: 'Unknown error occurred', + sourcePath, + }; +} + +function requireImportService(importService?: ImportService) { + if (!importService) { + throw new Error('Import service is unavailable'); + } + return importService; +} + type ImportConfig = { fileOptions: { acceptType: AcceptType; multiple: boolean }; - importFunction: ( - docCollection: Workspace, - files: File[], - handleImportAffineFile: () => Promise, - organizeService?: OrganizeService, - explorerIconService?: ExplorerIconService, - tagService?: TagService - ) => Promise; + nativeOnly?: boolean; + importFunction: (args: ImportFunctionArgs) => Promise; }; const importOptions = [ @@ -319,17 +203,6 @@ const importOptions = [ testId: 'editor-option-menu-import-notion', type: 'notion' as ImportType, }, - { - key: 'notionMarkdown', - label: 'com.affine.import.notion-markdown', - prefixIcon: , - suffixIcon: ( - - ), - suffixTooltip: 'com.affine.import.notion-markdown.tooltip', - testId: 'editor-option-menu-import-notion-markdown', - type: 'notionMarkdown' as ImportType, - }, { key: 'obsidian', label: 'com.affine.import.obsidian', @@ -356,6 +229,19 @@ const importOptions = [ testId: 'editor-option-menu-import-bear', type: 'bear' as ImportType, }, + { + key: 'oneNote', + label: 'com.affine.import.onenote', + prefixIcon: ( + + ), + suffixIcon: ( + + ), + suffixTooltip: 'com.affine.import.onenote.tooltip', + testId: 'editor-option-menu-import-onenote', + type: 'oneNote' as ImportType, + }, { key: 'docx', label: 'com.affine.import.docx', @@ -400,13 +286,7 @@ const importOptions = [ const importConfigs: Record = { markdown: { fileOptions: { acceptType: 'Markdown', multiple: true }, - importFunction: async ( - docCollection, - files, - _handleImportAffineFile, - _organizeService, - _explorerIconService - ) => { + importFunction: async ({ docCollection, files }) => { const docIds: string[] = []; for (const file of files) { const text = await file.text(); @@ -427,45 +307,20 @@ const importConfigs: Record = { }, markdownZip: { fileOptions: { acceptType: 'Zip', multiple: false }, - importFunction: async ( - docCollection, - files, - _handleImportAffineFile, - organizeService, - _explorerIconService - ) => { + importFunction: async ({ files, importService, context }) => { const file = files.length === 1 ? files[0] : null; if (!file) { throw new Error('Expected a single zip file for markdownZip import'); } - const { docIds, folderHierarchy } = - await MarkdownTransformer.importMarkdownZip({ - collection: docCollection, - schema: getAFFiNEWorkspaceSchema(), - imported: file, - extensions: getStoreManager().config.init().value.get('store'), - }); - - const rootFolderId = - folderHierarchy && organizeService - ? applyFolderHierarchy(organizeService, folderHierarchy) - : undefined; - - return { - docIds, - rootFolderId, - }; + return requireImportService(importService).importMarkdownZip( + file, + context + ); }, }, html: { fileOptions: { acceptType: 'Html', multiple: true }, - importFunction: async ( - docCollection, - files, - _handleImportAffineFile, - _organizeService, - _explorerIconService - ) => { + importFunction: async ({ docCollection, files }) => { const docIds: string[] = []; for (const file of files) { const text = await file.text(); @@ -486,220 +341,50 @@ const importConfigs: Record = { }, notion: { fileOptions: { acceptType: 'Zip', multiple: false }, - importFunction: async ( - docCollection, - files, - _handleImportAffineFile, - organizeService, - explorerIconService - ) => { + importFunction: async ({ files, importService, context }) => { const file = files.length === 1 ? files[0] : null; if (!file) { throw new Error('Expected a single zip file for notion import'); } - const { entryId, pageIds, isWorkspaceFile, folderHierarchy } = - await NotionHtmlTransformer.importNotionZip({ - collection: docCollection, - schema: getAFFiNEWorkspaceSchema(), - imported: file, - extensions: getStoreManager().config.init().value.get('store'), - }); - - const rootFolderId = - folderHierarchy && organizeService - ? applyFolderHierarchy( - organizeService, - folderHierarchy, - explorerIconService - ) - : undefined; - - return { - docIds: pageIds, - entryId, - isWorkspaceFile, - rootFolderId, - }; - }, - }, - notionMarkdown: { - fileOptions: { acceptType: 'Zip', multiple: false }, - importFunction: async ( - docCollection, - files, - _handleImportAffineFile, - organizeService, - explorerIconService - ) => { - const file = files.length === 1 ? files[0] : null; - if (!file) { - throw new Error('Expected a single zip file for notionMarkdown import'); - } - const { docIds, folderHierarchy } = - await MarkdownTransformer.importNotionMarkdownZip({ - collection: docCollection, - schema: getAFFiNEWorkspaceSchema(), - imported: file, - extensions: getStoreManager().config.init().value.get('store'), - }); - - const rootFolderId = - folderHierarchy && organizeService - ? applyFolderHierarchy( - organizeService, - folderHierarchy, - explorerIconService - ) - : undefined; - - return { - docIds, - rootFolderId, - }; + return requireImportService(importService).importNotionZip(file, context); }, }, obsidian: { fileOptions: { acceptType: 'Directory', multiple: false }, - importFunction: async ( - docCollection, - files, - _handleImportAffineFile, - _organizeService, - explorerIconService - ) => { - const { docIds, docEmojis } = - await ObsidianTransformer.importObsidianVault({ - collection: docCollection, - schema: getAFFiNEWorkspaceSchema(), - importedFiles: files, - extensions: getStoreManager().config.init().value.get('store'), - }); - - if (explorerIconService) { - for (const [id, emoji] of docEmojis.entries()) { - explorerIconService.setIcon({ - where: 'doc', - id, - icon: { type: IconType.Emoji, unicode: emoji }, - }); - } - } - - return { docIds }; + importFunction: async ({ files, importService, context }) => { + return requireImportService(importService).importObsidianVault( + files, + context + ); }, }, bear: { fileOptions: { acceptType: 'Zip', multiple: false }, - importFunction: async ( - docCollection, - files, - _handleImportAffineFile, - organizeService, - _explorerIconService, - tagService - ) => { + importFunction: async ({ files, importService, context }) => { const file = files.length === 1 ? files[0] : null; if (!file) { throw new Error('Expected a single .bear2bk file for Bear import'); } - let docIds: string[]; - let tags: Map; - let folderHierarchy: FolderHierarchy; - try { - const result = await BearTransformer.importBearBackup({ - collection: docCollection, - schema: getAFFiNEWorkspaceSchema(), - imported: file, - extensions: getStoreManager().config.init().value.get('store'), - }); - docIds = result.docIds; - tags = result.tags; - folderHierarchy = result.folderHierarchy; - } catch (err) { - logger.error('Bear import failed:', err); - throw err instanceof Error - ? err - : new Error(String(err) || 'Bear import failed'); - } - - // Create AFFiNE tags from Bear tags - if (tagService && tags.size > 0) { - try { - // Get existing tags for deduplication - const existingTags = tagService.tagList.tags$.value; - const existingTagMap = new Map(); // lowercase name → tag id - for (const tag of existingTags) { - const name = tag.value$.value.toLowerCase(); - existingTagMap.set(name, tag.id); - } - - // Consolidate tags by root segment (e.g., "privat/bike" → "privat"). - // Keyed by lowercase root for case-insensitive dedup, but the - // original capitalization of the first occurrence is preserved - // so new AFFiNE tags are created with the user's casing. - const rootTagDocMap = new Map< - string, - { displayName: string; docs: Set } - >(); - for (const [tagName, tagDocIds] of tags) { - const originalRoot = tagName.split('/')[0]; - const key = originalRoot.toLowerCase(); - let entry = rootTagDocMap.get(key); - if (!entry) { - entry = { displayName: originalRoot, docs: new Set() }; - rootTagDocMap.set(key, entry); - } - for (const docId of tagDocIds) { - entry.docs.add(docId); - } - } - - for (const [ - rootTagKey, - { displayName, docs: docIdSet }, - ] of rootTagDocMap) { - // Check if tag already exists (case-insensitive) - let tagId = existingTagMap.get(rootTagKey); - if (!tagId) { - const newTag = tagService.tagList.createTag( - displayName, - tagService.randomTagColor() - ); - tagId = newTag.id; - existingTagMap.set(rootTagKey, tagId); - } - - // Assign tag to each doc - for (const docId of docIdSet) { - const doc = docCollection.getDoc(docId); - const currentTags = doc?.meta?.tags ?? []; - if (!currentTags.includes(tagId)) { - docCollection.meta.setDocMeta(docId, { - tags: [...currentTags, tagId], - }); - } - } - } - } catch (error) { - logger.warn('Failed to create Bear tags:', error); - } + return requireImportService(importService).importBearBackup( + file, + context + ); + }, + }, + oneNote: { + fileOptions: { acceptType: 'OneNote', multiple: false }, + nativeOnly: true, + importFunction: async ({ files, importService, context }) => { + const file = files.length === 1 ? files[0] : null; + if (!file) { + throw new Error('Expected a single OneNote file'); } - - const rootFolderId = - folderHierarchy && organizeService - ? applyFolderHierarchy(organizeService, folderHierarchy) - : undefined; - - return { - docIds, - rootFolderId, - }; + return requireImportService(importService).importOneNote(file, context); }, }, docx: { fileOptions: { acceptType: 'Docx', multiple: false }, - importFunction: async (docCollection, file) => { - const files = Array.isArray(file) ? file : [file]; + importFunction: async ({ docCollection, files }) => { const docIds: string[] = []; for (const file of files) { const docId = await DocxTransformer.importDocx({ @@ -715,13 +400,7 @@ const importConfigs: Record = { }, snapshot: { fileOptions: { acceptType: 'Zip', multiple: false }, - importFunction: async ( - docCollection, - files, - _handleImportAffineFile, - _organizeService, - _explorerIconService - ) => { + importFunction: async ({ docCollection, files }) => { const file = files.length === 1 ? files[0] : null; if (!file) { throw new Error('Expected a single zip file for snapshot import'); @@ -743,14 +422,8 @@ const importConfigs: Record = { }, dotaffinefile: { fileOptions: { acceptType: 'Skip', multiple: false }, - importFunction: async ( - _, - __, - handleImportAffineFile, - _organizeService, - _explorerIconService - ) => { - const workspace = await handleImportAffineFile(); + importFunction: async ({ importAffineFile }) => { + const workspace = await importAffineFile(); return { docIds: [], entryId: undefined, @@ -768,6 +441,7 @@ const ImportOptionItem = ({ suffixTooltip, type, onImport, + disabled, ...props }: { label: string; @@ -776,10 +450,16 @@ const ImportOptionItem = ({ suffixTooltip?: string; type: ImportType; onImport: (type: ImportType) => void; + disabled?: boolean; }) => { const t = useI18n(); return ( -
onImport(type)} {...props}> +
onImport(type)} + aria-disabled={disabled} + {...props} + > {prefixIcon}
{t[label]()}
{suffixIcon && ( @@ -813,18 +493,24 @@ const ImportOptions = ({ suffixTooltip, testId, type, - }) => ( - - ) + }) => { + const disabled = Boolean( + importConfigs[type].nativeOnly && !BUILD_CONFIG.isElectron + ); + return ( + + ); + } )}
@@ -843,8 +529,18 @@ const ImportOptions = ({ ); }; -const ImportingStatus = () => { +const ImportingStatus = ({ + progress, + onCancel, +}: { + progress: { completed: number; total: number } | null; + onCancel: () => void; +}) => { const t = useI18n(); + const progressLabel = + progress && progress.total > 0 + ? `${progress.completed}/${progress.total}` + : null; return ( <>
@@ -853,11 +549,25 @@ const ImportingStatus = () => {

{t['com.affine.import.status.importing.message']()}

+ {progressLabel ? ( +
{progressLabel}
+ ) : null} +
+ +
); }; -const SuccessStatus = ({ onComplete }: { onComplete: () => void }) => { +const SuccessStatus = ({ + warnings, + onComplete, +}: { + warnings: string[]; + onComplete: () => void; +}) => { const t = useI18n(); return ( <> @@ -876,6 +586,13 @@ const SuccessStatus = ({ onComplete }: { onComplete: () => void }) => { .

+ {warnings.length ? ( +
+ {warnings.map((warning, index) => ( +
{warning}
+ ))} +
+ ) : null}

- {error || 'Unknown error occurred'} + {error?.message || 'Unknown error occurred'}

+ {error?.sourcePath ? ( +
{error.sourcePath}
+ ) : null}
- )} -
- ); -} diff --git a/packages/frontend/media-capture-playground/web/components/app-list.tsx b/packages/frontend/media-capture-playground/web/components/app-list.tsx deleted file mode 100644 index 4b222fffc8049..0000000000000 --- a/packages/frontend/media-capture-playground/web/components/app-list.tsx +++ /dev/null @@ -1,145 +0,0 @@ -import React from 'react'; -import useSWRSubscription from 'swr/subscription'; - -import type { App, AppGroup, RecordingStatus } from '../types'; -import { socket } from '../utils'; -import { AppItem } from './app-item'; - -export function AppList() { - const { data: apps = [] } = useSWRSubscription('apps', (_key, { next }) => { - let apps: App[] = []; - // Initial apps fetch - fetch('/api/apps') - .then(res => res.json()) - .then(data => { - apps = data.apps; - next(null, apps); - }) - .catch(err => next(err)); - - // Subscribe to app updates - socket.on('apps:all', data => { - next(null, data.apps); - apps = data.apps; - }); - socket.on('apps:state-changed', data => { - const index = apps.findIndex(a => a.processId === data.processId); - console.log('apps:state-changed', data, index); - if (index !== -1) { - next( - null, - apps.toSpliced(index, 1, { - ...apps[index], - isRunning: data.isRunning, - }) - ); - } - }); - socket.on('connect', () => { - // Refetch on reconnect - fetch('/api/apps') - .then(res => res.json()) - .then(data => next(null, data.apps)) - .catch(err => next(err)); - }); - - return () => { - socket.off('apps:all'); - socket.off('apps:state-changed'); - socket.off('connect'); - }; - }); - - const { data: recordings = [] } = useSWRSubscription( - 'recordings', - ( - _key: string, - { next }: { next: (err: Error | null, data?: RecordingStatus[]) => void } - ) => { - // Subscribe to recording updates - socket.on('apps:recording', (data: { recordings: RecordingStatus[] }) => { - next(null, data.recordings); - }); - - return () => { - socket.off('apps:recording'); - }; - } - ); - - const appGroups: AppGroup[] = React.useMemo(() => { - const mapping = apps.reduce((acc: Record, app: App) => { - if (!acc[app.processGroupId]) { - acc[app.processGroupId] = { - processGroupId: app.processGroupId, - apps: [], - rootApp: - apps.find((a: App) => a.processId === app.processGroupId) || app, - }; - } - acc[app.processGroupId].apps.push(app); - return acc; - }, {}); - return Object.values(mapping); - }, [apps]); - - const runningApps = (appGroups || []).filter(app => - app.apps.some(a => a.isRunning) - ); - const notRunningApps = (appGroups || []).filter( - app => !app.apps.some(a => a.isRunning) - ); - - return ( -
-
-
-

- Active Applications -

- - {runningApps.length} listening - -
-
- {runningApps.map(app => ( - - ))} - {runningApps.length === 0 && ( -
- No applications are currently listening -
- )} -
-
-
-
-

- Other Applications -

- - {notRunningApps.length} available - -
-
- {notRunningApps.map(app => ( - - ))} - {notRunningApps.length === 0 && ( -
- No other applications found -
- )} -
-
-
- ); -} diff --git a/packages/frontend/media-capture-playground/web/components/global-record-button.tsx b/packages/frontend/media-capture-playground/web/components/global-record-button.tsx deleted file mode 100644 index 6884b8b010ba9..0000000000000 --- a/packages/frontend/media-capture-playground/web/components/global-record-button.tsx +++ /dev/null @@ -1,70 +0,0 @@ -import { useCallback, useEffect, useState } from 'react'; - -import { socket } from '../utils'; - -export function GlobalRecordButton() { - const [isRecording, setIsRecording] = useState(false); - const [isLoading, setIsLoading] = useState(false); - - useEffect(() => { - function handleRecordingStatus(data: { - recordings: Array<{ processId: number }>; - }) { - // Global recording uses processId -1 - setIsRecording(data.recordings.some(r => r.processId === -1)); - } - - socket.on('apps:recording', handleRecordingStatus); - return () => { - socket.off('apps:recording', handleRecordingStatus); - }; - }, []); - - const handleClick = useCallback(() => { - setIsLoading(true); - const endpoint = isRecording ? '/api/global/stop' : '/api/global/record'; - fetch(endpoint, { method: 'POST' }) - .then(response => { - if (!response.ok) { - throw new Error('Failed to toggle global recording'); - } - }) - .catch(error => { - console.error('Error toggling global recording:', error); - }) - .finally(() => { - setIsLoading(false); - }); - }, [isRecording]); - - return ( - - ); -} diff --git a/packages/frontend/media-capture-playground/web/components/icons.tsx b/packages/frontend/media-capture-playground/web/components/icons.tsx deleted file mode 100644 index f7d359b581ef5..0000000000000 --- a/packages/frontend/media-capture-playground/web/components/icons.tsx +++ /dev/null @@ -1,163 +0,0 @@ -import type { ReactElement } from 'react'; - -export function PlayIcon(): ReactElement { - return ( - - - - ); -} - -export function PauseIcon(): ReactElement { - return ( - - - - ); -} - -export function RewindIcon(): ReactElement { - return ( - - - - ); -} - -export function ForwardIcon(): ReactElement { - return ( - - - - ); -} - -export function DeleteIcon(): ReactElement { - return ( - - - - ); -} - -export function LoadingSpinner(): ReactElement { - return ( - - - - - ); -} - -export function ErrorIcon(): ReactElement { - return ( - - - - ); -} - -export function MicrophoneIcon(): ReactElement { - return ( - - - - ); -} - -export function WarningIcon(): ReactElement { - return ( - - - - ); -} - -export function DefaultAppIcon(): ReactElement { - return ( - - - - - ); -} diff --git a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx deleted file mode 100644 index 78931fbb6992a..0000000000000 --- a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx +++ /dev/null @@ -1,895 +0,0 @@ -import type { ReactElement } from 'react'; -import React from 'react'; -import ReactMarkdown from 'react-markdown'; - -import type { SavedRecording, TranscriptionMetadata } from '../types'; -import { formatDuration, socket } from '../utils'; -import { - DefaultAppIcon, - DeleteIcon, - ErrorIcon, - ForwardIcon, - LoadingSpinner, - MicrophoneIcon, - PauseIcon, - PlayIcon, - RewindIcon, - WarningIcon, -} from './icons'; - -interface SavedRecordingItemProps { - recording: SavedRecording; -} - -// Audio player controls component -function AudioControls({ - audioRef, - playbackRate, - onPlaybackRateChange, - onSeek, - onPlayPause, -}: { - audioRef: React.RefObject; - playbackRate: number; - onPlaybackRateChange: () => void; - onSeek: (seconds: number) => void; - onPlayPause: () => void; -}): ReactElement { - const [currentTime, setCurrentTime] = React.useState('00:00'); - const [duration, setDuration] = React.useState('00:00'); - - React.useEffect(() => { - const audio = audioRef.current; - if (!audio) return; - - const formatTime = (time: number) => { - const minutes = Math.floor(time / 60); - const seconds = Math.floor(time % 60); - return `${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`; - }; - - const updateTime = () => { - setCurrentTime(formatTime(audio.currentTime)); - setDuration(formatTime(audio.duration)); - }; - - audio.addEventListener('timeupdate', updateTime); - audio.addEventListener('loadedmetadata', updateTime); - - return () => { - audio.removeEventListener('timeupdate', updateTime); - audio.removeEventListener('loadedmetadata', updateTime); - }; - }, [audioRef]); - - return ( -
-
- - - -
- {currentTime} / {duration} -
-
- -
- ); -} - -// Waveform visualization component -function WaveformVisualizer({ - containerRef, - waveformData, - currentTime, - fileName, -}: { - containerRef: React.RefObject; - waveformData: number[]; - currentTime: number; - fileName: string; -}): ReactElement { - return ( -
-
- {waveformData.map((amplitude, i) => ( -
- ))} -
-
- ); -} - -// Update TranscriptionMessage component -function TranscriptionMessage({ - item, - isNewSpeaker, - isCurrentMessage, -}: { - item: { - speaker: string; - start_time: string; - transcription: string; - }; - isNewSpeaker: boolean; - isCurrentMessage: boolean; -}): ReactElement { - return ( -
-
-
- {isNewSpeaker && ( -
- {item.speaker} -
- )} -
- {item.start_time} -
-
-
-
-
- {item.transcription} -
-
-
- ); -} - -// Add new Summary component -function TranscriptionSummary({ summary }: { summary: string }): ReactElement { - return ( -
-
- Summary -
-
- {summary} -
-
- ); -} - -// Update TranscriptionContent component -function TranscriptionContent({ - transcriptionData, - currentAudioTime, -}: { - transcriptionData: { - segments: Array<{ - speaker: string; - start_time: string; - transcription: string; - }>; - summary: string; - title: string; - }; - currentAudioTime: number; -}): ReactElement { - const parseTimestamp = (timestamp: string) => { - // Handle "MM:SS" format (without hours) - const [minutes, seconds] = timestamp.split(':'); - return parseInt(minutes, 10) * 60 + parseInt(seconds, 10); - }; - - return ( -
- - {transcriptionData.segments.map((item, index) => { - const isNewSpeaker = - index === 0 || - transcriptionData.segments[index - 1].speaker !== item.speaker; - - const startTime = parseTimestamp(item.start_time); - // Use next segment's start time as end time, or add 3 seconds for the last segment - const endTime = - index < transcriptionData.segments.length - 1 - ? parseTimestamp(transcriptionData.segments[index + 1].start_time) - : startTime + 3; - - const isCurrentMessage = - currentAudioTime >= startTime && currentAudioTime < endTime; - - return ( - - ); - })} -
- ); -} - -// Update TranscriptionStatus component -function TranscriptionStatus({ - transcription, - transcriptionError, - currentAudioTime, -}: { - transcription?: TranscriptionMetadata; - transcriptionError: string | null; - currentAudioTime: number; -}): ReactElement | null { - if (!transcription && !transcriptionError) { - return null; - } - - if (transcription?.transcriptionStatus === 'pending') { - return ( -
-
-
- - Processing Audio -
-
-
- -
- Starting transcription - ... -
-
- This may take a few moments depending on the length of the - recording -
-
-
-
-
- ); - } - - if (transcriptionError) { - return ( -
- - {transcriptionError} -
- ); - } - - if ( - transcription?.transcriptionStatus === 'completed' && - transcription.transcription - ) { - try { - const transcriptionData = transcription.transcription; - if ( - !transcriptionData.segments || - !Array.isArray(transcriptionData.segments) - ) { - throw new Error('Invalid transcription data format'); - } - - return ( -
-
-
- - Conversation Transcript -
- {transcriptionData.title && ( -
-
- Title -
-
- {transcriptionData.title} -
-
- )} - -
-
- ); - } catch (error) { - return ( -
- {error instanceof Error - ? error.message - : 'Failed to parse transcription data'} -
- ); - } - } - - return null; -} - -// Add new RecordingHeader component -function RecordingHeader({ - metadata, - fileName, - recordingDate, - duration, - error, - isDeleting, - showDeleteConfirm, - setShowDeleteConfirm, - handleDeleteClick, -}: { - metadata: SavedRecording['metadata']; - fileName: string; - recordingDate: string; - duration: string; - error: string | null; - isDeleting: boolean; - showDeleteConfirm: boolean; - setShowDeleteConfirm: (show: boolean) => void; - handleDeleteClick: () => void; - transcriptionError: string | null; -}): ReactElement { - const [imgError, setImgError] = React.useState(false); - const isGlobalRecording = metadata?.isGlobal; - - return ( -
-
- {!imgError && !isGlobalRecording ? ( - {metadata?.appName setImgError(true)} - /> - ) : ( -
- -
- )} -
-
-
-
- - {metadata?.appName || 'Unknown Application'} - - {isGlobalRecording && ( - - System Audio - - )} - - {duration} - -
-
- {showDeleteConfirm ? ( -
- - -
- ) : ( - - )} -
-
-
{recordingDate}
-
- {metadata?.bundleIdentifier || fileName} -
- {error && ( -
- - {error} -
- )} -
-
- ); -} - -// Add new AudioPlayer component -function AudioPlayer({ - isLoading, - error, - audioRef, - playbackRate, - handlePlaybackRateChange, - handleSeek, - handlePlayPause, - containerRef, - waveformData, - currentTime, - fileName, -}: { - isLoading: boolean; - error: string | null; - audioRef: React.RefObject; - playbackRate: number; - handlePlaybackRateChange: () => void; - handleSeek: (seconds: number) => void; - handlePlayPause: () => void; - containerRef: React.RefObject; - waveformData: number[]; - currentTime: number; - fileName: string; -}): ReactElement { - return ( -
- {isLoading && !error ? ( -
- - - Loading audio... - -
- ) : ( -
- - -
- )} -
- ); -} - -// Add new TranscribeButton component -function TranscribeButton({ - transcriptionStatus, - onTranscribe, -}: { - transcriptionStatus?: TranscriptionMetadata['transcriptionStatus']; - onTranscribe: () => void; -}): ReactElement { - return ( -
-
- -
-
- ); -} - -// Main SavedRecordingItem component (simplified) -export function SavedRecordingItem({ - recording, -}: SavedRecordingItemProps): ReactElement { - const [error, setError] = React.useState(null); - const [isLoading, setIsLoading] = React.useState(true); - const [isDeleting, setIsDeleting] = React.useState(false); - const [showDeleteConfirm, setShowDeleteConfirm] = React.useState(false); - const [playbackRate, setPlaybackRate] = React.useState(1); - const [waveformData, setWaveformData] = React.useState([]); - const [currentTime, setCurrentTime] = React.useState(0); - const audioRef = React.useRef(null); - const containerRef = React.useRef(null); - const [segments, setSegments] = React.useState(40); - const [currentAudioTime, setCurrentAudioTime] = React.useState(0); - const [transcriptionError, setTranscriptionError] = React.useState< - string | null - >(null); - - const metadata = recording.metadata; - // Ensure we have a valid filename, fallback to an empty string if undefined - const fileName = recording.wav || ''; - const recordingDate = metadata - ? new Date(metadata.recordingStartTime).toLocaleString() - : 'Unknown date'; - const duration = metadata - ? formatDuration(metadata.recordingDuration * 1000) - : 'Unknown duration'; - - // Update current audio time - React.useEffect(() => { - const audio = audioRef.current; - if (audio) { - const handleTimeUpdate = () => { - setCurrentAudioTime(audio.currentTime); - }; - audio.addEventListener('timeupdate', handleTimeUpdate); - return () => audio.removeEventListener('timeupdate', handleTimeUpdate); - } - return () => {}; - }, []); - - // Calculate number of segments based on container width - React.useEffect(() => { - const updateSegments = () => { - if (containerRef.current) { - // Each bar should be at least 2px wide (1px bar + 1px gap) - const width = containerRef.current.offsetWidth; - setSegments(Math.floor(width / 2)); - } - }; - - updateSegments(); - const resizeObserver = new ResizeObserver(updateSegments); - if (containerRef.current) { - resizeObserver.observe(containerRef.current); - } - - return () => resizeObserver.disconnect(); - }, []); - - const processAudioData = React.useCallback(async () => { - try { - // Check if fileName is empty - if (!fileName) { - throw new Error('Invalid recording filename'); - } - - const response = await fetch(`/api/recordings/${fileName}/recording.wav`); - if (!response.ok) { - throw new Error( - `Failed to fetch audio file (${response.status}): ${response.statusText}` - ); - } - - const audioContext = new AudioContext(); - const arrayBuffer = await response.arrayBuffer(); - - // Ensure we have data to process - if (!arrayBuffer || arrayBuffer.byteLength === 0) { - throw new Error('No audio data received'); - } - - const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); - const channelData = audioBuffer.getChannelData(0); - - // Process the audio data in chunks to create the waveform - const numberOfSamples = channelData.length; - const samplesPerSegment = Math.floor(numberOfSamples / segments); - - const waveform: number[] = []; - for (let i = 0; i < segments; i++) { - const start = i * samplesPerSegment; - const end = start + samplesPerSegment; - const segmentData = channelData.slice(start, end); - - // Calculate RMS (root mean square) for better amplitude representation - const rms = Math.sqrt( - segmentData.reduce((sum, sample) => sum + sample * sample, 0) / - segmentData.length - ); - - waveform.push(rms); - } - - // Normalize the waveform data to a 0-1 range - const maxAmplitude = Math.max(...waveform); - const normalizedWaveform = waveform.map(amp => amp / maxAmplitude); - - setWaveformData(normalizedWaveform); - setIsLoading(false); - } catch (err) { - console.error('Error processing audio:', err); - setError( - err instanceof Error ? err.message : 'Failed to process audio data' - ); - setIsLoading(false); - } - }, [fileName, segments]); - - React.useEffect(() => { - const audio = audioRef.current; - if (audio) { - const handleError = (e: ErrorEvent) => { - console.error('Audio error:', e); - setError('Failed to load audio'); - setIsLoading(false); - }; - - const handleLoadedMetadata = () => { - void processAudioData().catch(err => { - console.error('Error processing audio data:', err); - setError('Failed to process audio data'); - setIsLoading(false); - }); - }; - - const handleTimeUpdate = () => { - setCurrentTime(audio.currentTime / audio.duration); - }; - - audio.addEventListener('error', handleError as EventListener); - audio.addEventListener('loadedmetadata', handleLoadedMetadata); - audio.addEventListener('timeupdate', handleTimeUpdate); - - return () => { - audio.removeEventListener('error', handleError as EventListener); - audio.removeEventListener('loadedmetadata', handleLoadedMetadata); - audio.removeEventListener('timeupdate', handleTimeUpdate); - }; - } - return () => {}; - }, [processAudioData]); - - const handlePlayPause = React.useCallback(() => { - if (audioRef.current) { - if (audioRef.current.paused) { - // oxlint-disable-next-line @typescript-eslint/no-floating-promises - void audioRef.current.play(); - } else { - audioRef.current.pause(); - } - } - }, []); - - const handleSeek = React.useCallback((seconds: number) => { - if (audioRef.current) { - audioRef.current.currentTime += seconds; - } - }, []); - - const handlePlaybackRateChange = React.useCallback(() => { - if (audioRef.current) { - const newRate = playbackRate === 1 ? 1.5 : 1; - audioRef.current.playbackRate = newRate; - setPlaybackRate(newRate); - } - }, [playbackRate]); - - const handleDelete = React.useCallback(async () => { - setIsDeleting(true); - setError(null); // Clear any previous errors - - try { - // Check if filename is valid - if (!recording.wav) { - throw new Error('Invalid recording filename'); - } - - const response = await fetch(`/api/recordings/${recording.wav}`, { - method: 'DELETE', - }); - - if (!response.ok) { - let errorMessage: string; - try { - const errorData = await response.json(); - errorMessage = errorData.error; - } catch { - errorMessage = `Server error (${response.status}): ${response.statusText}`; - } - throw new Error(errorMessage); - } - - setShowDeleteConfirm(false); - } catch (err) { - console.error('Error deleting recording:', err); - setError( - err instanceof Error ? err.message : 'An unexpected error occurred' - ); - } finally { - setIsDeleting(false); - } - }, [recording.wav]); - - const handleDeleteClick = React.useCallback(() => { - void handleDelete().catch(err => { - console.error('Unexpected error during deletion:', err); - setError('An unexpected error occurred'); - }); - }, [handleDelete]); - - React.useEffect(() => { - // Listen for transcription events - socket.on( - 'apps:recording-transcription-start', - (data: { filename: string }) => { - if (recording.wav && data.filename === recording.wav) { - setTranscriptionError(null); - } - } - ); - - socket.on( - 'apps:recording-transcription-end', - (data: { - filename: string; - success: boolean; - transcription?: string; - error?: string; - }) => { - if (recording.wav && data.filename === recording.wav && !data.success) { - setTranscriptionError(data.error || 'Transcription failed'); - } - } - ); - - return () => { - socket.off('apps:recording-transcription-start'); - socket.off('apps:recording-transcription-end'); - }; - }, [recording.wav]); - - const handleTranscribe = React.useCallback(async () => { - try { - // Check if filename is valid - if (!recording.wav) { - throw new Error('Invalid recording filename'); - } - - const response = await fetch( - `/api/recordings/${recording.wav}/transcribe`, - { - method: 'POST', - } - ); - - if (!response.ok) { - const error = await response.json(); - throw new Error(error.error || 'Failed to start transcription'); - } - } catch (err) { - setTranscriptionError( - err instanceof Error ? err.message : 'Failed to start transcription' - ); - } - }, [recording.wav]); - - return ( -
- - } - playbackRate={playbackRate} - handlePlaybackRateChange={handlePlaybackRateChange} - handleSeek={handleSeek} - handlePlayPause={handlePlayPause} - containerRef={containerRef as React.RefObject} - waveformData={waveformData} - currentTime={currentTime} - fileName={fileName} - /> -
- ); -} diff --git a/packages/frontend/media-capture-playground/web/components/saved-recordings.tsx b/packages/frontend/media-capture-playground/web/components/saved-recordings.tsx deleted file mode 100644 index 2d6d25f184b6a..0000000000000 --- a/packages/frontend/media-capture-playground/web/components/saved-recordings.tsx +++ /dev/null @@ -1,41 +0,0 @@ -import useSWRSubscription from 'swr/subscription'; - -import type { SavedRecording } from '../types'; -import { socket } from '../utils'; -import { SavedRecordingItem } from './saved-recording-item'; - -export function SavedRecordings(): React.ReactElement { - const { data: recordings = [] } = useSWRSubscription( - 'saved-recordings', - ( - _key: string, - { next }: { next: (err: Error | null, data?: SavedRecording[]) => void } - ) => { - // Subscribe to saved recordings updates - socket.on('apps:saved', (data: { recordings: SavedRecording[] }) => { - next(null, data.recordings); - }); - - fetch('/api/apps/saved') - .then(res => res.json()) - .then(data => next(null, data.recordings)) - .catch(err => next(err)); - - return () => { - socket.off('apps:saved'); - }; - } - ); - - if (recordings.length === 0) { - return

No saved recordings

; - } - - return ( -
- {recordings.map(recording => ( - - ))} -
- ); -} diff --git a/packages/frontend/media-capture-playground/web/index.html b/packages/frontend/media-capture-playground/web/index.html deleted file mode 100644 index 14eaef1129424..0000000000000 --- a/packages/frontend/media-capture-playground/web/index.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - Media Capture Playground - - -
- - diff --git a/packages/frontend/media-capture-playground/web/main.css b/packages/frontend/media-capture-playground/web/main.css deleted file mode 100644 index d4b5078586e29..0000000000000 --- a/packages/frontend/media-capture-playground/web/main.css +++ /dev/null @@ -1 +0,0 @@ -@import 'tailwindcss'; diff --git a/packages/frontend/media-capture-playground/web/main.tsx b/packages/frontend/media-capture-playground/web/main.tsx deleted file mode 100644 index df8a85c6a455a..0000000000000 --- a/packages/frontend/media-capture-playground/web/main.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import './main.css'; - -import { createRoot } from 'react-dom/client'; - -import { App } from './app'; - -const rootElement = document.getElementById('root'); -if (!rootElement) { - throw new Error('Failed to find the root element'); -} -createRoot(rootElement).render(); diff --git a/packages/frontend/media-capture-playground/web/types.ts b/packages/frontend/media-capture-playground/web/types.ts deleted file mode 100644 index 4d0135be73c28..0000000000000 --- a/packages/frontend/media-capture-playground/web/types.ts +++ /dev/null @@ -1,60 +0,0 @@ -export interface App { - processId: number; - processGroupId: number; - bundleIdentifier: string; - name: string; - isRunning: boolean; -} - -export interface AppGroup { - processGroupId: number; - rootApp: App; - apps: App[]; -} - -export interface RecordingStatus { - processId: number; - bundleIdentifier: string; - name: string; - startTime: number; - duration: number; - isGlobal?: boolean; -} - -export interface RecordingMetadata { - appName: string; - bundleIdentifier: string; - processId: number; - recordingStartTime: number; - recordingEndTime: number; - recordingDuration: number; - sampleRate: number; - channels: number; - totalSamples: number; - icon?: Uint8Array; - mp3: string; - isGlobal?: boolean; -} - -export interface TranscriptionMetadata { - transcriptionStartTime: number; - transcriptionEndTime: number; - transcriptionStatus: 'not_started' | 'pending' | 'completed' | 'error'; - transcription?: { - title: string; - segments: Array<{ - speaker: string; - start_time: string; - end_time: string; - transcription: string; - }>; - summary: string; - }; - error?: string; -} - -export interface SavedRecording { - wav: string; - metadata?: RecordingMetadata; - transcription?: TranscriptionMetadata; -} diff --git a/packages/frontend/media-capture-playground/web/utils.ts b/packages/frontend/media-capture-playground/web/utils.ts deleted file mode 100644 index 61a924248f1fe..0000000000000 --- a/packages/frontend/media-capture-playground/web/utils.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { io } from 'socket.io-client'; - -// Create a singleton socket instance -export const socket = io('http://localhost:6544'); - -export function formatDuration(ms: number): string { - const seconds = Math.floor(ms / 1000); - const minutes = Math.floor(seconds / 60); - const hours = Math.floor(minutes / 60); - return `${hours.toString().padStart(2, '0')}:${(minutes % 60) - .toString() - .padStart(2, '0')}:${(seconds % 60).toString().padStart(2, '0')}`; -} - -// Helper function to convert timestamp (MM:SS.mmm) to seconds -export function timestampToSeconds(timestamp: string): number { - const [minutes, seconds] = timestamp.split(':').map(parseFloat); - return minutes * 60 + seconds; -} diff --git a/packages/frontend/native/Cargo.toml b/packages/frontend/native/Cargo.toml index 64976739029c4..8110bdab15f66 100644 --- a/packages/frontend/native/Cargo.toml +++ b/packages/frontend/native/Cargo.toml @@ -9,12 +9,19 @@ crate-type = ["cdylib", "rlib"] [dependencies] affine_common = { workspace = true, features = ["hashcash"] } +affine_doc_loader = { workspace = true } +affine_importer = { workspace = true } affine_media_capture = { path = "./media_capture" } affine_nbstore = { workspace = true, features = ["napi"] } affine_sqlite_v1 = { path = "./sqlite_v1" } +hex = { workspace = true } +infer = { workspace = true } napi = { workspace = true } napi-derive = { workspace = true } once_cell = { workspace = true } +onenote_parser = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } sqlx = { workspace = true, default-features = false, features = [ "chrono", "macros", @@ -24,6 +31,7 @@ sqlx = { workspace = true, default-features = false, features = [ ] } thiserror = { workspace = true } tokio = { workspace = true, features = ["full"] } +typed-path = { workspace = true } [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies] mermaid-rs-renderer = { workspace = true } @@ -38,9 +46,9 @@ mimalloc = { workspace = true } mimalloc = { workspace = true, features = ["local_dynamic_tls"] } [dev-dependencies] -chrono = { workspace = true } -serde_json = { workspace = true } -uuid = { workspace = true } +chrono = { workspace = true } +uuid = { workspace = true } +zip = { workspace = true } [build-dependencies] napi-build = { workspace = true } diff --git a/packages/frontend/native/__tests__/db.spec.mts b/packages/frontend/native/__tests__/db.spec.mts index b762f258553b3..43e6e79bbd3ad 100644 --- a/packages/frontend/native/__tests__/db.spec.mts +++ b/packages/frontend/native/__tests__/db.spec.mts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url'; import test from 'ava'; -import { SqliteConnection, ValidationResult } from '../index'; +import { SqliteConnection, ValidationResult } from '../index.js'; test('db validate', async t => { const path = fileURLToPath( diff --git a/packages/frontend/native/__tests__/pool.spec.mts b/packages/frontend/native/__tests__/pool.spec.mts index ce8eb19f04731..71a4bf99699bc 100644 --- a/packages/frontend/native/__tests__/pool.spec.mts +++ b/packages/frontend/native/__tests__/pool.spec.mts @@ -1,6 +1,6 @@ import test from 'ava'; -import { DocStoragePool } from '../index'; +import { DocStoragePool } from '../index.js'; test('can batch read/write pool', async t => { const pool = new DocStoragePool(); diff --git a/packages/frontend/native/index.d.ts b/packages/frontend/native/index.d.ts index 8e14011a35ea3..d13f8a7bcbe15 100644 --- a/packages/frontend/native/index.d.ts +++ b/packages/frontend/native/index.d.ts @@ -18,13 +18,6 @@ export declare class ApplicationStateChangedSubscriber { unsubscribe(): void } -export declare class AudioCaptureSession { - stop(): void - get sampleRate(): number - get channels(): number - get actualSampleRate(): number -} - export declare class ShareableContent { static onApplicationListChanged(callback: ((err: Error | null, ) => void)): ApplicationListChangedSubscriber static onAppStateChanged(app: ApplicationInfo, callback: ((err: Error | null, ) => void)): ApplicationStateChangedSubscriber @@ -32,8 +25,6 @@ export declare class ShareableContent { static applications(): Array static applicationWithProcessId(processId: number): ApplicationInfo | null static isUsingMicrophone(processId: number): boolean - static tapAudio(processId: number, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession - static tapGlobalAudio(excludedProcesses: Array | undefined | null, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession } export declare function abortRecording(id: string): Promise @@ -75,6 +66,29 @@ export interface RecordingStartOptions { export declare function startRecording(opts: RecordingStartOptions): Promise export declare function stopRecording(id: string): Promise +export declare function cancelImportSession(sessionId: string): void + +export interface CreateImportBatchLimits { + maxDocs?: number + maxBlobs?: number + maxBlobBytes?: number +} + +export declare function createImportSession(options: CreateImportSessionOptions): string + +export interface CreateImportSessionOptions { + format: string + source: CreateImportSessionSource + batchLimits?: CreateImportBatchLimits +} + +export interface CreateImportSessionSource { + kind: string + path: string +} + +export declare function disposeImportSession(sessionId: string): void + export interface MermaidRenderOptions { theme?: string fontFamily?: string @@ -92,6 +106,8 @@ export interface MermaidRenderResult { export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise +export declare function nextImportBatch(sessionId: string): string | null + export declare function renderMermaidSvg(request: MermaidRenderRequest): MermaidRenderResult export declare function renderTypstSvg(request: TypstRenderRequest): TypstRenderResult diff --git a/packages/frontend/native/index.js b/packages/frontend/native/index.js index 4e20bb5fd1abe..af1036fe55103 100644 --- a/packages/frontend/native/index.js +++ b/packages/frontend/native/index.js @@ -575,14 +575,17 @@ module.exports = nativeBinding module.exports.ApplicationInfo = nativeBinding.ApplicationInfo module.exports.ApplicationListChangedSubscriber = nativeBinding.ApplicationListChangedSubscriber module.exports.ApplicationStateChangedSubscriber = nativeBinding.ApplicationStateChangedSubscriber -module.exports.AudioCaptureSession = nativeBinding.AudioCaptureSession module.exports.ShareableContent = nativeBinding.ShareableContent module.exports.abortRecording = nativeBinding.abortRecording module.exports.decodeAudio = nativeBinding.decodeAudio module.exports.decodeAudioSync = nativeBinding.decodeAudioSync module.exports.startRecording = nativeBinding.startRecording module.exports.stopRecording = nativeBinding.stopRecording +module.exports.cancelImportSession = nativeBinding.cancelImportSession +module.exports.createImportSession = nativeBinding.createImportSession +module.exports.disposeImportSession = nativeBinding.disposeImportSession module.exports.mintChallengeResponse = nativeBinding.mintChallengeResponse +module.exports.nextImportBatch = nativeBinding.nextImportBatch module.exports.renderMermaidSvg = nativeBinding.renderMermaidSvg module.exports.renderTypstSvg = nativeBinding.renderTypstSvg module.exports.verifyChallengeResponse = nativeBinding.verifyChallengeResponse diff --git a/packages/frontend/native/media-capture-exapmle.ts b/packages/frontend/native/media-capture-exapmle.ts deleted file mode 100644 index 712d62bd3a6ad..0000000000000 --- a/packages/frontend/native/media-capture-exapmle.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -import { - Whisper, - WhisperFullParams, - WhisperSamplingStrategy, -} from '@napi-rs/whisper'; -import { BehaviorSubject, EMPTY, Observable } from 'rxjs'; -import { - distinctUntilChanged, - exhaustMap, - groupBy, - mergeMap, - switchMap, - tap, -} from 'rxjs/operators'; - -import { type Application, ShareableContent } from './index.js'; - -const rootDir = join(fileURLToPath(import.meta.url), '..'); - -const shareableContent = new ShareableContent(); - -const appList = new Set([ - 'com.tinyspeck.slackmacgap.helper', - 'us.zoom.xos', - 'org.mozilla.firefoxdeveloperedition', -]); - -console.info(shareableContent.applications().map(app => app.bundleIdentifier)); - -const GGLM_LARGE = join(rootDir, 'ggml-large-v3-turbo.bin'); - -const whisper = new Whisper(GGLM_LARGE, { - useGpu: true, - gpuDevice: 1, -}); - -const whisperParams = new WhisperFullParams(WhisperSamplingStrategy.Greedy); - -const SAMPLE_WINDOW_MS = 3000; // 3 seconds, similar to stream.cpp's step_ms -const SAMPLES_PER_WINDOW = (SAMPLE_WINDOW_MS / 1000) * 16000; // 16kHz sample rate - -// eslint-disable-next-line rxjs/finnish -const runningApplications = new BehaviorSubject( - shareableContent.applications() -); - -const applicationListChangedSubscriber = - ShareableContent.onApplicationListChanged(() => { - runningApplications.next(shareableContent.applications()); - }); - -runningApplications - .pipe( - mergeMap(apps => apps.filter(app => appList.has(app.bundleIdentifier))), - groupBy(app => app.bundleIdentifier), - mergeMap(app$ => - app$.pipe( - exhaustMap(app => - new Observable<[Application, boolean]>(subscriber => { - const stateSubscriber = ShareableContent.onAppStateChanged( - app, - err => { - if (err) { - subscriber.error(err); - return; - } - subscriber.next([app, app.isRunning]); - } - ); - return () => { - stateSubscriber.unsubscribe(); - }; - }).pipe( - distinctUntilChanged( - ([_, isRunningA], [__, isRunningB]) => isRunningA === isRunningB - ), - switchMap(([app]) => - !app.isRunning - ? EMPTY - : new Observable(observer => { - const buffers: Float32Array[] = []; - const audioStream = app.tapAudio((err, samples) => { - if (err) { - observer.error(err); - return; - } - - if (samples) { - buffers.push(samples); - observer.next(samples); - - // Calculate total samples in buffer - const totalSamples = buffers.reduce( - (acc, buf) => acc + buf.length, - 0 - ); - - // Process when we have enough samples for our window - if (totalSamples >= SAMPLES_PER_WINDOW) { - // Concatenate all buffers - const concatenated = new Float32Array(totalSamples); - let offset = 0; - buffers.forEach(buf => { - concatenated.set(buf, offset); - offset += buf.length; - }); - - // Transcribe the audio - const result = whisper.full( - whisperParams, - concatenated - ); - - // Print results - console.info(result); - - // Keep any remaining samples for next window - const remainingSamples = - totalSamples - SAMPLES_PER_WINDOW; - if (remainingSamples > 0) { - const lastBuffer = buffers[buffers.length - 1]; - buffers.length = 0; - buffers.push(lastBuffer.slice(-remainingSamples)); - } else { - buffers.length = 0; - } - } - } - }); - - return () => { - audioStream.stop(); - }; - }) - ) - ) - ) - ) - ), - tap({ - finalize: () => { - applicationListChangedSubscriber.unsubscribe(); - }, - }) - ) - .subscribe(); diff --git a/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs index c83d18bae9e15..58f0e297d8107 100644 --- a/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs +++ b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs @@ -682,14 +682,6 @@ impl ShareableContent { } } - #[napi] - pub fn tap_audio( - process_id: u32, - audio_stream_callback: ThreadsafeFunction, - ) -> Result { - ShareableContent::tap_audio_with_callback(process_id, AudioCallback::Js(Arc::new(audio_stream_callback))) - } - pub(crate) fn tap_global_audio_with_callback( excluded_processes: Option>, audio_stream_callback: AudioCallback, @@ -706,15 +698,4 @@ impl ShareableContent { let boxed_manager = Box::new(device_manager); Ok(AudioCaptureSession::new(boxed_manager)) } - - #[napi] - pub fn tap_global_audio( - excluded_processes: Option>, - audio_stream_callback: ThreadsafeFunction, - ) -> Result { - ShareableContent::tap_global_audio_with_callback( - excluded_processes, - AudioCallback::Js(Arc::new(audio_stream_callback)), - ) - } } diff --git a/packages/frontend/native/media_capture/src/macos/tap_audio.rs b/packages/frontend/native/media_capture/src/macos/tap_audio.rs index 46e62ae7e17a1..84dab004f25ba 100644 --- a/packages/frontend/native/media_capture/src/macos/tap_audio.rs +++ b/packages/frontend/native/media_capture/src/macos/tap_audio.rs @@ -21,7 +21,6 @@ use coreaudio::sys::{ kAudioSubTapUIDKey, }; use napi::bindgen_prelude::Result; -use napi_derive::napi; use objc2::runtime::AnyObject; use crate::{ @@ -887,8 +886,6 @@ impl Drop for AggregateDeviceManager { } } -// NEW NAPI Struct: AudioCaptureSession -#[napi] pub struct AudioCaptureSession { // Use Option> to allow taking ownership in stop() manager: Option>, @@ -896,9 +893,7 @@ pub struct AudioCaptureSession { channels: Option, } -#[napi] impl AudioCaptureSession { - // Constructor called internally, not directly via NAPI pub(crate) fn new(manager: Box) -> Self { Self { manager: Some(manager), @@ -907,7 +902,6 @@ impl AudioCaptureSession { } } - #[napi] pub fn stop(&mut self) -> Result<()> { if let Some(manager) = self.manager.take() { // Cache the stats before dropping @@ -926,7 +920,6 @@ impl AudioCaptureSession { } } - #[napi(getter)] pub fn get_sample_rate(&self) -> Result { if let Some(manager) = &self.manager { manager @@ -943,7 +936,6 @@ impl AudioCaptureSession { } } - #[napi(getter)] pub fn get_channels(&self) -> Result { if let Some(manager) = &self.manager { manager @@ -960,7 +952,6 @@ impl AudioCaptureSession { } } - #[napi(getter)] pub fn get_actual_sample_rate(&self) -> Result { if let Some(manager) = &self.manager { manager diff --git a/packages/frontend/native/media_capture/src/windows/audio_capture.rs b/packages/frontend/native/media_capture/src/windows/audio_capture.rs index 9967bddbe39e0..dcebb68c2fc33 100644 --- a/packages/frontend/native/media_capture/src/windows/audio_capture.rs +++ b/packages/frontend/native/media_capture/src/windows/audio_capture.rs @@ -200,7 +200,6 @@ fn emit_mixed_frames( } } -#[napi] pub struct AudioCaptureSession { mic_stream: Option, lb_stream: Option, @@ -258,25 +257,20 @@ where } } -#[napi] impl AudioCaptureSession { - #[napi(getter)] pub fn get_sample_rate(&self) -> f64 { self.sample_rate.0 as f64 } - #[napi(getter)] pub fn get_channels(&self) -> u32 { self.channels } - #[napi(getter)] pub fn get_actual_sample_rate(&self) -> f64 { // For CPAL we always operate at the target rate which is sample_rate self.sample_rate.0 as f64 } - #[napi] pub fn stop(&mut self) -> Result<()> { teardown_audio_capture_resources( &mut self.mic_stream, diff --git a/packages/frontend/native/media_capture/src/windows/screen_capture_kit.rs b/packages/frontend/native/media_capture/src/windows/screen_capture_kit.rs index c2bd65c888645..41d1a60da605c 100644 --- a/packages/frontend/native/media_capture/src/windows/screen_capture_kit.rs +++ b/packages/frontend/native/media_capture/src/windows/screen_capture_kit.rs @@ -225,16 +225,6 @@ impl ShareableContent { crate::windows::audio_capture::start_recording(audio_stream_callback, target) } - #[napi] - pub fn tap_audio( - _process_id: u32, // Currently unused - Windows captures global audio - audio_stream_callback: ThreadsafeFunction, - ) -> Result { - // On Windows with CPAL, we capture global audio (mic + loopback) - // since per-application audio tapping isn't supported the same way as macOS - ShareableContent::tap_audio_with_callback(_process_id, AudioCallback::Js(Arc::new(audio_stream_callback)), None) - } - pub(crate) fn tap_global_audio_with_callback( _excluded_processes: Option>, audio_stream_callback: AudioCallback, @@ -246,18 +236,6 @@ impl ShareableContent { crate::windows::audio_capture::start_recording(audio_stream_callback, target) } - #[napi] - pub fn tap_global_audio( - _excluded_processes: Option>, - audio_stream_callback: ThreadsafeFunction, - ) -> Result { - ShareableContent::tap_global_audio_with_callback( - _excluded_processes, - AudioCallback::Js(Arc::new(audio_stream_callback)), - None, - ) - } - #[napi] pub fn is_using_microphone(process_id: u32) -> Result { is_process_actively_using_microphone(process_id) diff --git a/packages/frontend/native/nbstore/Cargo.toml b/packages/frontend/native/nbstore/Cargo.toml index df4ccf1136dc4..259758bb9a343 100644 --- a/packages/frontend/native/nbstore/Cargo.toml +++ b/packages/frontend/native/nbstore/Cargo.toml @@ -13,7 +13,8 @@ napi = ["affine_common/napi"] use-as-lib = ["napi-derive/noop", "napi/noop"] [dependencies] -affine_common = { workspace = true, features = ["ydoc-loader"] } +affine_common = { workspace = true, features = ["napi"] } +affine_doc_loader = { workspace = true } affine_schema = { path = "../schema" } anyhow = { workspace = true } bincode = { version = "2.0.1", features = ["serde"] } diff --git a/packages/frontend/native/nbstore/src/error.rs b/packages/frontend/native/nbstore/src/error.rs index 673367640e829..90e1e3ecccbf1 100644 --- a/packages/frontend/native/nbstore/src/error.rs +++ b/packages/frontend/native/nbstore/src/error.rs @@ -1,4 +1,4 @@ -use affine_common::doc_parser::ParseError; +use affine_doc_loader::ParseError; pub type Result = std::result::Result; diff --git a/packages/frontend/native/nbstore/src/indexer.rs b/packages/frontend/native/nbstore/src/indexer.rs index 5a8ee894e3089..95e2165ce9db8 100644 --- a/packages/frontend/native/nbstore/src/indexer.rs +++ b/packages/frontend/native/nbstore/src/indexer.rs @@ -1,4 +1,4 @@ -use affine_common::doc_parser::{BlockInfo, CrawlResult, ParseError, parse_doc_from_binary}; +use affine_doc_loader::{BlockInfo, CrawlResult, ParseError, parse_doc_from_binary}; use memory_indexer::{SearchHit, SnapshotData}; use napi_derive::napi; use serde::Serialize; @@ -276,7 +276,7 @@ fn merge_updates(mut segments: Vec>, guid: &str) -> Result> { mod tests { use std::path::{Path, PathBuf}; - use affine_common::doc_parser::ParseError; + use affine_doc_loader::ParseError; use chrono::Utc; use serde_json::Value; use tokio::fs; diff --git a/packages/frontend/native/nbstore/src/storage.rs b/packages/frontend/native/nbstore/src/storage.rs index cb82dd2eb73ee..b25ecfd5ff09c 100644 --- a/packages/frontend/native/nbstore/src/storage.rs +++ b/packages/frontend/native/nbstore/src/storage.rs @@ -311,7 +311,7 @@ mod tests { storage .set_blob(crate::SetBlob { key: "large-blob".to_string(), - data: vec![7; 1024 * 1024], + data: Into::::into(vec![7; 1024 * 1024]), mime: "application/octet-stream".to_string(), }) .await diff --git a/packages/frontend/native/src/import/mod.rs b/packages/frontend/native/src/import/mod.rs new file mode 100644 index 0000000000000..d18b84530f33b --- /dev/null +++ b/packages/frontend/native/src/import/mod.rs @@ -0,0 +1,5 @@ +mod onenote; +mod session; + +use onenote::OneNoteImportProvider; +pub use session::*; diff --git a/packages/frontend/native/src/import/onenote/content/mod.rs b/packages/frontend/native/src/import/onenote/content/mod.rs new file mode 100644 index 0000000000000..86a2b8c9b2c32 --- /dev/null +++ b/packages/frontend/native/src/import/onenote/content/mod.rs @@ -0,0 +1,534 @@ +use std::{collections::HashSet, io::Read}; + +mod rich_text; + +use affine_importer::{ImportWarning, ImportedAsset}; +use onenote_parser::{ + contents::{Content, EmbeddedFile, Image, Outline, OutlineElement, OutlineGroup, OutlineItem, Table}, + page::Page, +}; +use rich_text::rich_text_markdown; +use serde_json::Value as JsonValue; +use sha2::{Digest, Sha256}; + +pub(super) fn page_markdown( + page: &Page, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let mut lines = Vec::new(); + for content in page.contents() { + if let Some(outline) = content.outline() { + lines.push(outline_markdown(outline, assets, warnings, source_path)); + } else if let Some(image) = content.image() { + lines.push(image_markdown(image, assets, warnings, source_path)); + } else if let Some(file) = content.embedded_file() { + lines.push(embedded_file_markdown(file, assets, warnings, source_path)); + } else if content.ink().is_some() { + push_warning(warnings, source_path, "onenote-ink", "Skipped OneNote ink content"); + } + } + let markdown = lines + .into_iter() + .filter(|line| !line.trim().is_empty()) + .collect::>() + .join("\n\n"); + if markdown.trim().is_empty() { + " ".to_string() + } else { + markdown + } +} + +pub(super) fn rewrite_non_image_assets_to_attachments(value: &mut JsonValue, assets: &[ImportedAsset]) { + match value { + JsonValue::Object(map) => { + let attachment = map + .get("flavour") + .and_then(JsonValue::as_str) + .filter(|flavour| *flavour == "affine:image") + .and_then(|_| map.get("props")) + .and_then(JsonValue::as_object) + .and_then(|props| props.get("sourceId")) + .and_then(JsonValue::as_str) + .and_then(|source_id| assets.iter().find(|asset| asset.blob_id == source_id)) + .filter(|asset| !asset.mime.starts_with("image/")); + + if let Some(asset) = attachment { + map.insert( + "flavour".to_string(), + JsonValue::String("affine:attachment".to_string()), + ); + if let Some(props) = map.get_mut("props").and_then(JsonValue::as_object_mut) { + props.insert("name".to_string(), JsonValue::String(asset.file_name.clone())); + props.insert("size".to_string(), JsonValue::Number(asset.bytes.len().into())); + props.insert("type".to_string(), JsonValue::String(asset.mime.clone())); + props.insert("embed".to_string(), JsonValue::Bool(false)); + props.insert("style".to_string(), JsonValue::String("horizontalThin".to_string())); + props.insert("footnoteIdentifier".to_string(), JsonValue::Null); + } + } + for value in map.values_mut() { + rewrite_non_image_assets_to_attachments(value, assets); + } + } + JsonValue::Array(values) => { + for value in values { + rewrite_non_image_assets_to_attachments(value, assets); + } + } + _ => {} + } +} + +pub(super) fn remove_blocks_with_source_ids(value: &mut JsonValue, source_ids: &HashSet) { + if source_ids.is_empty() { + return; + } + + match value { + JsonValue::Object(map) => { + for value in map.values_mut() { + remove_blocks_with_source_ids(value, source_ids); + } + } + JsonValue::Array(values) => { + for value in values.iter_mut() { + remove_blocks_with_source_ids(value, source_ids); + } + values.retain(|value| { + block_source_id(value) + .map(|source_id| !source_ids.contains(source_id)) + .unwrap_or(true) + }); + } + _ => {} + } +} + +fn block_source_id(value: &JsonValue) -> Option<&str> { + value + .as_object() + .and_then(|map| map.get("props")) + .and_then(JsonValue::as_object) + .and_then(|props| props.get("sourceId")) + .and_then(JsonValue::as_str) +} + +fn outline_markdown( + outline: &Outline, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + outline + .items() + .iter() + .map(|item| outline_item_markdown(item, 0, assets, warnings, source_path)) + .filter(|item| !item.trim().is_empty()) + .collect::>() + .join("\n\n") +} + +fn outline_item_markdown( + item: &OutlineItem, + depth: usize, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + match item { + OutlineItem::Element(element) => outline_element_markdown(element, depth, assets, warnings, source_path), + OutlineItem::Group(group) => outline_group_markdown(group, depth, assets, warnings, source_path), + } +} + +fn outline_group_markdown( + group: &OutlineGroup, + depth: usize, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let depth = depth + group.child_level() as usize; + group + .outlines() + .iter() + .map(|item| outline_item_markdown(item, depth, assets, warnings, source_path)) + .filter(|item| !item.trim().is_empty()) + .collect::>() + .join("\n") +} + +fn outline_element_markdown( + element: &OutlineElement, + depth: usize, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let mut content = element + .contents() + .iter() + .map(|content| content_markdown(content, assets, warnings, source_path)) + .filter(|content| !content.trim().is_empty()) + .collect::>() + .join("\n\n"); + + if !content.trim().is_empty() && element_is_list(element) { + content = list_item_markdown(element, depth, &content); + } + + let mut lines = Vec::new(); + if !content.trim().is_empty() { + lines.push(content); + } + for child in element.children() { + let child = outline_item_markdown(child, depth + 1, assets, warnings, source_path); + if !child.trim().is_empty() { + lines.push(child); + } + } + lines.join("\n") +} + +fn content_markdown( + content: &Content, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + if let Some(text) = content.rich_text() { + return rich_text_markdown(text); + } + if let Some(table) = content.table() { + return table_markdown(table, assets, warnings, source_path); + } + if let Some(image) = content.image() { + return image_markdown(image, assets, warnings, source_path); + } else if let Some(file) = content.embedded_file() { + return embedded_file_markdown(file, assets, warnings, source_path); + } else if content.ink().is_some() { + push_warning(warnings, source_path, "onenote-ink", "Skipped OneNote ink content"); + } + String::new() +} + +fn table_markdown( + table: &Table, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let rows = table + .contents() + .iter() + .map(|row| { + row + .contents() + .iter() + .map(|cell| { + let markdown = cell + .contents() + .iter() + .map(|element| outline_element_markdown(element, 0, assets, warnings, source_path)) + .collect::>() + .join("\n"); + markdown_table_cell(&markdown) + }) + .collect::>() + }) + .collect::>(); + if rows.is_empty() { + return String::new(); + } + let cols = rows.iter().map(Vec::len).max().unwrap_or(0).max(1); + let mut markdown = String::new(); + let header = rows.first().cloned().unwrap_or_default(); + markdown.push_str(&markdown_row(&header, cols)); + markdown.push('\n'); + markdown.push_str(&markdown_row(&vec!["---".to_string(); cols], cols)); + for row in rows.iter().skip(1) { + markdown.push('\n'); + markdown.push_str(&markdown_row(row, cols)); + } + markdown +} + +fn markdown_row(row: &[String], cols: usize) -> String { + let mut cells = row.to_vec(); + cells.resize(cols, String::new()); + format!("| {} |", cells.join(" | ")) +} + +fn markdown_table_cell(value: &str) -> String { + value + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .collect::>() + .join("
") + .replace('|', "\\|") +} + +fn escape_markdown_label(value: &str) -> String { + let mut output = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '\\' | '[' | ']' | '*' | '_' | '`' => { + output.push('\\'); + output.push(ch); + } + '\n' | '\r' => output.push(' '), + _ => output.push(ch), + } + } + output +} + +fn image_markdown( + image: &Image, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let Some(bytes) = read_optional_blob(image.read(), warnings, source_path, "onenote-image") else { + return image + .alt_text() + .or_else(|| image.text()) + .unwrap_or_default() + .to_string(); + }; + let detected = infer::get(&bytes); + let file_name = image_file_name(image, assets.len() + 1, detected.map(|kind| kind.extension())); + let blob_id = blob_id(&bytes); + let mime = image_mime(image, &file_name, detected.map(|kind| kind.mime_type())); + assets.push(ImportedAsset { + blob_id: blob_id.clone(), + source_path: format!("{source_path}/{file_name}"), + file_name, + mime, + bytes, + }); + let caption = image.alt_text().or_else(|| image.text()).unwrap_or(""); + format!("![{}](blob://{})", escape_markdown_label(caption), blob_id) +} + +fn element_is_list(element: &OutlineElement) -> bool { + !element.list_contents().is_empty() +} + +fn list_item_markdown(element: &OutlineElement, depth: usize, content: &str) -> String { + let marker = list_marker(element); + let indent = " ".repeat(depth); + content + .lines() + .enumerate() + .map(|(index, line)| { + if index == 0 { + format!("{indent}{marker} {line}") + } else if line.trim().is_empty() { + String::new() + } else { + format!("{indent} {line}") + } + }) + .collect::>() + .join("\n") +} + +fn list_marker(element: &OutlineElement) -> &'static str { + let ordered = element.list_contents().iter().any(|list| { + list.list_restart().is_some() + || list + .list_format() + .iter() + .any(|ch| ch.is_ascii_digit() || matches!(ch, '%' | '#')) + }); + if ordered { "1." } else { "-" } +} + +fn image_file_name(image: &Image, index: usize, detected_ext: Option<&'static str>) -> String { + let raw_name = image + .image_filename() + .filter(|name| !name.trim().is_empty()) + .map(|name| sanitize_file_name(name, "image")) + .unwrap_or_else(|| { + let ext = image.extension().or(detected_ext).unwrap_or("png"); + format!("image-{index}.{ext}") + }); + + if mime_from_path(&raw_name).starts_with("image/") { + return raw_name; + } + + image + .extension() + .or(detected_ext) + .map(|ext| format!("{raw_name}.{ext}")) + .unwrap_or(raw_name) +} + +fn embedded_file_markdown( + file: &EmbeddedFile, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let mut reader = file.read(); + let mut bytes = Vec::new(); + if let Err(error) = reader.read_to_end(&mut bytes) { + push_warning( + warnings, + source_path, + "onenote-attachment", + &format!("Skipped OneNote embedded file content: {error}"), + ); + return file.filename().to_string(); + } + let file_name = sanitize_file_name(file.filename(), "attachment"); + let blob_id = blob_id(&bytes); + assets.push(ImportedAsset { + blob_id: blob_id.clone(), + source_path: format!("{source_path}/{file_name}"), + file_name: file_name.clone(), + mime: mime_from_path(&file_name).to_string(), + bytes, + }); + format!("![{}](blob://{})", escape_markdown_label(&file_name), blob_id) +} + +fn read_optional_blob( + reader: Option>, + warnings: &mut Vec, + source_path: &str, + code: &str, +) -> Option> { + let Some(mut reader) = reader else { + push_warning( + warnings, + source_path, + code, + "Skipped OneNote blob content: missing data", + ); + return None; + }; + let mut bytes = Vec::new(); + if let Err(error) = reader.read_to_end(&mut bytes) { + push_warning( + warnings, + source_path, + code, + &format!("Skipped OneNote blob content: {error}"), + ); + return None; + } + Some(bytes) +} + +fn image_mime(image: &Image, file_name: &str, detected_mime: Option<&'static str>) -> String { + image + .extension() + .map(mime_from_extension) + .or(detected_mime) + .unwrap_or_else(|| mime_from_path(file_name)) + .to_string() +} + +fn blob_id(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + hex::encode(hasher.finalize()) +} + +fn sanitize_file_name(value: &str, fallback: &str) -> String { + let name = value + .split(['/', '\\', '\0']) + .filter(|part| !part.is_empty() && *part != "." && *part != "..") + .collect::>() + .join(" ") + .trim() + .to_string(); + if name.is_empty() { fallback.to_string() } else { name } +} + +fn mime_from_path(path: &str) -> &'static str { + match path.rsplit('.').next().unwrap_or("").to_lowercase().as_str() { + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + "gif" => "image/gif", + "webp" => "image/webp", + "bmp" => "image/bmp", + "svg" => "image/svg+xml", + "pdf" => "application/pdf", + "txt" => "text/plain", + "md" | "markdown" => "text/markdown", + "csv" => "text/csv", + "zip" => "application/zip", + "doc" => "application/msword", + "docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xls" => "application/vnd.ms-excel", + "xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "ppt" => "application/vnd.ms-powerpoint", + "pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation", + _ => "application/octet-stream", + } +} + +fn mime_from_extension(extension: &str) -> &'static str { + mime_from_path(extension) +} + +fn push_warning(warnings: &mut Vec, source_path: &str, code: &str, message: &str) { + warnings.push(ImportWarning { + code: code.to_string(), + source_path: Some(source_path.to_string()), + message: message.to_string(), + }); +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn markdown_table_cell_collapses_multiline_content() { + assert_eq!( + markdown_table_cell("first line\nsecond | line\n\n third line "), + "first line
second \\| line
third line" + ); + } + + #[test] + fn remove_blocks_with_source_ids_removes_matching_nested_blocks() { + let mut snapshot = json!({ + "children": [ + { + "flavour": "affine:paragraph", + "props": {}, + "children": [ + { + "flavour": "affine:image", + "props": { "sourceId": "skipped" }, + "children": [] + }, + { + "flavour": "affine:image", + "props": { "sourceId": "kept" }, + "children": [] + } + ] + } + ] + }); + let source_ids = HashSet::from(["skipped".to_string()]); + + remove_blocks_with_source_ids(&mut snapshot, &source_ids); + + assert_eq!(snapshot["children"][0]["children"].as_array().unwrap().len(), 1); + assert_eq!(snapshot["children"][0]["children"][0]["props"]["sourceId"], "kept"); + } +} diff --git a/packages/frontend/native/src/import/onenote/content/rich_text.rs b/packages/frontend/native/src/import/onenote/content/rich_text.rs new file mode 100644 index 0000000000000..ee63589a33c71 --- /dev/null +++ b/packages/frontend/native/src/import/onenote/content/rich_text.rs @@ -0,0 +1,171 @@ +use onenote_parser::{ + contents::{ParagraphStyling, RichText}, + property::common::ColorRef, +}; + +pub(super) fn rich_text_markdown(text: &RichText) -> String { + let mut markdown = styled_text_markdown(text); + let trimmed = markdown.trim(); + if trimmed.is_empty() { + return String::new(); + } + + if let Some(prefix) = heading_prefix(text, trimmed) { + markdown = format!("{prefix} {}", trimmed.trim_start_matches('#').trim_start()); + } + markdown +} + +fn styled_text_markdown(text: &RichText) -> String { + let chars = text.text().chars().collect::>(); + let styles = text.text_run_formatting(); + if chars.is_empty() { + return String::new(); + } + if styles.is_empty() { + return chars.into_iter().collect(); + } + + let mut output = String::new(); + let mut start = 0usize; + let mut pending_link: Option = None; + for (index, style) in styles.iter().enumerate() { + let end = text + .text_run_indices() + .get(index) + .map(|end| (*end as usize).min(chars.len())) + .unwrap_or(chars.len()); + if end < start { + continue; + } + let segment = chars[start..end].iter().collect::(); + start = end; + if segment.is_empty() { + continue; + } + if style.hidden() { + if let Some(url) = extract_hyperlink_url(&segment) { + pending_link = Some(url); + } + continue; + } + output.push_str(&format_segment(&segment, style, &mut pending_link)); + } + if start < chars.len() { + output.push_str(&chars[start..].iter().collect::()); + } + output +} + +fn heading_prefix(text: &RichText, value: &str) -> Option<&'static str> { + if value.chars().count() > 120 || value.lines().count() > 1 { + return None; + } + let font_size = text + .text_run_formatting() + .iter() + .filter(|style| !style.hidden()) + .filter_map(ParagraphStyling::font_size) + .max() + .or_else(|| text.paragraph_style().font_size()); + + match font_size { + Some(size) if size >= 28 => Some("##"), + Some(size) if size >= 24 => Some("###"), + _ => None, + } +} + +fn format_segment(segment: &str, style: &ParagraphStyling, pending_link: &mut Option) -> String { + let mut value = escape_html(segment); + if segment.trim().is_empty() { + return value; + } + + if style.bold() && should_keep_inline_style(segment) { + value = wrap_inline(&value, "", ""); + } + if style.italic() && should_keep_inline_style(segment) { + value = wrap_inline(&value, "", ""); + } + if style.strikethrough() { + value = wrap_inline(&value, "", ""); + } + if style.underline() { + value = wrap_inline(&value, "", ""); + } + if let Some(color) = style.font_color().and_then(hex_color) { + value = wrap_inline(&value, &format!(""), ""); + } + if let Some(color) = style.highlight().and_then(hex_color) { + value = wrap_inline(&value, &format!(""), ""); + } + + if style.hyperlink_protected() + && let Some(url) = pending_link.take() + { + return wrap_inline(&value, &format!("", escape_html_attr(&url)), ""); + } + value +} + +fn should_keep_inline_style(value: &str) -> bool { + value.chars().any(|ch| ch.is_alphanumeric()) +} + +fn wrap_inline(value: &str, open: &str, close: &str) -> String { + let leading_len = value.len() - value.trim_start().len(); + let trailing_len = value.len() - value.trim_end().len(); + let (leading, rest) = value.split_at(leading_len); + let (middle, trailing) = rest.split_at(rest.len().saturating_sub(trailing_len)); + if middle.is_empty() { + value.to_string() + } else { + format!("{leading}{open}{middle}{close}{trailing}") + } +} + +fn hex_color(color: ColorRef) -> Option { + match color { + ColorRef::Auto => None, + ColorRef::Manual { r, g, b } => Some(format!("#{r:02x}{g:02x}{b:02x}")), + } +} + +fn escape_html(value: &str) -> String { + let mut output = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '&' => output.push_str("&"), + '<' => output.push_str("<"), + '>' => output.push_str(">"), + _ => output.push(ch), + } + } + output +} + +fn escape_html_attr(value: &str) -> String { + let mut output = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '&' => output.push_str("&"), + '<' => output.push_str("<"), + '>' => output.push_str(">"), + '"' => output.push_str("""), + _ => output.push(ch), + } + } + output +} + +fn extract_hyperlink_url(value: &str) -> Option { + let marker = "HYPERLINK"; + let start = value.find(marker)? + marker.len(); + let rest = value[start..].trim(); + if let Some(quoted) = rest.strip_prefix('"') { + let end = quoted.find('"')?; + return Some(quoted[..end].to_string()); + } + rest.split_whitespace().next().map(ToString::to_string) +} diff --git a/packages/frontend/native/src/import/onenote/mod.rs b/packages/frontend/native/src/import/onenote/mod.rs new file mode 100644 index 0000000000000..53f65c69a28d7 --- /dev/null +++ b/packages/frontend/native/src/import/onenote/mod.rs @@ -0,0 +1,444 @@ +use std::{ + collections::{HashSet, hash_map::DefaultHasher}, + fs, + hash::{Hash, Hasher}, + path::{Path, PathBuf}, +}; + +mod content; + +use affine_importer::{ + FolderHierarchyDelta, ImportBatch, ImportBatchLimits, ImportCursor, ImportError, ImportOptions, ImportProgress, + ImportProvider, ImportResult, ImportSource, ImportWarning, ImportedAsset, ImportedDocMeta, ImportedDocSnapshot, +}; +use content::{page_markdown, remove_blocks_with_source_ids, rewrite_non_image_assets_to_attachments}; +use onenote_parser::{ + Parser, + notebook::Notebook, + page::Page, + section::{Section, SectionEntry, SectionGroup}, +}; +use typed_path::TypedPath; + +pub struct OneNoteImportProvider; + +impl OneNoteImportProvider { + pub fn new() -> Self { + Self + } +} + +impl ImportProvider for OneNoteImportProvider { + fn format(&self) -> &'static str { + "oneNote" + } + + fn create_cursor( + &self, + source: ImportSource, + _options: ImportOptions, + limits: ImportBatchLimits, + ) -> ImportResult> { + let mut planner = OneNotePlanner { + docs: Vec::new(), + folders: Vec::new(), + warnings: Vec::new(), + }; + planner.read_source(source)?; + Ok(Box::new(OneNoteImportCursor { + docs: planner.docs, + folders: planner.folders, + warnings: planner.warnings, + next_asset_doc: 0, + next_asset_index: 0, + next_doc: 0, + emitted_final: false, + skipped_blob_ids: HashSet::new(), + limits, + })) + } +} + +struct OneNoteDoc { + id: String, + title: String, + markdown: String, + assets: Vec, +} + +struct OneNotePlanner { + docs: Vec, + folders: Vec, + warnings: Vec, +} + +impl OneNotePlanner { + fn read_source(&mut self, source: ImportSource) -> ImportResult<()> { + match source { + ImportSource::FilePath(path) => self.read_file(&path), + ImportSource::DirectoryPath(path) => self.read_directory(&path), + } + } + + fn read_file(&mut self, path: &Path) -> ImportResult<()> { + let parser = Parser::new(); + match extension(path).as_deref() { + Some("one") => { + let source_path = path.to_string_lossy(); + let section = parser + .parse_section(TypedPath::derive(source_path.as_ref())) + .map_err(import_error)?; + self.add_section(§ion, None); + Ok(()) + } + Some("onetoc2") => { + let source_path = path.to_string_lossy(); + let notebook = parser + .parse_notebook(TypedPath::derive(source_path.as_ref())) + .map_err(import_error)?; + self.add_notebook(¬ebook, notebook_name(path)); + Ok(()) + } + Some("onepkg") => { + let source_path = path.to_string_lossy(); + let notebook = parser + .parse_package(TypedPath::derive(source_path.as_ref())) + .map_err(import_error)?; + self.add_notebook(¬ebook, notebook_name(path)); + Ok(()) + } + _ => Err(ImportError::InvalidSource(format!( + "unsupported OneNote source: {}", + path.to_string_lossy() + ))), + } + } + + fn read_directory(&mut self, path: &Path) -> ImportResult<()> { + let toc = find_first_onetoc2(path)?.ok_or_else(|| { + ImportError::InvalidSource(format!( + "directory contains no .onetoc2 file: {}", + path.to_string_lossy() + )) + })?; + let parser = Parser::new(); + let toc_path = toc.to_string_lossy(); + let notebook = parser + .parse_notebook(TypedPath::derive(toc_path.as_ref())) + .map_err(import_error)?; + self.add_notebook(¬ebook, notebook_name(path)); + Ok(()) + } + + fn add_notebook(&mut self, notebook: &Notebook, name: String) { + let root = sanitize_path_part(&name); + self.folders.push(FolderHierarchyDelta { + path: root.clone(), + name, + parent_path: None, + page_id: None, + icon: None, + }); + for entry in notebook.entries() { + self.add_section_entry(entry, Some(root.clone())); + } + } + + fn add_section_entry(&mut self, entry: &SectionEntry, parent_path: Option) { + match entry { + SectionEntry::Section(section) => self.add_section(section, parent_path), + SectionEntry::SectionGroup(group) => self.add_section_group(group, parent_path), + } + } + + fn add_section_group(&mut self, group: &SectionGroup, parent_path: Option) { + let path = join_import_path(parent_path.as_deref(), group.display_name()); + self.folders.push(FolderHierarchyDelta { + path: path.clone(), + name: group.display_name().to_string(), + parent_path, + page_id: None, + icon: None, + }); + for entry in group.entries() { + self.add_section_entry(entry, Some(path.clone())); + } + } + + fn add_section(&mut self, section: &Section, parent_path: Option) { + let path = join_import_path(parent_path.as_deref(), section.display_name()); + self.folders.push(FolderHierarchyDelta { + path: path.clone(), + name: section.display_name().to_string(), + parent_path, + page_id: None, + icon: None, + }); + let mut page_index = 0; + for series in section.page_series() { + for page in series.pages() { + self.add_page(page, &path, page_index); + page_index += 1; + } + } + } + + fn add_page(&mut self, page: &Page, parent_path: &str, page_index: usize) { + let title = page + .title_text() + .map(str::trim) + .filter(|title| !title.is_empty()) + .unwrap_or("Untitled") + .to_string(); + let source_path = join_import_path(Some(parent_path), &title); + let id = doc_id(page.link_target_id(), &title, parent_path, page_index); + let mut assets = Vec::new(); + let markdown = page_markdown(page, &mut assets, &mut self.warnings, &source_path); + self.folders.push(FolderHierarchyDelta { + path: source_path.clone(), + name: title.clone(), + parent_path: Some(parent_path.to_string()), + page_id: Some(id.clone()), + icon: None, + }); + self.docs.push(OneNoteDoc { + id, + title, + markdown, + assets, + }); + } +} + +struct OneNoteImportCursor { + docs: Vec, + folders: Vec, + warnings: Vec, + next_asset_doc: usize, + next_asset_index: usize, + next_doc: usize, + emitted_final: bool, + skipped_blob_ids: HashSet, + limits: ImportBatchLimits, +} + +impl ImportCursor for OneNoteImportCursor { + fn total(&self) -> usize { + self.docs.len() + } + + fn next_batch(&mut self) -> ImportResult> { + if self.next_doc >= self.docs.len() && self.emitted_final { + return Ok(None); + } + + if let Some(batch) = self.next_asset_batch() { + return Ok(Some(batch)); + } + + let end = (self.next_doc + self.limits.max_docs.max(1)).min(self.docs.len()); + let mut batch = empty_batch(self.docs.len()); + for doc in &self.docs[self.next_doc..end] { + let mut snapshot = affine_doc_loader::build_doc_snapshot(&doc.title, &doc.markdown, &doc.id)?; + rewrite_non_image_assets_to_attachments(&mut snapshot, &doc.assets); + remove_blocks_with_source_ids(&mut snapshot, &self.skipped_blob_ids); + batch.docs.push(ImportedDocSnapshot { + id: doc.id.clone(), + snapshot, + meta: Some(ImportedDocMeta { + title: Some(doc.title.clone()), + create_date: None, + updated_date: None, + tags: None, + favorite: None, + trash: None, + }), + }); + } + self.next_doc = end; + if self.next_doc >= self.docs.len() { + batch.folders = std::mem::take(&mut self.folders); + batch.warnings = std::mem::take(&mut self.warnings); + self.emitted_final = true; + } + batch.progress = ImportProgress { + completed: self.next_doc, + total: self.docs.len(), + }; + batch.done = self.emitted_final; + Ok(Some(batch)) + } +} + +impl OneNoteImportCursor { + fn next_asset_batch(&mut self) -> Option { + let max_blobs = self.limits.max_blobs.max(1); + let max_blob_bytes = self.limits.max_blob_bytes; + let mut batch = empty_batch(self.docs.len()); + let mut bytes_in_batch = 0u64; + + while self.next_asset_doc < self.docs.len() && batch.blobs.len() < max_blobs { + let doc = &self.docs[self.next_asset_doc]; + if self.next_asset_index >= doc.assets.len() { + self.next_asset_doc += 1; + self.next_asset_index = 0; + continue; + } + + let asset = &doc.assets[self.next_asset_index]; + self.next_asset_index += 1; + let asset_size = asset.bytes.len() as u64; + if asset_size > max_blob_bytes { + self.skipped_blob_ids.insert(asset.blob_id.clone()); + batch.warnings.push(ImportWarning { + code: "blob_too_large".to_string(), + source_path: Some(asset.source_path.clone()), + message: format!("Skipped {}: blob is larger than batch limit", asset.source_path), + }); + continue; + } + if !batch.blobs.is_empty() && bytes_in_batch + asset_size > max_blob_bytes { + self.next_asset_index -= 1; + break; + } + bytes_in_batch += asset_size; + batch.blobs.push(asset.clone()); + } + + if batch.blobs.is_empty() && batch.warnings.is_empty() { + return None; + } + batch.progress = ImportProgress { + completed: self.next_doc, + total: self.docs.len(), + }; + Some(batch) + } +} + +fn empty_batch(total: usize) -> ImportBatch { + ImportBatch { + docs: Vec::new(), + blobs: Vec::new(), + folders: Vec::new(), + tags: Vec::new(), + icons: Vec::new(), + warnings: Vec::new(), + progress: ImportProgress { completed: 0, total }, + entry_id: None, + is_workspace_file: false, + done: false, + } +} + +fn find_first_onetoc2(root: &Path) -> ImportResult> { + for entry in fs::read_dir(root)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + if let Some(found) = find_first_onetoc2(&path)? { + return Ok(Some(found)); + } + } else if extension(&path).as_deref() == Some("onetoc2") { + return Ok(Some(path)); + } + } + Ok(None) +} + +fn import_error(error: onenote_parser::errors::Error) -> ImportError { + ImportError::InvalidSource(error.to_string()) +} + +fn notebook_name(path: &Path) -> String { + path + .file_stem() + .or_else(|| path.file_name()) + .map(|name| name.to_string_lossy().to_string()) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| "OneNote".to_string()) +} + +fn extension(path: &Path) -> Option { + path.extension().map(|ext| ext.to_string_lossy().to_lowercase()) +} + +fn join_import_path(parent: Option<&str>, name: &str) -> String { + let name = sanitize_path_part(name); + parent + .filter(|parent| !parent.is_empty()) + .map(|parent| format!("{parent}/{name}")) + .unwrap_or(name) +} + +fn sanitize_path_part(value: &str) -> String { + let name = value + .split(['/', '\\']) + .filter(|part| !part.is_empty()) + .collect::>() + .join(" ") + .trim() + .to_string(); + if name.is_empty() { "Untitled".to_string() } else { name } +} + +fn doc_id(raw_id: &str, title: &str, parent_path: &str, page_index: usize) -> String { + let mut id = raw_id + .chars() + .filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_') + .collect::(); + if id.is_empty() { + let slug = title + .chars() + .filter(|ch| ch.is_ascii_alphanumeric()) + .take(32) + .collect::(); + let mut hasher = DefaultHasher::new(); + title.hash(&mut hasher); + parent_path.hash(&mut hasher); + page_index.hash(&mut hasher); + let hash = format!("{:x}", hasher.finish()); + id = if slug.is_empty() { + hash + } else { + format!("{slug}-{hash}") + }; + } + format!("onenote-{id}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sanitize_path_part_removes_path_separators() { + assert_eq!(sanitize_path_part("Group/Section\\Page"), "Group Section Page"); + assert_eq!(sanitize_path_part("///"), "Untitled"); + } + + #[test] + fn join_import_path_sanitizes_path_parts() { + assert_eq!( + join_import_path(Some("Notebook/Section"), "Page/Subpage\\Draft"), + "Notebook/Section/Page Subpage Draft" + ); + } + + #[test] + fn doc_id_keeps_ascii_ids_and_hashes_non_ascii_titles() { + assert_eq!(doc_id("{abc-123}", "Ignored", "Notebook/Section", 0), "onenote-abc-123"); + + let id = doc_id("", "中文标题", "Notebook/Section", 0); + assert!(id.starts_with("onenote-")); + assert!(id.len() > "onenote-".len()); + } + + #[test] + fn doc_id_fallback_distinguishes_duplicate_titles() { + let first = doc_id("", "Untitled", "Notebook/Section", 0); + let second = doc_id("", "Untitled", "Notebook/Section", 1); + + assert_ne!(first, second); + } +} diff --git a/packages/frontend/native/src/import/session.rs b/packages/frontend/native/src/import/session.rs new file mode 100644 index 0000000000000..8d6e817678873 --- /dev/null +++ b/packages/frontend/native/src/import/session.rs @@ -0,0 +1,279 @@ +use std::{ + collections::HashMap, + path::PathBuf, + sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, + }, +}; + +use affine_importer::{ + ImportBatchLimits, ImportOptions, ImportRegistry, ImportSession, ImportSessionOptions, ImportSource, +}; +use napi::{Status, bindgen_prelude::*}; +use napi_derive::napi; +use once_cell::sync::Lazy; + +static IMPORT_SESSIONS: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); +static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1); +static IMPORT_REGISTRY: Lazy> = Lazy::new(|| { + let mut registry = ImportRegistry::with_builtin(); + registry.register(super::OneNoteImportProvider::new()); + Arc::new(registry) +}); + +#[napi(object)] +pub struct CreateImportSessionOptions { + pub format: String, + pub source: CreateImportSessionSource, + pub batch_limits: Option, +} + +#[napi(object)] +pub struct CreateImportSessionSource { + pub kind: String, + pub path: String, +} + +#[napi(object)] +pub struct CreateImportBatchLimits { + pub max_docs: Option, + pub max_blobs: Option, + pub max_blob_bytes: Option, +} + +fn map_import_error(error: affine_importer::ImportError) -> Error { + let status = match error { + affine_importer::ImportError::Cancelled => Status::Cancelled, + affine_importer::ImportError::UnsupportedFormat(_) | affine_importer::ImportError::InvalidSource(_) => { + Status::InvalidArg + } + affine_importer::ImportError::Zip(_) + | affine_importer::ImportError::Io(_) + | affine_importer::ImportError::Document(_) => Status::GenericFailure, + }; + Error::new(status, error.to_string()) +} + +#[napi] +pub fn create_import_session(options: CreateImportSessionOptions) -> Result { + let source = match options.source.kind.as_str() { + "filePath" => ImportSource::FilePath(PathBuf::from(options.source.path)), + "directoryPath" => ImportSource::DirectoryPath(PathBuf::from(options.source.path)), + kind => { + return Err(Error::new( + Status::InvalidArg, + format!("unsupported import source kind: {kind}"), + )); + } + }; + let mut batch_limits = ImportBatchLimits::default(); + if let Some(limits) = options.batch_limits { + if let Some(max_docs) = limits.max_docs { + batch_limits.max_docs = max_docs as usize; + } + if let Some(max_blobs) = limits.max_blobs { + batch_limits.max_blobs = max_blobs as usize; + } + if let Some(max_blob_bytes) = limits.max_blob_bytes { + if !(0..=100 * 1024 * 1024).contains(&max_blob_bytes) { + return Err(Error::new(Status::InvalidArg, "maxBlobBytes not valid")); + } + batch_limits.max_blob_bytes = max_blob_bytes as u64; + } + } + let session = ImportSession::create( + ImportSessionOptions { + format: options.format, + source, + import_options: ImportOptions::default(), + batch_limits, + }, + IMPORT_REGISTRY.clone(), + ) + .map_err(map_import_error)?; + let id = format!("import-session-{}", NEXT_SESSION_ID.fetch_add(1, Ordering::Relaxed)); + IMPORT_SESSIONS + .lock() + .map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))? + .insert(id.clone(), session); + Ok(id) +} + +#[napi] +pub fn next_import_batch(session_id: String) -> Result> { + let mut sessions = IMPORT_SESSIONS + .lock() + .map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?; + let session = sessions + .get_mut(&session_id) + .ok_or_else(|| Error::new(Status::InvalidArg, format!("unknown import session: {session_id}")))?; + session + .next_batch() + .map_err(map_import_error)? + .map(|batch| serde_json::to_string(&batch).map_err(|err| Error::new(Status::GenericFailure, err.to_string()))) + .transpose() +} + +#[napi] +pub fn cancel_import_session(session_id: String) -> Result<()> { + let mut sessions = IMPORT_SESSIONS + .lock() + .map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?; + let session = sessions + .get_mut(&session_id) + .ok_or_else(|| Error::new(Status::InvalidArg, format!("unknown import session: {session_id}")))?; + session.cancel(); + Ok(()) +} + +#[napi] +pub fn dispose_import_session(session_id: String) -> Result<()> { + IMPORT_SESSIONS + .lock() + .map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))? + .remove(&session_id); + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::{ + fs, + io::Write, + path::PathBuf, + sync::atomic::{AtomicU64, Ordering}, + }; + + use zip::{ZipWriter, write::SimpleFileOptions}; + + use super::*; + + static NEXT_TEST_ARCHIVE_ID: AtomicU64 = AtomicU64::new(1); + + fn archive_path(entries: &[(&str, &[u8])]) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "affine-native-import-{}-{}.zip", + std::process::id(), + NEXT_TEST_ARCHIVE_ID.fetch_add(1, Ordering::Relaxed) + )); + let file = std::fs::File::create(&path).unwrap(); + let mut writer = ZipWriter::new(file); + for (path, bytes) in entries { + writer.start_file(path, SimpleFileOptions::default()).unwrap(); + writer.write_all(bytes).unwrap(); + } + writer.finish().unwrap(); + path + } + + fn directory_path(entries: &[(&str, &[u8])]) -> PathBuf { + let root = std::env::temp_dir().join(format!( + "affine-native-import-dir-{}-{}", + std::process::id(), + NEXT_TEST_ARCHIVE_ID.fetch_add(1, Ordering::Relaxed) + )); + for (path, bytes) in entries { + let path = root.join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, bytes).unwrap(); + } + root + } + + #[test] + fn import_session_returns_one_serialized_batch() { + let path = archive_path(&[("entry.md", b"entry")]); + let id = create_import_session(CreateImportSessionOptions { + format: "markdownZip".to_string(), + source: CreateImportSessionSource { + kind: "filePath".to_string(), + path: path.to_string_lossy().to_string(), + }, + batch_limits: None, + }) + .unwrap(); + + let batch = next_import_batch(id.clone()).unwrap().unwrap(); + assert!(batch.contains("\"docs\"")); + assert!(next_import_batch(id.clone()).unwrap().is_none()); + dispose_import_session(id).unwrap(); + let _ = std::fs::remove_file(path); + } + + #[test] + fn import_session_cancel_blocks_next_batch() { + let path = archive_path(&[("entry.md", b"entry")]); + let id = create_import_session(CreateImportSessionOptions { + format: "markdownZip".to_string(), + source: CreateImportSessionSource { + kind: "filePath".to_string(), + path: path.to_string_lossy().to_string(), + }, + batch_limits: None, + }) + .unwrap(); + + cancel_import_session(id.clone()).unwrap(); + assert!(next_import_batch(id.clone()).is_err()); + dispose_import_session(id).unwrap(); + let _ = std::fs::remove_file(path); + } + + #[test] + fn import_session_accepts_directory_source() { + let path = directory_path(&[("entry.md", b"entry")]); + let id = create_import_session(CreateImportSessionOptions { + format: "obsidian".to_string(), + source: CreateImportSessionSource { + kind: "directoryPath".to_string(), + path: path.to_string_lossy().to_string(), + }, + batch_limits: None, + }) + .unwrap(); + + let batch = next_import_batch(id.clone()).unwrap().unwrap(); + assert!(batch.contains("\"docs\"")); + assert!(next_import_batch(id.clone()).unwrap().is_none()); + dispose_import_session(id).unwrap(); + let _ = fs::remove_dir_all(path); + } + + #[test] + fn import_session_rejects_negative_blob_byte_limit() { + let path = archive_path(&[("entry.md", b"entry")]); + let result = create_import_session(CreateImportSessionOptions { + format: "markdownZip".to_string(), + source: CreateImportSessionSource { + kind: "filePath".to_string(), + path: path.to_string_lossy().to_string(), + }, + batch_limits: Some(CreateImportBatchLimits { + max_docs: None, + max_blobs: None, + max_blob_bytes: Some(-1), + }), + }); + + assert!(result.is_err()); + let _ = std::fs::remove_file(path); + } + + #[test] + fn import_session_routes_onenote_format_to_provider() { + let path = archive_path(&[("entry.md", b"entry")]); + let result = create_import_session(CreateImportSessionOptions { + format: "oneNote".to_string(), + source: CreateImportSessionSource { + kind: "filePath".to_string(), + path: path.to_string_lossy().to_string(), + }, + batch_limits: None, + }); + + let error = result.unwrap_err().to_string(); + assert!(error.contains("unsupported OneNote source")); + let _ = std::fs::remove_file(path); + } +} diff --git a/packages/frontend/native/src/lib.rs b/packages/frontend/native/src/lib.rs index f5b2a56d9424f..eec648226f8b9 100644 --- a/packages/frontend/native/src/lib.rs +++ b/packages/frontend/native/src/lib.rs @@ -1,4 +1,5 @@ pub mod hashcash; +mod import; #[cfg(not(any(target_os = "android", target_os = "ios")))] pub mod preview; @@ -10,3 +11,4 @@ static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; pub use affine_media_capture::*; pub use affine_nbstore::*; pub use affine_sqlite_v1::*; +pub use import::*; diff --git a/tools/cli/src/bundle-shared.ts b/tools/cli/src/bundle-shared.ts index 9afc1955e0972..a932c23e71c76 100644 --- a/tools/cli/src/bundle-shared.ts +++ b/tools/cli/src/bundle-shared.ts @@ -9,7 +9,6 @@ export const RSPACK_SUPPORTED_PACKAGES = [ '@affine/electron-renderer', '@affine/server', '@affine/reader', - '@affine/media-capture-playground', ] as const; const rspackSupportedPackageSet = new Set(RSPACK_SUPPORTED_PACKAGES); diff --git a/tools/cli/src/bundle.ts b/tools/cli/src/bundle.ts index a27ce78bdf58f..005ba6ab0ae56 100644 --- a/tools/cli/src/bundle.ts +++ b/tools/cli/src/bundle.ts @@ -194,41 +194,11 @@ function getRspackBundleConfigs(pkg: Package): MultiRspackOptions { }), ] as MultiRspackOptions; } - case '@affine/media-capture-playground': { - return [ - createRspackHTMLTargetConfig(pkg, pkg.join('web/main.tsx').value, { - template: pkg.join('web/index.html').value, - additionalEntryForSelfhost: false, - copySharedPublicAssets: false, - injectGlobalErrorHandler: false, - emitAssetsManifest: false, - }), - ] as MultiRspackOptions; - } } throw new Error(`Unsupported package: ${pkg.name}`); } -function getRspackDevServerConfig( - pkg: Package -): RspackDevServerConfiguration | undefined { - if (pkg.name !== '@affine/media-capture-playground') { - return; - } - - return { - proxy: [ - { - context: '/api', - target: 'http://localhost:6544', - changeOrigin: true, - pathRewrite: { '^/api': '' }, - }, - ], - }; -} - export class BundleCommand extends PackageCommand { static override paths = [['bundle'], ['pack'], ['bun']]; @@ -244,7 +214,7 @@ export class BundleCommand extends PackageCommand { const pkg = this.workspace.getPackage(this.package); if (this.dev) { - await BundleCommand.dev(pkg, getRspackDevServerConfig(pkg)); + await BundleCommand.dev(pkg); } else { await BundleCommand.build(pkg); } diff --git a/tools/cli/src/rspack/index.ts b/tools/cli/src/rspack/index.ts index e79e57f804a7d..f93e3c25667f5 100644 --- a/tools/cli/src/rspack/index.ts +++ b/tools/cli/src/rspack/index.ts @@ -78,9 +78,7 @@ export function createHTMLTargetConfig( ): RspackConfiguration { entry = typeof entry === 'string' ? { index: entry } : entry; const tailwindConfigPath = pkg.join('tailwind.config.js'); - const hasTailwind = - tailwindConfigPath.exists() || - pkg.name === '@affine/media-capture-playground'; + const hasTailwind = tailwindConfigPath.exists(); const tailwindPlugin = tailwindConfigPath.exists() ? ['@tailwindcss/postcss', require(tailwindConfigPath.value)] : ['@tailwindcss/postcss']; diff --git a/tools/utils/src/distribution.ts b/tools/utils/src/distribution.ts index 7c0b30992da1d..489b496c63dfd 100644 --- a/tools/utils/src/distribution.ts +++ b/tools/utils/src/distribution.ts @@ -6,7 +6,6 @@ export const PackageToDistribution = new Map< >([ ['@affine/admin', 'admin'], ['@affine/web', 'web'], - ['@affine/media-capture-playground', 'web'], ['@affine/electron-renderer', 'desktop'], ['@affine/electron', 'desktop'], ['@affine/mobile', 'mobile'], diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts index 1727fe14c9f02..bf2aece02937d 100644 --- a/tools/utils/src/workspace.gen.ts +++ b/tools/utils/src/workspace.gen.ts @@ -1325,11 +1325,6 @@ export const PackageList = [ 'tools/utils', ], }, - { - location: 'packages/frontend/media-capture-playground', - name: '@affine/media-capture-playground', - workspaceDependencies: ['packages/frontend/native'], - }, { location: 'packages/frontend/native', name: '@affine/native', @@ -1549,7 +1544,6 @@ export type PackageName = | '@affine/core' | '@affine/electron-api' | '@affine/i18n' - | '@affine/media-capture-playground' | '@affine/native' | '@affine/routes' | '@affine/templates' diff --git a/tsconfig.json b/tsconfig.json index bfc35da0e0ecb..187223efaf7c1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -146,7 +146,6 @@ { "path": "./packages/frontend/core" }, { "path": "./packages/frontend/electron-api" }, { "path": "./packages/frontend/i18n" }, - { "path": "./packages/frontend/media-capture-playground" }, { "path": "./packages/frontend/native" }, { "path": "./packages/frontend/routes" }, { "path": "./packages/frontend/track" }, diff --git a/yarn.lock b/yarn.lock index be5a6dea6c63c..16569bc54b56a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -688,34 +688,6 @@ __metadata: languageName: unknown linkType: soft -"@affine/media-capture-playground@workspace:packages/frontend/media-capture-playground": - version: 0.0.0-use.local - resolution: "@affine/media-capture-playground@workspace:packages/frontend/media-capture-playground" - dependencies: - "@affine/native": "workspace:*" - "@google/generative-ai": "npm:^0.24.0" - "@types/express": "npm:^5.0.0" - "@types/fs-extra": "npm:^11" - "@types/lodash-es": "npm:^4.17.12" - "@types/multer": "npm:^2.0.0" - "@types/react": "npm:^19.0.1" - "@types/react-dom": "npm:^19.0.2" - chokidar: "npm:^4.0.3" - express: "npm:^5.0.0" - express-rate-limit: "npm:^7.1.5" - fs-extra: "npm:^11.3.0" - lodash-es: "npm:^4.17.23" - multer: "npm:^2.2.0" - react: "npm:^19.2.1" - react-dom: "npm:^19.2.1" - react-markdown: "npm:^10.1.0" - socket.io: "npm:^4.8.3" - socket.io-client: "npm:^4.8.3" - swr: "npm:^2.3.7" - tailwindcss: "npm:^4.1.17" - languageName: unknown - linkType: soft - "@affine/mobile-shared@workspace:*, @affine/mobile-shared@workspace:packages/frontend/apps/mobile-shared": version: 0.0.0-use.local resolution: "@affine/mobile-shared@workspace:packages/frontend/apps/mobile-shared" @@ -5859,13 +5831,6 @@ __metadata: languageName: node linkType: hard -"@google/generative-ai@npm:^0.24.0": - version: 0.24.1 - resolution: "@google/generative-ai@npm:0.24.1" - checksum: 10/992a82e2d4a58d6eb0545254fe05e2da074c08560a9d51fb29c6f2a44db8c66ba44d87bc04e61b3676f5adc9ab94e527f171ee1395edf4ccbead3472fd8efff8 - languageName: node - linkType: hard - "@googleapis/androidpublisher@npm:^36.0.0": version: 36.0.0 resolution: "@googleapis/androidpublisher@npm:36.0.0" @@ -16769,15 +16734,6 @@ __metadata: languageName: node linkType: hard -"@types/estree-jsx@npm:^1.0.0": - version: 1.0.5 - resolution: "@types/estree-jsx@npm:1.0.5" - dependencies: - "@types/estree": "npm:*" - checksum: 10/a028ab0cd7b2950168a05c6a86026eb3a36a54a4adfae57f13911d7b49dffe573d9c2b28421b2d029b49b3d02fcd686611be2622dc3dad6d9791166c083f6008 - languageName: node - linkType: hard - "@types/estree@npm:*, @types/estree@npm:1.0.8, @types/estree@npm:^1.0.0, @types/estree@npm:^1.0.6, @types/estree@npm:^1.0.8": version: 1.0.8 resolution: "@types/estree@npm:1.0.8" @@ -16797,7 +16753,7 @@ __metadata: languageName: node linkType: hard -"@types/express@npm:*, @types/express@npm:^5.0.0, @types/express@npm:^5.0.1": +"@types/express@npm:*, @types/express@npm:^5.0.1": version: 5.0.2 resolution: "@types/express@npm:5.0.2" dependencies: @@ -16817,7 +16773,7 @@ __metadata: languageName: node linkType: hard -"@types/fs-extra@npm:^11, @types/fs-extra@npm:^11.0.4": +"@types/fs-extra@npm:^11.0.4": version: 11.0.4 resolution: "@types/fs-extra@npm:11.0.4" dependencies: @@ -17110,15 +17066,6 @@ __metadata: languageName: node linkType: hard -"@types/multer@npm:^2.0.0": - version: 2.0.0 - resolution: "@types/multer@npm:2.0.0" - dependencies: - "@types/express": "npm:*" - checksum: 10/87ddd5b39fd572b1d98f84908738eed9fef53fc20628bff066d43b3bcf70481f51010a997554ed65c97121e7b8fe44b59a24cd68ccf59c3d7c64bbffb8c5a127 - languageName: node - linkType: hard - "@types/mustache@npm:^4.2.5": version: 4.2.6 resolution: "@types/mustache@npm:4.2.6" @@ -17414,13 +17361,6 @@ __metadata: languageName: node linkType: hard -"@types/unist@npm:^2.0.0": - version: 2.0.11 - resolution: "@types/unist@npm:2.0.11" - checksum: 10/6d436e832bc35c6dde9f056ac515ebf2b3384a1d7f63679d12358766f9b313368077402e9c1126a14d827f10370a5485e628bf61aa91117cf4fc882423191a4e - languageName: node - linkType: hard - "@types/web-bluetooth@npm:^0.0.21": version: 0.0.21 resolution: "@types/web-bluetooth@npm:0.0.21" @@ -20102,13 +20042,6 @@ __metadata: languageName: node linkType: hard -"character-reference-invalid@npm:^2.0.0": - version: 2.0.1 - resolution: "character-reference-invalid@npm:2.0.1" - checksum: 10/98d3b1a52ae510b7329e6ee7f6210df14f1e318c5415975d4c9e7ee0ef4c07875d47c6e74230c64551f12f556b4a8ccc24d9f3691a2aa197019e72a95e9297ee - languageName: node - linkType: hard - "chardet@npm:^0.7.0": version: 0.7.0 resolution: "chardet@npm:0.7.0" @@ -23507,13 +23440,6 @@ __metadata: languageName: node linkType: hard -"estree-util-is-identifier-name@npm:^3.0.0": - version: 3.0.0 - resolution: "estree-util-is-identifier-name@npm:3.0.0" - checksum: 10/cdc9187614fdb269d714eddfdf72c270a79daa9ed51e259bb78527983be6dcc68da6a914ccc41175b662194c67fbd2a1cd262f85fac1eef7111cfddfaf6f77f8 - languageName: node - linkType: hard - "estree-walker@npm:2.0.2, estree-walker@npm:^2.0.2": version: 2.0.2 resolution: "estree-walker@npm:2.0.2" @@ -23668,16 +23594,7 @@ __metadata: languageName: node linkType: hard -"express-rate-limit@npm:^7.1.5": - version: 7.5.0 - resolution: "express-rate-limit@npm:7.5.0" - peerDependencies: - express: ^4.11 || 5 || ^5.0.0-beta.1 - checksum: 10/eff34c83bf586789933a332a339b66649e2cca95c8e977d193aa8bead577d3182ac9f0e9c26f39389287539b8038890ff023f910b54ebb506a26a2ce135b92ca - languageName: node - linkType: hard - -"express@npm:5.2.1, express@npm:^5.0.0, express@npm:^5.0.1, express@npm:^5.1.0": +"express@npm:5.2.1, express@npm:^5.0.1, express@npm:^5.1.0": version: 5.2.1 resolution: "express@npm:5.2.1" dependencies: @@ -24245,7 +24162,7 @@ __metadata: languageName: node linkType: hard -"fs-extra@npm:^11.1.0, fs-extra@npm:^11.1.1, fs-extra@npm:^11.2.0, fs-extra@npm:^11.3.0": +"fs-extra@npm:^11.1.0, fs-extra@npm:^11.1.1, fs-extra@npm:^11.2.0": version: 11.3.0 resolution: "fs-extra@npm:11.3.0" dependencies: @@ -25144,29 +25061,6 @@ __metadata: languageName: node linkType: hard -"hast-util-to-jsx-runtime@npm:^2.0.0": - version: 2.3.6 - resolution: "hast-util-to-jsx-runtime@npm:2.3.6" - dependencies: - "@types/estree": "npm:^1.0.0" - "@types/hast": "npm:^3.0.0" - "@types/unist": "npm:^3.0.0" - comma-separated-tokens: "npm:^2.0.0" - devlop: "npm:^1.0.0" - estree-util-is-identifier-name: "npm:^3.0.0" - hast-util-whitespace: "npm:^3.0.0" - mdast-util-mdx-expression: "npm:^2.0.0" - mdast-util-mdx-jsx: "npm:^3.0.0" - mdast-util-mdxjs-esm: "npm:^2.0.0" - property-information: "npm:^7.0.0" - space-separated-tokens: "npm:^2.0.0" - style-to-js: "npm:^1.0.0" - unist-util-position: "npm:^5.0.0" - vfile-message: "npm:^4.0.0" - checksum: 10/111bd69f482952c7591cb4e1d3face25f1c18849b310a4d6cacc91e2d2cbc965d455fad35c059b8f0cfd762e933b826a7090b6f3098dece08307a6569de8f1d8 - languageName: node - linkType: hard - "hast-util-whitespace@npm:^3.0.0": version: 3.0.0 resolution: "hast-util-whitespace@npm:3.0.0" @@ -25301,13 +25195,6 @@ __metadata: languageName: node linkType: hard -"html-url-attributes@npm:^3.0.0": - version: 3.0.1 - resolution: "html-url-attributes@npm:3.0.1" - checksum: 10/494074c2f730c5c0e517aa1b10111fb36732534a2d2b70427582c4a615472b47da472cf3a17562cc653826d378d20960f2783e0400f4f7cf0c3c2d91c6188d13 - languageName: node - linkType: hard - "html-validate@npm:^9.0.0": version: 9.5.3 resolution: "html-validate@npm:9.5.3" @@ -25779,13 +25666,6 @@ __metadata: languageName: node linkType: hard -"inline-style-parser@npm:0.2.4": - version: 0.2.4 - resolution: "inline-style-parser@npm:0.2.4" - checksum: 10/80814479d1f3c9cbd102f9de4cd6558cf43cc2e48640e81c4371c3634f1e8b6dfeb2f21063cfa31d46cc83e834c20cd59ed9eeed9bfd45ef5bc02187ad941faf - languageName: node - linkType: hard - "input-otp@npm:^1.4.1": version: 1.4.2 resolution: "input-otp@npm:1.4.2" @@ -25904,23 +25784,6 @@ __metadata: languageName: node linkType: hard -"is-alphabetical@npm:^2.0.0": - version: 2.0.1 - resolution: "is-alphabetical@npm:2.0.1" - checksum: 10/56207db8d9de0850f0cd30f4966bf731eb82cedfe496cbc2e97e7c3bacaf66fc54a972d2d08c0d93bb679cb84976a05d24c5ad63de56fabbfc60aadae312edaa - languageName: node - linkType: hard - -"is-alphanumerical@npm:^2.0.0": - version: 2.0.1 - resolution: "is-alphanumerical@npm:2.0.1" - dependencies: - is-alphabetical: "npm:^2.0.0" - is-decimal: "npm:^2.0.0" - checksum: 10/87acc068008d4c9c4e9f5bd5e251041d42e7a50995c77b1499cf6ed248f971aadeddb11f239cabf09f7975ee58cac7a48ffc170b7890076d8d227b24a68663c9 - languageName: node - linkType: hard - "is-arrayish@npm:^0.2.1": version: 0.2.1 resolution: "is-arrayish@npm:0.2.1" @@ -25960,13 +25823,6 @@ __metadata: languageName: node linkType: hard -"is-decimal@npm:^2.0.0": - version: 2.0.1 - resolution: "is-decimal@npm:2.0.1" - checksum: 10/97132de7acdce77caa7b797632970a2ecd649a88e715db0e4dbc00ab0708b5e7574ba5903962c860cd4894a14fd12b100c0c4ac8aed445cf6f55c6cf747a4158 - languageName: node - linkType: hard - "is-docker@npm:^2.0.0, is-docker@npm:^2.1.1": version: 2.2.1 resolution: "is-docker@npm:2.2.1" @@ -26031,13 +25887,6 @@ __metadata: languageName: node linkType: hard -"is-hexadecimal@npm:^2.0.0": - version: 2.0.1 - resolution: "is-hexadecimal@npm:2.0.1" - checksum: 10/66a2ea85994c622858f063f23eda506db29d92b52580709eb6f4c19550552d4dcf3fb81952e52f7cf972097237959e00adc7bb8c9400cd12886e15bf06145321 - languageName: node - linkType: hard - "is-inside-container@npm:^1.0.0": version: 1.0.0 resolution: "is-inside-container@npm:1.0.0" @@ -27955,54 +27804,6 @@ __metadata: languageName: node linkType: hard -"mdast-util-mdx-expression@npm:^2.0.0": - version: 2.0.1 - resolution: "mdast-util-mdx-expression@npm:2.0.1" - dependencies: - "@types/estree-jsx": "npm:^1.0.0" - "@types/hast": "npm:^3.0.0" - "@types/mdast": "npm:^4.0.0" - devlop: "npm:^1.0.0" - mdast-util-from-markdown: "npm:^2.0.0" - mdast-util-to-markdown: "npm:^2.0.0" - checksum: 10/70e860f8ee22c4f478449942750055d649d4380bf43b235d0710af510189d285fb057e401d20b59596d9789f4e270fce08ca892dc849676f9e3383b991d52485 - languageName: node - linkType: hard - -"mdast-util-mdx-jsx@npm:^3.0.0": - version: 3.2.0 - resolution: "mdast-util-mdx-jsx@npm:3.2.0" - dependencies: - "@types/estree-jsx": "npm:^1.0.0" - "@types/hast": "npm:^3.0.0" - "@types/mdast": "npm:^4.0.0" - "@types/unist": "npm:^3.0.0" - ccount: "npm:^2.0.0" - devlop: "npm:^1.1.0" - mdast-util-from-markdown: "npm:^2.0.0" - mdast-util-to-markdown: "npm:^2.0.0" - parse-entities: "npm:^4.0.0" - stringify-entities: "npm:^4.0.0" - unist-util-stringify-position: "npm:^4.0.0" - vfile-message: "npm:^4.0.0" - checksum: 10/62cd650a522e5d72ea6afd6d4a557fc86525b802d097a29a2fbe17d22e7b97c502a580611873e4d685777fe77c6ff8d39fb6e37d026b3acbc86c3b24927f4ad9 - languageName: node - linkType: hard - -"mdast-util-mdxjs-esm@npm:^2.0.0": - version: 2.0.1 - resolution: "mdast-util-mdxjs-esm@npm:2.0.1" - dependencies: - "@types/estree-jsx": "npm:^1.0.0" - "@types/hast": "npm:^3.0.0" - "@types/mdast": "npm:^4.0.0" - devlop: "npm:^1.0.0" - mdast-util-from-markdown: "npm:^2.0.0" - mdast-util-to-markdown: "npm:^2.0.0" - checksum: 10/05474226e163a3f407fccb5780b0d8585a95e548e5da4a85227df43f281b940c7941a9a9d4af1be4f885fe554731647addb057a728e87aa1f503ff9cc72c9163 - languageName: node - linkType: hard - "mdast-util-phrasing@npm:^4.0.0": version: 4.1.0 resolution: "mdast-util-phrasing@npm:4.1.0" @@ -30090,21 +29891,6 @@ __metadata: languageName: node linkType: hard -"parse-entities@npm:^4.0.0": - version: 4.0.2 - resolution: "parse-entities@npm:4.0.2" - dependencies: - "@types/unist": "npm:^2.0.0" - character-entities-legacy: "npm:^3.0.0" - character-reference-invalid: "npm:^2.0.0" - decode-named-character-reference: "npm:^1.0.0" - is-alphanumerical: "npm:^2.0.0" - is-decimal: "npm:^2.0.0" - is-hexadecimal: "npm:^2.0.0" - checksum: 10/b0ce693d0b3d7ed1cea6fe814e6e077c71532695f01178e846269e9a2bc2f7ff34ca4bb8db80b48af0451100f25bb010df6591c9bb6306e4680ccb423d1e4038 - languageName: node - linkType: hard - "parse-filepath@npm:^1.0.2": version: 1.0.2 resolution: "parse-filepath@npm:1.0.2" @@ -31681,28 +31467,6 @@ __metadata: languageName: node linkType: hard -"react-markdown@npm:^10.1.0": - version: 10.1.0 - resolution: "react-markdown@npm:10.1.0" - dependencies: - "@types/hast": "npm:^3.0.0" - "@types/mdast": "npm:^4.0.0" - devlop: "npm:^1.0.0" - hast-util-to-jsx-runtime: "npm:^2.0.0" - html-url-attributes: "npm:^3.0.0" - mdast-util-to-hast: "npm:^13.0.0" - remark-parse: "npm:^11.0.0" - remark-rehype: "npm:^11.0.0" - unified: "npm:^11.0.0" - unist-util-visit: "npm:^5.0.0" - vfile: "npm:^6.0.0" - peerDependencies: - "@types/react": ">=18" - react: ">=18" - checksum: 10/886c037d18266e94750abbc00c6096435bccd4da5f2d1727984bcbdf83e9f2a5cc9a6b0eecfe6c30456dc26546fee41c7f6aecbf176e6d7453963df4abefd7df - languageName: node - linkType: hard - "react-paginate@npm:^8.3.0": version: 8.3.0 resolution: "react-paginate@npm:8.3.0" @@ -32217,19 +31981,6 @@ __metadata: languageName: node linkType: hard -"remark-rehype@npm:^11.0.0": - version: 11.1.2 - resolution: "remark-rehype@npm:11.1.2" - dependencies: - "@types/hast": "npm:^3.0.0" - "@types/mdast": "npm:^4.0.0" - mdast-util-to-hast: "npm:^13.0.0" - unified: "npm:^11.0.0" - vfile: "npm:^6.0.0" - checksum: 10/b5374a0bf08398431c92740d0cd9b20aea9df44cee12326820ddcc1b7ee642706604006461ea9799554c347e7caf31e7432132a03b97c508e1f77d29c423bd86 - languageName: node - linkType: hard - "remark-stringify@npm:^11.0.0": version: 11.0.0 resolution: "remark-stringify@npm:11.0.0" @@ -33907,24 +33658,6 @@ __metadata: languageName: node linkType: hard -"style-to-js@npm:^1.0.0": - version: 1.1.16 - resolution: "style-to-js@npm:1.1.16" - dependencies: - style-to-object: "npm:1.0.8" - checksum: 10/a876cc49a29ac90c7723b4d6f002ac6c1ac5ccc6b5bc963d9c607cfc74b15927b704c9324df6f824f576c65689fe4b4ff79caabcd44a13d8a02641f721f1b316 - languageName: node - linkType: hard - -"style-to-object@npm:1.0.8": - version: 1.0.8 - resolution: "style-to-object@npm:1.0.8" - dependencies: - inline-style-parser: "npm:0.2.4" - checksum: 10/530b067325e3119bfaf75bdbe25cc86b02b559db00d881a74b98a2d5bb10ac953d1b455ed90c825963cf3b4bdaa1bda45f406d78d987391434b8d8ab3835df4e - languageName: node - linkType: hard - "stylehacks@npm:^7.0.5": version: 7.0.5 resolution: "stylehacks@npm:7.0.5"