From ce2df237ce313d8ae49b8ef4976d0e06dcc32e43 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 01:24:43 -0400 Subject: [PATCH 1/6] compression: expose static library interfaces Install and exercise the upstream libbz2 and liblzma consumer surfaces so dependent Kandelo formulae can link built-in compression support instead of spawning fallback tools. --- Formula/bzip2.rb | 38 ++++++++++++++++++++++++++++-- Formula/xz.rb | 60 +++++++++++++++++++++++++++++++++++++----------- README.md | 2 +- 3 files changed, 83 insertions(+), 17 deletions(-) diff --git a/Formula/bzip2.rb b/Formula/bzip2.rb index 17ece4e..c54cd81 100644 --- a/Formula/bzip2.rb +++ b/Formula/bzip2.rb @@ -9,7 +9,8 @@ class Bzip2 < Formula sha256 "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269" license "bzip2-1.0.6" - skip_clean "bin/bzip2" + skip_clean "bin/bzip2", "lib/libbz2.a" + link_overwrite "include/bzlib.h" def install kandelo_require_arch!("wasm32") @@ -20,12 +21,45 @@ def install "RANLIB=#{kandelo_ranlib}", "CFLAGS=-Wall -Winline -O2 -D_FILE_OFFSET_BITS=64", "LDFLAGS=", - "bzip2" + "libbz2.a", "bzip2" end kandelo_install_bin(buildpath, "bzip2", "bzip2") + lib.install "libbz2.a" + include.install "bzlib.h" end test do + assert_path_exists lib/"libbz2.a" + assert_path_exists include/"bzlib.h" + + source = testpath/"bzip2-smoke.c" + wasm = testpath/"bzip2-smoke.wasm" + source.write <<~C + #include + #include + #include + + int main(void) { + const char input[] = "Kandelo libbz2 round trip"; + char compressed[128]; + char output[128]; + unsigned int compressed_length = sizeof(compressed); + unsigned int output_length = sizeof(output); + + if (BZ2_bzBuffToBuffCompress(compressed, &compressed_length, + (char *)input, sizeof(input), 9, 0, 30) != BZ_OK) return 1; + if (BZ2_bzBuffToBuffDecompress(output, &output_length, + compressed, compressed_length, 0, 0) != BZ_OK) return 2; + if (output_length != sizeof(input) || memcmp(input, output, sizeof(input)) != 0) return 3; + puts("libbz2-ok"); + return 0; + } + C + kandelo_wasm_build do + system kandelo_cc, source, "-I#{include}", "-L#{lib}", "-lbz2", "-o", wasm + end + assert_equal "libbz2-ok\n", kandelo_run_wasm(wasm, []) + input = "Kandelo bzip2 round trip\n".b compressed = kandelo_run_wasm(bin/"bzip2", ["-c"], stdin: input) assert_equal input, kandelo_run_wasm(bin/"bzip2", ["-dc"], stdin: compressed).b diff --git a/Formula/xz.rb b/Formula/xz.rb index ff5605d..7d080c5 100644 --- a/Formula/xz.rb +++ b/Formula/xz.rb @@ -11,12 +11,17 @@ class Xz < Formula sha256 "a9db3bb3d64e248a0fae963f8fb6ba851a26ba1822e504dc0efd18a80c626caf" license all_of: ["GPL-2.0-or-later", "LGPL-2.1-or-later", "0BSD"] - skip_clean "bin/xz" - patch :DATA + skip_clean "bin", "lib/liblzma.a" def install kandelo_require_arch!("wasm32") + # Upstream treats every Wasm target as unable to provide sigprocmask(2), + # but Kandelo exposes the POSIX signal-mask API used by this code path. + inreplace "src/common/mythread.h", + "#if !(defined(_WIN32) && !defined(__CYGWIN__)) && !defined(__wasm__)", + "#if !(defined(_WIN32) && !defined(__CYGWIN__))" + kandelo_wasm_build do ENV["ac_cv_func_closedir_void"] = "no" ENV["ac_cv_func_malloc_0_nonnull"] = "yes" @@ -41,25 +46,52 @@ def install "--disable-lzmainfo", "--enable-sandbox=no" system "make" + system "make", "install" end - kandelo_install_bin(buildpath/"src/xz", "xz", "xz") + rm lib/"liblzma.la" if (lib/"liblzma.la").exist? end test do + assert_path_exists lib/"liblzma.a" + assert_path_exists include/"lzma.h" + assert_path_exists include/"lzma/base.h" + assert_path_exists lib/"pkgconfig/liblzma.pc" + + source = testpath/"liblzma-smoke.c" + wasm = testpath/"liblzma-smoke.wasm" + source.write <<~C + #include + #include + #include + + int main(void) { + const uint8_t input[] = "Kandelo liblzma round trip"; + uint8_t compressed[256]; + uint8_t output[128]; + size_t compressed_position = 0; + size_t input_position = 0; + size_t output_position = 0; + uint64_t memory_limit = UINT64_MAX; + + if (lzma_easy_buffer_encode(6, LZMA_CHECK_CRC64, NULL, + input, sizeof(input), compressed, &compressed_position, + sizeof(compressed)) != LZMA_OK) return 1; + if (lzma_stream_buffer_decode(&memory_limit, 0, NULL, + compressed, &input_position, compressed_position, + output, &output_position, sizeof(output)) != LZMA_OK) return 2; + if (output_position != sizeof(input) || memcmp(input, output, sizeof(input)) != 0) return 3; + puts("liblzma-ok"); + return 0; + } + C + kandelo_wasm_build do + system kandelo_cc, source, "-I#{include}", "-L#{lib}", "-llzma", "-o", wasm + end + assert_equal "liblzma-ok\n", kandelo_run_wasm(wasm, []) + input = "Kandelo xz round trip\n".b compressed = kandelo_run_wasm(bin/"xz", ["-c"], stdin: input) assert_equal input, kandelo_run_wasm(bin/"xz", ["-dc"], stdin: compressed).b end end - -__END__ -diff --git a/src/common/mythread.h b/src/common/mythread.h -index 1f17812..a34e02c 100644 ---- a/src/common/mythread.h -+++ b/src/common/mythread.h -@@ -81,3 +81,3 @@ --#if !(defined(_WIN32) && !defined(__CYGWIN__)) && !defined(__wasm__) -+#if !(defined(_WIN32) && !defined(__CYGWIN__)) - // Use sigprocmask() to set the signal mask in single-threaded programs. - #include diff --git a/README.md b/README.md index 2fd277f..372b036 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Current migration controls and pilots include: - `hello`, the original publication control; - `zlib` and `ruby`, the first dependency and heavy-runtime Formulae; -- `sqlite`, `bzip2`, and `xz`, the dependency-first source-build pilot; +- `sqlite` and the `bzip2`/`xz` compression tools and static libraries, the dependency-first source-build pilot; - `openssl`, the first dependency-root library migration; - `libpng` and `libxml2`, zlib-backed dependency-root libraries; - `libcxx`, the LLVM C++ standard library, ABI runtime, and bundled unwinder; From 9df7f2631f87d40adb12736eadba61f6c35604c8 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 02:36:59 -0400 Subject: [PATCH 2/6] formulae: expose complete compression library interfaces --- Formula/bzip2.rb | 27 +++++++++++++++++- Formula/xz.rb | 72 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 79 insertions(+), 20 deletions(-) diff --git a/Formula/bzip2.rb b/Formula/bzip2.rb index c54cd81..3897710 100644 --- a/Formula/bzip2.rb +++ b/Formula/bzip2.rb @@ -8,6 +8,9 @@ class Bzip2 < Formula url "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz" sha256 "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269" license "bzip2-1.0.6" + revision 1 + + depends_on "pkgconf" => :test skip_clean "bin/bzip2", "lib/libbz2.a" link_overwrite "include/bzlib.h" @@ -26,11 +29,25 @@ def install kandelo_install_bin(buildpath, "bzip2", "bzip2") lib.install "libbz2.a" include.install "bzlib.h" + (lib/"pkgconfig").mkpath + (lib/"pkgconfig/bzip2.pc").write <<~EOS + prefix=#{opt_prefix} + exec_prefix=${prefix} + libdir=${exec_prefix}/lib + includedir=${prefix}/include + + Name: bzip2 + Description: Lossless, block-sorting data compression + Version: #{version} + Libs: -L${libdir} -lbz2 + Cflags: -I${includedir} + EOS end test do assert_path_exists lib/"libbz2.a" assert_path_exists include/"bzlib.h" + assert_path_exists lib/"pkgconfig/bzip2.pc" source = testpath/"bzip2-smoke.c" wasm = testpath/"bzip2-smoke.wasm" @@ -56,7 +73,15 @@ def install } C kandelo_wasm_build do - system kandelo_cc, source, "-I#{include}", "-L#{lib}", "-lbz2", "-o", wasm + ENV["PKG_CONFIG_LIBDIR"] = (lib/"pkgconfig").to_s + ENV.delete("PKG_CONFIG_PATH") + ENV.delete("PKG_CONFIG_SYSROOT_DIR") + pkgconf = formula_opt_bin("pkgconf")/"pkg-config" + assert_equal opt_prefix.to_s, + Utils.safe_popen_read(pkgconf, "--variable=prefix", "bzip2").strip + flags = Utils.safe_popen_read(pkgconf, "--static", "--cflags", "--libs", "bzip2").split + assert_includes flags, "-lbz2" + system kandelo_cc, source, *flags, "-o", wasm end assert_equal "libbz2-ok\n", kandelo_run_wasm(wasm, []) diff --git a/Formula/xz.rb b/Formula/xz.rb index 7d080c5..3901281 100644 --- a/Formula/xz.rb +++ b/Formula/xz.rb @@ -6,22 +6,20 @@ class Xz < Formula desc "General-purpose data compression tools for Kandelo" homepage "https://tukaani.org/xz/" url "https://tukaani.org/xz/xz-5.6.2.tar.xz" - mirror "http://tukaani.org/xz/xz-5.6.2.tar.xz" mirror "https://github.com/tukaani-project/xz/releases/download/v5.6.2/xz-5.6.2.tar.xz" + mirror "https://downloads.sourceforge.net/project/lzmautils/xz-5.6.2.tar.xz" + mirror "http://downloads.sourceforge.net/project/lzmautils/xz-5.6.2.tar.xz" sha256 "a9db3bb3d64e248a0fae963f8fb6ba851a26ba1822e504dc0efd18a80c626caf" license all_of: ["GPL-2.0-or-later", "LGPL-2.1-or-later", "0BSD"] + revision 1 + + depends_on "pkgconf" => :test skip_clean "bin", "lib/liblzma.a" def install kandelo_require_arch!("wasm32") - # Upstream treats every Wasm target as unable to provide sigprocmask(2), - # but Kandelo exposes the POSIX signal-mask API used by this code path. - inreplace "src/common/mythread.h", - "#if !(defined(_WIN32) && !defined(__CYGWIN__)) && !defined(__wasm__)", - "#if !(defined(_WIN32) && !defined(__CYGWIN__))" - kandelo_wasm_build do ENV["ac_cv_func_closedir_void"] = "no" ENV["ac_cv_func_malloc_0_nonnull"] = "yes" @@ -37,7 +35,7 @@ def install system kandelo_configure, *kandelo_std_configure_args, "--disable-nls", - "--disable-threads", + "--enable-threads=posix", "--disable-shared", "--enable-static", "--disable-doc", @@ -50,6 +48,12 @@ def install end rm lib/"liblzma.la" if (lib/"liblzma.la").exist? + inreplace lib/"pkgconfig/liblzma.pc" do |s| + s.gsub!(/^prefix=.*/, "prefix=#{opt_prefix}") + s.gsub!(/^exec_prefix=.*/, "exec_prefix=${prefix}") + s.gsub!(/^libdir=.*/, "libdir=${exec_prefix}/lib") + s.gsub!(/^includedir=.*/, "includedir=${prefix}/include") + end end test do @@ -66,32 +70,62 @@ def install #include int main(void) { - const uint8_t input[] = "Kandelo liblzma round trip"; - uint8_t compressed[256]; - uint8_t output[128]; + static uint8_t input[262144]; + static uint8_t compressed[524288]; + static uint8_t output[sizeof(input)]; + lzma_stream stream = LZMA_STREAM_INIT; + lzma_mt options = { 0 }; size_t compressed_position = 0; size_t input_position = 0; size_t output_position = 0; uint64_t memory_limit = UINT64_MAX; + lzma_ret result; - if (lzma_easy_buffer_encode(6, LZMA_CHECK_CRC64, NULL, - input, sizeof(input), compressed, &compressed_position, - sizeof(compressed)) != LZMA_OK) return 1; + for (size_t i = 0; i < sizeof(input); ++i) input[i] = (uint8_t)(i * 31U + i / 257U); + options.threads = 2; + options.block_size = 65536; + options.preset = 6; + options.check = LZMA_CHECK_CRC64; + if (lzma_stream_encoder_mt(&stream, &options) != LZMA_OK) return 1; + stream.next_in = input; + stream.avail_in = sizeof(input); + stream.next_out = compressed; + stream.avail_out = sizeof(compressed); + do { + result = lzma_code(&stream, LZMA_FINISH); + } while (result == LZMA_OK); + if (result != LZMA_STREAM_END) return 2; + compressed_position = sizeof(compressed) - stream.avail_out; + lzma_end(&stream); if (lzma_stream_buffer_decode(&memory_limit, 0, NULL, compressed, &input_position, compressed_position, - output, &output_position, sizeof(output)) != LZMA_OK) return 2; - if (output_position != sizeof(input) || memcmp(input, output, sizeof(input)) != 0) return 3; - puts("liblzma-ok"); + output, &output_position, sizeof(output)) != LZMA_OK) return 3; + if (output_position != sizeof(input) || memcmp(input, output, sizeof(input)) != 0) return 4; + puts("liblzma-mt-ok"); return 0; } C kandelo_wasm_build do - system kandelo_cc, source, "-I#{include}", "-L#{lib}", "-llzma", "-o", wasm + ENV["PKG_CONFIG_LIBDIR"] = (lib/"pkgconfig").to_s + ENV.delete("PKG_CONFIG_PATH") + ENV.delete("PKG_CONFIG_SYSROOT_DIR") + pkgconf = formula_opt_bin("pkgconf")/"pkg-config" + assert_equal opt_prefix.to_s, + Utils.safe_popen_read(pkgconf, "--variable=prefix", "liblzma").strip + flags = Utils.safe_popen_read(pkgconf, "--static", "--cflags", "--libs", "liblzma").split + assert_includes flags, "-llzma" + system kandelo_cc, source, *flags, "-o", wasm + end + assert_equal "liblzma-mt-ok\n", kandelo_run_wasm(wasm, []) + + %w[lzcat lzma unlzma unxz xzcat].each do |name| + assert_equal "xz", (bin/name).readlink.to_s end - assert_equal "liblzma-ok\n", kandelo_run_wasm(wasm, []) + assert_path_exists bin/"xzdec" input = "Kandelo xz round trip\n".b compressed = kandelo_run_wasm(bin/"xz", ["-c"], stdin: input) assert_equal input, kandelo_run_wasm(bin/"xz", ["-dc"], stdin: compressed).b + assert_equal input, kandelo_run_wasm(bin/"xzdec", [], stdin: compressed).b end end From 6dd503b151b03768950aa597cadcd883db65173c Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 16:13:24 -0400 Subject: [PATCH 3/6] compression: keep static libraries side-module linkable Build libbz2 and liblzma objects as PIC so Kandelo shared Wasm consumers can link the published archives. Exercise each archive through pkg-config, a real side-module link, dlopen, and dlsym under the Kandelo runtime; retain liblzma's pthread metadata contract explicitly. --- Formula/bzip2.rb | 70 +++++++++++++++++++++++++++++++++++++++++++++++- Formula/xz.rb | 48 +++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/Formula/bzip2.rb b/Formula/bzip2.rb index 3897710..724a75b 100644 --- a/Formula/bzip2.rb +++ b/Formula/bzip2.rb @@ -22,7 +22,7 @@ def install "CC=#{kandelo_cc}", "AR=#{kandelo_ar}", "RANLIB=#{kandelo_ranlib}", - "CFLAGS=-Wall -Winline -O2 -D_FILE_OFFSET_BITS=64", + "CFLAGS=-Wall -Winline -O2 -fPIC -D_FILE_OFFSET_BITS=64", "LDFLAGS=", "libbz2.a", "bzip2" end @@ -51,6 +51,10 @@ def install source = testpath/"bzip2-smoke.c" wasm = testpath/"bzip2-smoke.wasm" + plugin_source = testpath/"bzip2-plugin.c" + plugin = testpath/"bzip2-plugin.so" + loader_source = testpath/"bzip2-loader.c" + loader = testpath/"bzip2-loader.wasm" source.write <<~C #include #include @@ -72,6 +76,66 @@ def install return 0; } C + plugin_source.write <<~C + #include + + const char *kandelo_bzip2_version(void) { + return BZ2_bzlibVersion(); + } + C + loader_source.write <<~C + #include + #include + #include + #include + + typedef const char *(*version_fn)(void); + + /* Wasm side modules import libc from main; retain the stdio surface used by bzlib.o. */ + static void retain_libbz2_imports(int argc, char **argv) { + char buffer[1] = { 0 }; + const char *path = argv[0] == NULL ? "" : argv[0]; + FILE *file; + + if (argc != 0) return; + file = fopen(path, "r"); + if (file != NULL) fclose(file); + file = fdopen(0, "r"); + if (file != NULL) fclose(file); + fputc(0, stdout); + (void)ferror(stderr); + (void)fflush(stdout); + (void)fgetc(stdin); + (void)ungetc(0, stdin); + (void)fread(buffer, 1, sizeof(buffer), stdin); + (void)fwrite(buffer, 1, sizeof(buffer), stdout); + exit((int)strlen(path)); + } + + int main(int argc, char **argv) { + void *handle; + void *allocation; + version_fn version; + + retain_libbz2_imports(argc, argv); + if (argc != 2) return 2; + allocation = calloc(1, 1); + if (allocation == NULL) return 5; + free(allocation); + handle = dlopen(argv[1], RTLD_NOW); + if (handle == NULL) { + fprintf(stderr, "dlopen: %s\\n", dlerror()); + return 3; + } + version = (version_fn)dlsym(handle, "kandelo_bzip2_version"); + if (version == NULL) { + fprintf(stderr, "dlsym: %s\\n", dlerror()); + return 4; + } + printf("bzip2-side-module %s ok\\n", version()); + return 0; + } + C kandelo_wasm_build do ENV["PKG_CONFIG_LIBDIR"] = (lib/"pkgconfig").to_s ENV.delete("PKG_CONFIG_PATH") @@ -82,8 +146,12 @@ def install flags = Utils.safe_popen_read(pkgconf, "--static", "--cflags", "--libs", "bzip2").split assert_includes flags, "-lbz2" system kandelo_cc, source, *flags, "-o", wasm + system kandelo_cc, "-shared", "-fPIC", plugin_source, *flags, "-o", plugin + system kandelo_cc, loader_source, "-ldl", "-Wl,--export-all", "-o", loader end assert_equal "libbz2-ok\n", kandelo_run_wasm(wasm, []) + assert_equal "bzip2-side-module #{version}, 13-Jul-2019 ok\n", + kandelo_run_wasm(loader, [plugin]) input = "Kandelo bzip2 round trip\n".b compressed = kandelo_run_wasm(bin/"bzip2", ["-c"], stdin: input) diff --git a/Formula/xz.rb b/Formula/xz.rb index 3901281..b5bad45 100644 --- a/Formula/xz.rb +++ b/Formula/xz.rb @@ -21,6 +21,7 @@ def install kandelo_require_arch!("wasm32") kandelo_wasm_build do + ENV["CFLAGS"] = "-O2 -fPIC" ENV["ac_cv_func_closedir_void"] = "no" ENV["ac_cv_func_malloc_0_nonnull"] = "yes" ENV["ac_cv_func_realloc_0_nonnull"] = "yes" @@ -64,6 +65,10 @@ def install source = testpath/"liblzma-smoke.c" wasm = testpath/"liblzma-smoke.wasm" + plugin_source = testpath/"liblzma-plugin.c" + plugin = testpath/"liblzma-plugin.so" + loader_source = testpath/"liblzma-loader.c" + loader = testpath/"liblzma-loader.wasm" source.write <<~C #include #include @@ -105,6 +110,43 @@ def install return 0; } C + plugin_source.write <<~C + #include + + const char *kandelo_liblzma_version(void) { + return lzma_version_string(); + } + C + loader_source.write <<~C + #include + #include + #include + + typedef const char *(*version_fn)(void); + + int main(int argc, char **argv) { + void *handle; + void *allocation; + version_fn version; + + if (argc != 2) return 2; + allocation = calloc(1, 1); + if (allocation == NULL) return 5; + free(allocation); + handle = dlopen(argv[1], RTLD_NOW); + if (handle == NULL) { + fprintf(stderr, "dlopen: %s\\n", dlerror()); + return 3; + } + version = (version_fn)dlsym(handle, "kandelo_liblzma_version"); + if (version == NULL) { + fprintf(stderr, "dlsym: %s\\n", dlerror()); + return 4; + } + printf("liblzma-side-module %s ok\\n", version()); + return 0; + } + C kandelo_wasm_build do ENV["PKG_CONFIG_LIBDIR"] = (lib/"pkgconfig").to_s ENV.delete("PKG_CONFIG_PATH") @@ -114,9 +156,15 @@ def install Utils.safe_popen_read(pkgconf, "--variable=prefix", "liblzma").strip flags = Utils.safe_popen_read(pkgconf, "--static", "--cflags", "--libs", "liblzma").split assert_includes flags, "-llzma" + assert_includes flags, "-pthread" + assert_includes flags, "-lpthread" system kandelo_cc, source, *flags, "-o", wasm + system kandelo_cc, "-shared", "-fPIC", plugin_source, *flags, "-o", plugin + system kandelo_cc, loader_source, "-ldl", "-Wl,--export-all", "-o", loader end assert_equal "liblzma-mt-ok\n", kandelo_run_wasm(wasm, []) + assert_equal "liblzma-side-module #{version} ok\n", + kandelo_run_wasm(loader, [plugin]) %w[lzcat lzma unlzma unxz xzcat].each do |name| assert_equal "xz", (bin/name).readlink.to_s From c874d7a6d2bb8497f751e477a04419fbbf7b931e Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 01:35:48 -0400 Subject: [PATCH 4/6] libmagic: ship the complete identification database Build the full version-matched magic.mgc with a native FILE_COMPILE helper, link the target library to declared compression dependencies, and validate the fixed-record database under Kandelo. --- Formula/libmagic.rb | 206 ++++++++++++++++++++++++++++++++++++++++++++ README.md | 1 + 2 files changed, 207 insertions(+) create mode 100644 Formula/libmagic.rb diff --git a/Formula/libmagic.rb b/Formula/libmagic.rb new file mode 100644 index 0000000..8c41865 --- /dev/null +++ b/Formula/libmagic.rb @@ -0,0 +1,206 @@ +require_relative "../Kandelo/formula_support/kandelo_formula_support" + +class Libmagic < Formula + include KandeloFormulaSupport + + desc "File type identification library for Kandelo" + homepage "https://www.darwinsys.com/file/" + url "https://astron.com/pub/file/file-5.45.tar.gz" + sha256 "fc97f51029bb0e2c9f4e3bffefdaf678f0e039ee872b9de5c002a6d09c784d82" + license "BSD-2-Clause" + + depends_on "pkgconf" => :test + depends_on "automattic/kandelo-homebrew/bzip2" + depends_on "automattic/kandelo-homebrew/xz" + depends_on "automattic/kandelo-homebrew/zlib" + + skip_clean "lib/libmagic.a" + + def install + kandelo_require_arch!("wasm32") + bzip2 = formula_opt_prefix("automattic/kandelo-homebrew/bzip2") + xz = formula_opt_prefix("automattic/kandelo-homebrew/xz") + zlib = formula_opt_prefix("automattic/kandelo-homebrew/zlib") + jobs = "-j#{ENV.make_jobs}" + + # A version-matched native file(1) compiles architecture-independent + # magic records for the cross build. The compiled format contains fixed + # width records and an endian marker, not host pointers. + host_build = buildpath/"host-build" + host_build.mkpath + cd host_build do + system buildpath/"configure", + "--disable-shared", + "--enable-static", + "--disable-zlib", + "--disable-bzlib", + "--disable-xzlib", + "--disable-zstdlib", + "--disable-lzlib", + "--disable-libseccomp", + "--disable-silent-rules" + system "make", "-C", "src", "magic.h" + system "make", jobs, "-C", "src", "file" + end + host_file = host_build/"src/file" + odie "native FILE_COMPILE helper was not built" unless host_file.executable? + + kandelo_wasm_build do + ENV["CPPFLAGS"] = "-I#{bzip2}/include -I#{xz}/include -I#{zlib}/include" + ENV["LDFLAGS"] = "-L#{bzip2}/lib -L#{xz}/lib -L#{zlib}/lib" + + # AC_CHECK_LIB calls these fully prototyped functions with no arguments, + # which is an invalid Wasm signature even though real consumers link. + ENV["ac_cv_lib_bz2_BZ2_bzCompressInit"] = "yes" + ENV["ac_cv_lib_lzma_lzma_stream_decoder"] = "yes" + ENV["ac_cv_lib_z_gzopen"] = "yes" + + system kandelo_configure, *kandelo_std_configure_args, + "--disable-shared", + "--enable-static", + "--enable-fsect-man5", + "--enable-zlib", + "--enable-bzlib", + "--enable-xzlib", + "--disable-zstdlib", + "--disable-lzlib", + "--disable-libseccomp", + "--disable-silent-rules" + + %w[ZLIBSUPPORT BZLIBSUPPORT XZLIBSUPPORT].each do |feature| + odie "#{feature} was not enabled" unless (buildpath/"config.h").read.include?("#define #{feature} 1") + end + %w[ZSTDLIBSUPPORT LZLIBSUPPORT].each do |feature| + odie "#{feature} was unexpectedly enabled" if (buildpath/"config.h").read.include?("#define #{feature} 1") + end + + system "make", jobs, "FILE_COMPILE=#{host_file}" + + # Embed the stable opt path in libmagic while installing data into the + # versioned keg. Make does not track command-line variable changes, so + # rebuild src explicitly before the normal recursive install. + system "make", "-C", "src", "clean" + system "make", jobs, "-C", "src", "MAGIC=#{opt_prefix}/share/misc/magic" + system "make", "install", "FILE_COMPILE=#{host_file}" + end + + rm_r bin + rm man1/"file.1" + rm lib/"libmagic.la" + + # Upstream's raw private library names omit keg search paths, and its + # src/Makefile links static consumers with libm without recording it. + # liblzma and zlib publish pkg-config metadata; bzip2 1.0.8 does not. + inreplace lib/"pkgconfig/libmagic.pc" do |s| + s.sub!(/^Libs\.private:.*$/, "Libs.private: -L#{bzip2}/lib -lbz2 -lm") + s.sub!(/^Cflags:/, "Requires.private: liblzma zlib\nCflags:") + end + + magic_sources = share/"misc/magic" + magic_sources.install buildpath/"magic/Header", buildpath/"magic/Localstuff" + magic_sources.install buildpath.glob("magic/Magdir/*") + + [man3/"libmagic.3", man5/"magic.5"].each do |manual| + inreplace manual, prefix.to_s, opt_prefix.to_s + end + + source_files = magic_sources.glob("*") + source_lines = source_files.sum { |source| source.each_line.count } + odie "incomplete magic source provenance" if source_files.length != 342 || source_lines != 45_082 + + database = share/"misc/magic.mgc" + database_bytes = File.binread(database) + header = database_bytes.unpack("L<2") + records, remainder = database_bytes.length.divmod(376) + odie "invalid magic database header" if header != [0xF11E041C, 18] + odie "incomplete compiled magic database" if records != 22_642 || !remainder.zero? + end + + test do + assert_path_exists lib/"libmagic.a" + assert_path_exists include/"magic.h" + assert_path_exists lib/"pkgconfig/libmagic.pc" + assert_path_exists share/"misc/magic.mgc" + assert_path_exists man3/"libmagic.3" + assert_path_exists man5/"magic.5" + assert_equal 342, (share/"misc/magic").glob("*").length + assert_equal 45_082, (share/"misc/magic").glob("*").sum { |source| source.each_line.count } + assert_equal 22_642, (share/"misc/magic.mgc").size/376 + archive = File.binread(lib/"libmagic.a") + assert_includes archive, "#{opt_prefix}/share/misc/magic" + refute_includes archive, (prefix/"share/misc/magic").to_s + + source = testpath/"libmagic-smoke.c" + wasm = testpath/"libmagic-smoke.wasm" + instrumented = testpath/"libmagic-smoke.instrumented.wasm" + source.write <<~C + #include + #include + #include + #include + #include + #include + #include + + static int classify(magic_t cookie, const char *label, + const void *data, size_t length, const char *expected) { + const char *result = magic_buffer(cookie, data, length); + if (result == NULL || strstr(result, expected) == NULL) { + fprintf(stderr, "%s: %s\\n", label, result == NULL ? magic_error(cookie) : result); + return -1; + } + printf("%s=%s\\n", label, result); + return 0; + } + + int main(void) { + static const uint8_t wasm_module[] = { 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00 }; + static const uint8_t pdf[] = "%PDF-1.7\\n"; + static const uint8_t text[] = + "Kandelo built-in compression support identifies this ASCII payload.\\n"; + uint8_t compressed[512]; + unsigned long zlib_length = sizeof(compressed); + unsigned int bzip2_length = sizeof(compressed); + size_t xz_length = 0; + magic_t cookie = magic_open(MAGIC_COMPRESS); + + if (cookie == NULL || magic_load(cookie, NULL) != 0) return 1; + if (classify(cookie, "wasm", wasm_module, sizeof(wasm_module), "WebAssembly") != 0) return 2; + if (classify(cookie, "pdf", pdf, sizeof(pdf) - 1, "PDF document") != 0) return 3; + + if (compress2(compressed, &zlib_length, text, sizeof(text) - 1, 9) != Z_OK) return 4; + if (classify(cookie, "zlib", compressed, zlib_length, "ASCII text") != 0) return 5; + + if (BZ2_bzBuffToBuffCompress((char *)compressed, &bzip2_length, + (char *)text, sizeof(text) - 1, 9, 0, 30) != BZ_OK) return 6; + if (classify(cookie, "bzip2", compressed, bzip2_length, "ASCII text") != 0) return 7; + + if (lzma_easy_buffer_encode(6, LZMA_CHECK_CRC64, NULL, + text, sizeof(text) - 1, compressed, &xz_length, sizeof(compressed)) != LZMA_OK) return 8; + if (classify(cookie, "xz", compressed, xz_length, "ASCII text") != 0) return 9; + + magic_close(cookie); + return 0; + } + C + + kandelo_wasm_build do |root| + bzip2 = formula_opt_prefix("automattic/kandelo-homebrew/bzip2") + xz = formula_opt_prefix("automattic/kandelo-homebrew/xz") + zlib = formula_opt_prefix("automattic/kandelo-homebrew/zlib") + ENV["PKG_CONFIG_LIBDIR"] = [lib/"pkgconfig", xz/"lib/pkgconfig", zlib/"lib/pkgconfig"].join(":") + ENV.delete("PKG_CONFIG_PATH") + ENV.delete("PKG_CONFIG_SYSROOT_DIR") + pkgconf = formula_opt_bin("pkgconf")/"pkg-config" + flags = shell_output("#{pkgconf} --static --cflags --libs libmagic").split + %w[-lmagic -llzma -lbz2 -lz -lm].each { |flag| assert_includes flags, flag } + system kandelo_cc, source, *flags, "-I#{bzip2}/include", "-o", wasm + system "#{root}/scripts/run-wasm-fork-instrument.sh", wasm, "-o", instrumented + end + + output = kandelo_run_wasm(instrumented, []) + assert_match(/^wasm=WebAssembly \(wasm\) binary module/, output) + assert_match(/^pdf=PDF document/, output) + %w[zlib bzip2 xz].each { |format| assert_match(/^#{format}=ASCII text/, output) } + end +end diff --git a/README.md b/README.md index 372b036..0d7bbd2 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Current migration controls and pilots include: - `hello`, the original publication control; - `zlib` and `ruby`, the first dependency and heavy-runtime Formulae; - `sqlite` and the `bzip2`/`xz` compression tools and static libraries, the dependency-first source-build pilot; +- `libmagic`, the full file-type database and compression-aware identification library; - `openssl`, the first dependency-root library migration; - `libpng` and `libxml2`, zlib-backed dependency-root libraries; - `libcxx`, the LLVM C++ standard library, ABI runtime, and bundled unwinder; From 991712b768243feebf9d1da27d40ffec7117ffca Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 02:06:51 -0400 Subject: [PATCH 5/6] libmagic: declare complete upstream licensing --- Formula/libmagic.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Formula/libmagic.rb b/Formula/libmagic.rb index 8c41865..5710428 100644 --- a/Formula/libmagic.rb +++ b/Formula/libmagic.rb @@ -7,7 +7,7 @@ class Libmagic < Formula homepage "https://www.darwinsys.com/file/" url "https://astron.com/pub/file/file-5.45.tar.gz" sha256 "fc97f51029bb0e2c9f4e3bffefdaf678f0e039ee872b9de5c002a6d09c784d82" - license "BSD-2-Clause" + license all_of: ["BSD-2-Clause-Darwin", "BSD-2-Clause", :public_domain] depends_on "pkgconf" => :test depends_on "automattic/kandelo-homebrew/bzip2" From 5735ce1816cee62a2928e9f6c963645691f69bd3 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 02:09:16 -0400 Subject: [PATCH 6/6] file: add the compression-aware CLI --- Aliases/file | 1 + Formula/file-formula.rb | 139 ++++++++++++++++++++++++++++++++++++++++ README.md | 4 +- 3 files changed, 143 insertions(+), 1 deletion(-) create mode 120000 Aliases/file create mode 100644 Formula/file-formula.rb diff --git a/Aliases/file b/Aliases/file new file mode 120000 index 0000000..bb915dc --- /dev/null +++ b/Aliases/file @@ -0,0 +1 @@ +../Formula/file-formula.rb \ No newline at end of file diff --git a/Formula/file-formula.rb b/Formula/file-formula.rb new file mode 100644 index 0000000..00f1480 --- /dev/null +++ b/Formula/file-formula.rb @@ -0,0 +1,139 @@ +require_relative "../Kandelo/formula_support/kandelo_formula_support" + +# File is a reserved Ruby class name. +class FileFormula < Formula + include KandeloFormulaSupport + + desc "Command-line file type identification tool for Kandelo" + homepage "https://www.darwinsys.com/file/" + url "https://astron.com/pub/file/file-5.45.tar.gz" + sha256 "fc97f51029bb0e2c9f4e3bffefdaf678f0e039ee872b9de5c002a6d09c784d82" + license "BSD-2-Clause-Darwin" + + depends_on "pkgconf" => :build + depends_on "automattic/kandelo-homebrew/bzip2" + depends_on "automattic/kandelo-homebrew/libmagic" + depends_on "automattic/kandelo-homebrew/xz" + depends_on "automattic/kandelo-homebrew/zlib" + + skip_clean "bin/file" + + def install + kandelo_require_arch!("wasm32") + bzip2 = formula_opt_prefix("automattic/kandelo-homebrew/bzip2") + libmagic = formula_opt_prefix("automattic/kandelo-homebrew/libmagic") + xz = formula_opt_prefix("automattic/kandelo-homebrew/xz") + zlib = formula_opt_prefix("automattic/kandelo-homebrew/zlib") + instrumented = buildpath/"src/file.instrumented" + + kandelo_wasm_build do |root| + ENV["PKG_CONFIG_LIBDIR"] = [ + libmagic/"lib/pkgconfig", + xz/"lib/pkgconfig", + zlib/"lib/pkgconfig", + ].join(":") + ENV.delete("PKG_CONFIG_PATH") + ENV.delete("PKG_CONFIG_SYSROOT_DIR") + + pkgconf = formula_opt_bin("pkgconf")/"pkg-config" + libmagic_version = Utils.safe_popen_read(pkgconf, "--modversion", "libmagic").strip + odie "file #{version} requires libmagic #{version}, found #{libmagic_version}" if libmagic_version != version.to_s + compile_flags = Utils.safe_popen_read(pkgconf, "--cflags", "libmagic").split + link_flags = Utils.safe_popen_read(pkgconf, "--static", "--libs", "libmagic").split + %W[-lmagic -llzma -L#{bzip2}/lib -lbz2 -lz -lm].each do |flag| + odie "libmagic static interface omits #{flag}" unless link_flags.include?(flag) + end + + ENV["CPPFLAGS"] = compile_flags.join(" ") + system kandelo_configure, *kandelo_std_configure_args, + "--disable-shared", + "--enable-static", + "--enable-fsect-man5", + "--disable-zlib", + "--disable-bzlib", + "--disable-xzlib", + "--disable-zstdlib", + "--disable-lzlib", + "--disable-libseccomp", + "--disable-silent-rules" + + system "make", "-C", "src", "magic.h" + + # Upstream's file target hardcodes the sibling libmagic.la. Build only + # the CLI and link it against the installed tap library contract. + system "make", "-C", "src", "file", + "MAGIC=#{libmagic}/share/misc/magic", + "file_DEPENDENCIES=", + "file_LDADD=#{link_flags.join(" ")}" + system "#{root}/scripts/run-wasm-fork-instrument.sh", + buildpath/"src/file", "-o", instrumented + + # The CLI owns file(1), with the installed libmagic opt path documented + # as its default database. libmagic owns the library manuals. + system "make", "-C", "doc", "file.1", + "MAGIC=#{libmagic}/share/misc/magic" + end + + kandelo_install_bin(buildpath/"src", "file.instrumented", "file") + man1.install buildpath/"doc/file.1" + end + + test do + assert_path_exists man1/"file.1" + + workspace = testpath/"workspace" + workspace.mkpath + (workspace/"module.wasm").binwrite("\0asm\x01\0\0\0") + (workspace/"document.pdf").write("%PDF-1.7\n") + (workspace/"plain.txt").write("Kandelo file utility test payload.\n") + (workspace/"plain-link").make_symlink("plain.txt") + + # These are the same ASCII payload encoded as zlib, bzip2, xz, and gzip streams. + (workspace/"payload.z").binwrite([ + "789cf34ecc4b49cdc95748cbcc4955282dc9ccc92ca95448cecf2d284a2d2e" \ + "4e4d512848acccc94f4cd1e3020047220f4a", + ].pack("H*")) + (workspace/"payload.bz2").binwrite([ + "425a6839314159265359ee7e1d9a00000355800010400100082f27de20200022" \ + "80d190da9885309a680d312b0ba3a62d91a230ac8d90e4d240bcc498dc29c6" \ + "cb89f177245385090ee7e1d9a0", + ].pack("H*")) + (workspace/"payload.xz").binwrite([ + "fd377a585a000004e6d6b44604c02d29210116000000000000000000727c299" \ + "a0100284b616e64656c6f2066696c65207574696c69747920636f6d70726573" \ + "736564207061796c6f61642e0a00000000a6f1317d726e0bb9000149290bd9" \ + "8f431fb6f37d010000000004595a", + ].pack("H*")) + (workspace/"payload.gz").binwrite([ + "1f8b0800000000000003f34ecc4b49cdc95748cbcc4955282dc9ccc92ca9544" \ + "8cecf2d284a2d2e4e4d512848acccc94f4cd1e3020006c327d729000000", + ].pack("H*")) + + env = { "KERNEL_CWD" => workspace } + version_output = kandelo_run_wasm(bin/"file", ["--version"]) + assert_match(/^file(?:\.wasm)?-5\.45$/, version_output) + assert_match(%r{magic file from .*/opt/libmagic/share/misc/magic$}, version_output) + + assert_match(/^WebAssembly \(wasm\) binary module/, + kandelo_run_wasm(bin/"file", ["-b", "module.wasm"], env: env)) + assert_equal "application/pdf\n", + kandelo_run_wasm(bin/"file", ["-b", "--mime-type", "document.pdf"], env: env) + assert_match(/^PDF document/, + kandelo_run_wasm(bin/"file", ["-b", "-"], stdin: "%PDF-1.7\n")) + + assert_equal "symbolic link to plain.txt\n", + kandelo_run_wasm(bin/"file", ["-b", "plain-link"], env: env) + assert_match(/^ASCII text/, + kandelo_run_wasm(bin/"file", ["-L", "-b", "plain-link"], env: env)) + + { + "payload.z" => "zlib compressed data", + "payload.bz2" => "bzip2 compressed data", + "payload.xz" => "XZ compressed data", + "payload.gz" => "gzip compressed data", + }.each do |path, compression| + output = kandelo_run_wasm(bin/"file", ["-z", "-b", path], env: env) + assert_match(/^ASCII text \(#{Regexp.escape(compression)}/, output) + end + end +end diff --git a/README.md b/README.md index 0d7bbd2..bb8891a 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,9 @@ Current migration controls and pilots include: - `curl`, the matching command-line transfer client linked against the tap library; - `sed`, the GNU stream-editing CLI used by shell and build workflows; - `gzip`, the GNU compression CLI with native gunzip and zcat aliases; -- `grep`, GNU regular-expression and file search for the leaf CLI wave. +- `grep`, GNU regular-expression and file search for the leaf CLI wave; +- `file`, compression-aware file type identification backed by the complete + `libmagic` database. The SDK is not yet a Homebrew dependency. Trusted builds supply an `HOMEBREW_KANDELO_ROOT` checkout containing the SDK, sysroot, kernel, and Node