file: add the compression-aware identification CLI

Aliases/file

@@ -0,0 +1 @@
+../Formula/file-formula.rb

Formula/bzip2.rb

@@ -8,8 +8,12 @@ class Bzip2 < Formula
   url "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz"
   sha256 "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269"
   license "bzip2-1.0.6"
+  revision 1
 
-  skip_clean "bin/bzip2"
+  depends_on "pkgconf" => :test
+
+  skip_clean "bin/bzip2", "lib/libbz2.a"
+  link_overwrite "include/bzlib.h"
 
   def install
     kandelo_require_arch!("wasm32")
@@ -18,14 +22,137 @@ def install
         "CC=#{kandelo_cc}",
         "AR=#{kandelo_ar}",
         "RANLIB=#{kandelo_ranlib}",
-        "CFLAGS=-Wall -Winline -O2 -D_FILE_OFFSET_BITS=64",
+        "CFLAGS=-Wall -Winline -O2 -fPIC -D_FILE_OFFSET_BITS=64",
         "LDFLAGS=",
-        "bzip2"
+        "libbz2.a", "bzip2"
     end
     kandelo_install_bin(buildpath, "bzip2", "bzip2")
+    lib.install "libbz2.a"
+    include.install "bzlib.h"
+    (lib/"pkgconfig").mkpath
+    (lib/"pkgconfig/bzip2.pc").write <<~EOS
+      prefix=#{opt_prefix}
+      exec_prefix=${prefix}
+      libdir=${exec_prefix}/lib
+      includedir=${prefix}/include
+
+      Name: bzip2
+      Description: Lossless, block-sorting data compression
+      Version: #{version}
+      Libs: -L${libdir} -lbz2
+      Cflags: -I${includedir}
+    EOS
   end
 
   test do
+    assert_path_exists lib/"libbz2.a"
+    assert_path_exists include/"bzlib.h"
+    assert_path_exists lib/"pkgconfig/bzip2.pc"
+
+    source = testpath/"bzip2-smoke.c"
+    wasm = testpath/"bzip2-smoke.wasm"
+    plugin_source = testpath/"bzip2-plugin.c"
+    plugin = testpath/"bzip2-plugin.so"
+    loader_source = testpath/"bzip2-loader.c"
+    loader = testpath/"bzip2-loader.wasm"
+    source.write <<~C
+      #include <bzlib.h>
+      #include <stdio.h>
+      #include <string.h>
+
+      int main(void) {
+        const char input[] = "Kandelo libbz2 round trip";
+        char compressed[128];
+        char output[128];
+        unsigned int compressed_length = sizeof(compressed);
+        unsigned int output_length = sizeof(output);
+
+        if (BZ2_bzBuffToBuffCompress(compressed, &compressed_length,
+              (char *)input, sizeof(input), 9, 0, 30) != BZ_OK) return 1;
+        if (BZ2_bzBuffToBuffDecompress(output, &output_length,
+              compressed, compressed_length, 0, 0) != BZ_OK) return 2;
+        if (output_length != sizeof(input) || memcmp(input, output, sizeof(input)) != 0) return 3;
+        puts("libbz2-ok");
+        return 0;
+      }
+    C
+    plugin_source.write <<~C
+      #include <bzlib.h>
+
+      const char *kandelo_bzip2_version(void) {
+        return BZ2_bzlibVersion();
+      }
+    C
+    loader_source.write <<~C
+      #include <dlfcn.h>
+      #include <stdio.h>
+      #include <stdlib.h>
+      #include <string.h>
+
+      typedef const char *(*version_fn)(void);
+
+      /* Wasm side modules import libc from main; retain the stdio surface used by bzlib.o. */
+      static void retain_libbz2_imports(int argc, char **argv) {
+        char buffer[1] = { 0 };
+        const char *path = argv[0] == NULL ? "" : argv[0];
+        FILE *file;
+
+        if (argc != 0) return;
+        file = fopen(path, "r");
+        if (file != NULL) fclose(file);
+        file = fdopen(0, "r");
+        if (file != NULL) fclose(file);
+        fputc(0, stdout);
+        (void)ferror(stderr);
+        (void)fflush(stdout);
+        (void)fgetc(stdin);
+        (void)ungetc(0, stdin);
+        (void)fread(buffer, 1, sizeof(buffer), stdin);
+        (void)fwrite(buffer, 1, sizeof(buffer), stdout);
+        exit((int)strlen(path));
+      }
+
+      int main(int argc, char **argv) {
+        void *handle;
+        void *allocation;
+        version_fn version;
+
+        retain_libbz2_imports(argc, argv);
+        if (argc != 2) return 2;
+        allocation = calloc(1, 1);
+        if (allocation == NULL) return 5;
+        free(allocation);
+        handle = dlopen(argv[1], RTLD_NOW);
+        if (handle == NULL) {
+          fprintf(stderr, "dlopen: %s\\n", dlerror());
+          return 3;
+        }
+        version = (version_fn)dlsym(handle, "kandelo_bzip2_version");
+        if (version == NULL) {
+          fprintf(stderr, "dlsym: %s\\n", dlerror());
+          return 4;
+        }
+        printf("bzip2-side-module %s ok\\n", version());
+        return 0;
+      }
+    C
+    kandelo_wasm_build do
+      ENV["PKG_CONFIG_LIBDIR"] = (lib/"pkgconfig").to_s
+      ENV.delete("PKG_CONFIG_PATH")
+      ENV.delete("PKG_CONFIG_SYSROOT_DIR")
+      pkgconf = formula_opt_bin("pkgconf")/"pkg-config"
+      assert_equal opt_prefix.to_s,
+        Utils.safe_popen_read(pkgconf, "--variable=prefix", "bzip2").strip
+      flags = Utils.safe_popen_read(pkgconf, "--static", "--cflags", "--libs", "bzip2").split
+      assert_includes flags, "-lbz2"
+      system kandelo_cc, source, *flags, "-o", wasm
+      system kandelo_cc, "-shared", "-fPIC", plugin_source, *flags, "-o", plugin
+      system kandelo_cc, loader_source, "-ldl", "-Wl,--export-all", "-o", loader
+    end
+    assert_equal "libbz2-ok\n", kandelo_run_wasm(wasm, [])
+    assert_equal "bzip2-side-module #{version}, 13-Jul-2019 ok\n",
+      kandelo_run_wasm(loader, [plugin])
+
     input = "Kandelo bzip2 round trip\n".b
     compressed = kandelo_run_wasm(bin/"bzip2", ["-c"], stdin: input)
     assert_equal input, kandelo_run_wasm(bin/"bzip2", ["-dc"], stdin: compressed).b

Formula/file-formula.rb

@@ -0,0 +1,139 @@
+require_relative "../Kandelo/formula_support/kandelo_formula_support"
+
+# File is a reserved Ruby class name.
+class FileFormula < Formula
+  include KandeloFormulaSupport
+
+  desc "Command-line file type identification tool for Kandelo"
+  homepage "https://www.darwinsys.com/file/"
+  url "https://astron.com/pub/file/file-5.45.tar.gz"
+  sha256 "fc97f51029bb0e2c9f4e3bffefdaf678f0e039ee872b9de5c002a6d09c784d82"
+  license "BSD-2-Clause-Darwin"
+
+  depends_on "pkgconf" => :build
+  depends_on "automattic/kandelo-homebrew/bzip2"
+  depends_on "automattic/kandelo-homebrew/libmagic"
+  depends_on "automattic/kandelo-homebrew/xz"
+  depends_on "automattic/kandelo-homebrew/zlib"
+
+  skip_clean "bin/file"
+
+  def install
+    kandelo_require_arch!("wasm32")
+    bzip2 = formula_opt_prefix("automattic/kandelo-homebrew/bzip2")
+    libmagic = formula_opt_prefix("automattic/kandelo-homebrew/libmagic")
+    xz = formula_opt_prefix("automattic/kandelo-homebrew/xz")
+    zlib = formula_opt_prefix("automattic/kandelo-homebrew/zlib")
+    instrumented = buildpath/"src/file.instrumented"
+
+    kandelo_wasm_build do |root|
+      ENV["PKG_CONFIG_LIBDIR"] = [
+        libmagic/"lib/pkgconfig",
+        xz/"lib/pkgconfig",
+        zlib/"lib/pkgconfig",
+      ].join(":")
+      ENV.delete("PKG_CONFIG_PATH")
+      ENV.delete("PKG_CONFIG_SYSROOT_DIR")
+
+      pkgconf = formula_opt_bin("pkgconf")/"pkg-config"
+      libmagic_version = Utils.safe_popen_read(pkgconf, "--modversion", "libmagic").strip
+      odie "file #{version} requires libmagic #{version}, found #{libmagic_version}" if libmagic_version != version.to_s
+      compile_flags = Utils.safe_popen_read(pkgconf, "--cflags", "libmagic").split
+      link_flags = Utils.safe_popen_read(pkgconf, "--static", "--libs", "libmagic").split
+      %W[-lmagic -llzma -L#{bzip2}/lib -lbz2 -lz -lm].each do |flag|
+        odie "libmagic static interface omits #{flag}" unless link_flags.include?(flag)
+      end
+
+      ENV["CPPFLAGS"] = compile_flags.join(" ")
+      system kandelo_configure, *kandelo_std_configure_args,
+        "--disable-shared",
+        "--enable-static",
+        "--enable-fsect-man5",
+        "--disable-zlib",
+        "--disable-bzlib",
+        "--disable-xzlib",
+        "--disable-zstdlib",
+        "--disable-lzlib",
+        "--disable-libseccomp",
+        "--disable-silent-rules"
+
+      system "make", "-C", "src", "magic.h"
+
+      # Upstream's file target hardcodes the sibling libmagic.la. Build only
+      # the CLI and link it against the installed tap library contract.
+      system "make", "-C", "src", "file",
+        "MAGIC=#{libmagic}/share/misc/magic",
+        "file_DEPENDENCIES=",
+        "file_LDADD=#{link_flags.join(" ")}"
+      system "#{root}/scripts/run-wasm-fork-instrument.sh",
+        buildpath/"src/file", "-o", instrumented
+
+      # The CLI owns file(1), with the installed libmagic opt path documented
+      # as its default database. libmagic owns the library manuals.
+      system "make", "-C", "doc", "file.1",
+        "MAGIC=#{libmagic}/share/misc/magic"
+    end
+
+    kandelo_install_bin(buildpath/"src", "file.instrumented", "file")
+    man1.install buildpath/"doc/file.1"
+  end
+
+  test do
+    assert_path_exists man1/"file.1"
+
+    workspace = testpath/"workspace"
+    workspace.mkpath
+    (workspace/"module.wasm").binwrite("\0asm\x01\0\0\0")
+    (workspace/"document.pdf").write("%PDF-1.7\n")
+    (workspace/"plain.txt").write("Kandelo file utility test payload.\n")
+    (workspace/"plain-link").make_symlink("plain.txt")
+
+    # These are the same ASCII payload encoded as zlib, bzip2, xz, and gzip streams.
+    (workspace/"payload.z").binwrite([
+      "789cf34ecc4b49cdc95748cbcc4955282dc9ccc92ca95448cecf2d284a2d2e" \
+      "4e4d512848acccc94f4cd1e3020047220f4a",
+    ].pack("H*"))
+    (workspace/"payload.bz2").binwrite([
+      "425a6839314159265359ee7e1d9a00000355800010400100082f27de20200022" \
+      "80d190da9885309a680d312b0ba3a62d91a230ac8d90e4d240bcc498dc29c6" \
+      "cb89f177245385090ee7e1d9a0",
+    ].pack("H*"))
+    (workspace/"payload.xz").binwrite([
+      "fd377a585a000004e6d6b44604c02d29210116000000000000000000727c299" \
+      "a0100284b616e64656c6f2066696c65207574696c69747920636f6d70726573" \
+      "736564207061796c6f61642e0a00000000a6f1317d726e0bb9000149290bd9" \
+      "8f431fb6f37d010000000004595a",
+    ].pack("H*"))
+    (workspace/"payload.gz").binwrite([
+      "1f8b0800000000000003f34ecc4b49cdc95748cbcc4955282dc9ccc92ca9544" \
+      "8cecf2d284a2d2e4e4d512848acccc94f4cd1e3020006c327d729000000",
+    ].pack("H*"))
+
+    env = { "KERNEL_CWD" => workspace }
+    version_output = kandelo_run_wasm(bin/"file", ["--version"])
+    assert_match(/^file(?:\.wasm)?-5\.45$/, version_output)
+    assert_match(%r{magic file from .*/opt/libmagic/share/misc/magic$}, version_output)
+
+    assert_match(/^WebAssembly \(wasm\) binary module/,
+      kandelo_run_wasm(bin/"file", ["-b", "module.wasm"], env: env))
+    assert_equal "application/pdf\n",
+      kandelo_run_wasm(bin/"file", ["-b", "--mime-type", "document.pdf"], env: env)
+    assert_match(/^PDF document/,
+      kandelo_run_wasm(bin/"file", ["-b", "-"], stdin: "%PDF-1.7\n"))
+
+    assert_equal "symbolic link to plain.txt\n",
+      kandelo_run_wasm(bin/"file", ["-b", "plain-link"], env: env)
+    assert_match(/^ASCII text/,
+      kandelo_run_wasm(bin/"file", ["-L", "-b", "plain-link"], env: env))
+
+    {
+      "payload.z"   => "zlib compressed data",
+      "payload.bz2" => "bzip2 compressed data",
+      "payload.xz"  => "XZ compressed data",
+      "payload.gz"  => "gzip compressed data",
+    }.each do |path, compression|
+      output = kandelo_run_wasm(bin/"file", ["-z", "-b", path], env: env)
+      assert_match(/^ASCII text \(#{Regexp.escape(compression)}/, output)
+    end
+  end
+end