Native port feature parity: haploid calling, select_variant_types, configurable extract geometry, CoreML strides, methylation build

.gitignore

@@ -40,3 +40,6 @@ testdata/reference/large/
 .DS_Store
 validation/work/
 validation/output/
+
+# Local Claude Code settings (do not commit)
+.claude/

CLAUDE.md

@@ -23,7 +23,7 @@ Running log: `PORT_LOG.md`.
 
 ## Working rules
 
-1. **Test before commit.** Every commit must leave the build green: `swift build && swift test` in `tools/conversion/` for Phase 0 work; `cmake --build build && ctest -V` for Phases 1+.
+1. **Test before commit.** Every commit must leave the build green: `swift build && swift test` in `tools/conversion/` for Phase 0 work; `cmake --build build-macos && ctest -V` for Phases 1+. (Use `build-macos`, not `build` — a `build/` dir collides with the Bazel `BUILD` file on macOS's case-insensitive filesystem.)
 2. **Never degrade scientific precision.** F1 thresholds are gates, not goals. If we slip below, we fix the root cause — we do not lower the bar.
 3. **Never bypass an error.** No `--no-verify`, no swallowed exceptions, no commenting out of failing tests. Diagnose the root cause.
 4. **Document every critical decision** in `PORT_LOG.md` with date, context, alternatives considered, and rationale.

cmake/deps.cmake

@@ -34,9 +34,18 @@ set_target_properties(htslib::htslib PROPERTIES
   IMPORTED_LOCATION "${HTSLIB_LIB}"
   INTERFACE_INCLUDE_DIRECTORIES "${HTSLIB_PREFIX}/include"
 )
+# Resolve libdeflate via Homebrew rather than a hardcoded /opt/homebrew path,
+# so the build works under a non-default Homebrew prefix or keg-only layout.
+execute_process(
+  COMMAND ${BREW_EXECUTABLE} --prefix libdeflate
+  OUTPUT_VARIABLE LIBDEFLATE_PREFIX
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+find_library(LIBDEFLATE_LIB NAMES libdeflate.a deflate
+  PATHS "${LIBDEFLATE_PREFIX}/lib" REQUIRED)
 target_link_libraries(htslib::htslib INTERFACE
   "-framework CoreFoundation"
-  /opt/homebrew/lib/libdeflate.a
+  "${LIBDEFLATE_LIB}"
   z bz2 lzma curl
 )
 message(STATUS "htslib: ${HTSLIB_LIB}")
@@ -83,10 +92,10 @@ FetchContent_Declare(
   URL      https://github.com/mengyao/Complete-Striped-Smith-Waterman-Library/archive/v1.2.5.tar.gz
   URL_HASH SHA256=b294c0cb6f0f3d578db11b4112a88b20583b9d4190b0a9cf04d83bb6a8704d9a
 )
-FetchContent_GetProperties(libssw)
-if(NOT libssw_POPULATED)
-  FetchContent_Populate(libssw)
-endif()
+# libssw ships no CMakeLists, so MakeAvailable just populates ${libssw_SOURCE_DIR}
+# (no add_subdirectory) — and avoids the deprecated single-arg
+# FetchContent_Populate(libssw) call.
+FetchContent_MakeAvailable(libssw)
 
 # OVERLAY: replace the vendored sse2neon.h (Ratcliff/NVIDIA early version,
 # 8798 lines, missing fixes) with the modern DLTcollab fork (11744 lines,

cmake/protos.cmake

@@ -46,7 +46,7 @@ endfunction()
 # 1. nucleus protos (self-contained, no TF imports)
 # ---------------------------------------------------------------------------
 set(NUCLEUS_PROTO_ROOT "${CMAKE_SOURCE_DIR}/third_party/nucleus/protos")
-file(GLOB NUCLEUS_PROTOS "${NUCLEUS_PROTO_ROOT}/*.proto")
+file(GLOB NUCLEUS_PROTOS CONFIGURE_DEPENDS "${NUCLEUS_PROTO_ROOT}/*.proto")
 
 set(NUCLEUS_PB_SRCS)
 foreach(_p ${NUCLEUS_PROTOS})
@@ -72,7 +72,7 @@ add_library(proto_tf_example ALIAS proto_nucleus)
 # 2. deepvariant protos
 # ---------------------------------------------------------------------------
 set(DV_PROTO_ROOT "${CMAKE_SOURCE_DIR}/deepvariant/protos")
-file(GLOB DV_PROTOS "${DV_PROTO_ROOT}/*.proto")
+file(GLOB DV_PROTOS CONFIGURE_DEPENDS "${DV_PROTO_ROOT}/*.proto")
 
 set(DV_PB_SRCS)
 foreach(_p ${DV_PROTOS})

deepvariant/CMakeLists.txt

@@ -36,7 +36,7 @@ dv_library(dv_utils
 # ---------------------------------------------------------------------------
 # dv_channels — all pileup channel implementations
 # ---------------------------------------------------------------------------
-file(GLOB DV_CHANNEL_SRCS "channels/*.cc")
+file(GLOB DV_CHANNEL_SRCS CONFIGURE_DEPENDS "channels/*.cc")
 list(FILTER DV_CHANNEL_SRCS EXCLUDE REGEX "_test\\.cc$")
 dv_library(dv_channels
   SRCS ${DV_CHANNEL_SRCS}

deepvariant/native/CMakeLists.txt

@@ -82,8 +82,13 @@ target_include_directories(dv_metal_conv_kahan PUBLIC
   "${CMAKE_SOURCE_DIR}"
   "${ABSL_PREFIX}/include"
 )
+# NB: dv_metal_conv_kahan only needs ConvDesc from metal_conv_serial.h (a
+# compile-time include available via target_include_directories), not any
+# compiled MetalConvSerial symbol — so there is no link edge back to
+# dv_metal_conv_serial. Adding one would create a static-library cycle
+# (serial -> kahan is the only real edge, via MetalConvKahan::Create) that
+# Apple ld reports as "ignoring duplicate libraries".
 target_link_libraries(dv_metal_conv_kahan PUBLIC
-  dv_metal_conv_serial   # reuse ConvDesc
   absl::log
   "-framework Metal"
   "-framework Foundation"
@@ -270,6 +275,24 @@ set(ME_COMPILE_OPTS
   "-include${CMAKE_SOURCE_DIR}/cmake/tf_stubs/tensorflow/core/platform/tf_compat.h"
 )
 
+# ---------------------------------------------------------------------------
+# dv_haploid_regions — sex-chromosome haploid-calling helpers shared by
+# make_examples (gVCF) and postprocess. The header is dependency-free and
+# header-only; only LoadParRegions lives here because it uses zlib to read
+# plaintext-or-gzipped PAR BEDs.
+# ---------------------------------------------------------------------------
+add_library(dv_haploid_regions STATIC
+  "${CMAKE_CURRENT_SOURCE_DIR}/haploid_regions.cc"
+)
+target_include_directories(dv_haploid_regions PUBLIC
+  "${CMAKE_SOURCE_DIR}"
+  "${ABSL_PREFIX}/include"
+)
+target_link_libraries(dv_haploid_regions PUBLIC
+  absl::strings
+  z  # zlib (also pulled in transitively by htslib); for gzopen/gzgets
+)
+
 add_library(dv_make_examples_lib STATIC
   "${CMAKE_CURRENT_SOURCE_DIR}/make_examples_main.cc"
   "${CMAKE_CURRENT_SOURCE_DIR}/regions.cc"
@@ -284,6 +307,7 @@ target_link_libraries(dv_make_examples_lib PUBLIC
   dv_tfrecord
   dv_small_model
   dv_direct_phasing       # Phase 9 / Step 4 — per-region read phasing
+  dv_haploid_regions      # haploid gVCF reference confidence (PAR BED loader)
   realigner
   proto_dv
   proto_nucleus
@@ -294,6 +318,34 @@ target_link_libraries(dv_make_examples_lib PUBLIC
   nucleus_io
 )
 
+# ---------------------------------------------------------------------------
+# dv_methylation_aware_phasing — faithful C++ port of upstream's
+# methylation_aware_phasing.{cc,h} (Wilcoxon rank-sum 5mC haplotype voting).
+#
+# NOTE: this algorithm is compiled and unit-tested here (so it cannot bitrot
+# against the native proto definitions), but it is NOT yet invoked by the
+# native make_examples pipeline. The native phasing step
+# (make_examples_main.cc) uses DirectPhasing only; there is no native
+# --enable_methylation_aware_phasing flag, so a user cannot currently engage
+# methylation-aware phasing. Wiring PerformMethylationAwarePhasing into the
+# make_examples region loop (mirroring make_examples_core.py's methylated-ref-
+# site partitioning + post-DirectPhasing call) is deferred follow-up work.
+# This target deliberately is not linked into dv_make_examples_lib.
+# ---------------------------------------------------------------------------
+add_library(dv_methylation_aware_phasing STATIC
+  "${CMAKE_SOURCE_DIR}/deepvariant/methylation_aware_phasing.cc"
+)
+target_include_directories(dv_methylation_aware_phasing PUBLIC ${ME_INCLUDE_DIRS})
+target_compile_options(dv_methylation_aware_phasing PRIVATE ${ME_COMPILE_OPTS})
+target_link_libraries(dv_methylation_aware_phasing PUBLIC
+  proto_dv
+  proto_nucleus
+  absl::log
+  absl::strings
+  absl::flat_hash_map   # raw_hash_set internals referenced by the .cc
+  absl::flat_hash_set
+)
+
 # ---------------------------------------------------------------------------
 # Phase 3 — postprocess_variants orchestration
 # ---------------------------------------------------------------------------
@@ -305,6 +357,7 @@ target_include_directories(dv_postprocess_lib PUBLIC ${ME_INCLUDE_DIRS})
 target_compile_options(dv_postprocess_lib PRIVATE ${ME_COMPILE_OPTS})
 target_link_libraries(dv_postprocess_lib PUBLIC
   dv_tfrecord
+  dv_haploid_regions      # haploid het-zeroing (PAR BED loader)
   proto_dv
   proto_nucleus
   absl::flags
@@ -323,8 +376,7 @@ add_executable(debug_metal "${CMAKE_CURRENT_SOURCE_DIR}/debug_metal_main.cc")
 target_include_directories(debug_metal PRIVATE ${ME_INCLUDE_DIRS})
 target_compile_options(debug_metal PRIVATE ${ME_COMPILE_OPTS})
 target_link_libraries(debug_metal PRIVATE
-  dv_metal_inference
-  dv_weights
+  dv_metal_inference  # transitively links dv_weights (PUBLIC); do not repeat it
 )
 
 # ---------------------------------------------------------------------------

deepvariant/native/call_variants_main.cc

@@ -751,7 +751,10 @@ int RunCallVariants(int argc, char** argv) {
   }
 
   reader->Close();
-  writer->Close();
+  if (!writer->Close()) {
+    LOG(ERROR) << "Failed to flush/close output: " << outfile_path;
+    return 1;
+  }
 
   LOG(INFO) << "call_variants done: " << total_examples << " examples, "
             << total_batches << " batches → " << outfile_path;