perf(gfx1100): speed up long-context DFlash verify 13.4% with FA2 split-KV and retain HipGraph/Redline PM4

crates/hipfire-arch-qwen35/examples/test_mq_f16_projection_producers_gfx1100.rs

@@ -527,6 +527,7 @@ fn main() {
                         up_m,
                         k,
                         n,
+                        false,
                     )
                     .expect("new gate_up gemm");
                     gpu.hip.device_synchronize().expect("sync gate_up");

crates/hipfire-arch-qwen35/map.md

@@ -28,8 +28,8 @@ _Generated by `scripts/check-crate-maps.py` from the tree — do not edit inside
 | [`src/carrier.rs`](src/carrier.rs) | 854 | 4 | 7 |
 | [`src/checkpoint.rs`](src/checkpoint.rs) | 108 | 3 | 0 |
 | [`src/dflash_slot.rs`](src/dflash_slot.rs) | 783 | 12 | 0 |
-| [`src/dflash_spec.rs`](src/dflash_spec.rs) | 2,048 | 19 | 16 |
-| [`src/dflash_verify_pm4.rs`](src/dflash_verify_pm4.rs) | 744 | 36 | 9 |
+| [`src/dflash_spec.rs`](src/dflash_spec.rs) | 2,088 | 19 | 17 |
+| [`src/dflash_verify_pm4.rs`](src/dflash_verify_pm4.rs) | 780 | 37 | 10 |
 | [`src/forward_slots.rs`](src/forward_slots.rs) | 4,346 | 24 | 6 |
 | [`src/grammar_config.rs`](src/grammar_config.rs) | 143 | 2 | 4 |
 | [`src/layer_driver.rs`](src/layer_driver.rs) | 646 | 0 | 2 |
@@ -42,18 +42,18 @@ _Generated by `scripts/check-crate-maps.py` from the tree — do not edit inside
 | [`src/mtp_speculator.rs`](src/mtp_speculator.rs) | 847 | 4 | 0 |
 | [`src/paro_moe.rs`](src/paro_moe.rs) | 575 | 0 | 0 |
 | [`src/qwen35/batch.rs`](src/qwen35/batch.rs) | 2,132 | 17 | 2 |
-| [`src/qwen35/config.rs`](src/qwen35/config.rs) | 1,947 | 43 | 25 |
+| [`src/qwen35/config.rs`](src/qwen35/config.rs) | 1,963 | 44 | 25 |
 | [`src/qwen35/ep_batch.rs`](src/qwen35/ep_batch.rs) | 5,641 | 23 | 8 |
 | [`src/qwen35/forward.rs`](src/qwen35/forward.rs) | 7,697 | 33 | 12 |
 | [`src/qwen35/load.rs`](src/qwen35/load.rs) | 7,635 | 16 | 14 |
 | [`src/qwen35/oracle.rs`](src/qwen35/oracle.rs) | 1,243 | 20 | 0 |
-| [`src/qwen35/prefill.rs`](src/qwen35/prefill.rs) | 17,159 | 18 | 70 |
+| [`src/qwen35/prefill.rs`](src/qwen35/prefill.rs) | 17,495 | 20 | 74 |
 | [`src/qwen35/weights.rs`](src/qwen35/weights.rs) | 2,991 | 43 | 11 |
 | [`src/qwen35.rs`](src/qwen35.rs) | 69 | 8 | 0 |
 | [`src/serve_engine.rs`](src/serve_engine.rs) | 7,434 | 12 | 33 |
 | [`src/spec_emit.rs`](src/spec_emit.rs) | 1,042 | 5 | 14 |
 | [`src/spec_impl.rs`](src/spec_impl.rs) | 643 | 1 | 0 |
-| [`src/speculative.rs`](src/speculative.rs) | 9,932 | 87 | 14 |
+| [`src/speculative.rs`](src/speculative.rs) | 10,016 | 88 | 14 |
 
 ### Public API surface
 
@@ -63,7 +63,7 @@ _Generated by `scripts/check-crate-maps.py` from the tree — do not edit inside
 - [`src/checkpoint.rs`](src/checkpoint.rs): `hipfire_runtime`, `capture_checkpoint`, `restore_private`
 - [`src/dflash_slot.rs`](src/dflash_slot.rs): `DflashShared`, `hidden_capture`, `free_gpu`, `DflashSlotState`, `DflashSlotDraft`, `n_verify`, `load_dflash_shared`, `new_dflash_slot_state`, `reset_dflash_slot`, `scatter_staging_rows_to_interleaved`, `dflash_slot_draft_step`, `dflash_slot_verify_accept`
 - [`src/dflash_spec.rs`](src/dflash_spec.rs): `DdtreeState`, `DflashState`, `free_gpu`, `DEFAULT_DFLASH_CTX_CAP`, `load_dflash_state`, `DflashSpeculator`, `new`, `verify_pm4`, `verify_pm4_mut`, `build_dflash_speculator`, `admit_dflash_verify_pm4`, `DenseTpDflashRankState`, +7 more
-- [`src/dflash_verify_pm4.rs`](src/dflash_verify_pm4.rs): `DFLASH_VERIFY_PM4_BLOCK`, `DflashVerifyPm4Phase`, `label`, `reason`, `is_live`, `DflashVerifyPm4Counters`, `DflashVerifyBinding`, `new`, `same_route`, `fingerprint_u64`, `DflashVerifyWindow`, `DflashVerifyRoute`, +24 more
+- [`src/dflash_verify_pm4.rs`](src/dflash_verify_pm4.rs): `DFLASH_VERIFY_PM4_BLOCK`, `DflashVerifyPm4Phase`, `label`, `reason`, `is_live`, `DflashVerifyPm4Counters`, `DflashVerifyBinding`, `new`, `same_route`, `fingerprint_u64`, `DflashVerifyWindow`, `DflashVerifyRoute`, +25 more
 - [`src/forward_slots.rs`](src/forward_slots.rs): `DESC_BYTES`, `SlotKvTier`, `q8`, `is_q8`, `free_gpu`, `SlotDescStaging`, `new`, `SpecVerifyCapture`, `SpecHiddenCapture`, `is_verify_slot`, `forward_batch_slots`, `DECODE_GRAPH_CTX_BUCKET`, +12 more
 - [`src/grammar_config.rs`](src/grammar_config.rs): `resolve_qwen35_grammar_config`, `resolve_grammar_config`
 - [`src/layer_driver.rs`](src/layer_driver.rs): —
@@ -76,18 +76,18 @@ _Generated by `scripts/check-crate-maps.py` from the tree — do not edit inside
 - [`src/mtp_speculator.rs`](src/mtp_speculator.rs): `Qwen35MtpDrafter`, `new`, `mtp_live_state`, `build_qwen35_mtp_speculator`
 - [`src/paro_moe.rs`](src/paro_moe.rs): —
 - [`src/qwen35/batch.rs`](src/qwen35/batch.rs): `PrefillBatchScratch`, `new`, `new_opt`, `new_opt_lean`, `free_gpu`, `Qwen35DecodeBatchState`, `reset`, `reset_lane`, `prefill_lane`, `sample`, `sample_product`, `sample_lane`, +5 more
-- [`src/qwen35/config.rs`](src/qwen35/config.rs): `LayerType`, `MaskEmbedOverride`, `DflashFusionCtx`, `TreeVerifyCtx`, `Qwen35Config`, `DenseTpRankLayout`, `dense_tp_rank_layouts`, `validate_dense_tp`, `local_dense_tp_config`, `Qwen35EpReduce`, `Qwen35BatchParallelism`, `Qwen35EpBatchReceipt`, +31 more
+- [`src/qwen35/config.rs`](src/qwen35/config.rs): `LayerType`, `MaskEmbedOverride`, `DflashFusionCtx`, `fn`, `TreeVerifyCtx`, `Qwen35Config`, `DenseTpRankLayout`, `dense_tp_rank_layouts`, `validate_dense_tp`, `local_dense_tp_config`, `Qwen35EpReduce`, `Qwen35BatchParallelism`, +32 more
 - [`src/qwen35/ep_batch.rs`](src/qwen35/ep_batch.rs): `validate_ep_batch_compatibility`, `Qwen35DecodeBatchEpState`, `max_batch`, `sequential_prefill_scratch_mut`, `peer_reduce_lease`, `lane_capacity`, `epoch`, `poison_mask`, `lane_state`, `download_rank_outputs`, `new`, `reset_all`, +11 more
 - [`src/qwen35/forward.rs`](src/qwen35/forward.rs): `dump_expert_stats`, `forward`, `Qwen35Scratch`, `new`, `new_with_kv_max`, `free_gpu`, `Qwen35ScratchSet`, `new_with_kv_max_multi`, `free_gpu_multi`, `forward_scratch`, `prepare_scratch_inputs`, `forward_scratch_with_hidden`, +21 more
 - [`src/qwen35/load.rs`](src/qwen35/load.rs): `hipfire_runtime`, `qwen35_tensor_name_candidates`, `load_weight_tensor`, `load_weight_tensor_host`, `report_cpu_exec_coverage`, `load_weights`, `load_weights_with_fault`, `HfqSource`, `new`, `ParoSource`, `preflight_weights_dense_tp`, `load_weights_dense_tp_rank`, +4 more
 - [`src/qwen35/oracle.rs`](src/qwen35/oracle.rs): `SCHEMA_VERSION`, `SCHEMA_FIELDS`, `ArrayRef`, `Snapshot`, `Observation`, `begin`, `set_sequence`, `set_decode_position`, `set_prefill_start`, `decode_position`, `prefill_start`, `decode_before`, +8 more
-- [`src/qwen35/prefill.rs`](src/qwen35/prefill.rs): `PREFILL_MAX_BATCH`, `prefill_max_batch`, `prefill_max_batch_tp`, `prefill_max_batch_ep`, `vmm_kv_token_bytes`, `minimum_prefill_reservation_bytes`, `ordinary_prefill_chunk_limit`, `ordinary_serve_prefill_chunk_len`, `upload_prefill_batch_inputs`, `forward_prefill_batch_single_chunk_captured`, `forward_prefill_batch_single_chunk_captured_opts`, `forward_prefill_batch`, +6 more
+- [`src/qwen35/prefill.rs`](src/qwen35/prefill.rs): `PREFILL_MAX_BATCH`, `prefill_max_batch`, `prefill_max_batch_tp`, `prefill_max_batch_ep`, `vmm_kv_token_bytes`, `minimum_prefill_reservation_bytes`, `ordinary_prefill_chunk_limit`, `ordinary_serve_prefill_chunk_len`, `upload_prefill_batch_inputs`, `forward_prefill_batch_single_chunk_captured`, `forward_prefill_batch_single_chunk_captured_opts`, `forward_prefill_batch`, +8 more
 - [`src/qwen35/weights.rs`](src/qwen35/weights.rs): `DeltaNetLayerWeights`, `FullAttnLayerWeights`, `ExpertWeights`, `mixed_expert_tag`, `SharedExpertWeights`, `MoeFfnWeights`, `MoeParoSidecars`, `DeltaNetMoeLayerWeights`, `FullAttnMoeLayerWeights`, `LayerWeights`, `free_gpu`, `Qwen35HfqSourceIdentity`, +31 more
 - [`src/qwen35.rs`](src/qwen35.rs): `batch`, `config`, `ep_batch`, `forward`, `load`, `oracle`, `prefill`, `weights`
 - [`src/serve_engine.rs`](src/serve_engine.rs): `EngineConfig`, `SlotEngine`, `submit`, `cancel_waiting`, `close`, `reset`, `stats`, `spawn`, `shutdown_engine`, `MTP_RETIRE_WINDOW`, `MTP_RETIRE_MIN_ADVANCE`, `MTP_RETIRE_WINDOWS`
 - [`src/spec_emit.rs`](src/spec_emit.rs): `Qwen35Emit`, `from_ctx`, `from_ctx_template_think_close`, `decoded_eot`, `visible_text`
 - [`src/spec_impl.rs`](src/spec_impl.rs): `Qwen35SpecScratch`
-- [`src/speculative.rs`](src/speculative.rs): `SeedOracleStats`, `read_seed_oracle_stats`, `reset_seed_oracle_stats`, `record_ddtree_meta_nodes`, `DdtreeMetaStats`, `read_ddtree_meta_stats`, `reset_ddtree_meta_stats`, `KvMode`, `ModelSlotConfig`, `ModelSlot`, `from_bundle`, `into_bundle`, +75 more
+- [`src/speculative.rs`](src/speculative.rs): `dflash_finish_retained_lm_head_argmax`, `SeedOracleStats`, `read_seed_oracle_stats`, `reset_seed_oracle_stats`, `record_ddtree_meta_nodes`, `DdtreeMetaStats`, `read_ddtree_meta_stats`, `reset_ddtree_meta_stats`, `KvMode`, `ModelSlotConfig`, `ModelSlot`, `from_bundle`, +76 more
 
 ### Dependencies (from `Cargo.toml`)
 
@@ -102,6 +102,6 @@ _Generated by `scripts/check-crate-maps.py` from the tree — do not edit inside
 
 ### Totals
 
-- 31 modules · 87,250 lines · 555 public items · 280 tests · 18 examples
+- 31 modules · 87,762 lines · 560 public items · 286 tests · 18 examples
 
 <!-- crate-map:generated:end -->

crates/hipfire-arch-qwen35/src/dflash_spec.rs

@@ -164,6 +164,7 @@ pub fn load_dflash_state(
     // Retained-PM4 admission facts owned by the loader/target, not the draft.
     target_weights: &Qwen35Weights,
     kv_is_q8: bool,
+    target_flash_partials_numel: usize,
     single_gpu: bool,
     // True when adaptive KV is engaged for this load (tier-switching cache).
     // Must be false for retained-PM4 admission.
@@ -466,12 +467,25 @@ pub fn load_dflash_state(
         .as_ref()
         .map(|p| p.max_batch)
         .unwrap_or(0);
+    let gfx1100_split_verify_min_ctx =
+        qwen35::prefill::gfx1100_split_verify_min_ctx(gpu, target_config).filter(|&threshold| {
+            qwen35::prefill::gfx1100_split_verify_admitted(
+                gpu,
+                target_config,
+                kv_is_q8,
+                DFLASH_VERIFY_PM4_BLOCK,
+                threshold.saturating_add(1),
+                target_flash_partials_numel,
+            )
+        });
+    let gfx1100_split_verify = gfx1100_split_verify_min_ctx.is_some();
     // The draft's selector/dynamic-conv shape is deliberately NOT a gate: the
     // draft forward is outside the tape, so DFlash2 and legacy DFlash yield an
     // identical target verify body.
     let verify_pm4 = match admit_dflash_verify_pm4(
         env_opt_in,
         &gpu.arch,
+        gfx1100_split_verify,
         single_gpu,
         target_config.num_experts,
         kv_is_q8,
@@ -496,7 +510,14 @@ pub fn load_dflash_state(
                 "  DFlash verify PM4: armed (B={}, exact {})",
                 DFLASH_VERIFY_PM4_BLOCK, gpu.arch
             );
-            DflashVerifyPm4::armed()
+            if let Some(min_ctx) = gfx1100_split_verify_min_ctx {
+                // Keep the faster incumbent below the resolved crossover, and
+                // do not capture a short-context tape that would silently omit
+                // the new partial + merge launches.
+                DflashVerifyPm4::armed_after_context(min_ctx)
+            } else {
+                DflashVerifyPm4::armed()
+            }
         }
         Err(reason) => {
             eprintln!("  DFlash verify PM4: disabled ({reason})");
@@ -1323,6 +1344,7 @@ pub fn build_dflash_speculator(
 pub fn admit_dflash_verify_pm4(
     env_opt_in: bool,
     arch: &str,
+    gfx1100_split_verify: bool,
     single_gpu: bool,
     num_experts: usize,
     kv_is_q8: bool,
@@ -1342,8 +1364,10 @@ pub fn admit_dflash_verify_pm4(
     if !env_opt_in {
         return Err("HIPFIRE_DFLASH_VERIFY_PM4 is not set to 1".into());
     }
-    if arch != "gfx1201" {
-        return Err(format!("arch is {arch}, not exact gfx1201"));
+    if arch != "gfx1201" && !(arch == "gfx1100" && gfx1100_split_verify) {
+        return Err(format!(
+            "arch is {arch}, not exact gfx1201 or admitted gfx1100 split verifier"
+        ));
     }
     if !single_gpu {
         return Err("multi-GPU load is not admitted".into());
@@ -1795,6 +1819,7 @@ mod admit_dflash_verify_pm4_tests {
     struct Args {
         env_opt_in: bool,
         arch: &'static str,
+        gfx1100_split_verify: bool,
         single_gpu: bool,
         num_experts: usize,
         kv_is_q8: bool,
@@ -1817,6 +1842,7 @@ mod admit_dflash_verify_pm4_tests {
             Self {
                 env_opt_in: true,
                 arch: "gfx1201",
+                gfx1100_split_verify: false,
                 single_gpu: true,
                 num_experts: 0,
                 kv_is_q8: true,
@@ -1840,6 +1866,7 @@ mod admit_dflash_verify_pm4_tests {
         admit_dflash_verify_pm4(
             a.env_opt_in,
             a.arch,
+            a.gfx1100_split_verify,
             a.single_gpu,
             a.num_experts,
             a.kv_is_q8,
@@ -1880,7 +1907,20 @@ mod admit_dflash_verify_pm4_tests {
             ..Args::default()
         })
         .unwrap_err();
-        assert_eq!(err, "arch is gfx1100, not exact gfx1201");
+        assert_eq!(
+            err,
+            "arch is gfx1100, not exact gfx1201 or admitted gfx1100 split verifier"
+        );
+    }
+
+    #[test]
+    fn admits_gfx1100_only_with_split_verifier() {
+        assert!(admit(Args {
+            arch: "gfx1100",
+            gfx1100_split_verify: true,
+            ..Args::default()
+        })
+        .is_ok());
     }
 
     #[test]

crates/hipfire-arch-qwen35/src/dflash_verify_pm4.rs

@@ -224,6 +224,11 @@ pub enum DflashVerifyRoute {
 pub struct DflashVerifyPm4 {
     phase: DflashVerifyPm4Phase,
     controller: Option<ReplayController>,
+    /// Do not build or submit the retained tape below this logical context.
+    /// The gfx1100 split-KV route wins only after 4K; deferring capture keeps
+    /// the short-context shipping kernel in place and guarantees that the
+    /// recorded body actually contains the split partial + merge pair.
+    min_context: usize,
     binding: Option<DflashVerifyBinding>,
     identity: Option<PreparedReplayIdentity>,
     /// First calibration recording and the position it was taken at.
@@ -239,6 +244,7 @@ impl DflashVerifyPm4 {
                 reason: reason.into(),
             },
             controller: None,
+            min_context: 0,
             binding: None,
             identity: None,
             calibration: None,
@@ -249,9 +255,17 @@ impl DflashVerifyPm4 {
     /// Admitted route with a dedicated manual-PM4 controller. Allocates no GPU
     /// resource until the first capture.
     pub fn armed() -> Self {
+        Self::armed_after_context(0)
+    }
+
+    /// Arm a route that remains on ordinary HIP until `start_pos + batch`
+    /// exceeds `min_context`. Once prepared, later short-context requests also
+    /// stay on HIP while the long-context tape remains ready for reuse.
+    pub fn armed_after_context(min_context: usize) -> Self {
         Self {
             phase: DflashVerifyPm4Phase::Armed,
             controller: Some(ReplayController::new_manual_pm4()),
+            min_context,
             binding: None,
             identity: None,
             calibration: None,
@@ -301,6 +315,10 @@ impl DflashVerifyPm4 {
             self.note_hip_window(window);
             return DflashVerifyRoute::HipAuto;
         }
+        if window.position.saturating_add(window.batch) <= self.min_context {
+            self.note_hip_window(window);
+            return DflashVerifyRoute::HipAuto;
+        }
         if !window.eligible_shape() {
             self.note_hip_window(window);
             return DflashVerifyRoute::HipAuto;
@@ -543,6 +561,7 @@ impl DflashVerifyPm4 {
         serde_json::json!({
             "phase": self.phase.label(),
             "reason": self.phase.reason(),
+            "min_context": self.min_context,
             "binding": self.binding,
             "kv_mode": "q8",
             "dn_state_quant": "q8",
@@ -660,6 +679,23 @@ mod tests {
         assert_eq!(route.counters().last_replay_position, Some(32));
     }
 
+    #[test]
+    fn minimum_context_defers_capture_without_changing_phase() {
+        let bound = binding(1, 8192);
+        let mut route = DflashVerifyPm4::armed_after_context(4096);
+        assert_eq!(
+            route.plan_route(&window(&bound, 16, 4080)),
+            DflashVerifyRoute::HipAuto
+        );
+        assert_eq!(*route.phase(), DflashVerifyPm4Phase::Armed);
+        assert_eq!(route.counters().full_hip, 1);
+        assert_eq!(
+            route.plan_route(&window(&bound, 16, 4081)),
+            DflashVerifyRoute::PrimeDirect
+        );
+        assert_eq!(route.counters().prime_windows, 1);
+    }
+
     #[test]
     fn position_beyond_prepared_max_rearms_instead_of_replaying() {
         let bound = binding(1, 64);

crates/hipfire-arch-qwen35/src/qwen35/config.rs

@@ -103,6 +103,22 @@ pub struct MaskEmbedOverride<'a> {
 pub enum DflashFusionCtx {
     Off,
     ChainVerify,
+    /// Linear-chain verify whose exact model/KV/context envelope admits the
+    /// gfx1100 FA2 split-KV route. This only widens capture-safe auxiliary
+    /// kernels; ordinary eager ChainVerify behavior is otherwise unchanged.
+    ChainVerifySplit,
+}
+
+impl DflashFusionCtx {
+    #[inline]
+    pub const fn is_chain_verify(self) -> bool {
+        matches!(self, Self::ChainVerify | Self::ChainVerifySplit)
+    }
+
+    #[inline]
+    pub const fn split_verify_active(self) -> bool {
+        matches!(self, Self::ChainVerifySplit)
+    }
 }
 
 #[derive(Clone, Copy)]