[skip benchmarks] Mapping-VGGT Branch

gtsfm/bundle/bundle_adjustment.py

@@ -169,6 +169,10 @@ class BundleAdjustmentOptions:
     min_tracks_per_camera: int = 15
     compute_pose_covariances: bool = False
     optimizer_relative_cost_tol: float = 1e-5
+    use_multi_view_retriangulation: bool = False
+    mv_retri_min_track_length: int = 3
+    mv_retri_reproj_error_thresh: float = 10.0
+    mv_retri_max_num_hypotheses: int = 100
 
     def to_optimizer(self, **overrides) -> "BundleAdjustmentOptimizer":
         """Construct a :class:`BundleAdjustmentOptimizer` from these options.
@@ -195,6 +199,10 @@ def to_optimizer(self, **overrides) -> "BundleAdjustmentOptimizer":
             min_tracks_per_camera=self.min_tracks_per_camera,
             compute_pose_covariances=self.compute_pose_covariances,
             optimizer_relative_cost_tol=self.optimizer_relative_cost_tol,
+            use_multi_view_retriangulation=self.use_multi_view_retriangulation,
+            mv_retri_min_track_length=self.mv_retri_min_track_length,
+            mv_retri_reproj_error_thresh=self.mv_retri_reproj_error_thresh,
+            mv_retri_max_num_hypotheses=self.mv_retri_max_num_hypotheses,
         )
         kwargs.update(overrides)
         return BundleAdjustmentOptimizer(**kwargs)
@@ -238,13 +246,6 @@ def __init__(
         min_tracks_per_camera: int = 15,
         compute_pose_covariances: bool = False,
         optimizer_relative_cost_tol: float = 1e-5,
-        # ── Optional post-BA multi-view retriangulation (opt-in) ──
-        # When `use_multi_view_retriangulation=True`: after the existing BA loop
-        # converges, re-triangulate the union-find 2D tracks against the post-BA
-        # cameras (recovers tracks dropped between union-find and BA's filter
-        # passes) and run a final BA on the augmented set. Requires `tracks_2d` to
-        # be passed to `create_computation_graph` / `_run_ba_and_evaluate`. The
-        # final BA reuses the existing `reproj_error_thresholds[-1]` for filtering.
         use_multi_view_retriangulation: bool = False,
         mv_retri_min_track_length: int = 3,
         mv_retri_reproj_error_thresh: float = 10.0,
@@ -952,12 +953,6 @@ def _run_ba_and_evaluate(
         )
         total_time = time.time() - start_time
 
-        # ── Optional post-BA multi-view retriangulation stage ──
-        # Re-triangulate union-find tracks against the post-BA cameras (recovers
-        # tracks dropped between union-find and BA's filter passes), then run a
-        # final BA on the augmented set. The final BA reuses the existing tightest
-        # `reproj_error_thresholds[-1]` for inline filtering — same mechanism as
-        # the upstream BA loop.
         if self._use_multi_view_retriangulation:
             if tracks_2d is None:
                 logger.warning(
@@ -984,10 +979,6 @@ def _run_ba_and_evaluate(
                     min_track_length=self._mv_retri_min_track_length,
                 )
                 if retri_data.number_tracks() > 0:
-                    # Final BA on the retri'd track set. No inline filter — pose AUC
-                    # is set by BA's converged cameras and is independent of any
-                    # downstream track filtering. Callers can filter the returned
-                    # GtsfmData themselves if they want.
                     (optimized_data, filtered_result, valid_mask, _) = self.run_ba_stage_with_filtering(
                         initial_data=retri_data,
                         absolute_pose_priors=absolute_pose_priors,

gtsfm/cluster_optimizer/cluster_mvo.py

@@ -162,6 +162,10 @@ def _run_two_view_estimation(
         all_two_view_results = cast(AnnotatedGraph[TwoViewResult], gathered_tve_futures)
         valid_two_view_results = {edge: result for edge, result in all_two_view_results.items() if result.valid()}
 
+        n_total = len(all_two_view_results)
+        n_valid = len(valid_two_view_results)
+        logger.info("Two-view estimation: %d/%d pairs valid, %d rejected.", n_valid, n_total, n_total - n_valid)
+
         if len(valid_two_view_results) == 0:
             logger.warning("🔵 ClusterMVO: Skipping cluster as it has no valid two-view results.")
 

gtsfm/cluster_optimizer/cluster_vggt.py

@@ -13,9 +13,10 @@
 
 import gtsfm.common.types as gtsfm_types
 import gtsfm.utils.metrics as metrics_utils
-from gtsfm.bundle.bundle_adjustment import BundleAdjustmentOptions
+from gtsfm.bundle.bundle_adjustment import BundleAdjustmentOptions, multi_view_retriangulate_from_2d_tracks
 from gtsfm.cluster_optimizer.cluster_optimizer_base import ClusterComputationGraph, ClusterContext, ClusterOptimizerBase
 from gtsfm.common.gtsfm_data import GtsfmData
+from gtsfm.common.sfm_track import SfmTrack2d
 from gtsfm.evaluation.metrics import GtsfmMetric, GtsfmMetricsGroup
 from gtsfm.frontend.multi_view_tracker import MultiViewTracker
 from gtsfm.frontend.vggt_geometry_transformer import (
@@ -50,11 +51,20 @@ def _run_cluster_ba(
     drop_camera_with_no_track: bool = False,
     min_track_length: int = 2,
     cluster_label: Optional[str] = None,
+    tracks_2d: Optional[list[SfmTrack2d]] = None,
+    use_multi_view_retriangulation: bool = False,
 ) -> tuple[GtsfmData, GtsfmData]:
     """Run cluster-level BA on a GtsfmData result.
 
     This is a module-level function so it can be used with ``dask.delayed``.
 
+    Args:
+        tracks_2d: (optional) Union-find 2D tracks. Required when
+            ``use_multi_view_retriangulation=True``.
+        use_multi_view_retriangulation: When True, after the initial BA, re-triangulate
+            ``tracks_2d`` against the post-BA cameras (recovers tracks dropped earlier
+            in the pipeline) and run a second BA on the augmented track set.
+
     Returns:
         Tuple of (post_ba_result, pre_ba_result).
     """
@@ -90,6 +100,20 @@ def _run_cluster_ba(
             post_ba_max_reproj_error
         )
 
+        # Optional retri stage: re-triangulate union-find tracks against the post-BA
+        # cameras and run another BA on the augmented set. Recovers tracks dropped
+        # earlier in the pipeline; mirrors the retri stage in
+        # BundleAdjustmentOptimizer._run_ba_and_evaluate.
+        if use_multi_view_retriangulation and tracks_2d is not None:
+            retri_data = multi_view_retriangulate_from_2d_tracks(
+                gtsfm_data_with_ba, tracks_2d, min_track_length=min_track_length,
+            )
+            if retri_data.number_tracks() > 0:
+                gtsfm_data_with_ba, _ = optimizer.run_simple_ba(retri_data)
+                gtsfm_data_with_ba = gtsfm_data_with_ba.filter_landmark_measurements(
+                    post_ba_max_reproj_error
+                )
+
         logger.info(
             "%s🔍 #valid tracks after BA: %d out of %d",
             f"[{cluster_label}] " if cluster_label else "",

gtsfm/cluster_optimizer/cluster_vggt_with_frontend.py

@@ -30,6 +30,7 @@
 from gtsfm.products.visibility_graph import visibility_graph_keys
 from gtsfm.two_view_estimator import TwoViewEstimator
 from gtsfm.ui.gtsfm_process import UiMetadata
+from gtsfm.view_graph_estimator.view_graph_calibration import calibrate_view_graph
 import gtsam
 
 from gtsfm.utils import torch as torch_utils
@@ -132,6 +133,7 @@ def _build_gtsfm_data_from_vggt_depth(
     image_indices: tuple[int, ...],
     num_images: int,
     min_track_length: int = 2,
+    refined_intrinsics: Optional[dict[int, gtsfm_types.CALIBRATION_TYPE]] = None,
 ) -> GtsfmData:
     """Build GtsfmData using VGGT cameras (rescaled to original resolution) and frontend 2D tracks.
 
@@ -159,14 +161,19 @@ def _build_gtsfm_data_from_vggt_depth(
     _, H_vggt, W_vggt = dense_points.shape[:3]
     global_to_local = {gidx: lidx for lidx, gidx in enumerate(image_indices)}
 
-    # Register cameras with intrinsics rescaled to original image resolution.
+    # Register cameras with intrinsics in original image resolution. If
+    # `refined_intrinsics` is supplied (from view-graph calibration), use those
+    # directly; otherwise rescale VGGT's predicted intrinsics from VGGT pixel space.
     gtsfm_data = GtsfmData(number_images=num_images)
     for global_idx, camera in cameras.items():
         if global_idx in image_shapes and global_idx in global_to_local:
-            _, orig_W = image_shapes[global_idx]
-            local_idx = global_to_local[global_idx]
-            scaled_W = float(original_coords[local_idx, 4])
-            camera = _scale_camera_intrinsics(camera, scale=orig_W / scaled_W)
+            if refined_intrinsics is not None:
+                camera = type(camera)(camera.pose(), refined_intrinsics[global_idx])
+            else:
+                _, orig_W = image_shapes[global_idx]
+                local_idx = global_to_local[global_idx]
+                scaled_W = float(original_coords[local_idx, 4])
+                camera = _scale_camera_intrinsics(camera, scale=orig_W / scaled_W)
         gtsfm_data.add_camera(global_idx, camera)
 
     for track_2d in tracks_2d:
@@ -220,6 +227,38 @@ def _build_gtsfm_data_from_vggt_depth(
     return gtsfm_data
 
 
+def _refine_vggt_intrinsics_via_view_graph(
+    vggt_result: VggtGeometryResult,
+    v_corr_idxs: dict,
+    keypoints_list: list,
+    image_shapes: dict[int, tuple[int, int]],
+    image_indices: tuple[int, ...],
+) -> dict[int, gtsfm_types.CALIBRATION_TYPE]:
+    """Refine focal lengths via Fetzer joint optimization over F-matrix edges.
+
+    Returns intrinsics in ORIGINAL image coordinates (suitable for use with the
+    frontend keypoints). VGGT's predicted intrinsics are rescaled to original
+    image coords first, then handed to `calibrate_view_graph` as the initial
+    estimate. VGGT's predicted poses are unchanged — only intrinsics are refined.
+    """
+    initial_intrinsics: dict[int, gtsfm_types.CALIBRATION_TYPE] = {}
+    for local_idx, global_idx in enumerate(image_indices):
+        if global_idx in vggt_result.cameras and global_idx in image_shapes:
+            _, orig_W = image_shapes[global_idx]
+            scaled_W = float(vggt_result.original_coords[local_idx, 4])
+            scale = orig_W / scaled_W if scaled_W > 0 else 1.0
+            scaled_cam = _scale_camera_intrinsics(vggt_result.cameras[global_idx], scale=scale)
+            initial_intrinsics[global_idx] = scaled_cam.calibration()
+
+    keypoints = {gidx: keypoints_list[gidx] for gidx in image_indices}
+    refined, _edges_to_remove = calibrate_view_graph(
+        v_corr_idxs_dict=v_corr_idxs,
+        keypoints=keypoints,
+        initial_intrinsics=initial_intrinsics,
+    )
+    return refined
+
+
 class ClusterVGGTWithFrontend(ClusterMVO):
     """Cluster optimizer that combines a traditional MVO frontend with VGGT poses.
 
@@ -250,6 +289,8 @@ def __init__(
         save_two_view_viz: bool = False,
         pose_angular_error_thresh: float = 3,
         output_worker: Optional[str] = None,
+        use_view_graph_calibration: bool = False,
+        use_multi_view_retriangulation: bool = False,
     ) -> None:
         super().__init__(
             correspondence_generator=correspondence_generator,
@@ -268,6 +309,8 @@ def __init__(
         self._metric_constructed_only = metric_constructed_only
         self._input_mode = input_mode
         self._seed = seed
+        self._use_view_graph_calibration = use_view_graph_calibration
+        self._use_multi_view_retriangulation = use_multi_view_retriangulation
 
         self._weights_path = Path(weights_path) if weights_path is not None else None
         self._loader_kwargs: dict[str, Any] = {}
@@ -343,6 +386,18 @@ def create_computation_graph(self, context: ClusterContext) -> ClusterComputatio
         # 4. Original image shapes (needed to map frontend pixel coords → VGGT pixel coords).
         image_shapes_graph = delayed(_get_image_shapes)(context.loader, global_indices)
 
+        # 4b. Optional: refine VGGT's predicted intrinsics via Fetzer joint
+        # optimization over the frontend's F-matrices (keeps VGGT's predicted poses).
+        refined_intrinsics_graph = None
+        if self._use_view_graph_calibration:
+            refined_intrinsics_graph = delayed(_refine_vggt_intrinsics_via_view_graph)(
+                vggt_result_graph,
+                v_corr_idxs_graph,
+                frontend_graphs.padded_keypoints,
+                image_shapes_graph,
+                global_indices,
+            )
+
         # 5. Build GtsfmData: lift 2D tracks to 3D using VGGT depth map.
         ba_input_graph = delayed(_build_gtsfm_data_from_vggt_depth)(
             vggt_result_graph,
@@ -351,6 +406,7 @@ def create_computation_graph(self, context: ClusterContext) -> ClusterComputatio
             image_indices=global_indices,
             num_images=context.num_images,
             min_track_length=self._min_track_length,
+            refined_intrinsics=refined_intrinsics_graph,
         )
 
         # 6. Cluster-level BA.
@@ -362,6 +418,8 @@ def create_computation_graph(self, context: ClusterContext) -> ClusterComputatio
             drop_camera_with_no_track=self._drop_camera_with_no_track,
             min_track_length=self._min_track_length,
             cluster_label=context.label,
+            tracks_2d=tracks_2d_graph,
+            use_multi_view_retriangulation=self._use_multi_view_retriangulation,
         )
 
         # 7. Metrics + I/O.

gtsfm/configs/vggt_sift_frontend_megaloc.yaml

@@ -97,6 +97,10 @@ cluster_optimizer:
     input_mode: crop
     seed: 42
     model_cache_key: null
+    # Refine VGGT predicted intrinsics via Fetzer joint optimization on F-matrices.
+    use_view_graph_calibration: false
+    # Re-triangulate union-find 2D tracks against post-BA cameras and run another BA.
+    use_multi_view_retriangulation: false
 
 # --- Merging options ---
 merging_options:

gtsfm/configs/vggt_sift_frontend_megaloc_phototourism.yaml

@@ -97,6 +97,10 @@ cluster_optimizer:
     input_mode: crop
     seed: 42
     model_cache_key: null
+    # Refine VGGT predicted intrinsics via Fetzer joint optimization on F-matrices.
+    use_view_graph_calibration: false
+    # Re-triangulate union-find 2D tracks against post-BA cameras and run another BA.
+    use_multi_view_retriangulation: false
 
 # --- Merging options ---
 merging_options:

gtsfm/configs/vggt_unified_frontend_megaloc.yaml

@@ -92,6 +92,10 @@ cluster_optimizer:
     input_mode: crop
     seed: 42
     model_cache_key: null
+    # Refine VGGT predicted intrinsics via Fetzer joint optimization on F-matrices.
+    use_view_graph_calibration: false
+    # Re-triangulate union-find 2D tracks against post-BA cameras and run another BA.
+    use_multi_view_retriangulation: false
 
 # --- Merging options ---
 merging_options:

gtsfm/graph_partitioner/metis_partitioner.py

@@ -38,6 +38,7 @@ def __init__(
         min_child_overlap_for_split: int = 2,
         min_parent_overlap_for_split: int = 2,
         split_oversized_nodes: bool = False,
+        include_all_edges_in_cluster: bool = False,
     ) -> None:
         super().__init__(process_name="MetisPartitioner")
         if min_cameras_to_partition is not None and min_cameras_to_partition < 1:
@@ -55,6 +56,7 @@ def __init__(
         self._min_child_overlap_for_split = min_child_overlap_for_split
         self._min_parent_overlap_for_split = min_parent_overlap_for_split
         self._split_oversized_nodes = split_oversized_nodes
+        self._include_all_edges_in_cluster = include_all_edges_in_cluster
 
     @staticmethod
     def _is_connected(graph: VisibilityGraph) -> bool:
@@ -684,11 +686,13 @@ def _cluster_from_clique(self, clique: SymbolicBayesTreeClique, graph: Visibilit
         child_results = self._merge_small_children_at_level(child_results, graph)
         descendant_edges = set.union(*(result.edges for result in child_results)) if child_results else set()
 
-        # Only keep edges that touch at least one frontal variable from this clique.
-        candidate_edges = {
-            (i, j) for i, j in graph if i in keys and j in keys and (i in frontals or j in frontals or not frontals)
-        }
-        current_edges = candidate_edges - descendant_edges
+        if self._include_all_edges_in_cluster:
+            current_edges = {(i, j) for i, j in graph if i in keys and j in keys}
+        else:
+            candidate_edges = {
+                (i, j) for i, j in graph if i in keys and j in keys and (i in frontals or j in frontals or not frontals)
+            }
+            current_edges = candidate_edges - descendant_edges
 
         def sorted_edges(edges: set[tuple[int, int]]) -> list[tuple[int, int]]:
             return sorted(edges)

gtsfm/multi_view_optimizer.py

@@ -183,8 +183,8 @@ def create_computation_graph(
 
         # View graph calibration: refine focal lengths from F-matrices
         if self._run_view_graph_calibration:
-            all_intrinsics, edges_to_remove = delayed(view_graph_calibration.calibrate_view_graph, nout=2)(
-                viewgraph_v_corr_idxs_graph, keypoints_graph, all_intrinsics, num_images
+            all_intrinsics, edges_to_remove = delayed(_calibrate_view_graph_from_lists, nout=2)(
+                viewgraph_v_corr_idxs_graph, keypoints_graph, all_intrinsics
             )
             # Remove edges with high calibration error
             viewgraph_i2Ri1_graph, viewgraph_i2Ui1_graph, viewgraph_v_corr_idxs_graph = delayed(_filter_edges, nout=3)(
@@ -336,6 +336,22 @@ def _sync_two_view_reports_after_calibration(
     return synced_reports
 
 
+def _calibrate_view_graph_from_lists(
+    v_corr_idxs_dict,
+    keypoints_list: list[Keypoints],
+    intrinsics_list: list[gtsfm_types.CALIBRATION_TYPE],
+    **kwargs,
+):
+    """List-interface wrapper around calibrate_view_graph for use in multi_view_optimizer."""
+    kp_dict = {i: kp for i, kp in enumerate(keypoints_list)}
+    intr_dict = {i: cal for i, cal in enumerate(intrinsics_list)}
+    refined_dict, edges_to_remove = view_graph_calibration.calibrate_view_graph(
+        v_corr_idxs_dict, kp_dict, intr_dict, **kwargs
+    )
+    refined_list = [refined_dict.get(i, intrinsics_list[i]) for i in range(len(intrinsics_list))]
+    return refined_list, edges_to_remove
+
+
 def reestimate_relative_poses(
     i2Ri1_dict: Dict[Tuple[int, int], Rot3],
     i2Ui1_dict: Dict[Tuple[int, int], Unit3],