diff --git a/gtsfm/bundle/bundle_adjustment.py b/gtsfm/bundle/bundle_adjustment.py index 4f079d95e..b89fe0a0b 100644 --- a/gtsfm/bundle/bundle_adjustment.py +++ b/gtsfm/bundle/bundle_adjustment.py @@ -169,6 +169,10 @@ class BundleAdjustmentOptions: min_tracks_per_camera: int = 15 compute_pose_covariances: bool = False optimizer_relative_cost_tol: float = 1e-5 + use_multi_view_retriangulation: bool = False + mv_retri_min_track_length: int = 3 + mv_retri_reproj_error_thresh: float = 10.0 + mv_retri_max_num_hypotheses: int = 100 def to_optimizer(self, **overrides) -> "BundleAdjustmentOptimizer": """Construct a :class:`BundleAdjustmentOptimizer` from these options. @@ -195,6 +199,10 @@ def to_optimizer(self, **overrides) -> "BundleAdjustmentOptimizer": min_tracks_per_camera=self.min_tracks_per_camera, compute_pose_covariances=self.compute_pose_covariances, optimizer_relative_cost_tol=self.optimizer_relative_cost_tol, + use_multi_view_retriangulation=self.use_multi_view_retriangulation, + mv_retri_min_track_length=self.mv_retri_min_track_length, + mv_retri_reproj_error_thresh=self.mv_retri_reproj_error_thresh, + mv_retri_max_num_hypotheses=self.mv_retri_max_num_hypotheses, ) kwargs.update(overrides) return BundleAdjustmentOptimizer(**kwargs) @@ -238,13 +246,6 @@ def __init__( min_tracks_per_camera: int = 15, compute_pose_covariances: bool = False, optimizer_relative_cost_tol: float = 1e-5, - # ── Optional post-BA multi-view retriangulation (opt-in) ── - # When `use_multi_view_retriangulation=True`: after the existing BA loop - # converges, re-triangulate the union-find 2D tracks against the post-BA - # cameras (recovers tracks dropped between union-find and BA's filter - # passes) and run a final BA on the augmented set. Requires `tracks_2d` to - # be passed to `create_computation_graph` / `_run_ba_and_evaluate`. The - # final BA reuses the existing `reproj_error_thresholds[-1]` for filtering. use_multi_view_retriangulation: bool = False, mv_retri_min_track_length: int = 3, mv_retri_reproj_error_thresh: float = 10.0, @@ -952,12 +953,6 @@ def _run_ba_and_evaluate( ) total_time = time.time() - start_time - # ── Optional post-BA multi-view retriangulation stage ── - # Re-triangulate union-find tracks against the post-BA cameras (recovers - # tracks dropped between union-find and BA's filter passes), then run a - # final BA on the augmented set. The final BA reuses the existing tightest - # `reproj_error_thresholds[-1]` for inline filtering — same mechanism as - # the upstream BA loop. if self._use_multi_view_retriangulation: if tracks_2d is None: logger.warning( @@ -984,10 +979,6 @@ def _run_ba_and_evaluate( min_track_length=self._mv_retri_min_track_length, ) if retri_data.number_tracks() > 0: - # Final BA on the retri'd track set. No inline filter — pose AUC - # is set by BA's converged cameras and is independent of any - # downstream track filtering. Callers can filter the returned - # GtsfmData themselves if they want. (optimized_data, filtered_result, valid_mask, _) = self.run_ba_stage_with_filtering( initial_data=retri_data, absolute_pose_priors=absolute_pose_priors, diff --git a/gtsfm/cluster_optimizer/cluster_mvo.py b/gtsfm/cluster_optimizer/cluster_mvo.py index fc1b61907..e243e8c71 100644 --- a/gtsfm/cluster_optimizer/cluster_mvo.py +++ b/gtsfm/cluster_optimizer/cluster_mvo.py @@ -162,6 +162,10 @@ def _run_two_view_estimation( all_two_view_results = cast(AnnotatedGraph[TwoViewResult], gathered_tve_futures) valid_two_view_results = {edge: result for edge, result in all_two_view_results.items() if result.valid()} + n_total = len(all_two_view_results) + n_valid = len(valid_two_view_results) + logger.info("Two-view estimation: %d/%d pairs valid, %d rejected.", n_valid, n_total, n_total - n_valid) + if len(valid_two_view_results) == 0: logger.warning("🔵 ClusterMVO: Skipping cluster as it has no valid two-view results.") diff --git a/gtsfm/cluster_optimizer/cluster_vggt.py b/gtsfm/cluster_optimizer/cluster_vggt.py index 78aeabede..808a0f279 100644 --- a/gtsfm/cluster_optimizer/cluster_vggt.py +++ b/gtsfm/cluster_optimizer/cluster_vggt.py @@ -13,9 +13,10 @@ import gtsfm.common.types as gtsfm_types import gtsfm.utils.metrics as metrics_utils -from gtsfm.bundle.bundle_adjustment import BundleAdjustmentOptions +from gtsfm.bundle.bundle_adjustment import BundleAdjustmentOptions, multi_view_retriangulate_from_2d_tracks from gtsfm.cluster_optimizer.cluster_optimizer_base import ClusterComputationGraph, ClusterContext, ClusterOptimizerBase from gtsfm.common.gtsfm_data import GtsfmData +from gtsfm.common.sfm_track import SfmTrack2d from gtsfm.evaluation.metrics import GtsfmMetric, GtsfmMetricsGroup from gtsfm.frontend.multi_view_tracker import MultiViewTracker from gtsfm.frontend.vggt_geometry_transformer import ( @@ -50,11 +51,20 @@ def _run_cluster_ba( drop_camera_with_no_track: bool = False, min_track_length: int = 2, cluster_label: Optional[str] = None, + tracks_2d: Optional[list[SfmTrack2d]] = None, + use_multi_view_retriangulation: bool = False, ) -> tuple[GtsfmData, GtsfmData]: """Run cluster-level BA on a GtsfmData result. This is a module-level function so it can be used with ``dask.delayed``. + Args: + tracks_2d: (optional) Union-find 2D tracks. Required when + ``use_multi_view_retriangulation=True``. + use_multi_view_retriangulation: When True, after the initial BA, re-triangulate + ``tracks_2d`` against the post-BA cameras (recovers tracks dropped earlier + in the pipeline) and run a second BA on the augmented track set. + Returns: Tuple of (post_ba_result, pre_ba_result). """ @@ -90,6 +100,20 @@ def _run_cluster_ba( post_ba_max_reproj_error ) + # Optional retri stage: re-triangulate union-find tracks against the post-BA + # cameras and run another BA on the augmented set. Recovers tracks dropped + # earlier in the pipeline; mirrors the retri stage in + # BundleAdjustmentOptimizer._run_ba_and_evaluate. + if use_multi_view_retriangulation and tracks_2d is not None: + retri_data = multi_view_retriangulate_from_2d_tracks( + gtsfm_data_with_ba, tracks_2d, min_track_length=min_track_length, + ) + if retri_data.number_tracks() > 0: + gtsfm_data_with_ba, _ = optimizer.run_simple_ba(retri_data) + gtsfm_data_with_ba = gtsfm_data_with_ba.filter_landmark_measurements( + post_ba_max_reproj_error + ) + logger.info( "%s🔍 #valid tracks after BA: %d out of %d", f"[{cluster_label}] " if cluster_label else "", diff --git a/gtsfm/cluster_optimizer/cluster_vggt_with_frontend.py b/gtsfm/cluster_optimizer/cluster_vggt_with_frontend.py index e8246d617..030881f43 100644 --- a/gtsfm/cluster_optimizer/cluster_vggt_with_frontend.py +++ b/gtsfm/cluster_optimizer/cluster_vggt_with_frontend.py @@ -30,6 +30,7 @@ from gtsfm.products.visibility_graph import visibility_graph_keys from gtsfm.two_view_estimator import TwoViewEstimator from gtsfm.ui.gtsfm_process import UiMetadata +from gtsfm.view_graph_estimator.view_graph_calibration import calibrate_view_graph import gtsam from gtsfm.utils import torch as torch_utils @@ -132,6 +133,7 @@ def _build_gtsfm_data_from_vggt_depth( image_indices: tuple[int, ...], num_images: int, min_track_length: int = 2, + refined_intrinsics: Optional[dict[int, gtsfm_types.CALIBRATION_TYPE]] = None, ) -> GtsfmData: """Build GtsfmData using VGGT cameras (rescaled to original resolution) and frontend 2D tracks. @@ -159,14 +161,19 @@ def _build_gtsfm_data_from_vggt_depth( _, H_vggt, W_vggt = dense_points.shape[:3] global_to_local = {gidx: lidx for lidx, gidx in enumerate(image_indices)} - # Register cameras with intrinsics rescaled to original image resolution. + # Register cameras with intrinsics in original image resolution. If + # `refined_intrinsics` is supplied (from view-graph calibration), use those + # directly; otherwise rescale VGGT's predicted intrinsics from VGGT pixel space. gtsfm_data = GtsfmData(number_images=num_images) for global_idx, camera in cameras.items(): if global_idx in image_shapes and global_idx in global_to_local: - _, orig_W = image_shapes[global_idx] - local_idx = global_to_local[global_idx] - scaled_W = float(original_coords[local_idx, 4]) - camera = _scale_camera_intrinsics(camera, scale=orig_W / scaled_W) + if refined_intrinsics is not None: + camera = type(camera)(camera.pose(), refined_intrinsics[global_idx]) + else: + _, orig_W = image_shapes[global_idx] + local_idx = global_to_local[global_idx] + scaled_W = float(original_coords[local_idx, 4]) + camera = _scale_camera_intrinsics(camera, scale=orig_W / scaled_W) gtsfm_data.add_camera(global_idx, camera) for track_2d in tracks_2d: @@ -220,6 +227,38 @@ def _build_gtsfm_data_from_vggt_depth( return gtsfm_data +def _refine_vggt_intrinsics_via_view_graph( + vggt_result: VggtGeometryResult, + v_corr_idxs: dict, + keypoints_list: list, + image_shapes: dict[int, tuple[int, int]], + image_indices: tuple[int, ...], +) -> dict[int, gtsfm_types.CALIBRATION_TYPE]: + """Refine focal lengths via Fetzer joint optimization over F-matrix edges. + + Returns intrinsics in ORIGINAL image coordinates (suitable for use with the + frontend keypoints). VGGT's predicted intrinsics are rescaled to original + image coords first, then handed to `calibrate_view_graph` as the initial + estimate. VGGT's predicted poses are unchanged — only intrinsics are refined. + """ + initial_intrinsics: dict[int, gtsfm_types.CALIBRATION_TYPE] = {} + for local_idx, global_idx in enumerate(image_indices): + if global_idx in vggt_result.cameras and global_idx in image_shapes: + _, orig_W = image_shapes[global_idx] + scaled_W = float(vggt_result.original_coords[local_idx, 4]) + scale = orig_W / scaled_W if scaled_W > 0 else 1.0 + scaled_cam = _scale_camera_intrinsics(vggt_result.cameras[global_idx], scale=scale) + initial_intrinsics[global_idx] = scaled_cam.calibration() + + keypoints = {gidx: keypoints_list[gidx] for gidx in image_indices} + refined, _edges_to_remove = calibrate_view_graph( + v_corr_idxs_dict=v_corr_idxs, + keypoints=keypoints, + initial_intrinsics=initial_intrinsics, + ) + return refined + + class ClusterVGGTWithFrontend(ClusterMVO): """Cluster optimizer that combines a traditional MVO frontend with VGGT poses. @@ -250,6 +289,8 @@ def __init__( save_two_view_viz: bool = False, pose_angular_error_thresh: float = 3, output_worker: Optional[str] = None, + use_view_graph_calibration: bool = False, + use_multi_view_retriangulation: bool = False, ) -> None: super().__init__( correspondence_generator=correspondence_generator, @@ -268,6 +309,8 @@ def __init__( self._metric_constructed_only = metric_constructed_only self._input_mode = input_mode self._seed = seed + self._use_view_graph_calibration = use_view_graph_calibration + self._use_multi_view_retriangulation = use_multi_view_retriangulation self._weights_path = Path(weights_path) if weights_path is not None else None self._loader_kwargs: dict[str, Any] = {} @@ -343,6 +386,18 @@ def create_computation_graph(self, context: ClusterContext) -> ClusterComputatio # 4. Original image shapes (needed to map frontend pixel coords → VGGT pixel coords). image_shapes_graph = delayed(_get_image_shapes)(context.loader, global_indices) + # 4b. Optional: refine VGGT's predicted intrinsics via Fetzer joint + # optimization over the frontend's F-matrices (keeps VGGT's predicted poses). + refined_intrinsics_graph = None + if self._use_view_graph_calibration: + refined_intrinsics_graph = delayed(_refine_vggt_intrinsics_via_view_graph)( + vggt_result_graph, + v_corr_idxs_graph, + frontend_graphs.padded_keypoints, + image_shapes_graph, + global_indices, + ) + # 5. Build GtsfmData: lift 2D tracks to 3D using VGGT depth map. ba_input_graph = delayed(_build_gtsfm_data_from_vggt_depth)( vggt_result_graph, @@ -351,6 +406,7 @@ def create_computation_graph(self, context: ClusterContext) -> ClusterComputatio image_indices=global_indices, num_images=context.num_images, min_track_length=self._min_track_length, + refined_intrinsics=refined_intrinsics_graph, ) # 6. Cluster-level BA. @@ -362,6 +418,8 @@ def create_computation_graph(self, context: ClusterContext) -> ClusterComputatio drop_camera_with_no_track=self._drop_camera_with_no_track, min_track_length=self._min_track_length, cluster_label=context.label, + tracks_2d=tracks_2d_graph, + use_multi_view_retriangulation=self._use_multi_view_retriangulation, ) # 7. Metrics + I/O. diff --git a/gtsfm/configs/vggt_sift_frontend_megaloc.yaml b/gtsfm/configs/vggt_sift_frontend_megaloc.yaml index 9790e1669..a13640396 100644 --- a/gtsfm/configs/vggt_sift_frontend_megaloc.yaml +++ b/gtsfm/configs/vggt_sift_frontend_megaloc.yaml @@ -97,6 +97,10 @@ cluster_optimizer: input_mode: crop seed: 42 model_cache_key: null + # Refine VGGT predicted intrinsics via Fetzer joint optimization on F-matrices. + use_view_graph_calibration: false + # Re-triangulate union-find 2D tracks against post-BA cameras and run another BA. + use_multi_view_retriangulation: false # --- Merging options --- merging_options: diff --git a/gtsfm/configs/vggt_sift_frontend_megaloc_phototourism.yaml b/gtsfm/configs/vggt_sift_frontend_megaloc_phototourism.yaml index 2379ccf97..706030fdb 100644 --- a/gtsfm/configs/vggt_sift_frontend_megaloc_phototourism.yaml +++ b/gtsfm/configs/vggt_sift_frontend_megaloc_phototourism.yaml @@ -97,6 +97,10 @@ cluster_optimizer: input_mode: crop seed: 42 model_cache_key: null + # Refine VGGT predicted intrinsics via Fetzer joint optimization on F-matrices. + use_view_graph_calibration: false + # Re-triangulate union-find 2D tracks against post-BA cameras and run another BA. + use_multi_view_retriangulation: false # --- Merging options --- merging_options: diff --git a/gtsfm/configs/vggt_unified_frontend_megaloc.yaml b/gtsfm/configs/vggt_unified_frontend_megaloc.yaml index 88a3e629b..bbac38fad 100644 --- a/gtsfm/configs/vggt_unified_frontend_megaloc.yaml +++ b/gtsfm/configs/vggt_unified_frontend_megaloc.yaml @@ -92,6 +92,10 @@ cluster_optimizer: input_mode: crop seed: 42 model_cache_key: null + # Refine VGGT predicted intrinsics via Fetzer joint optimization on F-matrices. + use_view_graph_calibration: false + # Re-triangulate union-find 2D tracks against post-BA cameras and run another BA. + use_multi_view_retriangulation: false # --- Merging options --- merging_options: diff --git a/gtsfm/graph_partitioner/metis_partitioner.py b/gtsfm/graph_partitioner/metis_partitioner.py index dd35fd84b..8ce812d3c 100644 --- a/gtsfm/graph_partitioner/metis_partitioner.py +++ b/gtsfm/graph_partitioner/metis_partitioner.py @@ -38,6 +38,7 @@ def __init__( min_child_overlap_for_split: int = 2, min_parent_overlap_for_split: int = 2, split_oversized_nodes: bool = False, + include_all_edges_in_cluster: bool = False, ) -> None: super().__init__(process_name="MetisPartitioner") if min_cameras_to_partition is not None and min_cameras_to_partition < 1: @@ -55,6 +56,7 @@ def __init__( self._min_child_overlap_for_split = min_child_overlap_for_split self._min_parent_overlap_for_split = min_parent_overlap_for_split self._split_oversized_nodes = split_oversized_nodes + self._include_all_edges_in_cluster = include_all_edges_in_cluster @staticmethod def _is_connected(graph: VisibilityGraph) -> bool: @@ -684,11 +686,13 @@ def _cluster_from_clique(self, clique: SymbolicBayesTreeClique, graph: Visibilit child_results = self._merge_small_children_at_level(child_results, graph) descendant_edges = set.union(*(result.edges for result in child_results)) if child_results else set() - # Only keep edges that touch at least one frontal variable from this clique. - candidate_edges = { - (i, j) for i, j in graph if i in keys and j in keys and (i in frontals or j in frontals or not frontals) - } - current_edges = candidate_edges - descendant_edges + if self._include_all_edges_in_cluster: + current_edges = {(i, j) for i, j in graph if i in keys and j in keys} + else: + candidate_edges = { + (i, j) for i, j in graph if i in keys and j in keys and (i in frontals or j in frontals or not frontals) + } + current_edges = candidate_edges - descendant_edges def sorted_edges(edges: set[tuple[int, int]]) -> list[tuple[int, int]]: return sorted(edges) diff --git a/gtsfm/multi_view_optimizer.py b/gtsfm/multi_view_optimizer.py index f5f68a5ca..c7dcfd590 100644 --- a/gtsfm/multi_view_optimizer.py +++ b/gtsfm/multi_view_optimizer.py @@ -183,8 +183,8 @@ def create_computation_graph( # View graph calibration: refine focal lengths from F-matrices if self._run_view_graph_calibration: - all_intrinsics, edges_to_remove = delayed(view_graph_calibration.calibrate_view_graph, nout=2)( - viewgraph_v_corr_idxs_graph, keypoints_graph, all_intrinsics, num_images + all_intrinsics, edges_to_remove = delayed(_calibrate_view_graph_from_lists, nout=2)( + viewgraph_v_corr_idxs_graph, keypoints_graph, all_intrinsics ) # Remove edges with high calibration error viewgraph_i2Ri1_graph, viewgraph_i2Ui1_graph, viewgraph_v_corr_idxs_graph = delayed(_filter_edges, nout=3)( @@ -336,6 +336,22 @@ def _sync_two_view_reports_after_calibration( return synced_reports +def _calibrate_view_graph_from_lists( + v_corr_idxs_dict, + keypoints_list: list[Keypoints], + intrinsics_list: list[gtsfm_types.CALIBRATION_TYPE], + **kwargs, +): + """List-interface wrapper around calibrate_view_graph for use in multi_view_optimizer.""" + kp_dict = {i: kp for i, kp in enumerate(keypoints_list)} + intr_dict = {i: cal for i, cal in enumerate(intrinsics_list)} + refined_dict, edges_to_remove = view_graph_calibration.calibrate_view_graph( + v_corr_idxs_dict, kp_dict, intr_dict, **kwargs + ) + refined_list = [refined_dict.get(i, intrinsics_list[i]) for i in range(len(intrinsics_list))] + return refined_list, edges_to_remove + + def reestimate_relative_poses( i2Ri1_dict: Dict[Tuple[int, int], Rot3], i2Ui1_dict: Dict[Tuple[int, int], Unit3], diff --git a/gtsfm/view_graph_estimator/view_graph_calibration.py b/gtsfm/view_graph_estimator/view_graph_calibration.py index 087e5a624..50f33eece 100644 --- a/gtsfm/view_graph_estimator/view_graph_calibration.py +++ b/gtsfm/view_graph_estimator/view_graph_calibration.py @@ -121,30 +121,28 @@ def _fetzer_residuals( def calibrate_view_graph( v_corr_idxs_dict: Dict[Tuple[int, int], np.ndarray], - keypoints_list: List[Keypoints], - initial_intrinsics: List[gtsfm_types.CALIBRATION_TYPE], - num_images: int, + keypoints: Dict[int, Keypoints], + initial_intrinsics: Dict[int, gtsfm_types.CALIBRATION_TYPE], min_correspondences: int = 30, min_focal_ratio: float = 0.5, max_focal_ratio: float = 2.0, max_edge_error: float = 0.5, -) -> Tuple[List[gtsfm_types.CALIBRATION_TYPE], set]: +) -> Tuple[Dict[int, gtsfm_types.CALIBRATION_TYPE], set]: """Refine camera focal lengths via joint Fetzer optimization over all F-matrix edges. Also filters edges with high calibration error (GLOMAP FilterImagePairs). Args: v_corr_idxs_dict: Verified correspondence indices per image pair. - keypoints_list: Keypoints for all images. - initial_intrinsics: Initial intrinsics (e.g., from EXIF or heuristic). - num_images: Total number of images. + keypoints: Keypoints keyed by image index. + initial_intrinsics: Initial intrinsics keyed by image index. min_correspondences: Minimum correspondences to attempt F estimation. min_focal_ratio: Minimum allowed ratio of optimized/initial focal length. max_focal_ratio: Maximum allowed ratio of optimized/initial focal length. max_edge_error: Maximum Fetzer residual norm to keep an edge. Returns: - Refined intrinsics list (same length as initial_intrinsics). + Refined intrinsics dict (same keys as initial_intrinsics). Set of edge keys (i1, i2) to remove from the view graph. """ # Step 1: Estimate F-matrices and collect optimization edges. @@ -155,8 +153,8 @@ def calibrate_view_graph( if v_corr_idxs.shape[0] < min_correspondences: continue - coords_i1 = keypoints_list[i1].coordinates[v_corr_idxs[:, 0]] - coords_i2 = keypoints_list[i2].coordinates[v_corr_idxs[:, 1]] + coords_i1 = keypoints[i1].coordinates[v_corr_idxs[:, 0]] + coords_i2 = keypoints[i2].coordinates[v_corr_idxs[:, 1]] F = estimate_fundamental_from_correspondences(coords_i1, coords_i2) if F is None: @@ -212,7 +210,7 @@ def calibrate_view_graph( optimized_focals = result.x # Step 4: Validate and build refined intrinsics. - refined = list(initial_intrinsics) + refined = dict(initial_intrinsics) num_refined = 0 num_rejected = 0