fix default debug

rapida/cli/__init__.py

@@ -26,7 +26,7 @@
 
 
 
-@click.group(cls=RapidaCommandGroup, context_settings=dict(help_option_names=['-h', '--help']))
+@click.group(cls=RapidaCommandGroup, )
 
 @click.pass_context
 def cli(ctx):
@@ -36,8 +36,7 @@ def cli(ctx):
     representing exposure and vulnerability aspects of geospatial risk induced
     by natural hazards.
     """
-    # 1. Initialize your structured logging engine
-    logger = setup_logger(name='rapida', make_root=False)
+
 
     # 2. Ensure ctx.obj is initialized as a container dictionary
     ctx.ensure_object(dict)

rapida/cli/aclick.py

@@ -52,7 +52,7 @@ def list_commands(self, ctx):
 
     def add_command(self, cmd: click.Command, name: str = None) -> None:
         # Catch-all: forces help display if a subcommand is run empty
-        cmd.no_args_is_help = True
+        #cmd.no_args_is_help = True
         # 2. Automatically inject --debug into every registered subcommand
         cmd.params.append(
             click.Option(

rapida/cli/admin.py

@@ -76,12 +76,7 @@ def admin():
     default=False,
     help="Keep admin polygons that are disputed areas with invalid country code"
 )
-@click.option('--debug',
 
-    is_flag=True,
-    default=False,
-    help="Set log level to debug"
-)
 def osm(bbox=None,admin_level=None, osm_level=None, clip=False, h3id_precision=7, destination_path=None, debug=False, keep_disputed_areas=None):
     """
     Fetch admin boundaries from OSM
@@ -141,12 +136,8 @@ def osm(bbox=None,admin_level=None, osm_level=None, clip=False, h3id_precision=7
     default=False,
     help="Keep admin polygons that are disputed areas with invalid country code"
 )
-@click.option('--debug',
-    is_flag=True,
-    default=False,
-    help="Set log level to debug"
-)
-def cgaz(bbox=None,admin_level=None,  clip=False, destination_path=None, debug=False, keep_disputed_areas=None, h3id_precision=None):
+
+def cgaz(bbox=None,admin_level=None,  clip=False, destination_path=None,  keep_disputed_areas=None, h3id_precision=None):
     """
     Fetch admin boundaries from CGAZ
 
@@ -161,7 +152,6 @@ def cgaz(bbox=None,admin_level=None,  clip=False, destination_path=None, debug=F
     Rapida admin cgaz --bbox 33.681335,-0.131836,35.966492,1.158979 -l 2 --clip /data/admin2_cgaz.gpkg
 
     """
-    setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO)
 
     if not is_rapida_initialized():
         return

rapida/cli/assess.py

@@ -179,13 +179,9 @@ def build_variable_help():
               help="Optional. A project folder with rapida.json can be specified. If not, current directory is considered as a project folder.")
 @click.option('--force', '-f', default=False, show_default=True,is_flag=True,
               help=f'Force assess components. Downloaded data or computed data will be ignored and recomputed.')
-@click.option('--debug',
-              is_flag=True,
-              default=False,
-              help="Set log level to debug"
-              )
+
 @click.pass_context
-def assess(ctx, all=False, components=None,  variables=None, year=None, datetime_range=None, outage_date=None, cloud_cover=None, project: str = None, force=False, debug=False):
+def assess(ctx, all=False, components=None,  variables=None, year=None, datetime_range=None, outage_date=None, cloud_cover=None, project: str = None, force=False):
     """
     Assess/evaluate a specific geospatial exposure components/variables
 
@@ -214,7 +210,6 @@ def assess(ctx, all=False, components=None,  variables=None, year=None, datetime
     rapida assess -c landuse -dt 2025-02-01/2025-05-31 -cc 10: Search Sentinel 2 item which is less than 10% of cloud cover from February to May 2025.
 
     """
-    setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO)
 
     if not is_rapida_initialized():
         return

rapida/cli/auth.py

@@ -13,16 +13,12 @@
 @click.command(short_help="authenticate with UNDP account")
 @click.option('-c', '--cache_dir', default=None, type=click.Path(),
               help="Optional. Folder path to store token_cache.json. Default is ~/.rapida folder to store cache file.")
-@click.option('--debug',
-              is_flag=True,
-              default=False,
-              help="Set log level to debug"
-              )
-def auth(cache_dir, debug=False):
+
+def auth(cache_dir):
     """
     Authenticate with  UNDP account
     """
-    setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO)
+    # setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO)
 
     credential = SurgeTokenCredential(cache_dir=cache_dir)
     token, exp_ts = credential.get_token("https://storage.azure.com/.default",)

rapida/cli/create.py

@@ -18,12 +18,8 @@
               help='Full path to the mask dataset in any GDAL/OGR supported format. Can be vector or raster' )
 @click.option('-c', '--comment', required=False, type=str,
               help='Any comment you might want to add into the project config' )
-@click.option('--debug',
-              is_flag=True,
-              default=False,
-              help="Set log level to debug"
-              )
-def create(project_name: str, destination_folder: str = None, polygons=None, mask=None, comment=None, debug=False):
+
+def create(project_name: str, destination_folder: str = None, polygons=None, mask=None, comment=None, ):
     """
     Create a RAPIDA project in a new folder. As default, it creates a project folder with the same name as user specified.
 
@@ -42,7 +38,6 @@ def create(project_name: str, destination_folder: str = None, polygons=None, mas
         Second positional argument is optional. If skipped, current directory is used.
 
     """
-    setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO)
 
     if not is_rapida_initialized():
         return

rapida/cli/delete.py

@@ -18,11 +18,8 @@
               is_flag=True,
               default=False,
               help="Optional. If True, it will automatically answer yes to prompts. Default is False.")
-@click.option('--debug',
-              is_flag=True,
-              default=False,
-              help="Set log level to debug")
-def delete(project: str, no_input: bool = False, debug: bool =False):
+
+def delete(project: str, no_input: bool = False, ):
     """
     Delete a project from local storage and Azure File Share if it was uploaded.
 
@@ -37,7 +34,7 @@ def delete(project: str, no_input: bool = False, debug: bool =False):
         rapida delete --project=<project folder path>: If you are not in a project folder
 
     """
-    setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO)
+    #setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO)
 
     if not is_rapida_initialized():
         return

rapida/cli/init.py

@@ -78,11 +78,7 @@ def setup_prompt(session: Session):
               is_flag=True,
               default=False,
               help="Optional. If True, it will automatically answer yes to prompts. Default is False.")
-@click.option('--debug',
-              is_flag=True,
-              default=False,
-              help="Set log level to debug"
-              )
+
 def init(no_input: bool = False, debug=False):
     """
     Initialize RAPIDA tool. This command will creates a config file under user home directory (~/.rapida/config.json) to setup RAPIDA tool.

rapida/cli/ntl.py

@@ -415,6 +415,7 @@ async def bulk_download(ctx, bbox:tuple[float, float, float, float]=None, start_
     )
 
 
+
 @ntl.command(short_help=f'Find and download best NTL data for a specific event associated with an area and date ')
 
 @click.option('-b', '--bbox',

rapida/cli/publish.py

@@ -56,12 +56,8 @@ def validate_layers(ctx, param, value):
               is_flag=True,
               default=False,
               help="Optional. If True, it will automatically answer yes to prompts. Default is False.")
-@click.option('--debug',
-              is_flag=True,
-              default=False,
-              help="Set log level to debug"
-              )
-def publish(project: str, no_input: bool = False, debug: bool =False, layer=None):
+
+def publish(project: str, no_input: bool = False,  layer=None):
     """
     Publish project data to Azure and open GeoHub registration page URL.
 
@@ -80,7 +76,6 @@ def publish(project: str, no_input: bool = False, debug: bool =False, layer=None
         rapida publish -l stats.elegrid -l elegrid: publish only two layers from geopackage
 
     """
-    setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO)
 
     if not is_rapida_initialized():
         return

rapida/cli/upload.py

@@ -28,11 +28,8 @@
               is_flag=True,
               default=False,
               help="Optional. If True, it will automatically answer yes to prompts. Default is False.")
-@click.option('--debug',
-              is_flag=True,
-              default=False,
-              help="Set log level to debug")
-def upload(project=None, max_concurrency=4, force=False, no_input: bool = False, debug: bool =False):
+
+def upload(project=None, max_concurrency=4, force=False, no_input: bool = False):
     """
     Upload an entire project folder to Azure File Share
 
@@ -46,8 +43,6 @@ def upload(project=None, max_concurrency=4, force=False, no_input: bool = False,
 
     Use `--no-input` to disable prompting. Default is False.
     """
-    setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO)
-
     if not is_rapida_initialized():
         return
 

rapida/ntl/nasa/io.py

@@ -391,7 +391,6 @@ async def download_tile(
 async def bulk_download(bbox:tuple[numbers.Number]=None, start_date:datetime=None, end_date:datetime=None,
                            stream:str = None, products:str=None, dst_dir:str=None, progress=None):
 
-
     results = stac_search(stream=stream, products=products,
                        dt=[start_date, end_date], bbox=bbox,push_to_cache=False)
 
@@ -407,6 +406,7 @@ async def bulk_download(bbox:tuple[numbers.Number]=None, start_date:datetime=Non
     tasks = []
     semaphore = asyncio.Semaphore(5)
     progress_task = None
+    downloaded_file = None
     if results:
         dest_path = Path(dst_dir)
         dest_path.mkdir(parents=True, exist_ok=True)
@@ -433,7 +433,7 @@ async def bulk_download(bbox:tuple[numbers.Number]=None, start_date:datetime=Non
 
                     if progress and progress_task is not None:
                         progress.update(progress_task,description=f'[green]🡇 {downloaded_file.name}', advance=1)
-                    downloaded_files.append(str(downloaded_file))
+                    downloaded_files.append(downloaded_file)
                 except Exception as e:
                     logger.error(e)
 
@@ -442,4 +442,20 @@ async def bulk_download(bbox:tuple[numbers.Number]=None, start_date:datetime=Non
                         if not atask.done():
                             atask.cancel()
                     await asyncio.gather(*tasks, return_exceptions=True)
-                    raise
+                    raise
+            files = {}
+            for df in downloaded_files:
+                m = nasaconst.NTL_FILENAME_PATTERN.match(df.name)
+                meta = m.groupdict()
+                if '_' in meta['product']:
+                    product = meta['product'].split('_')
+                else:
+                    product = meta['product']
+                _, level = product.split('46')
+
+
+            # if len(products) == 1:
+            #     product = len(produ)
+            #     return extract(image_files=downloaded_files,sds_name=nasaconst.SUB_DATASETS[level])
+
+    return downloaded_file

rapida/ntl/nasa/search.py

@@ -1,4 +1,7 @@
 import os.path
+
+from sympy.physics.units import minute, second
+
 from rapida.ntl import cache
 from rapida.ntl.nasa import const
 from rapida.ntl.utils import timestamp_format
@@ -290,6 +293,14 @@ def stac_search(stream:str=None, processing_level:Optional[str]=None, products:l
     catalog_collections = const.COLLECTIONS[catalog_name]
     catalog_processing_levels = catalog_collections.keys()
 
+    try:
+        s,e = dt
+        s = s.replace(hour=0, minute=0, second=0)
+        e = e.replace(hour=23, minute=59, second=59)
+        dt = [dt.isoformat() + "Z" for dt in (s, e)]
+
+    except TypeError:
+        pass
     if not products:
 
         available_collections = sorted(catalog_collections[processing_level], reverse=True)

rapida/ntl/noaa/cmask.py

@@ -83,6 +83,7 @@ def select_required_granules(sorted_granules: list, bbox: tuple, progress:Progre
     try:
         if progress:
             progress_task = progress.add_task(description=f'Selecting best granules tha cover {bbox}', total=None)
+        combos = {}
         for combo_size in range(2, len(sorted_granules) + 1):
             if progress and progress_task:
                 progress.update(progress_task, description=f'Evaluating granules by pairs of {combo_size} ')
@@ -94,13 +95,16 @@ def select_required_granules(sorted_granules: list, bbox: tuple, progress:Progre
                 # 3. FLOATING-POINT SAFE COVERAGE CHECK
                 # We use difference() because .within() can fail on microscopic 1e-15 gap artifacts
                 uncovered = boxpoly.difference(merged_poly)
-
+                combos[int((boxpoly.area-uncovered.area)/boxpoly.area*100)] = combo
                 if uncovered.is_empty or uncovered.area < 1e-6:
                     logger.debug(f"Success: BBOX covered perfectly by {combo_size} granule(s).")
                     return combo  # Exits immediately with the absolute minimum required set
 
         logger.warning("Exhausted all combinations. BBOX cannot be fully covered by available data.")
-        return tuple()
+        logger.warning(f"Selecting max of {combos}.")
+
+        w = max(combos)
+        return combos[w]
     finally:
         if progress and progress_task:
             progress.remove_task(progress_task)

rapida/ntl/outage.py

@@ -78,19 +78,28 @@ async def detect_outage(
                 cmask_path=product_files[noaa_const.CM], target_area=target_area
             )
 
+
             # 3. Create a STRICTLY DAILY invalid mask
             # It inherits the base analysis_mask (water/bg), but does NOT inherit yesterday's clouds
             daily_invalid_mask = np.isnan(daily_data) | analysis_mask
-
+            daily_data_label = f'{noaa_const.SDR}_{timestamp}'
             if mask_clouds is True:
                 is_cloudy = cloud_mask > 1
-                daily_invalid_mask |= is_cloudy
+                cloudy_percentage = is_cloudy[is_cloudy].size/is_cloudy.size *100
+                if round(cloudy_percentage ) >= 95:
+                    logger.info(f'There are too many clouds in the bbox: {cloudy_percentage:.2f}%. Skipping cloud mask' )
+                else:
+                    daily_invalid_mask |= is_cloudy
+            daily_invalid_percentage = daily_invalid_mask[daily_invalid_mask].size/daily_invalid_mask.size*100
+            if round(daily_invalid_percentage) >= 97:
+                logger.info(f'There are too many masked/invalid pixels in the bbox for {daily_data_label}: {daily_invalid_percentage}%. It will be skipped' )
+                continue
 
             log_daily_data = np.zeros_like(daily_data)
             log_daily_data[~daily_invalid_mask] = np.log1p(daily_data[~daily_invalid_mask])
 
             # Save individual daily raw data if needed
-            daily_data_label = f'{noaa_const.SDR}_{timestamp}'
+
             arrays[daily_data_label] = log_daily_data
 
             # Run the NTL statistics engine on today's clear pixels
@@ -114,8 +123,7 @@ async def detect_outage(
             # Update our tracking mask to protect these pixels from future overlaps
             valid_mask |= daily_valid
 
-            # Update our tracking mask to remember we saw these pixels
-            valid_mask |= daily_valid
+
 
         ts = f'{nominal_date:%Y%m%d}'