diff --git a/rapida/cli/__init__.py b/rapida/cli/__init__.py index 9ac19b6..6df47f8 100644 --- a/rapida/cli/__init__.py +++ b/rapida/cli/__init__.py @@ -26,7 +26,7 @@ -@click.group(cls=RapidaCommandGroup, context_settings=dict(help_option_names=['-h', '--help'])) +@click.group(cls=RapidaCommandGroup, ) @click.pass_context def cli(ctx): @@ -36,8 +36,7 @@ def cli(ctx): representing exposure and vulnerability aspects of geospatial risk induced by natural hazards. """ - # 1. Initialize your structured logging engine - logger = setup_logger(name='rapida', make_root=False) + # 2. Ensure ctx.obj is initialized as a container dictionary ctx.ensure_object(dict) diff --git a/rapida/cli/aclick.py b/rapida/cli/aclick.py index 62baeae..311317a 100644 --- a/rapida/cli/aclick.py +++ b/rapida/cli/aclick.py @@ -52,7 +52,7 @@ def list_commands(self, ctx): def add_command(self, cmd: click.Command, name: str = None) -> None: # Catch-all: forces help display if a subcommand is run empty - cmd.no_args_is_help = True + #cmd.no_args_is_help = True # 2. Automatically inject --debug into every registered subcommand cmd.params.append( click.Option( diff --git a/rapida/cli/admin.py b/rapida/cli/admin.py index cc5a2b3..25f6dcd 100644 --- a/rapida/cli/admin.py +++ b/rapida/cli/admin.py @@ -76,12 +76,7 @@ def admin(): default=False, help="Keep admin polygons that are disputed areas with invalid country code" ) -@click.option('--debug', - is_flag=True, - default=False, - help="Set log level to debug" -) def osm(bbox=None,admin_level=None, osm_level=None, clip=False, h3id_precision=7, destination_path=None, debug=False, keep_disputed_areas=None): """ Fetch admin boundaries from OSM @@ -141,12 +136,8 @@ def osm(bbox=None,admin_level=None, osm_level=None, clip=False, h3id_precision=7 default=False, help="Keep admin polygons that are disputed areas with invalid country code" ) -@click.option('--debug', - is_flag=True, - default=False, - help="Set log level to debug" -) -def cgaz(bbox=None,admin_level=None, clip=False, destination_path=None, debug=False, keep_disputed_areas=None, h3id_precision=None): + +def cgaz(bbox=None,admin_level=None, clip=False, destination_path=None, keep_disputed_areas=None, h3id_precision=None): """ Fetch admin boundaries from CGAZ @@ -161,7 +152,6 @@ def cgaz(bbox=None,admin_level=None, clip=False, destination_path=None, debug=F Rapida admin cgaz --bbox 33.681335,-0.131836,35.966492,1.158979 -l 2 --clip /data/admin2_cgaz.gpkg """ - setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO) if not is_rapida_initialized(): return diff --git a/rapida/cli/assess.py b/rapida/cli/assess.py index 60b14d2..42bc31e 100644 --- a/rapida/cli/assess.py +++ b/rapida/cli/assess.py @@ -179,13 +179,9 @@ def build_variable_help(): help="Optional. A project folder with rapida.json can be specified. If not, current directory is considered as a project folder.") @click.option('--force', '-f', default=False, show_default=True,is_flag=True, help=f'Force assess components. Downloaded data or computed data will be ignored and recomputed.') -@click.option('--debug', - is_flag=True, - default=False, - help="Set log level to debug" - ) + @click.pass_context -def assess(ctx, all=False, components=None, variables=None, year=None, datetime_range=None, outage_date=None, cloud_cover=None, project: str = None, force=False, debug=False): +def assess(ctx, all=False, components=None, variables=None, year=None, datetime_range=None, outage_date=None, cloud_cover=None, project: str = None, force=False): """ Assess/evaluate a specific geospatial exposure components/variables @@ -214,7 +210,6 @@ def assess(ctx, all=False, components=None, variables=None, year=None, datetime rapida assess -c landuse -dt 2025-02-01/2025-05-31 -cc 10: Search Sentinel 2 item which is less than 10% of cloud cover from February to May 2025. """ - setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO) if not is_rapida_initialized(): return diff --git a/rapida/cli/auth.py b/rapida/cli/auth.py index 06a9999..90e60ae 100644 --- a/rapida/cli/auth.py +++ b/rapida/cli/auth.py @@ -13,16 +13,12 @@ @click.command(short_help="authenticate with UNDP account") @click.option('-c', '--cache_dir', default=None, type=click.Path(), help="Optional. Folder path to store token_cache.json. Default is ~/.rapida folder to store cache file.") -@click.option('--debug', - is_flag=True, - default=False, - help="Set log level to debug" - ) -def auth(cache_dir, debug=False): + +def auth(cache_dir): """ Authenticate with UNDP account """ - setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO) + # setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO) credential = SurgeTokenCredential(cache_dir=cache_dir) token, exp_ts = credential.get_token("https://storage.azure.com/.default",) diff --git a/rapida/cli/create.py b/rapida/cli/create.py index 3e88a37..9d6fd37 100644 --- a/rapida/cli/create.py +++ b/rapida/cli/create.py @@ -18,12 +18,8 @@ help='Full path to the mask dataset in any GDAL/OGR supported format. Can be vector or raster' ) @click.option('-c', '--comment', required=False, type=str, help='Any comment you might want to add into the project config' ) -@click.option('--debug', - is_flag=True, - default=False, - help="Set log level to debug" - ) -def create(project_name: str, destination_folder: str = None, polygons=None, mask=None, comment=None, debug=False): + +def create(project_name: str, destination_folder: str = None, polygons=None, mask=None, comment=None, ): """ Create a RAPIDA project in a new folder. As default, it creates a project folder with the same name as user specified. @@ -42,7 +38,6 @@ def create(project_name: str, destination_folder: str = None, polygons=None, mas Second positional argument is optional. If skipped, current directory is used. """ - setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO) if not is_rapida_initialized(): return diff --git a/rapida/cli/delete.py b/rapida/cli/delete.py index 06c3369..e0ac3dd 100644 --- a/rapida/cli/delete.py +++ b/rapida/cli/delete.py @@ -18,11 +18,8 @@ is_flag=True, default=False, help="Optional. If True, it will automatically answer yes to prompts. Default is False.") -@click.option('--debug', - is_flag=True, - default=False, - help="Set log level to debug") -def delete(project: str, no_input: bool = False, debug: bool =False): + +def delete(project: str, no_input: bool = False, ): """ Delete a project from local storage and Azure File Share if it was uploaded. @@ -37,7 +34,7 @@ def delete(project: str, no_input: bool = False, debug: bool =False): rapida delete --project=: If you are not in a project folder """ - setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO) + #setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO) if not is_rapida_initialized(): return diff --git a/rapida/cli/init.py b/rapida/cli/init.py index 3ff4f36..d2bcd07 100644 --- a/rapida/cli/init.py +++ b/rapida/cli/init.py @@ -78,11 +78,7 @@ def setup_prompt(session: Session): is_flag=True, default=False, help="Optional. If True, it will automatically answer yes to prompts. Default is False.") -@click.option('--debug', - is_flag=True, - default=False, - help="Set log level to debug" - ) + def init(no_input: bool = False, debug=False): """ Initialize RAPIDA tool. This command will creates a config file under user home directory (~/.rapida/config.json) to setup RAPIDA tool. diff --git a/rapida/cli/ntl.py b/rapida/cli/ntl.py index 2454dc9..c170ca0 100644 --- a/rapida/cli/ntl.py +++ b/rapida/cli/ntl.py @@ -415,6 +415,7 @@ async def bulk_download(ctx, bbox:tuple[float, float, float, float]=None, start_ ) + @ntl.command(short_help=f'Find and download best NTL data for a specific event associated with an area and date ') @click.option('-b', '--bbox', diff --git a/rapida/cli/publish.py b/rapida/cli/publish.py index 38797e1..48475f6 100644 --- a/rapida/cli/publish.py +++ b/rapida/cli/publish.py @@ -56,12 +56,8 @@ def validate_layers(ctx, param, value): is_flag=True, default=False, help="Optional. If True, it will automatically answer yes to prompts. Default is False.") -@click.option('--debug', - is_flag=True, - default=False, - help="Set log level to debug" - ) -def publish(project: str, no_input: bool = False, debug: bool =False, layer=None): + +def publish(project: str, no_input: bool = False, layer=None): """ Publish project data to Azure and open GeoHub registration page URL. @@ -80,7 +76,6 @@ def publish(project: str, no_input: bool = False, debug: bool =False, layer=None rapida publish -l stats.elegrid -l elegrid: publish only two layers from geopackage """ - setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO) if not is_rapida_initialized(): return diff --git a/rapida/cli/upload.py b/rapida/cli/upload.py index a3e355d..56bb03f 100644 --- a/rapida/cli/upload.py +++ b/rapida/cli/upload.py @@ -28,11 +28,8 @@ is_flag=True, default=False, help="Optional. If True, it will automatically answer yes to prompts. Default is False.") -@click.option('--debug', - is_flag=True, - default=False, - help="Set log level to debug") -def upload(project=None, max_concurrency=4, force=False, no_input: bool = False, debug: bool =False): + +def upload(project=None, max_concurrency=4, force=False, no_input: bool = False): """ Upload an entire project folder to Azure File Share @@ -46,8 +43,6 @@ def upload(project=None, max_concurrency=4, force=False, no_input: bool = False, Use `--no-input` to disable prompting. Default is False. """ - setup_logger(name='rapida', level=logging.DEBUG if debug else logging.INFO) - if not is_rapida_initialized(): return diff --git a/rapida/ntl/nasa/io.py b/rapida/ntl/nasa/io.py index b07554f..ba8cfab 100644 --- a/rapida/ntl/nasa/io.py +++ b/rapida/ntl/nasa/io.py @@ -391,7 +391,6 @@ async def download_tile( async def bulk_download(bbox:tuple[numbers.Number]=None, start_date:datetime=None, end_date:datetime=None, stream:str = None, products:str=None, dst_dir:str=None, progress=None): - results = stac_search(stream=stream, products=products, dt=[start_date, end_date], bbox=bbox,push_to_cache=False) @@ -407,6 +406,7 @@ async def bulk_download(bbox:tuple[numbers.Number]=None, start_date:datetime=Non tasks = [] semaphore = asyncio.Semaphore(5) progress_task = None + downloaded_file = None if results: dest_path = Path(dst_dir) dest_path.mkdir(parents=True, exist_ok=True) @@ -433,7 +433,7 @@ async def bulk_download(bbox:tuple[numbers.Number]=None, start_date:datetime=Non if progress and progress_task is not None: progress.update(progress_task,description=f'[green]🡇 {downloaded_file.name}', advance=1) - downloaded_files.append(str(downloaded_file)) + downloaded_files.append(downloaded_file) except Exception as e: logger.error(e) @@ -442,4 +442,20 @@ async def bulk_download(bbox:tuple[numbers.Number]=None, start_date:datetime=Non if not atask.done(): atask.cancel() await asyncio.gather(*tasks, return_exceptions=True) - raise \ No newline at end of file + raise + files = {} + for df in downloaded_files: + m = nasaconst.NTL_FILENAME_PATTERN.match(df.name) + meta = m.groupdict() + if '_' in meta['product']: + product = meta['product'].split('_') + else: + product = meta['product'] + _, level = product.split('46') + + + # if len(products) == 1: + # product = len(produ) + # return extract(image_files=downloaded_files,sds_name=nasaconst.SUB_DATASETS[level]) + + return downloaded_file \ No newline at end of file diff --git a/rapida/ntl/nasa/search.py b/rapida/ntl/nasa/search.py index 5ea3d6f..9d46628 100644 --- a/rapida/ntl/nasa/search.py +++ b/rapida/ntl/nasa/search.py @@ -1,4 +1,7 @@ import os.path + +from sympy.physics.units import minute, second + from rapida.ntl import cache from rapida.ntl.nasa import const from rapida.ntl.utils import timestamp_format @@ -290,6 +293,14 @@ def stac_search(stream:str=None, processing_level:Optional[str]=None, products:l catalog_collections = const.COLLECTIONS[catalog_name] catalog_processing_levels = catalog_collections.keys() + try: + s,e = dt + s = s.replace(hour=0, minute=0, second=0) + e = e.replace(hour=23, minute=59, second=59) + dt = [dt.isoformat() + "Z" for dt in (s, e)] + + except TypeError: + pass if not products: available_collections = sorted(catalog_collections[processing_level], reverse=True) diff --git a/rapida/ntl/noaa/cmask.py b/rapida/ntl/noaa/cmask.py index a181878..9762566 100644 --- a/rapida/ntl/noaa/cmask.py +++ b/rapida/ntl/noaa/cmask.py @@ -83,6 +83,7 @@ def select_required_granules(sorted_granules: list, bbox: tuple, progress:Progre try: if progress: progress_task = progress.add_task(description=f'Selecting best granules tha cover {bbox}', total=None) + combos = {} for combo_size in range(2, len(sorted_granules) + 1): if progress and progress_task: progress.update(progress_task, description=f'Evaluating granules by pairs of {combo_size} ') @@ -94,13 +95,16 @@ def select_required_granules(sorted_granules: list, bbox: tuple, progress:Progre # 3. FLOATING-POINT SAFE COVERAGE CHECK # We use difference() because .within() can fail on microscopic 1e-15 gap artifacts uncovered = boxpoly.difference(merged_poly) - + combos[int((boxpoly.area-uncovered.area)/boxpoly.area*100)] = combo if uncovered.is_empty or uncovered.area < 1e-6: logger.debug(f"Success: BBOX covered perfectly by {combo_size} granule(s).") return combo # Exits immediately with the absolute minimum required set logger.warning("Exhausted all combinations. BBOX cannot be fully covered by available data.") - return tuple() + logger.warning(f"Selecting max of {combos}.") + + w = max(combos) + return combos[w] finally: if progress and progress_task: progress.remove_task(progress_task) diff --git a/rapida/ntl/outage.py b/rapida/ntl/outage.py index 09d332f..04c9e8d 100644 --- a/rapida/ntl/outage.py +++ b/rapida/ntl/outage.py @@ -78,19 +78,28 @@ async def detect_outage( cmask_path=product_files[noaa_const.CM], target_area=target_area ) + # 3. Create a STRICTLY DAILY invalid mask # It inherits the base analysis_mask (water/bg), but does NOT inherit yesterday's clouds daily_invalid_mask = np.isnan(daily_data) | analysis_mask - + daily_data_label = f'{noaa_const.SDR}_{timestamp}' if mask_clouds is True: is_cloudy = cloud_mask > 1 - daily_invalid_mask |= is_cloudy + cloudy_percentage = is_cloudy[is_cloudy].size/is_cloudy.size *100 + if round(cloudy_percentage ) >= 95: + logger.info(f'There are too many clouds in the bbox: {cloudy_percentage:.2f}%. Skipping cloud mask' ) + else: + daily_invalid_mask |= is_cloudy + daily_invalid_percentage = daily_invalid_mask[daily_invalid_mask].size/daily_invalid_mask.size*100 + if round(daily_invalid_percentage) >= 97: + logger.info(f'There are too many masked/invalid pixels in the bbox for {daily_data_label}: {daily_invalid_percentage}%. It will be skipped' ) + continue log_daily_data = np.zeros_like(daily_data) log_daily_data[~daily_invalid_mask] = np.log1p(daily_data[~daily_invalid_mask]) # Save individual daily raw data if needed - daily_data_label = f'{noaa_const.SDR}_{timestamp}' + arrays[daily_data_label] = log_daily_data # Run the NTL statistics engine on today's clear pixels @@ -114,8 +123,7 @@ async def detect_outage( # Update our tracking mask to protect these pixels from future overlaps valid_mask |= daily_valid - # Update our tracking mask to remember we saw these pixels - valid_mask |= daily_valid + ts = f'{nominal_date:%Y%m%d}'