diff --git a/CHANGELOG.md b/CHANGELOG.md index 257671eb..aa74517d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,11 +25,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - --> -## [v3.2.3](https://github.com/TUDelftGeodesy/caroline/tree/main) (28-Aug-2026, [diff](https://github.com/TUDelftGeodesy/caroline/compare/e46493f87582bfbdb2a8ae17dd46da4879538b27...main)) +## [v4.0.0](https://github.com/TUDelftGeodesy/caroline/tree/main) (22-Sep-2026, [diff](https://github.com/TUDelftGeodesy/caroline/compare/888f022ba797e4cb63b5904854efacbdc4cc9f6c...main)) ### Changed -- `snap_run` now runs on 12 cores instead of 8 +- `crop_to_raw` has been renamed to `reduce_SLC_matlab` (abbr. `RM`) +- `crop_to_zarr` has been renamed to `reduce_SLC_python` (abbr. `RP`) +- `znap_to_raw` has been renamed to `merge_to_stack_matlab` (abbr. `MM`) +- `znap_to_zarr` has been renamed to `merge_to_stack_python` (abbr. `MP`) +- `doris` has been renamed to `doris_v5` +- `deinsar` has been renamed to `doris_v4` +- `doris_cleanup` has been renamed to `doris_v5_cleanup` +- `snap_run` has been renamed to `snap` (abbr. `SN`) +- `snap_permissions` has been renamed to `snap_fix_permissions` (abbr. `SF`) +- `depsi` has been renamed to `depsi_matlab` (abbr. `DM`) +- `stm_generation` has been renamed to `generate_partitioned_stm` (abbr. `GS`) +- `mrm` has been renamed to `create_mrm` (abbr. `CM`) +- `tarball` has been renamed to `create_tarball` (abbr. `CT`) +- `portal_upload` has been renamed to `set_portal_upload_flag` +- The Coregistration submodule and Cropping submodule have been merged into the Stack Generation submodule +- `asf_search` dependency version bumped to `v13.0.0` to avoid an error being thrown during `utils.identify_s1_orbits_in_aoi` +- All instances of job names in variable names have been replaced by their new names +- The parameter group `stm_generation:stm_generation-settings:ps-selection:initialization-settings` has been renamed to `generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:init-settings` +- The parameter `stm_generation:stm_generation-settings:ps-selection:initialization-settings:initialization-length` has been renamed to `generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:init-settings:init-length` +- The parameter `stm_generation:stm_generation-settings:partitioning:undifferenced-output-layers` has been renamed to `generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:undifferenced-output-lyrs` +- The parameter `stm_generation:stm_generation-settings:partitioning:single-difference-output-layers` has been renamed to `generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:single-difference-output-lyrs` +- The parameter `stm_generation:stm_generation-settings:incremental-statistics:recalibration-jump-size` has been renamed to `generate_partitioned_stm:generate_partitioned_stm-settings:incremental-statistics:recal-jump-size` + +### Fixed +- Separate `create_mrm` and `depsi_post` on one side, and `set_portal_upload_flag` on the other side into different step machine fields, so that it is possible to run `depsi_post` without pushing to the portal + +## [v3.2.3](https://github.com/TUDelftGeodesy/caroline/tree/888f022ba797e4cb63b5904854efacbdc4cc9f6c) (28-Aug-2026, [diff](https://github.com/TUDelftGeodesy/caroline/compare/e46493f87582bfbdb2a8ae17dd46da4879538b27...888f022ba797e4cb63b5904854efacbdc4cc9f6c)) + +### Changed +- `snap_run` now runs on 12 cores instead of 8 ## [v3.2.2](https://github.com/TUDelftGeodesy/caroline/tree/e46493f87582bfbdb2a8ae17dd46da4879538b27) (07-Aug-2026, [diff](https://github.com/TUDelftGeodesy/caroline/compare/d7a236c3a1a1464d77e26d7f1a37f4ef833818ad...e46493f87582bfbdb2a8ae17dd46da4879538b27)) diff --git a/README.md b/README.md index 56f48678..28822664 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Welcome to the CAROLINE Project source code repository. CAROLINE (Contextual and Autonomous processing of satellite Radar Observations for Learning and Interpreting the Natural and built Environment) is an InSAR data processing system that automates the InSAR processing chain from download to final output. It allows for continuous product generation, where predefined products are created as new datasets are downloaded, as wel as ad-hoc product creation. -The current documentation still partly pertains to v0.1.0. In v3.X, the continuous and ad hoc product generation is functional, but the database integration with docker-compose.yml is still missing. +The current documentation still partly pertains to v0.1.0. In v4.X, the continuous and ad hoc product generation is functional, but the database integration with docker-compose.yml is still missing. **Table of Contents** diff --git a/caroline/jobarray_preparation.py b/caroline/jobarray_preparation.py index 23083fd4..777474e6 100644 --- a/caroline/jobarray_preparation.py +++ b/caroline/jobarray_preparation.py @@ -38,7 +38,7 @@ def jobarray_preparation_scheduler_hook(parameter_file: str, njobs_function: str return eval(f"{njobs_function}('{parameter_file}')") -def njobs_snap_run(parameter_file: str) -> int: +def njobs_snap(parameter_file: str) -> int: """Figure out how many jobs are necessary in the array to successfully run the job run_snap. Parameters diff --git a/caroline/parameter_file.py b/caroline/parameter_file.py index 2eb8bef1..e9891274 100644 --- a/caroline/parameter_file.py +++ b/caroline/parameter_file.py @@ -10,7 +10,7 @@ CONFIG_PARAMETERS = get_config() # Only in the following list of keys, new keys in the configuration are allowed (as here tracks can be added) # The ':' character is used as a split between keys to allow for going deeper into the dictionaries -NEW_CONFIG_KEYS_ALLOWED = ["deinsar:input:data-directories", "depsi:depsi-settings:general:ref-cn"] +NEW_CONFIG_KEYS_ALLOWED = ["doris_v4:input:data-directories", "depsi_matlab:depsi_matlab-settings:general:ref-cn"] def generate_full_parameter_file( diff --git a/caroline/preparation.py b/caroline/preparation.py index f66d2083..e371f4d6 100644 --- a/caroline/preparation.py +++ b/caroline/preparation.py @@ -286,8 +286,8 @@ def finish_installation() -> None: os.system(f"rm -rf {download_config}") -def prepare_crop_to_raw(parameter_file: str, do_track: int | list | None = None) -> None: - """Set up the directories and run files for cropping. +def prepare_create_mrm(parameter_file: str, do_track: int | list | None = None) -> None: + """Set up the directories and files for mrm creation, part of DePSI-post. Parameters ---------- @@ -301,6 +301,8 @@ def prepare_crop_to_raw(parameter_file: str, do_track: int | list | None = None) "general:tracks:track", "general:tracks:asc_dsc", "general:input-data:sensor", + "depsi_post:general:cpxfiddle-directory", + "general:workflow:filters:coregistration-mode", ] out_parameters = read_parameter_file(parameter_file, search_parameters) @@ -315,67 +317,80 @@ def prepare_crop_to_raw(parameter_file: str, do_track: int | list | None = None) if tracks[track] not in do_track: continue - crop_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["crop_to_raw"], track=tracks[track] - ) - - if out_parameters["general:input-data:sensor"] == "S1": - coregistration_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["doris"], track=tracks[track] + # determine if we came from reduce_slc_matlab or merge_to_stack_matlab + if ( + out_parameters["general:workflow:filters:coregistration-mode"] == "doris" + or out_parameters["general:input-data:sensor"].lower() != "s1" + ): + crop_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["reduce_slc_matlab"], track=tracks[track] ) - else: - coregistration_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["deinsar"], track=tracks[track] + crop_directory = format_process_folder( + parameter_file=parameter_file, + job_description=JOB_DEFINITIONS["merge_to_stack_matlab"], + track=tracks[track], ) - os.makedirs(crop_directory, exist_ok=True) + depsi_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["depsi_matlab"], track=tracks[track] + ) - # soft-link the processing directory without job_id.txt, dir_contents.txt and queue.txt - # Sentinel-1 has more files starting with d as Doris-v5 output, other sensors do not have that - if out_parameters["general:input-data:sensor"] == "S1": - link_keys = ["[bgiprs]*", "doris*", "dem"] - else: - link_keys = ["[bgiprs]*"] - for key in link_keys: - # run the soft-link command - os.system(f"ln -sfn {coregistration_directory}/{key} {crop_directory}") + # we need to run cpxfiddle first. This requires two parameters: n_lines, and the project ID + fr = open(f"{crop_directory}/cropped_stack/nlines_crp.txt") + data = fr.read().split("\n") + fr.close() + n_lines = data[0] + + project_id = depsi_directory.split("/")[-2].split("-")[0] + + # format the arguments in the correct order + command_args = ( + f"{project_id} {n_lines} 1 1 {out_parameters['depsi_post:general:cpxfiddle-directory']} {depsi_directory}" + ) + os.system( + f"bash {CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/scripts/create_mrm_ras_header.sh " + f"{command_args}" + ) - # generate crop.sh write_run_file( - save_path=f"{crop_directory}/crop-to-raw.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/crop-to-raw/crop-to-raw.sh", + save_path=f"{depsi_directory}/create_mrm.m", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/create_mrm/create_mrm.m", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, - parameter_file_parameters=["crop_to_raw:general:AoI-name"], - config_parameters=["caroline_work_directory", "matlab_module"], - other_parameters={"track": tracks[track], "crop_base_directory": crop_directory}, + parameter_file_parameters=[ + "depsi_matlab:general:AoI-name", + ["general:input-data:sensor", "lowercase"], + ], + other_parameters={ + "fill_track": f"{tracks[track]:0>3d}", + "asc_dsc": asc_dsc[track], + }, ) - # generate crop.m write_run_file( - save_path=f"{crop_directory}/crop_to_raw.m", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/crop-to-raw/crop-to-raw.m", + save_path=f"{depsi_directory}/create_mrm.sh", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/create_mrm/create_mrm.sh", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, - parameter_file_parameters=[ - "general:shape-file:aoi-name", - "general:shape-file:directory", - "general:input-data:sensor", - ], - config_parameters=["caroline_install_directory"], + parameter_file_parameters=["depsi_matlab:general:AoI-name"], + config_parameters=["caroline_work_directory", "matlab_module"], + other_parameters={ + "track": tracks[track], + "depsi_base_directory": depsi_directory, + }, ) write_directory_contents( - crop_directory, - filename=f'dir_contents{JOB_DEFINITIONS["crop_to_raw"]["directory-contents-file-appendix"]}.txt', + depsi_directory, + filename=f'dir_contents{JOB_DEFINITIONS["create_mrm"]["directory-contents-file-appendix"]}.txt', ) -def prepare_crop_to_zarr(parameter_file: str, do_track: int | list | None = None) -> None: - """Set up the directories and run files for crop_to_zarr. +def prepare_create_tarball(parameter_file: str, do_track: int | list | None = None) -> None: + """Create the tarball after DePSI-post. Parameters ---------- @@ -384,21 +399,12 @@ def prepare_crop_to_zarr(parameter_file: str, do_track: int | list | None = None do_track: int | list | None, optional Track number, or list of track numbers, of the track(s) to prepare. `None` (default) prepares all tracks in the parameter file - - Raises - ------ - ValueError - If the mother image cannot be detected from doris_input.xml (S1) or deinsar.py (otherwise) """ - search_parameters = [ - "general:tracks:track", - "general:tracks:asc_dsc", - "general:input-data:sensor", - ] + search_parameters = ["track"] out_parameters = read_parameter_file(parameter_file, search_parameters) - tracks = out_parameters["general:tracks:track"] - asc_dsc = out_parameters["general:tracks:asc_dsc"] + tracks = out_parameters["track"] + for track in range(len(tracks)): if isinstance(do_track, int): if tracks[track] != do_track: @@ -407,102 +413,19 @@ def prepare_crop_to_zarr(parameter_file: str, do_track: int | list | None = None if tracks[track] not in do_track: continue - crop_to_zarr_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["crop_to_zarr"], track=tracks[track] - ) - - if out_parameters["general:input-data:sensor"] == "S1": - coregistration_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["doris"], track=tracks[track] - ) - - else: - coregistration_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["deinsar"], track=tracks[track] - ) - - os.makedirs(crop_to_zarr_directory, exist_ok=True) - - # detect the mother image - if out_parameters["general:input-data:sensor"].lower() == "s1": - f = open(f"{coregistration_directory}/doris_input.xml") - data = f.read().split("\n") - f.close() - mother = None - for line in data: - if "" in line: - mother = line.split(">")[1].split("<")[0].replace("-", "") - break - - if mother is None: - raise ValueError(f"Failed to detect mother in {coregistration_directory}/doris_input.xml!") - - else: - f = open(f"{coregistration_directory}/run_deinsar.py") - data = f.read().split("\n") - f.close() - mother = None - for line in data: - if "master = " in line: - mother = line.split('"')[1] - break - - if mother is None: - raise ValueError(f"Failed to detect mother in {coregistration_directory}/run_deinsar.py !") - - # generate crop-to-zarr.py - crop_to_zarr_output_name = crop_to_zarr_directory.split("/")[-1] - - write_run_file( - save_path=f"{crop_to_zarr_directory}/crop-to-zarr.py", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/crop-to-zarr/crop-to-zarr.py", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=[ - "general:shape-file:aoi-name", - "general:input-data:sensor", - "general:shape-file:directory", - ], - other_parameters={ - "coregistration_directory": coregistration_directory, - "stack_folder_name": "stack" if out_parameters["general:input-data:sensor"] == "S1" else "process", - "mother": mother, - "mother_slc_name": "slave_rsmp_reramped.raw" - if out_parameters["general:input-data:sensor"] == "S1" - else "slave_rsmp.raw", - "crop_to_zarr_output_filename": crop_to_zarr_output_name, - }, - ) - - # generate crop-to-zarr.sh - write_run_file( - save_path=f"{crop_to_zarr_directory}/crop-to-zarr.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/crop-to-zarr/crop-to-zarr.sh", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=[ - "crop_to_zarr:general:AoI-name", - "crop_to_zarr:general:crop_to_zarr-code-directory", - ], - config_parameters=[ - "caroline_work_directory", - "caroline_virtual_environment_directory", - "python3_module", - "gdal_module", - ], - other_parameters={"track": tracks[track]}, + depsi_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["depsi_matlab"], track=tracks[track] ) - write_directory_contents( - crop_to_zarr_directory, - filename=f'dir_contents{JOB_DEFINITIONS["crop_to_zarr"]["directory-contents-file-appendix"]}.txt', + project_id = depsi_directory.split("/")[-2].split("-")[0] + os.system( + f"cd {depsi_directory}; " + f"bash {CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/scripts/create_post_project_tar.sh {project_id}" ) -def prepare_deinsar(parameter_file: str, do_track: int | list | None = None) -> None: - """Set up the directories and run files for DeInSAR. +def prepare_depsi_matlab(parameter_file: str, do_track: int | list | None = None) -> None: + """Set up the directories and files for DePSI matlab. Parameters ---------- @@ -515,45 +438,30 @@ def prepare_deinsar(parameter_file: str, do_track: int | list | None = None) -> Raises ------ AssertionError - If one of the tracks is not provided in `di_data_directories` + If a dictionary is passed to `ref_cn` in the parameter file, but the track key is missing ValueError - If an unknown sensor is provided in the parameter file + If an invalid mode is passed to `ref_cn` in the parameter file """ search_parameters = [ "general:tracks:track", "general:tracks:asc_dsc", "general:input-data:sensor", - "deinsar:input:data-directories", + "depsi_matlab:general:depsi_matlab-code-directory", + "depsi_matlab:general:rdnaptrans-directory", + "depsi_matlab:general:geocoding-directory", "general:timeframe:start", "general:timeframe:end", - "general:timeframe:mother", - "general:dem:file", - "general:dem:upperleft", - "general:dem:delta", - "general:shape-file:directory", - "general:shape-file:aoi-name", - "deinsar:deinsar-settings:finecoreg:finecoreg-mode", - "default:input-data:polarisation", + "depsi_matlab:depsi_matlab-settings:general:ref-cn", + "depsi_matlab:depsi_matlab-settings:psc:do-water-mask", + "depsi_matlab:general:AoI-name", + "general:workflow:filters:coregistration-mode", ] out_parameters = read_parameter_file(parameter_file, search_parameters) tracks = out_parameters["general:tracks:track"] asc_dsc = out_parameters["general:tracks:asc_dsc"] - - datadirs = out_parameters["deinsar:input:data-directories"] - - start_date = eval(out_parameters["general:timeframe:start"].replace("-", "")) - master_date = eval(out_parameters["general:timeframe:mother"].replace("-", "")) - end_date = eval(out_parameters["general:timeframe:end"].replace("-", "")) - - polarisation = out_parameters["general:input-data:polarisation"] - polarisation = [f"_{pol}" for pol in polarisation] - if "_HH" in polarisation: - polarisation[polarisation.index("_HH")] = "" - - dem_delta = out_parameters["general:dem:delta"] - dem_size = out_parameters["general:dem:size"] - dem_upperleft = out_parameters["general:dem:upperleft"] + start_date = out_parameters["general:timeframe:start"].replace("-", "") + end_date = out_parameters["general:timeframe:end"].replace("-", "") for track in range(len(tracks)): if isinstance(do_track, int): @@ -563,379 +471,257 @@ def prepare_deinsar(parameter_file: str, do_track: int | list | None = None) -> if tracks[track] not in do_track: continue - assert ( - f"{out_parameters['general:input-data:sensor'].lower()}_{asc_dsc[track]}_t{tracks[track]:0>3d}" - in datadirs.keys() - ), ( - f"{out_parameters['general:input-data:sensor'].lower()}_" - f"{asc_dsc[track]}_t{tracks[track]:0>3d} is not in deinsar:input:data-directories!" + depsi_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["depsi_matlab"], track=tracks[track] ) - coregistration_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["deinsar"], track=tracks[track] - ) + # determine if we came from reduce_slc_matlab or merge_to_stack_matlab + if ( + out_parameters["general:workflow:filters:coregistration-mode"] == "doris" + or out_parameters["general:input-data:sensor"].lower() != "s1" + ): + crop_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["reduce_slc_matlab"], track=tracks[track] + ) + else: + crop_directory = format_process_folder( + parameter_file=parameter_file, + job_description=JOB_DEFINITIONS["merge_to_stack_matlab"], + track=tracks[track], + ) - # we need a process folder in the coregistration directory, so we can combine that command - os.makedirs(f"{coregistration_directory}/process", exist_ok=True) + # we need a psi and boxes folder in the depsi directory + os.makedirs(f"{depsi_directory}", exist_ok=True) + os.makedirs(f"{depsi_directory}/../boxes", exist_ok=True) - # generate deinsar.sh - write_run_file( - save_path=f"{coregistration_directory}/run_deinsar.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/deinsar/run_deinsar.sh", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=[ - "deinsar:general:deinsar-code-directory", - "deinsar:general:doris-v4-code-directory", - "deinsar:general:AoI-name", - ], - config_parameters=["caroline_work_directory", "orbit_directory", "python2_module", "gdal_module"], - other_parameters={"track": tracks[track], "coregistration_base_directory": coregistration_directory}, + # link the necessary boxes + os.system( + f"cp -Rp {out_parameters['depsi_matlab:general:depsi_matlab-code-directory']} {depsi_directory}/../boxes" ) + os.system(f"cp -Rp {out_parameters['depsi_matlab:general:rdnaptrans-directory']} {depsi_directory}/../boxes") + os.system(f"cp -Rp {out_parameters['depsi_matlab:general:geocoding-directory']} {depsi_directory}/../boxes") - # generate run_deinsar.py + # detect the mother and dem_radar from the mother + mother = glob.glob(f"{crop_directory}/*cropped_stack/2*/master.res")[0] + # cut off master.res, and add dem_radar.raw + dem_radar = mother.replace("/master.res", "/dem_radar.raw") + mother_date = mother.split("/")[-2] - # first search for the start, end, and master dates by parsing all data in the data directory, - # which is different per sensor - datadir = datadirs[ - f"{out_parameters['general:input-data:sensor'].lower()}_{asc_dsc[track]}_t{tracks[track]:0>3d}" - ] - if out_parameters["general:input-data:sensor"] in ["ALOS2", "ERS"]: - dirs = glob.glob(f"{datadir}/[12]*") - images = list(sorted([eval(image.split("/")[-1]) for image in dirs])) - elif out_parameters["general:input-data:sensor"] in ["RSAT2"]: - dirs = glob.glob(f"{datadir}/RS2*") - images = list(sorted([eval(image.split("/")[-1].split("FQ2_")[1].split("_")[0]) for image in dirs])) - elif out_parameters["general:input-data:sensor"] in ["TSX"]: - dirs = glob.glob(f"{datadir}/*/iif/*") - images = list(sorted([eval(image.split("/")[-1].split("SRA_")[1].split("T")[0]) for image in dirs])) - elif out_parameters["general:input-data:sensor"] in ["SAOCOM"]: - dirs = glob.glob(f"{datadir}/*/*.xemt") - images = list(sorted([eval(image.split("/")[-1].split("OLF_")[1].split("T")[0]) for image in dirs])) - elif out_parameters["general:input-data:sensor"] in ["ENV"]: - # 2 different formats for some reason - dirs1 = glob.glob(f"{datadir}/*.N1") - dirs2 = glob.glob(f"{datadir}/*/*.N1") - dirs = [] - for d in dirs1: - dirs.append(d) - for d in dirs2: - dirs.append(d) - images = list(sorted([eval(image.split("/")[-1].split("PA")[1].split("_")[0]) for image in dirs])) - else: - raise ValueError(f'Unknown directory format for sensor {out_parameters["general:input-data:sensor"]}!') - - # then select the start, end, and master dates - act_start_date = str(min([image for image in images if image >= start_date])) - act_end_date = str(max([image for image in images if image <= end_date])) - act_master_date = str(min([image for image in images if image >= master_date])) - - # finally, write run_deinsar.py - write_run_file( - save_path=f"{coregistration_directory}/run_deinsar.py", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/deinsar/run_deinsar.py", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=[ - ["deinsar:input:data-directories", "dictionary"], - "general:input-data:sensor", - "general:input-data:polarisation", - "deinsar:deinsar-settings:do-orbit", - "deinsar:deinsar-settings:do-crop", - "deinsar:deinsar-settings:do-tsx-deramp", - "deinsar:deinsar-settings:do-simamp", - "deinsar:deinsar-settings:do-mtiming", - "deinsar:deinsar-settings:do-ovs", - "deinsar:deinsar-settings:do-choose-master", - "deinsar:deinsar-settings:do-coarseorb", - "deinsar:deinsar-settings:do-coarsecorr", - "deinsar:deinsar-settings:finecoreg:do-finecoreg", - "deinsar:deinsar-settings:do-reltiming", - "deinsar:deinsar-settings:do-dembased", - "deinsar:deinsar-settings:do-coregpm", - "deinsar:deinsar-settings:do-comprefpha", - "deinsar:deinsar-settings:do-comprefdem", - "deinsar:deinsar-settings:do-resample", - "deinsar:deinsar-settings:do-tsx-reramp", - "deinsar:deinsar-settings:do-interferogram", - "deinsar:deinsar-settings:do-subtrrefpha", - "deinsar:deinsar-settings:do-subtrrefdem", - "deinsar:deinsar-settings:do-coherence", - "deinsar:deinsar-settings:do-geocoding", - ], - other_parameters={"master": act_master_date, "startdate": act_start_date, "enddate": act_end_date}, - ) + # link the mother resfile and dem_radar + os.system(f"ln -sf {mother} {depsi_directory}/slave.res") + os.system(f"ln -sf {dem_radar} {depsi_directory}/dem_radar.raw") - # finally, create the input files + # find the first and last valid dates within range + if os.path.exists(f"{crop_directory}/cropped_stack/path_slcs.txt"): + f = open(f"{crop_directory}/cropped_stack/path_slcs.txt") + resfiles = f.read().split("\n") + f.close() + dates = [i.split("/")[-2] for i in resfiles if i != ""] + valid_dates = [date for date in dates if start_date <= date <= end_date] + else: + valid_dates = [] - # these ones can be copied directly - for file in [ - "input.baselines", - "input.coarsecorr", - "input.coarseorb", - "input.comprefpha", - "input.coregpm", - "input.mtiming", - "input.reltiming", - "input.geocoding", - ]: - write_run_file( - save_path=f"{coregistration_directory}/process/{file}", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/deinsar/input_files/{file}", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, + if len(valid_dates) == 0: + # From #77 , not doing this will cause the following in multi-track starts: + # Looping over A,B,C,D , if C has no valid_dates, the parameter file for D will not be generated + # as the generation in C will throw an error with the min/max below + print( + "WARNING: Did not identify any properly cropped images! Cannot determine start and " + "end date for DePSI, setting to None. This will crash DePSI." ) + act_start_date = None + act_end_date = None + else: + act_start_date = min(valid_dates) + act_end_date = max(valid_dates) - # these ones are polarisation-dependent - for file in ["input.coherence", "input.interferogram", "input.subtrrefdem", "input.subtrrefpha", "input.ovs"]: - for pol in polarisation: - write_run_file( - save_path=f"{coregistration_directory}/process/{file}{pol}", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" - f"templates/deinsar/input_files/{file}", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - other_parameters={"pol": pol}, - ) - - # these ones need the DEM variables - for file in ["input.comprefdem", "input.dembased", "input.simamp"]: - write_run_file( - save_path=f"{coregistration_directory}/process/{file}", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/deinsar/input_files/{file}", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=["general:dem:file", "general:dem:format", "general:dem:nodata"], - other_parameters={ - "dem_s1": dem_size[0], - "dem_s2": dem_size[1], - "dem_d1": dem_delta[0], - "dem_d2": dem_delta[1], - "dem_ul1": dem_upperleft[0], - "dem_ul2": dem_upperleft[1], - }, + # generate the water mask link + if out_parameters["depsi_matlab:depsi_matlab-settings:psc:do-water-mask"] == "yes": + filename_water_mask = ( + f"{CONFIG_PARAMETERS['CAROLINE_WATER_MASK_DIRECTORY']}/water_mask_" + f"{out_parameters['depsi_matlab:general:AoI-name']}_" + f"{out_parameters['general:input-data:sensor'].lower()}_{asc_dsc[track]}_t{tracks[track]:0>3d}.raw" ) + else: + filename_water_mask = "[]" - # finecoreg changes based on the fine coregistration mode - if out_parameters["deinsar:deinsar-settings:finecoreg:finecoreg-mode"] == "simple": - write_run_file( - save_path=f"{coregistration_directory}/process/input.finecoreg_simple", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" - f"templates/deinsar/input_files/input.finecoreg", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - other_parameters={"nwin": 5000}, - ) - else: # normal mode - write_run_file( - save_path=f"{coregistration_directory}/process/input.finecoreg", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" - f"templates/deinsar/input_files/input.finecoreg", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - other_parameters={"nwin": 8000}, - ) + # #62 -> figure out the reference point + key = f"{out_parameters['general:input-data:sensor'].lower()}_{asc_dsc[track]}_t{tracks[track]:0>3d}" - # porbit is only necessary for ERS and ENV - if out_parameters["general:input-data:sensor"] == "ERS": - # this one requires two copies - for satellite in [1, 2]: - write_run_file( - save_path=f"{coregistration_directory}/process/input.porbit_ERS{satellite}", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" - f"templates/deinsar/input_files/input.porbit", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - other_parameters={"directory": f"ERS{satellite}"}, - ) - elif out_parameters["general:input-data:sensor"] == "ENV": - write_run_file( - save_path=f"{coregistration_directory}/process/input.porbit", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" - f"templates/deinsar/input_files/input.porbit", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - other_parameters={"directory": "envisat/dor_vor_odr"}, + if not isinstance(out_parameters["depsi_matlab:depsi_matlab-settings:general:ref-cn"], dict): + print( + "WARNING: Invalid value for ref-cn " + f"({out_parameters['depsi_matlab:depsi_matlab-settings:general:ref-cn']}) " + "encountered. Using mode 'constant'..." ) + mode = "constant" - # for input.crop and input.resample we need to read the shapefile extent and calculate the amount of pixels - coordinates = np.array( - read_shp_extent( - f"{out_parameters['general:shape-file:directory']}/" - f"{out_parameters['general:shape-file:aoi-name']}_shape.shp", - shp_type="AoI", - )["0"] - ) - min_lat = min(coordinates[:, 1]) - max_lat = max(coordinates[:, 1]) - min_lon = min(coordinates[:, 0]) - max_lon = max(coordinates[:, 0]) - - # get the central coordinates - center_lon = (max_lon + min_lon) / 2 - center_lat = (max_lat + min_lat) / 2 - - # get the latitude at which the crop is widest - if min_lat < 0: - if max_lat > 0: - ref_lat = 0 + if key not in out_parameters["depsi_matlab:depsi_matlab-settings:general:ref-cn"]: + if "all" not in out_parameters["depsi_matlab:depsi_matlab-settings:general:ref-cn"]: + raise ValueError( + f"Cannot find {key} in ref-cn {out_parameters['depsi_matlab:depsi_matlab-settings:general:ref-cn']}" + ) else: - ref_lat = max_lat + mode = str(out_parameters["depsi_matlab:depsi_matlab-settings:general:ref-cn"]["all"]) else: - ref_lat = min_lat - - # calculate the extent of the AoI - dist_lat = haversine(min_lat, max_lat, min_lon, min_lon) - dist_lon = haversine(ref_lat, ref_lat, min_lon, max_lon) # calculated at the widest part of the AoI - - # determine the number of pixels - d_az, d_r = detect_sensor_pixelsize(out_parameters["general:input-data:sensor"]) - pix_dr = int(np.ceil(dist_lon / d_r * 1.05)) - pix_daz = int(np.ceil(dist_lat / d_az * 1.05)) + mode = str(out_parameters["depsi_matlab:depsi_matlab-settings:general:ref-cn"][key]) - # for input.crop we will add 500 to eliminate edge effects - if out_parameters["general:input-data:sensor"] == "ALOS2": - img_name = "IMG.1" - elif out_parameters["general:input-data:sensor"] == "Cosmo": - img_name = "image.h5" - elif out_parameters["general:input-data:sensor"] == "ENV": - img_name = "image.N1" - elif out_parameters["general:input-data:sensor"] == "ERS": - img_name = "DAT_01.001" - elif out_parameters["general:input-data:sensor"] == "RSAT2": - img_name = "imagery{pol}.tif" - # requires loop over polarisations to get additional crop files - elif out_parameters["general:input-data:sensor"] == "TSX": - img_name = "image.cos" - else: - raise ValueError(f'Unknown sensor {out_parameters["general:input-data:sensor"]}!') + if mode in ["independent", "[]"]: + ref_cn = "[]" + elif mode[0] == "[": # hardcoded + ref_cn = mode.replace(" ", "") # remove spaces since Matlab doesn't like them + elif mode == "constant": + # find the old runs + directories = glob.glob(f"{'-'.join(depsi_directory.split('-')[:-1])}-*") + ref_cn = "[]" + if len(directories) == 0: + # no old runs are present, so we run on mode 'independent' for the initialization + pass + else: + # sort and reverse them to find the most recent one + rev_order_runs = list(sorted(directories))[::-1] + for i in range(len(rev_order_runs)): # loop in case one crashed. If all crashed, + # ref_cn is defined before the if/else, and we run on mode 'independent' + ref_file = ( + f"{rev_order_runs[i]}/psi/{out_parameters['depsi_matlab:general:AoI-name']}_" + f"{out_parameters['general:input-data:sensor'].lower()}_" + f"{asc_dsc[track]}_t{tracks[track]:0>3d}_ref_sel1.raw" + ) # this file saves the selected reference + if os.path.exists(ref_file): + ref_data = np.memmap(ref_file, mode="r", shape=(3,), dtype="float64") + # this outputs the reference point in [index, az, r]. We need [az,r] + ref_cn = f"[{int(round(ref_data[1]))},{int(round(ref_data[2]))}]" + break # we found one, so we can stop - # write input.crop - if ( - out_parameters["general:input-data:sensor"] == "RSAT2" - ): # for RSAT2 this is per polarisation, otherwise there just is one - for pol in polarisation: - write_run_file( - save_path=f"{coregistration_directory}/process/input.crop{pol}", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" - f"templates/deinsar/input_files/input.crop", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - other_parameters={ - "img_name": img_name.format(pol), - "pol": pol, - "center_lat": center_lat, - "center_lon": center_lon, - "pix_az": pix_daz + 500, - "pix_r": pix_dr + 500, - }, - ) else: - write_run_file( - save_path=f"{coregistration_directory}/process/input.crop", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" - f"templates/deinsar/input_files/input.crop", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - other_parameters={ - "img_name": img_name, - "pol": "", - "center_lat": center_lat, - "center_lon": center_lon, - "pix_az": pix_daz + 500, - "pix_r": pix_dr + 500, - }, - ) - - # write input.resample - for pol in polarisation: - write_run_file( - save_path=f"{coregistration_directory}/process/input.resample{pol}", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" - f"templates/deinsar/input_files/input.resample", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - other_parameters={ - "pol": pol, - "center_lat": center_lat, - "center_lon": center_lon, - "pix_az": pix_daz, - "pix_r": pix_dr, - }, + raise ValueError( + f"Expected types are dictionary, 'independent', '[]', '[az, r]', or 'constant', got {mode}" ) - # finally, we need input.readfiles, which requires a data string composed of sensor-specific data - if out_parameters["general:input-data:sensor"] == "ALOS2": - data_string = """S_IN_METHOD ALOS2 -S_IN_DAT IMG.1 -S_IN_LEA LED.1 -S_IN_VOL VOL.1""" - - elif out_parameters["general:input-data:sensor"] == "Cosmo": - data_string = """S_IN_METHOD CSK -S_IN_DAT image.h5""" - - elif out_parameters["general:input-data:sensor"] == "ERS": - data_string = """S_IN_METHOD ERS -S_IN_VOL VRD_DAT.001 -S_IN_DAT DAT_01.001 -S_IN_LEA LEA_01.001 -S_IN_NULL dummy""" - - elif out_parameters["general:input-data:sensor"] == "ENV": - data_string = """S_IN_METHOD ASAR -S_IN_DAT image.N1""" - - elif out_parameters["general:input-data:sensor"] == "RSAT": - data_string = """S_IN_METHOD RSAT -S_IN_VOL VDF_DAT.001 -S_IN_DAT DAT_01.001 -S_IN_LEA LEA_01.001 -S_IN_NULL dummy""" - - elif out_parameters["general:input-data:sensor"] == "RSAT2": - data_string = """S_IN_METHOD RADARSAT-2 -S_IN_DAT imagery_HH.tif -S_IN_LEA product.xml""" - - elif out_parameters["general:input-data:sensor"] == "TSX": - data_string = """S_IN_METHOD TSX -S_IN_DAT image.cos -S_IN_LEA leader.xml""" + # write depsi.m + write_run_file( + save_path=f"{depsi_directory}/depsi.m", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/depsi_matlab/depsi.m", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + other_parameters={ + "geocoding_version": out_parameters["depsi_matlab:general:geocoding-directory"].split("/")[-1].rstrip(), + "depsi_version": out_parameters["depsi_matlab:general:depsi_matlab-code-directory"] + .split("/")[-1] + .rstrip(), + }, + ) - else: - raise ValueError(f'Unknown sensor {out_parameters["general:input-data:sensor"]} for input.readfiles!') + # write depsi.sh + write_run_file( + save_path=f"{depsi_directory}/depsi_matlab.sh", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/depsi_matlab/depsi_matlab.sh", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=["depsi_matlab:general:AoI-name"], + config_parameters=["caroline_work_directory", "matlab_module"], + other_parameters={"depsi_base_directory": depsi_directory, "track": tracks[track]}, + ) + # create param_file_depsi.txt + # write_run_file( - save_path=f"{coregistration_directory}/process/input.readfiles", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" - f"templates/deinsar/input_files/input.readfiles", + save_path=f"{depsi_directory}/param_file_depsi.txt", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/depsi_matlab/param_file.txt", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, - other_parameters={"data_string": data_string}, + parameter_file_parameters=[ + "depsi_matlab:general:AoI-name", + "depsi_matlab:depsi_matlab-settings:general:max-mem-buffer", + "depsi_matlab:depsi_matlab-settings:general:visible-plots", + "depsi_matlab:depsi_matlab-settings:general:detail-plots", + "depsi_matlab:depsi_matlab-settings:general:processing-groups", + "depsi_matlab:depsi_matlab-settings:general:run-mode", + ["general:input-data:sensor", "lowercase"], + "depsi_matlab:depsi_matlab-settings:general:exclude-date", + "depsi_matlab:depsi_matlab-settings:general:az-spacing", + "depsi_matlab:depsi_matlab-settings:general:r-spacing", + "depsi_matlab:depsi_matlab-settings:general:slc-selection-input", + "depsi_matlab:depsi_matlab-settings:general:ifg-selection-input", + "depsi_matlab:depsi_matlab-settings:general:Ncv", + "depsi_matlab:depsi_matlab-settings:general:ps-method", + "depsi_matlab:depsi_matlab-settings:general:psc-model", + "depsi_matlab:depsi_matlab-settings:general:ps-model", + "depsi_matlab:depsi_matlab-settings:general:final-model", + "depsi_matlab:depsi_matlab-settings:general:breakpoint", + "depsi_matlab:depsi_matlab-settings:general:breakpoint2", + "depsi_matlab:depsi_matlab-settings:general:ens-coh-threshold", + "depsi_matlab:depsi_matlab-settings:general:varfac-threshold", + "depsi_matlab:depsi_matlab-settings:general:detrend-method", + "depsi_matlab:depsi_matlab-settings:general:output-format", + "depsi_matlab:depsi_matlab-settings:general:do-apriori-sidelobe-mask", + "depsi_matlab:depsi_matlab-settings:general:do-aposteriori-sidelobe-mask", + "depsi_matlab:depsi_matlab-settings:geocoding:ref-height", + "depsi_matlab:depsi_matlab-settings:psc:amplitude-calibration", + "depsi_matlab:depsi_matlab-settings:psc:psc-selection-method", + "depsi_matlab:depsi_matlab-settings:psc:psc-selection-gridsize", + "depsi_matlab:depsi_matlab-settings:psc:psc-threshold", + "depsi_matlab:depsi_matlab-settings:psc:max-arc-length", + "depsi_matlab:depsi_matlab-settings:psc:network-method", + "depsi_matlab:depsi_matlab-settings:psc:Ncon", + "depsi_matlab:depsi_matlab-settings:psc:Nparts", + "depsi_matlab:depsi_matlab-settings:psc:Npsc-selections", + "depsi_matlab:depsi_matlab-settings:psc:gamma-threshold", + "depsi_matlab:depsi_matlab-settings:psc:psc-distribution", + "depsi_matlab:depsi_matlab-settings:psc:weighted-unwrap", + "depsi_matlab:depsi_matlab-settings:psc:livetime-threshold", + "depsi_matlab:depsi_matlab-settings:psc:peak-tolerance", + "depsi_matlab:depsi_matlab-settings:psp:psp-selection-method", + "depsi_matlab:depsi_matlab-settings:psp:psp-threshold1", + "depsi_matlab:depsi_matlab-settings:psp:psp-threshold2", + "depsi_matlab:depsi_matlab-settings:psp:ps-eval-method", + "depsi_matlab:depsi_matlab-settings:psp:Namp-disp-bins", + "depsi_matlab:depsi_matlab-settings:psp:Ndens-iterations", + "depsi_matlab:depsi_matlab-settings:psp:densification-flag", + "depsi_matlab:depsi_matlab-settings:psp:ps-area-of-interest", + "depsi_matlab:depsi_matlab-settings:psp:dens-method", + "depsi_matlab:depsi_matlab-settings:psp:dens-check", + "depsi_matlab:depsi_matlab-settings:psp:Nest", + "depsi_matlab:depsi_matlab-settings:stochastic-model:defo-range", + "depsi_matlab:depsi_matlab-settings:stochastic-model:weighting", + "depsi_matlab:depsi_matlab-settings:stochastic-model:ts-atmo-filter", + "depsi_matlab:depsi_matlab-settings:stochastic-model:ts-atmo-filter-length", + "depsi_matlab:depsi_matlab-settings:stochastic-model:ts-noise-filter", + "depsi_matlab:depsi_matlab-settings:stochastic-model:ts-noise-filter-length", + "depsi_matlab:depsi_matlab-settings:bowl:defo-method", + "depsi_matlab:depsi_matlab-settings:bowl:xc0", + "depsi_matlab:depsi_matlab-settings:bowl:yc0", + "depsi_matlab:depsi_matlab-settings:bowl:zc0", + "depsi_matlab:depsi_matlab-settings:bowl:r0", + "depsi_matlab:depsi_matlab-settings:bowl:r10", + "depsi_matlab:depsi_matlab-settings:bowl:epoch", + ["depsi_matlab:depsi_matlab-settings:general:stc-min-max", "strip", "[] "], + ["depsi_matlab:depsi_matlab-settings:stochastic-model:std-param", "strip", "[] "], + ], + other_parameters={ + "crop_base_directory": crop_directory, + "track": f"{tracks[track]:0>3d}", + "asc_dsc": asc_dsc[track], + "asc_dsc_fmt": "desc" if asc_dsc[track] == "dsc" else asc_dsc[track], + "start_date": act_start_date, + "stop_date": act_end_date, + "master_date": mother_date, + "ref_cn": ref_cn, + "filename_water_mask": filename_water_mask, + }, ) write_directory_contents( - coregistration_directory, - filename=f'dir_contents{JOB_DEFINITIONS["deinsar"]["directory-contents-file-appendix"]}.txt', + depsi_directory, + filename=f'dir_contents{JOB_DEFINITIONS["depsi_matlab"]["directory-contents-file-appendix"]}.txt', ) -def prepare_depsi(parameter_file: str, do_track: int | list | None = None) -> None: - """Set up the directories and files for DePSI. +def prepare_depsi_post(parameter_file: str, do_track: int | list | None = None) -> None: + """Set up the directories and files for DePSI-post. Parameters ---------- @@ -947,31 +733,42 @@ def prepare_depsi(parameter_file: str, do_track: int | list | None = None) -> No Raises ------ - AssertionError - If a dictionary is passed to `ref_cn` in the parameter file, but the track key is missing ValueError - If an invalid mode is passed to `ref_cn` in the parameter file + If `depsi_post_mode` is not 'tarball' or 'csv' """ search_parameters = [ "general:tracks:track", "general:tracks:asc_dsc", "general:input-data:sensor", - "depsi:general:depsi-code-directory", - "depsi:general:rdnaptrans-directory", - "depsi:general:geocoding-directory", - "general:timeframe:start", - "general:timeframe:end", - "depsi:depsi-settings:general:ref-cn", - "depsi:depsi-settings:psc:do-water-mask", - "depsi:general:AoI-name", - "general:workflow:filters:coregistration-mode", + "depsi_post:general:depsi_post-code-directory", + "depsi_post:depsi_post-settings:defo-clim", + "depsi_post:depsi_post-settings:height-clim", + "depsi_matlab:general:rdnaptrans-directory", + "depsi_matlab:general:geocoding-directory", + "general:workflow:filters:depsi_post-output", ] out_parameters = read_parameter_file(parameter_file, search_parameters) tracks = out_parameters["general:tracks:track"] asc_dsc = out_parameters["general:tracks:asc_dsc"] - start_date = out_parameters["general:timeframe:start"].replace("-", "") - end_date = out_parameters["general:timeframe:end"].replace("-", "") + + defo_clim_raw = out_parameters["depsi_post:depsi_post-settings:defo-clim"] + defo_clim_min = defo_clim_raw[0] + defo_clim_max = defo_clim_raw[1] + + height_clim_raw = out_parameters["depsi_post:depsi_post-settings:height-clim"] + height_clim_min = height_clim_raw[0] + height_clim_max = height_clim_raw[1] + + if out_parameters["general:workflow:filters:depsi_post-output"] == "tarball": + do_csv = 0 + elif out_parameters["general:workflow:filters:depsi_post-output"] == "csv": + do_csv = 1 + else: + raise ValueError( + "general:workflow:filters:depsi_post-output is set to " + f"{out_parameters['general:workflow:filters:depsi_post-output']}, only know 'tarball' and 'csv'!" + ) for track in range(len(tracks)): if isinstance(do_track, int): @@ -982,390 +779,528 @@ def prepare_depsi(parameter_file: str, do_track: int | list | None = None) -> No continue depsi_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["depsi"], track=tracks[track] + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["depsi_post"], track=tracks[track] ) - # determine if we came from crop_to_raw or znap_to_raw - if ( - out_parameters["general:workflow:filters:coregistration-mode"] == "doris" - or out_parameters["general:input-data:sensor"].lower() != "s1" - ): - crop_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["crop_to_raw"], track=tracks[track] - ) - else: - crop_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["znap_to_raw"], track=tracks[track] - ) + # link the DePSI-post box + os.system(f"cp -Rp {out_parameters['depsi_post:general:depsi_post-code-directory']} {depsi_directory}/../boxes") - # we need a psi and boxes folder in the depsi directory - os.makedirs(f"{depsi_directory}", exist_ok=True) - os.makedirs(f"{depsi_directory}/../boxes", exist_ok=True) + # write depsi_post.m + write_run_file( + save_path=f"{depsi_directory}/depsi_post.m", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/depsi_post/depsi_post.m", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=[ + "depsi_post:depsi_post-settings:dlat", + "depsi_post:depsi_post-settings:dlon", + "depsi_post:depsi_post-settings:drdx", + "depsi_post:depsi_post-settings:drdy", + "general:input-data:sensor", + "depsi_matlab:general:AoI-name", + "depsi_post:depsi_post-settings:proj", + "depsi_post:depsi_post-settings:ref-dheight", + "depsi_post:depsi_post-settings:posteriori-scale-factor", + ["depsi_post:depsi_post-settings:pred-model", "strip", " "], + "depsi_post:depsi_post-settings:plot-mode", + ["depsi_post:depsi_post-settings:do-plots", "strip", "{} "], + ["depsi_post:depsi_post-settings:output", "strip", "{} "], + "depsi_post:depsi_post-settings:fontsize", + "depsi_post:depsi_post-settings:markersize", + "depsi_post:depsi_post-settings:do-print", + "depsi_post:depsi_post-settings:output-format", + "depsi_post:depsi_post-settings:az0", + "depsi_post:depsi_post-settings:azN", + "depsi_post:depsi_post-settings:r0", + "depsi_post:depsi_post-settings:rN", + "depsi_post:depsi_post-settings:result", + "depsi_post:depsi_post-settings:psc-selection", + "depsi_post:depsi_post-settings:do-remove-filtered", + "depsi_post:depsi_post-settings:which-sl-mask", + "depsi_post:depsi_post-settings:shift-to-mean", + "depsi_post:depsi_post-settings:new-ref-cn", + "depsi_post:depsi_post-settings:map-to-vert", + "depsi_post:depsi_post-settings:defo-lim", + "depsi_post:depsi_post-settings:height-lim", + "depsi_post:depsi_post-settings:ens-coh-lim", + "depsi_post:depsi_post-settings:ens-coh-local-lim", + "depsi_post:depsi_post-settings:stc-lim", + "depsi_post:depsi_post-settings:ens-coh-clim", + "depsi_post:depsi_post-settings:ens-coh-local-clim", + "depsi_post:depsi_post-settings:stc-clim", + ], + other_parameters={ + "geocoding_version": out_parameters["depsi_matlab:general:geocoding-directory"].split("/")[-1].rstrip(), + "depsi_post_version": out_parameters["depsi_post:general:depsi_post-code-directory"] + .split("/")[-1] + .rstrip(), + "rdnaptrans_version": out_parameters["depsi_matlab:general:rdnaptrans-directory"] + .split("/")[-1] + .rstrip(), + "do_csv": do_csv, + "asc_dsc": asc_dsc[track], + "track": tracks[track], + "fill_track": f"{tracks[track]:0>3d}", + "dp_defo_clim_min": defo_clim_min, + "dp_defo_clim_max": defo_clim_max, + "dp_height_clim_min": height_clim_min, + "dp_height_clim_max": height_clim_max, + }, + ) - # link the necessary boxes - os.system(f"cp -Rp {out_parameters['depsi:general:depsi-code-directory']} {depsi_directory}/../boxes") - os.system(f"cp -Rp {out_parameters['depsi:general:rdnaptrans-directory']} {depsi_directory}/../boxes") - os.system(f"cp -Rp {out_parameters['depsi:general:geocoding-directory']} {depsi_directory}/../boxes") + # write depsi_post.sh + write_run_file( + save_path=f"{depsi_directory}/depsi_post.sh", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/depsi_post/depsi_post.sh", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=["depsi_matlab:general:AoI-name"], + config_parameters=["caroline_work_directory", "matlab_module"], + other_parameters={"track": tracks[track], "depsi_base_directory": depsi_directory}, + ) - # detect the mother and dem_radar from the mother - mother = glob.glob(f"{crop_directory}/*cropped_stack/2*/master.res")[0] - # cut off master.res, and add dem_radar.raw - dem_radar = mother.replace("/master.res", "/dem_radar.raw") - mother_date = mother.split("/")[-2] + write_directory_contents( + depsi_directory, + filename=f'dir_contents{JOB_DEFINITIONS["depsi_post"]["directory-contents-file-appendix"]}.txt', + ) + + +def prepare_doris_v4(parameter_file: str, do_track: int | list | None = None) -> None: + """Set up the directories and run files for doris_v4. + + Parameters + ---------- + parameter_file: str + Absolute path to the parameter file. + do_track: int | list | None, optional + Track number, or list of track numbers, of the track(s) to prepare. `None` (default) prepares all tracks in + the parameter file + + Raises + ------ + AssertionError + If one of the tracks is not provided in `di_data_directories` + ValueError + If an unknown sensor is provided in the parameter file + """ + search_parameters = [ + "general:tracks:track", + "general:tracks:asc_dsc", + "general:input-data:sensor", + "doris_v4:input:data-directories", + "general:timeframe:start", + "general:timeframe:end", + "general:timeframe:mother", + "general:dem:file", + "general:dem:upperleft", + "general:dem:delta", + "general:shape-file:directory", + "general:shape-file:aoi-name", + "doris_v4:doris_v4-settings:finecoreg:finecoreg-mode", + "default:input-data:polarisation", + ] + out_parameters = read_parameter_file(parameter_file, search_parameters) - # link the mother resfile and dem_radar - os.system(f"ln -sf {mother} {depsi_directory}/slave.res") - os.system(f"ln -sf {dem_radar} {depsi_directory}/dem_radar.raw") + tracks = out_parameters["general:tracks:track"] + asc_dsc = out_parameters["general:tracks:asc_dsc"] - # find the first and last valid dates within range - if os.path.exists(f"{crop_directory}/cropped_stack/path_slcs.txt"): - f = open(f"{crop_directory}/cropped_stack/path_slcs.txt") - resfiles = f.read().split("\n") - f.close() - dates = [i.split("/")[-2] for i in resfiles if i != ""] - valid_dates = [date for date in dates if start_date <= date <= end_date] - else: - valid_dates = [] + datadirs = out_parameters["doris_v4:input:data-directories"] - if len(valid_dates) == 0: - # From #77 , not doing this will cause the following in multi-track starts: - # Looping over A,B,C,D , if C has no valid_dates, the parameter file for D will not be generated - # as the generation in C will throw an error with the min/max below - print( - "WARNING: Did not identify any properly cropped images! Cannot determine start and " - "end date for DePSI, setting to None. This will crash DePSI." - ) - act_start_date = None - act_end_date = None - else: - act_start_date = min(valid_dates) - act_end_date = max(valid_dates) + start_date = eval(out_parameters["general:timeframe:start"].replace("-", "")) + master_date = eval(out_parameters["general:timeframe:mother"].replace("-", "")) + end_date = eval(out_parameters["general:timeframe:end"].replace("-", "")) - # generate the water mask link - if out_parameters["depsi:depsi-settings:psc:do-water-mask"] == "yes": - filename_water_mask = ( - f"{CONFIG_PARAMETERS['CAROLINE_WATER_MASK_DIRECTORY']}/water_mask_" - f"{out_parameters['depsi:general:AoI-name']}_" - f"{out_parameters['general:input-data:sensor'].lower()}_{asc_dsc[track]}_t{tracks[track]:0>3d}.raw" - ) - else: - filename_water_mask = "[]" + polarisation = out_parameters["general:input-data:polarisation"] + polarisation = [f"_{pol}" for pol in polarisation] + if "_HH" in polarisation: + polarisation[polarisation.index("_HH")] = "" - # #62 -> figure out the reference point - key = f"{out_parameters['general:input-data:sensor'].lower()}_{asc_dsc[track]}_t{tracks[track]:0>3d}" + dem_delta = out_parameters["general:dem:delta"] + dem_size = out_parameters["general:dem:size"] + dem_upperleft = out_parameters["general:dem:upperleft"] - if not isinstance(out_parameters["depsi:depsi-settings:general:ref-cn"], dict): - print( - f"WARNING: Invalid value for ref-cn ({out_parameters['depsi:depsi-settings:general:ref-cn']}) " - "encountered. Using mode 'constant'..." - ) - mode = "constant" + for track in range(len(tracks)): + if isinstance(do_track, int): + if tracks[track] != do_track: + continue + elif isinstance(do_track, list): + if tracks[track] not in do_track: + continue - if key not in out_parameters["depsi:depsi-settings:general:ref-cn"]: - if "all" not in out_parameters["depsi:depsi-settings:general:ref-cn"]: - raise ValueError(f"Cannot find {key} in ref-cn {out_parameters['depsi:depsi-settings:general:ref-cn']}") - else: - mode = str(out_parameters["depsi:depsi-settings:general:ref-cn"]["all"]) - else: - mode = str(out_parameters["depsi:depsi-settings:general:ref-cn"][key]) + assert ( + f"{out_parameters['general:input-data:sensor'].lower()}_{asc_dsc[track]}_t{tracks[track]:0>3d}" + in datadirs.keys() + ), ( + f"{out_parameters['general:input-data:sensor'].lower()}_" + f"{asc_dsc[track]}_t{tracks[track]:0>3d} is not in doris_v4:input:data-directories!" + ) - if mode in ["independent", "[]"]: - ref_cn = "[]" - elif mode[0] == "[": # hardcoded - ref_cn = mode.replace(" ", "") # remove spaces since Matlab doesn't like them - elif mode == "constant": - # find the old runs - directories = glob.glob(f"{'-'.join(depsi_directory.split('-')[:-1])}-*") - ref_cn = "[]" - if len(directories) == 0: - # no old runs are present, so we run on mode 'independent' for the initialization - pass - else: - # sort and reverse them to find the most recent one - rev_order_runs = list(sorted(directories))[::-1] - for i in range(len(rev_order_runs)): # loop in case one crashed. If all crashed, - # ref_cn is defined before the if/else, and we run on mode 'independent' - ref_file = ( - f"{rev_order_runs[i]}/psi/{out_parameters['depsi:general:AoI-name']}_" - f"{out_parameters['general:input-data:sensor'].lower()}_" - f"{asc_dsc[track]}_t{tracks[track]:0>3d}_ref_sel1.raw" - ) # this file saves the selected reference - if os.path.exists(ref_file): - ref_data = np.memmap(ref_file, mode="r", shape=(3,), dtype="float64") - # this outputs the reference point in [index, az, r]. We need [az,r] - ref_cn = f"[{int(round(ref_data[1]))},{int(round(ref_data[2]))}]" - break # we found one, so we can stop + coregistration_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["doris_v4"], track=tracks[track] + ) - else: - raise ValueError( - f"Expected types are dictionary, 'independent', '[]', '[az, r]', or 'constant', got {mode}" - ) + # we need a process folder in the coregistration directory, so we can combine that command + os.makedirs(f"{coregistration_directory}/process", exist_ok=True) - # write depsi.m + # generate doris_v4.sh write_run_file( - save_path=f"{depsi_directory}/depsi.m", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/depsi/depsi.m", + save_path=f"{coregistration_directory}/run_doris_v4.sh", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/doris_v4/run_doris_v4.sh", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, - other_parameters={ - "geocoding_version": out_parameters["depsi:general:geocoding-directory"].split("/")[-1].rstrip(), - "depsi_version": out_parameters["depsi:general:depsi-code-directory"].split("/")[-1].rstrip(), - }, + parameter_file_parameters=[ + "doris_v4:general:deinsar-code-directory", + "doris_v4:general:doris-v4-code-directory", + "doris_v4:general:AoI-name", + ], + config_parameters=["caroline_work_directory", "orbit_directory", "python2_module", "gdal_module"], + other_parameters={"track": tracks[track], "coregistration_base_directory": coregistration_directory}, ) - # write depsi.sh - write_run_file( - save_path=f"{depsi_directory}/depsi.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/depsi/depsi.sh", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=["depsi:general:AoI-name"], - config_parameters=["caroline_work_directory", "matlab_module"], - other_parameters={"depsi_base_directory": depsi_directory, "track": tracks[track]}, - ) + # generate run_doris_v4.py - # create param_file_depsi.txt - # + # first search for the start, end, and master dates by parsing all data in the data directory, + # which is different per sensor + datadir = datadirs[ + f"{out_parameters['general:input-data:sensor'].lower()}_{asc_dsc[track]}_t{tracks[track]:0>3d}" + ] + if out_parameters["general:input-data:sensor"] in ["ALOS2", "ERS"]: + dirs = glob.glob(f"{datadir}/[12]*") + images = list(sorted([eval(image.split("/")[-1]) for image in dirs])) + elif out_parameters["general:input-data:sensor"] in ["RSAT2"]: + dirs = glob.glob(f"{datadir}/RS2*") + images = list(sorted([eval(image.split("/")[-1].split("FQ2_")[1].split("_")[0]) for image in dirs])) + elif out_parameters["general:input-data:sensor"] in ["TSX"]: + dirs = glob.glob(f"{datadir}/*/iif/*") + images = list(sorted([eval(image.split("/")[-1].split("SRA_")[1].split("T")[0]) for image in dirs])) + elif out_parameters["general:input-data:sensor"] in ["SAOCOM"]: + dirs = glob.glob(f"{datadir}/*/*.xemt") + images = list(sorted([eval(image.split("/")[-1].split("OLF_")[1].split("T")[0]) for image in dirs])) + elif out_parameters["general:input-data:sensor"] in ["ENV"]: + # 2 different formats for some reason + dirs1 = glob.glob(f"{datadir}/*.N1") + dirs2 = glob.glob(f"{datadir}/*/*.N1") + dirs = [] + for d in dirs1: + dirs.append(d) + for d in dirs2: + dirs.append(d) + images = list(sorted([eval(image.split("/")[-1].split("PA")[1].split("_")[0]) for image in dirs])) + else: + raise ValueError(f'Unknown directory format for sensor {out_parameters["general:input-data:sensor"]}!') + + # then select the start, end, and master dates + act_start_date = str(min([image for image in images if image >= start_date])) + act_end_date = str(max([image for image in images if image <= end_date])) + act_master_date = str(min([image for image in images if image >= master_date])) + + # finally, write run_doris_v4.py write_run_file( - save_path=f"{depsi_directory}/param_file_depsi.txt", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/depsi/param_file.txt", + save_path=f"{coregistration_directory}/run_doris_v4.py", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/doris_v4/run_doris_v4.py", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, parameter_file_parameters=[ - "depsi:general:AoI-name", - "depsi:depsi-settings:general:max-mem-buffer", - "depsi:depsi-settings:general:visible-plots", - "depsi:depsi-settings:general:detail-plots", - "depsi:depsi-settings:general:processing-groups", - "depsi:depsi-settings:general:run-mode", - ["general:input-data:sensor", "lowercase"], - "depsi:depsi-settings:general:exclude-date", - "depsi:depsi-settings:general:az-spacing", - "depsi:depsi-settings:general:r-spacing", - "depsi:depsi-settings:general:slc-selection-input", - "depsi:depsi-settings:general:ifg-selection-input", - "depsi:depsi-settings:general:Ncv", - "depsi:depsi-settings:general:ps-method", - "depsi:depsi-settings:general:psc-model", - "depsi:depsi-settings:general:ps-model", - "depsi:depsi-settings:general:final-model", - "depsi:depsi-settings:general:breakpoint", - "depsi:depsi-settings:general:breakpoint2", - "depsi:depsi-settings:general:ens-coh-threshold", - "depsi:depsi-settings:general:varfac-threshold", - "depsi:depsi-settings:general:detrend-method", - "depsi:depsi-settings:general:output-format", - "depsi:depsi-settings:general:do-apriori-sidelobe-mask", - "depsi:depsi-settings:general:do-aposteriori-sidelobe-mask", - "depsi:depsi-settings:geocoding:ref-height", - "depsi:depsi-settings:psc:amplitude-calibration", - "depsi:depsi-settings:psc:psc-selection-method", - "depsi:depsi-settings:psc:psc-selection-gridsize", - "depsi:depsi-settings:psc:psc-threshold", - "depsi:depsi-settings:psc:max-arc-length", - "depsi:depsi-settings:psc:network-method", - "depsi:depsi-settings:psc:Ncon", - "depsi:depsi-settings:psc:Nparts", - "depsi:depsi-settings:psc:Npsc-selections", - "depsi:depsi-settings:psc:gamma-threshold", - "depsi:depsi-settings:psc:psc-distribution", - "depsi:depsi-settings:psc:weighted-unwrap", - "depsi:depsi-settings:psc:livetime-threshold", - "depsi:depsi-settings:psc:peak-tolerance", - "depsi:depsi-settings:psp:psp-selection-method", - "depsi:depsi-settings:psp:psp-threshold1", - "depsi:depsi-settings:psp:psp-threshold2", - "depsi:depsi-settings:psp:ps-eval-method", - "depsi:depsi-settings:psp:Namp-disp-bins", - "depsi:depsi-settings:psp:Ndens-iterations", - "depsi:depsi-settings:psp:densification-flag", - "depsi:depsi-settings:psp:ps-area-of-interest", - "depsi:depsi-settings:psp:dens-method", - "depsi:depsi-settings:psp:dens-check", - "depsi:depsi-settings:psp:Nest", - "depsi:depsi-settings:stochastic-model:defo-range", - "depsi:depsi-settings:stochastic-model:weighting", - "depsi:depsi-settings:stochastic-model:ts-atmo-filter", - "depsi:depsi-settings:stochastic-model:ts-atmo-filter-length", - "depsi:depsi-settings:stochastic-model:ts-noise-filter", - "depsi:depsi-settings:stochastic-model:ts-noise-filter-length", - "depsi:depsi-settings:bowl:defo-method", - "depsi:depsi-settings:bowl:xc0", - "depsi:depsi-settings:bowl:yc0", - "depsi:depsi-settings:bowl:zc0", - "depsi:depsi-settings:bowl:r0", - "depsi:depsi-settings:bowl:r10", - "depsi:depsi-settings:bowl:epoch", - ["depsi:depsi-settings:general:stc-min-max", "strip", "[] "], - ["depsi:depsi-settings:stochastic-model:std-param", "strip", "[] "], + ["doris_v4:input:data-directories", "dictionary"], + "general:input-data:sensor", + "general:input-data:polarisation", + "doris_v4:doris_v4-settings:do-orbit", + "doris_v4:doris_v4-settings:do-crop", + "doris_v4:doris_v4-settings:do-tsx-deramp", + "doris_v4:doris_v4-settings:do-simamp", + "doris_v4:doris_v4-settings:do-mtiming", + "doris_v4:doris_v4-settings:do-ovs", + "doris_v4:doris_v4-settings:do-choose-master", + "doris_v4:doris_v4-settings:do-coarseorb", + "doris_v4:doris_v4-settings:do-coarsecorr", + "doris_v4:doris_v4-settings:finecoreg:do-finecoreg", + "doris_v4:doris_v4-settings:do-reltiming", + "doris_v4:doris_v4-settings:do-dembased", + "doris_v4:doris_v4-settings:do-coregpm", + "doris_v4:doris_v4-settings:do-comprefpha", + "doris_v4:doris_v4-settings:do-comprefdem", + "doris_v4:doris_v4-settings:do-resample", + "doris_v4:doris_v4-settings:do-tsx-reramp", + "doris_v4:doris_v4-settings:do-interferogram", + "doris_v4:doris_v4-settings:do-subtrrefpha", + "doris_v4:doris_v4-settings:do-subtrrefdem", + "doris_v4:doris_v4-settings:do-coherence", + "doris_v4:doris_v4-settings:do-geocoding", ], - other_parameters={ - "crop_base_directory": crop_directory, - "track": f"{tracks[track]:0>3d}", - "asc_dsc": asc_dsc[track], - "asc_dsc_fmt": "desc" if asc_dsc[track] == "dsc" else asc_dsc[track], - "start_date": act_start_date, - "stop_date": act_end_date, - "master_date": mother_date, - "ref_cn": ref_cn, - "filename_water_mask": filename_water_mask, - }, + other_parameters={"master": act_master_date, "startdate": act_start_date, "enddate": act_end_date}, ) - write_directory_contents( - depsi_directory, filename=f'dir_contents{JOB_DEFINITIONS["depsi"]["directory-contents-file-appendix"]}.txt' + # finally, create the input files + + # these ones can be copied directly + for file in [ + "input.baselines", + "input.coarsecorr", + "input.coarseorb", + "input.comprefpha", + "input.coregpm", + "input.mtiming", + "input.reltiming", + "input.geocoding", + ]: + write_run_file( + save_path=f"{coregistration_directory}/process/{file}", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/doris_v4/input_files/{file}", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + ) + + # these ones are polarisation-dependent + for file in ["input.coherence", "input.interferogram", "input.subtrrefdem", "input.subtrrefpha", "input.ovs"]: + for pol in polarisation: + write_run_file( + save_path=f"{coregistration_directory}/process/{file}{pol}", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" + f"templates/doris_v4/input_files/{file}", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + other_parameters={"pol": pol}, + ) + + # these ones need the DEM variables + for file in ["input.comprefdem", "input.dembased", "input.simamp"]: + write_run_file( + save_path=f"{coregistration_directory}/process/{file}", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/doris_v4/input_files/{file}", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=["general:dem:file", "general:dem:format", "general:dem:nodata"], + other_parameters={ + "dem_s1": dem_size[0], + "dem_s2": dem_size[1], + "dem_d1": dem_delta[0], + "dem_d2": dem_delta[1], + "dem_ul1": dem_upperleft[0], + "dem_ul2": dem_upperleft[1], + }, + ) + + # finecoreg changes based on the fine coregistration mode + if out_parameters["doris_v4:doris_v4-settings:finecoreg:finecoreg-mode"] == "simple": + write_run_file( + save_path=f"{coregistration_directory}/process/input.finecoreg_simple", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" + f"templates/doris_v4/input_files/input.finecoreg", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + other_parameters={"nwin": 5000}, + ) + else: # normal mode + write_run_file( + save_path=f"{coregistration_directory}/process/input.finecoreg", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" + f"templates/doris_v4/input_files/input.finecoreg", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + other_parameters={"nwin": 8000}, + ) + + # porbit is only necessary for ERS and ENV + if out_parameters["general:input-data:sensor"] == "ERS": + # this one requires two copies + for satellite in [1, 2]: + write_run_file( + save_path=f"{coregistration_directory}/process/input.porbit_ERS{satellite}", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" + f"templates/doris_v4/input_files/input.porbit", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + other_parameters={"directory": f"ERS{satellite}"}, + ) + elif out_parameters["general:input-data:sensor"] == "ENV": + write_run_file( + save_path=f"{coregistration_directory}/process/input.porbit", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" + f"templates/doris_v4/input_files/input.porbit", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + other_parameters={"directory": "envisat/dor_vor_odr"}, + ) + + # for input.crop and input.resample we need to read the shapefile extent and calculate the amount of pixels + coordinates = np.array( + read_shp_extent( + f"{out_parameters['general:shape-file:directory']}/" + f"{out_parameters['general:shape-file:aoi-name']}_shape.shp", + shp_type="AoI", + )["0"] ) + min_lat = min(coordinates[:, 1]) + max_lat = max(coordinates[:, 1]) + min_lon = min(coordinates[:, 0]) + max_lon = max(coordinates[:, 0]) + # get the central coordinates + center_lon = (max_lon + min_lon) / 2 + center_lat = (max_lat + min_lat) / 2 -def prepare_depsi_post(parameter_file: str, do_track: int | list | None = None) -> None: - """Set up the directories and files for DePSI-post. + # get the latitude at which the crop is widest + if min_lat < 0: + if max_lat > 0: + ref_lat = 0 + else: + ref_lat = max_lat + else: + ref_lat = min_lat - Parameters - ---------- - parameter_file: str - Absolute path to the parameter file. - do_track: int | list | None, optional - Track number, or list of track numbers, of the track(s) to prepare. `None` (default) prepares all tracks in - the parameter file + # calculate the extent of the AoI + dist_lat = haversine(min_lat, max_lat, min_lon, min_lon) + dist_lon = haversine(ref_lat, ref_lat, min_lon, max_lon) # calculated at the widest part of the AoI - Raises - ------ - ValueError - If `depsi_post_mode` is not 'tarball' or 'csv' - """ - search_parameters = [ - "general:tracks:track", - "general:tracks:asc_dsc", - "general:input-data:sensor", - "depsi_post:general:depsi_post-code-directory", - "depsi_post:depsi_post-settings:defo-clim", - "depsi_post:depsi_post-settings:height-clim", - "depsi:general:rdnaptrans-directory", - "depsi:general:geocoding-directory", - "general:workflow:filters:depsi_post-output", - ] - out_parameters = read_parameter_file(parameter_file, search_parameters) + # determine the number of pixels + d_az, d_r = detect_sensor_pixelsize(out_parameters["general:input-data:sensor"]) + pix_dr = int(np.ceil(dist_lon / d_r * 1.05)) + pix_daz = int(np.ceil(dist_lat / d_az * 1.05)) - tracks = out_parameters["general:tracks:track"] - asc_dsc = out_parameters["general:tracks:asc_dsc"] + # for input.crop we will add 500 to eliminate edge effects + if out_parameters["general:input-data:sensor"] == "ALOS2": + img_name = "IMG.1" + elif out_parameters["general:input-data:sensor"] == "Cosmo": + img_name = "image.h5" + elif out_parameters["general:input-data:sensor"] == "ENV": + img_name = "image.N1" + elif out_parameters["general:input-data:sensor"] == "ERS": + img_name = "DAT_01.001" + elif out_parameters["general:input-data:sensor"] == "RSAT2": + img_name = "imagery{pol}.tif" + # requires loop over polarisations to get additional crop files + elif out_parameters["general:input-data:sensor"] == "TSX": + img_name = "image.cos" + else: + raise ValueError(f'Unknown sensor {out_parameters["general:input-data:sensor"]}!') - defo_clim_raw = out_parameters["depsi_post:depsi_post-settings:defo-clim"] - defo_clim_min = defo_clim_raw[0] - defo_clim_max = defo_clim_raw[1] + # write input.crop + if ( + out_parameters["general:input-data:sensor"] == "RSAT2" + ): # for RSAT2 this is per polarisation, otherwise there just is one + for pol in polarisation: + write_run_file( + save_path=f"{coregistration_directory}/process/input.crop{pol}", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" + f"templates/doris_v4/input_files/input.crop", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + other_parameters={ + "img_name": img_name.format(pol), + "pol": pol, + "center_lat": center_lat, + "center_lon": center_lon, + "pix_az": pix_daz + 500, + "pix_r": pix_dr + 500, + }, + ) + else: + write_run_file( + save_path=f"{coregistration_directory}/process/input.crop", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" + f"templates/doris_v4/input_files/input.crop", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + other_parameters={ + "img_name": img_name, + "pol": "", + "center_lat": center_lat, + "center_lon": center_lon, + "pix_az": pix_daz + 500, + "pix_r": pix_dr + 500, + }, + ) - height_clim_raw = out_parameters["depsi_post:depsi_post-settings:height-clim"] - height_clim_min = height_clim_raw[0] - height_clim_max = height_clim_raw[1] + # write input.resample + for pol in polarisation: + write_run_file( + save_path=f"{coregistration_directory}/process/input.resample{pol}", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" + f"templates/doris_v4/input_files/input.resample", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + other_parameters={ + "pol": pol, + "center_lat": center_lat, + "center_lon": center_lon, + "pix_az": pix_daz, + "pix_r": pix_dr, + }, + ) - if out_parameters["general:workflow:filters:depsi_post-output"] == "tarball": - do_csv = 0 - elif out_parameters["general:workflow:filters:depsi_post-output"] == "csv": - do_csv = 1 - else: - raise ValueError( - "general:workflow:filters:depsi_post-output is set to " - f"{out_parameters['general:workflow:filters:depsi_post-output']}, only know 'tarball' and 'csv'!" - ) + # finally, we need input.readfiles, which requires a data string composed of sensor-specific data + if out_parameters["general:input-data:sensor"] == "ALOS2": + data_string = """S_IN_METHOD ALOS2 +S_IN_DAT IMG.1 +S_IN_LEA LED.1 +S_IN_VOL VOL.1""" - for track in range(len(tracks)): - if isinstance(do_track, int): - if tracks[track] != do_track: - continue - elif isinstance(do_track, list): - if tracks[track] not in do_track: - continue + elif out_parameters["general:input-data:sensor"] == "Cosmo": + data_string = """S_IN_METHOD CSK +S_IN_DAT image.h5""" - depsi_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["depsi_post"], track=tracks[track] - ) + elif out_parameters["general:input-data:sensor"] == "ERS": + data_string = """S_IN_METHOD ERS +S_IN_VOL VRD_DAT.001 +S_IN_DAT DAT_01.001 +S_IN_LEA LEA_01.001 +S_IN_NULL dummy""" - # link the DePSI-post box - os.system(f"cp -Rp {out_parameters['depsi_post:general:depsi_post-code-directory']} {depsi_directory}/../boxes") + elif out_parameters["general:input-data:sensor"] == "ENV": + data_string = """S_IN_METHOD ASAR +S_IN_DAT image.N1""" - # write depsi_post.m - write_run_file( - save_path=f"{depsi_directory}/depsi_post.m", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/depsi_post/depsi_post.m", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=[ - "depsi_post:depsi_post-settings:dlat", - "depsi_post:depsi_post-settings:dlon", - "depsi_post:depsi_post-settings:drdx", - "depsi_post:depsi_post-settings:drdy", - "general:input-data:sensor", - "depsi:general:AoI-name", - "depsi_post:depsi_post-settings:proj", - "depsi_post:depsi_post-settings:ref-dheight", - "depsi_post:depsi_post-settings:posteriori-scale-factor", - ["depsi_post:depsi_post-settings:pred-model", "strip", " "], - "depsi_post:depsi_post-settings:plot-mode", - ["depsi_post:depsi_post-settings:do-plots", "strip", "{} "], - ["depsi_post:depsi_post-settings:output", "strip", "{} "], - "depsi_post:depsi_post-settings:fontsize", - "depsi_post:depsi_post-settings:markersize", - "depsi_post:depsi_post-settings:do-print", - "depsi_post:depsi_post-settings:output-format", - "depsi_post:depsi_post-settings:az0", - "depsi_post:depsi_post-settings:azN", - "depsi_post:depsi_post-settings:r0", - "depsi_post:depsi_post-settings:rN", - "depsi_post:depsi_post-settings:result", - "depsi_post:depsi_post-settings:psc-selection", - "depsi_post:depsi_post-settings:do-remove-filtered", - "depsi_post:depsi_post-settings:which-sl-mask", - "depsi_post:depsi_post-settings:shift-to-mean", - "depsi_post:depsi_post-settings:new-ref-cn", - "depsi_post:depsi_post-settings:map-to-vert", - "depsi_post:depsi_post-settings:defo-lim", - "depsi_post:depsi_post-settings:height-lim", - "depsi_post:depsi_post-settings:ens-coh-lim", - "depsi_post:depsi_post-settings:ens-coh-local-lim", - "depsi_post:depsi_post-settings:stc-lim", - "depsi_post:depsi_post-settings:ens-coh-clim", - "depsi_post:depsi_post-settings:ens-coh-local-clim", - "depsi_post:depsi_post-settings:stc-clim", - ], - other_parameters={ - "geocoding_version": out_parameters["depsi:general:geocoding-directory"].split("/")[-1].rstrip(), - "depsi_post_version": out_parameters["depsi_post:general:depsi_post-code-directory"] - .split("/")[-1] - .rstrip(), - "rdnaptrans_version": out_parameters["depsi:general:rdnaptrans-directory"].split("/")[-1].rstrip(), - "do_csv": do_csv, - "asc_dsc": asc_dsc[track], - "track": tracks[track], - "fill_track": f"{tracks[track]:0>3d}", - "dp_defo_clim_min": defo_clim_min, - "dp_defo_clim_max": defo_clim_max, - "dp_height_clim_min": height_clim_min, - "dp_height_clim_max": height_clim_max, - }, - ) + elif out_parameters["general:input-data:sensor"] == "RSAT": + data_string = """S_IN_METHOD RSAT +S_IN_VOL VDF_DAT.001 +S_IN_DAT DAT_01.001 +S_IN_LEA LEA_01.001 +S_IN_NULL dummy""" + + elif out_parameters["general:input-data:sensor"] == "RSAT2": + data_string = """S_IN_METHOD RADARSAT-2 +S_IN_DAT imagery_HH.tif +S_IN_LEA product.xml""" + + elif out_parameters["general:input-data:sensor"] == "TSX": + data_string = """S_IN_METHOD TSX +S_IN_DAT image.cos +S_IN_LEA leader.xml""" + + else: + raise ValueError(f'Unknown sensor {out_parameters["general:input-data:sensor"]} for input.readfiles!') - # write depsi_post.sh write_run_file( - save_path=f"{depsi_directory}/depsi_post.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/depsi_post/depsi_post.sh", + save_path=f"{coregistration_directory}/process/input.readfiles", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/" + f"templates/doris_v4/input_files/input.readfiles", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, - parameter_file_parameters=["depsi:general:AoI-name"], - config_parameters=["caroline_work_directory", "matlab_module"], - other_parameters={"track": tracks[track], "depsi_base_directory": depsi_directory}, + other_parameters={"data_string": data_string}, ) write_directory_contents( - depsi_directory, - filename=f'dir_contents{JOB_DEFINITIONS["depsi_post"]["directory-contents-file-appendix"]}.txt', + coregistration_directory, + filename=f'dir_contents{JOB_DEFINITIONS["doris_v4"]["directory-contents-file-appendix"]}.txt', ) -def prepare_doris(parameter_file: str, do_track: int | list | None = None) -> None: +def prepare_doris_v5(parameter_file: str, do_track: int | list | None = None) -> None: """Set up the directories and run files for Doris v5. Parameters @@ -1406,7 +1341,7 @@ def prepare_doris(parameter_file: str, do_track: int | list | None = None) -> No continue coregistration_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["doris"], track=tracks[track] + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["doris_v5"], track=tracks[track] ) # we need a process folder in the coregistration directory, so we can combine that command @@ -1436,7 +1371,7 @@ def prepare_doris(parameter_file: str, do_track: int | list | None = None) -> No # generate the input files input_files = glob.glob( - f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/" "doris/input_files/input.*" + f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/doris_v5/input_files/input.*" ) for file in input_files: if file.split("/")[-1] in ["input.comprefdem", "input.dembased"]: @@ -1471,30 +1406,30 @@ def prepare_doris(parameter_file: str, do_track: int | list | None = None) -> No # we need to transform all the 1/0 from the parameter file into Yes/No other_parameters = {} for parameter in [ - "doris:doris-settings:do-coarse-orbits", - "doris:doris-settings:do-deramp", - "doris:doris-settings:do-reramp", - "doris:doris-settings:do-fake-fine-coreg-bursts", - "doris:doris-settings:do-dac-bursts", - "doris:doris-settings:do-fake-coreg-bursts", - "doris:doris-settings:do-fake-master-resample", - "doris:doris-settings:do-resample", - "doris:doris-settings:do-reramp2", - "doris:doris-settings:do-interferogram", - "doris:doris-settings:do-compref-phase", - "doris:doris-settings:do-compref-dem", - "doris:doris-settings:do-coherence", - "doris:doris-settings:do-esd", - "doris:doris-settings:do-network-esd", - "doris:doris-settings:do-ESD-correct", - "doris:doris-settings:do-combine-master", - "doris:doris-settings:do-combine-slave", - "doris:doris-settings:do-ref-phase", - "doris:doris-settings:do-ref-dem", - "doris:doris-settings:do-phasefilt", - "doris:doris-settings:do-calc-coordinates", - "doris:doris-settings:do-multilooking", - "doris:doris-settings:do-unwrap", + "doris_v5:doris_v5-settings:do-coarse-orbits", + "doris_v5:doris_v5-settings:do-deramp", + "doris_v5:doris_v5-settings:do-reramp", + "doris_v5:doris_v5-settings:do-fake-fine-coreg-bursts", + "doris_v5:doris_v5-settings:do-dac-bursts", + "doris_v5:doris_v5-settings:do-fake-coreg-bursts", + "doris_v5:doris_v5-settings:do-fake-master-resample", + "doris_v5:doris_v5-settings:do-resample", + "doris_v5:doris_v5-settings:do-reramp2", + "doris_v5:doris_v5-settings:do-interferogram", + "doris_v5:doris_v5-settings:do-compref-phase", + "doris_v5:doris_v5-settings:do-compref-dem", + "doris_v5:doris_v5-settings:do-coherence", + "doris_v5:doris_v5-settings:do-esd", + "doris_v5:doris_v5-settings:do-network-esd", + "doris_v5:doris_v5-settings:do-ESD-correct", + "doris_v5:doris_v5-settings:do-combine-master", + "doris_v5:doris_v5-settings:do-combine-slave", + "doris_v5:doris_v5-settings:do-ref-phase", + "doris_v5:doris_v5-settings:do-ref-dem", + "doris_v5:doris_v5-settings:do-phasefilt", + "doris_v5:doris_v5-settings:do-calc-coordinates", + "doris_v5:doris_v5-settings:do-multilooking", + "doris_v5:doris_v5-settings:do-unwrap", ]: value = read_parameter_file(parameter_file, [parameter])[parameter] if value == 1: @@ -1540,7 +1475,7 @@ def prepare_doris(parameter_file: str, do_track: int | list | None = None) -> No # write doris_input.xml write_run_file( save_path=f"{coregistration_directory}/doris_input.xml", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/doris/doris_input.xml", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/doris_v5/doris_input.xml", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, @@ -1552,11 +1487,11 @@ def prepare_doris(parameter_file: str, do_track: int | list | None = None) -> No # write doris_stack.sh write_run_file( save_path=f"{coregistration_directory}/doris_stack.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/doris/doris_stack.sh", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/doris_v5/doris_stack.sh", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, - parameter_file_parameters=["doris:general:AoI-name", "doris:general:code-directory"], + parameter_file_parameters=["doris_v5:general:AoI-name", "doris_v5:general:code-directory"], config_parameters=[ "caroline_work_directory", "caroline_virtual_environment_directory", @@ -1568,11 +1503,11 @@ def prepare_doris(parameter_file: str, do_track: int | list | None = None) -> No write_directory_contents( coregistration_directory, - filename=f'dir_contents{JOB_DEFINITIONS["doris"]["directory-contents-file-appendix"]}.txt', + filename=f'dir_contents{JOB_DEFINITIONS["doris_v5"]["directory-contents-file-appendix"]}.txt', ) -def prepare_doris_cleanup(parameter_file: str, do_track: int | list | None = None) -> None: +def prepare_doris_v5_cleanup(parameter_file: str, do_track: int | list | None = None) -> None: """Set up the cleanup script to clean the directories produced by Doris v5. Parameters @@ -1601,13 +1536,13 @@ def prepare_doris_cleanup(parameter_file: str, do_track: int | list | None = Non continue coregistration_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["doris_cleanup"], track=tracks[track] + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["doris_v5_cleanup"], track=tracks[track] ) write_run_file( save_path=f"{coregistration_directory}/cleanup.sh", template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/" - "doris/cleanup-doris-s1-stack.sh", + "doris_v5/cleanup-doris-s1-stack.sh", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, @@ -1659,8 +1594,8 @@ def prepare_email(parameter_file: str, do_track: int | list | None = None) -> No {body}" | {CONFIG_PARAMETERS['SENDMAIL_EXECUTABLE']} {out_parameters['general:email:recipients']}""") -def prepare_mrm(parameter_file: str, do_track: int | list | None = None) -> None: - """Set up the directories and files for mrm creation, part of DePSI-post. +def prepare_generate_partitioned_stm(parameter_file: str, do_track: int | list | None = None) -> None: + """Set up the directories and run files for STM generation. Parameters ---------- @@ -1671,10 +1606,11 @@ def prepare_mrm(parameter_file: str, do_track: int | list | None = None) -> None the parameter file """ search_parameters = [ + "generate_partitioned_stm:general:AoI-name", + "generate_partitioned_stm:general:directory", "general:tracks:track", "general:tracks:asc_dsc", "general:input-data:sensor", - "depsi_post:general:cpxfiddle-directory", "general:workflow:filters:coregistration-mode", ] out_parameters = read_parameter_file(parameter_file, search_parameters) @@ -1690,77 +1626,361 @@ def prepare_mrm(parameter_file: str, do_track: int | list | None = None) -> None if tracks[track] not in do_track: continue - # determine if we came from crop_to_raw or znap_to_raw + stm_directory = format_process_folder( + parameter_file=parameter_file, + job_description=JOB_DEFINITIONS["generate_partitioned_stm"], + track=tracks[track], + ) + + # determine if we came from reduce_slc_python or merge_to_stack_python if ( out_parameters["general:workflow:filters:coregistration-mode"] == "doris" or out_parameters["general:input-data:sensor"].lower() != "s1" ): - crop_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["crop_to_raw"], track=tracks[track] + reduce_slc_python_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["reduce_slc_python"], track=tracks[track] ) else: - crop_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["znap_to_raw"], track=tracks[track] + reduce_slc_python_directory = format_process_folder( + parameter_file=parameter_file, + job_description=JOB_DEFINITIONS["merge_to_stack_python"], + track=tracks[track], ) - depsi_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["depsi"], track=tracks[track] + os.makedirs(stm_directory, exist_ok=True) + + # generate stm-generation.py + stm_output_name = stm_directory.split("/")[-1] + reduce_slc_python_output_name = reduce_slc_python_directory.split("/")[-1] + + write_run_file( + save_path=f"{stm_directory}/generate-partitioned-stm.py", + template_path=( + f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/" + "generate-partitioned-stm/generate-partitioned-stm.py" + ), + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=[ + "generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:mode", + "generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:init-settings:start-date", + "generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:init-settings:init-length", + "generate_partitioned_stm:generate_partitioned_stm-settings:incremental-statistics:increment-mode", + "generate_partitioned_stm:generate_partitioned_stm-settings:incremental-statistics:recal-jump-size", + "generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:method", + "generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:threshold", + "generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:do-outlier-detection", + "generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:window-size", + "generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:db-mode", + "generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:n-sigma", + "generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:do-partitioning", + "generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:search-method", + "generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:cost-function", + "generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:db-mode", + "generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:min-partition-length", + "generate_partitioned_stm:generate_partitioned_stm-settings:single-differences:mother", + "generate_partitioned_stm:generate_partitioned_stm-settings:extra-projection", + "generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:undifferenced-output-lyrs", + "generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:single-difference-output-lyrs", + ], + other_parameters={ + "reduce_slc_python_directory": reduce_slc_python_directory, + "reduce_slc_python_output_name": reduce_slc_python_output_name, + "stm_output_directory": stm_directory, + "stm_output_name": stm_output_name, + }, ) - # we need to run cpxfiddle first. This requires two parameters: n_lines, and the project ID - fr = open(f"{crop_directory}/cropped_stack/nlines_crp.txt") - data = fr.read().split("\n") - fr.close() - n_lines = data[0] + # generate stm-generation.sh + write_run_file( + save_path=f"{stm_directory}/generate-partitioned-stm.sh", + template_path=( + f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/generate-partitioned-stm/" + "generate-partitioned-stm.sh" + ), + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=[ + "generate_partitioned_stm:general:AoI-name", + "reduce_slc_python:general:depsi_group-code-directory", + ], + config_parameters=[ + "caroline_work_directory", + "caroline_virtual_environment_directory", + "python3_module", + "gdal_module", + ], + other_parameters={"track": tracks[track]}, + ) - project_id = depsi_directory.split("/")[-2].split("-")[0] + write_directory_contents( + stm_directory, + filename=( + f'dir_contents{JOB_DEFINITIONS["generate_partitioned_stm"]["directory-contents-file-appendix"]}.txt' + ), + ) - # format the arguments in the correct order - command_args = ( - f"{project_id} {n_lines} 1 1 {out_parameters['depsi_post:general:cpxfiddle-directory']} {depsi_directory}" + +def prepare_merge_to_stack_matlab(parameter_file: str, do_track: int | list | None = None) -> None: + """Set up the directories and run files for merge_to_stack_matlab. + + Parameters + ---------- + parameter_file: str + Absolute path to the parameter file. + do_track: int | list | None, optional + Track number, or list of track numbers, of the track(s) to prepare. `None` (default) prepares all tracks in + the parameter file + """ + search_parameters = [ + "general:tracks:track", + "general:tracks:asc_dsc", + "general:input-data:sensor", + ] + out_parameters = read_parameter_file(parameter_file, search_parameters) + + tracks = out_parameters["general:tracks:track"] + asc_dsc = out_parameters["general:tracks:asc_dsc"] + for track in range(len(tracks)): + if isinstance(do_track, int): + if tracks[track] != do_track: + continue + elif isinstance(do_track, list): + if tracks[track] not in do_track: + continue + + merge_to_stack_matlab_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["merge_to_stack_matlab"], track=tracks[track] ) - os.system( - f"bash {CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/scripts/create_mrm_ras_header.sh " - f"{command_args}" + + coregistration_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["snap"], track=tracks[track] ) + os.makedirs(merge_to_stack_matlab_directory, exist_ok=True) + + # generate znap-to-raw.py write_run_file( - save_path=f"{depsi_directory}/read_mrm.m", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/mrm/read_mrm.m", + save_path=f"{merge_to_stack_matlab_directory}/merge-to-stack-matlab.py", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/merge-to-stack-matlab/merge-to-stack-matlab.py", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, parameter_file_parameters=[ - "depsi:general:AoI-name", - ["general:input-data:sensor", "lowercase"], + "general:shape-file:aoi-name", + "general:shape-file:directory", ], other_parameters={ - "fill_track": f"{tracks[track]:0>3d}", - "asc_dsc": asc_dsc[track], + "snap-output-path": coregistration_directory, + "raw-output-path": merge_to_stack_matlab_directory, }, ) + # generate znap-to-raw.sh write_run_file( - save_path=f"{depsi_directory}/read_mrm.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/mrm/read_mrm.sh", + save_path=f"{merge_to_stack_matlab_directory}/merge-to-stack-matlab.sh", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/merge-to-stack-matlab/merge-to-stack-matlab.sh", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, - parameter_file_parameters=["depsi:general:AoI-name"], - config_parameters=["caroline_work_directory", "matlab_module"], + parameter_file_parameters=[ + "merge_to_stack_matlab:general:AoI-name", + "merge_to_stack_matlab:general:depsi_group-code-directory", + ], + config_parameters=[ + "caroline_work_directory", + "caroline_virtual_environment_directory", + "python3_module", + "gdal_module", + ], + other_parameters={"track": tracks[track]}, + ) + + write_directory_contents( + merge_to_stack_matlab_directory, + filename=f'dir_contents{JOB_DEFINITIONS["merge_to_stack_matlab"]["directory-contents-file-appendix"]}.txt', + ) + + +def prepare_merge_to_stack_python(parameter_file: str, do_track: int | list | None = None) -> None: + """Set up the directories and run files for merge_to_stack_python. + + Parameters + ---------- + parameter_file: str + Absolute path to the parameter file. + do_track: int | list | None, optional + Track number, or list of track numbers, of the track(s) to prepare. `None` (default) prepares all tracks in + the parameter file + """ + search_parameters = [ + "general:tracks:track", + "general:tracks:asc_dsc", + "general:input-data:sensor", + ] + out_parameters = read_parameter_file(parameter_file, search_parameters) + + tracks = out_parameters["general:tracks:track"] + asc_dsc = out_parameters["general:tracks:asc_dsc"] + for track in range(len(tracks)): + if isinstance(do_track, int): + if tracks[track] != do_track: + continue + elif isinstance(do_track, list): + if tracks[track] not in do_track: + continue + + merge_to_stack_python_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["merge_to_stack_python"], track=tracks[track] + ) + + coregistration_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["snap"], track=tracks[track] + ) + + os.makedirs(merge_to_stack_python_directory, exist_ok=True) + + # generate crop-to-zarr.py + merge_to_stack_python_output_name = merge_to_stack_python_directory.split("/")[-1] + + write_run_file( + save_path=f"{merge_to_stack_python_directory}/merge-to-stack-python.py", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/merge-to-stack-python/merge-to-stack-python.py", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=[ + "general:shape-file:aoi-name", + "general:shape-file:directory", + ], other_parameters={ - "track": tracks[track], - "depsi_base_directory": depsi_directory, + "snap-output-path": coregistration_directory, + "merge_to_stack_python_output_filename": merge_to_stack_python_output_name, }, ) + # generate crop-to-zarr.sh + write_run_file( + save_path=f"{merge_to_stack_python_directory}/merge-to-stack-python.sh", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/merge-to-stack-python/merge-to-stack-python.sh", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=[ + "merge_to_stack_python:general:AoI-name", + "merge_to_stack_python:general:depsi_group-code-directory", + ], + config_parameters=[ + "caroline_work_directory", + "caroline_virtual_environment_directory", + "python3_module", + "gdal_module", + ], + other_parameters={"track": tracks[track]}, + ) + + write_directory_contents( + merge_to_stack_python_directory, + filename=f'dir_contents{JOB_DEFINITIONS["merge_to_stack_python"]["directory-contents-file-appendix"]}.txt', + ) + + +def prepare_reduce_slc_matlab(parameter_file: str, do_track: int | list | None = None) -> None: + """Set up the directories and run files for cropping. + + Parameters + ---------- + parameter_file: str + Absolute path to the parameter file. + do_track: int | list | None, optional + Track number, or list of track numbers, of the track(s) to prepare. `None` (default) prepares all tracks in + the parameter file + """ + search_parameters = [ + "general:tracks:track", + "general:tracks:asc_dsc", + "general:input-data:sensor", + ] + out_parameters = read_parameter_file(parameter_file, search_parameters) + + tracks = out_parameters["general:tracks:track"] + asc_dsc = out_parameters["general:tracks:asc_dsc"] + + for track in range(len(tracks)): + if isinstance(do_track, int): + if tracks[track] != do_track: + continue + elif isinstance(do_track, list): + if tracks[track] not in do_track: + continue + + crop_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["reduce_slc_matlab"], track=tracks[track] + ) + + if out_parameters["general:input-data:sensor"] == "S1": + coregistration_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["doris_v5"], track=tracks[track] + ) + + else: + coregistration_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["doris_v4"], track=tracks[track] + ) + + os.makedirs(crop_directory, exist_ok=True) + + # soft-link the processing directory without job_id.txt, dir_contents.txt and queue.txt + # Sentinel-1 has more files starting with d as Doris-v5 output, other sensors do not have that + if out_parameters["general:input-data:sensor"] == "S1": + link_keys = ["[bgiprs]*", "doris*", "dem"] + else: + link_keys = ["[bgiprs]*"] + for key in link_keys: + # run the soft-link command + os.system(f"ln -sfn {coregistration_directory}/{key} {crop_directory}") + + # generate crop.sh + write_run_file( + save_path=f"{crop_directory}/reduce-slc-matlab.sh", + template_path=( + f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/reduce-slc-matlab/reduce-slc-matlab.sh" + ), + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=["reduce_slc_matlab:general:AoI-name"], + config_parameters=["caroline_work_directory", "matlab_module"], + other_parameters={"track": tracks[track], "crop_base_directory": crop_directory}, + ) + + # generate crop.m + write_run_file( + save_path=f"{crop_directory}/reduce_slc_matlab.m", + template_path=( + f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/reduce-slc-matlab/reduce_slc_matlab.m" + ), + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=[ + "general:shape-file:aoi-name", + "general:shape-file:directory", + "general:input-data:sensor", + ], + config_parameters=["caroline_install_directory"], + ) + write_directory_contents( - depsi_directory, filename=f'dir_contents{JOB_DEFINITIONS["mrm"]["directory-contents-file-appendix"]}.txt' + crop_directory, + filename=f'dir_contents{JOB_DEFINITIONS["reduce_slc_matlab"]["directory-contents-file-appendix"]}.txt', ) -def prepare_portal_upload(parameter_file: str, do_track: int | list | None = None) -> None: - """Create the indication for a portal upload. +def prepare_reduce_slc_python(parameter_file: str, do_track: int | list | None = None) -> None: + """Set up the directories and run files for reduce_slc_python. Parameters ---------- @@ -1769,16 +1989,21 @@ def prepare_portal_upload(parameter_file: str, do_track: int | list | None = Non do_track: int | list | None, optional Track number, or list of track numbers, of the track(s) to prepare. `None` (default) prepares all tracks in the parameter file + + Raises + ------ + ValueError + If the mother image cannot be detected from doris_input.xml (S1) or doris_v4.py (otherwise) """ search_parameters = [ "general:tracks:track", - "general:portal:skygeo-customer", - "general:portal:skygeo-viewer", + "general:tracks:asc_dsc", + "general:input-data:sensor", ] out_parameters = read_parameter_file(parameter_file, search_parameters) tracks = out_parameters["general:tracks:track"] - + asc_dsc = out_parameters["general:tracks:asc_dsc"] for track in range(len(tracks)): if isinstance(do_track, int): if tracks[track] != do_track: @@ -1787,23 +2012,98 @@ def prepare_portal_upload(parameter_file: str, do_track: int | list | None = Non if tracks[track] not in do_track: continue - depsi_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["depsi"], track=tracks[track] + reduce_slc_python_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["reduce_slc_python"], track=tracks[track] ) - # The parameter file already contains a datestamp so we don't need to redo that - portal_upload_file = ( - f"{CONFIG_PARAMETERS['PORTAL_UPLOAD_FLAG_DIRECTORY']}/" - f"{parameter_file.split('/')[-1].split('.')[0]}_t{tracks[track]:0>3d}_upload.txt" + if out_parameters["general:input-data:sensor"] == "S1": + coregistration_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["doris_v5"], track=tracks[track] + ) + + else: + coregistration_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["doris_v4"], track=tracks[track] + ) + + os.makedirs(reduce_slc_python_directory, exist_ok=True) + + # detect the mother image + if out_parameters["general:input-data:sensor"].lower() == "s1": + f = open(f"{coregistration_directory}/doris_input.xml") + data = f.read().split("\n") + f.close() + mother = None + for line in data: + if "" in line: + mother = line.split(">")[1].split("<")[0].replace("-", "") + break + + if mother is None: + raise ValueError(f"Failed to detect mother in {coregistration_directory}/doris_input.xml!") + + else: + f = open(f"{coregistration_directory}/run_doris_v4.py") + data = f.read().split("\n") + f.close() + mother = None + for line in data: + if "master = " in line: + mother = line.split('"')[1] + break + + if mother is None: + raise ValueError(f"Failed to detect mother in {coregistration_directory}/run_doris_v4.py !") + + # generate crop-to-zarr.py + reduce_slc_python_output_name = reduce_slc_python_directory.split("/")[-1] + + write_run_file( + save_path=f"{reduce_slc_python_directory}/reduce-slc-python.py", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/reduce-slc-python/reduce-slc-python.py", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=[ + "general:shape-file:aoi-name", + "general:input-data:sensor", + "general:shape-file:directory", + ], + other_parameters={ + "coregistration_directory": coregistration_directory, + "stack_folder_name": "stack" if out_parameters["general:input-data:sensor"] == "S1" else "process", + "mother": mother, + "mother_slc_name": "slave_rsmp_reramped.raw" + if out_parameters["general:input-data:sensor"] == "S1" + else "slave_rsmp.raw", + "reduce_slc_python_output_filename": reduce_slc_python_output_name, + }, ) - f = open(portal_upload_file, "w") - f.write( - f"Status: TBD\n" - f"Directory: {depsi_directory}\n" - f"Viewer: {out_parameters['general:portal:skygeo-viewer']}\n" - f"Customer: {out_parameters['general:portal:skygeo-customer']}" + + # generate crop-to-zarr.sh + write_run_file( + save_path=f"{reduce_slc_python_directory}/reduce-slc-python.sh", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/reduce-slc-python/reduce-slc-python.sh", + asc_dsc=asc_dsc[track], + track=tracks[track], + parameter_file=parameter_file, + parameter_file_parameters=[ + "reduce_slc_python:general:AoI-name", + "reduce_slc_python:general:depsi_group-code-directory", + ], + config_parameters=[ + "caroline_work_directory", + "caroline_virtual_environment_directory", + "python3_module", + "gdal_module", + ], + other_parameters={"track": tracks[track]}, + ) + + write_directory_contents( + reduce_slc_python_directory, + filename=f'dir_contents{JOB_DEFINITIONS["reduce_slc_python"]["directory-contents-file-appendix"]}.txt', ) - f.close() def prepare_s1_download(parameter_file: str, do_track: int | list | None = None) -> None: @@ -1940,171 +2240,14 @@ def prepare_s1_download(parameter_file: str, do_track: int | list | None = None) except zipfile.BadZipFile: # zip file cannot be opened --> incomplete download bad_zip_files.append(zip_file) - if len(bad_zip_files) > 0: - os.system('''echo "\n\n\nZIP File Integrity Check Failed!\n"''') - os.system(f'''echo "Failed on zip file(s) {str(bad_zip_files).replace('"', "'")}. Exiting with code 5...\n"''') - exit(5) # Make the code exit with a non-zero exit code so the next steps won't run - - -def prepare_snap_permissions(parameter_file: str, do_track: int | list | None = None) -> None: - """Change all the permissions for the SNAP output to 775. - - Parameters - ---------- - parameter_file: str - Absolute path to the parameter file. - do_track: int | list | None, optional - Track number, or list of track numbers, of the track(s) to prepare. `None` (default) prepares all tracks in - the parameter file - """ - search_parameters = [ - "general:tracks:track", - "general:tracks:asc_dsc", - ] - out_parameters = read_parameter_file(parameter_file, search_parameters) - - tracks = out_parameters["general:tracks:track"] - - for track in range(len(tracks)): - if isinstance(do_track, int): - if tracks[track] != do_track: - continue - elif isinstance(do_track, list): - if tracks[track] not in do_track: - continue - - snap_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["snap_run"], track=tracks[track] - ) - - znaps = glob.glob(f"{snap_directory}/*-coreg.znap") - for zf in znaps: - os.system(f"chmod 775 {zf}; chmod 775 {zf}/*; chmod 775 {zf}/*/*; chmod 775 {zf}/*/*/*") - - -def prepare_snap_preparation(parameter_file: str, do_track: int | list | None = None) -> None: - """Set up the directories and run files for SNAP preparation. - - Parameters - ---------- - parameter_file: str - Absolute path to the parameter file. - do_track: int | list | None, optional - Track number, or list of track numbers, of the track(s) to prepare. `None` (default) prepares all tracks in - the parameter file - """ - search_parameters = [ - "snap:general:AoI-name", - "snap:general:directory", - "general:tracks:track", - "general:tracks:asc_dsc", - "general:input-data:sensor", - "general:shape-file:aoi-name", - "general:shape-file:directory", - "general:timeframe:start", - "general:timeframe:end", - "general:timeframe:mother", - ] - out_parameters = read_parameter_file(parameter_file, search_parameters) - - tracks = out_parameters["general:tracks:track"] - asc_dsc = out_parameters["general:tracks:asc_dsc"] - - shapefile_name = ( - f"{out_parameters['general:shape-file:directory']}/{out_parameters['general:shape-file:aoi-name']}_shape.shp" - ) - - for track in range(len(tracks)): - if isinstance(do_track, int): - if tracks[track] != do_track: - continue - elif isinstance(do_track, list): - if tracks[track] not in do_track: - continue - - snap_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["snap_preparation"], track=tracks[track] - ) - - os.makedirs(snap_directory, exist_ok=True) - - track_fmt = f"{out_parameters['general:input-data:sensor'].lower()}_{asc_dsc[track]}_t{tracks[track]:0>3d}" - - other_parameters = { - "track": tracks[track], - "snap-output-path": snap_directory, - "dry_run": "0", - "track_formatted": track_fmt, - } - - # and the start, end, and mother dates - images = glob.glob(f"{CONFIG_PARAMETERS['SLC_BASE_DIRECTORY']}/{track_fmt}/IW_SLC__1SDV_VVVH/2*") - images = [eval(image.split("/")[-1]) for image in images] - - start_date = eval(out_parameters["general:timeframe:start"].replace("-", "")) - end_date = eval(out_parameters["general:timeframe:end"].replace("-", "")) - mother_date = eval(out_parameters["general:timeframe:mother"].replace("-", "")) - - # then select and format the start, end, and master dates - other_parameters["start_date"] = str(min([image for image in images if image >= start_date])) - other_parameters["start_date"] = ( - f"{other_parameters['start_date'][:4]}-" - f"{other_parameters['start_date'][4:6]}-" - f"{other_parameters['start_date'][6:]}" - ) - other_parameters["end_date"] = str(max([image for image in images if image <= end_date])) - other_parameters["end_date"] = ( - f"{other_parameters['end_date'][:4]}-" - f"{other_parameters['end_date'][4:6]}-" - f"{other_parameters['end_date'][6:]}" - ) - other_parameters["mother_date"] = str(min([image for image in images if image >= mother_date])) - other_parameters["mother_date"] = ( - f"{other_parameters['mother_date'][:4]}-" - f"{other_parameters['mother_date'][4:6]}-" - f"{other_parameters['mother_date'][6:]}" - ) - - # generate the WKT file - write_run_file( - save_path=f"{snap_directory}/aoi.wkt", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/snap/aoi.wkt", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - other_parameters={"wkt_string": convert_shp_to_wkt(shapefile_name)}, - ) - - # generate generate-snap-graphs.sh - write_run_file( - save_path=f"{snap_directory}/generate-snap-graphs.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/snap/generate-snap-graphs.sh", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=[ - "snap:general:AoI-name", - ], - config_parameters=[ - "caroline_work_directory", - "caroline_virtual_environment_directory", - "caroline_install_directory", - "slc_base_directory", - "python3_module", - "gdal_module", - "snap_module", - ], - other_parameters=other_parameters, - ) - - write_directory_contents( - snap_directory, - filename=f'dir_contents{JOB_DEFINITIONS["snap_preparation"]["directory-contents-file-appendix"]}.txt', - ) + if len(bad_zip_files) > 0: + os.system('''echo "\n\n\nZIP File Integrity Check Failed!\n"''') + os.system(f'''echo "Failed on zip file(s) {str(bad_zip_files).replace('"', "'")}. Exiting with code 5...\n"''') + exit(5) # Make the code exit with a non-zero exit code so the next steps won't run -def prepare_snap_run(parameter_file: str, do_track: int | list | None = None) -> None: - """Set up the directories and run files for SNAP run. +def prepare_set_portal_upload_flag(parameter_file: str, do_track: int | list | None = None) -> None: + """Create the indication for a portal upload. Parameters ---------- @@ -2115,16 +2258,13 @@ def prepare_snap_run(parameter_file: str, do_track: int | list | None = None) -> the parameter file """ search_parameters = [ - "snap:general:AoI-name", - "snap:general:directory", "general:tracks:track", - "general:tracks:asc_dsc", - "general:input-data:sensor", + "general:portal:skygeo-customer", + "general:portal:skygeo-viewer", ] out_parameters = read_parameter_file(parameter_file, search_parameters) tracks = out_parameters["general:tracks:track"] - asc_dsc = out_parameters["general:tracks:asc_dsc"] for track in range(len(tracks)): if isinstance(do_track, int): @@ -2134,47 +2274,27 @@ def prepare_snap_run(parameter_file: str, do_track: int | list | None = None) -> if tracks[track] not in do_track: continue - snap_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["snap_run"], track=tracks[track] + depsi_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["depsi_matlab"], track=tracks[track] ) - if "--constraint=rome" in JOB_DEFINITIONS["snap_run"]["sbatch-args"]: - rome_constrained = "1" - else: - rome_constrained = "0" - - os.makedirs(snap_directory, exist_ok=True) - - # generate run-snap-graph.sh - write_run_file( - save_path=f"{snap_directory}/run-snap-graph.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/snap/run-snap-graph.sh", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=["snap:general:AoI-name"], - config_parameters=[ - "caroline_work_directory", - "caroline_virtual_environment_directory", - "python3_module", - "gdal_module", - "snap_module", - ], - other_parameters={ - "track": tracks[track], - "snap-output-path": snap_directory, - "rome-constrained": rome_constrained, - }, + # The parameter file already contains a datestamp so we don't need to redo that + portal_upload_file = ( + f"{CONFIG_PARAMETERS['PORTAL_UPLOAD_FLAG_DIRECTORY']}/" + f"{parameter_file.split('/')[-1].split('.')[0]}_t{tracks[track]:0>3d}_upload.txt" ) - - write_directory_contents( - snap_directory, - filename=f'dir_contents{JOB_DEFINITIONS["snap_run"]["directory-contents-file-appendix"]}.txt', + f = open(portal_upload_file, "w") + f.write( + f"Status: TBD\n" + f"Directory: {depsi_directory}\n" + f"Viewer: {out_parameters['general:portal:skygeo-viewer']}\n" + f"Customer: {out_parameters['general:portal:skygeo-customer']}" ) + f.close() -def prepare_stm_generation(parameter_file: str, do_track: int | list | None = None) -> None: - """Set up the directories and run files for STM generation. +def prepare_snap(parameter_file: str, do_track: int | list | None = None) -> None: + """Set up the directories and run files for SNAP run. Parameters ---------- @@ -2185,12 +2305,11 @@ def prepare_stm_generation(parameter_file: str, do_track: int | list | None = No the parameter file """ search_parameters = [ - "stm_generation:general:AoI-name", - "stm_generation:general:directory", + "snap:general:AoI-name", + "snap:general:directory", "general:tracks:track", "general:tracks:asc_dsc", "general:input-data:sensor", - "general:workflow:filters:coregistration-mode", ] out_parameters = read_parameter_file(parameter_file, search_parameters) @@ -2205,93 +2324,47 @@ def prepare_stm_generation(parameter_file: str, do_track: int | list | None = No if tracks[track] not in do_track: continue - stm_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["stm_generation"], track=tracks[track] + snap_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["snap"], track=tracks[track] ) - # determine if we came from crop_to_zarr or znap_to_zarr - if ( - out_parameters["general:workflow:filters:coregistration-mode"] == "doris" - or out_parameters["general:input-data:sensor"].lower() != "s1" - ): - crop_to_zarr_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["crop_to_zarr"], track=tracks[track] - ) + if "--constraint=rome" in JOB_DEFINITIONS["snap"]["sbatch-args"]: + rome_constrained = "1" else: - crop_to_zarr_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["znap_to_zarr"], track=tracks[track] - ) - - os.makedirs(stm_directory, exist_ok=True) - - # generate stm-generation.py - stm_output_name = stm_directory.split("/")[-1] - crop_to_zarr_output_name = crop_to_zarr_directory.split("/")[-1] + rome_constrained = "0" - write_run_file( - save_path=f"{stm_directory}/generate-stm.py", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/stm-generation/generate-stm.py", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=[ - "stm_generation:stm_generation-settings:ps-selection:mode", - "stm_generation:stm_generation-settings:ps-selection:initialization-mode-settings:start-date", - "stm_generation:stm_generation-settings:ps-selection:initialization-mode-settings:initialization-length", - "stm_generation:stm_generation-settings:incremental-statistics:increment-mode", - "stm_generation:stm_generation-settings:incremental-statistics:recalibration-jump-size", - "stm_generation:stm_generation-settings:ps-selection:method", - "stm_generation:stm_generation-settings:ps-selection:threshold", - "stm_generation:stm_generation-settings:outlier-detection:do-outlier-detection", - "stm_generation:stm_generation-settings:outlier-detection:window-size", - "stm_generation:stm_generation-settings:outlier-detection:db-mode", - "stm_generation:stm_generation-settings:outlier-detection:n-sigma", - "stm_generation:stm_generation-settings:partitioning:do-partitioning", - "stm_generation:stm_generation-settings:partitioning:search-method", - "stm_generation:stm_generation-settings:partitioning:cost-function", - "stm_generation:stm_generation-settings:partitioning:db-mode", - "stm_generation:stm_generation-settings:partitioning:min-partition-length", - "stm_generation:stm_generation-settings:single-differences:mother", - "stm_generation:stm_generation-settings:extra-projection", - "stm_generation:stm_generation-settings:partitioning:undifferenced-output-layers", - "stm_generation:stm_generation-settings:partitioning:single-difference-output-layers", - ], - other_parameters={ - "crop_to_zarr_directory": crop_to_zarr_directory, - "crop_to_zarr_output_name": crop_to_zarr_output_name, - "stm_output_directory": stm_directory, - "stm_output_name": stm_output_name, - }, - ) + os.makedirs(snap_directory, exist_ok=True) - # generate stm-generation.sh + # generate run-snap-graph.sh write_run_file( - save_path=f"{stm_directory}/generate-stm.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/stm-generation/generate-stm.sh", + save_path=f"{snap_directory}/run-snap-graph.sh", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/snap/run-snap-graph.sh", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, - parameter_file_parameters=[ - "stm_generation:general:AoI-name", - "crop_to_zarr:general:crop_to_zarr-code-directory", - ], + parameter_file_parameters=["snap:general:AoI-name"], config_parameters=[ "caroline_work_directory", "caroline_virtual_environment_directory", "python3_module", "gdal_module", + "snap_module", ], - other_parameters={"track": tracks[track]}, + other_parameters={ + "track": tracks[track], + "snap-output-path": snap_directory, + "rome-constrained": rome_constrained, + }, ) write_directory_contents( - stm_directory, - filename=f'dir_contents{JOB_DEFINITIONS["stm_generation"]["directory-contents-file-appendix"]}.txt', + snap_directory, + filename=f'dir_contents{JOB_DEFINITIONS["snap"]["directory-contents-file-appendix"]}.txt', ) -def prepare_tarball(parameter_file: str, do_track: int | list | None = None) -> None: - """Create the tarball after DePSI-post. +def prepare_snap_fix_permissions(parameter_file: str, do_track: int | list | None = None) -> None: + """Change all the permissions for the SNAP output to 775. Parameters ---------- @@ -2301,10 +2374,13 @@ def prepare_tarball(parameter_file: str, do_track: int | list | None = None) -> Track number, or list of track numbers, of the track(s) to prepare. `None` (default) prepares all tracks in the parameter file """ - search_parameters = ["track"] + search_parameters = [ + "general:tracks:track", + "general:tracks:asc_dsc", + ] out_parameters = read_parameter_file(parameter_file, search_parameters) - tracks = out_parameters["track"] + tracks = out_parameters["general:tracks:track"] for track in range(len(tracks)): if isinstance(do_track, int): @@ -2314,19 +2390,17 @@ def prepare_tarball(parameter_file: str, do_track: int | list | None = None) -> if tracks[track] not in do_track: continue - depsi_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["depsi"], track=tracks[track] + snap_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["snap"], track=tracks[track] ) - project_id = depsi_directory.split("/")[-2].split("-")[0] - os.system( - f"cd {depsi_directory}; " - f"bash {CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/scripts/create_post_project_tar.sh {project_id}" - ) + znaps = glob.glob(f"{snap_directory}/*-coreg.znap") + for zf in znaps: + os.system(f"chmod 775 {zf}; chmod 775 {zf}/*; chmod 775 {zf}/*/*; chmod 775 {zf}/*/*/*") -def prepare_znap_to_raw(parameter_file: str, do_track: int | list | None = None) -> None: - """Set up the directories and run files for znap_to_raw. +def prepare_snap_preparation(parameter_file: str, do_track: int | list | None = None) -> None: + """Set up the directories and run files for SNAP preparation. Parameters ---------- @@ -2337,14 +2411,26 @@ def prepare_znap_to_raw(parameter_file: str, do_track: int | list | None = None) the parameter file """ search_parameters = [ + "snap:general:AoI-name", + "snap:general:directory", "general:tracks:track", "general:tracks:asc_dsc", "general:input-data:sensor", + "general:shape-file:aoi-name", + "general:shape-file:directory", + "general:timeframe:start", + "general:timeframe:end", + "general:timeframe:mother", ] out_parameters = read_parameter_file(parameter_file, search_parameters) tracks = out_parameters["general:tracks:track"] asc_dsc = out_parameters["general:tracks:asc_dsc"] + + shapefile_name = ( + f"{out_parameters['general:shape-file:directory']}/{out_parameters['general:shape-file:aoi-name']}_shape.shp" + ) + for track in range(len(tracks)): if isinstance(do_track, int): if tracks[track] != do_track: @@ -2353,139 +2439,84 @@ def prepare_znap_to_raw(parameter_file: str, do_track: int | list | None = None) if tracks[track] not in do_track: continue - znap_to_raw_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["znap_to_raw"], track=tracks[track] - ) - - coregistration_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["snap_run"], track=tracks[track] - ) - - os.makedirs(znap_to_raw_directory, exist_ok=True) - - # generate znap-to-raw.py - write_run_file( - save_path=f"{znap_to_raw_directory}/znap-to-raw.py", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/znap-to-raw/znap-to-raw.py", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=[ - "general:shape-file:aoi-name", - "general:shape-file:directory", - ], - other_parameters={ - "snap-output-path": coregistration_directory, - "raw-output-path": znap_to_raw_directory, - }, - ) - - # generate znap-to-raw.sh - write_run_file( - save_path=f"{znap_to_raw_directory}/znap-to-raw.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/znap-to-raw/znap-to-raw.sh", - asc_dsc=asc_dsc[track], - track=tracks[track], - parameter_file=parameter_file, - parameter_file_parameters=[ - "znap_to_raw:general:AoI-name", - "znap_to_raw:general:znap_to_raw-code-directory", - ], - config_parameters=[ - "caroline_work_directory", - "caroline_virtual_environment_directory", - "python3_module", - "gdal_module", - ], - other_parameters={"track": tracks[track]}, + snap_directory = format_process_folder( + parameter_file=parameter_file, job_description=JOB_DEFINITIONS["snap_preparation"], track=tracks[track] ) - write_directory_contents( - znap_to_raw_directory, - filename=f'dir_contents{JOB_DEFINITIONS["znap_to_raw"]["directory-contents-file-appendix"]}.txt', - ) + os.makedirs(snap_directory, exist_ok=True) + track_fmt = f"{out_parameters['general:input-data:sensor'].lower()}_{asc_dsc[track]}_t{tracks[track]:0>3d}" -def prepare_znap_to_zarr(parameter_file: str, do_track: int | list | None = None) -> None: - """Set up the directories and run files for znap_to_zarr. + other_parameters = { + "track": tracks[track], + "snap-output-path": snap_directory, + "dry_run": "0", + "track_formatted": track_fmt, + } - Parameters - ---------- - parameter_file: str - Absolute path to the parameter file. - do_track: int | list | None, optional - Track number, or list of track numbers, of the track(s) to prepare. `None` (default) prepares all tracks in - the parameter file - """ - search_parameters = [ - "general:tracks:track", - "general:tracks:asc_dsc", - "general:input-data:sensor", - ] - out_parameters = read_parameter_file(parameter_file, search_parameters) + # and the start, end, and mother dates + images = glob.glob(f"{CONFIG_PARAMETERS['SLC_BASE_DIRECTORY']}/{track_fmt}/IW_SLC__1SDV_VVVH/2*") + images = [eval(image.split("/")[-1]) for image in images] - tracks = out_parameters["general:tracks:track"] - asc_dsc = out_parameters["general:tracks:asc_dsc"] - for track in range(len(tracks)): - if isinstance(do_track, int): - if tracks[track] != do_track: - continue - elif isinstance(do_track, list): - if tracks[track] not in do_track: - continue + start_date = eval(out_parameters["general:timeframe:start"].replace("-", "")) + end_date = eval(out_parameters["general:timeframe:end"].replace("-", "")) + mother_date = eval(out_parameters["general:timeframe:mother"].replace("-", "")) - znap_to_zarr_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["znap_to_zarr"], track=tracks[track] + # then select and format the start, end, and master dates + other_parameters["start_date"] = str(min([image for image in images if image >= start_date])) + other_parameters["start_date"] = ( + f"{other_parameters['start_date'][:4]}-" + f"{other_parameters['start_date'][4:6]}-" + f"{other_parameters['start_date'][6:]}" ) - - coregistration_directory = format_process_folder( - parameter_file=parameter_file, job_description=JOB_DEFINITIONS["snap_run"], track=tracks[track] + other_parameters["end_date"] = str(max([image for image in images if image <= end_date])) + other_parameters["end_date"] = ( + f"{other_parameters['end_date'][:4]}-" + f"{other_parameters['end_date'][4:6]}-" + f"{other_parameters['end_date'][6:]}" + ) + other_parameters["mother_date"] = str(min([image for image in images if image >= mother_date])) + other_parameters["mother_date"] = ( + f"{other_parameters['mother_date'][:4]}-" + f"{other_parameters['mother_date'][4:6]}-" + f"{other_parameters['mother_date'][6:]}" ) - os.makedirs(znap_to_zarr_directory, exist_ok=True) - - # generate crop-to-zarr.py - znap_to_zarr_output_name = znap_to_zarr_directory.split("/")[-1] - + # generate the WKT file write_run_file( - save_path=f"{znap_to_zarr_directory}/znap-to-zarr.py", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/znap-to-zarr/znap-to-zarr.py", + save_path=f"{snap_directory}/aoi.wkt", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/snap/aoi.wkt", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, - parameter_file_parameters=[ - "general:shape-file:aoi-name", - "general:shape-file:directory", - ], - other_parameters={ - "snap-output-path": coregistration_directory, - "znap_to_zarr_output_filename": znap_to_zarr_output_name, - }, + other_parameters={"wkt_string": convert_shp_to_wkt(shapefile_name)}, ) - # generate crop-to-zarr.sh + # generate generate-snap-graphs.sh write_run_file( - save_path=f"{znap_to_zarr_directory}/znap-to-zarr.sh", - template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/znap-to-zarr/znap-to-zarr.sh", + save_path=f"{snap_directory}/generate-snap-graphs.sh", + template_path=f"{CONFIG_PARAMETERS['CAROLINE_INSTALL_DIRECTORY']}/templates/snap/generate-snap-graphs.sh", asc_dsc=asc_dsc[track], track=tracks[track], parameter_file=parameter_file, parameter_file_parameters=[ - "znap_to_zarr:general:AoI-name", - "znap_to_zarr:general:znap_to_zarr-code-directory", + "snap:general:AoI-name", ], config_parameters=[ "caroline_work_directory", "caroline_virtual_environment_directory", + "caroline_install_directory", + "slc_base_directory", "python3_module", "gdal_module", + "snap_module", ], - other_parameters={"track": tracks[track]}, + other_parameters=other_parameters, ) write_directory_contents( - znap_to_zarr_directory, - filename=f'dir_contents{JOB_DEFINITIONS["znap_to_zarr"]["directory-contents-file-appendix"]}.txt', + snap_directory, + filename=f'dir_contents{JOB_DEFINITIONS["snap_preparation"]["directory-contents-file-appendix"]}.txt', ) diff --git a/caroline/utils.py b/caroline/utils.py index f3ae29f4..51ca4313 100644 --- a/caroline/utils.py +++ b/caroline/utils.py @@ -81,7 +81,7 @@ def remove_incomplete_sentinel1_images(parameter_file: str) -> None: doris_job_definition = get_config( f'{CONFIG_PARAMETERS["CAROLINE_INSTALL_DIRECTORY"]}/config/job-definitions.yaml', flatten=False - )["jobs"]["doris"] + )["jobs"]["doris_v5"] for track in range(len(tracks)): base_folder = format_process_folder( @@ -290,7 +290,7 @@ def _generate_email(parameter_file: str) -> str: else: directory = CONFIG_PARAMETERS["SLURM_OUTPUT_DIRECTORY"] if check["successful_finish"]: - if job == "portal_upload": + if job == "set_portal_upload_flag": status_checks += ( "NOTE: it can take a few hours for the results to show up in the portal.\n" + "The DePSI-post results can be accessed at " @@ -666,6 +666,9 @@ def identify_s1_orbits_in_aoi(shp_filename: str) -> tuple[list[str], dict]: except (asf.exceptions.ASFSearch5xxError, asf.exceptions.ASFSearchError, TimeoutError): counter += 1 os.system(f'''echo "ASF encountered an internal error. Retrying... (#{counter})"''') + if counter > 5: + os.system('''echo "ASF encountered an internal error more than 5 times. Continuing without results..."''') + return [], {} # return empty lists orbits = [ f"s1_{slc.properties['flightDirection'].lower().replace('e', '')[:3]}_t{slc.properties['pathNumber']:0>3d}" diff --git a/config/job-definitions.yaml b/config/job-definitions.yaml index 8498a9ac..9fcc1f9f 100644 --- a/config/job-definitions.yaml +++ b/config/job-definitions.yaml @@ -51,9 +51,9 @@ jobs: filters: coregistration-mode: "snap" - snap_run: + snap: requirement: "snap_preparation" - two-letter-id: "SR" + two-letter-id: "SN" parameter-file-step-key: "general:steps:do-coregistration" partition: "normal" sbatch-args: "--qos=long --ntasks=1 --cpus-per-task=12 --constraint=rome" @@ -69,7 +69,7 @@ jobs: bash-file-slurm-cluster: job-array: run-as-array: True - njobs-in-array-function: "njobs_snap_run" + njobs-in-array-function: "njobs_snap" filters: general: input-data: @@ -78,9 +78,9 @@ jobs: filters: coregistration-mode: "snap" - snap_permissions: - requirement: "snap_run" - two-letter-id: "SE" + snap_fix_permissions: + requirement: "snap" + two-letter-id: "SF" parameter-file-step-key: "general:steps:do-coregistration" partition: "short" sbatch-args: "--ntasks=1 --cpus-per-task=1" @@ -100,11 +100,11 @@ jobs: filters: coregistration-mode: "snap" - doris: + doris_v5: requirement: "s1_download" two-letter-id: "D5" parameter-file-step-key: "general:steps:do-coregistration" - partition: "doris:general:partition" + partition: "doris_v5:general:partition" sbatch-args: "--qos=long --ntasks=1 --cpus-per-task=8 --mem-per-cpu=8000" directory-contents-file-appendix: "" email: @@ -112,7 +112,7 @@ jobs: status-file-search-key: "profile_log*" bash-file: bash-file-name: "doris_stack.sh" - bash-file-base-directory: "doris" # expects doris:general:AoI-name and doris:general:directory + bash-file-base-directory: "doris_v5" # expects doris_v5:general:AoI-name and doris_v5:general:directory bash-file-directory-appendix: "" bash-file-directory-is-reusable: True bash-file-slurm-cluster: @@ -127,19 +127,19 @@ jobs: filters: coregistration-mode: "doris" - deinsar: + doris_v4: requirement: two-letter-id: "D4" parameter-file-step-key: "general:steps:do-coregistration" - partition: "deinsar:general:partition" + partition: "doris_v4:general:partition" sbatch-args: "--qos=long --ntasks=1 --cpus-per-task=8 --mem-per-cpu=8000" directory-contents-file-appendix: "" email: include-in-email: True status-file-search-key: bash-file: - bash-file-name: "run_deinsar.sh" - bash-file-base-directory: "deinsar" # expects deinsar:general:AoI-name and deinsar:general:directory + bash-file-name: "run_doris_v4.sh" + bash-file-base-directory: "doris_v4" # expects doris_v4:general:AoI-name and doris_v4:general:directory bash-file-directory-appendix: "" bash-file-directory-is-reusable: True bash-file-slurm-cluster: @@ -151,8 +151,8 @@ jobs: input-data: sensor: ["ERS", "ENV", "TSX", "TDX", "PAZ", "RSAT2", "Cosmo", "ALOS2"] - doris_cleanup: - requirement: "doris" + doris_v5_cleanup: + requirement: "doris_v5" two-letter-id: "DC" parameter-file-step-key: "general:steps:do-coregistration" partition: "short" @@ -163,7 +163,7 @@ jobs: status-file-search-key: bash-file: bash-file-name: "cleanup.sh" - bash-file-base-directory: "doris" + bash-file-base-directory: "doris_v5" bash-file-directory-appendix: "" bash-file-directory-is-reusable: True bash-file-slurm-cluster: @@ -178,19 +178,19 @@ jobs: filters: coregistration-mode: "doris" - crop_to_raw: - requirement: ["doris", "deinsar"] - two-letter-id: "CR" - parameter-file-step-key: "general:steps:do-crop_to_raw" - partition: "crop_to_raw:general:partition" + reduce_slc_matlab: + requirement: ["doris_v5", "doris_v4"] + two-letter-id: "RM" + parameter-file-step-key: "general:steps:do-reduce_slc_matlab" + partition: "reduce_slc_matlab:general:partition" sbatch-args: "--qos=long --ntasks=1 --cpus-per-task=8" directory-contents-file-appendix: "" email: include-in-email: True status-file-search-key: bash-file: - bash-file-name: "crop-to-raw.sh" - bash-file-base-directory: "crop_to_raw" + bash-file-name: "reduce-slc-matlab.sh" + bash-file-base-directory: "reduce_slc_matlab" bash-file-directory-appendix: "" bash-file-directory-is-reusable: True bash-file-slurm-cluster: @@ -203,19 +203,19 @@ jobs: filters: coregistration-mode: "doris" - znap_to_raw: - requirement: "snap_permissions" - two-letter-id: "ZR" - parameter-file-step-key: "general:steps:do-znap_to_raw" - partition: "znap_to_raw:general:partition" + merge_to_stack_matlab: + requirement: "snap_fix_permissions" + two-letter-id: "MM" + parameter-file-step-key: "general:steps:do-merge_to_stack_matlab" + partition: "merge_to_stack_matlab:general:partition" sbatch-args: "--qos=long --ntasks=1 --cpus-per-task=8 --nodes=1" directory-contents-file-appendix: "" email: include-in-email: True status-file-search-key: bash-file: - bash-file-name: "znap-to-raw.sh" - bash-file-base-directory: "znap_to_raw" + bash-file-name: "merge-to-stack-matlab.sh" + bash-file-base-directory: "merge_to_stack_matlab" bash-file-directory-appendix: "" bash-file-directory-is-reusable: True bash-file-slurm-cluster: @@ -230,19 +230,19 @@ jobs: filters: coregistration-mode: "snap" - znap_to_zarr: - requirement: "snap_permissions" - two-letter-id: "ZZ" - parameter-file-step-key: "general:steps:do-znap_to_zarr" - partition: "znap_to_zarr:general:partition" + merge_to_stack_python: + requirement: "snap_fix_permissions" + two-letter-id: "MP" + parameter-file-step-key: "general:steps:do-merge_to_stack_python" + partition: "merge_to_stack_python:general:partition" sbatch-args: "--qos=long --ntasks=1 --cpus-per-task=8 --nodes=1" directory-contents-file-appendix: "" email: include-in-email: True status-file-search-key: bash-file: - bash-file-name: "znap-to-zarr.sh" - bash-file-base-directory: "znap_to_zarr" + bash-file-name: "merge-to-stack-python.sh" + bash-file-base-directory: "merge_to_stack_python" bash-file-directory-appendix: "" bash-file-directory-is-reusable: True bash-file-slurm-cluster: @@ -257,19 +257,19 @@ jobs: filters: coregistration-mode: "snap" - crop_to_zarr: - requirement: ["doris", "deinsar"] - two-letter-id: "CZ" - parameter-file-step-key: "general:steps:do-crop_to_zarr" - partition: "crop_to_zarr:general:partition" + reduce_slc_python: + requirement: ["doris_v5", "doris_v4"] + two-letter-id: "RP" + parameter-file-step-key: "general:steps:do-reduce_slc_python" + partition: "reduce_slc_python:general:partition" sbatch-args: "--qos=long --ntasks=1 --cpus-per-task=8 --nodes=1" directory-contents-file-appendix: "" email: include-in-email: True status-file-search-key: bash-file: - bash-file-name: "crop-to-zarr.sh" - bash-file-base-directory: "crop_to_zarr" + bash-file-name: "reduce-slc-python.sh" + bash-file-base-directory: "reduce_slc_python" bash-file-directory-appendix: "" bash-file-directory-is-reusable: True bash-file-slurm-cluster: @@ -284,19 +284,19 @@ jobs: filters: coregistration-mode: "doris" - stm_generation: - requirement: ["crop_to_zarr", "znap_to_zarr"] - two-letter-id: "SG" - parameter-file-step-key: "general:steps:do-stm_generation" - partition: "stm_generation:general:partition" + generate_partitioned_stm: + requirement: ["reduce_slc_python", "merge_to_stack_python"] + two-letter-id: "GS" + parameter-file-step-key: "general:steps:do-generate_partitioned_stm" + partition: "generate_partitioned_stm:general:partition" sbatch-args: "--qos=long --ntasks=1 --cpus-per-task=8 --nodes=1" directory-contents-file-appendix: "" email: include-in-email: True status-file-search-key: bash-file: - bash-file-name: "generate-stm.sh" - bash-file-base-directory: "stm_generation" + bash-file-name: "generate-partitioned-stm.sh" + bash-file-base-directory: "generate_partitioned_stm" bash-file-directory-appendix: "" bash-file-directory-is-reusable: True bash-file-slurm-cluster: @@ -307,19 +307,19 @@ jobs: njobs-in-array-function: filters: - depsi: - requirement: ["crop_to_raw", "znap_to_raw"] - two-letter-id: "DE" - parameter-file-step-key: "general:steps:do-depsi" - partition: "depsi:general:partition" + depsi_matlab: + requirement: ["reduce_slc_matlab", "merge_to_stack_matlab"] + two-letter-id: "DM" + parameter-file-step-key: "general:steps:do-depsi_matlab" + partition: "depsi_matlab:general:partition" sbatch-args: "--qos=long --ntasks=1 --cpus-per-task=8 --mem-per-cpu=8000" directory-contents-file-appendix: "" email: include-in-email: True status-file-search-key: "*resfile.txt" bash-file: - bash-file-name: "depsi.sh" - bash-file-base-directory: "depsi" + bash-file-name: "depsi_matlab.sh" + bash-file-base-directory: "depsi_matlab" bash-file-directory-appendix: "/psi" bash-file-directory-is-reusable: False bash-file-slurm-cluster: @@ -328,9 +328,9 @@ jobs: njobs-in-array-function: filters: - mrm: - requirement: "depsi" - two-letter-id: "MR" + create_mrm: + requirement: "depsi_matlab" + two-letter-id: "CM" parameter-file-step-key: "general:steps:do-depsi_post" partition: "normal" sbatch-args: "--qos=long --ntasks=1 --cpus-per-task=4 --mem-per-cpu=8000" @@ -339,8 +339,8 @@ jobs: include-in-email: False status-file-search-key: bash-file: - bash-file-name: "read_mrm.sh" - bash-file-base-directory: "depsi" + bash-file-name: "create_mrm.sh" + bash-file-base-directory: "depsi_matlab" bash-file-directory-appendix: "/psi" bash-file-directory-is-reusable: False bash-file-slurm-cluster: @@ -350,7 +350,7 @@ jobs: filters: depsi_post: - requirement: "mrm" + requirement: "create_mrm" two-letter-id: "DP" parameter-file-step-key: "general:steps:do-depsi_post" partition: "depsi_post:general:partition" @@ -361,7 +361,7 @@ jobs: status-file-search-key: bash-file: bash-file-name: "depsi_post.sh" - bash-file-base-directory: "depsi" + bash-file-base-directory: "depsi_matlab" bash-file-directory-appendix: "/psi" bash-file-directory-is-reusable: False bash-file-slurm-cluster: @@ -370,9 +370,9 @@ jobs: njobs-in-array-function: filters: - tarball: + create_tarball: requirement: "depsi_post" - two-letter-id: "TB" + two-letter-id: "CT" parameter-file-step-key: "general:steps:do-depsi_post" partition: "short" sbatch-args: "--ntasks=1 --cpus-per-task=1" @@ -390,15 +390,15 @@ jobs: filters: depsi_post-output: "tarball" - portal_upload: + set_portal_upload_flag: requirement: "depsi_post" two-letter-id: "PU" - parameter-file-step-key: "general:steps:do-depsi_post" + parameter-file-step-key: "general:steps:do-set_portal_upload_flag" partition: "short" sbatch-args: "--ntasks=1 --cpus-per-task=1" directory-contents-file-appendix: "_portalupload" email: - include-in-email: True + include-in-email: True # this one also generates the link to the portal into the email status-file-search-key: bash-file: job-array: diff --git a/config/parameter-files/default-job-param-file-crop_to_zarr.yaml b/config/parameter-files/default-job-param-file-crop_to_zarr.yaml deleted file mode 100644 index fdd06b71..00000000 --- a/config/parameter-files/default-job-param-file-crop_to_zarr.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# the default settings for crop-to-zarr, running in the crop_to_zarr job - -crop_to_zarr: - general: - AoI-name: "**AoI_name**" - directory: '/project/caroline/Share/stacks_zarr' - partition: 'normal' - crop_to_zarr-code-directory: '**CAROLINE_PLUGINS_DIRECTORY**/DePSI_group' diff --git a/config/parameter-files/default-job-param-file-depsi.yaml b/config/parameter-files/default-job-param-file-depsi_matlab.yaml similarity index 95% rename from config/parameter-files/default-job-param-file-depsi.yaml rename to config/parameter-files/default-job-param-file-depsi_matlab.yaml index d457362d..da2ca31d 100644 --- a/config/parameter-files/default-job-param-file-depsi.yaml +++ b/config/parameter-files/default-job-param-file-depsi_matlab.yaml @@ -1,15 +1,15 @@ -# the default settings for DePSI, running in the depsi job +# the default settings for DePSI, running in the depsi_matlab job -depsi: +depsi_matlab: general: AoI-name: "**AoI_name**" directory: '/project/caroline/Share/projects/**AoI_name**/depsi' partition: 'normal' - depsi-code-directory: '**CAROLINE_PLUGINS_DIRECTORY**/depsi_v2.2.1.1' + depsi_matlab-code-directory: '**CAROLINE_PLUGINS_DIRECTORY**/depsi_v2.2.1.1' rdnaptrans-directory: '**CAROLINE_PLUGINS_DIRECTORY**/rdnaptrans' geocoding-directory: '**CAROLINE_PLUGINS_DIRECTORY**/geocoding_v0.9' - depsi-settings: + depsi_matlab-settings: general: max-mem-buffer: 50e7 visible-plots: 'n' diff --git a/config/parameter-files/default-job-param-file-deinsar.yaml b/config/parameter-files/default-job-param-file-doris_v4.yaml similarity index 93% rename from config/parameter-files/default-job-param-file-deinsar.yaml rename to config/parameter-files/default-job-param-file-doris_v4.yaml index 86ee9fbf..b39f16bc 100644 --- a/config/parameter-files/default-job-param-file-deinsar.yaml +++ b/config/parameter-files/default-job-param-file-doris_v4.yaml @@ -1,18 +1,18 @@ # the default settings for Doris v4, running in the DeInSAR job -deinsar: +doris_v4: general: AoI-name: "**AoI_name**" directory: '/project/caroline/Share/stacks' partition: 'normal' deinsar-code-directory: '**CAROLINE_PLUGINS_DIRECTORY**/deinsar_v0.3.4' - doris-v4-code-directory: '/project/caroline/Software/doris/doris_v4.13.1_cosmo2_alos2fix_tsxfix' + doris_v4-code-directory: '/project/caroline/Software/doris/doris_v4.13.1_cosmo2_alos2fix_tsxfix' input: data-directories: # this one needs one line for every track you wish to run, with as key the track and as value the directory tsx_asc_t116: '/project/caroline/Data/radar_data/eurasia/netherlands/tsx/nl_amsterdam_tsx_asc_t116_T171816_171824_007_hh/data_backup' - deinsar-settings: + doris_v4-settings: do-orbit: 1 # only for ENV, ERS, RSAT2, will turn off for other sensors automatically do-crop: 1 do-tsx-deramp: 1 # only for TSX, will turn off for other sensors automatically diff --git a/config/parameter-files/default-job-param-file-doris.yaml b/config/parameter-files/default-job-param-file-doris_v5.yaml similarity index 96% rename from config/parameter-files/default-job-param-file-doris.yaml rename to config/parameter-files/default-job-param-file-doris_v5.yaml index 77153f19..0c043ad4 100644 --- a/config/parameter-files/default-job-param-file-doris.yaml +++ b/config/parameter-files/default-job-param-file-doris_v5.yaml @@ -1,13 +1,13 @@ # the default settings for Doris v5, running in the doris job -doris: +doris_v5: general: AoI-name: "**AoI_name**" directory: '/project/caroline/Share/stacks' partition: 'normal' code-directory: '/project/caroline/Software/doris/doris_v5_current' - doris-settings: + doris_v5-settings: do-coarse-orbits: 1 do-deramp: 1 do-reramp: 1 diff --git a/config/parameter-files/default-job-param-file-stm_generation.yaml b/config/parameter-files/default-job-param-file-generate_partitioned_stm.yaml similarity index 63% rename from config/parameter-files/default-job-param-file-stm_generation.yaml rename to config/parameter-files/default-job-param-file-generate_partitioned_stm.yaml index cfc5eda0..554afd7c 100644 --- a/config/parameter-files/default-job-param-file-stm_generation.yaml +++ b/config/parameter-files/default-job-param-file-generate_partitioned_stm.yaml @@ -1,26 +1,26 @@ -# the default settings for the STM generation, running in the stm_generation job +# the default settings for the partitioned STM generation, running in the generate_partitioned_stm job -stm_generation: +generate_partitioned_stm: general: AoI-name: "**AoI_name**" directory: '/project/caroline/Share/stms_zarr' partition: 'normal' - stm_generation-code-directory: '**CAROLINE_PLUGINS_DIRECTORY**/DePSI_group' + depsi_group-code-directory: '**CAROLINE_PLUGINS_DIRECTORY**/DePSI_group' - stm_generation-settings: + generate_partitioned_stm-settings: ps-selection: mode: 'full' # initialization / full - initialization-mode-settings: # ignored if mode is full + init-settings: # settings for initialization mode, ignored if mode is full start-date: '2013-03-01' - initialization-length: 50 + init-length: 50 method: "nmad" # nmad / nad threshold: 0.35 incremental-statistics: increment-mode: "incremental" # incremental (every epoch) / recalibration (every jump-size epochs) - recalibration-jump-size: 10 + recal-jump-size: 10 single-differences: mother: "auto" # YYYY-MM-DD or auto, latter uses already existing mother @@ -33,8 +33,8 @@ stm_generation: cost-function: 'l2' db-mode: 0 min-partition-length: 27 - undifferenced-output-layers: ['nmad', 'nad', 'quality_nmad_2sigma'] - single-difference-output-layers: ['mad', 'amplitude_sigma', 'amplitude_mean', 'amplitude_median'] + undifferenced-output-lyrs: ['nmad', 'nad', 'quality_nmad_2sigma'] + single-difference-output-lyrs: ['mad', 'amplitude_sigma', 'amplitude_mean', 'amplitude_median'] outlier-detection: do-outlier-detection: 1 diff --git a/config/parameter-files/default-job-param-file-merge_to_stack_matlab.yaml b/config/parameter-files/default-job-param-file-merge_to_stack_matlab.yaml new file mode 100644 index 00000000..83b3bfc5 --- /dev/null +++ b/config/parameter-files/default-job-param-file-merge_to_stack_matlab.yaml @@ -0,0 +1,10 @@ +# the default settings for merge_to_stack_matlab, running in the merge_to_stack_matlab job + +merge_to_stack_matlab: + general: + AoI-name: "**AoI_name**" + directory: '/project/caroline/Share/crops' + partition: 'normal' + depsi_group-code-directory: '**CAROLINE_PLUGINS_DIRECTORY**/DePSI_group' + + diff --git a/config/parameter-files/default-job-param-file-merge_to_stack_python.yaml b/config/parameter-files/default-job-param-file-merge_to_stack_python.yaml new file mode 100644 index 00000000..7220864d --- /dev/null +++ b/config/parameter-files/default-job-param-file-merge_to_stack_python.yaml @@ -0,0 +1,8 @@ +# the default settings for merge_to_stack_python, running in the merge_to_stack_python job + +merge_to_stack_python: + general: + AoI-name: "**AoI_name**" + directory: '/project/caroline/Share/stacks_zarr' + partition: 'normal' + depsi_group-code-directory: '**CAROLINE_PLUGINS_DIRECTORY**/DePSI_group' diff --git a/config/parameter-files/default-job-param-file-crop_to_raw.yaml b/config/parameter-files/default-job-param-file-reduce_slc_matlab.yaml similarity index 52% rename from config/parameter-files/default-job-param-file-crop_to_raw.yaml rename to config/parameter-files/default-job-param-file-reduce_slc_matlab.yaml index 02122954..de2cb9ee 100644 --- a/config/parameter-files/default-job-param-file-crop_to_raw.yaml +++ b/config/parameter-files/default-job-param-file-reduce_slc_matlab.yaml @@ -1,6 +1,6 @@ -# the default settings for crop_to_raw, running in the crop_to_raw job +# the default settings for reduce_slc_matlab, running in the reduce_slc_matlab job -crop_to_raw: +reduce_slc_matlab: general: AoI-name: "**AoI_name**" directory: '/project/caroline/Share/crops' diff --git a/config/parameter-files/default-job-param-file-reduce_slc_python.yaml b/config/parameter-files/default-job-param-file-reduce_slc_python.yaml new file mode 100644 index 00000000..67f03e2f --- /dev/null +++ b/config/parameter-files/default-job-param-file-reduce_slc_python.yaml @@ -0,0 +1,8 @@ +# the default settings for reduce_slc_python, running in the reduce_slc_python job + +reduce_slc_python: + general: + AoI-name: "**AoI_name**" + directory: '/project/caroline/Share/stacks_zarr' + partition: 'normal' + depsi_group-code-directory: '**CAROLINE_PLUGINS_DIRECTORY**/DePSI_group' diff --git a/config/parameter-files/default-job-param-file-snap_run.yaml b/config/parameter-files/default-job-param-file-snap.yaml similarity index 100% rename from config/parameter-files/default-job-param-file-snap_run.yaml rename to config/parameter-files/default-job-param-file-snap.yaml diff --git a/config/parameter-files/default-job-param-file-znap_to_raw.yaml b/config/parameter-files/default-job-param-file-znap_to_raw.yaml deleted file mode 100644 index a47b8e51..00000000 --- a/config/parameter-files/default-job-param-file-znap_to_raw.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# the default settings for crop_to_raw, running in the crop_to_raw job - -znap_to_raw: - general: - AoI-name: "**AoI_name**" - directory: '/project/caroline/Share/crops' - partition: 'normal' - znap_to_raw-code-directory: '**CAROLINE_PLUGINS_DIRECTORY**/DePSI_group' - - diff --git a/config/parameter-files/default-job-param-file-znap_to_zarr.yaml b/config/parameter-files/default-job-param-file-znap_to_zarr.yaml deleted file mode 100644 index f9d57261..00000000 --- a/config/parameter-files/default-job-param-file-znap_to_zarr.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# the default settings for crop-to-zarr, running in the crop_to_zarr job - -znap_to_zarr: - general: - AoI-name: "**AoI_name**" - directory: '/project/caroline/Share/stacks_zarr' - partition: 'normal' - znap_to_zarr-code-directory: '**CAROLINE_PLUGINS_DIRECTORY**/DePSI_group' diff --git a/config/parameter-files/default-machine-fields-param-file.yaml b/config/parameter-files/default-machine-fields-param-file.yaml index 353275a2..5cc0ff59 100644 --- a/config/parameter-files/default-machine-fields-param-file.yaml +++ b/config/parameter-files/default-machine-fields-param-file.yaml @@ -9,13 +9,14 @@ general: steps: do-s1_download: 0 do-coregistration: 0 - do-crop_to_raw: 0 - do-znap_to_raw: 0 - do-crop_to_zarr: 0 - do-znap_to_zarr: 0 - do-stm_generation: 0 - do-depsi: 0 + do-reduce_slc_matlab: 0 + do-merge_to_stack_matlab: 0 + do-reduce_slc_python: 0 + do-merge_to_stack_python: 0 + do-generate_partitioned_stm: 0 + do-depsi_matlab: 0 do-depsi_post: 0 + do-set_portal_upload_flag: 0 shape-file: aoi-name: "**AoI_name**" diff --git a/config/parameter-files/example-user-param-file-nl_amsterdam.yaml b/config/parameter-files/example-user-param-file-nl_amsterdam.yaml index f0cba504..af790781 100644 --- a/config/parameter-files/example-user-param-file-nl_amsterdam.yaml +++ b/config/parameter-files/example-user-param-file-nl_amsterdam.yaml @@ -13,8 +13,10 @@ general: workflow: dependency: aoi-name: 'nl_veenweiden' - input-step: 'doris' - output-steps: ["stm_generation", "portal_upload"] + input-step: 'doris_v5' + output-steps: ["generate_partitioned_stm", "set_portal_upload_flag"] + filters: + coregistration-mode: "doris" shape-file: rectangular-shape-file: @@ -26,7 +28,7 @@ general: email: recipients: "s.a.n.vandiepen@tudelft.nl,y.wang-29@tudelft.nl,w.s.brouwer@tudelft.nl,r.f.hanssen@tudelft.nl" -doris: +doris_v5: general: AoI-name: "nl_veenweiden" diff --git a/docs/abbreviations.md b/docs/abbreviations.md index 025acf25..1abce849 100644 --- a/docs/abbreviations.md +++ b/docs/abbreviations.md @@ -12,22 +12,19 @@ and `NNN` the three-letter area of interest abbreviation. - `SD`: Sentinel-1 Download (for one-time full-period downloads, periodic downloads of the last month are managed by [manage-s1-download.sh](../scripts/manage-s1-download.sh)) -#### Coregistration submodule +#### Stack generation submodule -- `D4`: DeInSAR +- `D4`: Doris v4 - `D5`: Doris v5 - `DC`: Doris v5 cleanup -- `SE`: SNAP-permissions +- `MM`: merge-to-stack-matlab +- `MP`: merge-to-stack-python +- `RM`: reduce-slc-matlab +- `RP`: reduce-slc-python +- `SF`: SNAP-fix permissions +- `SN`: SNAP - `SP`: SNAP Preparation -- `SR`: SNAP-run -#### Cropping submodule - -- `CR`: crop-to-raw -- `CZ`: crop-to-zarr -- `RE`: Re-SLC (deprecated) -- `ZR`: znap-to-raw -- `ZZ`: znap-to-zarr ### AAA Job Definition module @@ -41,8 +38,8 @@ None yet, managed by [contextual-data-definitions.yaml](../config/contextual-dat #### PSI-batch submodule -- `DE`: DePSI -- `SG`: STM_generation +- `DM`: DePSI_matlab +- `GS`: generate_partitioned_STM ### Autonomous Analysis module @@ -56,10 +53,11 @@ None yet #### PSI-batch submodule +- `CM`: Create mrm +- `CT`: Create tarball - `DP`: DePSI-post -- `MR`: Read mrm -- `PU`: Portal upload preparation (the actual upload is managed by [manage-portal-upload.sh](../scripts/manage-portal-upload.sh)) -- `TB`: Tarball creation +- `PU`: Set portal upload flag (the actual upload is managed by [manage-portal-upload.sh](../scripts/manage-portal-upload.sh)) + ### Push module diff --git a/docs/architecture.md b/docs/architecture.md index 2d0de42e..bb2c98b3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,7 +1,7 @@ # CAROLINE Architecture ## Current architecture overview -![Architecture_v3.2.0](assets/Caroline_v3.2.0.png) +![Architecture_v4.0.0](assets/Caroline_v4.0.0.png) ## Modules @@ -30,10 +30,8 @@ There are nine modules: ### Autonomous coregistered stack building module - Download submodule - Goal: download radar data from all satellites. -- Coregistration submodule - - Goal: perform the basic interferometric procedure per image pair. This includes orbit corrections, coregistration, resampling, burst merging, interferogram generation, reference phase and DEM (including reference ellipsoid) subtraction, geocoding, and coherence estimation. -- Cropping submodule - - Goal: create reduced SLCs of the AoI, and crop as preparation for the Recursive parameter estimation module. +- Stack generation submodule + - Goal: perform the basic interferometric procedure per image pair, and create a stack of reduced SLCs. This includes orbit corrections, coregistration, resampling, burst merging, interferogram generation, reference phase and DEM (including reference ellipsoid) subtraction, geocoding, and coherence estimation. ### Recursive parameter estimation module - PSI batch submodule diff --git a/docs/assets/Caroline_drawings.pptx b/docs/assets/Caroline_drawings.pptx index 1afb1f5c..34dcdc9d 100644 Binary files a/docs/assets/Caroline_drawings.pptx and b/docs/assets/Caroline_drawings.pptx differ diff --git a/docs/assets/Caroline_v4.0.0.png b/docs/assets/Caroline_v4.0.0.png new file mode 100644 index 00000000..36716659 Binary files /dev/null and b/docs/assets/Caroline_v4.0.0.png differ diff --git a/docs/glossary.md b/docs/glossary.md index 9532e169..4eb5e810 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -4,14 +4,14 @@ This file details the definitions of terms used in the [CAROLINE architecture](# ## CAROLINE Architecture - module: a block in the CAROLINE [architecture](architecture.md). An example is the autonomous stack building module. A module has one or more submodules. -- submodule: a component of a module. An example is coregistration, part of the autonomous stack building module. A submodule has one or more jobs. -- job: a single program that achieves a clearly specified goal, that is individually submitted to the SLURM manager. The coregistration submodule contains three jobs: Doris v5 (Sentinel-1 coregistration), Doris v5 cleanup, and DeInSAR (for coregistration of other sensors). A job consists of exactly one function call to a preparation function, and optionally one bash script to be executed. -- job array: a group of jobs running the same single program that achieves a clearly specified goal, where each individual subjob within the job array uses different parameter settings. The snap_run job is submitted as a job array. See the [SLURM documentation](https://slurm.schedmd.com/job_array.html) for more details on how this works. +- submodule: a component of a module. An example is the stack generation submodule, part of the autonomous stack building module. A submodule has one or more jobs. +- job: a single program that achieves a clearly specified goal, that is individually submitted to the SLURM manager. The stack generation submodule contains 10 jobs: Doris v5, Doris v5 cleanup, Doris v4, SNAP preparation, SNAP, SNAP fix permissions, reduce SLC matlab, reduce SLC python, merge to stack matlab, and merge to stcack python. A job consists of exactly one function call to a preparation function, and optionally one bash script to be executed. +- job array: a group of jobs running the same single program that achieves a clearly specified goal, where each individual subjob within the job array uses different parameter settings. The SNAP job is submitted as a job array. See the [SLURM documentation](https://slurm.schedmd.com/job_array.html) for more details on how this works. - subjob: one of the jobs running in a job array. They differentiate themselves from a job since they also have a task ID (e.g. 12345_1 instead of just 12345) - function: a Python function. - plugin: an external software package that is called by CAROLINE to execute a job. An example is the Doris v5.0.4 plugin, used in the job Doris v5 in the coregistration submodule. - patch: an amendment to a plugin, where the original plugin code does not function as intended for CAROLINE. All patches are located in the `patches` directory, using the exact same folder structure as will be generated in the directory read from the `CAROLINE_PLUGINS_DIRECTORY` setting. -- workflow: the string of consecutive jobs required to reach a specific outcome. E.g., for a psi_batch portal layer starting from a coregistered stack, the workflow is `crop_to_raw` > `DePSI` > `mrm` > `DePSI_post` > `portal upload` +- workflow: the string of consecutive jobs required to reach a specific outcome. E.g., for a psi_batch portal layer starting from a Doris v5-reduced SLC stack, the workflow is `reduce_slc_matlab` > `DePSI` > `mrm` > `DePSI_post` > `portal upload` - status file: a file detailing the status, progress, and errors of a job, produced by the job itself (so not the command line output). @@ -34,7 +34,7 @@ All jobs run on a single AoI on a single track. The following specifications wil * AoI in `.shp` format * output: * Original SLCs -- doris: this job uses Doris v5 to perform the basic interferometric Sentinel-1 procedure per image pair. This includes orbit corrections, coregistration, resampling, burst merging, interferogram generation, reference phase and DEM (including reference ellipsoid) subtraction, geocoding, and coherence estimation. +- doris_v5: this job uses Doris v5 to perform the basic interferometric Sentinel-1 procedure per image pair. This includes orbit corrections, coregistration, resampling, burst merging, interferogram generation, reference phase and DEM (including reference ellipsoid) subtraction, geocoding, and coherence estimation. * input: * Original SLCs * AoI in `.shp` format @@ -73,7 +73,7 @@ All jobs run on a single AoI on a single track. The following specifications wil * output: * XML graphs to be processed by SNAP called `PROCESSID----graph.xml`, one for each mother/daughter combination to be processed (they will be processed by individual subjobs in `snap_run`, hence the subjob ID identifier). * A datestamped archive folder containing all previously present XML graphs in the processing folder -- snap_run: this job uses SNAP to perform the basic interferometric Sentinel-1 procedure per image pair as a job array (one subjob per mother/daughter combination). This includes orbit corrections, coregistration, resampling, burst merging, interferogram generation, reference phase and DEM (including reference ellipsoid) subtraction, geocoding, and coherence estimation. +- snap: this job uses SNAP to perform the basic interferometric Sentinel-1 procedure per image pair as a job array (one subjob per mother/daughter combination). This includes orbit corrections, coregistration, resampling, burst merging, interferogram generation, reference phase and DEM (including reference ellipsoid) subtraction, geocoding, and coherence estimation. * input: * XML graphs, one for each subjob * output: @@ -93,12 +93,12 @@ All jobs run on a single AoI on a single track. The following specifications wil * product metadata * ground control points (empty) * pins (empty) -- snap_permissions: this job sets the correct permissions on the `.znap` archives. +- snap_fix_permissions: this job sets the correct permissions on the `.znap` archives. * input: - * The `.znap` archives outputted by `snap_run` + * The `.znap` archives outputted by `snap` * output: * `.znap`-archives, one per acquisition, with permissions `775` -- crop_to_raw: this job crops the output complex interferograms, height-to-phase screens, geocoded coordinates and mother SLC of `deinsar` or `doris` to a provided AoI. The crop is taken to be the smallest rectangle in line/pixel coordinates that completely encloses the AoI. It then creates the (now resampled and reference DEM-subtracted, i.e., _reduced_) SLCs from the cropped complex interferograms and the mother SLC. +- reduce_slc_matlab: this job crops the output complex interferograms, height-to-phase screens, geocoded coordinates and mother SLC of `doris_v4` or `doris_v5` to a provided AoI. The crop is taken to be the smallest rectangle in line/pixel coordinates that completely encloses the AoI. It then creates the (now resampled and reference DEM-subtracted, i.e., _reduced_) SLCs from the cropped complex interferograms and the mother SLC. * input: * Complex interferograms with reference DEM subtracted (`cint_srd.raw`) * Height-to-phase screens with the reference DEM subtracted (`h2ph_srd.raw`) @@ -114,7 +114,7 @@ All jobs run on a single AoI on a single track. The following specifications wil * Height-to-phase screens with reference DEM subtracted cropped to the AoI (`h2ph_srd.raw`) * Reduced SLCs with reference DEM subtracted cropped to the AoI (`slc_srd.raw`) * Line and pixel specification of the crop (`nlines_crop.txt` and `npixels_crop.txt`) -- python_preparation: this job converts the output complex interferograms, height-to-phase screens, geocoded coordinates and mother SLC of `doris_v4` or `doris_v5` into a [sarxarray](https://github.com/TUDelftGeodesy/sarxarray) stack. It then crops all data to a provided AoI. The crop is taken to be the smallest rectangle in line/pixel coordinates that completely encloses the AoI. Finally, it creates the (now resampled and reference DEM-subtracted, i.e., _reduced_) SLCs from the cropped complex interferograms and the mother SLC. +- reduce_slc_python: this job converts the output complex interferograms, height-to-phase screens, geocoded coordinates and mother SLC of `doris_v4` or `doris_v5` into a [sarxarray](https://github.com/TUDelftGeodesy/sarxarray) stack. It then crops all data to a provided AoI. The crop is taken to be the smallest rectangle in line/pixel coordinates that completely encloses the AoI. Finally, it creates the (now resampled and reference DEM-subtracted, i.e., _reduced_) SLCs from the cropped complex interferograms and the mother SLC. * input: * Complex interferograms with reference DEM subtracted (`cint_srd.raw`) * Height-to-phase screens with the reference DEM subtracted (`h2ph_srd.raw`) @@ -135,7 +135,7 @@ All jobs run on a single AoI on a single track. The following specifications wil * `lat`: latitude coordinates cropped to the AoI * `lon`: longitude coordinates cropped to the AoI * `time`: epochs of the acquisitions -- znap_to_raw: this job crops the output SLCs, height-to-phase screens, geocoded coordinates and mother SLC of `snap_permissions` to a provided AoI. The crop is taken to be the smallest rectangle in line/pixel coordinates that completely encloses the AoI. It then creates the (now resampled and reference DEM-subtracted, i.e., _reduced_) complex interferograms from the cropped SLCs and the mother SLC, before writing everything to `.raw` format +- merge_to_stack_matlab: this job crops the output SLCs, height-to-phase screens, geocoded coordinates and mother SLC of `snap_permissions` to a provided AoI. The crop is taken to be the smallest rectangle in line/pixel coordinates that completely encloses the AoI. It then creates the (now resampled and reference DEM-subtracted, i.e., _reduced_) complex interferograms from the cropped SLCs and the mother SLC, before writing everything to `.raw` format * input: * `.znap`-archives, one per acquisition, with permissions `775` * AoI in `.shp` format @@ -146,7 +146,7 @@ All jobs run on a single AoI on a single track. The following specifications wil * Height-to-phase screens with reference DEM subtracted cropped to the AoI (`h2ph_srd.raw`) * Reduced SLCs with reference DEM subtracted cropped to the AoI (`slc_srd.raw`) * Line and pixel specification of the crop (`nlines_crop.txt` and `npixels_crop.txt`) -- znap_to_zarr: this job converts the output SLCs, height-to-phase screens, geocoded coordinates and mother SLC of `snap_permissions` into a [sarxarray](https://github.com/TUDelftGeodesy/sarxarray) stack. It then crops all data to a provided AoI. The crop is taken to be the smallest rectangle in line/pixel coordinates that completely encloses the AoI. +- merge_to_stack_python: this job converts the output SLCs, height-to-phase screens, geocoded coordinates and mother SLC of `snap_permissions` into a [sarxarray](https://github.com/TUDelftGeodesy/sarxarray) stack. It then crops all data to a provided AoI. The crop is taken to be the smallest rectangle in line/pixel coordinates that completely encloses the AoI. * input: * `.znap`-archives, one per acquisition, with permissions `775` * AoI in `.shp` format @@ -209,7 +209,7 @@ All jobs run on a single AoI on a single track. The following specifications wil - optional coordinates: * `rd_x` / `epsg:xxx_x`: x coordinate of requested projection from variable `stm_extra_projection` * `rd_y` / `epsg:xxx_y`: y coordinate of requested projection from variable `stm_extra_projection` -- depsi: this job runs [Delft Persistent Scatterer Interferometry (DePSI)](https://repository.tudelft.nl/record/uuid:5dba48d7-ee26-4449-b674-caa8df93e71e) on the output of `crop_to_raw`. +- depsi: this job runs [Delft Persistent Scatterer Interferometry (DePSI)](https://repository.tudelft.nl/record/uuid:5dba48d7-ee26-4449-b674-caa8df93e71e) on the output of `reduce_slc_matlab`. * input: * Radarcoded DEM cropped to the AoI (`dem_radar.raw`) * Geocoded pixel coordinates cropped to the AoI (`lam.raw` and `phi.raw`) diff --git a/docs/parameter-file.md b/docs/parameter-file.md index e614d26b..d91061e7 100644 --- a/docs/parameter-file.md +++ b/docs/parameter-file.md @@ -69,7 +69,7 @@ Required fields for any AoI are marked with a bold **R** - Possible values: any `string` containing lowercase letters and underscores, typically matching the AoI name itself - `general:shape-file:directory` (machine field) - Function: specify the base directory where the shapefile should be stored. - - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Software/roi//_'`, where `stacks` indicates `coregistration` and `crops` indicates `crop_to_raw`. + - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Software/roi//_'`, where `stacks` indicates `coregistration` and `crops` indicates `reduce_slc_matlab`. - **R** `general:shape-file:shape-file-link` - Function: provide a link to a predetermined shapefile. If this field is provided, the `rectangular-shape-file` keys are ignored - Possible values: `''` for rectangular AoI generation, or `string` with any valid path to a shapefile on Spider, including the name of the shapefile itself ending in `.shp`. It is assumed the corresponding `.dbf`, `.prj` and `.shx` also exist with the same name (except the format) in the same directory. @@ -128,165 +128,165 @@ Required fields for any AoI are marked with a bold **R** - Function: specify which jobs should run, linked to the step keys in [job-definitions.yaml](../config/job-definitions.yaml) -## Crop_to_raw parameters +## reduce_slc_matlab parameters -These parameters are used in the job `crop_to_raw`. Defaults in [the default crop_to_raw config file](../config/parameter-files/default-job-param-file-crop_to_raw.yaml). +These parameters are used in the job `reduce_slc_matlab`. Defaults in [the default reduce_slc_matlab config file](../config/parameter-files/default-job-param-file-reduce_slc_matlab.yaml). -- `crop_to_raw:general:AoI-name`: - - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `crop_to_raw`. For cross-AoI dependencies, specify the same AoI name as the dependency. +- `reduce_slc_matlab:general:AoI-name`: + - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `reduce_slc_matlab`. For cross-AoI dependencies, specify the same AoI name as the dependency. - Possible values: any `string` containing lowercase letters and underscores, typically matching the AoI name itself -- `crop_to_raw:general:directory` - - Function: specify the base directory where the [job](glossary.md#jobs) `crop_to_raw` should run. - - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/stacks'` -- `crop_to_raw:general:partition` - - Function: specify the partition on which the [job](glossary.md#jobs) `crop_to_raw` should be run - - Possible values: `'short'` (10h time limit, max 2 jobs), `'normal'` (5 day time limit), `'infinite'` (12 day time limit) +- `reduce_slc_matlab:general:directory` + - Function: specify the base directory where the [job](glossary.md#jobs) `reduce_slc_matlab` should run. + - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/crops'` +- `reduce_slc_matlab:general:partition` + - Function: specify the partition on which the [job](glossary.md#jobs) `reduce_slc_matlab` should be run + - Possible values: `'short'` (10h time limit, max 2 jobs), `'normal'` (5 day time limit), `'infinite'` (30 day time limit) -## Crop_to_zarr parameters +## reduce_slc_python parameters -These parameters are used in the job `crop_to_zarr`. Defaults in [the default crop_to_zarr config file](../config/parameter-files/default-job-param-file-crop_to_zarr.yaml). +These parameters are used in the job `reduce_slc_python`. Defaults in [the default reduce_slc_python config file](../config/parameter-files/default-job-param-file-reduce_slc_python.yaml). -- `crop_to_zarr:general:AoI-name`: - - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `crop_to_zarr`. For cross-AoI dependencies, specify the same AoI name as the dependency. +- `reduce_slc_python:general:AoI-name`: + - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `reduce_slc_python`. For cross-AoI dependencies, specify the same AoI name as the dependency. - Possible values: any `string` containing lowercase letters and underscores, typically matching the AoI name itself -- `crop_to_zarr:general:directory` - - Function: specify the base directory where the [job](glossary.md#jobs) `crop_to_zarr` should run. - - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/stacks'` -- `crop_to_zarr:general:partition` - - Function: specify the partition on which the [job](glossary.md#jobs) `crop_to_zarr` should be run - - Possible values: `'short'` (10h time limit, max 2 jobs), `'normal'` (5 day time limit), `'infinite'` (12 day time limit) -- `crop_to_zarr:general:crop_to_zarr-code-directory` - - Function: specify where the DePSI_group code is, containing the functionality for `crop_to_zarr` +- `reduce_slc_python:general:directory` + - Function: specify the base directory where the [job](glossary.md#jobs) `reduce_slc_python` should run. + - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/stacks_zarr'` +- `reduce_slc_python:general:partition` + - Function: specify the partition on which the [job](glossary.md#jobs) `reduce_slc_python` should be run + - Possible values: `'short'` (10h time limit, max 2 jobs), `'normal'` (5 day time limit), `'infinite'` (30 day time limit) +- `reduce_slc_python:general:depsi_group-code-directory` + - Function: specify where the DePSI_group code is, containing the functionality for `reduce_slc_python` - Possible values: `string` with the absolute path to the base directory of `DePSI_group` -## DeInSAR parameters +## doris_v4 parameters -These parameters are used in the job `deinsar`. Defaults in [the default DeInSAR config file](../config/parameter-files/default-job-param-file-deinsar.yaml). +These parameters are used in the job `doris_v4`. Defaults in [the default doris_v4 config file](../config/parameter-files/default-job-param-file-doris_v4.yaml). -- `deinsar:general:AoI-name`: - - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `deinsar`. For cross-AoI dependencies, specify the same AoI name as the dependency. +- `doris_v4:general:AoI-name`: + - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `doris_v4`. For cross-AoI dependencies, specify the same AoI name as the dependency. - Possible values: any `string` containing lowercase letters and underscores, typically matching the AoI name itself -- `deinsar:general:directory` - - Function: specify the base directory where the [job](glossary.md#jobs) `deinsar` should run. +- `doris_v4:general:directory` + - Function: specify the base directory where the [job](glossary.md#jobs) `doris_v4` should run. - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/stacks'` -- `deinsar:general:partition` - - Function: specify the partition on which the [job](glossary.md#jobs) `deinsar` should be run +- `doris_v4:general:partition` + - Function: specify the partition on which the [job](glossary.md#jobs) `doris_v4` should be run - Possible values: `'short'` (10h time limit, max 2 jobs), `'normal'` (5 day time limit), `'infinite'` (12 day time limit) -- `deinsar:general:deinsar-code-directory` +- `doris_v4:general:deinsar-code-directory` - Function: specify the absolute path to the DeINSAR code - Possible values: `string` with the absolute path to the base directory of `DeInSAR` -- `deinsar:general:doris-v4-code-directory` +- `doris_v4:general:doris-v4-code-directory` - Function: specify the absolute path to the Doris v4 code - Possible values: `string` with the absolute path to the base directory of `doris_v4` -- `deinsar:input:data-directories` +- `doris_v4:input:data-directories` - Function: specify the data directories where the original images are stored (as the non-Sentinel-1 data archive is not sorted in a machine-readable way) - Possible values: e.g. `{'tsx_asc_t116': '/project/caroline/Data/radar_data/eurasia/netherlands/tsx/nl_amsterdam_tsx_asc_t116_T171816_171824_007_hh/data_backup'}`, as keys the tracks, as arguments the full path to the data directory, each key on a new line one tab in. -- `deinsar:deinsar-settings:do-orbit` +- `doris_v4:doris_v4-settings:do-orbit` - Function: specify whether the step `orbit` should be run (note: only for ENV, ERS and RSAT2) - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-crop` +- `doris_v4:doris_v4-settings:do-crop` - Function: specify whether the step `crop` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-tsx-deramp` +- `doris_v4:doris_v4-settings:do-tsx-deramp` - Function: specify whether the step `tsx-deramp` should be run (note: only for TSX) - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-simamp` +- `doris_v4:doris_v4-settings:do-simamp` - Function: specify whether the step `simamp` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-mtiming` +- `doris_v4:doris_v4-settings:do-mtiming` - Function: specify whether the step `mtiming` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-ovs` +- `doris_v4:doris_v4-settings:do-ovs` - Function: specify whether the step `ovs` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-choose-master` +- `doris_v4:doris_v4-settings:do-choose-master` - Function: specify whether the step `choose-master` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-coarseorb` +- `doris_v4:doris_v4-settings:do-coarseorb` - Function: specify whether the step `coarseorb` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-coarsecorr` +- `doris_v4:doris_v4-settings:do-coarsecorr` - Function: specify whether the step `coarsecorr` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:finecoreg:do-finecoreg` +- `doris_v4:doris_v4-settings:finecoreg:do-finecoreg` - Function: specify whether the step `finecoreg` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:finecoreg:finecoreg-mode` +- `doris_v4:doris_v4-settings:finecoreg:finecoreg-mode` - Function: specify which version of `finecoreg` should be run - Possible values: `'simple'`, `'normal'` -- `deinsar:deinsar-settings:do-reltiming` +- `doris_v4:doris_v4-settings:do-reltiming` - Function: specify whether the step `reltiming` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-dembased` +- `doris_v4:doris_v4-settings:do-dembased` - Function: specify whether the step `dembased` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-coregpm` +- `doris_v4:doris_v4-settings:do-coregpm` - Function: specify whether the step `coregpm` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-resample` +- `doris_v4:doris_v4-settings:do-resample` - Function: specify whether the step `resample` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-tsx-reramp` +- `doris_v4:doris_v4-settings:do-tsx-reramp` - Function: specify whether the step `tsx-reramp` should be run (note: only for TSX) - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-comprefpha` +- `doris_v4:doris_v4-settings:do-comprefpha` - Function: specify whether the step `comprefpha` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-comprefdem` +- `doris_v4:doris_v4-settings:do-comprefdem` - Function: specify whether the step `comprefdem` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-interferogram` +- `doris_v4:doris_v4-settings:do-interferogram` - Function: specify whether the step `interferogram` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-subtrrefpha` +- `doris_v4:doris_v4-settings:do-subtrrefpha` - Function: specify whether the step `subtrrefpha` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-subtrrefdem` +- `doris_v4:doris_v4-settings:do-subtrrefdem` - Function: specify whether the step `subtrrefdem` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-coherence` +- `doris_v4:doris_v4-settings:do-coherence` - Function: specify whether the step `coherence` should be run - Possible values: `0`, `1` -- `deinsar:deinsar-settings:do-geocoding` +- `doris_v4:doris_v4-settings:do-geocoding` - Function: specify whether the step `geocoding` should be run - Possible values: `0`, `1` -## DePSI parameters +## DePSI_matlab parameters -These parameters are used in the job `depsi`. Defaults in [the default DePSI config file](../config/parameter-files/default-job-param-file-depsi.yaml) +These parameters are used in the job `depsi_matlab`. Defaults in [the default DePSI config file](../config/parameter-files/default-job-param-file-depsi_matlab.yaml) Most parameters are explained in the [PhD thesis of Freek](https://repository.tudelft.nl/record/uuid:5dba48d7-ee26-4449-b674-caa8df93e71e) using the same naming (underscores and dashes may be swapped). Those that aren't: -- `depsi:general:AoI-name`: - - Function: specify the AoI name for the directory naming in the [jobs](glossary.md#jobs) `depsi`, `mrm`, and `depsi_post`. For cross-AoI dependencies, specify the same AoI name as the dependency. +- `depsi_matlab:general:AoI-name`: + - Function: specify the AoI name for the directory naming in the [jobs](glossary.md#jobs) `depsi_matlab`, `create_mrm`, and `depsi_post`. For cross-AoI dependencies, specify the same AoI name as the dependency. - Possible values: any `string` containing lowercase letters and underscores, typically matching the AoI name itself -- `depsi:general:directory` - - Function: specify the base directory where the [jobs](glossary.md#jobs) `depsi`, `mrm`, and `depsi_post` should run. +- `depsi_matlab:general:directory` + - Function: specify the base directory where the [jobs](glossary.md#jobs) `depsi_matlab`, `create_mrm`, and `depsi_post` should run. - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/stacks'` -- `depsi:general:partition` - - Function: specify the partition on which the [job](glossary.md#jobs) `depsi` should be run +- `depsi_matlab:general:partition` + - Function: specify the partition on which the [job](glossary.md#jobs) `depsi_matlab` should be run - Possible values: `'short'` (10h time limit, max 2 jobs), `'normal'` (5 day time limit), `'infinite'` (12 day time limit) -- `depsi:general:depsi-code-directory` +- `depsi_matlab:general:depsi_matlab-code-directory` - Function: specify where the DePSI code is - - Possible values: `string` with the absolute path to the base directory of `depsi` -- `depsi:general:rdnaptrans-directory` + - Possible values: `string` with the absolute path to the base directory of `depsi_matlab` +- `depsi_matlab:general:rdnaptrans-directory` - Function: specify where the [RDNAPtrans](https://www.nsgi.nl/coordinatenstelsels-en-transformaties/coordinatentransformaties/rdnap-etrs89-rdnaptrans) code is - Possible values: `string` with the absolute path to the base directory of RDNAPtrans -- `depsi:general:geocoding-directory` +- `depsi_matlab:general:geocoding-directory` - Function: specify where the Geocoding code is - Possible values: `string` with the absolute path to the base directory of Geocoding -- `depsi:depsi-settings:general:ref-cn` +- `depsi_matlab:depsi_matlab-settings:general:ref-cn` - Function: specify the mode of how the reference point in DePSI should be determined - Possible values for the key `all`: - `[]` or `'independent'`: consecutive runs are treated as completely independent, and can therefore have different reference points. This can have unintended consequences as the behaviour of the reference point can change drastically. The reference point is determined using the NAD metric. - `'constant'`: consecutive runs on the same track use the same reference point, where the first run runs on mode `'independent'` to select a reference point, and further runs retain this point. - `[azimuth, range]`: all runs are forced to the specified reference point regardless of what is there. If the reference point is not in the selection, DePSI will throw an error - Additionally, one can add keys for individual tracks as new lines one tab in from `ref-cn`. This generates a dictionary that looks like this. `{'s1_asc_t088': 'constant', 's1_dsc_t110': [100, 300], 'all': 'constant'}`. Different behaviours are specified for different tracks. This is almost always necessary for the `[azimuth, range]` mode. If tracks are missing from this specification, the `'all'` key is used for those instead. All of the above options are allowed in this mode as arguments. -- `depsi:depsi-settings:psc:do-water-mask` - - Function: specify whether a water mask should be applied. If it should be applied, a water mask named `water_mask____t3d>.raw` is expected in the water-mask directory in the [configuration file](../config/spider-config.yaml). +- `depsi_matlab:depsi_matlab-settings:psc:do-water-mask` + - Function: specify whether a water mask should be applied. If it should be applied, a water mask named `water_mask____t3d>.raw` is expected in the water-mask directory in the [configuration file](../config/spider-config.yaml). - Possible values: `'yes'`, `'no'` ## DePSI_post parameters @@ -305,207 +305,207 @@ Most parameters are explained in [How to DePSI-post](https://sites.google.com/si - Possible values: `string` with the absolute path to `cpxfiddle` -## Doris parameters -These parameters are used in the job `doris`. Defaults in [the default doris config file](../config/parameter-files/default-job-param-file-doris.yaml). +## Doris_v5 parameters +These parameters are used in the job `doris_v5`. Defaults in [the default doris_v5 config file](../config/parameter-files/default-job-param-file-doris_v5.yaml). -- `doris:general:AoI-name`: - - Function: specify the AoI name for the directory naming in the [jobs](glossary.md#jobs) `doris` and `doris_cleanup`. For cross-AoI dependencies, specify the same AoI name as the dependency. +- `doris_v5:general:AoI-name`: + - Function: specify the AoI name for the directory naming in the [jobs](glossary.md#jobs) `doris_v5` and `doris_v5_cleanup`. For cross-AoI dependencies, specify the same AoI name as the dependency. - Possible values: any `string` containing lowercase letters and underscores, typically matching the AoI name itself -- `doris:general:directory` - - Function: specify the base directory where the [jobs](glossary.md#jobs) `doris` and `doris_cleanup` should run. +- `doris_v5:general:directory` + - Function: specify the base directory where the [jobs](glossary.md#jobs) `doris_v5` and `doris_v5_cleanup` should run. - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/stacks'` -- `doris:general:partition` - - Function: specify the partition on which the [job](glossary.md#jobs) `doris` should be run +- `doris_v5:general:partition` + - Function: specify the partition on which the [job](glossary.md#jobs) `doris_v5` should be run - Possible values: `'short'` (10h time limit, max 2 jobs), `'normal'` (5 day time limit), `'infinite'` (12 day time limit) -- `doris:general:code-directory` +- `doris_v5:general:code-directory` - Function: specify the location of the Doris code. - - Possible values: `string` with absolute path to the base of the Doris code -- `doris:doris-settings:do-coarse-orbits` + - Possible values: `string` with absolute path to the base of the Doris v5 code +- `doris_v5:doris_v5-settings:do-coarse-orbits` - Function: specify whether the Doris step `coarse_orbits` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-deramp` +- `doris_v5:doris_v5-settings:do-deramp` - Function: specify whether the Doris step `deramp` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-reramp` +- `doris_v5:doris_v5-settings:do-reramp` - Function: specify whether the Doris step `reramp` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-fake-fine-coreg-bursts` +- `doris_v5:doris_v5-settings:do-fake-fine-coreg-bursts` - Function: specify whether the Doris step `fake_fine_coreg_bursts` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-fake-master-resample` +- `doris_v5:doris_v5-settings:do-fake-master-resample` - Function: specify whether the Doris step `fake_master_resample` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-dac-bursts` +- `doris_v5:doris_v5-settings:do-dac-bursts` - Function: specify whether the Doris step `dac_bursts` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-fake-coreg-bursts` +- `doris_v5:doris_v5-settings:do-fake-coreg-bursts` - Function: specify whether the Doris step `fake_coreg_bursts` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-resample` +- `doris_v5:doris_v5-settings:do-resample` - Function: specify whether the Doris step `resample` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-reramp2` +- `doris_v5:doris_v5-settings:do-reramp2` - Function: specify whether the Doris step `reramp` should be run (the second time it appears in the parameter file) - Possible values: `0`, `1` -- `doris:doris-settings:do-interferogram` +- `doris_v5:doris_v5-settings:do-interferogram` - Function: specify whether the Doris step `interferogram` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-compref-phase` +- `doris_v5:doris_v5-settings:do-compref-phase` - Function: specify whether the Doris step `compref_phase` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-compref-dem` +- `doris_v5:doris_v5-settings:do-compref-dem` - Function: specify whether the Doris step `compref_dem` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-coherence` +- `doris_v5:doris_v5-settings:do-coherence` - Function: specify whether the Doris step `coherence` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-esd` +- `doris_v5:doris_v5-settings:do-esd` - Function: specify whether the Doris step `esd` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-network-esd` +- `doris_v5:doris_v5-settings:do-network-esd` - Function: specify whether the Doris step `network_esd` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-ESD-correct` +- `doris_v5:doris_v5-settings:do-ESD-correct` - Function: specify whether the Doris step `ESD_correct` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-combine-master` +- `doris_v5:doris_v5-settings:do-combine-master` - Function: specify whether the Doris step `combine_master` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-combine-slave` +- `doris_v5:doris_v5-settings:do-combine-slave` - Function: specify whether the Doris step `combine_slave` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-ref-phase` +- `doris_v5:doris_v5-settings:do-ref-phase` - Function: specify whether the Doris step `ref_phase` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-ref-dem` +- `doris_v5:doris_v5-settings:do-ref-dem` - Function: specify whether the Doris step `ref_dem` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-phasefilt` +- `doris_v5:doris_v5-settings:do-phasefilt` - Function: specify whether the Doris step `phasefilt` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-calc-coordinates` +- `doris_v5:doris_v5-settings:do-calc-coordinates` - Function: specify whether the Doris step `calc_coordinates` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-multilooking` +- `doris_v5:doris_v5-settings:do-multilooking` - Function: specify whether the Doris step `multilooking` should be run - Possible values: `0`, `1` -- `doris:doris-settings:do-unwrap` +- `doris_v5:doris_v5-settings:do-unwrap` - Function: specify whether the Doris step `unwrap` should be run - Possible values: `0`, `1` -## STM_generation parameters -These parameters are used in the job `stm_generation`. Defaults in [the default stm_generation config file](../config/parameter-files/default-job-param-file-stm_generation.yaml). +## generate_partitioned_stm parameters +These parameters are used in the job `generate_partitioned_stm`. Defaults in [the default generate_partitioned_stm config file](../config/parameter-files/default-job-param-file-generate_partitioned_stm.yaml). -- `stm_generation:general:AoI-name`: - - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `stm_generation`. For cross-AoI dependencies, specify the same AoI name as the dependency. +- `generate_partitioned_stm:general:AoI-name`: + - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `generate_partitioned_stm`. For cross-AoI dependencies, specify the same AoI name as the dependency. - Possible values: any `string` containing lowercase letters and underscores, typically matching the AoI name itself -- `stm_generation:general:directory` - - Function: specify the base directory where the [job](glossary.md#jobs) `stm_generation` should run. - - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/stacks'` -- `stm_generation:general:partition` - - Function: specify the partition on which the [job](glossary.md#jobs) `stm_generation` should be run +- `generate_partitioned_stm:general:directory` + - Function: specify the base directory where the [job](glossary.md#jobs) `generate_partitioned_stm` should run. + - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/stms_zarr'` +- `generate_partitioned_stm:general:partition` + - Function: specify the partition on which the [job](glossary.md#jobs) `generate_partitioned_stm` should be run - Possible values: `'short'` (10h time limit, max 2 jobs), `'normal'` (5 day time limit), `'infinite'` (12 day time limit) -- `stm_generation:general:stm_generation-code-directory` - - Function: specify where the DePSI_group code is, containing the functionality for `stm_generation` +- `generate_partitioned_stm:general:depsi_group-code-directory` + - Function: specify where the DePSI_group code is, containing the functionality for `generate_partitioned_stm` - Possible values: `string` with the absolute path to the base directory of `DePSI_group` -- `stm_generation:stm_generation-settings:ps-selection:mode` +- `generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:mode` - Function: specify the mode to be used for the time frame selection during the PS selection - Possible values: `'full'` (full time series), `'initialization'` (using part of the time series defined by `stm_start_date_ps_selection` and `stm_initialization_length`) -- `stm_generation:stm_generation-settings:ps-selection:initialization-mode-settings:start-date` +- `generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:init-settings:start-date` - Function: specify the start date of the time frame to be used for PS selection in `initialization` mode - Possible values: `'YYYY-MM-DD'` -- `stm_generation:stm_generation-settings:ps-selection:initialization-mode-settings:initialization-length` +- `generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:init-settings:initialization-length` - Function: specify the length of the time frame to be used for PS selection in `initialization` mode - Possible values: any positive integer (# of epochs), or `'YYYY-MM-DD'` (end date) -- `stm_generation:stm_generation-settings:ps-selection:method` +- `generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:method` - Function: specify the method to be used for the PS selection - Possible values: `'nmad'`, `'nad'` -- `stm_generation:stm_generation-settings:ps-selection:threshold` +- `generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:threshold` - Function: set the threshold for when a point is considered a PS (all PS below the threshold are accepted) - Possible values: any positive `float` -- `stm_generation:stm_generation-settings:incremental-statistics:increment-mode` +- `generate_partitioned_stm:generate_partitioned_stm-settings:incremental-statistics:increment-mode` - Function: specify the mode to add either the [incremental or recalibration](https://github.com/TUDelftGeodesy/DePSI_group/blob/dev/depsi/point_quality.py#L148) NAD or NMAD (based on `stm_ps_selection_method`) - Possible values: `'incremental'` (update every epoch), `'recalibration'` (update every `stm_nad_nmad_recalibration_jump_size` epochs) -- `stm_generation:stm_generation-settings:incremental-statistics:recalibration-jump-size` +- `generate_partitioned_stm:generate_partitioned_stm-settings:incremental-statistics:recal-jump-size` - Function: specify the jump size to be used for `recalibration` mode for the updating NAD or NMAD - Possible values: any positive integer -- `stm_generation:stm_generation-settings:single-differences:mother` +- `generate_partitioned_stm:generate_partitioned_stm-settings:single-differences:mother` - Function: specify the mother epoch for single difference computations - Possible values: `'auto'` (uses the mother from the input `.zarr` archive), `'YYYY-MM-DD'` -- `stm_generation:stm_generation-settings:extra-projection` +- `generate_partitioned_stm:generate_partitioned_stm-settings:extra-projection` - Function: specify an extra projection to project the geolocation coordinates into - Possible values: `'RD'` (Dutch Rijksdriehoek, Netherlands only), `'EPSG:###'` (any code works), `''` (no new projection) -- `stm_generation:stm_generation-settings:partitioning:do-partitioning` +- `generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:do-partitioning` - Function: switch to do or not do partitioning in time - Possible values: `0`, `1` -- `stm_generation:stm_generation-settings:partitioning:search-method` +- `generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:search-method` - Function: specify the method to search for the partitions - Possible values: `'pelt'`, `'binseg'` -- `stm_generation:stm_generation-settings:partitioning:cost-function` +- `generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:cost-function` - Function: specify the cost function in the partition search - Possible values: `'l2'` -- `stm_generation:stm_generation-settings:partitioning:db-mode` +- `generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:db-mode` - Function: specify whether or not to do the partition search on the Decibel scale - Possible values: `0` (advised), `1` -- `stm_generation:stm_generation-settings:partitioning:min-partition-length` +- `generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:min-partition-length` - Function: specify the minimum number of acquisitions per partition - Possible values: any positive integer -- `stm_generation:stm_generation-settings:partitioning:undifferenced-output-layers` +- `generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:undifferenced-output-lyrs` - Function: specify the output data layers per partition using the undifferenced input data - Possible values: `list` containing a subset of the following: `'nad'`, `'nmad'`, `mad`, `'quality_nmad_2sigma'`, `'quality_nmad_mean'`, `'quality_nad_2sigma'`, `'quality_nad_mean'`, `'amplitude_mean'`, `'amplitude_sigma'`, `'amplitude_median'`. -- `stm_generation:stm_generation-settings:partitioning:single-difference-output-layers` +- `generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:single-difference-output-lyrs` - Function: specify the output data layers per partition using the input data with a single difference in time with respect to `stm_single_difference_mother` - Possible values: `list` containing a subset of the following: `'nad'`, `'nmad'`, `mad`, `'quality_nmad_2sigma'`, `'quality_nmad_mean'`, `'quality_nad_2sigma'`, `'quality_nad_mean'`, `'amplitude_mean'`, `'amplitude_sigma'`, `'amplitude_median'`. -- `stm_generation:stm_generation-settings:outlier-detection:do-outlier-detection` +- `generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:do-outlier-detection` - Function: switch to do or not do outlier detection in time - Possible values: `0`, `1` -- `stm_generation:stm_generation-settings:outlier-detection:window-size` +- `generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:window-size` - Function: specify the size of the rolling window across which the statistics are computed to determine whether or not an observation is an outlier - Possible values: any positive integer -- `stm_generation:stm_generation-settings:outlier-detection:db-mode` +- `generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:db-mode` - Function: specify whether or not to do outlier detection on the Decibel scale - Possible values: `0`, `1` (advised) -- `stm_generation:stm_generation-settings:outlier-detection:n-sigma` +- `generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:n-sigma` - Function: specify the minimum number of sigma deviation from the median of the observations in the window before an observation is considered an outlier - Possible values: any positive `float` (advised 3) -## znap_to_raw parameters +## merge_to_stack_matlab parameters -These parameters are used in the job `znap_to_raw`. Defaults in [the default znap_to_raw config file](../config/parameter-files/default-job-param-file-znap_to_raw.yaml). +These parameters are used in the job `merge_to_stack_matlab`. Defaults in [the default merge_to_stack_matlab config file](../config/parameter-files/default-job-param-file-merge_to_stack_matlab.yaml). -- `znap_to_raw:general:AoI-name`: - - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `znap_to_raw`. For cross-AoI dependencies, specify the same AoI name as the dependency. +- `merge_to_stack_matlab:general:AoI-name`: + - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `merge_to_stack_matlab`. For cross-AoI dependencies, specify the same AoI name as the dependency. - Possible values: any `string` containing lowercase letters and underscores, typically matching the AoI name itself -- `znap_to_raw:general:directory` - - Function: specify the base directory where the [job](glossary.md#jobs) `znap_to_raw` should run. - - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/stacks'` -- `znap_to_raw:general:partition` - - Function: specify the partition on which the [job](glossary.md#jobs) `znap_to_raw` should be run - - Possible values: `'short'` (10h time limit, max 2 jobs), `'normal'` (5 day time limit), `'infinite'` (12 day time limit) -- `znap_to_raw:general:znap_to_raw-code-directory` - - Function: specify where the DePSI_group code is, containing the functionality for `znap_to_raw` +- `merge_to_stack_matlab:general:directory` + - Function: specify the base directory where the [job](glossary.md#jobs) `merge_to_stack_matlab` should run. + - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/crops'` +- `merge_to_stack_matlab:general:partition` + - Function: specify the partition on which the [job](glossary.md#jobs) `merge_to_stack_matlab` should be run + - Possible values: `'short'` (10h time limit, max 2 jobs), `'normal'` (5 day time limit), `'infinite'` (30 day time limit) +- `merge_to_stack_matlab:general:depsi_group-code-directory` + - Function: specify where the DePSI_group code is, containing the functionality for `merge_to_stack_matlab` - Possible values: `string` with the absolute path to the base directory of `DePSI_group` -## znap_to_zarr parameters +## merge_to_stack_python parameters -These parameters are used in the job `znap_to_zarr`. Defaults in [the default znap_to_zarr config file](../config/parameter-files/default-job-param-file-znap_to_zarr.yaml). +These parameters are used in the job `merge_to_stack_python`. Defaults in [the default merge_to_stack_python config file](../config/parameter-files/default-job-param-file-merge_to_stack_python.yaml). -- `znap_to_zarr:general:AoI-name`: - - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `znap_to_zarr`. For cross-AoI dependencies, specify the same AoI name as the dependency. +- `merge_to_stack_python:general:AoI-name`: + - Function: specify the AoI name for the directory naming in the [job](glossary.md#jobs) `merge_to_stack_python`. For cross-AoI dependencies, specify the same AoI name as the dependency. - Possible values: any `string` containing lowercase letters and underscores, typically matching the AoI name itself -- `znap_to_zarr:general:directory` - - Function: specify the base directory where the [job](glossary.md#jobs) `znap_to_zarr` should run. - - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/stacks'` -- `znap_to_zarr:general:partition` - - Function: specify the partition on which the [job](glossary.md#jobs) `znap_to_zarr` should be run +- `merge_to_stack_python:general:directory` + - Function: specify the base directory where the [job](glossary.md#jobs) `merge_to_stack_python` should run. + - Possible values: `string` with any valid path on Spider. If it does not exist, it will be created. Default is `'/project/caroline/Share/stacks_zarr'` +- `merge_to_stack_python:general:partition` + - Function: specify the partition on which the [job](glossary.md#jobs) `merge_to_stack_python` should be run - Possible values: `'short'` (10h time limit, max 2 jobs), `'normal'` (5 day time limit), `'infinite'` (12 day time limit) -- `znap_to_zarr:general:znap_to_zarr-code-directory` - - Function: specify where the DePSI_group code is, containing the functionality for `znap_to_zarr` +- `merge_to_stack_python:general:depsi_group-code-directory` + - Function: specify where the DePSI_group code is, containing the functionality for `merge_to_stack_python` - Possible values: `string` with the absolute path to the base directory of `DePSI_group` diff --git a/pyproject.toml b/pyproject.toml index e40ec77f..de0b8d4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "Caroline" -version = "3.2.3" +version = "4.0.0" requires-python = ">=3.10" dependencies = [ "apache-airflow", @@ -15,7 +15,8 @@ dependencies = [ "sarxarray>=1.4.0b2", "caroline-download@git+https://github.com/TUDelftGeodesy/caroline-download.git@v0.2.0rc2", "snap-run@git+ssh://git@github.com/TUDelftGeodesy/snap-coregistration.git@caroline-clone-branch", - "depsi@git+ssh://git@github.com/TUDelftGeodesy/DePSI_group.git@caroline-clone-branch" + "depsi@git+ssh://git@github.com/TUDelftGeodesy/DePSI_group.git@caroline-clone-branch", + "asf_search>=13.0.0" ] description = "End-to-end processing of Interferometric Synthetic Aperture Radar (InSAR) data" readme = "README.md" @@ -53,8 +54,8 @@ docs = [ plugins = [ "scipy", # necessary for Doris v5 "gdal==3.4.1", # necessary for Doris v5, version 3.4.1 since that reflects the status of libgdal on Spider - "dask_jobqueue", # necessary for crop_to_zarr - "ruptures", # necessary for stm_generation + "dask_jobqueue", # necessary for reduce_slc_python + "ruptures", # necessary for generate_partitioned_stm ] [tool.setuptools] diff --git a/templates/mrm/read_mrm.m b/templates/create_mrm/create_mrm.m similarity index 88% rename from templates/mrm/read_mrm.m rename to templates/create_mrm/create_mrm.m index 0a9ec7c9..945bcfa4 100644 --- a/templates/mrm/read_mrm.m +++ b/templates/create_mrm/create_mrm.m @@ -1,7 +1,7 @@ clear all close all -load **depsi:general:AoI-name**_**general:input-data:sensor**_**asc_dsc**_t**fill_track**_project.mat +load **depsi_matlab:general:AoI-name**_**general:input-data:sensor**_**asc_dsc**_t**fill_track**_project.mat if ~exist([project_id '_mrm_uint8_orig.raw'],'file'); copyfile([project_id '_mrm_uint8.raw'],[project_id '_mrm_uint8_orig.raw']); diff --git a/templates/mrm/read_mrm.sh b/templates/create_mrm/create_mrm.sh similarity index 78% rename from templates/mrm/read_mrm.sh rename to templates/create_mrm/create_mrm.sh index d6de6f6d..8a972ae2 100644 --- a/templates/mrm/read_mrm.sh +++ b/templates/create_mrm/create_mrm.sh @@ -34,8 +34,8 @@ module --ignore-cache load **matlab_module** -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started read_mrm.sh (AoI **depsi:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started create_mrm.sh (AoI **depsi_matlab:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log -srun matlab -nodisplay -nosplash -nodesktop -r "run('**depsi_base_directory**/read_mrm.m');exit;" || exit 5 +srun matlab -nodisplay -nosplash -nodesktop -r "run('**depsi_base_directory**/create_mrm.m');exit;" || exit 5 -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished read_mrm.sh (AoI **depsi:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished create_mrm.sh (AoI **depsi_matlab:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/crop-to-zarr/crop-to-zarr.sh b/templates/crop-to-zarr/crop-to-zarr.sh deleted file mode 100644 index 8c8bd1dd..00000000 --- a/templates/crop-to-zarr/crop-to-zarr.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --time=24:00:00 -#SBATCH --cpus-per-task=4 -#SBATCH --partition=normal - -source ~/.bashrc - -source /etc/profile.d/modules.sh -source /project/caroline/Software/bin/init.sh -module load **python3_module** **gdal_module** -source **caroline_virtual_environment_directory**/bin/activate - -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started crop-to-zarr.sh (AoI **crop_to_zarr:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log - -export PATH="**crop_to_zarr:general:crop_to_zarr-code-directory**:$PATH" -export PYTHONPATH="**crop_to_zarr:general:crop_to_zarr-code-directory**:$PYTHONPATH" - -python3 crop-to-zarr.py || exit 5 - -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished crop-to-zarr.sh (AoI **crop_to_zarr:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/depsi/param_file.txt b/templates/depsi/param_file.txt deleted file mode 100755 index aef45aa0..00000000 --- a/templates/depsi/param_file.txt +++ /dev/null @@ -1,121 +0,0 @@ -Input_file template for ps_analysis.m -(See the function ps_readinput_parameters.m for more information) - -% General parameters -% ---------------------------------------------------------------------- - -max_mem_buffer = **depsi:depsi-settings:general:max-mem-buffer** -visible_plots = '**depsi:depsi-settings:general:visible-plots**' -detail_plots = '**depsi:depsi-settings:general:detail-plots**' -processing_groups = **depsi:depsi-settings:general:processing-groups** -run_mode = '**depsi:depsi-settings:general:run-mode**' - - -% Project parameters -% ---------------------------------------------------------------------- - -project_id = '**depsi:general:AoI-name**_**general:input-data:sensor**_**asc_dsc**_t**track**' -input_file = [] -processDir = '**crop_base_directory**/cropped_stack' -startDate = '**start_date**' -stopDate = '**stop_date**' -excludeDate = '**depsi:depsi-settings:general:exclude-date**' -ifgsVersion = '_srd' -altimg = '' -master = '**master_date**' -swath_burst = [] -sensor = '**general:input-data:sensor**' -orbit = '**asc_dsc_fmt**' -crop = [] -processor = 'doris_flinsar' -project = '**depsi:general:AoI-name**' - -az_spacing = **depsi:depsi-settings:general:az-spacing** -r_spacing = **depsi:depsi-settings:general:r-spacing** -slc_selection_input = **depsi:depsi-settings:general:slc-selection-input** -ifg_selection_input = **depsi:depsi-settings:general:ifg-selection-input** -ref_cn = **ref_cn** -Ncv = **depsi:depsi-settings:general:Ncv** -ps_method = '**depsi:depsi-settings:general:ps-method**' -psc_model = **depsi:depsi-settings:general:psc-model** -ps_model = **depsi:depsi-settings:general:ps-model** -final_model = **depsi:depsi-settings:general:final-model** -breakpoint = **depsi:depsi-settings:general:breakpoint** -breakpoint2 = **depsi:depsi-settings:general:breakpoint2** -ens_coh_threshold = **depsi:depsi-settings:general:ens-coh-threshold** -varfac_threshold = **depsi:depsi-settings:general:varfac-threshold** -detrend_method = '**depsi:depsi-settings:general:detrend-method**' -output_format = **depsi:depsi-settings:general:output-format** -stc_min_max = [**depsi:depsi-settings:general:stc-min-max**] -do_apriori_sidelobe_mask = '**depsi:depsi-settings:general:do-apriori-sidelobe-mask**' -do_aposteriori_sidelobe_mask = '**depsi:depsi-settings:general:do-aposteriori-sidelobe-mask**' - - -% Geocoding parameters -%---------------------------------------------------------------------- - -master_res = 'slave.res' -ref_height = **depsi:depsi-settings:geocoding:ref-height** -demFile = 'dem_radar.raw' - -% Psc parameters -%---------------------------------------------------------------------- - -amplitude_calibration = '**depsi:depsi-settings:psc:amplitude-calibration**' -psc_selection_method = '**depsi:depsi-settings:psc:psc-selection-method**' -psc_selection_gridsize = **depsi:depsi-settings:psc:psc-selection-gridsize** -psc_threshold = **depsi:depsi-settings:psc:psc-threshold** -max_arc_length = **depsi:depsi-settings:psc:max-arc-length** -network_method = '**depsi:depsi-settings:psc:network-method**' -Ncon = **depsi:depsi-settings:psc:Ncon** -Nparts = **depsi:depsi-settings:psc:Nparts** -Npsc_selections = **depsi:depsi-settings:psc:Npsc-selections** -filename_water_mask = **filename_water_mask** -gamma_threshold = **depsi:depsi-settings:psc:gamma-threshold** -psc_distribution = '**depsi:depsi-settings:psc:psc-distribution**' -weighted_unwrap = '**depsi:depsi-settings:psc:weighted-unwrap**' - -% threshold is percentage of slc's that has an amplitude peak -livetime_threshold = **depsi:depsi-settings:psc:livetime-threshold**; -% include local near maxima -peak_tolerance = **depsi:depsi-settings:psc:peak-tolerance**; - - -% Ps parameters -% ---------------------------------------------------------------------- - -psp_selection_method = '**depsi:depsi-settings:psp:psp-selection-method**' -psp_threshold1 = **depsi:depsi-settings:psp:psp-threshold1** -psp_threshold2 = **depsi:depsi-settings:psp:psp-threshold2** -ps_eval_method = '**depsi:depsi-settings:psp:ps-eval-method**' -Namp_disp_bins = **depsi:depsi-settings:psp:Namp-disp-bins** -Ndens_iterations = **depsi:depsi-settings:psp:Ndens-iterations** -densification_flag = '**depsi:depsi-settings:psp:densification-flag**' -ps_area_of_interest = **depsi:depsi-settings:psp:ps-area-of-interest** -dens_method = '**depsi:depsi-settings:psp:dens-method**' -dens_check = '**depsi:depsi-settings:psp:dens-check**' -Nest = **depsi:depsi-settings:psp:Nest**; - - -% Stochastic model parameters -% ---------------------------------------------------------------------- - -std_param = [**depsi:depsi-settings:stochastic-model:std-param**]; -defo_range = **depsi:depsi-settings:stochastic-model:defo-range** -weighting = '**depsi:depsi-settings:stochastic-model:weighting**' -ts_atmo_filter = '**depsi:depsi-settings:stochastic-model:ts-atmo-filter**' -ts_atmo_filter_length = **depsi:depsi-settings:stochastic-model:ts-atmo-filter-length** -ts_noise_filter = '**depsi:depsi-settings:stochastic-model:ts-noise-filter**' -ts_noise_filter_length = **depsi:depsi-settings:stochastic-model:ts-noise-filter-length** - - -% Bowl parameters -%----------------------------------------------------------------------- - -defo_method = **depsi:depsi-settings:bowl:defo-method** -xc0 = **depsi:depsi-settings:bowl:xc0** -yc0 = **depsi:depsi-settings:bowl:yc0** -zc0 = **depsi:depsi-settings:bowl:zc0** -r0 = **depsi:depsi-settings:bowl:r0** -r10 = **depsi:depsi-settings:bowl:r10** -epoch = **depsi:depsi-settings:bowl:epoch** diff --git a/templates/depsi/depsi.m b/templates/depsi_matlab/depsi.m similarity index 100% rename from templates/depsi/depsi.m rename to templates/depsi_matlab/depsi.m diff --git a/templates/depsi/depsi.sh b/templates/depsi_matlab/depsi_matlab.sh similarity index 81% rename from templates/depsi/depsi.sh rename to templates/depsi_matlab/depsi_matlab.sh index 9d7d1d2b..2d78b8a7 100644 --- a/templates/depsi/depsi.sh +++ b/templates/depsi_matlab/depsi_matlab.sh @@ -34,8 +34,8 @@ module --ignore-cache load **matlab_module** -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started depsi.sh (AoI **depsi:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started depsi_matlab.sh (AoI **depsi_matlab:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log srun matlab -nodisplay -nosplash -nodesktop -r "run('**depsi_base_directory**/depsi.m');exit;" || exit 5 -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished depsi.sh (AoI **depsi:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log \ No newline at end of file +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished depsi_matlab.sh (AoI **depsi_matlab:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log \ No newline at end of file diff --git a/templates/depsi_matlab/param_file.txt b/templates/depsi_matlab/param_file.txt new file mode 100755 index 00000000..d0896dcd --- /dev/null +++ b/templates/depsi_matlab/param_file.txt @@ -0,0 +1,121 @@ +Input_file template for ps_analysis.m +(See the function ps_readinput_parameters.m for more information) + +% General parameters +% ---------------------------------------------------------------------- + +max_mem_buffer = **depsi_matlab:depsi_matlab-settings:general:max-mem-buffer** +visible_plots = '**depsi_matlab:depsi_matlab-settings:general:visible-plots**' +detail_plots = '**depsi_matlab:depsi_matlab-settings:general:detail-plots**' +processing_groups = **depsi_matlab:depsi_matlab-settings:general:processing-groups** +run_mode = '**depsi_matlab:depsi_matlab-settings:general:run-mode**' + + +% Project parameters +% ---------------------------------------------------------------------- + +project_id = '**depsi_matlab:general:AoI-name**_**general:input-data:sensor**_**asc_dsc**_t**track**' +input_file = [] +processDir = '**crop_base_directory**/cropped_stack' +startDate = '**start_date**' +stopDate = '**stop_date**' +excludeDate = '**depsi_matlab:depsi_matlab-settings:general:exclude-date**' +ifgsVersion = '_srd' +altimg = '' +master = '**master_date**' +swath_burst = [] +sensor = '**general:input-data:sensor**' +orbit = '**asc_dsc_fmt**' +crop = [] +processor = 'doris_flinsar' +project = '**depsi_matlab:general:AoI-name**' + +az_spacing = **depsi_matlab:depsi_matlab-settings:general:az-spacing** +r_spacing = **depsi_matlab:depsi_matlab-settings:general:r-spacing** +slc_selection_input = **depsi_matlab:depsi_matlab-settings:general:slc-selection-input** +ifg_selection_input = **depsi_matlab:depsi_matlab-settings:general:ifg-selection-input** +ref_cn = **ref_cn** +Ncv = **depsi_matlab:depsi_matlab-settings:general:Ncv** +ps_method = '**depsi_matlab:depsi_matlab-settings:general:ps-method**' +psc_model = **depsi_matlab:depsi_matlab-settings:general:psc-model** +ps_model = **depsi_matlab:depsi_matlab-settings:general:ps-model** +final_model = **depsi_matlab:depsi_matlab-settings:general:final-model** +breakpoint = **depsi_matlab:depsi_matlab-settings:general:breakpoint** +breakpoint2 = **depsi_matlab:depsi_matlab-settings:general:breakpoint2** +ens_coh_threshold = **depsi_matlab:depsi_matlab-settings:general:ens-coh-threshold** +varfac_threshold = **depsi_matlab:depsi_matlab-settings:general:varfac-threshold** +detrend_method = '**depsi_matlab:depsi_matlab-settings:general:detrend-method**' +output_format = **depsi_matlab:depsi_matlab-settings:general:output-format** +stc_min_max = [**depsi_matlab:depsi_matlab-settings:general:stc-min-max**] +do_apriori_sidelobe_mask = '**depsi_matlab:depsi_matlab-settings:general:do-apriori-sidelobe-mask**' +do_aposteriori_sidelobe_mask = '**depsi_matlab:depsi_matlab-settings:general:do-aposteriori-sidelobe-mask**' + + +% Geocoding parameters +%---------------------------------------------------------------------- + +master_res = 'slave.res' +ref_height = **depsi_matlab:depsi_matlab-settings:geocoding:ref-height** +demFile = 'dem_radar.raw' + +% Psc parameters +%---------------------------------------------------------------------- + +amplitude_calibration = '**depsi_matlab:depsi_matlab-settings:psc:amplitude-calibration**' +psc_selection_method = '**depsi_matlab:depsi_matlab-settings:psc:psc-selection-method**' +psc_selection_gridsize = **depsi_matlab:depsi_matlab-settings:psc:psc-selection-gridsize** +psc_threshold = **depsi_matlab:depsi_matlab-settings:psc:psc-threshold** +max_arc_length = **depsi_matlab:depsi_matlab-settings:psc:max-arc-length** +network_method = '**depsi_matlab:depsi_matlab-settings:psc:network-method**' +Ncon = **depsi_matlab:depsi_matlab-settings:psc:Ncon** +Nparts = **depsi_matlab:depsi_matlab-settings:psc:Nparts** +Npsc_selections = **depsi_matlab:depsi_matlab-settings:psc:Npsc-selections** +filename_water_mask = **filename_water_mask** +gamma_threshold = **depsi_matlab:depsi_matlab-settings:psc:gamma-threshold** +psc_distribution = '**depsi_matlab:depsi_matlab-settings:psc:psc-distribution**' +weighted_unwrap = '**depsi_matlab:depsi_matlab-settings:psc:weighted-unwrap**' + +% threshold is percentage of slc's that has an amplitude peak +livetime_threshold = **depsi_matlab:depsi_matlab-settings:psc:livetime-threshold**; +% include local near maxima +peak_tolerance = **depsi_matlab:depsi_matlab-settings:psc:peak-tolerance**; + + +% Ps parameters +% ---------------------------------------------------------------------- + +psp_selection_method = '**depsi_matlab:depsi_matlab-settings:psp:psp-selection-method**' +psp_threshold1 = **depsi_matlab:depsi_matlab-settings:psp:psp-threshold1** +psp_threshold2 = **depsi_matlab:depsi_matlab-settings:psp:psp-threshold2** +ps_eval_method = '**depsi_matlab:depsi_matlab-settings:psp:ps-eval-method**' +Namp_disp_bins = **depsi_matlab:depsi_matlab-settings:psp:Namp-disp-bins** +Ndens_iterations = **depsi_matlab:depsi_matlab-settings:psp:Ndens-iterations** +densification_flag = '**depsi_matlab:depsi_matlab-settings:psp:densification-flag**' +ps_area_of_interest = **depsi_matlab:depsi_matlab-settings:psp:ps-area-of-interest** +dens_method = '**depsi_matlab:depsi_matlab-settings:psp:dens-method**' +dens_check = '**depsi_matlab:depsi_matlab-settings:psp:dens-check**' +Nest = **depsi_matlab:depsi_matlab-settings:psp:Nest**; + + +% Stochastic model parameters +% ---------------------------------------------------------------------- + +std_param = [**depsi_matlab:depsi_matlab-settings:stochastic-model:std-param**]; +defo_range = **depsi_matlab:depsi_matlab-settings:stochastic-model:defo-range** +weighting = '**depsi_matlab:depsi_matlab-settings:stochastic-model:weighting**' +ts_atmo_filter = '**depsi_matlab:depsi_matlab-settings:stochastic-model:ts-atmo-filter**' +ts_atmo_filter_length = **depsi_matlab:depsi_matlab-settings:stochastic-model:ts-atmo-filter-length** +ts_noise_filter = '**depsi_matlab:depsi_matlab-settings:stochastic-model:ts-noise-filter**' +ts_noise_filter_length = **depsi_matlab:depsi_matlab-settings:stochastic-model:ts-noise-filter-length** + + +% Bowl parameters +%----------------------------------------------------------------------- + +defo_method = **depsi_matlab:depsi_matlab-settings:bowl:defo-method** +xc0 = **depsi_matlab:depsi_matlab-settings:bowl:xc0** +yc0 = **depsi_matlab:depsi_matlab-settings:bowl:yc0** +zc0 = **depsi_matlab:depsi_matlab-settings:bowl:zc0** +r0 = **depsi_matlab:depsi_matlab-settings:bowl:r0** +r10 = **depsi_matlab:depsi_matlab-settings:bowl:r10** +epoch = **depsi_matlab:depsi_matlab-settings:bowl:epoch** diff --git a/templates/depsi_post/depsi_post.sh b/templates/depsi_post/depsi_post.sh index 9ad2aee6..c25a9932 100644 --- a/templates/depsi_post/depsi_post.sh +++ b/templates/depsi_post/depsi_post.sh @@ -34,10 +34,10 @@ module --ignore-cache load **matlab_module** -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started depsi_post.sh (AoI **depsi:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started depsi_post.sh (AoI **depsi_matlab:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log srun matlab -nodisplay -nosplash -nodesktop -r "run('**depsi_base_directory**/depsi_post.m');exit;" || exit 5 -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished depsi_post.sh (AoI **depsi:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished depsi_post.sh (AoI **depsi_matlab:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/doris/doris_input.xml b/templates/doris/doris_input.xml deleted file mode 100755 index 809d8b1f..00000000 --- a/templates/doris/doris_input.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - **coregistration_directory** - **coregistration_directory**/good_images - vv - **general:shape-file:directory**/**general:shape-file:aoi-name**_shape.shp - **track** - **asc_dsc** - **orbit_directory**/sentinel1 - yes - **coregistration_directory**/dem - **coregistration_directory**/dem - yes - 8 - **start_date** - **end_date** - **master_date** - - - **doris:doris-settings:do-coarse-orbits** - **doris:doris-settings:do-deramp** - **doris:doris-settings:do-reramp** - **doris:doris-settings:do-fake-fine-coreg-bursts** - **doris:doris-settings:do-dac-bursts** - **doris:doris-settings:do-fake-coreg-bursts** - **doris:doris-settings:do-fake-master-resample** - **doris:doris-settings:do-resample** - **doris:doris-settings:do-reramp2** - **doris:doris-settings:do-interferogram** - **doris:doris-settings:do-compref-phase** - **doris:doris-settings:do-compref-dem** - **doris:doris-settings:do-coherence** - **doris:doris-settings:do-esd** - **doris:doris-settings:do-network-esd** - **doris:doris-settings:do-ESD-correct** - **doris:doris-settings:do-combine-master** - **doris:doris-settings:do-combine-slave** - **doris:doris-settings:do-ref-phase** - **doris:doris-settings:do-ref-dem** - **doris:doris-settings:do-phasefilt** - **doris:doris-settings:do-calc-coordinates** - **doris:doris-settings:do-multilooking** - **doris:doris-settings:do-unwrap** - - diff --git a/templates/deinsar/input_files/input.baselines b/templates/doris_v4/input_files/input.baselines similarity index 100% rename from templates/deinsar/input_files/input.baselines rename to templates/doris_v4/input_files/input.baselines diff --git a/templates/deinsar/input_files/input.coarsecorr b/templates/doris_v4/input_files/input.coarsecorr similarity index 100% rename from templates/deinsar/input_files/input.coarsecorr rename to templates/doris_v4/input_files/input.coarsecorr diff --git a/templates/deinsar/input_files/input.coarseorb b/templates/doris_v4/input_files/input.coarseorb similarity index 100% rename from templates/deinsar/input_files/input.coarseorb rename to templates/doris_v4/input_files/input.coarseorb diff --git a/templates/deinsar/input_files/input.coherence b/templates/doris_v4/input_files/input.coherence similarity index 100% rename from templates/deinsar/input_files/input.coherence rename to templates/doris_v4/input_files/input.coherence diff --git a/templates/deinsar/input_files/input.comprefdem b/templates/doris_v4/input_files/input.comprefdem similarity index 100% rename from templates/deinsar/input_files/input.comprefdem rename to templates/doris_v4/input_files/input.comprefdem diff --git a/templates/deinsar/input_files/input.comprefpha b/templates/doris_v4/input_files/input.comprefpha similarity index 100% rename from templates/deinsar/input_files/input.comprefpha rename to templates/doris_v4/input_files/input.comprefpha diff --git a/templates/deinsar/input_files/input.coregpm b/templates/doris_v4/input_files/input.coregpm similarity index 100% rename from templates/deinsar/input_files/input.coregpm rename to templates/doris_v4/input_files/input.coregpm diff --git a/templates/deinsar/input_files/input.crop b/templates/doris_v4/input_files/input.crop similarity index 100% rename from templates/deinsar/input_files/input.crop rename to templates/doris_v4/input_files/input.crop diff --git a/templates/deinsar/input_files/input.dembased b/templates/doris_v4/input_files/input.dembased similarity index 100% rename from templates/deinsar/input_files/input.dembased rename to templates/doris_v4/input_files/input.dembased diff --git a/templates/deinsar/input_files/input.finecoreg b/templates/doris_v4/input_files/input.finecoreg similarity index 100% rename from templates/deinsar/input_files/input.finecoreg rename to templates/doris_v4/input_files/input.finecoreg diff --git a/templates/deinsar/input_files/input.geocoding b/templates/doris_v4/input_files/input.geocoding similarity index 100% rename from templates/deinsar/input_files/input.geocoding rename to templates/doris_v4/input_files/input.geocoding diff --git a/templates/deinsar/input_files/input.interferogram b/templates/doris_v4/input_files/input.interferogram similarity index 100% rename from templates/deinsar/input_files/input.interferogram rename to templates/doris_v4/input_files/input.interferogram diff --git a/templates/deinsar/input_files/input.mtiming b/templates/doris_v4/input_files/input.mtiming similarity index 100% rename from templates/deinsar/input_files/input.mtiming rename to templates/doris_v4/input_files/input.mtiming diff --git a/templates/deinsar/input_files/input.ovs b/templates/doris_v4/input_files/input.ovs similarity index 100% rename from templates/deinsar/input_files/input.ovs rename to templates/doris_v4/input_files/input.ovs diff --git a/templates/deinsar/input_files/input.porbit b/templates/doris_v4/input_files/input.porbit similarity index 100% rename from templates/deinsar/input_files/input.porbit rename to templates/doris_v4/input_files/input.porbit diff --git a/templates/deinsar/input_files/input.readfiles b/templates/doris_v4/input_files/input.readfiles similarity index 100% rename from templates/deinsar/input_files/input.readfiles rename to templates/doris_v4/input_files/input.readfiles diff --git a/templates/deinsar/input_files/input.reltiming b/templates/doris_v4/input_files/input.reltiming similarity index 100% rename from templates/deinsar/input_files/input.reltiming rename to templates/doris_v4/input_files/input.reltiming diff --git a/templates/deinsar/input_files/input.resample b/templates/doris_v4/input_files/input.resample similarity index 100% rename from templates/deinsar/input_files/input.resample rename to templates/doris_v4/input_files/input.resample diff --git a/templates/deinsar/input_files/input.simamp b/templates/doris_v4/input_files/input.simamp similarity index 100% rename from templates/deinsar/input_files/input.simamp rename to templates/doris_v4/input_files/input.simamp diff --git a/templates/deinsar/input_files/input.subtrrefdem b/templates/doris_v4/input_files/input.subtrrefdem similarity index 100% rename from templates/deinsar/input_files/input.subtrrefdem rename to templates/doris_v4/input_files/input.subtrrefdem diff --git a/templates/deinsar/input_files/input.subtrrefpha b/templates/doris_v4/input_files/input.subtrrefpha similarity index 100% rename from templates/deinsar/input_files/input.subtrrefpha rename to templates/doris_v4/input_files/input.subtrrefpha diff --git a/templates/deinsar/run_deinsar.py b/templates/doris_v4/run_doris_v4.py similarity index 78% rename from templates/deinsar/run_deinsar.py rename to templates/doris_v4/run_doris_v4.py index 87f73a8e..88a212e2 100644 --- a/templates/deinsar/run_deinsar.py +++ b/templates/doris_v4/run_doris_v4.py @@ -46,7 +46,7 @@ ) # Parameters -datadir = "**deinsar:input:data-directories**" +datadir = "**doris_v4:input:data-directories**" os.chdir(os.path.dirname(os.path.realpath(__file__))) processdir = os.path.abspath("process") @@ -61,28 +61,28 @@ if "_HH" in polarisations: polarisations[polarisations.index("_HH")] = "" -do_orbit = "**deinsar:deinsar-settings:do-orbit**" -do_crop = "**deinsar:deinsar-settings:do-crop**" -do_tsx_deramp = "**deinsar:deinsar-settings:do-tsx-deramp**" -do_simamp = "**deinsar:deinsar-settings:do-simamp**" -do_mtiming = "**deinsar:deinsar-settings:do-mtiming**" -do_ovs = "**deinsar:deinsar-settings:do-ovs**" -do_choose_master = "**deinsar:deinsar-settings:do-choose-master**" -do_coarseorb = "**deinsar:deinsar-settings:do-coarseorb**" -do_coarsecorr = "**deinsar:deinsar-settings:do-coarsecorr**" -do_finecoreg = "**deinsar:deinsar-settings:finecoreg:do-finecoreg**" -do_reltiming = "**deinsar:deinsar-settings:do-reltiming**" -do_dembased = "**deinsar:deinsar-settings:do-dembased**" -do_coregpm = "**deinsar:deinsar-settings:do-coregpm**" -do_comprefpha = "**deinsar:deinsar-settings:do-comprefpha**" -do_comprefdem = "**deinsar:deinsar-settings:do-comprefdem**" -do_resample = "**deinsar:deinsar-settings:do-resample**" -do_tsx_reramp = "**deinsar:deinsar-settings:do-tsx-reramp**" -do_interferogram = "**deinsar:deinsar-settings:do-interferogram**" -do_subtrrefpha = "**deinsar:deinsar-settings:do-subtrrefpha**" -do_subtrrefdem = "**deinsar:deinsar-settings:do-subtrrefdem**" -do_coherence = "**deinsar:deinsar-settings:do-coherence**" -do_geocoding = "**deinsar:deinsar-settings:do-geocoding**" +do_orbit = "**doris_v4:doris_v4-settings:do-orbit**" +do_crop = "**doris_v4:doris_v4-settings:do-crop**" +do_tsx_deramp = "**doris_v4:doris_v4-settings:do-tsx-deramp**" +do_simamp = "**doris_v4:doris_v4-settings:do-simamp**" +do_mtiming = "**doris_v4:doris_v4-settings:do-mtiming**" +do_ovs = "**doris_v4:doris_v4-settings:do-ovs**" +do_choose_master = "**doris_v4:doris_v4-settings:do-choose-master**" +do_coarseorb = "**doris_v4:doris_v4-settings:do-coarseorb**" +do_coarsecorr = "**doris_v4:doris_v4-settings:do-coarsecorr**" +do_finecoreg = "**doris_v4:doris_v4-settings:finecoreg:do-finecoreg**" +do_reltiming = "**doris_v4:doris_v4-settings:do-reltiming**" +do_dembased = "**doris_v4:doris_v4-settings:do-dembased**" +do_coregpm = "**doris_v4:doris_v4-settings:do-coregpm**" +do_comprefpha = "**doris_v4:doris_v4-settings:do-comprefpha**" +do_comprefdem = "**doris_v4:doris_v4-settings:do-comprefdem**" +do_resample = "**doris_v4:doris_v4-settings:do-resample**" +do_tsx_reramp = "**doris_v4:doris_v4-settings:do-tsx-reramp**" +do_interferogram = "**doris_v4:doris_v4-settings:do-interferogram**" +do_subtrrefpha = "**doris_v4:doris_v4-settings:do-subtrrefpha**" +do_subtrrefdem = "**doris_v4:doris_v4-settings:do-subtrrefdem**" +do_coherence = "**doris_v4:doris_v4-settings:do-coherence**" +do_geocoding = "**doris_v4:doris_v4-settings:do-geocoding**" # Processing steps link_files(datadir, processdir, sensor) diff --git a/templates/deinsar/run_deinsar.sh b/templates/doris_v4/run_doris_v4.sh similarity index 72% rename from templates/deinsar/run_deinsar.sh rename to templates/doris_v4/run_doris_v4.sh index 2145a770..32e435b5 100755 --- a/templates/deinsar/run_deinsar.sh +++ b/templates/doris_v4/run_doris_v4.sh @@ -33,11 +33,11 @@ source /project/caroline/Software/bin/init.sh module load **python2_module** **gdal_module** -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started deinsar.sh (AoI **deinsar:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started run_doris_v4.sh (AoI **doris_v4:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log -export PYTHONPATH=**deinsar:general:deinsar-code-directory** -export PATH=**deinsar:general:doris-v4-code-directory**:$PATH +export PYTHONPATH=**doris_v4:general:deinsar-code-directory** +export PATH=**doris_v4:general:doris_v4-code-directory**:$PATH export SAR_ODR_DIR=**orbit_directory** -python **coregistration_base_directory**/run_deinsar.py || exit 5 +python **coregistration_base_directory**/run_doris_v4.py || exit 5 -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished deinsar.sh (AoI **deinsar:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished run_doris_v4.sh (AoI **doris_v4:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/doris/cleanup-doris-s1-stack.sh b/templates/doris_v5/cleanup-doris-s1-stack.sh similarity index 100% rename from templates/doris/cleanup-doris-s1-stack.sh rename to templates/doris_v5/cleanup-doris-s1-stack.sh diff --git a/templates/doris_v5/doris_input.xml b/templates/doris_v5/doris_input.xml new file mode 100755 index 00000000..6a1418c4 --- /dev/null +++ b/templates/doris_v5/doris_input.xml @@ -0,0 +1,45 @@ + + + **coregistration_directory** + **coregistration_directory**/good_images + vv + **general:shape-file:directory**/**general:shape-file:aoi-name**_shape.shp + **track** + **asc_dsc** + **orbit_directory**/sentinel1 + yes + **coregistration_directory**/dem + **coregistration_directory**/dem + yes + 8 + **start_date** + **end_date** + **master_date** + + + **doris_v5:doris_v5-settings:do-coarse-orbits** + **doris_v5:doris_v5-settings:do-deramp** + **doris_v5:doris_v5-settings:do-reramp** + **doris_v5:doris_v5-settings:do-fake-fine-coreg-bursts** + **doris_v5:doris_v5-settings:do-dac-bursts** + **doris_v5:doris_v5-settings:do-fake-coreg-bursts** + **doris_v5:doris_v5-settings:do-fake-master-resample** + **doris_v5:doris_v5-settings:do-resample** + **doris_v5:doris_v5-settings:do-reramp2** + **doris_v5:doris_v5-settings:do-interferogram** + **doris_v5:doris_v5-settings:do-compref-phase** + **doris_v5:doris_v5-settings:do-compref-dem** + **doris_v5:doris_v5-settings:do-coherence** + **doris_v5:doris_v5-settings:do-esd** + **doris_v5:doris_v5-settings:do-network-esd** + **doris_v5:doris_v5-settings:do-ESD-correct** + **doris_v5:doris_v5-settings:do-combine-master** + **doris_v5:doris_v5-settings:do-combine-slave** + **doris_v5:doris_v5-settings:do-ref-phase** + **doris_v5:doris_v5-settings:do-ref-dem** + **doris_v5:doris_v5-settings:do-phasefilt** + **doris_v5:doris_v5-settings:do-calc-coordinates** + **doris_v5:doris_v5-settings:do-multilooking** + **doris_v5:doris_v5-settings:do-unwrap** + + diff --git a/templates/doris/doris_stack.sh b/templates/doris_v5/doris_stack.sh similarity index 62% rename from templates/doris/doris_stack.sh rename to templates/doris_v5/doris_stack.sh index 7102a64d..6acea3f2 100755 --- a/templates/doris/doris_stack.sh +++ b/templates/doris_v5/doris_stack.sh @@ -35,11 +35,11 @@ module load **python3_module** **gdal_module** source ~/.bashrc source **caroline_virtual_environment_directory**/bin/activate -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started doris_stack.sh (AoI **doris:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started doris_stack.sh (AoI **doris_v5:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log -source_path=**doris:general:code-directory**:**doris:general:code-directory**/doris/doris_stack/functions:**doris:general:code-directory**/doris/doris_stack/main_code +source_path=**doris_v5:general:code-directory**:**doris_v5:general:code-directory**/doris/doris_stack/functions:**doris_v5:general:code-directory**/doris/doris_stack/main_code export PYTHONPATH=$source_path:$PYTHONPATH -export PATH=**doris:general:code-directory**:**doris:general:code-directory**/doris/doris_stack/functions:**doris:general:code-directory**/doris/doris_stack/main_code:/project/caroline/Software/snaphu:$PATH -python3 **doris:general:code-directory**/doris/doris_stack/main_code/doris_main.py -p **coregistration_directory** || exit 5 +export PATH=**doris_v5:general:code-directory**:**doris_v5:general:code-directory**/doris/doris_stack/functions:**doris_v5:general:code-directory**/doris/doris_stack/main_code:/project/caroline/Software/snaphu:$PATH +python3 **doris_v5:general:code-directory**/doris/doris_stack/main_code/doris_main.py -p **coregistration_directory** || exit 5 -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished doris_stack.sh (AoI **doris:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished doris_stack.sh (AoI **doris_v5:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/doris/input_files/input.coarsecorr b/templates/doris_v5/input_files/input.coarsecorr similarity index 100% rename from templates/doris/input_files/input.coarsecorr rename to templates/doris_v5/input_files/input.coarsecorr diff --git a/templates/doris/input_files/input.coarseorb b/templates/doris_v5/input_files/input.coarseorb similarity index 100% rename from templates/doris/input_files/input.coarseorb rename to templates/doris_v5/input_files/input.coarseorb diff --git a/templates/doris/input_files/input.coherence b/templates/doris_v5/input_files/input.coherence similarity index 100% rename from templates/doris/input_files/input.coherence rename to templates/doris_v5/input_files/input.coherence diff --git a/templates/doris/input_files/input.coherence_network b/templates/doris_v5/input_files/input.coherence_network similarity index 100% rename from templates/doris/input_files/input.coherence_network rename to templates/doris_v5/input_files/input.coherence_network diff --git a/templates/doris/input_files/input.comprefdem b/templates/doris_v5/input_files/input.comprefdem similarity index 100% rename from templates/doris/input_files/input.comprefdem rename to templates/doris_v5/input_files/input.comprefdem diff --git a/templates/doris/input_files/input.comprefpha b/templates/doris_v5/input_files/input.comprefpha similarity index 100% rename from templates/doris/input_files/input.comprefpha rename to templates/doris_v5/input_files/input.comprefpha diff --git a/templates/doris/input_files/input.coregpm b/templates/doris_v5/input_files/input.coregpm similarity index 100% rename from templates/doris/input_files/input.coregpm rename to templates/doris_v5/input_files/input.coregpm diff --git a/templates/doris/input_files/input.dembased b/templates/doris_v5/input_files/input.dembased similarity index 100% rename from templates/doris/input_files/input.dembased rename to templates/doris_v5/input_files/input.dembased diff --git a/templates/doris/input_files/input.finecoreg b/templates/doris_v5/input_files/input.finecoreg similarity index 100% rename from templates/doris/input_files/input.finecoreg rename to templates/doris_v5/input_files/input.finecoreg diff --git a/templates/doris/input_files/input.geocode b/templates/doris_v5/input_files/input.geocode similarity index 100% rename from templates/doris/input_files/input.geocode rename to templates/doris_v5/input_files/input.geocode diff --git a/templates/doris/input_files/input.interferogram b/templates/doris_v5/input_files/input.interferogram similarity index 100% rename from templates/doris/input_files/input.interferogram rename to templates/doris_v5/input_files/input.interferogram diff --git a/templates/doris/input_files/input.phasefilt b/templates/doris_v5/input_files/input.phasefilt similarity index 100% rename from templates/doris/input_files/input.phasefilt rename to templates/doris_v5/input_files/input.phasefilt diff --git a/templates/doris/input_files/input.resample b/templates/doris_v5/input_files/input.resample similarity index 100% rename from templates/doris/input_files/input.resample rename to templates/doris_v5/input_files/input.resample diff --git a/templates/doris/input_files/input.subtrrefdem b/templates/doris_v5/input_files/input.subtrrefdem similarity index 100% rename from templates/doris/input_files/input.subtrrefdem rename to templates/doris_v5/input_files/input.subtrrefdem diff --git a/templates/doris/input_files/input.subtrrefpha b/templates/doris_v5/input_files/input.subtrrefpha similarity index 100% rename from templates/doris/input_files/input.subtrrefpha rename to templates/doris_v5/input_files/input.subtrrefpha diff --git a/templates/doris/input_files/input.unwrap b/templates/doris_v5/input_files/input.unwrap similarity index 100% rename from templates/doris/input_files/input.unwrap rename to templates/doris_v5/input_files/input.unwrap diff --git a/templates/stm-generation/generate-stm.py b/templates/generate-partitioned-stm/generate-partitioned-stm.py similarity index 67% rename from templates/stm-generation/generate-stm.py rename to templates/generate-partitioned-stm/generate-partitioned-stm.py index e82f0549..a08bfc97 100644 --- a/templates/stm-generation/generate-stm.py +++ b/templates/generate-partitioned-stm/generate-partitioned-stm.py @@ -29,64 +29,82 @@ # ############## INPUT VARIABLES -slc_path = "**crop_to_zarr_directory**/**crop_to_zarr_output_name**.zarr" +slc_path = "**reduce_slc_python_directory**/**reduce_slc_python_output_name**.zarr" # STM save path stm_save_path = "**stm_output_directory**/**stm_output_name**.zarr" # PS Selection based on initialization -ps_selection_mode = "**stm_generation:stm_generation-settings:ps-selection:mode**" +ps_selection_mode = "**generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:mode**" if ps_selection_mode == "initialization": start_date_ps_selection = ( - "**stm_generation:stm_generation-settings:ps-selection:initialization-mode-settings:start-date**".replace( + "**generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:init-settings:start-date**".replace( "-", "" ) ) initialization_length = int( - "**stm_generation:stm_generation-settings:ps-selection:initialization-mode-settings:initialization-length**" + "**generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:init-settings:init-length**" ) else: start_date_ps_selection = None initialization_length = None # Recalibrated NAD and NMAD settings -increment_mode = "**stm_generation:stm_generation-settings:incremental-statistics:increment-mode**" +increment_mode = "**generate_partitioned_stm:generate_partitioned_stm-settings:incremental-statistics:increment-mode**" recalibration_jump_size = eval( - "**stm_generation:stm_generation-settings:incremental-statistics:recalibration-jump-size**" + "**generate_partitioned_stm:generate_partitioned_stm-settings:incremental-statistics:recal-jump-size**" ) # PS selection method -ps_selection_method = "**stm_generation:stm_generation-settings:ps-selection:method**" -threshold = eval("**stm_generation:stm_generation-settings:ps-selection:threshold**") +ps_selection_method = "**generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:method**" +threshold = eval("**generate_partitioned_stm:generate_partitioned_stm-settings:ps-selection:threshold**") chunks_ps_selection = 1000 # Input variables for the outlier detection do_ps_outlier_detection = ( - True if "**stm_generation:stm_generation-settings:outlier-detection:do-outlier-detection**" == "1" else False + True + if "**generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:do-outlier-detection**" == "1" + else False +) +ps_window_size_outliers = int( + "**generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:window-size**" ) -ps_window_size_outliers = int("**stm_generation:stm_generation-settings:outlier-detection:window-size**") ps_outlier_detection_db = ( - True if "**stm_generation:stm_generation-settings:outlier-detection:db-mode**" == "1" else False + True if "**generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:db-mode**" == "1" else False ) -ps_n_sigma_outliers = int("**stm_generation:stm_generation-settings:outlier-detection:n-sigma**") +ps_n_sigma_outliers = int("**generate_partitioned_stm:generate_partitioned_stm-settings:outlier-detection:n-sigma**") # Input variables for the partitioning -do_ps_partitioning = True if "**stm_generation:stm_generation-settings:partitioning:do-partitioning**" == "1" else False -ps_partitioning_search_method = "**stm_generation:stm_generation-settings:partitioning:search-method**" -ps_partitioning_cost_function = "**stm_generation:stm_generation-settings:partitioning:cost-function**" -ps_db_partitioning = True if "**stm_generation:stm_generation-settings:partitioning:db-mode**" == "1" else False -ps_min_obs_partition = int("**stm_generation:stm_generation-settings:partitioning:min-partition-length**") +do_ps_partitioning = ( + True + if "**generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:do-partitioning**" == "1" + else False +) +ps_partitioning_search_method = ( + "**generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:search-method**" +) +ps_partitioning_cost_function = ( + "**generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:cost-function**" +) +ps_db_partitioning = ( + True if "**generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:db-mode**" == "1" else False +) +ps_min_obs_partition = int( + "**generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:min-partition-length**" +) partitioning_output_layers = tuple( - eval("**stm_generation:stm_generation-settings:partitioning:undifferenced-output-layers**") + eval("**generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:undifferenced-output-lyrs**") ) partitioning_sd_output_layers = tuple( - eval("**stm_generation:stm_generation-settings:partitioning:single-difference-output-layers**") + eval("**generate_partitioned_stm:generate_partitioned_stm-settings:partitioning:single-difference-output-lyrs**") ) # Compute temporal differences -ps_mother_epoch_sd = "**stm_generation:stm_generation-settings:single-differences:mother**".replace("-", "") +ps_mother_epoch_sd = "**generate_partitioned_stm:generate_partitioned_stm-settings:single-differences:mother**".replace( + "-", "" +) -projection = "**stm_generation:stm_generation-settings:extra-projection**" +projection = "**generate_partitioned_stm:generate_partitioned_stm-settings:extra-projection**" do_projection = False if projection not in ["", "None"]: do_projection = True @@ -105,7 +123,7 @@ def get_free_port(): return freesock -N_WORKERS = JOB_DEFINITIONS["jobs"]["stm_generation"]["bash-file"]["bash-file-slurm-cluster"][ +N_WORKERS = JOB_DEFINITIONS["jobs"]["generate_partitioned_stm"]["bash-file"]["bash-file-slurm-cluster"][ "slurm-cluster-n-workers" ] # Manual input: number of workers to spin-up FREE_SOCKET = get_free_port() # Get a free port @@ -115,7 +133,7 @@ def get_free_port(): cores=4, # Number of cores per worker memory="30 GB", # Total amount of memory per worker processes=1, # Number of Python processes per worker - walltime=JOB_DEFINITIONS["jobs"]["stm_generation"]["bash-file"]["bash-file-slurm-cluster"][ + walltime=JOB_DEFINITIONS["jobs"]["generate_partitioned_stm"]["bash-file"]["bash-file-slurm-cluster"][ "slurm-cluster-worker-time" ], # Reserve each worker for X hour scheduler_options={"dashboard_address": f":{FREE_SOCKET}"}, # Host Dashboard in a free socket diff --git a/templates/generate-partitioned-stm/generate-partitioned-stm.sh b/templates/generate-partitioned-stm/generate-partitioned-stm.sh new file mode 100644 index 00000000..be090e3d --- /dev/null +++ b/templates/generate-partitioned-stm/generate-partitioned-stm.sh @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --time=24:00:00 +#SBATCH --cpus-per-task=4 +#SBATCH --partition=normal + +source ~/.bashrc + +source /etc/profile.d/modules.sh +source /project/caroline/Software/bin/init.sh +module load **python3_module** **gdal_module** +source **caroline_virtual_environment_directory**/bin/activate + +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started generate_partitioned_stm.sh (AoI **generate_partitioned_stm:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log + +export PATH="**reduce_slc_python:general:depsi_group-code-directory**:$PATH" +export PYTHONPATH="**reduce_slc_python:general:depsi_group-code-directory**:$PYTHONPATH" + +python3 generate-partitioned-stm.py || exit 5 + +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished generate_partitioned_stm.sh (AoI **generate_partitioned_stm:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/znap-to-raw/master.res b/templates/merge-to-stack-matlab/master.res similarity index 100% rename from templates/znap-to-raw/master.res rename to templates/merge-to-stack-matlab/master.res diff --git a/templates/znap-to-raw/znap-to-raw.py b/templates/merge-to-stack-matlab/merge-to-stack-matlab.py similarity index 97% rename from templates/znap-to-raw/znap-to-raw.py rename to templates/merge-to-stack-matlab/merge-to-stack-matlab.py index ac79b8e4..790eb21f 100644 --- a/templates/znap-to-raw/znap-to-raw.py +++ b/templates/merge-to-stack-matlab/merge-to-stack-matlab.py @@ -37,7 +37,7 @@ def get_free_port(): # Option 1: Initiate a new SLURMCluster # Uncomment the following part to setup a new Dask SLURMCluster -N_WORKERS = JOB_DEFINITIONS["jobs"]["znap_to_raw"]["bash-file"]["bash-file-slurm-cluster"][ +N_WORKERS = JOB_DEFINITIONS["jobs"]["merge_to_stack_matlab"]["bash-file"]["bash-file-slurm-cluster"][ "slurm-cluster-n-workers" ] # Manual input: number of workers to spin-up FREE_SOCKET = get_free_port() # Get a free port @@ -47,7 +47,7 @@ def get_free_port(): cores=4, # Number of cores per worker memory="30 GB", # Total amount of memory per worker processes=1, # Number of Python processes per worker - walltime=JOB_DEFINITIONS["jobs"]["znap_to_raw"]["bash-file"]["bash-file-slurm-cluster"][ + walltime=JOB_DEFINITIONS["jobs"]["merge_to_stack_matlab"]["bash-file"]["bash-file-slurm-cluster"][ "slurm-cluster-worker-time" ], # Reserve each worker for X hour scheduler_options={ @@ -141,7 +141,7 @@ def get_free_port(): write_run_file( save_path=f"{write_path}/{filename}", - template_path=f"{CONFIG['CAROLINE_INSTALL_DIRECTORY']}/templates/znap-to-raw/master.res", + template_path=f"{CONFIG['CAROLINE_INSTALL_DIRECTORY']}/templates/merge-to-stack-matlab/master.res", asc_dsc=None, track=None, parameter_file=None, diff --git a/templates/merge-to-stack-matlab/merge-to-stack-matlab.sh b/templates/merge-to-stack-matlab/merge-to-stack-matlab.sh new file mode 100644 index 00000000..2f0a16fe --- /dev/null +++ b/templates/merge-to-stack-matlab/merge-to-stack-matlab.sh @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --time=24:00:00 +#SBATCH --cpus-per-task=4 +#SBATCH --partition=normal + +source ~/.bashrc + +source /etc/profile.d/modules.sh +source /project/caroline/Software/bin/init.sh +module load **python3_module** **gdal_module** +source **caroline_virtual_environment_directory**/bin/activate + +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started merge-to-stack-matlab.sh (AoI **merge_to_stack_matlab:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log + +export PATH="**merge_to_stack_matlab:general:depsi_group-code-directory**:$PATH" +export PYTHONPATH="**merge_to_stack_matlab:general:depsi_group-code-directory**:$PYTHONPATH" + +python3 merge-to-stack-matlab.py || exit 5 + +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished merge-to-stack-matlab.sh (AoI **merge_to_stack_matlab:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/znap-to-zarr/znap-to-zarr.py b/templates/merge-to-stack-python/merge-to-stack-python.py similarity index 90% rename from templates/znap-to-zarr/znap-to-zarr.py rename to templates/merge-to-stack-python/merge-to-stack-python.py index 80b7a1c7..e7d2b1f7 100644 --- a/templates/znap-to-zarr/znap-to-zarr.py +++ b/templates/merge-to-stack-python/merge-to-stack-python.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) znap_output_path = "**snap-output-path**" -zarr_output_path = "**znap_to_zarr_output_filename**.zarr" +zarr_output_path = "**merge_to_stack_python_output_filename**.zarr" aoi_path = "**general:shape-file:directory**/**general:shape-file:aoi-name**_shape.shp" @@ -33,7 +33,7 @@ def get_free_port(): # Option 1: Initiate a new SLURMCluster # Uncomment the following part to setup a new Dask SLURMCluster -N_WORKERS = JOB_DEFINITIONS["jobs"]["znap_to_zarr"]["bash-file"]["bash-file-slurm-cluster"][ +N_WORKERS = JOB_DEFINITIONS["jobs"]["merge_to_stack_python"]["bash-file"]["bash-file-slurm-cluster"][ "slurm-cluster-n-workers" ] # Manual input: number of workers to spin-up FREE_SOCKET = get_free_port() # Get a free port @@ -43,7 +43,7 @@ def get_free_port(): cores=4, # Number of cores per worker memory="30 GB", # Total amount of memory per worker processes=1, # Number of Python processes per worker - walltime=JOB_DEFINITIONS["jobs"]["znap_to_zarr"]["bash-file"]["bash-file-slurm-cluster"][ + walltime=JOB_DEFINITIONS["jobs"]["merge_to_stack_python"]["bash-file"]["bash-file-slurm-cluster"][ "slurm-cluster-worker-time" ], # Reserve each worker for X hour scheduler_options={ diff --git a/templates/merge-to-stack-python/merge-to-stack-python.sh b/templates/merge-to-stack-python/merge-to-stack-python.sh new file mode 100644 index 00000000..1b69785a --- /dev/null +++ b/templates/merge-to-stack-python/merge-to-stack-python.sh @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --time=24:00:00 +#SBATCH --cpus-per-task=4 +#SBATCH --partition=normal + +source ~/.bashrc + +source /etc/profile.d/modules.sh +source /project/caroline/Software/bin/init.sh +module load **python3_module** **gdal_module** +source **caroline_virtual_environment_directory**/bin/activate + +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started merge-to-stack-python.sh (AoI **merge_to_stack_python:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log + +export PATH="**merge_to_stack_python:general:depsi_group-code-directory**:$PATH" +export PYTHONPATH="**merge_to_stack_python:general:depsi_group-code-directory**:$PYTHONPATH" + +python3 merge-to-stack-python.py || exit 5 + +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished merge-to-stack-python.sh (AoI **merge_to_stack_python:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/crop-to-raw/crop-to-raw.sh b/templates/reduce-slc-matlab/reduce-slc-matlab.sh similarity index 75% rename from templates/crop-to-raw/crop-to-raw.sh rename to templates/reduce-slc-matlab/reduce-slc-matlab.sh index 49498a7e..2789e972 100644 --- a/templates/crop-to-raw/crop-to-raw.sh +++ b/templates/reduce-slc-matlab/reduce-slc-matlab.sh @@ -30,8 +30,8 @@ module --ignore-cache load **matlab_module** -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started crop-to-raw.sh (AoI **crop_to_raw:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started reduce-slc-matlab.sh (AoI **reduce_slc_matlab:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log -srun matlab -nodisplay -nosplash -nodesktop -r "run('**crop_base_directory**/crop_to_raw.m');exit;" || exit 5 +srun matlab -nodisplay -nosplash -nodesktop -r "run('**crop_base_directory**/reduce_slc_matlab.m');exit;" || exit 5 -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished crop-to-raw.sh (AoI **crop_to_raw:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished reduce-slc-matlab.sh (AoI **reduce_slc_matlab:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/crop-to-raw/crop-to-raw.m b/templates/reduce-slc-matlab/reduce_slc_matlab.m similarity index 100% rename from templates/crop-to-raw/crop-to-raw.m rename to templates/reduce-slc-matlab/reduce_slc_matlab.m diff --git a/templates/crop-to-zarr/crop-to-zarr.py b/templates/reduce-slc-python/reduce-slc-python.py similarity index 94% rename from templates/crop-to-zarr/crop-to-zarr.py rename to templates/reduce-slc-python/reduce-slc-python.py index f717a0f7..b6f8dbe2 100644 --- a/templates/crop-to-zarr/crop-to-zarr.py +++ b/templates/reduce-slc-python/reduce-slc-python.py @@ -61,7 +61,7 @@ def get_free_port(): # Option 1: Initiate a new SLURMCluster # Uncomment the following part to setup a new Dask SLURMCluster -N_WORKERS = JOB_DEFINITIONS["jobs"]["crop_to_zarr"]["bash-file"]["bash-file-slurm-cluster"][ +N_WORKERS = JOB_DEFINITIONS["jobs"]["reduce_slc_python"]["bash-file"]["bash-file-slurm-cluster"][ "slurm-cluster-n-workers" ] # Manual input: number of workers to spin-up FREE_SOCKET = get_free_port() # Get a free port @@ -71,7 +71,7 @@ def get_free_port(): cores=4, # Number of cores per worker memory="30 GB", # Total amount of memory per worker processes=1, # Number of Python processes per worker - walltime=JOB_DEFINITIONS["jobs"]["crop_to_zarr"]["bash-file"]["bash-file-slurm-cluster"][ + walltime=JOB_DEFINITIONS["jobs"]["reduce_slc_python"]["bash-file"]["bash-file-slurm-cluster"][ "slurm-cluster-worker-time" ], # Reserve each worker for X hour scheduler_options={ @@ -209,10 +209,10 @@ def get_free_port(): ) # Rechunk and write as zarr slcs_output = slcs_output.chunk(writing_chunks) - if not os.path.exists("**crop_to_zarr_output_filename**.zarr"): - slcs_output.to_zarr("**crop_to_zarr_output_filename**.zarr", mode="w") + if not os.path.exists("**reduce_slc_python_output_filename**.zarr"): + slcs_output.to_zarr("**reduce_slc_python_output_filename**.zarr", mode="w") else: - slcs_output.to_zarr("**crop_to_zarr_output_filename**.zarr", mode="w") + slcs_output.to_zarr("**reduce_slc_python_output_filename**.zarr", mode="w") logger.info("Finishing... Closing client.") # Close the client when finishing diff --git a/templates/reduce-slc-python/reduce-slc-python.sh b/templates/reduce-slc-python/reduce-slc-python.sh new file mode 100644 index 00000000..aff280b7 --- /dev/null +++ b/templates/reduce-slc-python/reduce-slc-python.sh @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --time=24:00:00 +#SBATCH --cpus-per-task=4 +#SBATCH --partition=normal + +source ~/.bashrc + +source /etc/profile.d/modules.sh +source /project/caroline/Software/bin/init.sh +module load **python3_module** **gdal_module** +source **caroline_virtual_environment_directory**/bin/activate + +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started reduce-slc-python.sh (AoI **reduce_slc_python:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log + +export PATH="**reduce_slc_python:general:depsi_group-code-directory**:$PATH" +export PYTHONPATH="**reduce_slc_python:general:depsi_group-code-directory**:$PYTHONPATH" + +python3 reduce-slc-python.py || exit 5 + +echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished reduce-slc-python.sh (AoI **reduce_slc_python:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/stm-generation/generate-stm.sh b/templates/stm-generation/generate-stm.sh deleted file mode 100644 index 67dfd449..00000000 --- a/templates/stm-generation/generate-stm.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --time=24:00:00 -#SBATCH --cpus-per-task=4 -#SBATCH --partition=normal - -source ~/.bashrc - -source /etc/profile.d/modules.sh -source /project/caroline/Software/bin/init.sh -module load **python3_module** **gdal_module** -source **caroline_virtual_environment_directory**/bin/activate - -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started generate-stm.sh (AoI **stm_generation:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log - -export PATH="**crop_to_zarr:general:crop_to_zarr-code-directory**:$PATH" -export PYTHONPATH="**crop_to_zarr:general:crop_to_zarr-code-directory**:$PYTHONPATH" - -python3 generate-stm.py || exit 5 - -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished generate-stm.sh (AoI **stm_generation:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/znap-to-raw/znap-to-raw.sh b/templates/znap-to-raw/znap-to-raw.sh deleted file mode 100644 index a132aac7..00000000 --- a/templates/znap-to-raw/znap-to-raw.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --time=24:00:00 -#SBATCH --cpus-per-task=4 -#SBATCH --partition=normal - -source ~/.bashrc - -source /etc/profile.d/modules.sh -source /project/caroline/Software/bin/init.sh -module load **python3_module** **gdal_module** -source **caroline_virtual_environment_directory**/bin/activate - -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started znap-to-raw.sh (AoI **znap_to_raw:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log - -export PATH="**znap_to_raw:general:znap_to_raw-code-directory**:$PATH" -export PYTHONPATH="**znap_to_raw:general:znap_to_raw-code-directory**:$PYTHONPATH" - -python3 znap-to-raw.py || exit 5 - -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished znap-to-raw.sh (AoI **znap_to_raw:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log diff --git a/templates/znap-to-zarr/znap-to-zarr.sh b/templates/znap-to-zarr/znap-to-zarr.sh deleted file mode 100644 index f76cf90c..00000000 --- a/templates/znap-to-zarr/znap-to-zarr.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --time=24:00:00 -#SBATCH --cpus-per-task=4 -#SBATCH --partition=normal - -source ~/.bashrc - -source /etc/profile.d/modules.sh -source /project/caroline/Software/bin/init.sh -module load **python3_module** **gdal_module** -source **caroline_virtual_environment_directory**/bin/activate - -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has started znap-to-zarr.sh (AoI **znap_to_zarr:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log - -export PATH="**znap_to_zarr:general:znap_to_zarr-code-directory**:$PATH" -export PYTHONPATH="**znap_to_zarr:general:znap_to_zarr-code-directory**:$PYTHONPATH" - -python3 znap-to-zarr.py || exit 5 - -echo "$(date '+%Y-%m-%dT%H:%M:%S'): $(whoami) in $(pwd) has finished znap-to-zarr.sh (AoI **znap_to_zarr:general:AoI-name**, track **track**) with slurm-ID $SLURM_JOB_ID)" >> **caroline_work_directory**/submitted_jobs.log