From 0c31644f849a83c808b641a91c748207e1ae9b47 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Thu, 16 Nov 2023 05:57:16 +0000 Subject: [PATCH 01/14] initial work --- nci_environment/build_environment_module.py | 34 ++++++++-------- nci_environment/dea/modulefile.template | 44 +++++++++++++++------ nci_environment/dea/modulespec.yaml | 19 +++++---- 3 files changed, 59 insertions(+), 38 deletions(-) diff --git a/nci_environment/build_environment_module.py b/nci_environment/build_environment_module.py index 98750fe0..e2c0c404 100755 --- a/nci_environment/build_environment_module.py +++ b/nci_environment/build_environment_module.py @@ -9,11 +9,11 @@ - (opt) Conda environment to create - (opt) Pip style requirements.txt to install to a directory -It requires python 3.8+ and pyyaml. +It requires python 3.9+ and pyyaml. Use a qsub interactive copyq job on raijin with sufficient memory to run the following commands at the NCI: New DEA-Env Module $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 + $ module load python3/3.9.2 # if pyyaml is not installed in gadi $ pip install PyYAML --user @@ -22,7 +22,7 @@ New DEA Module $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 + $ module load python3/3.9.2 $ # Building a new DEA Module $ python3 build_environment_module.py dea/modulespec.yaml @@ -262,8 +262,8 @@ def copy_and_fill_templates(template_tasks, variables): src = Path(task["src"]) dest = Path(task["dest"]) - LOG.info("Copy and fill dea-env modulefile %s in %s", src, dest) - # Write the module file template to modulefiles/dea-env directory + LOG.info("Copy and fill dea modulefile %s in %s", src, dest) + # Write the module file template to modulefiles/dea directory write_template(src, variables, dest) if "chmod" in task: @@ -361,18 +361,18 @@ def run_final_commands_on_module(commands, module_path): run_command(cmd) -def include_stable_module_dep_versions(config): - """ - Include stable module dependency versions +# def include_stable_module_dep_versions(config): +# """ +# Include stable module dependency versions - :param config: Dictionary of configuration variables - :return: None - """ - stable_module_deps = config.get("stable_module_deps", []) - for dep in stable_module_deps: - default_version = find_default_version(dep) - dep = dep.replace("-", "_") - config["variables"][f"fixed_{dep}"] = default_version +# :param config: Dictionary of configuration variables +# :return: None +# """ +# stable_module_deps = config.get("stable_module_deps", []) +# for dep in stable_module_deps: +# default_version = find_default_version(dep) +# dep = dep.replace("-", "_") +# config["variables"][f"fixed_{dep}"] = default_version def main(config_path): @@ -394,7 +394,7 @@ def main(config_path): if "module_version" not in variables: variables["module_version"] = date() include_templated_vars(config) - include_stable_module_dep_versions(config) + # include_stable_module_dep_versions(config) pre_check(config) prep(config_path) diff --git a/nci_environment/dea/modulefile.template b/nci_environment/dea/modulefile.template index d451542e..abf61993 100644 --- a/nci_environment/dea/modulefile.template +++ b/nci_environment/dea/modulefile.template @@ -16,7 +16,35 @@ set base ${module_path} module-whatis "${module_description} ${module_version}" -module load ${fixed_dea_env} +# Enable segfault tracebacks in py3. https://docs.python.org/3/library/faulthandler.html +setenv PYTHONFAULTHANDLER 1 + +# Set PYTHONUSERBASE based on the version of dea module. +# This allows users to install python packages with "pip install --user ", +setenv PYTHONUSERBASE ~/.dea-sandbox/${module_name}/${module_version}/local +setenv GDAL_DATA ${module_path}/share/gdal +setenv PROJ_LIB ${module_path}/share/proj +setenv PROJ_NETWORK ON +setenv UDUNITS2_XML_PATH ${module_path}/share/udunits/udunits2.xml +setenv GSETTINGS_SCHEMA_DIR ${module_path}/share/glib-2.0/schemas + +if {[module-info mode load] && [info exists env(PYTHONPATH)] && $$env(PYTHONPATH) != ""} { + puts stderr "Warning: ${module_name}/${module_version} exists in the python env ($$env(PYTHONPATH))" + puts stderr "Unload all python modules, if you experience any issues." +} + +prepend-path PYTHONPATH ${module_path}/share/qgis/python + +# Remove duplicate entries for module path dir and prepend again +remove-path PATH ${module_path}/bin +prepend-path PATH ${module_path}/bin + +# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH +# variables to point to a directory based on the Environment Module version which is loaded so that extra +# packages must be re-installed when a new dea module is released +# Remove duplicate entries for HOME dir and prepend again +remove-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin +prepend-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin setenv DATACUBE_CONFIG_PATH ${module_path}/datacube.conf setenv LC_ALL en_AU.utf8 @@ -26,19 +54,9 @@ if {[module-info mode load] && [is-loaded $$name/$$version]} { puts stderr "Warning: $$name/$$version exists in the python env ($$env(PYTHONPATH))" } -# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH -# variables to point to a directory based on the Environment Module version which is loaded so that extra -# packages must be re-installed when a new dea-env module is released -prepend-path PATH ${module_path}/bin -prepend-path PYTHONPATH ${python_path} - -# Remove duplicate entries for HOME dir and prepend at the top -remove-path PATH ~/.dea-sandbox/${fixed_dea_env}/local/bin -prepend-path PATH ~/.dea-sandbox/${fixed_dea_env}/local/bin - # Remove duplicate entries for HOME dir and prepend at the top -remove-path PYTHONPATH ~/.dea-sandbox/${fixed_dea_env}/local/lib/python3.6/site-packages -prepend-path PYTHONPATH ~/.dea-sandbox/${fixed_dea_env}/local/lib/python3.6/site-packages +remove-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.9/site-packages +prepend-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.9/site-packages ############################################################# diff --git a/nci_environment/dea/modulespec.yaml b/nci_environment/dea/modulespec.yaml index 42919bab..09ac26f8 100644 --- a/nci_environment/dea/modulespec.yaml +++ b/nci_environment/dea/modulespec.yaml @@ -1,23 +1,25 @@ variables: module_name: dea module_description: DEA tools for the NCI - modules_base: "/g/data/v10/public/modules" + modules_dir: "/g/data/v10/public/modules" + conda_path: "/g/data/v10/private/mambaforge/bin/mamba" dbhost: dea-db.nci.org.au dbport: 6432 - python_version: 3.8 + python_version: 3.9 templated_variables: - module_path: "{modules_base}/{module_name}/{module_version}" - python_path: "{modules_base}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" + module_path: "{modules_dir}/{module_name}/{module_version}" + python_path: "{modules_dir}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" dea_module: "{module_name}/{module_version}" -stable_module_deps: -- dea-env +install_conda_packages: ../../docker/env.yaml install_pip_packages: - pip_cmd: "module load {fixed_dea_env}; pip install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; pip install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt" + pip_cmd: "module load python3/3.9.2; pip install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; pip install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt" copy_files: +- src: ../../docker/env.yaml + dest: "{modules_dir}/{module_name}/{module_version}/environment.yaml" - src: requirements.txt dest: "{module_path}/requirements.txt" - src: datacube.conf @@ -39,4 +41,5 @@ wget_files: template_files: - src: modulefile.template - dest: "{modules_base}/modulefiles/{module_name}/{module_version}" + dest: "{modules_dir}/modulefiles/{module_name}/{module_version}" + chmod: 0o444 \ No newline at end of file From 4f28d76af5b09c949a10dbe448ba541a7c19277e Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Wed, 22 Nov 2023 03:52:25 +0000 Subject: [PATCH 02/14] remove need for dea-env module, move some pip packages to conda env --- docker/env.yaml | 4 ++ docker/requirements.txt | 4 -- nci_environment/README.md | 49 +++++++-------------- nci_environment/build_environment_module.py | 27 +----------- nci_environment/dea/modulefile.template | 8 +++- nci_environment/dea/modulespec.yaml | 6 ++- nci_environment/dea/requirements.txt | 4 -- 7 files changed, 32 insertions(+), 70 deletions(-) diff --git a/docker/env.yaml b/docker/env.yaml index 88a2b191..35e75823 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -55,6 +55,7 @@ dependencies: - distributed - docutils - ephem + - eodatasets3 - fiona - Flask - Flask-Babel @@ -92,6 +93,9 @@ dependencies: - nodejs - numexpr - numpy + - odc-algo + - odc-dscache + - odc-io - ordered-set - packaging - pandas diff --git a/docker/requirements.txt b/docker/requirements.txt index 698318df..1a2c2f6c 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -9,7 +9,6 @@ jupyter-nbextensions-configurator # ODC/DEA: these are installed in builder stage otps -eodatasets3 # Dale's s2cloudmask # https://github.com/daleroberts/s2cloudmask @@ -19,10 +18,7 @@ opencv-python-headless opencv-contrib-python-headless datacube[performance,s3] -odc-algo odc-cloud[ASYNC] -odc-dscache -odc-io odc-stac odc-stats[ows] odc-ui diff --git a/nci_environment/README.md b/nci_environment/README.md index 743e859f..cf334ab4 100644 --- a/nci_environment/README.md +++ b/nci_environment/README.md @@ -2,25 +2,22 @@ These scripts are used to update and deploy DEA modules on NCI. -There are two modules, with date-based version numbers: +The *dea* module, with a date-based version number, contains third party dependencies +of all of the DEA code, installed via a `conda` environment. This `conda` environment is the same +as the one for dea-sandbox. - 1. The Python Environment module *dea-env* - - * This contains third party dependencies of all of the DEA code, installed via - a `conda` environment. - - 2. A *dea* module, which depends on the _environment module_: +Additionally, the environment includes: * [Open Data Cube Core](https://github.com/opendatacube/datacube-core/) - * [EO Datasets](https://github.com/GeoscienceAustralia/eo-datasets/) - * [Digital Earth AU](https://github.com/GeoscienceAustralia/digitalearthau/) * [Data Cube Stats](https://github.com/GeoscienceAustralia/datacube-stats/) * [Fractional Cover](https://github.com/GeoscienceAustralia/fc/) + * [Water Observation From Space](https://github.com/GeoscienceAustralia/wofs) + * Creates users accounts in the Production Database the first time it is loaded by a user. @@ -34,14 +31,11 @@ There are two modules, with date-based version numbers: This will load the latest version of `dea/` module. -It will also load `dea-env/` which contains all of the software -dependencies for using DEA. - ## Notes -Loading these module might conflict with other python modules you have loaded. +Loading this module might conflict with other python modules you have loaded. -The `dea-env` module will prevent conflicts with locally installed python packages by +The `dea` module will prevent conflicts with locally installed python packages by changing `PYTHONUSERBASE` for each release; pip install --user @@ -54,33 +48,23 @@ It includes a config file, which it specifies by setting the # Maintainer Instructions -Only run these scripts from Raijin. We've seen filesystem sync issues when +Only run these scripts from Gadi. We've seen filesystem sync issues when run from VDI. - module load python3/3.8.5 + module load python3/3.10.4 pip3 install --user pyyaml jinja2 -## Building a new _Environment Module_ - -It requires python 3.8+ and pyyaml. Run the following on raijin at the NCI: - - $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 - $ ./build_environment_module.py dea-env/modulespec.yaml - -This will build a new environment module for today. - -The module version number is the current date in format YYYYMMDD, as it is a snapshot -of all of our pip/conda dependencies on that date. - ## Building a new _DEA Module_ -A DEA module will specify one exact environment module. +It requires python 3.10+ and pyyaml. Run the following on gadi at the NCI: $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 + $ module load python3/3.10.4 $ ./build_environment_module.py dea/modulespec.yaml +The module version number is the current date in format YYYYMMDD, as it is a snapshot +of all of our pip/conda dependencies on that date. + ## Updating the Default Version Once a module has been tested and approved, it can be made the default. @@ -108,14 +92,13 @@ Eg. For `dea` this is: `/g/data/v10/public/modules/modulefiles/dea/.version` ## Setup Copy the 3 lines below and modify the VERSION value - to the dea and dea-env module version you would + to the dea module version you would like the tests to be run on. Paste them in a brand new shell session/terminal VERSION="20230710" module use /g/data/v10/public/modules/modulefiles module load dea/$VERSION - module load dea-env/$VERSION ## Execution On gadi, just run the tests with in this fashion: diff --git a/nci_environment/build_environment_module.py b/nci_environment/build_environment_module.py index e2c0c404..c70cafc8 100755 --- a/nci_environment/build_environment_module.py +++ b/nci_environment/build_environment_module.py @@ -9,21 +9,13 @@ - (opt) Conda environment to create - (opt) Pip style requirements.txt to install to a directory -It requires python 3.9+ and pyyaml. +It requires python 3.10+ and pyyaml. Use a qsub interactive copyq job on raijin with sufficient memory to run the following commands at the NCI: -New DEA-Env Module $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.9.2 + $ module load python3/3.10.4 # if pyyaml is not installed in gadi $ pip install PyYAML --user - $ # Building a new Environment Module: - $ python3 build_environment_module.py dea-env/modulespec.yaml - -New DEA Module - $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.9.2 - $ # Building a new DEA Module $ python3 build_environment_module.py dea/modulespec.yaml @@ -361,20 +353,6 @@ def run_final_commands_on_module(commands, module_path): run_command(cmd) -# def include_stable_module_dep_versions(config): -# """ -# Include stable module dependency versions - -# :param config: Dictionary of configuration variables -# :return: None -# """ -# stable_module_deps = config.get("stable_module_deps", []) -# for dep in stable_module_deps: -# default_version = find_default_version(dep) -# dep = dep.replace("-", "_") -# config["variables"][f"fixed_{dep}"] = default_version - - def main(config_path): """ Build new environment module @@ -394,7 +372,6 @@ def main(config_path): if "module_version" not in variables: variables["module_version"] = date() include_templated_vars(config) - # include_stable_module_dep_versions(config) pre_check(config) prep(config_path) diff --git a/nci_environment/dea/modulefile.template b/nci_environment/dea/modulefile.template index abf61993..e58d87e1 100644 --- a/nci_environment/dea/modulefile.template +++ b/nci_environment/dea/modulefile.template @@ -39,6 +39,10 @@ prepend-path PYTHONPATH ${module_path}/share/qgis/python remove-path PATH ${module_path}/bin prepend-path PATH ${module_path}/bin +# Remove duplicate entries for python path and prepend again +remove-path PYTHONPATH ${python_path} +prepend-path PYTHONPATH ${python_path} + # To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH # variables to point to a directory based on the Environment Module version which is loaded so that extra # packages must be re-installed when a new dea module is released @@ -55,8 +59,8 @@ if {[module-info mode load] && [is-loaded $$name/$$version]} { } # Remove duplicate entries for HOME dir and prepend at the top -remove-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.9/site-packages -prepend-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.9/site-packages +remove-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.10/site-packages +prepend-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.10/site-packages ############################################################# diff --git a/nci_environment/dea/modulespec.yaml b/nci_environment/dea/modulespec.yaml index 09ac26f8..4173d096 100644 --- a/nci_environment/dea/modulespec.yaml +++ b/nci_environment/dea/modulespec.yaml @@ -5,17 +5,19 @@ variables: conda_path: "/g/data/v10/private/mambaforge/bin/mamba" dbhost: dea-db.nci.org.au dbport: 6432 - python_version: 3.9 + python_version: '3.10' templated_variables: module_path: "{modules_dir}/{module_name}/{module_version}" python_path: "{modules_dir}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" dea_module: "{module_name}/{module_version}" + pip_path: "{modules_dir}/{module_name}/{module_version}/bin/pip3" install_conda_packages: ../../docker/env.yaml install_pip_packages: - pip_cmd: "module load python3/3.9.2; pip install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; pip install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt" + # need to specify which python/pip to use otherwise it defaults to the incorrect one + pip_cmd: "{pip_path} install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; {pip_path} install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt" copy_files: - src: ../../docker/env.yaml diff --git a/nci_environment/dea/requirements.txt b/nci_environment/dea/requirements.txt index bad0c985..ff29c81f 100644 --- a/nci_environment/dea/requirements.txt +++ b/nci_environment/dea/requirements.txt @@ -1,8 +1,4 @@ datacube -eodatasets3 -odc-algo odc-apps-dc-tools -odc-dscache odc-geom -odc-io odc-ui From 3b0d1afb24967ae8a50a16d1687b79e6fad76dc4 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Wed, 22 Nov 2023 03:54:41 +0000 Subject: [PATCH 03/14] remove dea-env and dea_unstable --- nci_environment/README.md | 8 ---- nci_environment/dea-env/modulefile.template | 47 ------------------- nci_environment/dea-env/modulespec.yaml | 23 --------- nci_environment/dea_unstable/modulespec.yaml | 43 ----------------- nci_environment/dea_unstable/requirements.txt | 2 - 5 files changed, 123 deletions(-) delete mode 100644 nci_environment/dea-env/modulefile.template delete mode 100644 nci_environment/dea-env/modulespec.yaml delete mode 100644 nci_environment/dea_unstable/modulespec.yaml delete mode 100644 nci_environment/dea_unstable/requirements.txt diff --git a/nci_environment/README.md b/nci_environment/README.md index cf334ab4..c0f5bb86 100644 --- a/nci_environment/README.md +++ b/nci_environment/README.md @@ -73,14 +73,6 @@ Edit the `.version` file in the modulefiles directory. Eg. For `dea` this is: `/g/data/v10/public/modules/modulefiles/dea/.version` - -## Re-Building _dea-unstable_ - - module load python3/3.6.2 - rm -rf /g/data/v10/public/modules/dea/unstable - python3 build_environment_module.py dea_unstable/modulespec.yaml - - ## Archiving an old module [TO DO]... diff --git a/nci_environment/dea-env/modulefile.template b/nci_environment/dea-env/modulefile.template deleted file mode 100644 index 81b03495..00000000 --- a/nci_environment/dea-env/modulefile.template +++ /dev/null @@ -1,47 +0,0 @@ -#%Module######################################################################## -## -## ${module_name} modulefile -## - -proc ModulesHelp { } { - global version - - puts stderr " ${module_description}" - puts stderr " Version ${module_version}" -} - -set name ${module_name} -set version ${module_version} -set base ${module_path} - -module-whatis "${module_description} ${module_version}" - -# Enable segfault tracebacks in py3. https://docs.python.org/3/library/faulthandler.html -setenv PYTHONFAULTHANDLER 1 - -# Set PYTHONUSERBASE based on the version of dea-env module. -# This allows users to install python packages with "pip install --user ", -setenv PYTHONUSERBASE ~/.dea-sandbox/${module_name}/${module_version}/local -setenv GDAL_DATA ${module_path}/share/gdal -setenv PROJ_LIB ${module_path}/share/proj -setenv PROJ_NETWORK ON -setenv UDUNITS2_XML_PATH ${module_path}/share/udunits/udunits2.xml -setenv GSETTINGS_SCHEMA_DIR ${module_path}/share/glib-2.0/schemas - -if {[module-info mode load] && [info exists env(PYTHONPATH)] && $$env(PYTHONPATH) != ""} { - puts stderr "Warning: ${module_name}/${module_version} exists in the python env ($$env(PYTHONPATH))" - puts stderr "Unload all python modules, if you experience any issues." -} - -prepend-path PYTHONPATH ${module_path}/share/qgis/python - -# Remove duplicate entries for module path dir and prepend again -remove-path PATH ${module_path}/bin -prepend-path PATH ${module_path}/bin - -# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH -# variables to point to a directory based on the Environment Module version which is loaded so that extra -# packages must be re-installed when a new dea-env module is released -# Remove duplicate entries for HOME dir and prepend again -remove-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin -prepend-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin diff --git a/nci_environment/dea-env/modulespec.yaml b/nci_environment/dea-env/modulespec.yaml deleted file mode 100644 index a0c5c474..00000000 --- a/nci_environment/dea-env/modulespec.yaml +++ /dev/null @@ -1,23 +0,0 @@ -variables: - module_name: dea-env - module_description: DEA Environment Module - modules_dir: "/g/data/v10/public/modules" - conda_path: "/g/data/v10/private/mambaforge/bin/mamba" - python_version: 3.8 - -# These templated variables are filled and included in the available variables used -# in template files and configuration sections below -templated_variables: - module_path: "{modules_dir}/{module_name}/{module_version}" - python_path: "{modules_dir}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" - -install_conda_packages: ../../docker/env.yaml - -copy_files: -- src: ../../docker/env.yaml - dest: "{modules_dir}/{module_name}/{module_version}/environment.yaml" - -template_files: -- src: modulefile.template - dest: "{modules_dir}/modulefiles/{module_name}/{module_version}" - chmod: 0o444 diff --git a/nci_environment/dea_unstable/modulespec.yaml b/nci_environment/dea_unstable/modulespec.yaml deleted file mode 100644 index 05c189f6..00000000 --- a/nci_environment/dea_unstable/modulespec.yaml +++ /dev/null @@ -1,43 +0,0 @@ -variables: - module_name: dea - module_description: DEA tools for the NCI - modules_base: "/g/data/v10/public/modules" - dbhost: dea-db.nci.org.au - dbport: 6432 - python_version: 3.6 - module_version: unstable - -templated_variables: - module_path: "{modules_base}/{module_name}/{module_version}" - python_path: "{modules_base}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" - dea_module: "{module_name}/{module_version}" - -stable_module_deps: -- dea-env - -install_pip_packages: - pip_cmd: "module load {fixed_dea_env}; pip install --no-warn-script-location --prefix {module_path} --requirement requirements.txt" - -copy_files: -- src: ../dea/requirements.txt - dest: "{module_path}/requirements.txt" -- src: ../dea/datacube.conf - dest: "{module_path}/datacube.conf" -- src: ../dea/scripts/datacube-ensure-user.py - dest: "{module_path}/bin/datacube-ensure-user.py" - chmod: 0o755 -- src: ../dea/scripts/add-aws-vault - dest: "{module_path}/bin/add-aws-vault" - chmod: 0o755 -- src: ../dea/scripts/login-aws-vault - dest: "{module_path}/bin/login-aws-vault" - chmod: 0o755 - -wget_files: -- src: "https://github.com/99designs/aws-vault/releases/download/v6.3.1/aws-vault-linux-amd64" - dest: "{module_path}/bin/aws-vault" - chmod: 0o755 - -template_files: -- src: ../dea/modulefile.template - dest: "{modules_base}/modulefiles/{module_name}/{module_version}" diff --git a/nci_environment/dea_unstable/requirements.txt b/nci_environment/dea_unstable/requirements.txt deleted file mode 100644 index 5ed66a4a..00000000 --- a/nci_environment/dea_unstable/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ ---pre --r ../dea/requirements.txt From b2a32d64e9ef4b6cfbce2261eec8407d330c1ee4 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Thu, 23 Nov 2023 01:35:27 +0000 Subject: [PATCH 04/14] use double quotes around python version --- nci_environment/dea/modulespec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nci_environment/dea/modulespec.yaml b/nci_environment/dea/modulespec.yaml index 4173d096..f7f4b379 100644 --- a/nci_environment/dea/modulespec.yaml +++ b/nci_environment/dea/modulespec.yaml @@ -5,7 +5,7 @@ variables: conda_path: "/g/data/v10/private/mambaforge/bin/mamba" dbhost: dea-db.nci.org.au dbport: 6432 - python_version: '3.10' + python_version: "3.10" templated_variables: module_path: "{modules_dir}/{module_name}/{module_version}" From 649387e5749d3806d8dd7bcf1d1ae70c306c9f12 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Fri, 24 Nov 2023 03:27:02 +0000 Subject: [PATCH 05/14] update test script to not need full module path and test against notebooks --- nci_environment/test-module.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/nci_environment/test-module.sh b/nci_environment/test-module.sh index caaf74e5..a73e489a 100644 --- a/nci_environment/test-module.sh +++ b/nci_environment/test-module.sh @@ -2,6 +2,10 @@ # Module to test +if [[-z $1 ]]; then + echo "ERROR: no module provided" + exit 1 +fi module_to_test="$1" echo Testing module: "$module_to_test" @@ -10,7 +14,7 @@ echo Testing module: "$module_to_test" export MODULEPATH=/apps/.mf:/opt/Modules/modulefiles:/apps/Modules/modulefiles: # Ensure that all files are world readable -if [[ $(find "$module_to_test" ! -perm /004) ]]; then +if [[ $(find /g/data/v10/public/modules/modulefiles/"$module_to_test" ! -perm /004) ]]; then echo "ERROR: Some files in $module_to_test are not world readable." exit 1 fi @@ -26,6 +30,13 @@ module load "$module_to_test" # TODO Check the output from: datacube system check +echo Testing notebooks +cd ../.. +export DEA_TOOLS_TIDE_MODELS=/g/data/v10/tide_models +if [[ ! -d dea-notebooks ]]; then + git clone --depth 1 https://github.com/GeoscienceAustralia/dea-notebooks.git +fi +./dea-notebooks/Tests/test_notebooks.sh # Run the initialise Test Database Script From fad9ec5133bca597609a5ad35b1a2e106d487b4a Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Fri, 24 Nov 2023 05:50:20 +0000 Subject: [PATCH 06/14] set tide modelling env var on module load --- nci_environment/dea/modulefile.template | 2 ++ nci_environment/test-module.sh | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nci_environment/dea/modulefile.template b/nci_environment/dea/modulefile.template index e58d87e1..b8d86eec 100644 --- a/nci_environment/dea/modulefile.template +++ b/nci_environment/dea/modulefile.template @@ -54,6 +54,8 @@ setenv DATACUBE_CONFIG_PATH ${module_path}/datacube.conf setenv LC_ALL en_AU.utf8 setenv LANG C.UTF-8 +setenv DEA_TOOLS_TIDE_MODELS /g/data/v10/tide_models + if {[module-info mode load] && [is-loaded $$name/$$version]} { puts stderr "Warning: $$name/$$version exists in the python env ($$env(PYTHONPATH))" } diff --git a/nci_environment/test-module.sh b/nci_environment/test-module.sh index a73e489a..1b316029 100644 --- a/nci_environment/test-module.sh +++ b/nci_environment/test-module.sh @@ -32,7 +32,6 @@ datacube system check echo Testing notebooks cd ../.. -export DEA_TOOLS_TIDE_MODELS=/g/data/v10/tide_models if [[ ! -d dea-notebooks ]]; then git clone --depth 1 https://github.com/GeoscienceAustralia/dea-notebooks.git fi From c6141c8679e82b60526c6d534f8c50461ff5bff2 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Wed, 29 Nov 2023 05:58:57 +0000 Subject: [PATCH 07/14] add pytest-odc to requirements --- nci_environment/dea/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/nci_environment/dea/requirements.txt b/nci_environment/dea/requirements.txt index ff29c81f..bca7b28a 100644 --- a/nci_environment/dea/requirements.txt +++ b/nci_environment/dea/requirements.txt @@ -1,4 +1,5 @@ datacube +pytest-odc odc-apps-dc-tools odc-geom odc-ui From d5fec890f6d5c188a19562542dce342447f432e0 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Mon, 4 Dec 2023 02:23:56 +0000 Subject: [PATCH 08/14] fix bash syntax --- nci_environment/test-module.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nci_environment/test-module.sh b/nci_environment/test-module.sh index 1b316029..fb33a1c2 100644 --- a/nci_environment/test-module.sh +++ b/nci_environment/test-module.sh @@ -2,7 +2,7 @@ # Module to test -if [[-z $1 ]]; then +if [[ -z $1 ]]; then echo "ERROR: no module provided" exit 1 fi From 687cf05454b02940084db8cbb0ee4712c455001e Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Mon, 1 Jul 2024 03:22:28 +0000 Subject: [PATCH 09/14] fix env vs reqs package listing --- docker/env.yaml | 3 ++- docker/requirements.txt | 4 +--- nci_environment/dea/requirements.txt | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/env.yaml b/docker/env.yaml index 57a42312..b531bdbd 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -94,9 +94,10 @@ dependencies: - nodejs - numexpr - numpy - - odc-algo - odc-dscache + - odc-geo - odc-io + - odc-stac - ordered-set - packaging - pandas diff --git a/docker/requirements.txt b/docker/requirements.txt index 331c2634..0367d222 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -20,13 +20,11 @@ s2cloudmask opencv-python-headless opencv-contrib-python-headless -datacube[performance,s3] >= 1.8.17 +datacube[performance,s3] >= 1.8.18 odc-algo @ git+https://github.com/opendatacube/odc-algo@b8dcfce odc-cloud[ASYNC] -odc-stac odc-stats[ows] odc-ui -odc-geo dea-tools thredds-crawler diff --git a/nci_environment/dea/requirements.txt b/nci_environment/dea/requirements.txt index bca7b28a..f3992ffd 100644 --- a/nci_environment/dea/requirements.txt +++ b/nci_environment/dea/requirements.txt @@ -1,5 +1,6 @@ -datacube +datacube >= 1.8.18 pytest-odc +odc-algo @ git+https://github.com/opendatacube/odc-algo@b8dcfce odc-apps-dc-tools odc-geom odc-ui From 060570376daaad620a9ba69ae75fc197a474d05f Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Tue, 2 Jul 2024 05:14:07 +0000 Subject: [PATCH 10/14] use forked dive action, try with pytest 8.1.1 --- .github/workflows/dive.yml | 2 +- docker/env.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dive.yml b/.github/workflows/dive.yml index 6d7863ab..53ae5efd 100644 --- a/.github/workflows/dive.yml +++ b/.github/workflows/dive.yml @@ -81,7 +81,7 @@ jobs: sudo rm -rf ${GITHUB_WORKSPACE}/.git - name: Dive - uses: yuichielectric/dive-action@0.0.3 + uses: MaxymVlasov/dive-action@1.0.1 with: image: ${{ env.ORG }}/${{ env.IMAGE}}:_build config-file: ${{ github.workspace }}/dive-ci.yml diff --git a/docker/env.yaml b/docker/env.yaml index b531bdbd..8dfda6cc 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -115,7 +115,7 @@ dependencies: - pyrsistent - pystac - pystac-client - - pytest!=8.1.0 + - pytest=8.1.1 - python-dateutil - python-rapidjson - python-slugify From 5262eda48230b3ade359277fea6a08b71bdbc024 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Tue, 2 Jul 2024 05:15:07 +0000 Subject: [PATCH 11/14] change dive action ver --- .github/workflows/dive.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dive.yml b/.github/workflows/dive.yml index 53ae5efd..4562e9a8 100644 --- a/.github/workflows/dive.yml +++ b/.github/workflows/dive.yml @@ -81,7 +81,7 @@ jobs: sudo rm -rf ${GITHUB_WORKSPACE}/.git - name: Dive - uses: MaxymVlasov/dive-action@1.0.1 + uses: MaxymVlasov/dive-action@1.0.0 with: image: ${{ env.ORG }}/${{ env.IMAGE}}:_build config-file: ${{ github.workspace }}/dive-ci.yml From b97371d6ca4fe6b8113c9df5101363464303928a Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Tue, 2 Jul 2024 05:18:09 +0000 Subject: [PATCH 12/14] actually fix dive action version this time --- .github/workflows/dive.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dive.yml b/.github/workflows/dive.yml index 4562e9a8..b577ab51 100644 --- a/.github/workflows/dive.yml +++ b/.github/workflows/dive.yml @@ -81,7 +81,7 @@ jobs: sudo rm -rf ${GITHUB_WORKSPACE}/.git - name: Dive - uses: MaxymVlasov/dive-action@1.0.0 + uses: MaxymVlasov/dive-action@v1.0.1 with: image: ${{ env.ORG }}/${{ env.IMAGE}}:_build config-file: ${{ github.workspace }}/dive-ci.yml From c73a6360cfd199db205daa949affd73b4713cec6 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Tue, 9 Jul 2024 07:52:43 +0000 Subject: [PATCH 13/14] update datacube version in nci reqs --- nci_environment/dea/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nci_environment/dea/requirements.txt b/nci_environment/dea/requirements.txt index f3992ffd..c831b9f0 100644 --- a/nci_environment/dea/requirements.txt +++ b/nci_environment/dea/requirements.txt @@ -1,4 +1,4 @@ -datacube >= 1.8.18 +datacube >= 1.8.19 pytest-odc odc-algo @ git+https://github.com/opendatacube/odc-algo@b8dcfce odc-apps-dc-tools From 5483f8141700261282cb99c452f87342192f85bb Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Tue, 25 Mar 2025 05:26:43 +0000 Subject: [PATCH 14/14] don't require host to be * --- nci_environment/dea/scripts/datacube-ensure-user.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nci_environment/dea/scripts/datacube-ensure-user.py b/nci_environment/dea/scripts/datacube-ensure-user.py index b53ad045..e2acde31 100755 --- a/nci_environment/dea/scripts/datacube-ensure-user.py +++ b/nci_environment/dea/scripts/datacube-ensure-user.py @@ -25,6 +25,8 @@ DBCreds = namedtuple("DBCreds", ["host", "port", "database", "username", "password"]) +DB_HOST = "dea-db.nci.org.au" + CANNOT_CONNECT_MSG = """ Unable to connect to the Data Cube database (host={}, port={}, db={}, username={}) Please contact earth.observation@ga.gov.au for help. @@ -77,7 +79,7 @@ def main(hostname, port, dbusername): creds = create_db_account(dbcreds) print("Created new database account.") # Append new credentials to ~/.pgpass file - append_credentials(pgpass, creds) + append_credentials(pgpass, creds._replace(port="*")) if not can_connect(dbcreds): print_stderr( @@ -129,11 +131,11 @@ def find_credentials(pgpass, dbcreds): # Ignore comments and empty lines if not line.strip().startswith("#") and line.strip(): creds = DBCreds(*line.strip().split(":")) - if creds.host == "*" and creds.port == "*" and creds.username == dbcreds.username: + if creds.host in ["*", DB_HOST] and creds.port in ["*", dbcreds.port] and creds.username == dbcreds.username: found_creds = True if not found_creds: raise ValueError("No valid credentials found in .pgpass file. " - "Please ensure that pgpass includes credentials in the format *:*:*::") + "Please ensure that pgpass includes credentials in the format :*:*::") def append_credentials(pgpass, dbcreds): @@ -206,8 +208,6 @@ def gen_password(length=20): # Tests # ######### -DB_HOST = "dea-db.nci.org.au" - def test_no_pgpass(tmpdir): # Create a pgpass.txt file in temp folder path = tmpdir.join("pgpass.txt") @@ -345,7 +345,7 @@ def test_against_comment_in_pgpass(tmpdir): creds = DBCreds("*", "*", "*", "foo_user", "asdf") - newcreds = find_credentials(pgpass, DB_HOST, creds) + newcreds = find_credentials(pgpass, creds) assert newcreds is not None assert newcreds.password == "asdf"