diff --git a/example-config.yaml b/example-config.yaml index db867af..cb604a9 100755 --- a/example-config.yaml +++ b/example-config.yaml @@ -290,3 +290,21 @@ rules: command: 'swfo-convert mcd43a1 h5-md --fname {brdf_base}/{collection}/{ymd}/{basename}{hdf_ext} --outdir /tmp/data/conversion/BRDF/{collection}/{ymd}/ --filter-opts ''{{"aggression": 6}}'' --compression BLOSC_ZSTANDARD' input_files: ['^(?P.*BRDF)/(?P.*)/(?P[0-9]{4}\.[0-9]{2}\.[0-9]{2})/(?P.*)(?P.hdf)(?P.xml)?', ['{brdf_base}/{collection}/{ymd}/{basename}{hdf_ext}', '{brdf_base}/{collection}/{ymd}/{basename}{hdf_ext}.xml']] expect_file: '/tmp/data/conversion/BRDF/{collection}/{ymd}/{basename}.h5' + +# +#------------------------------- +# Example using the OpenSearch API | +#------------------------------- +# + RESORB: + schedule: '0 */6 * * *' + source: !opensearch-api + username: + password: + api_url: https://scihub.copernicus.eu/gnss + query: + platformname: Sentinel-1 + producttype: AUX_RESORB + generationdate: '[NOW-5DAYS TO NOW]' + filename_transform: !regexp-extract '(?PS1[A,B]{1})_[A-Z].*_[A-Z].*_(?P[A-Z].*)_[A-Z].*_(?P[0-9,T]{15})_V(?P[0-9,T]{15})_(?P[0-9,T]{15}).EOF' + target_dir: /tmp/data/sensor-specific/Sentinel-1/{product}/{platform} \ No newline at end of file diff --git a/fetch/load.py b/fetch/load.py index 18b4c0d..0cf4435 100755 --- a/fetch/load.py +++ b/fetch/load.py @@ -13,7 +13,7 @@ import yaml.resolver from croniter import croniter -from . import ftp, http, ecmwf +from . import ftp, http, ecmwf, opensearch from ._core import RegexpOutputPathTransform, DateRangeSource, DateFilenameTransform, \ RsyncMirrorSource, SimpleObject, ShellFileProcessor from .util import remove_nones @@ -367,6 +367,7 @@ def add_item_constructor(source, type_annotation, attribute): add_default_constructor(http.HttpPostAction, '!http-post') add_default_constructor(http.HttpAuthAction, '!http-auth') add_default_constructor(ecmwf.EcmwfApiSource, '!ecmwf-api') + add_default_constructor(opensearch.OpenSearchApiSource, '!opensearch-api') add_item_constructor(RegexpOutputPathTransform, '!regexp-extract', 'pattern') add_item_constructor(DateFilenameTransform, '!date-pattern', 'format_') diff --git a/fetch/opensearch.py b/fetch/opensearch.py new file mode 100644 index 0000000..5532f70 --- /dev/null +++ b/fetch/opensearch.py @@ -0,0 +1,64 @@ +from __future__ import absolute_import + +import logging +import os + +from sentinelsat import SentinelAPI + +from ._core import DataSource, fetch_file + +DEFAULT_CONNECT_TIMEOUT_SECS = 100 + +_log = logging.getLogger(__name__) + + +class OpenSearchApiSource(DataSource): + """ + Class for data retrievals using the OpenSearch API. + """ + + def __init__(self, target_dir, api_url, username, password, query, show_progressbars=False, + timeout=DEFAULT_CONNECT_TIMEOUT_SECS, filename_transform=None, override_existing=False): + self.target_dir = target_dir + self.filename_transform = filename_transform + self.override_existing = override_existing + self.api_url = api_url + self.username = username + self.password = password + self.query = query + self.show_progressbars = show_progressbars + self.timeout = timeout + + self.api = SentinelAPI(self.username, self.password, self.api_url, self.show_progressbars, self.timeout) + + def trigger(self, reporter): + """ + :type reporter: ResultHandler + """ + + query_results = self.api.query(**self.query) + + for (uuid, result) in query_results.items(): + _log.info('Found %s with uuid %s', result['filename'], uuid) + + def create_fetch_function(key): + def opensearch_fetch(target): + download = self.api.download(key) + + # Workaround for fixed filename + _log.debug('Renaming %s to %s', download['path'], target) + os.rename(download['path'], target) + + return True + + return opensearch_fetch + + fetch_file( + result['link'].replace("'", '%27'), + create_fetch_function(uuid), + reporter, + os.path.basename(result['filename']), + self.target_dir, + filename_transform=self.filename_transform, + override_existing=self.override_existing + ) diff --git a/setup.py b/setup.py index d953bd9..7d0c903 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ 'croniter', 'feedparser', 'lxml', + 'sentinelsat', 'pathlib;python_version<"3.4"', 'pyyaml<5.1', 'requests>=2.21.0',