From 21163379dff0d2fc87c9739f9c9ff1ef47b2e57b Mon Sep 17 00:00:00 2001 From: Stefan Codrescu Date: Tue, 8 Apr 2025 23:43:33 -0600 Subject: [PATCH 1/7] adds type annotations, move netcdf utils to io module Conflicts: .github/workflows/lint-and-test.yml CHANGELOG.md ncflag/__init__.py ncflag/cli.py ncflag/flag_wrapper.py ncflag/tests/test_misc_api.py ncflag/tests/test_real_netcdf.py ncflag/tests/test_theoretical.py setup.py --- .editorconfig | 13 ++ .github/actions/setup/action.yml | 21 ++ .github/workflows/ci.yml | 55 +++++ .gitignore | 114 +--------- CHANGELOG.md | 32 +-- environment.yml | 1 - ncflag/__init__.py | 11 +- ncflag/__main__.py | 3 + ncflag/cli.py | 67 +++--- ncflag/flag_wrapper.py | 212 +++++++++--------- ncflag/io.py | 63 ++++++ test/__init__.py => ncflag/py.typed | 0 ncflag/tests/__init__.py | 0 .../ops_exis-l1b-sfxr_g16_d20180402_v0-0-0.nc | Bin {test => ncflag/tests}/test_misc_api.py | 12 +- {test => ncflag/tests}/test_real_netcdf.py | 16 +- {test => ncflag/tests}/test_reduce.py | 14 +- {test => ncflag/tests}/test_theoretical.py | 33 +-- pyproject.toml | 19 ++ setup.py | 7 +- 20 files changed, 402 insertions(+), 291 deletions(-) create mode 100644 .editorconfig create mode 100644 .github/actions/setup/action.yml create mode 100644 .github/workflows/ci.yml create mode 100644 ncflag/io.py rename test/__init__.py => ncflag/py.typed (100%) create mode 100644 ncflag/tests/__init__.py rename {test => ncflag/tests}/data/ops_exis-l1b-sfxr_g16_d20180402_v0-0-0.nc (100%) rename {test => ncflag/tests}/test_misc_api.py (76%) rename {test => ncflag/tests}/test_real_netcdf.py (86%) rename {test => ncflag/tests}/test_reduce.py (81%) rename {test => ncflag/tests}/test_theoretical.py (92%) create mode 100644 pyproject.toml diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..70c6b68 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +# EditorConfig is awesome: https://EditorConfig.org + +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +trim_trailing_whitespace = true + +[*.py] +indent_size = 4 diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 0000000..1d989ec --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,21 @@ +name: "setup" +description: "setup python" + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + - name: "install" + uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: '2.0.5-0' # from https://github.com/mamba-org/micromamba-releases + environment-file: environment.yml + cache-environment: true + post-cleanup: 'none' + create-args: >- + python=3.12 + black + mypy + isort + ruff + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..eddccb2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,55 @@ +name: ci + +on: + pull_request: + push: + branches: + - main + +defaults: + run: + shell: bash -leo pipefail {0} + +jobs: + black: + name: black + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + - name: black + run: "python -m black --check ." + isort: + name: isort + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + - name: isort + run: "python -m isort -c -v ." + lint: + name: ruff + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + - name: lint + run: "python -m ruff check" + mypy: + name: mypy + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + - name: mypy + run: "python -m mypy ." + test: + name: unittest + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + - name: test + run: "python -m unittest discover -v" + + diff --git a/.gitignore b/.gitignore index eb36223..da41f3e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,125 +1,27 @@ -# Created by .ignore support plugin (hsz.mobi) -### JetBrains template -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 +# IDEs / editors .idea - -## Plugin-specific files: - -# IntelliJ -/out/ - -# mpeltonen/sbt-idea plugin -.idea_modules/ - -# JIRA plugin -atlassian-ide-plugin.xml - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties -### Vim template -# swap -[._]*.s[a-w][a-z] -[._]s[a-w][a-z] -# session -Session.vim -# temporary -.netrwhist -*~ -# auto-generated tag files -tags -### Python template -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so +*.swo +*.swp +.zed # Distribution / packaging -.Python -env/ -build/ -develop-eggs/ dist/ -downloads/ +build/ eggs/ .eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ -.tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml -*,cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# dotenv -.env # virtualenv .venv/ venv/ -ENV/ - -# Spyder project settings -.spyderproject - -# Rope project settings -.ropeproject +# Tools +.mypy_cache/ +.ruff_cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a08812..77642e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,25 @@ -# v0.3.3 - 2024 04 08 +# v1.0.0 - 2022-12-27 +- Adds type annotations and `py.typed` marker. +- Replaces asserts with proper checks and custom exceptions +- Change (breaking!): remove netcdf read/write FlagWrap instance methods +- Adds netcdf read/write utils in a new `io` module + - Use `from ncflag.io import read_flag_from_netcdf, write_flag_to_netcdf` + - Replace `FlagWrap.init_from_netcdf(...)` with `read_flag_from_netcdf(...)` + - Replace `FlagWrap.write_to_netcdf(...)` with `write_flag_to_netcdf(...)` +# v0.3.3 - 2024 04 08 - Bug fix: compatibility with numpy 1.24+ new out-of-bounds int conversion failure https://numpy.org/devdocs/release/1.24.0-notes.html#conversion-of-out-of-bound-python-integers -# v0.3.2 - 2022 10 02 - +# v0.3.2 - 2022-10-02 - use long_description_content_type="text/markdown" in setup.py - unpin click dep, tested with latest 8.1.3 -# v0.3.1 - 2022 06 26 - +# v0.3.1 - 2022-06-26 - `ignore_missing` option on `get_flag` to bitwise and flag meanings, but not raise exception if a meaning is missing from the flag. -# v0.3.0 - 2020 12 11 +# v0.3.0 - 2020-12-11 - New method: `FlagWrap.is_valid_meaning` for convenience. - Change reduce method to exclude _entire_ flag vectors (all bits) from reduction if any bits in the exclude_mask are set. @@ -21,16 +27,16 @@ - Adds name attribute, available from init as kwarg, for misc use. - Drops support for Python 2.7 -# v0.2.6 - 2020 04 03 +# v0.2.6 - 2020-04-03 - Bug fix: cli: fix `--show_flags` option not working for Python 3 -# v0.2.5 - 2019 12 16 +# v0.2.5 - 2019-12-16 - cli: Support Python 3 by removing relative import. - setup.py move to `convert_file` from deprecated pypandoc `convert` - Adds shape, fill shortcuts for `init_from_netcdf` - Persist internal reference to netcdf object for convenience in `write_to_netcdf`. -# v0.2.0 - 2018 09 25 +# v0.2.0 - 2018-09-25 - netcdf agnostic, just need flag_meanings, flag_values, and optionally flag_masks. - Previous FlagWrap(netcdf.Variable) construction should be replaced with @@ -41,17 +47,17 @@ FlagWrap.write_to_netcdf(netcdf.Variable). - Fixed bug in setting mutually exclusive flag_meanings. - Added comprehensive testing. -# v0.1.1 - 2018 06 05 +# v0.1.1 - 2018-06-05 - Add reset and init to custom value other than 0. -# v0.1.0 - 2018 06 05 +# v0.1.0 - 2018-06-05 - Fix method for setting flags. Previous method did not work properly in some cases, eg. good_quality_qf. Now clears bits in flag under the mask before setting value. - Adds a get_mask_for_meaning function. - Improves documentation and testing. -# v0.0.3 - 2018 05 31 +# v0.0.3 - 2018-05-31 - Handle flags that are set to fill values, by definition, no flags are set when the value is filled. @@ -62,6 +68,6 @@ FlagWrap.write_to_netcdf(netcdf.Variable). - Optimization: exit_on_good option for get_flags_set_at_index. If known that no other flags can be set on "good" flag and good flag == 0, skip extra work of searching through all other flags. - - New: get_value_for_meaning function. Use case eg: start by setting all + - New: get_value_for_meaning function. Use case eg: start by setting all flags to a "missing_data" value, but avoid hard coding the "missing_data" value. diff --git a/environment.yml b/environment.yml index 5afd084..da4f976 100644 --- a/environment.yml +++ b/environment.yml @@ -6,4 +6,3 @@ dependencies: - numpy - netCDF4 - click - diff --git a/ncflag/__init__.py b/ncflag/__init__.py index 3a2d062..8a237f4 100644 --- a/ncflag/__init__.py +++ b/ncflag/__init__.py @@ -1 +1,10 @@ -from .flag_wrapper import FlagWrap as FlagWrap +from __future__ import annotations + +from .flag_wrapper import FlagWrap, InvalidFlagWrapMetadata, NoFlagFound + + +__all__ = [ + "FlagWrap", + "InvalidFlagWrapMetadata", + "NoFlagFound", +] diff --git a/ncflag/__main__.py b/ncflag/__main__.py index 4cafccb..67704f9 100644 --- a/ncflag/__main__.py +++ b/ncflag/__main__.py @@ -1,3 +1,6 @@ +from __future__ import annotations + from .cli import cli + cli() diff --git a/ncflag/cli.py b/ncflag/cli.py index 9fb41d0..96ee7dd 100644 --- a/ncflag/cli.py +++ b/ncflag/cli.py @@ -1,10 +1,13 @@ +from __future__ import annotations + import logging import click import netCDF4 as nc import pkg_resources -from .flag_wrapper import FlagWrap +from .io import read_flag_from_netcdf + try: version = pkg_resources.require("ncflag")[0].version @@ -12,7 +15,7 @@ version = "unknown" -def show_flags(ctx, param, ncfile): +def show_flags(ctx: click.Context, param: str, ncfile: str) -> None: if not ncfile or ctx.resilient_parsing: return valid_flags = [] @@ -31,7 +34,7 @@ def show_flags(ctx, param, ncfile): @click.command() @click.version_option(version, "-v", "--version") @click.option( - "--show_flags", + "--show-flags", callback=show_flags, expose_value=False, is_eager=True, @@ -40,37 +43,49 @@ def show_flags(ctx, param, ncfile): ) @click.argument("ncfile", type=click.Path(exists=True, dir_okay=False)) @click.argument("flag", type=click.STRING) -@click.option("--use_time_var", type=click.STRING, default=None) +@click.option("--use-time-var", type=click.STRING, default=None) @click.option( - "-l", + "--log-level", help="log level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), default="WARNING", ) -def cli(ncfile, flag, use_time_var, log_level): +def cli(ncfile: str, flag: str, use_time_var: str | None, log_level: str) -> None: logging.getLogger().setLevel(log_level) - with nc.Dataset(ncfile) as nc_in: # type: nc.Dataset + with nc.Dataset(ncfile) as nc_in: # initial checks - v = nc_in.variables[flag] # type: nc.Variable - assert hasattr(v, "flag_values"), ( - "%s is not CF compliant flag, missing flag_values" % flag - ) - assert hasattr(v, "flag_meanings"), ( - "%s is not CF compliant flag, missing flag_meanings" % flag - ) - assert ( - len(v.dimensions) == 1 - ), "multidimensional flags are not supported, see docs and use ipython instead" - w = FlagWrap.init_from_netcdf(v) - if use_time_var is not None: - t = nc_in.variables[use_time_var] # type: nc.Variable - assert ( - t.dimensions == v.dimensions - ), "To print flags by time, time flag must share dimensions" - assert hasattr(t, "units"), ( - "did not find units on time flag %s" % use_time_var + v = nc_in.variables[flag] + + missing = [] + if not hasattr(v, "flag_values"): + missing.append("flag_values") + + if not hasattr(v, "flag_meanings"): + missing.append("flag_meanings") + + if missing: + raise Exception( + "not a flag variable: missing attributes", ) - for i, dt in enumerate(nc.num2date(t[:], t.units)): + + if not len(v.dimensions) == 1: + raise Exception("multidimensional flags are not supported") + + w = read_flag_from_netcdf(v) + + if use_time_var is not None: + t = nc_in.variables[use_time_var] + + if not (t.dimensions == v.dimensions): + raise Exception("To print flags by time, time must share dimensions") + + if not hasattr(t, "units"): + raise Exception( + "did not find units on time variable", + ) + + dts = nc.num2date(t[:], t.units) + for i, dt in enumerate(dts): # type: ignore out_time = ( dt.isoformat() if dt is not None else "__________________________" ) diff --git a/ncflag/flag_wrapper.py b/ncflag/flag_wrapper.py index 05f5108..9dfe559 100644 --- a/ncflag/flag_wrapper.py +++ b/ncflag/flag_wrapper.py @@ -1,63 +1,81 @@ +from __future__ import annotations + +from typing import Any + import numpy as np +import numpy.typing as npt + + +class InvalidFlagWrapMetadata(Exception): + pass + + +class NoFlagFound(Exception): + pass -class FlagWrap(object): +class FlagWrap: """ - A convenience wrapper for flag bit vectors implemented with proper :flag_values, :flag_masks, - and :flag_meanings arrays so that flags can be queried and set in code by their meanings instead - of using (error prone) hardcoded masks and values. + Implements an API for bitwise flag vectors given metadata. + + Metadata that defines a flagging scheme includes: + - required flag_meanings + - required flag_values + - optional flag_masks + + The flag_meanings assign a label to each of the flag_values. The + flag_meaning[i] is said to be set when flag_values[i] == flag & flag_masks[i]. + In other words, a flag meaning is set when the masked flag is equal to the flag value. + + When flag_masks are not provided, no mask is applied. + + See the CF Convetions for more information and examples: + https://cfconventions.org/cf-conventions/cf-conventions.html#flags """ - def __init__(self, flags, flag_meanings, flag_values, flag_masks=None, name=None): + flag_meanings: list[str] + flag_values: npt.NDArray[Any] + flag_masks: npt.NDArray[Any] + + def __init__( + self, + flags: npt.NDArray[Any], + flag_meanings: list[str] | npt.NDArray[Any], + flag_values: list[int] | npt.NDArray[Any], + flag_masks: list[int] | npt.NDArray[Any] | None = None, + ) -> None: """ Initialize a FlagWrapper for a set of flags with associated metadata. - :type flags: np.ndarray - :param flags: array, flag values indicated - :type flag_meanings: list[str] | str - :param flag_meanings: list[str] | str, flag_meaning, string name of each designated flag_meaning - :type flag_values: np.array - :param flag_values: array, value of flag indicating corresponding flag_meaning set - :type flag_masks: np.array - :param flag_masks: array, optional mask the isolates bits from flag to indicate flag_meaning + :param flags: flag values indicated + :param flag_meanings: mnemonic labels for each flag value + :param flag_values: value that indicates a flag meaning is set + :param flag_masks: optional mask values to isolate bits from flag """ if np.ma.is_masked(flags): - # support for masked arrays is essential because init_from_netcdf is - # somewhat likely to give masked arrays, especially if someone is using - # FlagWrap to, eg, build a product. self.flags = np.ma.array(flags) - # FlagWrap carries around a masked array for a while, but note set_flag + # FlagWrap carries around a masked array for a while, but set_flag # enthusiastically casts to np.ndarray as soon as nothing is masked. else: self.flags = np.array(flags) - if isinstance(flag_meanings, str): - self.flag_meanings = flag_meanings.split() # split on spaces - else: - assert isinstance( - flag_meanings, list - ), "expected flag_meanings as either list of flag_meanings, or space separated string of flag_meanings" - self.flag_meanings = flag_meanings - - self._flag_values = np.array(flag_values).astype(self.flags.dtype) - assert len(self._flag_values) == len( - self.flag_meanings - ), "flag_meanings vs flag_values length mismatch: found {} and {}".format( - len(self.flag_meanings), len(self._flag_values) - ) + self.flag_meanings = list(flag_meanings) + + self.flag_values = np.array(flag_values).astype(self.flags.dtype) + if len(self.flag_values) != len(self.flag_meanings): + raise InvalidFlagWrapMetadata( + f"flag_meanings vs flag_values length mismatch: {len(self.flag_meanings)} != {len(self.flag_values)}" + ) if flag_masks is None: self._flag_masks = np.invert(np.zeros_like(self._flag_values)) else: - self._flag_masks = np.array(flag_masks).astype(self.flags.dtype) - assert len(self._flag_masks) == len( - self.flag_meanings - ), "flag_meanings vs flag_masks length mismatch: found {} and {}".format( - len(self.flag_meanings), len(self._flag_masks) + self.flag_masks = np.array(flag_masks).astype(self.flags.dtype) + if len(self.flag_masks) != len(self.flag_meanings): + raise InvalidFlagWrapMetadata( + f"flag_meanings vs flag_masks length mismatch: {len(self.flag_meanings)} != {len(self.flag_masks)}" ) - self.name = name - # This is only for use with init_from_netcdf to hold the reference to nc_var so that # the caller doesn't have to associate the write_to_netcdf call with an nc_var if it's the same as # it was initialized with. Otherwise, self._nc_var should never be used in the FlagWrap!!! @@ -127,42 +145,44 @@ def write_to_netcdf(self, nc_var=None): # only write masks if they aren't the default all bits 1 or somethign existed before nc_var.flag_masks = self._flag_masks - def get_flag(self, flag_meaning, ignore_missing=False): + def get_flag( + self, + flag_meaning: str | list[str], + ignore_missing: bool = False, + ) -> npt.NDArray[np.bool]: """ Get an array of booleans, same length as flags, at each index indicating if flag_meaning was set. - :type flag_meaning: str | list[str] | tuple[str] :param flag_meaning: flag meaning(s) to be looked up, bitwise_or reduced across first axis if multiple - :type ignore_missing: bool :param ignore_missing: if False, raises exception for missing flags, if True assumes missing flags are not set. - :rtype: np.array :return: array of booleans where flag_meaning(s) is(are) set """ - def get(meaning): + def get(meaning: str) -> npt.NDArray[np.bool]: """Get the meaning of an individual flag_meaning.""" if ignore_missing and meaning not in self.flag_meanings: return np.full_like(self.flags, False, dtype=bool) + index = self.flag_meanings.index(meaning) # start by default assuming there are no flags set. # Do not return a masked array! All value should be either True or False. # A flag is either set or not set. There is no inbetween. - default = np.full(self.flags.shape, False, dtype=bool) - if not np.ma.is_masked(self.flags): - mask = np.full_like(self.flags, True, dtype=bool) - else: + default = np.full_like(self.flags, False, dtype=bool) + if np.ma.is_masked(self.flags): mask = ~np.ma.getmask(self.flags) + else: + mask = np.full_like(self.flags, True, dtype=bool) # only the booleans to True potentially only where flags are not masked in the first place. default[mask] = ( - self.flags[mask] & self._flag_masks[index] - ) == self._flag_values[index] + self.flags[mask] & self.flag_masks[index] + ) == self.flag_values[index] return default - any_set = np.zeros(self.flags.shape, dtype=bool) - if isinstance(flag_meaning, (list, tuple)): + any_set = np.full(self.flags.shape, False, dtype=bool) + if isinstance(flag_meaning, list): # if receive a sequence, or them together. for each in flag_meaning: any_set |= get(each) @@ -171,18 +191,15 @@ def get(meaning): any_set |= get(flag_meaning) return any_set - def reduce(self, exclude_mask=0, axis=-1): + def reduce(self, exclude_mask: int = 0, axis: int = -1) -> FlagWrap: """ Return a new FlagWrap with the current flags reduced along some axis, possibly with some bit vectors excluded when anything from exclude_mask is set. Primary purpose: reduce a multidimensional flag. - :type exclude_mask: int :param exclude_mask: mask indicating (where bits are 1) where to exclude from reduced - :type axis: int :param axis: what axis to reduce, default -1 (last) - :rtype: FlagWrap :return: FlagWrap instance wrapping a flag with one fewer dimensions """ if exclude_mask != 0: @@ -197,53 +214,39 @@ def reduce(self, exclude_mask=0, axis=-1): return FlagWrap( new_flags, self.flag_meanings, - self._flag_values, - self._flag_masks, + self.flag_values, + self.flag_masks, ) - def get_flag_at_index(self, flag_meaning, i): + def get_flag_at_index(self, flag_meaning: str, i: int) -> bool: """ Returns True or False if flag_meaning set at index i? - :type flag_meaning: str :param flag_meaning: flag meaning intended to be set - :type i: int :param i: index in wrapped flags array to inspect. - :rtype: bool :return: bool indicating if flag_meaning was set at index i """ index = self.flag_meanings.index(flag_meaning) flag_value = self.flags[i] if np.ma.is_masked(flag_value): return False - else: - return (flag_value & self._flag_masks[index]) == self._flag_values[index] + result = (flag_value & self.flag_masks[index]) == self.flag_values[index] + return bool(result) - def get_flags_set_at_index(self, i, exit_on_good=False): + def get_flags_set_at_index(self, i: int) -> list[str]: """ Get a list of the flag_meanings set at a particular index. - :type i: int :param i: the index to examine - :type exit_on_good: bool - :param exit_on_good: shortcut, return good as soon as good is found - :rtype: list[str] :return: a list of flags_meanings set at index i """ flags_set = [] - # if exit_on_good, exit if a good_quality_qf type flag is found - # assumptions: good_quality_qf has the substring "good" and flag value can only be 0. - if exit_on_good and self.flags[i] == 0: - good_meaning = next((f for f in self.flag_meanings if "good" in f), None) - if good_meaning is not None: - return [good_meaning] - # otherwise, go into nominal search through all flags. If set, accumulate. for flag_meaning in self.flag_meanings: if self.get_flag_at_index(flag_meaning, i): flags_set.append(flag_meaning) return flags_set - def find_flag(self, options): + def find_flag(self, options: list[str]) -> npt.NDArray[Any]: """ Treat a list of flag_meanins as options, and return the result of the first one that exists. @@ -255,17 +258,20 @@ def find_flag(self, options): History: dealing with misspelled flag_meanings that will eventually be fixed, but now we must think about making the code work properly, transparently over time, over the point when the input is fixed. - :type options: list[str] :param options: list of potential flag_meanings to seek - :rtype: np.array :return: array of booleans indicating where first flag_meaning found is set. """ for flag in options: if flag in self.flag_meanings: return self.get_flag(flag) - raise ValueError("None of %s found." % options) - - def set_flag(self, flag_meaning, should_be_set, zero_if_unset=False): + raise NoFlagFound(f"None of find_flag flags found {options}") + + def set_flag( + self, + flag_meaning: str, + should_be_set: list[int] | npt.NDArray[Any], + zero_if_unset: bool = False, + ) -> None: """ Set flag_meaning in all flags where should_be_set is True. zero_if_unset (default True) controls whether bits targeted by flag_meaning are cleared (zeroed) even if should_be_set is False there. @@ -277,11 +283,8 @@ def set_flag(self, flag_meaning, should_be_set, zero_if_unset=False): flags where the whole value indicates the flag_meaning, so zeroing the target bits before setting on should_be_set will clear any previous flag_meanings or fill values that were set. - :type flag_meaning: str :param flag_meaning: flag meaning intended to be set - :type should_be_set: np.array :param should_be_set: array of booleans to set - :type zero_if_unset: bool :param zero_if_unset: explicitly set flag to false when flags indicates false :return: None """ @@ -297,10 +300,10 @@ def set_flag(self, flag_meaning, should_be_set, zero_if_unset=False): self.flags = np.array(self.flags) # zero_if_unset is True => all should_be_set will have targeted field zeroed... # otherwise only targets that that will be set. - self.flags[bool_flags | zero_if_unset] &= ~self._flag_masks[index] - self.flags |= bool_flags * self._flag_values[index] + self.flags[bool_flags | zero_if_unset] &= ~self.flag_masks[index] + self.flags |= bool_flags * self.flag_values[index] - def set_flag_at_index(self, flag_meaning, i): + def set_flag_at_index(self, flag_meaning: str, i: int) -> None: """ Set a flag at index i. @@ -308,9 +311,7 @@ def set_flag_at_index(self, flag_meaning, i): of the flag_meaning within the bit vec while leaving the rest set or unset as they were. Then, OR the flag value onto the target, preserves all other independent flags set. - :type flag_meaning: str :param flag_meaning: flag meaning intended to be set - :type i: int :param i: index at which to set flag_meaning :return: None """ @@ -321,11 +322,11 @@ def set_flag_at_index(self, flag_meaning, i): self.flags.mask[i] = False else: # otherwise, it has been set before, - self.flags[i] &= ~self._flag_masks[index] + self.flags[i] &= ~self.flag_masks[index] - self.flags[i] |= self._flag_values[index] + self.flags[i] |= self.flag_values[index] - def get_value_for_meaning(self, flag_meaning): + def get_value_for_meaning(self, flag_meaning: str) -> int: """ Get the value that sets flag_meaning. @@ -335,15 +336,15 @@ def get_value_for_meaning(self, flag_meaning): Note: Raises ValueError if flag_meaning is not found. - :type flag_meaning: str :param flag_meaning: string flag name to return value of - :rtype: int :return: value of flag that sets flag_meaning """ + if not self.is_valid_meaning(flag_meaning): + raise NoFlagFound(flag_meaning) index = self.flag_meanings.index(flag_meaning) - return self._flag_values[index] + return int(self.flag_values[index]) - def get_mask_for_meaning(self, flag_meaning): + def get_mask_for_meaning(self, flag_meaning: str) -> int: """ Get the the mask that would be used to test if flag_meaning is set. @@ -355,24 +356,19 @@ def get_mask_for_meaning(self, flag_meaning): Note: Raises ValueError if flag_meaning not found. - :type flag_meaning: str :param flag_meaning: string flag name to return corrsponding mask of - :rtype: int :return: flag_mask value corresponding to flag_meaning """ + if not self.is_valid_meaning(flag_meaning): + raise NoFlagFound(flag_meaning) index = self.flag_meanings.index(flag_meaning) - return self._flag_masks[index] + return int(self.flag_masks[index]) - def is_valid_meaning(self, flag_meaning): + def is_valid_meaning(self, flag_meaning: str) -> bool: """ - Test if a flag_meaning is valid or exists. - - Convenience function to handle common use case of checking if a - flag_meaning is valid for the wrapped flag. + Determine if the flag_meaning is valid. - :type flag_meaning: str - :param flag_meaning: string flag name to test existence of. - :rtype: bool - :return: if flag_meaning is a valid meaning for the flag. + :param flag_meaning: string flag name to test for existence. + :return: whether flag_meaning is valid """ return flag_meaning in self.flag_meanings diff --git a/ncflag/io.py b/ncflag/io.py new file mode 100644 index 0000000..df0314c --- /dev/null +++ b/ncflag/io.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from typing import Any + +import netCDF4 as nc +import numpy as np + +from .flag_wrapper import FlagWrap + + +def read_flag_from_netcdf( + nc_var: nc.Variable[Any], + shape: tuple[int] | None = None, + fill: int | None = None, +) -> FlagWrap: + """ + Initialize a FlagWrap instance from a reference to a NetCDF Variable. + + Default behavior, if no shape or fill are specified, the flags are read from the variable, + otherwise, if shape and fill are specified, the flag values will be initialized to that shape with + the fill value specified. + + :param nc_var: reference to a NetCDF flag variable + :param shape: If initializing data, secify shape (dimensions). + :param fill: If initializing data, what fill value to use. + :return: FlagWrap instance + """ + if shape is not None and fill is not None: + flags = np.full(shape, fill, dtype=nc_var.dtype) + return FlagWrap( + flags, + nc_var.flag_meanings.split(), + nc_var.flag_values, + getattr(nc_var, "flag_masks", None), + ) + return FlagWrap( + nc_var[:], + nc_var.flag_meanings.split(), + nc_var.flag_values, + getattr(nc_var, "flag_masks", None), + ) + + +def write_flag_to_netcdf( + flag_wrap: FlagWrap, + nc_var: nc.Variable[Any], +) -> None: + """ + Write a FlagWrap values and metadata to a NetCDF4 Variable. + + :type nc_var: netCDF4.Variable + :param nc_var: reference to NetCDF variable to write FlagWrap to. + :return: None + """ + nc_var[:] = flag_wrap.flags + nc_var.setncattr("flag_meanings", " ".join(flag_wrap.flag_meanings)) + nc_var.setncattr("flag_values", flag_wrap.flag_values) + + if not np.all(flag_wrap.flag_masks == np.full_like(flag_wrap.flag_values, -1)): + # only write masks if they aren't the default all bits are 1 + nc_var.flag_masks = flag_wrap.flag_masks + else: + nc_var.delncattr("flag_masks") diff --git a/test/__init__.py b/ncflag/py.typed similarity index 100% rename from test/__init__.py rename to ncflag/py.typed diff --git a/ncflag/tests/__init__.py b/ncflag/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/data/ops_exis-l1b-sfxr_g16_d20180402_v0-0-0.nc b/ncflag/tests/data/ops_exis-l1b-sfxr_g16_d20180402_v0-0-0.nc similarity index 100% rename from test/data/ops_exis-l1b-sfxr_g16_d20180402_v0-0-0.nc rename to ncflag/tests/data/ops_exis-l1b-sfxr_g16_d20180402_v0-0-0.nc diff --git a/test/test_misc_api.py b/ncflag/tests/test_misc_api.py similarity index 76% rename from test/test_misc_api.py rename to ncflag/tests/test_misc_api.py index 8f8ef36..efc018b 100644 --- a/test/test_misc_api.py +++ b/ncflag/tests/test_misc_api.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from unittest import TestCase import numpy as np @@ -6,20 +8,20 @@ class TestApi(TestCase): - def setUp(self): + def setUp(self) -> None: self.flags = np.array([0, 0, 1, 2, 3], dtype=np.ubyte) - self.flag_meanings = "good medium bad extra_bad" + self.flag_meanings = "good medium bad extra_bad".split() self.flag_values = np.array([0, 1, 2, 3]) self.f = FlagWrap(self.flags, self.flag_meanings, self.flag_values) - def test_valid_meaning(self): - for flag_meaning in self.flag_meanings.split(): + def test_valid_meaning(self) -> None: + for flag_meaning in self.flag_meanings: self.assertTrue(self.f.is_valid_meaning(flag_meaning)) for not_a_meaning in ["test", "not", "valid", "good1", "extra"]: self.assertFalse(self.f.is_valid_meaning(not_a_meaning)) - def test_get_flag_on_missing_meaning(self): + def test_get_flag_on_missing_meaning(self) -> None: for not_a_meaning in ["test", "not", "valid", "good1", "extra"]: flags = self.f.get_flag(not_a_meaning, ignore_missing=True) self.assertEqual(np.count_nonzero(flags), 0) diff --git a/test/test_real_netcdf.py b/ncflag/tests/test_real_netcdf.py similarity index 86% rename from test/test_real_netcdf.py rename to ncflag/tests/test_real_netcdf.py index ecefafd..c1670f8 100644 --- a/test/test_real_netcdf.py +++ b/ncflag/tests/test_real_netcdf.py @@ -1,13 +1,15 @@ +from __future__ import annotations + import os from unittest import TestCase import netCDF4 as nc import numpy as np -from ncflag import FlagWrap +from ncflag.io import read_flag_from_netcdf -def get_dataset(): +def get_dataset() -> nc.Dataset: datafile = os.path.join( os.path.dirname(__file__), "data/ops_exis-l1b-sfxr_g16_d20180402_v0-0-0.nc" ) @@ -15,9 +17,9 @@ def get_dataset(): class TestXrs(TestCase): - def test_yaw_flip_flag(self): + def test_yaw_flip_flag(self) -> None: ds = get_dataset() - f = FlagWrap.init_from_netcdf(ds.variables["yaw_flip_flag"]) + f = read_flag_from_netcdf(ds.variables["yaw_flip_flag"]) # the flag value at masked_index is masked, i.e. has no value. # below, make sure that's accounted for properly. Should not return @@ -25,7 +27,7 @@ def test_yaw_flip_flag(self): masked_index = 34799 self.assertEqual(len(f.get_flags_set_at_index(masked_index)), 0) - # but, for example, the one before it should be set to "upright" + # but, for example, the ones before it should be set to "upright" first_flags = f.get_flags_set_at_index(0) self.assertEqual(first_flags, ["upright"]) @@ -34,9 +36,9 @@ def test_yaw_flip_flag(self): self.assertFalse(np.ma.is_masked(upright)) self.assertFalse(upright[masked_index]) - def test_misc_consistency(self): + def test_misc_consistency(self) -> None: ds = get_dataset() - f = FlagWrap.init_from_netcdf(ds.variables["quality_flags"]) + f = read_flag_from_netcdf(ds.variables["quality_flags"]) # good_quality_qf is first in the list and easy to manually establish... # value should be 0... diff --git a/test/test_reduce.py b/ncflag/tests/test_reduce.py similarity index 81% rename from test/test_reduce.py rename to ncflag/tests/test_reduce.py index 4625354..515ecd3 100644 --- a/test/test_reduce.py +++ b/ncflag/tests/test_reduce.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from unittest import TestCase import numpy as np @@ -6,27 +8,27 @@ class TestReduce(TestCase): - def test_reduce_axis0(self): + def test_reduce_axis0(self) -> None: flags = np.array([[0, 1], [1, 0], [0, 0], [1, 1]], dtype=np.ubyte) - f = FlagWrap(flags, "good bad", [0, 1]) + f = FlagWrap(flags, ["good", "bad"], [0, 1]) f_reduced = f.reduce(axis=0) np.testing.assert_array_equal(f_reduced.get_flag("good"), [0, 0]) np.testing.assert_array_equal(f_reduced.get_flag("bad"), [1, 1]) - def test_reduce_axis1(self): + def test_reduce_axis1(self) -> None: flags = np.array([[0, 1], [1, 0], [0, 0], [1, 1]], dtype=np.ubyte) - f = FlagWrap(flags, "good bad", [0, 1]) + f = FlagWrap(flags, ["good", "bad"], [0, 1]) f_reduced = f.reduce(axis=1) np.testing.assert_array_equal(f_reduced.get_flag("good"), [0, 0, 1, 0]) np.testing.assert_array_equal(f_reduced.get_flag("bad"), [1, 1, 0, 1]) - def test_reduce_mask(self): + def test_reduce_mask(self) -> None: flags = np.array( [ @@ -38,7 +40,7 @@ def test_reduce_mask(self): dtype=np.ubyte, ) - f = FlagWrap(flags, "red green blue", [1, 2, 4], [1, 2, 4]) + f = FlagWrap(flags, ["red", "green", "blue"], [1, 2, 4], [1, 2, 4]) # exclude mask == 1 means that any flag vectors that have red should # not be included in the reduced version... diff --git a/test/test_theoretical.py b/ncflag/tests/test_theoretical.py similarity index 92% rename from test/test_theoretical.py rename to ncflag/tests/test_theoretical.py index e4e7e33..be86fd9 100644 --- a/test/test_theoretical.py +++ b/ncflag/tests/test_theoretical.py @@ -1,28 +1,32 @@ +from __future__ import annotations + from unittest import TestCase import numpy as np -from ncflag import FlagWrap +from ncflag import FlagWrap, InvalidFlagWrapMetadata, NoFlagFound class TestTheoretical(TestCase): - def test_teject_misformed_flags(self): - with self.assertRaises(AssertionError): + def test_teject_misformed_flags(self) -> None: + with self.assertRaises(InvalidFlagWrapMetadata): # wrong number of flag_values for flag_meanings - FlagWrap(np.array([]), "flag1 flag2", [1, 2, 3]) + FlagWrap(np.array([]), ["flag1", "flag2"], [1, 2, 3]) - with self.assertRaises(AssertionError): + with self.assertRaises(InvalidFlagWrapMetadata): # wrong number of flag_masks for flag_meanings - FlagWrap(np.array([]), "flag1 flag2", [1, 2], [-1, -1, -1]) + FlagWrap(np.array([]), ["flag1", "flag2"], [1, 2], [-1, -1, -1]) - def test_exclusive_flag_type(self): + def test_exclusive_flag_type(self) -> None: """A barrage of tests to make sure everything works properly for flag variables defined so that every flag_meaning is mutually exclusive.""" original_flags = np.array([0, 0, 1, 2, 3, 255], dtype=np.ubyte) f = FlagWrap( - original_flags.copy(), "good medium bad extra_bad", np.array([0, 1, 2, 3]) + original_flags.copy(), + ["good", "medium", "bad", "extra_bad"], + np.array([0, 1, 2, 3]), ) np.testing.assert_array_equal( @@ -64,10 +68,7 @@ def test_exclusive_flag_type(self): for i in range(len(f.flags)): # exit_on_good option applies here since flags are mutually exclusive # result should be the same regardless of exit_on_good - self.assertTrue( - f.get_flags_set_at_index(i, exit_on_good=True) - == f.get_flags_set_at_index(i, exit_on_good=False) - ) + self.assertTrue(f.get_flags_set_at_index(i) == f.get_flags_set_at_index(i)) self.assertTrue(f.get_flag_at_index("good", 0)) self.assertFalse(f.get_flag_at_index("good", -1)) @@ -115,7 +116,7 @@ def test_exclusive_flag_type(self): # actually, one more thing while we have this FlagWrap instance, check that find_flag works... these_dont_exist = ["thegrinch", "lochnessmonster", "dreams"] - with self.assertRaises(ValueError): + with self.assertRaises(NoFlagFound): f.find_flag(these_dont_exist) for flag_meaning in f.flag_meanings: np.testing.assert_array_equal( @@ -123,7 +124,7 @@ def test_exclusive_flag_type(self): ) # ok, we'll call it good there. - def test_maskedarray_initial_flags(self): + def test_maskedarray_initial_flags(self) -> None: """Similar to the previous test, except start with a masked array and fill... This will be what it looks like if someone does init_from_netcdf for an unwritten variable that actually has shape.""" @@ -169,7 +170,7 @@ def test_maskedarray_initial_flags(self): f.set_flag("bad", [0, 0, 1, 0, 0], zero_if_unset=True) np.testing.assert_array_equal(f.get_flag("good"), np.array([1, 1, 0, 1, 1])) - def test_inclusive_flag(self): + def test_inclusive_flag(self) -> None: # as opposed to an exclusive flag where every flag_meaning is exclusive, in other words, there can only # every be one flag meaning, an inclusive flag can have multiple flag_meanings set at once. @@ -196,7 +197,7 @@ def test_inclusive_flag(self): f = FlagWrap( original_flags.copy(), - "good degraded middle left right red blue", + ["good", "degraded", "middle", "left", "right", "red", "blue"], [0, 1, 6, 2, 4, 8, 16], [1, 1, 6, 6, 6, 8, 16], ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8a0b768 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ + +[tool.isort] +profile = "black" +lines_after_imports = 2 +add_imports = "from __future__ import annotations" + +[tool.mypy] +plugins = [ + "numpy.typing.mypy_plugin" +] +strict = true +disallow_untyped_calls = false +disallow_untyped_defs = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_return_any = true +warn_unreachable = true + diff --git a/setup.py b/setup.py index 03b4e01..ee70351 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,12 @@ +from __future__ import annotations + from setuptools import setup + setup( name="ncflag", - version="0.3.3", - description="Utility and library to interface with CF-Compliant NetCDF flag variables.", + version="1.0.0", + description="Implements an API for bitwise flag vectors given metadata.", author="Stefan Codrescu", author_email="stefan.codrescu@noaa.gov", url="https://github.com/5tefan/ncflag", From 3538e364f090924fa2c66e7c3e7f2d832ce4d75d Mon Sep 17 00:00:00 2001 From: Stefan Codrescu Date: Sat, 28 Dec 2024 15:36:48 -0700 Subject: [PATCH 2/7] fix dtype casting for numpy 2.0+ Conflicts: ncflag/flag_wrapper.py --- ncflag/flag_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ncflag/flag_wrapper.py b/ncflag/flag_wrapper.py index 9dfe559..00a4d3f 100644 --- a/ncflag/flag_wrapper.py +++ b/ncflag/flag_wrapper.py @@ -68,7 +68,7 @@ def __init__( ) if flag_masks is None: - self._flag_masks = np.invert(np.zeros_like(self._flag_values)) + self.flag_masks = np.invert(np.zeros_like(self.flag_values)) else: self.flag_masks = np.array(flag_masks).astype(self.flags.dtype) if len(self.flag_masks) != len(self.flag_meanings): From 52ff653e9a3b5d2cc2d264353c5c74c0c8603815 Mon Sep 17 00:00:00 2001 From: Stefan Codrescu Date: Sat, 28 Dec 2024 15:54:36 -0700 Subject: [PATCH 3/7] add types-setuptools dependency (conda only) --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index da4f976..78cc8a1 100644 --- a/environment.yml +++ b/environment.yml @@ -6,3 +6,4 @@ dependencies: - numpy - netCDF4 - click + - types-setuptools From fa7de087b15f452a314973e4bcd1ed785823d1e2 Mon Sep 17 00:00:00 2001 From: Stefan Codrescu Date: Sat, 28 Dec 2024 17:05:56 -0700 Subject: [PATCH 4/7] update README --- README.md | 130 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 97 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index ec4778b..b45bb29 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,104 @@ -# NetCDF Flag Wrapper (ncflag) +# ncflag -So... you want to inspect CF Compliant NetCDF flag variables? +An API that makes working with bitwise quality flag vectors easy! -CF Compliant NetCDF Flag variables are integer flags associated with, or having: +## Motivation + +Scientific datasets often pack multiple flags into a single integer by assigning individual bits or +combinations of bits to indicate certain conditions. This library allows code to be written in terms +of human readable labels instead of hard coding arbitrary flag values and masks. This makes code +more readable and robust against changes to flagging schemes. + +For example, consider a satellite observation with a 1 byte flag (i.e. 8 bits -- eg `np.ubyte` +datatype). In this example, suppose: + +1. The least significant bit indicates a temperature anomaly. +2. The next least significant bit indicates a satellite maneuver. +3. Data is good when neither temperature anomaly nor satellite maneuver is indicated. + +Several combinations are possible: + +1. `0b0000_0001`: temperature anomaly is indicated, bad data. +2. `0b0000_0010`: satellite maneuver is indicated, bad data. +3. `0b0000_0011`: temperature anomaly AND satellite manuever indicated, bad data. +4. `0b0000_0000`: good data. + +The [CF Conventions](http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html#flags) +define a standard set of metadata for defining a flagging scheme that includes the following pieces +of data: + +- `flag_meanings`: a list of strings giving labels to each flag +- `flag_values`: a list of values, used to test if the corresponding flag is set +- `flag_masks`: a mask used to isolate the relevant bits for the corresponding flag + +For the example satellite obersvations we would have the following: +- `flag_meanings`: `["good", "temperature_anomaly", "satellite_maneuver"]` +- `flag_values`: `[0, 1, 2]` +- `flag_masks`: `[3, 1, 2]` + +To test if index `1` of a 1D array of flags is "good", compare two approaches. With `ncflag`: +`flag.get_flag_at_index("good", 1)` versus without `ncflag`: `flag[1] & 3 == 0`. Future maintainers +are unlikely to immediately know where the `3` (mask) and `0` (value) came from or intuit that these +values are correct in the context of the program. On the other hand, it's relatively easy to +understand that we're reading some kind of "good" flag when encountering the `ncflag` API. + +Spot the bug: `temperature_anomaly = flag & 2 == 0`... a bit subtle. But what about now? +`temperature_anomaly = flag.get_flag("satellite_maneuver")`... a little less subtle! + +What happens if the data provider realizes that data is still good during a satellite maneuver? They +would adjust the "good" mask from 3 to 1. If you had hardcoded `3` in your code, you would need to +ensure you noticed this change, and fix your code. By using `ncflag` and the metadata provided in +the dataset, this change would be transparent to you. + +## Brief + +An API to interact with data quality flags defined by: - flag_values - flag_meanings - - flag_masks (optionally) + - flag_masks (optional) -Read the [CF Conventions on flags](http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html#flags) +Read the [CF Conventions on flags](http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html#flags) for more information. -## TL;DR +## Install + +Install the utility with either `pip` or `conda` -Install the utility with with pip: ``` pip install ncflag ``` +``` +conda install -c conda-forge ncflag +``` + + +## Command Line Interface + On the command line, use `ncflag`: ``` -Usage: ncflag [OPTIONS] NCFILE FLAG +Usage: python -m ncflag [OPTIONS] NCFILE FLAG Options: -v, --version Show the version and exit. - --show_flags PATH Print the flags this tool can inspect. - --use_time_var TEXT - -l [DEBUG|INFO|WARNING|ERROR|CRITICAL] - log level + --show-flags FILE Print the flags this tool can inspect. + --use-time-var TEXT Variable in NetCDF file to use to display + timestamps + --log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL] + Log level --help Show this message and exit. - ``` Notes: -Use --show_flags to discover what flags in a given file can be inspected. +Use `--show-flags` to discover what flags in a given file can be inspected. Limitation: can only inspect flags of at most one dimension. See details below for dealing with multidimensional flags. -The nominal output with --use_time_var specified is shown below. Without use_time_var, the index along the -dimension will be printed instead of a iso 8601 timestamp. +The output below uses `--use-time-var` to show a human readable timestamp rather than an index along the dimension. ```text 2017-11-27T21:07:41.543778: [u'data_quality_error'] 2017-11-27T21:07:42.543812: [u'good_data'] @@ -51,20 +108,21 @@ dimension will be printed instead of a iso 8601 timestamp. ## Multidimensional Flags -Occasionally, by some poor misfortune, you may encounter multidimensional flag variables. These are currently not +You may encounter multidimensional flag variables. These are currently not supported by the Command Line Interface (CLI), however, the FlagWrap class can still be used in code, or through an -interactive (IPython) session. The `FlagWrap.get_flags_set_at_index` can be passed a tuple of indicies to get the -flags set in a multidimensional flag variable. Below is an example. +interactive (IPython) session. The `FlagWrap.get_flags_set_at_index` can be passed a tuple of indicies to get the +flags set in a multidimensional flag variable. Below is an example. ```python from ncflag import FlagWrap +from ncflag.io import read_flag_from_netcdf import netCDF4 as nc with nc.Dataset("somenetcdf.nc") as nc_in: v = nc_in.variables["mutidim_variable"] print(v.shape) # --> (2, 10), is multidim. - w = FlagWrap.init_from_netcdf(v) - print(w.get_flags_set_at_index((0, 0))) # --> ["good_quality_qf"] + f = read_flag_from_netcdf(v) + print(f.get_flags_set_at_index((0, 0))) # --> ["good_quality_qf"] ``` ## API and Documentation @@ -76,28 +134,34 @@ For documentation, please read `flag_wrapper.py`. It is one file and is documented with comprehensive docstrings. The functions are named descriptively. A following functions are available from a FlagWrap instance. - - get_flag(self, flag_meaning) - - reduce(self, exclude_mask, axis=-1) - - get_flag_at_index(self, flag_meaning, i) - - get_flags_set_at_index(self, i, exit_on_good=False) - - find_flag(self, options) - - set_flag(self, flag_meaning, should_be_set, zero_if_unset=True) - - set_flag_at_index(self, flag_meaning, i) - - get_value_for_meaning(self, flag_meaning) - - get_mask_for_meaning(self, flag_meaning) + - get_flag(self, flag_meaning: str | list[str], ignore_missing: bool = False) -> npt.NDArray[Any] + - reduce(self, exclude_mask: int = 0, axis: int = -1) -> FlagWrap + - get_flag_at_index(self, flag_meaning: str, i: int) -> bool + - get_flags_set_at_index(self, i: int) -> list[str] + - find_flag(self, options: list[str]) -> npt.NDArray[Any] + - set_flag(self, flag_meaning: str, should_be_set: list[int] | npt.NDArray[Any], zero_if_unset: bool = False) -> None + - set_flag_at_index(self, flag_meaning: str, i: int) -> None + - get_value_for_meaning(self, flag_meaning: str) -> int + - get_mask_for_meaning(self, flag_meaning: str) -> int + - is_valid_meaning(self, flag_meaning: str) -> bool + +## Naming +The package name `ncflag` is inspired by classic utilities for the NetCDF file format (`.nc`) like `ncdump`, +and `ncgen` etc. However, all NetCDF specificity has been removed from the core +`FlagWrap` object. This library can be used independently of NetCDF, for example with hardcoded metadata +or reading metadata from other sources like `.fits`. ## Testing There are tests, using both synthetic flags, as well as some more serious tests for some fairly complex "in the wild" flags taken from a sample GOES-16 EXIS-L1b-SFXR product file. -`test/test_theoretical.py` is actually a very thourough read to help anyone really understand -what's possible and what's going on with these flags. +`ncflag/test/test_theoretical.py` is a resource to understand what's possible in a toy example. --------------------- -Deploy to pip, after testing with python2 and python3: +Deploy to pip, after testing: ```bash rm -r dist/ From 4677ac3a0110b9728f4f2b742edb6d05b8811f48 Mon Sep 17 00:00:00 2001 From: Stefan Codrescu Date: Sat, 28 Dec 2024 17:06:12 -0700 Subject: [PATCH 5/7] cli help, mention cli _ -> - in changelog, setuptools classifiers --- CHANGELOG.md | 2 ++ ncflag/cli.py | 8 ++++++-- setup.py | 6 ++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77642e6..5c92c18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - Use `from ncflag.io import read_flag_from_netcdf, write_flag_to_netcdf` - Replace `FlagWrap.init_from_netcdf(...)` with `read_flag_from_netcdf(...)` - Replace `FlagWrap.write_to_netcdf(...)` with `write_flag_to_netcdf(...)` +- Change CLI options to dash word separators (`-`). `--show_flags` -> + `--show-flags` and `--use_time_var` -> `--use-time-var`. # v0.3.3 - 2024 04 08 - Bug fix: compatibility with numpy 1.24+ new out-of-bounds int conversion failure diff --git a/ncflag/cli.py b/ncflag/cli.py index 96ee7dd..b74c537 100644 --- a/ncflag/cli.py +++ b/ncflag/cli.py @@ -43,10 +43,14 @@ def show_flags(ctx: click.Context, param: str, ncfile: str) -> None: ) @click.argument("ncfile", type=click.Path(exists=True, dir_okay=False)) @click.argument("flag", type=click.STRING) -@click.option("--use-time-var", type=click.STRING, default=None) +@click.option( + "--use-time-var", + help="Variable in NetCDF file to use to display timestamps", + type=click.STRING, default=None, +) @click.option( "--log-level", - help="log level", + help="Log level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), default="WARNING", ) diff --git a/setup.py b/setup.py index ee70351..03a4fee 100644 --- a/setup.py +++ b/setup.py @@ -22,8 +22,10 @@ classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", + "Topic :: Scientific/Engineering :: Atmospheric Science", + "Topic :: Scientific/Engineering :: Hydrology", + "Topic :: Scientific/Engineering :: Oceanography", + "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], From 97a576a9d807fb9da22bfae70d34f3eae1a47c2f Mon Sep 17 00:00:00 2001 From: Stefan Codrescu Date: Sat, 28 Dec 2024 17:11:16 -0700 Subject: [PATCH 6/7] more README updates --- README.md | 7 ++++--- ncflag/cli.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b45bb29..f2d7a90 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ncflag -An API that makes working with bitwise quality flag vectors easy! +An API that makes working with bitwise quality flags easy! ## Motivation @@ -42,8 +42,9 @@ are unlikely to immediately know where the `3` (mask) and `0` (value) came from values are correct in the context of the program. On the other hand, it's relatively easy to understand that we're reading some kind of "good" flag when encountering the `ncflag` API. -Spot the bug: `temperature_anomaly = flag & 2 == 0`... a bit subtle. But what about now? -`temperature_anomaly = flag.get_flag("satellite_maneuver")`... a little less subtle! +Spot the bug: `temperature_anomaly = flag & 2 == 0`... not easy! But what about now? +`temperature_anomaly = flag.get_flag("satellite_maneuver")`... a little less subtle! Looks like +we're reading the "satellite_maneuver" flag into a variable called "temperature_anomaly"... sus. What happens if the data provider realizes that data is still good during a satellite maneuver? They would adjust the "good" mask from 3 to 1. If you had hardcoded `3` in your code, you would need to diff --git a/ncflag/cli.py b/ncflag/cli.py index b74c537..464412e 100644 --- a/ncflag/cli.py +++ b/ncflag/cli.py @@ -46,7 +46,8 @@ def show_flags(ctx: click.Context, param: str, ncfile: str) -> None: @click.option( "--use-time-var", help="Variable in NetCDF file to use to display timestamps", - type=click.STRING, default=None, + type=click.STRING, + default=None, ) @click.option( "--log-level", From e4d1459858f3ad9f69267fd802902356f0e3b911 Mon Sep 17 00:00:00 2001 From: Stefan Codrescu Date: Tue, 8 Apr 2025 23:54:22 -0600 Subject: [PATCH 7/7] Fix wonky rebase conflict resolution --- ncflag/flag_wrapper.py | 69 ------------------------------------------ 1 file changed, 69 deletions(-) diff --git a/ncflag/flag_wrapper.py b/ncflag/flag_wrapper.py index 00a4d3f..cc291c6 100644 --- a/ncflag/flag_wrapper.py +++ b/ncflag/flag_wrapper.py @@ -76,75 +76,6 @@ def __init__( f"flag_meanings vs flag_masks length mismatch: {len(self.flag_meanings)} != {len(self.flag_masks)}" ) - # This is only for use with init_from_netcdf to hold the reference to nc_var so that - # the caller doesn't have to associate the write_to_netcdf call with an nc_var if it's the same as - # it was initialized with. Otherwise, self._nc_var should never be used in the FlagWrap!!! - self._nc_var = None - - @classmethod - def init_from_netcdf(cls, nc_var, shape=None, fill=None): - """ - Initialize a FlagWrap instance from a reference to a NetCDF Variable. - - Default behavior, if no shape or fill are specified, the flags are read from the variable, - otherwise, if shape and fill are specified, the flag values will be initialized to that shape with - the fill value specified. - - :type nc_var: netCDF4.Variable - :param nc_var: reference to NetCDF variable to wrap - :type shape: tuple - :param shape: If data is to be initialized instead of read from variable, secify shape (dimensions). - :type fill: int - :param fill: Again, if initializing data, what fill value to use. - :return: FlagWrap instance for nc_var - """ - if shape is not None and fill is not None: - flags = np.full(shape, fill, dtype=nc_var.dtype) - instance = cls( - flags, - nc_var.flag_meanings, - nc_var.flag_values, - getattr(nc_var, "flag_masks", None), - name=nc_var.name, - ) - else: - instance = cls( - nc_var[:], - nc_var.flag_meanings, - nc_var.flag_values, - getattr(nc_var, "flag_masks", None), - name=nc_var.name, - ) - - instance._nc_var = nc_var - - return instance - - def write_to_netcdf(self, nc_var=None): - """ - Write a FlagWrap values and metadata to a NetCDF4 Variable. - - :type nc_var: netCDF4.Variable - :param nc_var: reference to NetCDF variable to write FlagWrap to. - :return: None - """ - if nc_var is None and self._nc_var is not None: - nc_var = self._nc_var - elif nc_var is None: - raise RuntimeError( - "write_to_netcdf called w/o target nc_var and appears not to be init from an nc_var." - ) - - nc_var[:] = self.flags - nc_var.flag_meanings = " ".join(self.flag_meanings) - nc_var.flag_values = self._flag_values - if ( - not np.all(self._flag_masks == np.invert(np.zeros_like(self._flag_values))) - or getattr(nc_var, "flag_masks", None) is not None - ): - # only write masks if they aren't the default all bits 1 or somethign existed before - nc_var.flag_masks = self._flag_masks - def get_flag( self, flag_meaning: str | list[str],