diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a5acbe14..88c15f88 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,31 +17,36 @@ env: GITHUB_TOKEN: ${{ secrets.COVERALLS_TOKEN }} jobs: - source_check: + lint: name: Linting (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: [ '3.12' ] + python-version: [ '3.13' ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install black 'isort[colors]<6' - python -m pip install --editable . + python -m pip install --group lint + - name: Linting checks + run: | + black --check src/spotpy + isort --check src/spotpy - build_sdist: - name: sdist on ${{ matrix.os }} with Python ${{ matrix.python-version }} + test_build: + name: Test and build (${{ matrix.os }}, Python ${{ matrix.python-version }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -50,20 +55,24 @@ jobs: python-version: [ '3.10', '3.13' ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: '0' + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install build pytest-cov - python -m pip install --editable .[test] + python -m pip install --group build --group test + + - name: Install spotpy with optional dependencies + run: | + python -m pip install --editable ".[database,describe,mpi,plotting]" - name: Run tests run: | @@ -82,8 +91,8 @@ jobs: run: | python -m build - - uses: actions/upload-artifact@v4 - if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' + - uses: actions/upload-artifact@v7 + if: (matrix.os == 'ubuntu-latest') && (matrix.python-version == '3.13') with: name: dist-sdist path: dist/ @@ -91,11 +100,12 @@ jobs: upload_to_pypi: name: Upload to PyPI/Test PyPI needs: - - build_sdist + - test_build + if: (github.ref == 'refs/heads/master') || (startsWith(github.ref, 'refs/tags')) runs-on: ubuntu-latest steps: - name: Download all the dists - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: dist-sdist path: dist/ diff --git a/MANIFEST.in b/MANIFEST.in index f2da5cec..b2936ac7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,5 +2,5 @@ prune ** graft src/spotpy graft tests graft tutorials -include LICENSE README.md pyproject.toml setup.py +include LICENSE README.md pyproject.toml global-exclude __pycache__ *.py[co] diff --git a/README.md b/README.md index 95f18c6c..b59ae637 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A Statistical Parameter Optimization Tool for Python [![Python Versions][pypi-pyv-image]][pypi-pyv-link] [![License][license-image]][license-link] [![Coverage Status](https://coveralls.io/repos/github/thouska/spotpy/badge.svg?branch=master)](https://coveralls.io/github/thouska/spotpy?branch=master) -[![DOI](https://zenodo.org/badge/47562322.svg)](https://zenodo.org/badge/latestdoi/47562322) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4075177.svg)](https://doi.org/10.5281/zenodo.4075177) [pypi-v-image]: https://img.shields.io/pypi/v/spotpy.png [pypi-v-link]: https://pypi.python.org/pypi/spotpy diff --git a/pyproject.toml b/pyproject.toml index b0e9b39e..13321981 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,23 @@ [build-system] requires = [ - "setuptools>=64", - "setuptools_scm[toml]>=6.4", + "setuptools>=80", + "setuptools_scm[toml]>=9.2", ] build-backend = "setuptools.build_meta" +[dependency-groups] +build = [ + "build" +] +lint = [ + "black >=26.3.0", + "isort >=8.0.1" +] +test = [ + "pytest >=9.0", + "pytest-cov >=7.1.0", +] + [project] requires-python = ">=3.10" name = "spotpy" @@ -21,7 +34,8 @@ maintainers = [ {name = "Tobias Houska", email = "tobias.houska@umwelt.uni-giessen.de"}, ] readme = "README.md" -license = {file = "LICENSE"} +license = "MIT" +license-files = ["LICENSE"] dynamic = ["version"] keywords = [ "Monte Carlo", @@ -47,7 +61,6 @@ classifiers = [ "Intended Audience :: End Users/Desktop", "Intended Audience :: Science/Research", "Intended Audience :: Education", - "License :: OSI Approved :: MIT License", "Natural Language :: English", "Operating System :: Unix", "Operating System :: Microsoft", @@ -66,25 +79,29 @@ classifiers = [ "Topic :: Utilities", ] dependencies = [ - "numpy>=2.0", - "scipy>=1.5.0", + "click >=8.1", + "numpy >=2.0", + "scipy >=1.12.0", ] [project.optional-dependencies] -plotting = [ - "pandas>=1", - "matplotlib>=3", +database = [ + "tables" ] -test = [ - "pytest-cov>=3", - "numba", - "pathos", - "matplotlib", - "click", - "pandas", - "tables", - "docutils", +describe = [ + "docutils" +] +mpi = [ + "mpi4py", + "pathos" ] +plotting = [ + "pandas >=2.2.0", + "matplotlib >=3.8.0", +] + +[project.scripts] +spotpy = "spotpy.cli:cli" [project.urls] Changelog = "https://github.com/thouska/spotpy/blob/master/CHANGELOG.md" @@ -108,11 +125,10 @@ multi_line_output = 3 [tool.black] exclude = "_version.py" target-version = [ - "py39", "py310", "py311", "py312", - "py313", + "py313" ] [tool.coverage] diff --git a/src/spotpy/__init__.py b/src/spotpy/__init__.py index 22b920e6..3d107998 100644 --- a/src/spotpy/__init__.py +++ b/src/spotpy/__init__.py @@ -31,6 +31,7 @@ Please cite our paper, if you are using SPOTPY. """ + from . import algorithms # Contains all the different algorithms implemented in SPOTPY from . import ( analyser, # Contains some examples to analyse the results of the different algorithms diff --git a/src/spotpy/algorithms/__init__.py b/src/spotpy/algorithms/__init__.py index 0053e192..7aee608a 100644 --- a/src/spotpy/algorithms/__init__.py +++ b/src/spotpy/algorithms/__init__.py @@ -14,7 +14,6 @@ To reduce dependencies, one may select here just the needed algorithm. """ - from ._algorithm import _algorithm from .abc import abc # Artificial Bee Colony from .dds import dds # Dynamically Dimensioned Search algorithm @@ -28,6 +27,7 @@ from .mc import mc # Monte Carlo from .mcmc import mcmc # Metropolis Markov Chain Monte Carlo from .mle import mle # Maximum Likelihood Estimation +from .morris import morris # Morris Screening Sensitivity Test from .nsgaii import ( NSGAII, # A Fast and Elitist Multiobjective Genetic Algorithm: NSGA-II ) @@ -35,4 +35,3 @@ from .rope import rope # RObust Parameter Estimation from .sa import sa # Simulated annealing from .sceua import sceua # Shuffled Complex Evolution -from .morris import morris # Morris Screening Sensitivity Test diff --git a/src/spotpy/algorithms/efast.py b/src/spotpy/algorithms/efast.py index eb73068b..a7c3b3fa 100644 --- a/src/spotpy/algorithms/efast.py +++ b/src/spotpy/algorithms/efast.py @@ -564,4 +564,4 @@ def calc_sensitivity(self, results, dbname, freq="cukier"): np.savetxt(f, [sens_data], delimiter=",", fmt="%1.5f") f.close() - return sens_data \ No newline at end of file + return sens_data diff --git a/src/spotpy/algorithms/fast.py b/src/spotpy/algorithms/fast.py index d72d4daa..37b37317 100644 --- a/src/spotpy/algorithms/fast.py +++ b/src/spotpy/algorithms/fast.py @@ -149,12 +149,10 @@ def analyze(self, problem, Y, D, parnames, M=4, print_to_console=False): """ ) else: - print( - """ + print(""" Error: Number of samples in model output file must be a multiple of D, where D is the number of parameters in your parameter file. - """ - ) + """) exit() # Recreate the vector omega used in the sampling diff --git a/src/spotpy/algorithms/list_sampler.py b/src/spotpy/algorithms/list_sampler.py index 8f386e2a..854fe4b9 100644 --- a/src/spotpy/algorithms/list_sampler.py +++ b/src/spotpy/algorithms/list_sampler.py @@ -4,6 +4,7 @@ This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska """ + from .. import analyser from . import _algorithm diff --git a/src/spotpy/algorithms/morris.py b/src/spotpy/algorithms/morris.py index 580c2970..eac4b902 100644 --- a/src/spotpy/algorithms/morris.py +++ b/src/spotpy/algorithms/morris.py @@ -19,13 +19,13 @@ class morris(_algorithm): This class holds the Morris Screening Sensitivity Test (MORRIS) based on Morris (1991), Campolongo et al (2007) and Ruano et al. (2012): Morris, M.D., 1991, Factorial Sampling Plans for Preliminary Computational Experiments. Technometrics 33, 161-174. - + Campolongo, F., Cariboni, J., & Saltelli, A. 2007. An effective screening design for sensitivity analysis of large models. Environmental Modelling & Software, 22(10), 1509-1518. Ruano, M.V., Ribes, J., Seco, A., Ferrer, J., 2012. An improved sampling strategy based on trajectory design for application of the Morris method to systems with many input factors. Environmental Modelling & Software 37, 103-109. - + The presented code is based on SALib Copyright (C) 2013-2015 Jon Herman and others. Licensed under the GNU Lesser General Public License. The Sensitivity Analysis Library is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. @@ -98,18 +98,29 @@ def scale_samples(self, params, bounds): def matrix(self, bounds, problem, N, M=4): from SALib.sample import morris as morris_sample - X = morris_sample.sample(problem, N=N, num_levels=M, - optimal_trajectories=None, # set int to use Campolongo opt design - local_optimization=True, # Ruano-style distance improvement - seed=123) + X = morris_sample.sample( + problem, + N=N, + num_levels=M, + optimal_trajectories=None, # set int to use Campolongo opt design + local_optimization=True, # Ruano-style distance improvement + seed=123, + ) self.scale_samples(X, bounds) return X def analyze(self, problem, X, Y, num_levels=4, print_to_console=False): from SALib.analyze import morris as morris_analyze - Si = morris_analyze.analyze(problem, X, Y, conf_level=0.95, num_levels=num_levels, - print_to_console=print_to_console) + + Si = morris_analyze.analyze( + problem, + X, + Y, + conf_level=0.95, + num_levels=num_levels, + print_to_console=print_to_console, + ) return Si def sample(self, repetitions, num_levels=4): @@ -132,7 +143,6 @@ def sample(self, repetitions, num_levels=4): # distribution parmin, parmax = self.parameter()["minbound"], self.parameter()["maxbound"] - bounds = [] for i in range(len(parmin)): bounds.append([parmin[i], parmax[i]]) @@ -142,14 +152,14 @@ def sample(self, repetitions, num_levels=4): # Assume df_params from your existing script # columns: name, change_type, lower_bound, upper_bound, subbasins problem = { - 'num_vars': len(names), - 'names': names.tolist(), # or name_spotpy if you expanded names - 'bounds': bounds + "num_vars": len(names), + "names": names.tolist(), # or name_spotpy if you expanded names + "bounds": bounds, #'bounds': self.parameter()[['minbound','maxbound']].values.tolist() } - k = problem['num_vars'] - N = 15 # number of trajectories (adjust for robustness vs runtime) + k = problem["num_vars"] + N = 15 # number of trajectories (adjust for robustness vs runtime) num_levels = num_levels # classic Morris grid # Create an Matrix to store the parameter sets @@ -179,10 +189,16 @@ def sample(self, repetitions, num_levels=4): try: data = self.datawriter.getdata() # this is likely to crash if database does not assign name 'like1' - Si = self.analyze(problem, Matrix, data["like1"], num_levels=num_levels, print_to_console=False) + Si = self.analyze( + problem, + Matrix, + data["like1"], + num_levels=num_levels, + print_to_console=False, + ) # Si = self.analyze( # bounds, data["like1"], len(bounds), names, M=num_levels, print_to_console=True # ) - print(Si['mu_star']) + print(Si["mu_star"]) except AttributeError: # Happens if no database was assigned pass diff --git a/src/spotpy/algorithms/padds.py b/src/spotpy/algorithms/padds.py index d5d6f09b..a50aefbb 100644 --- a/src/spotpy/algorithms/padds.py +++ b/src/spotpy/algorithms/padds.py @@ -382,7 +382,7 @@ def calc_initial_pareto_front(self, its): ] ) else: - (self.pareto_front, dominance_flag) = nd_check( + self.pareto_front, dominance_flag = nd_check( self.pareto_front, self.obj_func_current, self.parameter_current.copy(), diff --git a/src/spotpy/algorithms/rope.py b/src/spotpy/algorithms/rope.py index 7cf4aff5..f206559a 100644 --- a/src/spotpy/algorithms/rope.py +++ b/src/spotpy/algorithms/rope.py @@ -4,6 +4,7 @@ This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska and Alejandro Chamorro-Chavez """ + import random import time diff --git a/src/spotpy/analyser.py b/src/spotpy/analyser.py index e3eed865..10718fd8 100644 --- a/src/spotpy/analyser.py +++ b/src/spotpy/analyser.py @@ -29,7 +29,11 @@ def load_csv_results(filename, usecols=None): """ if usecols == None: return np.genfromtxt( - filename + ".csv", delimiter=",", names=True, dtype=np.float32, invalid_raise=False + filename + ".csv", + delimiter=",", + names=True, + dtype=np.float32, + invalid_raise=False, ) else: return np.genfromtxt( @@ -39,7 +43,7 @@ def load_csv_results(filename, usecols=None): skip_footer=1, invalid_raise=False, usecols=usecols, - dtype=np.float32 + dtype=np.float32, )[1:] @@ -1404,85 +1408,105 @@ def plot_efast(dbname, spot_setup, fig_name="efast_sensitivities.png"): fig: saves figure as .png """ - import matplotlib.pyplot as plt import matplotlib.cm as cm + import matplotlib.pyplot as plt + senstivities = load_csv_results(dbname) parameters = senstivities.dtype.names sens_values = void_to_arr(senstivities).T - - color= cm.brg(np.linspace(0, 1, len(parameters))) + + color = cm.brg(np.linspace(0, 1, len(parameters))) fig, axs = plt.subplots(len(parameters), figsize=(10, 5), sharey=True) for i in range(len(parameters)): # WATCH OUT: First 5 results get discarded - axs[i].plot(spot_setup.date[364:], sens_values[i, :][5:], color=color[i]) + axs[i].plot(spot_setup.date[364:], sens_values[i, :][5:], color=color[i]) axs[i].set_ylabel(parameters[i][3:]) - if i < len(parameters)-1: + if i < len(parameters) - 1: axs[i].set_xticklabels([]) - print('hello') - axs[i].set_xlabel('Date') - axs[i].set_ylim(0,1) + print("hello") + axs[i].set_xlabel("Date") + axs[i].set_ylim(0, 1) plt.tight_layout() fig.savefig(fig_name, dpi=300) + def calculate_morris_sensitivity(problem, X, Y, num_levels=4, print_to_console=False): from SALib.analyze import morris as morris_analyze - Si = morris_analyze.analyze(problem, X, Y, conf_level=0.95, num_levels=num_levels, - print_to_console=print_to_console) + + Si = morris_analyze.analyze( + problem, + X, + Y, + conf_level=0.95, + num_levels=num_levels, + print_to_console=print_to_console, + ) return Si -def plot_morris_sensitivity(results, spot_setup, like_index=0, num_levels=4, fig_name='Morris_interaction.png'): + +def plot_morris_sensitivity( + results, spot_setup, like_index=0, num_levels=4, fig_name="Morris_interaction.png" +): + import matplotlib.cm as cm import matplotlib.pyplot as plt import pandas as pd - import matplotlib.cm as cm + parameter = spotpy.parameter.get_parameters_array(spot_setup) names = parameter["name"] - + bounds = [] for i in range(len(names)): bounds.append([parameter["minbound"][i], parameter["maxbound"][i]]) problem = { - 'num_vars': len(names), - 'names': names.tolist(), # or name_spotpy if you expanded names - 'bounds': bounds + "num_vars": len(names), + "names": names.tolist(), # or name_spotpy if you expanded names + "bounds": bounds, #'bounds': self.parameter()[['minbound','maxbound']].values.tolist() } par_fields = get_parameter_fields(results) - + par_results = results[par_fields] X = pd.DataFrame(par_results).values - - like_results = results["like"+str(like_index+1)] + + like_results = results["like" + str(like_index + 1)] Y = like_results - - SI = calculate_morris_sensitivity(problem, X, Y, num_levels=num_levels) + + SI = calculate_morris_sensitivity(problem, X, Y, num_levels=num_levels) fig = plt.figure(figsize=(6, 6)) - + colors = cm.brg(np.linspace(0, 1, len(par_fields))) - + ax = plt.subplot(1, 1, 1) - + for i, label in enumerate(par_fields): - plt.scatter(SI['mu_star'][i], SI['sigma'][i], color=colors[i]) - plt.text(SI['mu_star'][i]+0.03, SI['sigma'][i]+0.03, par_fields[i][3:]) + plt.scatter(SI["mu_star"][i], SI["sigma"][i], color=colors[i]) + plt.text(SI["mu_star"][i] + 0.03, SI["sigma"][i] + 0.03, par_fields[i][3:]) print(SI) xmin, xmax = ax.get_xlim() - ymin, ymax =ax.get_ylim() - #lims = [ - #np.min([ax.get_xlim(), ax.get_ylim()]), # min of both axes - #np.max([ax.get_xlim(), ax.get_ylim()]), # max of both axes - #] - #print(lims) + ymin, ymax = ax.get_ylim() + # lims = [ + # np.min([ax.get_xlim(), ax.get_ylim()]), # min of both axes + # np.max([ax.get_xlim(), ax.get_ylim()]), # max of both axes + # ] + # print(lims) # now plot both limits against eachother - #ax.plot(lims, lims, 'k--', alpha=0.75, zorder=0) - ax.plot([0,np.max([ax.get_xlim(), ax.get_ylim()])], [0,np.max([ax.get_xlim(), ax.get_ylim()])*2], color='dimgrey', linestyle='--', alpha=0.75, zorder=0) - plt.xlabel("$\mu*$"+'\nParameter Sensitivity') - plt.ylabel("$\sigma$"+'\nParameter Interaction') - plt.xlim(0,xmax*1.1) - plt.ylim(0,ymax*1.1) - + # ax.plot(lims, lims, 'k--', alpha=0.75, zorder=0) + ax.plot( + [0, np.max([ax.get_xlim(), ax.get_ylim()])], + [0, np.max([ax.get_xlim(), ax.get_ylim()]) * 2], + color="dimgrey", + linestyle="--", + alpha=0.75, + zorder=0, + ) + plt.xlabel("$\mu*$" + "\nParameter Sensitivity") + plt.ylabel("$\sigma$" + "\nParameter Interaction") + plt.xlim(0, xmax * 1.1) + plt.ylim(0, ymax * 1.1) + plt.tight_layout() plt.show() - fig.savefig(fig_name, dpi=150) \ No newline at end of file + fig.savefig(fig_name, dpi=150) diff --git a/src/spotpy/database/base.py b/src/spotpy/database/base.py index 5e64b149..5166cb42 100644 --- a/src/spotpy/database/base.py +++ b/src/spotpy/database/base.py @@ -9,6 +9,7 @@ This is the parent class of all algorithms, which can handle the database structure during the sample. """ + import sys import time from importlib import import_module @@ -170,12 +171,10 @@ class custom(database): def __init__(self, *args, **kwargs): if "setup" not in kwargs: - raise ValueError( - """ + raise ValueError(""" You are using the 'custom' Datawriter. To use it, the setup must be specified on creation, but it is missing - """ - ) + """) self.setup = kwargs["setup"] if not hasattr(self.setup, "save"): raise AttributeError( diff --git a/src/spotpy/database/sql.py b/src/spotpy/database/sql.py index 2231fc1c..56c350cd 100644 --- a/src/spotpy/database/sql.py +++ b/src/spotpy/database/sql.py @@ -48,14 +48,8 @@ def __init__(self, *args, **kwargs): # Create Table # self.db_cursor.execute('''CREATE TABLE IF NOT EXISTS '''+self.dbname+''' # (like1 real, parx real, pary real, simulation1 real, chain int)''') - self.db_cursor.execute( - """CREATE TABLE IF NOT EXISTS """ - + self.dbname - + """ - (""" - + " real ,".join(self.header) - + """)""" - ) + self.db_cursor.execute("""CREATE TABLE IF NOT EXISTS """ + self.dbname + """ + (""" + " real ,".join(self.header) + """)""") def save(self, objectivefunction, parameterlist, simulations=None, chains=1): coll = ( diff --git a/src/spotpy/describe.py b/src/spotpy/describe.py index cc15884f..7e4d6e3b 100644 --- a/src/spotpy/describe.py +++ b/src/spotpy/describe.py @@ -11,6 +11,7 @@ >>> spotpy.describe.sampler(sampler) >>> spotpy.describe.setup(model) """ + import sys from inspect import getdoc as _getdoc diff --git a/src/spotpy/examples/spot_setup_hymod_python.py b/src/spotpy/examples/spot_setup_hymod_python.py index 0d5c9f89..00091eb6 100644 --- a/src/spotpy/examples/spot_setup_hymod_python.py +++ b/src/spotpy/examples/spot_setup_hymod_python.py @@ -7,12 +7,12 @@ This example implements the python version of hymod into SPOTPY. """ +import datetime as dt import os from spotpy.examples.hymod_python.hymod import hymod from spotpy.objectivefunctions import rmse from spotpy.parameter import Uniform -import datetime as dt class spot_setup(object): @@ -46,7 +46,7 @@ def __init__(self, obj_func=None): self.header = headerline.split(self.delimiter) for line in climatefile: values = line.strip().split(self.delimiter) - self.date.append(dt.datetime.strptime(str(values[0]), '%d.%m.%Y')) + self.date.append(dt.datetime.strptime(str(values[0]), "%d.%m.%Y")) self.Precip.append(float(values[1])) self.PET.append(float(values[2])) self.trueObs.append(float(values[3])) diff --git a/src/spotpy/hydrology/signatures.py b/src/spotpy/hydrology/signatures.py index 10fb97ea..40004475 100644 --- a/src/spotpy/hydrology/signatures.py +++ b/src/spotpy/hydrology/signatures.py @@ -56,7 +56,6 @@ cf. to QuantileSignature and the get_qXXX methods """ - import inspect import sys diff --git a/src/spotpy/parameter.py b/src/spotpy/parameter.py index a52a06d6..89407b8e 100644 --- a/src/spotpy/parameter.py +++ b/src/spotpy/parameter.py @@ -5,6 +5,7 @@ :author: Philipp Kraft and Tobias Houska Contains classes to generate random parameter sets """ + import copy import sys from itertools import cycle