From 1c986de9dfebcd3eaa57d2612e697401b457adf7 Mon Sep 17 00:00:00 2001 From: Bob Yantosca Date: Thu, 25 Jan 2024 10:27:07 -0500 Subject: [PATCH 01/10] Add create_test_plot script; can check if GCPy is installed properly gcpy/examples/plotting/create_test_plot.py - Script to create a dummy plot that renders to the screen. Useful for checking if "import gcpy" works properly. gcpy/examples/plotting/__init__.py - Added create_test_plot.py CHANGELOG.md - Updated accordingly Signed-off-by: Bob Yantosca --- CHANGELOG.md | 5 ++ gcpy/examples/plotting/__init__.py | 1 + gcpy/examples/plotting/create_test_plot.py | 55 ++++++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100755 gcpy/examples/plotting/create_test_plot.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d6258b7e..86335626 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to GCPy will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] - TBD +### Added +- Example script `create_test_plot.py`, which can be used to check that GCPy has been installed properly + + ## [1.4.1] - 2023-12-08 ### Fixed - Now use the proper default value for the `--weightsdir` argument to `gcpy/file_regrid.py` diff --git a/gcpy/examples/plotting/__init__.py b/gcpy/examples/plotting/__init__.py index b9f7dd64..7aa3eac5 100644 --- a/gcpy/examples/plotting/__init__.py +++ b/gcpy/examples/plotting/__init__.py @@ -1,5 +1,6 @@ """ GCPy import script """ +from .create_test_plot import * from .plot_single_panel import * from .plot_comparisons import * diff --git a/gcpy/examples/plotting/create_test_plot.py b/gcpy/examples/plotting/create_test_plot.py new file mode 100755 index 00000000..848677ff --- /dev/null +++ b/gcpy/examples/plotting/create_test_plot.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +""" +Creates a dummy xarray DataArray object for testing the single_panel +plotting routine. This script can be a useful check to see whether the +GCPy mamba/conda environment has been successfully installed. +""" + +import numpy as np +import xarray as xr +import matplotlib.pyplot as plt +import gcpy + + +def main(): + """ + Main program: Creates a dummy plot + """ + + # Get X and Y coordinate arrays + n_lat = 181 + n_lon = 361 + lat_vals = np.linspace(-90, 90, n_lat) + lon_vals = np.linspace(-180, 180, n_lon) + + # Create a dummy numpy array for plotting + # Populate it with the distance from the center of the plot + data = np.zeros([n_lat, n_lon]) + for lat in range(n_lat): + for lon in range(n_lon): + data[lat, lon] = np.sqrt(lon_vals[lon]**2 + lat_vals[lat]**2) + + # Create a Dataarray from the numpy array + darr = xr.DataArray( + data=data, + dims=["lat", "lon"], + coords=dict( + lat=("lat", lat_vals), + lon=("lon", lon_vals), + ), + ) + + # Create a plot + gcpy.plot.single_panel( + darr, + plot_type="single_level", + title="Test plot to check GCPy import", + extent=[-180, 180, -90, 90], + comap=plt.get_cmap("RdBu_r") + ) + plt.show() + + +if __name__ == '__main__': + main() From 730cb82f51d6ce430dd0f6f7046222f14b527f94 Mon Sep 17 00:00:00 2001 From: Bob Yantosca Date: Thu, 25 Jan 2024 11:23:35 -0500 Subject: [PATCH 02/10] Add GitHub action "build-gcpy-environment" .github/workflows/build-gcpy-environment.yml - Added this configuration file for the "build-gcpy-environment" GitHub action. This runs on pushes or PRs made to the main or dev branches. The PR will use setup-micromamba to build the gcpy environment, import gcpy, and call a simple plotting script. CHANGELOG.md - Updated accordingly Signed-off-by: Bob Yantosca --- .github/workflows/build-gcpy-environment.yml | 34 ++++++++++++++++++++ CHANGELOG.md | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-gcpy-environment.yml diff --git a/.github/workflows/build-gcpy-environment.yml b/.github/workflows/build-gcpy-environment.yml new file mode 100644 index 00000000..33bd0063 --- /dev/null +++ b/.github/workflows/build-gcpy-environment.yml @@ -0,0 +1,34 @@ +--- +# +# GitHub action to build the GCPy environment with micromamba +# See: https://github.com/marketplace/actions/setup-micromamba +# +name: build-gcpy-environment + +on: + push: + branches: [ "main", "dev" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main", "dev" ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9"] + steps: + - uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: 'latest' + environment-file: docs/source/environment.yml + init-shell: bash + cache-environment: true + post-cleanup: 'all' + - name: Import gcpy in micromamba environment (bash) + run: python -c "import numpy" + shell: bash -el {0} + - name: Run custom command in micromamba environment + run: python -m gcpy.examples.plotting.create_test_plot + shell: micromamba-shell {0} diff --git a/CHANGELOG.md b/CHANGELOG.md index 86335626..7c68b041 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] - TBD ### Added - Example script `create_test_plot.py`, which can be used to check that GCPy has been installed properly - +- GitHub action `build-gcpy-environment` which tests installation of the mamba environment specified in in `docs/source/environment.yml` ## [1.4.1] - 2023-12-08 ### Fixed From 5afddd3a2d9dd3793ab6df7689ed5d272f5b2e67 Mon Sep 17 00:00:00 2001 From: Bob Yantosca Date: Thu, 25 Jan 2024 13:11:55 -0500 Subject: [PATCH 03/10] Add GitHub action to build the GCPy python environment .github/workflows/build-gcpy-environment.yml - Add configuration file to trigger a GitHub action on pushes to main,dev or PRs to main,dev. CHANGELOG.md - Updated accordingly Signed-off-by: Bob Yantosca --- .github/workflows/build-gcpy-environment.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-gcpy-environment.yml b/.github/workflows/build-gcpy-environment.yml index 33bd0063..35b8fb19 100644 --- a/.github/workflows/build-gcpy-environment.yml +++ b/.github/workflows/build-gcpy-environment.yml @@ -19,16 +19,21 @@ jobs: matrix: python-version: ["3.9"] steps: - - uses: mamba-org/setup-micromamba@v1 + - name: Checkout the GCPy repository + uses: actions/checkout@v2 + - name: Create the GCPy environment + uses: mamba-org/setup-micromamba@v1 with: micromamba-version: 'latest' - environment-file: docs/source/environment.yml + environment-file: environment.yml + environment-name: gcpy-test-env init-shell: bash cache-environment: true + generate-run-shell: true post-cleanup: 'all' - - name: Import gcpy in micromamba environment (bash) - run: python -c "import numpy" - shell: bash -el {0} - - name: Run custom command in micromamba environment + - name: Test if "import gcpy" works + run: python -c "import gcpy" + shell: micromamba-shell {0} + - name: Test if we can create a plot run: python -m gcpy.examples.plotting.create_test_plot shell: micromamba-shell {0} From 0252fc5b32e6fb8d91eeb9a27087be3893fc7fe7 Mon Sep 17 00:00:00 2001 From: Bob Yantosca Date: Thu, 25 Jan 2024 16:13:36 -0500 Subject: [PATCH 04/10] Prevent overwriting the "results" variable when parallelization is off. This commit fixes the issue reported by @lizziel in #285. The "results" variable was being overwritten instead of appended to when plots are generated sequentially (i.e. with "n_cores: 1" in the YAML input). gcpy/benchmark/modules/run_1yr_*_fullchem.py gcpy/benchmark_funcs.py gcpy/plot/compare_*.py - For the case when parallelization is off: 1. Declare "results" as an empty list 2. Append the output of the routine being called into "results" This will prevent a dictionary key error as described in #285. Signed-off-by: Bob Yantosca --- CHANGELOG.md | 3 +++ .../modules/run_1yr_fullchem_benchmark.py | 18 ++++++++++++------ gcpy/benchmark/modules/run_1yr_tt_benchmark.py | 9 ++++++--- gcpy/benchmark_funcs.py | 16 ++++++++++++---- gcpy/plot/compare_single_level.py | 3 ++- gcpy/plot/compare_zonal_mean.py | 3 ++- 6 files changed, 37 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c68b041..53d545e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Example script `create_test_plot.py`, which can be used to check that GCPy has been installed properly - GitHub action `build-gcpy-environment` which tests installation of the mamba environment specified in in `docs/source/environment.yml` +### Fixed +- Prevent overwriting of the `results` variable when parallel plotting is deactivated (`n_cores: 1`) + ## [1.4.1] - 2023-12-08 ### Fixed - Now use the proper default value for the `--weightsdir` argument to `gcpy/file_regrid.py` diff --git a/gcpy/benchmark/modules/run_1yr_fullchem_benchmark.py b/gcpy/benchmark/modules/run_1yr_fullchem_benchmark.py index 7321c50a..3582bfba 100644 --- a/gcpy/benchmark/modules/run_1yr_fullchem_benchmark.py +++ b/gcpy/benchmark/modules/run_1yr_fullchem_benchmark.py @@ -694,8 +694,9 @@ def gcc_vs_gcc_mass_table(mon): for mon in range(bmk_n_months) ) else: + results = [] for mon in range(bmk_n_months): - results = gcc_vs_gcc_mass_table(mon) + results.append(gcc_vs_gcc_mass_table(mon)) # ================================================================== # GCC vs GCC operations budgets tables @@ -741,8 +742,9 @@ def gcc_vs_gcc_ops_budg(mon): for mon in range(bmk_n_months) ) else: + results = [] for mon in range(bmk_n_months): - results = gcc_vs_gcc_ops_budg(mon) + results.append(gcc_vs_gcc_ops_budg(mon)) # ================================================================== # GCC vs GCC aerosols budgets/burdens tables @@ -1255,8 +1257,9 @@ def gchp_vs_gcc_mass_table(mon): for mon in range(bmk_n_months) ) else: + results = [] for mon in range(bmk_n_months): - results = gchp_vs_gcc_mass_table(mon) + results.append(gchp_vs_gcc_mass_table(mon)) # ================================================================== # GCHP vs GCC operations budgets tables @@ -1311,8 +1314,9 @@ def gchp_vs_gcc_ops_budg(mon): for mon in range(bmk_n_months) ) else: + results = [] for mon in range(bmk_n_months): - results = gchp_vs_gcc_ops_budg(mon) + results.append(gchp_vs_gcc_ops_budg(mon)) # ================================================================== # GCHP vs GCC aerosol budgets and burdens tables @@ -1863,8 +1867,9 @@ def gchp_vs_gchp_mass_table(mon): for mon in range(bmk_n_months) ) else: + results = [] for mon in range(bmk_n_months): - results = gchp_vs_gchp_mass_table(mon) + results.append(gchp_vs_gchp_mass_table(mon)) # ================================================================== # GCHP vs GCHP operations budgets tables @@ -1921,8 +1926,9 @@ def gchp_vs_gchp_ops_budg(mon): for mon in range(bmk_n_months) ) else: + results = [] for mon in range(bmk_n_months): - results = gchp_vs_gchp_ops_budg(mon) + results.append(gchp_vs_gchp_ops_budg(mon)) # ================================================================== # GCHP vs GCHP aerosol budgets and burdens tables diff --git a/gcpy/benchmark/modules/run_1yr_tt_benchmark.py b/gcpy/benchmark/modules/run_1yr_tt_benchmark.py index 878dad76..7d10d149 100644 --- a/gcpy/benchmark/modules/run_1yr_tt_benchmark.py +++ b/gcpy/benchmark/modules/run_1yr_tt_benchmark.py @@ -540,8 +540,9 @@ def gcc_vs_gcc_mass_table(mon): for mon in range(bmk_n_months) ) else: + results = [] for mon in range(bmk_n_months): - results = gcc_vs_gcc_mass_table(mon) + results.append(gcc_vs_gcc_mass_table(mon)) # ================================================================== # GCC vs GCC operations budgets tables @@ -837,8 +838,9 @@ def gchp_vs_gcc_mass_table(mon): for mon in range(bmk_n_months) ) else: + results = [] for mon in range(bmk_n_months): - results = gchp_vs_gcc_mass_table(mon) + results.append(gchp_vs_gcc_mass_table(mon)) # ================================================================== # GCHP vs GCC operations budgets tables @@ -1149,8 +1151,9 @@ def gchp_vs_gchp_mass_table(mon): for mon in range(bmk_n_months) ) else: + results = [] for mon in range(bmk_n_months): - results = gchp_vs_gchp_mass_table(mon) + results.append(gchp_vs_gchp_mass_table(mon)) # ================================================================== # GCHP vs GCHP operations budgets tables diff --git a/gcpy/benchmark_funcs.py b/gcpy/benchmark_funcs.py index e6fe88c6..36ea5360 100644 --- a/gcpy/benchmark_funcs.py +++ b/gcpy/benchmark_funcs.py @@ -1503,10 +1503,13 @@ def createplots(filecat): for _, filecat in enumerate(catdict) ) else: + results = [] for _, filecat in enumerate(catdict): - results = createplots(filecat) + results.append(createplots(filecat)) # -------------------------------------------- - + print(results) + quit() + dict_sfc = {list(result.keys())[0]: result[list( result.keys())[0]]['sfc'] for result in results} dict_500 = {list(result.keys())[0]: result[list( @@ -1514,6 +1517,9 @@ def createplots(filecat): dict_zm = {list(result.keys())[0]: result[list( result.keys())[0]]['zm'] for result in results} + print("stop here") + quit() + # ============================================================== # Write the list of species having significant differences, # which we need to fill out the benchmark approval forms. @@ -1868,8 +1874,9 @@ def createfile_hco_cat(c): for c in emis_cats ) else: + results = [] for c in emis_cats: - results = createfile_hco_cat(c) + results.append(createfile_hco_cat(c)) # --------------------------------------- dict_emis = {list(result.keys())[0]: result[list(result.keys())[0]] @@ -1981,8 +1988,9 @@ def createfile_bench_cat(filecat): for _, filecat in enumerate(catdict) ) else: + results = [] for _, filecat in enumerate(catdict): - results = createfile_bench_cat(filecat) + results.append(createfile_bench_cat(filecat)) #------------------------------------------------ allcatspc = [spc for result in results for spc in result] diff --git a/gcpy/plot/compare_single_level.py b/gcpy/plot/compare_single_level.py index 5a9db5ef..faf82650 100644 --- a/gcpy/plot/compare_single_level.py +++ b/gcpy/plot/compare_single_level.py @@ -1152,8 +1152,9 @@ def get_extent_for_colors(dset, minlon, maxlon, minlat, maxlat): for i in range(n_var) ) else: + results = [] for i in range(n_var): - results = createfig(i, temp_dir) + results.append(createfig(i, temp_dir)) # --------------------------------------- # update sig diffs after parallel calls diff --git a/gcpy/plot/compare_zonal_mean.py b/gcpy/plot/compare_zonal_mean.py index 3999bb3e..f7b41577 100644 --- a/gcpy/plot/compare_zonal_mean.py +++ b/gcpy/plot/compare_zonal_mean.py @@ -1046,8 +1046,9 @@ def createfig(ivar, temp_dir=''): for i in range(n_var) ) else: + results = [] for i in range(n_var): - results = createfig(i, temp_dir) + results.append(createfig(i, temp_dir)) # --------------------------------------- # update sig diffs after parallel calls From 7a2c321a87f261f80e5b4fa0fada956734a8a483 Mon Sep 17 00:00:00 2001 From: Bob Yantosca Date: Thu, 25 Jan 2024 16:18:06 -0500 Subject: [PATCH 05/10] Remove leftover debug output gcpy/benchmark_funcs.py - Removed a leftover print statement and quit() statement from debugging Signed-off-by: Bob Yantosca --- gcpy/benchmark_funcs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gcpy/benchmark_funcs.py b/gcpy/benchmark_funcs.py index 36ea5360..18f7ffd3 100644 --- a/gcpy/benchmark_funcs.py +++ b/gcpy/benchmark_funcs.py @@ -1507,8 +1507,6 @@ def createplots(filecat): for _, filecat in enumerate(catdict): results.append(createplots(filecat)) # -------------------------------------------- - print(results) - quit() dict_sfc = {list(result.keys())[0]: result[list( result.keys())[0]]['sfc'] for result in results} From 1e4a090415954407dfa904c93bac7799e00f6c16 Mon Sep 17 00:00:00 2001 From: Bob Yantosca Date: Thu, 25 Jan 2024 16:39:26 -0500 Subject: [PATCH 06/10] Update environment.yml file to use pegged versions docs/environment_files/environment.yml - Now use pegged package versions, as described in issue #284 Signed-off-by: Bob Yantosca --- docs/environment_files/environment.yml | 89 ++++++++++++-------------- 1 file changed, 40 insertions(+), 49 deletions(-) diff --git a/docs/environment_files/environment.yml b/docs/environment_files/environment.yml index f5fc704d..994d13e9 100644 --- a/docs/environment_files/environment.yml +++ b/docs/environment_files/environment.yml @@ -1,59 +1,50 @@ +--- # ====================================================================== # GCPy environment file # # Recommended installation: with Mambaforge # $ mamba env create -n gcpy_env --file=/path/to/gcpy/environment.yml # -# Some package versions are not the most recent, but these -# have been proven to work together. (Bob Yantosca, 14 Aug 2023) +# These package versions have been proven to work together. +# See: https://github.com/geoschem/gcpy/issues/284 # ====================================================================== name: gcpy_env channels: - - conda-forge - - nodefaults + - conda-forge + - nodefaults dependencies: - - awscli # Utilities for AWS cloud - - cartopy # Geospatial data processing - - cf_xarray # CF conventions for xarray - - dask # Parallel library; backend for xarray - - gridspec # Define Earth System Model grids - - ipython # Interactive Python (used by Jupyter) - - joblib # Parallelize python code - - jupyter # Jupyter Notebook - - matplotlib # Creates plots and visualizations - - netcdf4 # Python wrapper for netCDF - - netcdf-fortran # Python wrapper for netCDF-Fortran - - numpy # Optimized mathematical functions - - pandas # Tables/timeseries manipulation - - pip # Install packages from PyPi - - pylint # Python linter - - pyproj # Python map projections library - - python<3.10 # Any python version prior to 3.10 - - pypdf # PDF utilities (bookmarks, etc.) - - recommonmark # Dependency for Sphinx - - requests # HTTP library - - scipy # Scientific python package - - sparselt>=0.1.3 # Regridding earth system model data - - tabulate # Pretty-printing for column data - - tk # Tcl/tk library - - xarray # Read data from netCDF etc files - # - # NOTE: These packages need to be pegged at specific versions - # in order to avoid an ImportError. - # -- Bob Yantosca (14 Aug 2023) - # - - esmf==8.1.1 # Earth system modeling framework - - esmpy==8.1.1 # Python wrapper for ESMF - - xesmf==0.5.1 # Universal regridder - # - # NOTE: These packages need to be pegged at specific versions - # or else the ReadTheDocs output won't render properly. - # -- Bob Yantosca (14 Aug 2023) - # - - docutils==0.16 # Convert text to other formats - - jinja2==3.0.3 # Dependency for Sphinx - - sphinx==3.5.4 # Generate ReadTheDocs output - - sphinx-autoapi==1.9.0 # Sphinx autodoc style documentation - - sphinx-autobuild==2021.3.14 # Build ReadTheDos live in browser - - sphinxcontrib-bibtex==2.2.0 # ReadTheDocs bibliography style - - sphinx_rtd_theme==0.5.2 # ReadTheDocs HTML theme files + - awscli==2.13.39 # Utilities for AWS cloud + - cartopy==0.22.0 # Geospatial data processing + - cf_xarray==0.8.4 # CF conventions for xarray + - dask==2023.9.2 # Parallel library; backend for xarray + - gridspec==0.1.0 # Define Earth System Model grids + - ipython==8.15.0 # Interactive Python (used by Jupyter) + - joblib==1.3.2 # Parallelize python code + - jupyter==1.0.0 # Jupyter Notebook + - matplotlib==3.8.0 # Creates plots and visualizations + - netcdf4==1.6.0 # Python wrapper for netCDF + - netcdf-fortran==4.5.4 # Python wrapper for netCDF-Fortran + - numpy==1.26.0 # Optimized mathematical functions + - pandas==2.1.1 # Tables/timeseries manipulation + - pip==23.2.1 # Install packages from PyPi + - pylint==2.17.5 # Python linter + - pyproj==3.6.1 # Python map projections library + - python==3.9.18 # Any python version prior to 3.10 + - pypdf==3.16.1 # PDF utilities (bookmarks, etc.) + - recommonmark==0.7.1 # Dependency for Sphinx + - requests==2.31.0 # HTTP library + - scipy==1.11.2 # Scientific python package + - sparselt==0.1.3 # Regridding earth system model data + - tabulate==0.9.0 # Pretty-printing for column data + - tk==8.6.12 # Tcl/tk library + - xarray==2023.8.0 # Read data from netCDF etc files + - esmf==8.1.1 # Earth system modeling framework + - esmpy==8.1.1 # Python wrapper for ESMF + - xesmf==0.5.1 # Universal regridder + - docutils==0.16 # Convert text to other formats + - jinja2==3.0.3 # Dependency for Sphinx + - sphinx==3.5.4 # Generate ReadTheDocs output + - sphinx-autoapi==1.9.0 # Sphinx autodoc style documentation + - sphinx-autobuild==2021.3.14 # Build ReadTheDos live in browser + - sphinxcontrib-bibtex==2.2.0 # ReadTheDocs bibliography style + - sphinx_rtd_theme==0.5.2 # ReadTheDocs HTML theme files From 1a6723644df011d6fe4d59bb78e18facbb17e621 Mon Sep 17 00:00:00 2001 From: Bob Yantosca Date: Thu, 25 Jan 2024 17:06:16 -0500 Subject: [PATCH 07/10] Add test environment file; Add new GitHub action docs/environment_files/testing.yml - Add this new environment file, based on the environment.yml file, but without pegged version numbers for most packages. .github/workflows/build-test-environment.yml - YAML config file to start a new GitHub action to build the mamba environment from "testing.yml" .github/workflows/build-gcpy-environment.yml - Updated names and comments - Now read docs/environment_files/environment.yml CHANGEKOG.md - Updated accordingly Signed-off-by: Bob Yantosca --- .github/workflows/build-gcpy-environment.yml | 7 ++- .github/workflows/build-test-environment.yml | 38 +++++++++++++++ CHANGELOG.md | 4 +- docs/environment_files/testing.yml | 49 ++++++++++++++++++++ 4 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/build-test-environment.yml create mode 100644 docs/environment_files/testing.yml diff --git a/.github/workflows/build-gcpy-environment.yml b/.github/workflows/build-gcpy-environment.yml index 35b8fb19..96e7a161 100644 --- a/.github/workflows/build-gcpy-environment.yml +++ b/.github/workflows/build-gcpy-environment.yml @@ -1,6 +1,6 @@ --- # -# GitHub action to build the GCPy environment with micromamba +# GitHub action to build the GCPy production environment with micromamba # See: https://github.com/marketplace/actions/setup-micromamba # name: build-gcpy-environment @@ -21,12 +21,11 @@ jobs: steps: - name: Checkout the GCPy repository uses: actions/checkout@v2 - - name: Create the GCPy environment + - name: Create "gcpy_env" environment uses: mamba-org/setup-micromamba@v1 with: micromamba-version: 'latest' - environment-file: environment.yml - environment-name: gcpy-test-env + environment-file: docs/environment_files/environment.yml init-shell: bash cache-environment: true generate-run-shell: true diff --git a/.github/workflows/build-test-environment.yml b/.github/workflows/build-test-environment.yml new file mode 100644 index 00000000..42fdb1c4 --- /dev/null +++ b/.github/workflows/build-test-environment.yml @@ -0,0 +1,38 @@ +--- +# +# GitHub action to build the GCPy test environment with micromamba +# See: https://github.com/marketplace/actions/setup-micromamba +# +name: build-test-environment + +on: + push: + branches: [ "main", "dev" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main", "dev" ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9"] + steps: + - name: Checkout the GCPy repository + uses: actions/checkout@v2 + - name: Create "testing" environment + uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: 'latest' + environment-file: docs/environment_files/testing.yml + init-shell: bash + cache-environment: true + generate-run-shell: true + post-cleanup: 'all' + - name: Test if "import gcpy" works + run: python -c "import gcpy" + shell: micromamba-shell {0} + - name: Test if we can create a plot + run: python -m gcpy.examples.plotting.create_test_plot + shell: micromamba-shell {0} diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c68b041..23b4c04a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] - TBD ### Added - Example script `create_test_plot.py`, which can be used to check that GCPy has been installed properly -- GitHub action `build-gcpy-environment` which tests installation of the mamba environment specified in in `docs/source/environment.yml` +- GitHub action `build-gcpy-environment` which tests installation of the mamba environment specified in in `docs/environment_files/environment.yml` +- YAML file`docs/environment_files/testing.yml` for building an environment without pegged package versions (for testing) +- GitHub action `build-test-environment` to test the environment specified in `testing.yml` ## [1.4.1] - 2023-12-08 ### Fixed diff --git a/docs/environment_files/testing.yml b/docs/environment_files/testing.yml new file mode 100644 index 00000000..85a15fac --- /dev/null +++ b/docs/environment_files/testing.yml @@ -0,0 +1,49 @@ +--- +# ====================================================================== +# GCPy testing environment file. +# Intended to be used by the GCST for testing purposes. +# +# GCPy users should build the environmet from "environment.yml", +# as this will specify package versions that are known to work. +# See: https://github.com/geoschem/gcpy/issues/284 +# ====================================================================== +name: testing +channels: + - conda-forge + - nodefaults +dependencies: + - awscli # Utilities for AWS cloud + - cartopy # Geospatial data processing + - cf_xarray # CF conventions for xarray + - dask # Parallel library; backend for xarray + - gridspec # Define Earth System Model grids + - ipython # Interactive Python (used by Jupyter) + - joblib # Parallelize python code + - jupyter # Jupyter Notebook + - matplotlib # Creates plots and visualizations + - netcdf4 # Python wrapper for netCDF + - netcdf-fortran # Python wrapper for netCDF-Fortran + - numpy # Optimized mathematical functions + - pandas # Tables/timeseries manipulation + - pip # Install packages from PyPi + - pylint # Python linter + - pyproj # Python map projections library + - python # Any python version prior to 3.10 + - pypdf # PDF utilities (bookmarks, etc.) + - recommonmark # Dependency for Sphinx + - requests # HTTP library + - scipy # Scientific python package + - sparselt # Regridding earth system model data + - tabulate # Pretty-printing for column data + - tk # Tcl/tk library + - xarray # Read data from netCDF etc files + - esmf==8.1.1 # Earth system modeling framework + - esmpy==8.1.1 # Python wrapper for ESMF + - xesmf==0.5.1 # Universal regridder + - docutils # Convert text to other formats + - jinja2 # Dependency for Sphinx + - sphinx # Generate ReadTheDocs output + - sphinx-autoapi # Sphinx autodoc style documentation + - sphinx-autobuild # Build ReadTheDos live in browser + - sphinxcontrib-bibtex # ReadTheDocs bibliography style + - sphinx_rtd_theme # ReadTheDocs HTML theme files From 8595565e6afd73c2468e006b232e33f80eff745d Mon Sep 17 00:00:00 2001 From: Bob Yantosca Date: Fri, 26 Jan 2024 10:16:49 -0500 Subject: [PATCH 08/10] build-gcpy-environment now tests env build w/ several Python versions .github/workflows/build-gcpy-environment - Now attempt to build the environment with Python versions 3.8 thru 3.13. CHANGELOG.md - Updated currently Signed-off-by: Bob Yantosca --- .github/workflows/build-gcpy-environment.yml | 2 +- CHANGELOG.md | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-gcpy-environment.yml b/.github/workflows/build-gcpy-environment.yml index 96e7a161..765a18a7 100644 --- a/.github/workflows/build-gcpy-environment.yml +++ b/.github/workflows/build-gcpy-environment.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout the GCPy repository uses: actions/checkout@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 23b4c04a..ad30b2e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - YAML file`docs/environment_files/testing.yml` for building an environment without pegged package versions (for testing) - GitHub action `build-test-environment` to test the environment specified in `testing.yml` +### Changed +- `build-gcpy-environment` + + ## [1.4.1] - 2023-12-08 ### Fixed - Now use the proper default value for the `--weightsdir` argument to `gcpy/file_regrid.py` @@ -164,7 +168,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Now use restarts_subdir tag from YAML file for paths to restart files (@yantosca) - GCPy now uses proper year for dev in 1-yr benchmarks (@laestrada) - Fixed date string issue in benchmarking scripts (@lizziel) - - Updates for new GCHP restart file format (@lizziel) + - Updates for new GCHP restart file format (@lizziel) - Updated environment.yml with package versions that work together (@yantosca) - Updated the AUTHORS.txt and LICENSE.txt files (@yantosca) From 43c783a549ede5a3619ac2ef9e454f25857b33b0 Mon Sep 17 00:00:00 2001 From: Bob Yantosca Date: Fri, 26 Jan 2024 10:41:32 -0500 Subject: [PATCH 09/10] Fix typo in testing.yml; peg versions for RTD packages docs/environment_files/testing.yml - Fixed typo: "environmet" -> "environment" in comments - Now peg the RTD package versions, as we know these work. - Also peg matplotlib>=3.8 Signed-off-by: Bob Yantosca --- docs/environment_files/testing.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/environment_files/testing.yml b/docs/environment_files/testing.yml index 85a15fac..3c6c9de0 100644 --- a/docs/environment_files/testing.yml +++ b/docs/environment_files/testing.yml @@ -3,7 +3,7 @@ # GCPy testing environment file. # Intended to be used by the GCST for testing purposes. # -# GCPy users should build the environmet from "environment.yml", +# GCPy users should build the environment from "environment.yml", # as this will specify package versions that are known to work. # See: https://github.com/geoschem/gcpy/issues/284 # ====================================================================== @@ -20,7 +20,7 @@ dependencies: - ipython # Interactive Python (used by Jupyter) - joblib # Parallelize python code - jupyter # Jupyter Notebook - - matplotlib # Creates plots and visualizations + - matplotlib>=3.8 # Creates plots and visualizations - netcdf4 # Python wrapper for netCDF - netcdf-fortran # Python wrapper for netCDF-Fortran - numpy # Optimized mathematical functions @@ -37,13 +37,16 @@ dependencies: - tabulate # Pretty-printing for column data - tk # Tcl/tk library - xarray # Read data from netCDF etc files + # + # NOTE: These packages are known to work, so peg these versions + # - esmf==8.1.1 # Earth system modeling framework - esmpy==8.1.1 # Python wrapper for ESMF - xesmf==0.5.1 # Universal regridder - - docutils # Convert text to other formats - - jinja2 # Dependency for Sphinx - - sphinx # Generate ReadTheDocs output - - sphinx-autoapi # Sphinx autodoc style documentation - - sphinx-autobuild # Build ReadTheDos live in browser - - sphinxcontrib-bibtex # ReadTheDocs bibliography style - - sphinx_rtd_theme # ReadTheDocs HTML theme files + - docutils==0.16 # Convert text to other formats + - jinja2==3.0.3 # Dependency for Sphinx + - sphinx==3.5.4 # Generate ReadTheDocs output + - sphinx-autoapi==1.9.0 # Sphinx autodoc style documentation + - sphinx-autobuild==2021.3.14 # Build ReadTheDos live in browser + - sphinxcontrib-bibtex==2.2.0 # ReadTheDocs bibliography style + - sphinx_rtd_theme==0.5.2 # ReadTheDocs HTML theme files From 70d649a12b81735521005d48dee1adfbf360da37 Mon Sep 17 00:00:00 2001 From: Bob Yantosca Date: Fri, 26 Jan 2024 11:20:43 -0500 Subject: [PATCH 10/10] Update version numbers and prepare for GCPy 1.4.2 release CHANGELOG.md docs/source/conf.py gcpy/_version.py gcpy/benchmark/run_benchmark.py gcpy/benchmark/modules/run_1yr_fullchem_benchmark.py gcpy/benchmark/modules/run_1yr_tt_benchmark.py CHANGELOG.md - Updated version numbers setup.py - Restored, now use same version numbers as in environment.yml Signed-off-by: Bob Yantosca --- CHANGELOG.md | 5 +- docs/source/Release_guide.rst | 6 +- docs/source/conf.py | 2 +- gcpy/_version.py | 2 +- .../modules/run_1yr_fullchem_benchmark.py | 2 +- .../benchmark/modules/run_1yr_tt_benchmark.py | 2 +- gcpy/benchmark/run_benchmark.py | 2 +- setup.py | 144 ++++++++++++++++++ 8 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 setup.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b9c797db..1cab4aa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to GCPy will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] - TBD +## [1.4.2] - 2024-01-26 ### Added - Example script `create_test_plot.py`, which can be used to check that GCPy has been installed properly - GitHub action `build-gcpy-environment` which tests installation of the mamba environment specified in in `docs/environment_files/environment.yml` @@ -12,8 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - GitHub action `build-test-environment` to test the environment specified in `testing.yml` ### Changed -- `build-gcpy-environment` - +- `build-gcpy-environment` GitHub action now runs with several Python versions ### Fixed - Prevent overwriting of the `results` variable when parallel plotting is deactivated (`n_cores: 1`) diff --git a/docs/source/Release_guide.rst b/docs/source/Release_guide.rst index 129861fe..6b12a762 100644 --- a/docs/source/Release_guide.rst +++ b/docs/source/Release_guide.rst @@ -17,9 +17,9 @@ versions of GCPy on Github, PyPi, and conda-forge. - :file:`setup.py` - :file:`gcpy/_version.py` - :file:`docs/source/conf.py` - - :file:`benchmark/run_benchmark.py` - - :file:`benchmark/modules/run_1yr_fullchem_benchmark.py` - - :file:`benchmark/modules/run_1yr_tt_benchmark.py` + - :file:`gcpy/benchmark/run_benchmark.py` + - :file:`gcpy/benchmark/modules/run_1yr_fullchem_benchmark.py` + - :file:`gcpy/benchmark/modules/run_1yr_tt_benchmark.py` |br| #. Update :file:`CHANGELOG.md` |br| diff --git a/docs/source/conf.py b/docs/source/conf.py index f01ae723..62776896 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -22,7 +22,7 @@ author = 'GEOS-Chem Support Team' # The full version, including alpha/beta/rc tags -release = '1.4.1' +release = '1.4.2' # -- General configuration --------------------------------------------------- diff --git a/gcpy/_version.py b/gcpy/_version.py index 8bad8dce..b58b8b9c 100644 --- a/gcpy/_version.py +++ b/gcpy/_version.py @@ -1,2 +1,2 @@ +__version__ = '1.4.2' -__version__ = '1.4.1' diff --git a/gcpy/benchmark/modules/run_1yr_fullchem_benchmark.py b/gcpy/benchmark/modules/run_1yr_fullchem_benchmark.py index 3582bfba..ecb2491f 100644 --- a/gcpy/benchmark/modules/run_1yr_fullchem_benchmark.py +++ b/gcpy/benchmark/modules/run_1yr_fullchem_benchmark.py @@ -43,7 +43,7 @@ https://github.com/ipython/ipython/issues/10627 -This script corresponds with GCPy 1.4.0. Edit this version ID if releasing +This script corresponds with GCPy 1.4.2. Edit this version ID if releasing a new version of GCPy. """ diff --git a/gcpy/benchmark/modules/run_1yr_tt_benchmark.py b/gcpy/benchmark/modules/run_1yr_tt_benchmark.py index 7d10d149..5864bcda 100644 --- a/gcpy/benchmark/modules/run_1yr_tt_benchmark.py +++ b/gcpy/benchmark/modules/run_1yr_tt_benchmark.py @@ -42,7 +42,7 @@ https://github.com/ipython/ipython/issues/10627 -This script corresponds with GCPy 1.4.0. Edit this version ID if releasing +This script corresponds with GCPy 1.4.2. Edit this version ID if releasing a new version of GCPy. """ diff --git a/gcpy/benchmark/run_benchmark.py b/gcpy/benchmark/run_benchmark.py index 42d36418..0b5db597 100755 --- a/gcpy/benchmark/run_benchmark.py +++ b/gcpy/benchmark/run_benchmark.py @@ -31,7 +31,7 @@ https://github.com/ipython/ipython/issues/10627 -This script corresponds with GCPy 1.4.0. Edit this version ID if releasing +This script corresponds with GCPy 1.4.2. Edit this version ID if releasing a new version of GCPy. """ diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..2fae9011 --- /dev/null +++ b/setup.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python + +import os +import warnings + +from setuptools import setup, find_packages + +from textwrap import dedent + +DESCRIPTION = "" +LONG_DESCRIPTION = """\ +Python toolkit for working with GEOS-Chem output. +""" + +DISTNAME = "geoschem-gcpy" +AUTHOR = "GEOS-Chem Support Team" +AUTHOR_EMAIL = "geos-chem-support@g.harvard.edu" +URL = "https://github.com/geoschem/gcpy" +LICENSE = "MIT" + +CLASSIFIERS = [ + 'Development Status :: 5 - Production/Stable', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + 'Intended Audience :: Science/Research', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.9', + 'Topic :: Scientific/Engineering', +] + +MAJOR = 1 +MINOR = 4 +MICRO = 2 +EXTRA = '' # for alpha (aN), beta (bN), rc (rcN) versions + +VERSION = f"{MAJOR}.{MINOR}.{MICRO}{EXTRA}" +''' +#DEV format (using git hash) is intriguing but incompatible with PEP 440 +#No hashes can be used in version field +DEV = True + + +# Correct versioning with git info if DEV + +if DEV: + import subprocess + + pipe = subprocess.Popen( + ['git', "describe", "--always", "--match", "v[0-9]*"], + stdout=subprocess.PIPE) + so, err = pipe.communicate() + + if pipe.returncode != 0: + # no git or something wrong with git (not in dir?) + warnings.warn("WARNING: Couldn't identify git revision, using generic version string") + VERSION += ".dev" + else: + git_rev = so.strip() + git_rev = git_rev.decode('ascii') # necessary for Python >= 3 + + VERSION += ".dev-{}".format(git_rev) +''' + +def _write_version_file(): + + fn = os.path.join(os.path.dirname(__file__), 'gcpy', '_version.py') + + version_str = dedent(""" + __version__ = '{}' + """) + + # Write version file + with open(fn, 'w') as version_file: + version_file.write(version_str.format(VERSION)) + +# Write version and install +_write_version_file() + +setup( + name = DISTNAME, + author = AUTHOR, + author_email = AUTHOR_EMAIL, + maintainer = AUTHOR, + maintainer_email = AUTHOR_EMAIL, + description = DESCRIPTION, + long_description = LONG_DESCRIPTION, + license = LICENSE, + url = URL, + version = VERSION, + packages = find_packages(), + include_package_data=True, + install_requires=[ + "awscli==2.13.39", + "cartopy==0.22.0", + "cf_xarray==0.8.4", + "dask==2023.9.2", + "gridspec==0.1.0", + "ipython==8.15.0", + "joblib==1.3.2", + "jupyter==1.0.0", + "matplotlib==3.8.0", + "netcdf4==1.6.0", + "netcdf-fortran==4.5.4", + "numpy==1.26.0", + "pandas==2.1.1", + "pip==23.2.1", + "pylint==2.17.5", + "pyproj==3.6.1", + "python==3.9.18", + "pypdf==3.16.1", + "recommonmark==0.7.1", + "requests==2.31.0", + "scipy==1.11.2", + "sparselt==0.1.3", + "tabulate==0.9.0", + "tk==8.6.12", + "xarray==2023.8.0", + "esmf==8.1.1", + "esmpy==8.1.1", + "xesmf==0.5.1", + "docutils==0.16", + "jinja2==3.0.3", + "sphinx==3.5.4", + "sphinx-autoapi==1.9.0", + "sphinx-autobuild==2021.3.14", + "sphinxcontrib-bibtex==2.2.0", + "sphinx_rtd_theme==0.5.2", + ], + classifiers = CLASSIFIERS +) + + +''' +Instructions for publishing new version to TestPyPi (mimics PyPi) and PyPi: + +1. Update version numbers above and in _version.py and commit before releasing +2. Install twine if not available (pip install twine) +3. run python setup.py sdist bdist_wheel +4. run twine check dist/* +5. run twine upload --repository-url https://test.pypi.org/legacy/ dist/* +6. Test installation in a virtual environment using pip install --extra-index-url https://test.pypi.org/simple/ geoschem-gcpy +7. To publish to PyPi, run twine upload dist/* + +'''